diff --git a/README.md b/README.md index 0423b732..47b776cb 100644 --- a/README.md +++ b/README.md @@ -185,7 +185,7 @@ A WebUI developed based on Gradio, with a simple interface and only core parsing Parsing Backend pipeline - *-auto-engine + *-engine *-http-client diff --git a/README_zh-CN.md b/README_zh-CN.md index d6e33077..7ee80344 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -183,7 +183,7 @@ https://github.com/user-attachments/assets/4bea02c9-6d54-4cd6-97ed-dff14340982c 解析后端 pipeline - *-auto-engine + *-engine *-http-client diff --git a/demo/demo.py b/demo/demo.py index f1c48a76..e83d05a3 100644 --- a/demo/demo.py +++ b/demo/demo.py @@ -95,7 +95,7 @@ async def run_demo( output_dir: str | Path, *, api_url: str | None = None, - backend: str = "hybrid-auto-engine", + backend: str = "hybrid-engine", parse_method: str = "auto", language: str = "ch", formula_enable: bool = True, @@ -212,12 +212,12 @@ def main() -> None: api_url = None # Available examples: - # "hybrid-auto-engine" -> local hybrid parsing, recommended default + # "hybrid-engine" -> local hybrid parsing, recommended default # "pipeline" -> more general OCR/text pipeline - # "vlm-auto-engine" -> local VLM parsing + # "vlm-engine" -> local VLM parsing # "vlm-http-client" -> remote OpenAI-compatible VLM server # "hybrid-http-client" -> remote OpenAI-compatible hybrid server - backend = "hybrid-auto-engine" + backend = "hybrid-engine" # Available options: # "auto" -> let MinerU choose between text extraction and OCR # "txt" -> force text extraction diff --git a/docs/en/quick_start/index.md b/docs/en/quick_start/index.md index fbf76fae..b4234a1a 100644 --- a/docs/en/quick_start/index.md +++ b/docs/en/quick_start/index.md @@ -33,7 +33,7 @@ A WebUI developed based on Gradio, with a simple interface and only core parsing Parsing Backend pipeline - *-auto-engine + *-engine *-http-client diff --git a/docs/en/usage/cli_tools.md b/docs/en/usage/cli_tools.md index 01d93c1f..25c30d03 100644 --- a/docs/en/usage/cli_tools.md +++ b/docs/en/usage/cli_tools.md @@ -12,8 +12,8 @@ Options: -o, --output PATH Output directory (required) --api-url TEXT MinerU FastAPI base URL; if omitted, `mineru` starts a temporary local `mineru-api` -m, --method [auto|txt|ocr] Parsing method: auto (default), txt, ocr (pipeline and hybrid* backend only) - -b, --backend [pipeline|hybrid-auto-engine|hybrid-http-client|vlm-auto-engine|vlm-http-client] - Parsing backend (default: hybrid-auto-engine) + -b, --backend [pipeline|vlm-engine|hybrid-engine|hybrid-flash-engine|vlm-http-client|hybrid-http-client|hybrid-flash-http-client] + Parsing backend (default: hybrid-flash-engine) -l, --lang [ch|ch_server|ch_lite|en|korean|japan|chinese_cht|ta|te|ka|th|el|latin|arabic|east_slavic|cyrillic|devanagari] Specify document language (improves OCR accuracy, pipeline and hybrid* backend only) -u, --url TEXT OpenAI-compatible backend URL passed through to the server when using http-client diff --git a/docs/zh/quick_start/index.md b/docs/zh/quick_start/index.md index eb9792a8..04fea421 100644 --- a/docs/zh/quick_start/index.md +++ b/docs/zh/quick_start/index.md @@ -33,7 +33,7 @@ 解析后端 pipeline - *-auto-engine + *-engine *-http-client diff --git a/docs/zh/usage/acceleration_cards/Ascend.md b/docs/zh/usage/acceleration_cards/Ascend.md index 7aa643f7..ccfd25a0 100644 --- a/docs/zh/usage/acceleration_cards/Ascend.md +++ b/docs/zh/usage/acceleration_cards/Ascend.md @@ -116,7 +116,7 @@ docker run -u root --name mineru_docker --privileged=true \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 @@ -132,7 +132,7 @@ docker run -u root --name mineru_docker --privileged=true \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 @@ -148,7 +148,7 @@ docker run -u root --name mineru_docker --privileged=true \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 diff --git a/docs/zh/usage/acceleration_cards/Biren.md b/docs/zh/usage/acceleration_cards/Biren.md index acf9a2cc..154d2d94 100644 --- a/docs/zh/usage/acceleration_cards/Biren.md +++ b/docs/zh/usage/acceleration_cards/Biren.md @@ -57,7 +57,7 @@ docker run -it --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -70,7 +70,7 @@ docker run -it --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -83,7 +83,7 @@ docker run -it --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 diff --git a/docs/zh/usage/acceleration_cards/Cambricon.md b/docs/zh/usage/acceleration_cards/Cambricon.md index fc8a9af4..83943a03 100644 --- a/docs/zh/usage/acceleration_cards/Cambricon.md +++ b/docs/zh/usage/acceleration_cards/Cambricon.md @@ -95,7 +95,7 @@ docker run --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟡 🟡 @@ -111,7 +111,7 @@ docker run --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🔴 🟢 @@ -127,7 +127,7 @@ docker run --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🔴 🟢 diff --git a/docs/zh/usage/acceleration_cards/Enflame.md b/docs/zh/usage/acceleration_cards/Enflame.md index 19f66272..0ee1cbaa 100644 --- a/docs/zh/usage/acceleration_cards/Enflame.md +++ b/docs/zh/usage/acceleration_cards/Enflame.md @@ -55,7 +55,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -68,7 +68,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -81,7 +81,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 diff --git a/docs/zh/usage/acceleration_cards/Hygon.md b/docs/zh/usage/acceleration_cards/Hygon.md index faaada2a..d6ad7947 100644 --- a/docs/zh/usage/acceleration_cards/Hygon.md +++ b/docs/zh/usage/acceleration_cards/Hygon.md @@ -62,7 +62,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -75,7 +75,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -88,7 +88,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 diff --git a/docs/zh/usage/acceleration_cards/IluvatarCorex.md b/docs/zh/usage/acceleration_cards/IluvatarCorex.md index f1ef7c71..8022461d 100644 --- a/docs/zh/usage/acceleration_cards/IluvatarCorex.md +++ b/docs/zh/usage/acceleration_cards/IluvatarCorex.md @@ -69,7 +69,7 @@ docker run --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -82,7 +82,7 @@ docker run --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -95,7 +95,7 @@ docker run --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 diff --git a/docs/zh/usage/acceleration_cards/Kunlunxin.md b/docs/zh/usage/acceleration_cards/Kunlunxin.md index bb54cbdd..48576492 100644 --- a/docs/zh/usage/acceleration_cards/Kunlunxin.md +++ b/docs/zh/usage/acceleration_cards/Kunlunxin.md @@ -69,7 +69,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -82,7 +82,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -95,7 +95,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 diff --git a/docs/zh/usage/acceleration_cards/METAX.md b/docs/zh/usage/acceleration_cards/METAX.md index 8056d10b..ceef098a 100644 --- a/docs/zh/usage/acceleration_cards/METAX.md +++ b/docs/zh/usage/acceleration_cards/METAX.md @@ -88,7 +88,7 @@ docker run --ipc host \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 @@ -104,7 +104,7 @@ docker run --ipc host \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 @@ -120,7 +120,7 @@ docker run --ipc host \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 diff --git a/docs/zh/usage/acceleration_cards/MooreThreads.md b/docs/zh/usage/acceleration_cards/MooreThreads.md index 1015a637..31dbc8e8 100644 --- a/docs/zh/usage/acceleration_cards/MooreThreads.md +++ b/docs/zh/usage/acceleration_cards/MooreThreads.md @@ -63,7 +63,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -76,7 +76,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🔴 @@ -89,7 +89,7 @@ docker run -u root --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🔴 diff --git a/docs/zh/usage/acceleration_cards/THead.md b/docs/zh/usage/acceleration_cards/THead.md index 98e9dee0..677dde27 100644 --- a/docs/zh/usage/acceleration_cards/THead.md +++ b/docs/zh/usage/acceleration_cards/THead.md @@ -79,7 +79,7 @@ docker run --privileged=true \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 @@ -95,7 +95,7 @@ docker run --privileged=true \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 @@ -111,7 +111,7 @@ docker run --privileged=true \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 🟢 diff --git a/docs/zh/usage/acceleration_cards/Tecorigin.md b/docs/zh/usage/acceleration_cards/Tecorigin.md index 07180dce..3c44409e 100644 --- a/docs/zh/usage/acceleration_cards/Tecorigin.md +++ b/docs/zh/usage/acceleration_cards/Tecorigin.md @@ -65,7 +65,7 @@ docker run -dit --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -78,7 +78,7 @@ docker run -dit --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 @@ -91,7 +91,7 @@ docker run -dit --name mineru_docker \ 🟢 - <vlm/hybrid>-auto-engine + <vlm/hybrid>-engine 🟢 diff --git a/docs/zh/usage/acceleration_cards/VastAI.md b/docs/zh/usage/acceleration_cards/VastAI.md index 5698763d..a1938670 100644 --- a/docs/zh/usage/acceleration_cards/VastAI.md +++ b/docs/zh/usage/acceleration_cards/VastAI.md @@ -72,7 +72,7 @@ ## 4. MinerU功能 > [!NOTE] -> - `VastAI`加速卡仅支持使用`vlm-auto-engine`和`vlm-http-client`形式进行`VLM`模型推理加速 +> - `VastAI`加速卡仅支持使用`vlm-engine`和`vlm-http-client`形式进行`VLM`模型推理加速 - 进入容器 ```bash @@ -83,15 +83,15 @@ - 模型准备，参考官方介绍：[model_source.md](https://github.com/opendatalab/MinerU/blob/master/docs/zh/usage/model_source.md) - - 方式一：`vlm-auto-engine` + - 方式一：`vlm-engine` ```bash export MINERU_MODEL_SOURCE=modelscope - # step1, 以`vlm-auto-engine`方式启动MinerU解析任务 + # step1, 以`vlm-engine`方式启动MinerU解析任务 mineru -p image.png \ -o ./output \ - -b vlm-auto-engine \ + -b vlm-engine \ --http-timeout 1200 \ --tensor-parallel-size 2 \ --enforce_eager \ @@ -147,11 +147,11 @@ 🔴 - hybrid-auto-engine + hybrid-engine 🔴 - vlm-auto-engine + vlm-engine 🟢 @@ -168,11 +168,11 @@ 🔴 - hybrid-auto-engine + hybrid-engine 🔴 - vlm-auto-engine + vlm-engine 🟢 @@ -189,11 +189,11 @@ 🔴 - hybrid-auto-engine + hybrid-engine 🔴 - vlm-auto-engine + vlm-engine 🟢 @@ -212,4 +212,4 @@ > - 🟢: 支持，运行较稳定，精度与NVIDIA GPU基本一致 > - 🟡: 支持但较不稳定，在某些场景下可能出现异常，或精度存在一定差异 > - 🔴: 不支持，无法运行，或精度存在较大差异 -> - `vlm-auto-engine`：VastAI仅支持vLLM后端 \ No newline at end of file +> - `vlm-engine`：VastAI仅支持vLLM后端 \ No newline at end of file diff --git a/docs/zh/usage/cli_tools.md b/docs/zh/usage/cli_tools.md index e99d780b..52f52899 100644 --- a/docs/zh/usage/cli_tools.md +++ b/docs/zh/usage/cli_tools.md @@ -12,8 +12,8 @@ Options: -o, --output PATH 输出目录（必填） --api-url TEXT MinerU FastAPI 服务地址；不传时自动拉起本地临时 mineru-api -m, --method [auto|txt|ocr] 解析方法：auto（默认）、txt、ocr（仅用于 pipeline 与 hybrid* 后端） - -b, --backend [pipeline|hybrid-auto-engine|hybrid-http-client|vlm-auto-engine|vlm-http-client] - 解析后端（默认为 hybrid-auto-engine） + -b, --backend [pipeline|vlm-engine|hybrid-engine|hybrid-flash-engine|vlm-http-client|hybrid-http-client|hybrid-flash-http-client] + 解析后端（默认为 hybrid-flash-engine） -l, --lang [ch|ch_server|ch_lite|en|korean|japan|chinese_cht|ta|te|ka|th|el|latin|arabic|east_slavic|cyrillic|devanagari] 指定文档语言（可提升 OCR 准确率，仅用于 pipeline 与 hybrid* 后端） -u, --url TEXT 当使用 http-client 时，传给服务端后端的 OpenAI 兼容地址 diff --git a/mineru/backend/hybrid/hybrid_analyze.py b/mineru/backend/hybrid/hybrid_analyze.py index 095aa2d4..0a1213c0 100644 --- a/mineru/backend/hybrid/hybrid_analyze.py +++ b/mineru/backend/hybrid/hybrid_analyze.py @@ -9,7 +9,7 @@ import numpy as np import pypdfium2 as pdfium from loguru import logger from mineru_vl_utils import MinerUClient -from mineru_vl_utils.structs import BlockType +from mineru_vl_utils.structs import BlockType, ContentBlock from tqdm import tqdm from mineru.backend.hybrid.hybrid_model_output_to_middle_json import ( @@ -25,6 +25,8 @@ from mineru.backend.pipeline.model_init import ( run_mfr_inference, run_ocr_inference, ) +from mineru.backend.pipeline.model_list import AtomicModel +from mineru.backend.utils.formula_number import optimize_flash_formula_number_blocks from mineru.backend.vlm.vlm_analyze import ( ModelSingleton, aio_predictor_execution_guard, @@ -59,6 +61,51 @@ LAYOUT_TITLE_SPLIT_OVERLAP_THRESHOLD = 0.8 not_extract_list = [item.value for item in NotExtractType] HYBRID_OCR_DET_TEXT_TYPES = set(not_extract_list) +HYBRID_ANALYZE_MODES = {"pro", "flash"} +INLINE_FORMULA_CONTAINER_LABELS = {"table", "image", "chart", "display_formula"} +FLASH_LAYOUT_LABEL_TO_VLM_TYPE = { + "abstract": BlockType.TEXT, + "algorithm": BlockType.CODE, + "aside_text": BlockType.ASIDE_TEXT, + "content": BlockType.TEXT, + "doc_title": BlockType.TITLE, + "footer": BlockType.FOOTER, + "footer_image": BlockType.FOOTER, + "footnote": BlockType.PAGE_FOOTNOTE, + "formula_number": BlockType.FORMULA_NUMBER, + "header": BlockType.HEADER, + "header_image": BlockType.HEADER, + "number": BlockType.PAGE_NUMBER, + "paragraph_title": BlockType.TITLE, + "reference_content": BlockType.REF_TEXT, + "text": BlockType.TEXT, + "vertical_text": BlockType.TEXT, + "figure_title": BlockType.IMAGE_CAPTION, + "vision_footnote": BlockType.IMAGE_FOOTNOTE, + "image": BlockType.IMAGE, + "chart": BlockType.CHART, + "seal": BlockType.IMAGE, + "table": BlockType.TABLE, + "display_formula": BlockType.EQUATION, +} + + +def _validate_hybrid_mode(mode: str) -> str: + """校验 Hybrid 运行模式，避免静默走错解析分支。""" + if mode not in HYBRID_ANALYZE_MODES: + raise ValueError('mode must be "pro" or "flash"') + return mode + + +def _vlm_type_for_flash_layout_label(label: str | None) -> str | None: + """将 pipeline layout 标签映射为 mineru-vl-utils 支持的 VLM 抽取类型。""" + return FLASH_LAYOUT_LABEL_TO_VLM_TYPE.get(label) + + +def _apply_flash_visual_sub_type(block, label: str | None): + """为视觉块补充下游需要透传的子类型。""" + if label == "seal": + block["sub_type"] = "seal" def _is_hybrid_ocr_det_candidate(block): @@ -281,6 +328,129 @@ def normalize_bbox_to_unit(item, page_width, page_height): return True +def _layout_det_bbox_to_unit(layout_det, page_width, page_height): + """复制并归一化 layout bbox，避免构造 VLM 输入时改动 pipeline 原始结果。""" + bbox = layout_det.get("bbox") + if bbox is None or len(bbox) != 4: + return None + bbox_item = {"bbox": list(bbox)} + if not normalize_bbox_to_unit(bbox_item, page_width, page_height): + return None + return bbox_item["bbox"] + + +def _layout_det_bbox_to_pixel(layout_det, page_width, page_height): + """将layout bbox转换为页面像素坐标，兼容归一化和像素两种输入。""" + bbox = layout_det.get("bbox") + if bbox is None or len(bbox) != 4: + return None + + x0, y0, x1, y1 = [float(v) for v in bbox] + if ( + 0.0 <= x0 <= 1.0 + and 0.0 <= y0 <= 1.0 + and 0.0 <= x1 <= 1.0 + and 0.0 <= y1 <= 1.0 + ): + x0, x1 = x0 * page_width, x1 * page_width + y0, y1 = y0 * page_height, y1 * page_height + + x0 = max(0, min(page_width, x0)) + y0 = max(0, min(page_height, y0)) + x1 = max(0, min(page_width, x1)) + y1 = max(0, min(page_height, y1)) + if x1 <= x0 or y1 <= y0: + return None + return [x0, y0, x1, y1] + + +def _normalize_flash_vlm_angle(angle): + """将pipeline方向标签转换为mineru-vl-utils接受的整数角度。""" + try: + normalized_angle = int(angle) + except (TypeError, ValueError): + return 0 + if normalized_angle in {0, 90, 180, 270}: + return normalized_angle + return 0 + + +def _build_flash_vlm_layout_blocks(layout_dets, page_width, page_height): + """用 pipeline layout 构造 VLM 外部 layout 输入，跳过 VLM 自身 layout 解析。""" + blocks = [] + for layout_det in layout_dets or []: + label = layout_det.get("label") + vlm_type = _vlm_type_for_flash_layout_label(label) + if vlm_type is None: + continue + bbox = _layout_det_bbox_to_unit(layout_det, page_width, page_height) + if bbox is None: + continue + try: + block = ContentBlock( + vlm_type, + bbox, + angle=_normalize_flash_vlm_angle(layout_det.get("angle", 0)), + content=layout_det.get("content"), + ) + except AssertionError as exc: + logger.warning(f"Skip invalid Hybrid flash VLM block: {layout_det}, error: {exc}") + continue + _apply_flash_visual_sub_type(block, label) + blocks.append(block) + return blocks + + +def _apply_flash_table_orientation_labels( + images_pil_list, + images_layout_res, + hybrid_pipeline_model, + batch_ratio: int = 1, +): + """复用pipeline表格方向分类，为Hybrid flash的table layout写入VLM旋转角度。""" + table_inputs = [] + table_layout_refs = [] + for pil_img, layout_res in zip(images_pil_list, images_layout_res): + page_width, page_height = pil_img.size + for layout_det in layout_res or []: + if layout_det.get("label") != "table": + continue + pixel_bbox = _layout_det_bbox_to_pixel(layout_det, page_width, page_height) + if pixel_bbox is None: + continue + try: + table_img, _ = crop_img({"bbox": pixel_bbox}, pil_img) + except Exception as exc: + logger.warning( + f"Skip Hybrid flash table orientation crop: {layout_det}, error: {exc}" + ) + continue + table_inputs.append({"table_img": table_img}) + table_layout_refs.append(layout_det) + + if not table_inputs: + return + + try: + table_orientation_cls_model = hybrid_pipeline_model.atom_model_manager.get_atom_model( + atom_model_name=AtomicModel.TableOrientationCls, + lang=getattr(hybrid_pipeline_model, "lang", None), + ) + rotate_labels = table_orientation_cls_model.batch_predict( + table_inputs, + det_batch_size=max(1, batch_ratio * OCR_DET_BASE_BATCH_SIZE), + tqdm_enable=True, + ) + if len(rotate_labels) != len(table_layout_refs): + raise ValueError("Table orientation prediction result count mismatch") + for layout_det, rotate_label in zip(table_layout_refs, rotate_labels): + layout_det["angle"] = str(rotate_label or "0") + except Exception as exc: + logger.warning( + f"Hybrid flash table orientation classification failed: {exc}, using original table images" + ) + + def _formula_item_to_pixel_bbox(item): bbox = item.get('bbox') if bbox is not None and len(bbox) == 4: @@ -289,12 +459,79 @@ def _formula_item_to_pixel_bbox(item): return None +def _layout_item_to_float_bbox(item): + """校验并读取layout检测框，异常或无效bbox返回None。""" + bbox = item.get("bbox") + if bbox is None or len(bbox) != 4: + return None + + try: + x0, y0, x1, y1 = [float(v) for v in bbox] + except (TypeError, ValueError): + return None + + if x1 < x0 or y1 < y0: + return None + + return [x0, y0, x1, y1] + + +def _bbox_center_point(bbox): + """计算bbox中心点，用于判断行内公式是否落入视觉容器。""" + return (float(bbox[0] + bbox[2]) / 2.0, float(bbox[1] + bbox[3]) / 2.0) + + +def _is_point_inside_bbox(point, bbox): + """判断点是否位于bbox内部，边界点按内部处理。""" + x, y = point + return bbox[0] <= x <= bbox[2] and bbox[1] <= y <= bbox[3] + + +def _is_inline_formula_inside_container(inline_formula_bbox, container_bboxes): + """判断行内公式中心点是否落入任一视觉/行间公式容器。""" + inline_formula_center = _bbox_center_point(inline_formula_bbox) + return any( + _is_point_inside_bbox(inline_formula_center, container_bbox) + for container_bbox in container_bboxes + ) + + +def _filter_inline_formulas_inside_containers(images_layout_res): + """原地移除位于table/image/chart/display_formula内的行内公式。""" + for layout_res in images_layout_res: + container_bboxes = [] + for res in layout_res: + if res.get("label") not in INLINE_FORMULA_CONTAINER_LABELS: + continue + bbox = _layout_item_to_float_bbox(res) + if bbox is not None: + container_bboxes.append(bbox) + + if not container_bboxes: + continue + + kept_layout_res = [] + for res in layout_res: + if res.get("label") != "inline_formula": + kept_layout_res.append(res) + continue + + inline_formula_bbox = _layout_item_to_float_bbox(res) + if inline_formula_bbox is None or not _is_inline_formula_inside_container( + inline_formula_bbox, + container_bboxes, + ): + kept_layout_res.append(res) + + layout_res[:] = kept_layout_res + + def _build_inline_formula_inputs(images_layout_res): inline_formula_inputs = [] for layout_res in images_layout_res: page_inline_formula_inputs = [] for res in layout_res: - if res.get('label') not in ['inline_formula', 'display_formula']: + if res.get('label') != 'inline_formula': continue bbox = res.get('bbox') if bbox is None or len(bbox) != 4: @@ -326,6 +563,28 @@ def _build_formula_mask_inputs(images_layout_res): return page_formula_masks +def _build_inline_formula_det_inputs(images_layout_res): + """从 layout 检测结果提取行内公式框，供 VLM-OCR 作为 OCR det hint 使用。""" + inline_formula_inputs = [] + for layout_res in images_layout_res: + page_inline_formula_inputs = [] + for res in layout_res: + if res.get('label') != 'inline_formula': + continue + bbox = _formula_item_to_pixel_bbox(res) + if bbox is None: + continue + page_inline_formula_inputs.append( + { + "bbox": bbox, + "score": float(res.get('score', 0.0)), + "latex": "", + } + ) + inline_formula_inputs.append(page_inline_formula_inputs) + return inline_formula_inputs + + def _normalize_page_size(page_image): """从PIL或numpy图像中读取页面宽高，供归一化bbox还原为像素bbox。""" if hasattr(page_image, "size"): @@ -411,13 +670,37 @@ def _predict_layout_for_title_split( ) +def _predict_layout_for_window( + images_pil_list, + language, + inline_formula_enable, + batch_ratio, + vlm_ocr_enable, +): + """为单个处理窗口执行一次 pipeline layout，并返回可复用的小模型实例。""" + hybrid_model_singleton = HybridModelSingleton() + hybrid_pipeline_model = hybrid_model_singleton.get_model( + lang=language, + formula_enable=inline_formula_enable and not vlm_ocr_enable, + ) + images_layout_res = _predict_layout_for_title_split( + hybrid_pipeline_model, + images_pil_list, + batch_ratio, + ) + _filter_inline_formulas_inside_containers(images_layout_res) + return images_layout_res, hybrid_pipeline_model + + def _process_ocr_and_formulas( images_pil_list, model_list, - language, inline_formula_enable, _ocr_enable, batch_ratio: int = 1, + *, + images_layout_res, + hybrid_pipeline_model, ): """处理OCR和公式识别""" @@ -428,21 +711,6 @@ def _process_ocr_and_formulas( # 将PIL图片转换为numpy数组 np_images = [np.asarray(pil_image).copy() for pil_image in images_pil_list] - # 获取混合模型实例 - hybrid_model_singleton = HybridModelSingleton() - hybrid_pipeline_model = hybrid_model_singleton.get_model( - lang=language, - formula_enable=inline_formula_enable, - ) - - # 在进行`行内`公式检测和识别前，先将图像中的图片、表格、`行间`公式区域mask掉 - layout_images = mask_image_regions(np_images, model_list) if inline_formula_enable else np_images - images_layout_res = _predict_layout_for_title_split( - hybrid_pipeline_model, - layout_images, - batch_ratio, - ) - if inline_formula_enable: images_mfd_res = _build_inline_formula_inputs(images_layout_res) # 公式识别 @@ -538,61 +806,43 @@ def _process_ocr_and_formulas( if need_ocr_res in page_ocr_res_list: page_ocr_res_list.remove(need_ocr_res) - _apply_layout_title_split( - model_list, - images_layout_res, - [_normalize_page_size(image) for image in images_pil_list], - ) - _normalize_bbox(inline_formula_list, ocr_res_list, images_pil_list) merged_model_list = _merge_page_sidecar_items( model_list, inline_formula_list, ocr_res_list, ) - return merged_model_list, hybrid_pipeline_model + return merged_model_list -def _apply_layout_title_split_for_window( +def _apply_vlm_ocr_det_sidecars_for_window( images_pil_list, model_list, - language, batch_ratio, + *, + images_layout_res, + hybrid_pipeline_model, ): - """为VLM-OCR路径补跑layout小模型，先基于VLM原始title做OCR det，再拆分标题。""" - hybrid_model_singleton = HybridModelSingleton() - hybrid_pipeline_model = hybrid_model_singleton.get_model( - lang=language, - formula_enable=False, - ) - images_layout_res = _predict_layout_for_title_split( - hybrid_pipeline_model, - images_pil_list, - batch_ratio, - ) + """为VLM-OCR路径追加OCR det空文本行和行内公式框sidecar。""" + formula_mask_inputs = _build_formula_mask_inputs(images_layout_res) + inline_formula_list = _build_inline_formula_det_inputs(images_layout_res) np_images = [np.asarray(pil_image).copy() for pil_image in images_pil_list] ocr_res_list = ocr_det( hybrid_pipeline_model, np_images, model_list, - _build_formula_mask_inputs(images_layout_res), + formula_mask_inputs, False, batch_ratio=batch_ratio, fill_text=False, ) - _normalize_bbox([[] for _ in images_pil_list], ocr_res_list, images_pil_list) + _normalize_bbox(inline_formula_list, ocr_res_list, images_pil_list) model_list[:] = _merge_page_sidecar_items( model_list, - [[] for _ in images_pil_list], + inline_formula_list, ocr_res_list, keep_ocr_text=False, ) - _apply_layout_title_split( - model_list, - images_layout_res, - [_normalize_page_size(image) for image in images_pil_list], - ) - return hybrid_pipeline_model def _normalize_bbox( @@ -740,8 +990,10 @@ def doc_analyze( model_path: str | None = None, server_url: str | None = None, image_analysis: bool = True, + mode: str = "pro", **kwargs, ): + mode = _validate_hybrid_mode(mode) client_side_output_generation = bool( kwargs.pop("client_side_output_generation", False) ) @@ -754,7 +1006,11 @@ def doc_analyze( _vlm_ocr_enable = _should_enable_vlm_ocr(_ocr_enable, language, inline_formula_enable) pdf_doc = open_pdfium_document(pdfium.PdfDocument, pdf_bytes) - middle_json = init_middle_json(_ocr_enable, _vlm_ocr_enable) + middle_json = init_middle_json( + _ocr_enable, + _vlm_ocr_enable, + hybrid_mode=mode, + ) model_list = [] doc_closed = False hybrid_pipeline_model = None @@ -789,22 +1045,72 @@ def doc_analyze( ) try: images_pil_list = [image_dict["img_pil"] for image_dict in images_list] + page_sizes = [_normalize_page_size(image) for image in images_pil_list] logger.info( f'Hybrid processing window {window_index + 1}/{total_windows}: ' f'pages {window_start + 1}-{window_end + 1}/{page_count} ' f'({len(images_pil_list)} pages)' ) - if _vlm_ocr_enable: + images_layout_res, hybrid_pipeline_model = _predict_layout_for_window( + images_pil_list, + language, + inline_formula_enable, + batch_ratio, + _vlm_ocr_enable, + ) + if mode == "flash": + _apply_flash_table_orientation_labels( + images_pil_list, + images_layout_res, + hybrid_pipeline_model, + batch_ratio=batch_ratio, + ) + vlm_blocks_list = [ + _build_flash_vlm_layout_blocks( + page_layout_res, + pil_img.width, + pil_img.height, + ) + for page_layout_res, pil_img in zip(images_layout_res, images_pil_list) + ] + with predictor_execution_guard(predictor): + window_model_list = predictor.batch_extract_with_layout( + images_pil_list, + vlm_blocks_list, + not_extract_list=None if _vlm_ocr_enable else not_extract_list, + image_analysis=image_analysis, + ) + optimize_flash_formula_number_blocks(window_model_list) + if _vlm_ocr_enable: + _apply_vlm_ocr_det_sidecars_for_window( + images_pil_list, + window_model_list, + batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, + ) + else: + window_model_list = _process_ocr_and_formulas( + images_pil_list, + window_model_list, + inline_formula_enable, + _ocr_enable, + batch_ratio=batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, + ) + elif _vlm_ocr_enable: with predictor_execution_guard(predictor): window_model_list = predictor.batch_two_step_extract( images=images_pil_list, image_analysis=image_analysis, ) - hybrid_pipeline_model = _apply_layout_title_split_for_window( + _apply_vlm_ocr_det_sidecars_for_window( images_pil_list, window_model_list, - language, batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, ) else: with predictor_execution_guard(predictor): @@ -813,15 +1119,21 @@ def doc_analyze( not_extract_list=not_extract_list, image_analysis=image_analysis, ) - window_model_list, hybrid_pipeline_model = _process_ocr_and_formulas( + window_model_list = _process_ocr_and_formulas( images_pil_list, window_model_list, - language, inline_formula_enable, _ocr_enable, batch_ratio=batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, ) + _apply_layout_title_split( + window_model_list, + images_layout_res, + page_sizes, + ) model_list.extend(window_model_list) if progress_bar is None: progress_bar = tqdm(total=page_count, desc="Processing pages") @@ -869,6 +1181,7 @@ def doc_analyze( hybrid_pipeline_model, _ocr_enable, _vlm_ocr_enable, + hybrid_mode=mode, ) close_pdfium_document(pdf_doc) doc_closed = True @@ -890,8 +1203,10 @@ async def aio_doc_analyze( model_path: str | None = None, server_url: str | None = None, image_analysis: bool = True, + mode: str = "pro", **kwargs, ): + mode = _validate_hybrid_mode(mode) client_side_output_generation = bool( kwargs.pop("client_side_output_generation", False) ) @@ -904,7 +1219,11 @@ async def aio_doc_analyze( _vlm_ocr_enable = _should_enable_vlm_ocr(_ocr_enable, language, inline_formula_enable) pdf_doc = open_pdfium_document(pdfium.PdfDocument, pdf_bytes) - middle_json = init_middle_json(_ocr_enable, _vlm_ocr_enable) + middle_json = init_middle_json( + _ocr_enable, + _vlm_ocr_enable, + hybrid_mode=mode, + ) model_list = [] doc_closed = False hybrid_pipeline_model = None @@ -938,23 +1257,77 @@ async def aio_doc_analyze( ) try: images_pil_list = [image_dict["img_pil"] for image_dict in images_list] + page_sizes = [_normalize_page_size(image) for image in images_pil_list] logger.info( f'Hybrid processing window {window_index + 1}/{total_windows}: ' f'pages {window_start + 1}-{window_end + 1}/{page_count} ' f'({len(images_pil_list)} pages)' ) - if _vlm_ocr_enable: + images_layout_res, hybrid_pipeline_model = await asyncio.to_thread( + _predict_layout_for_window, + images_pil_list, + language, + inline_formula_enable, + batch_ratio, + _vlm_ocr_enable, + ) + if mode == "flash": + await asyncio.to_thread( + _apply_flash_table_orientation_labels, + images_pil_list, + images_layout_res, + hybrid_pipeline_model, + batch_ratio, + ) + vlm_blocks_list = [ + _build_flash_vlm_layout_blocks( + page_layout_res, + pil_img.width, + pil_img.height, + ) + for page_layout_res, pil_img in zip(images_layout_res, images_pil_list) + ] + async with aio_predictor_execution_guard(predictor): + window_model_list = await predictor.aio_batch_extract_with_layout( + images_pil_list, + vlm_blocks_list, + not_extract_list=None if _vlm_ocr_enable else not_extract_list, + image_analysis=image_analysis, + ) + optimize_flash_formula_number_blocks(window_model_list) + if _vlm_ocr_enable: + await asyncio.to_thread( + _apply_vlm_ocr_det_sidecars_for_window, + images_pil_list, + window_model_list, + batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, + ) + else: + window_model_list = await asyncio.to_thread( + _process_ocr_and_formulas, + images_pil_list, + window_model_list, + inline_formula_enable, + _ocr_enable, + batch_ratio=batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, + ) + elif _vlm_ocr_enable: async with aio_predictor_execution_guard(predictor): window_model_list = await predictor.aio_batch_two_step_extract( images=images_pil_list, image_analysis=image_analysis, ) - hybrid_pipeline_model = await asyncio.to_thread( - _apply_layout_title_split_for_window, + await asyncio.to_thread( + _apply_vlm_ocr_det_sidecars_for_window, images_pil_list, window_model_list, - language, batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, ) else: async with aio_predictor_execution_guard(predictor): @@ -963,16 +1336,23 @@ async def aio_doc_analyze( not_extract_list=not_extract_list, image_analysis=image_analysis, ) - window_model_list, hybrid_pipeline_model = await asyncio.to_thread( + window_model_list = await asyncio.to_thread( _process_ocr_and_formulas, images_pil_list, window_model_list, - language, inline_formula_enable, _ocr_enable, batch_ratio=batch_ratio, + images_layout_res=images_layout_res, + hybrid_pipeline_model=hybrid_pipeline_model, ) + await asyncio.to_thread( + _apply_layout_title_split, + window_model_list, + images_layout_res, + page_sizes, + ) model_list.extend(window_model_list) if progress_bar is None: progress_bar = tqdm(total=page_count, desc="Processing pages") @@ -1022,6 +1402,7 @@ async def aio_doc_analyze( hybrid_pipeline_model, _ocr_enable, _vlm_ocr_enable, + hybrid_mode=mode, ) close_pdfium_document(pdf_doc) doc_closed = True diff --git a/mineru/backend/hybrid/hybrid_magic_model.py b/mineru/backend/hybrid/hybrid_magic_model.py index 21555ec2..a2a19629 100644 --- a/mineru/backend/hybrid/hybrid_magic_model.py +++ b/mineru/backend/hybrid/hybrid_magic_model.py @@ -74,7 +74,7 @@ class MagicModel: for inline_formula in self.page_inline_formula: inline_formula["bbox"] = list(self.cal_real_bbox(inline_formula["bbox"])) inline_formula_latex = inline_formula.pop("latex", "") - if inline_formula_latex: + if inline_formula_latex or _vlm_ocr_enable: page_text_inline_formula_spans.append( { "bbox": inline_formula["bbox"], diff --git a/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py b/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py index 06821c64..f6da7d4b 100644 --- a/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py +++ b/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py @@ -180,10 +180,11 @@ def _normalize_split_title_blocks(pdf_info_list): block["level"] = title_level -def init_middle_json(_ocr_enable, _vlm_ocr_enable): +def init_middle_json(_ocr_enable, _vlm_ocr_enable, hybrid_mode="pro"): return { "pdf_info": [], "_backend": "hybrid", + "_hybrid_mode": hybrid_mode, "_ocr_enable": _ocr_enable, "_vlm_ocr_enable": _vlm_ocr_enable, "_version_name": __version__ @@ -260,12 +261,13 @@ def apply_server_side_postprocess( _apply_post_ocr(pdf_info_list, hybrid_pipeline_model) -def finalize_middle_json_from_preproc(pdf_info_list): +def finalize_middle_json_from_preproc(pdf_info_list, hybrid_mode="pro"): """从 Hybrid preproc_blocks 执行完整 finalize，供服务端完整路径和客户端复用。""" build_para_blocks_from_preproc(pdf_info_list) merge_para_text_blocks( pdf_info_list, auto_merge_by_det=True, + auto_merge_vertical_by_det=hybrid_mode == "flash", ) table_enable = get_table_enable(os.getenv('MINERU_VLM_TABLE_ENABLE', 'True').lower() == 'true') @@ -282,6 +284,7 @@ def finalize_middle_json( hybrid_pipeline_model, _ocr_enable, _vlm_ocr_enable, + hybrid_mode="pro", ): """保持旧入口语义：服务端先做必要 post-OCR，再执行完整 finalize。""" apply_server_side_postprocess( @@ -290,7 +293,7 @@ def finalize_middle_json( _ocr_enable, _vlm_ocr_enable, ) - finalize_middle_json_from_preproc(pdf_info_list) + finalize_middle_json_from_preproc(pdf_info_list, hybrid_mode=hybrid_mode) def result_to_middle_json( diff --git a/mineru/backend/pipeline/batch_analyze.py b/mineru/backend/pipeline/batch_analyze.py index 291a01da..bc60237d 100644 --- a/mineru/backend/pipeline/batch_analyze.py +++ b/mineru/backend/pipeline/batch_analyze.py @@ -408,8 +408,6 @@ class BatchAnalyze: # 移除所有的"inline_formula" layout_res[:] = [res for res in layout_res if res.get("label") != "inline_formula"] - - ocr_res_list_all_page = [] table_res_list_all_page = [] for index in range(len(np_images)): @@ -474,6 +472,7 @@ class BatchAnalyze: rotate_labels = table_orientation_cls_model.batch_predict( table_res_list_all_page, det_batch_size=self.batch_ratio * OCR_DET_BASE_BATCH_SIZE, + tqdm_enable=True, ) if len(rotate_labels) != len(table_res_list_all_page): raise ValueError( diff --git a/mineru/backend/pipeline/model_json_to_middle_json.py b/mineru/backend/pipeline/model_json_to_middle_json.py index 5c4cc9b9..dcd66c40 100644 --- a/mineru/backend/pipeline/model_json_to_middle_json.py +++ b/mineru/backend/pipeline/model_json_to_middle_json.py @@ -4,13 +4,13 @@ import copy from tqdm import tqdm from mineru.backend.utils.html_image_utils import replace_inline_table_images +from mineru.backend.utils.formula_number import optimize_formula_number_blocks from mineru.backend.utils.runtime_utils import cross_page_table_merge from mineru.backend.pipeline.model_init import ( AtomModelSingleton, run_ocr_inference, ) from mineru.backend.pipeline.para_split import para_split -from mineru.utils.char_utils import full_to_half from mineru.utils.cut_image import cut_image_and_table from mineru.utils.enum_class import ContentType, BlockType from mineru.utils.title_level_postprocess import apply_title_leveling_to_pdf_info @@ -136,15 +136,6 @@ def append_batch_results_to_middle_json( ) -def _extract_text_from_block(block): - text_parts = [] - for line in block.get("lines", []): - for span in line.get("spans", []): - if span.get("type") == ContentType.TEXT: - text_parts.append(span.get("content", "")) - return "".join(text_parts).strip() - - def _iter_block_spans(block): for line in block.get("lines", []): for span in line.get("spans", []): @@ -154,61 +145,6 @@ def _iter_block_spans(block): yield from _iter_block_spans(sub_block) -def _normalize_formula_tag_content(tag_content): - tag_content = full_to_half(tag_content.strip()) - if tag_content.startswith("("): - tag_content = tag_content[1:].strip() - if tag_content.endswith(")"): - tag_content = tag_content[:-1].strip() - return tag_content - - -def _get_interline_equation_span(block): - for line in block.get("lines", []): - for span in line.get("spans", []): - if span.get("type") == ContentType.INTERLINE_EQUATION: - return span - return None - - -def _append_formula_number_tag(equation_block, formula_number_block): - equation_span = _get_interline_equation_span(equation_block) - tag_content = _normalize_formula_tag_content(_extract_text_from_block(formula_number_block)) - if equation_span is not None: - formula = equation_span.get("content", "") - equation_span["content"] = f"{formula}\\tag{{{tag_content}}}" - - -def _optimize_formula_number_blocks(pdf_info_list): - for page_info in pdf_info_list: - optimized_blocks = [] - blocks = page_info.get("preproc_blocks", []) - for index, block in enumerate(blocks): - if block.get("type") != BlockType.FORMULA_NUMBER: - optimized_blocks.append(block) - continue - - prev_block = blocks[index - 1] if index > 0 else None - if prev_block and prev_block.get("type") == BlockType.INTERLINE_EQUATION: - _append_formula_number_tag(prev_block, block) - continue - - next_block = blocks[index + 1] if index + 1 < len(blocks) else None - next_next_block = blocks[index + 2] if index + 2 < len(blocks) else None - if ( - next_block - and next_block.get("type") == BlockType.INTERLINE_EQUATION - and (next_next_block is None or next_next_block.get("type") != BlockType.FORMULA_NUMBER) - ): - _append_formula_number_tag(next_block, block) - continue - - block["type"] = BlockType.TEXT - optimized_blocks.append(block) - - page_info["preproc_blocks"] = optimized_blocks - - def _apply_post_ocr(pdf_info_list, lang=None): need_ocr_list = [] img_crop_list = [] @@ -279,7 +215,7 @@ def apply_server_side_postprocess(pdf_info_list, lang=None): def finalize_middle_json_from_preproc(pdf_info_list): """从 preproc_blocks 执行确定性 finalize，供服务端完整路径和客户端复用。""" - _optimize_formula_number_blocks(pdf_info_list) + optimize_formula_number_blocks(pdf_info_list) para_split(pdf_info_list) cross_page_table_merge(pdf_info_list) apply_title_leveling_to_pdf_info(pdf_info_list) diff --git a/mineru/backend/pipeline/pipeline_analyze.py b/mineru/backend/pipeline/pipeline_analyze.py index 1fcb517a..b288712b 100644 --- a/mineru/backend/pipeline/pipeline_analyze.py +++ b/mineru/backend/pipeline/pipeline_analyze.py @@ -376,7 +376,13 @@ def batch_image_analyze( os.environ["TORCH_CUDNN_V8_API_DISABLED"] = "1" enable_ocr_det_batch = True - batch_model = BatchAnalyze(model_manager, batch_ratio, formula_enable, table_enable, enable_ocr_det_batch) + batch_model = BatchAnalyze( + model_manager, + batch_ratio, + formula_enable, + table_enable, + enable_ocr_det_batch, + ) results = batch_model(images_with_extra_info) clean_memory(get_device()) diff --git a/mineru/backend/pipeline/pipeline_magic_model.py b/mineru/backend/pipeline/pipeline_magic_model.py index fa82e223..30f43b05 100644 --- a/mineru/backend/pipeline/pipeline_magic_model.py +++ b/mineru/backend/pipeline/pipeline_magic_model.py @@ -1,10 +1,7 @@ # Copyright (c) Opendatalab. All rights reserved. from mineru.backend.pipeline.para_split import ListLineTag from mineru.backend.pipeline.pipeline_middle_json_mkcontent import _merge_para_text -from mineru.utils.boxbase import ( - calculate_overlap_area_2_minbox_area_ratio, - calculate_overlap_area_in_bbox1_area_ratio, -) +from mineru.backend.utils.formula_number import formula_number_max_overlap_ratio from mineru.utils.enum_class import ContentType, BlockType from mineru.utils.guess_suffix_or_lang import guess_language_by_text from mineru.utils.span_block_fix import merge_spans_to_vertical_line, vertical_line_sort_spans_from_top_to_bottom, \ @@ -286,7 +283,7 @@ class MagicModel: if block["type"] == BlockType.FORMULA_NUMBER: block_spans = span_matcher.collect_for_block( block["bbox"], - overlap_ratio_getter=self.__formula_number_overlap_ratio, + overlap_ratio_getter=formula_number_max_overlap_ratio, ) else: block_spans = span_matcher.collect_for_block(block["bbox"]) @@ -295,14 +292,6 @@ class MagicModel: block = self.__fix_text_block(block) self.page_text_inline_formula_spans = span_matcher.remaining_spans() - @staticmethod - def __formula_number_overlap_ratio(span, block_bbox): - """公式编号框较窄时，沿用最小框重叠比例提高回填召回。""" - return max( - calculate_overlap_area_in_bbox1_area_ratio(span['bbox'], block_bbox), - calculate_overlap_area_2_minbox_area_ratio(span['bbox'], block_bbox), - ) - def __fix_axis(self): need_remove_list = [] layout_dets = self.__page_model_info["layout_dets"] diff --git a/mineru/backend/utils/formula_number.py b/mineru/backend/utils/formula_number.py new file mode 100644 index 00000000..3a29efae --- /dev/null +++ b/mineru/backend/utils/formula_number.py @@ -0,0 +1,161 @@ +# Copyright (c) Opendatalab. All rights reserved. +from collections.abc import Callable, Iterable, Sequence +from typing import Any + +from mineru.utils.boxbase import ( + calculate_overlap_area_2_minbox_area_ratio, + calculate_overlap_area_in_bbox1_area_ratio, +) +from mineru.utils.char_utils import full_to_half +from mineru.utils.enum_class import BlockType, ContentType +from mineru.utils.visual_magic_model_utils import isolated_formula_clean + +Block = dict[str, Any] + + +def formula_number_max_overlap_ratio(span: Block, block_bbox: Sequence[float]) -> float: + """取公式编号span与block的两种重叠比例最大值，兼容block窄于span的情况。""" + return max( + calculate_overlap_area_in_bbox1_area_ratio(span["bbox"], block_bbox), + calculate_overlap_area_2_minbox_area_ratio(span["bbox"], block_bbox), + ) + + +def extract_formula_number_text(block: Block) -> str: + """从公式编号块中提取文本，优先使用VLM直接返回的content。""" + content = block.get("content") + if isinstance(content, str) and content.strip(): + return content.strip() + + text_parts = [] + for line in block.get("lines", []): + for span in line.get("spans", []): + if span.get("type") == ContentType.TEXT: + text_parts.append(span.get("content", "")) + return "".join(text_parts).strip() + + +def normalize_formula_tag_content(tag_content: str) -> str: + """归一化公式编号文本，去掉外层括号并转换全角字符。""" + tag_content = full_to_half(tag_content.strip()) + if tag_content.startswith("("): + tag_content = tag_content[1:].strip() + if tag_content.endswith(")"): + tag_content = tag_content[:-1].strip() + return tag_content + + +def _normalize_formula_content_for_tag(formula_content: str) -> str: + """归一化待合并编号的公式正文，去掉VLM可能携带的展示公式分隔符。""" + return isolated_formula_clean(formula_content or "") + + +def build_tagged_formula_content( + formula_content: str, + formula_number_block: Block, +) -> str: + """将公式正文和公式编号合成带LaTeX tag的公式内容。""" + formula_content = _normalize_formula_content_for_tag(formula_content) + tag_content = normalize_formula_tag_content( + extract_formula_number_text(formula_number_block) + ) + return f"{formula_content}\\tag{{{tag_content}}}" + + +def get_interline_equation_span(block: Block) -> Block | None: + """查找行间公式块中的公式span。""" + for line in block.get("lines", []): + for span in line.get("spans", []): + if span.get("type") == ContentType.INTERLINE_EQUATION: + return span + return None + + +def append_formula_number_tag( + equation_block: Block, + formula_number_block: Block, +) -> None: + """将公式编号写入pipeline middle-json行间公式span。""" + equation_span = get_interline_equation_span(equation_block) + if equation_span is not None: + equation_span["content"] = build_tagged_formula_content( + equation_span.get("content", ""), + formula_number_block, + ) + + +def _optimize_formula_number_sequence( + blocks: Sequence[Block], + is_formula_number: Callable[[Block], bool], + is_equation: Callable[[Block], bool], + append_tag: Callable[[Block, Block], None], + downgrade_block: Callable[[Block], None], +) -> list[Block]: + """按统一相邻规则优化公式编号序列，调用方负责适配不同block结构。""" + optimized_blocks = [] + for index, block in enumerate(blocks): + if not is_formula_number(block): + optimized_blocks.append(block) + continue + + prev_block = blocks[index - 1] if index > 0 else None + if prev_block and is_equation(prev_block): + append_tag(prev_block, block) + continue + + next_block = blocks[index + 1] if index + 1 < len(blocks) else None + next_next_block = blocks[index + 2] if index + 2 < len(blocks) else None + if ( + next_block + and is_equation(next_block) + and (next_next_block is None or not is_formula_number(next_next_block)) + ): + append_tag(next_block, block) + continue + + downgrade_block(block) + optimized_blocks.append(block) + + return optimized_blocks + + +def _downgrade_formula_number_to_text(block: Block) -> None: + """将未匹配公式编号降级为普通文本块。""" + block["type"] = BlockType.TEXT + + +def _append_flash_formula_number_tag( + equation_block: Block, + formula_number_block: Block, +) -> None: + """将公式编号写入Hybrid flash的VLM行间公式内容。""" + equation_block["content"] = build_tagged_formula_content( + equation_block.get("content", ""), + formula_number_block, + ) + + +def optimize_formula_number_blocks(pdf_info_list: Iterable[Block]) -> None: + """按pipeline规则合并公式编号块，未匹配的编号降级为普通文本。""" + for page_info in pdf_info_list: + blocks = page_info.get("preproc_blocks", []) + page_info["preproc_blocks"] = _optimize_formula_number_sequence( + blocks, + lambda block: block.get("type") == BlockType.FORMULA_NUMBER, + lambda block: block.get("type") == BlockType.INTERLINE_EQUATION, + append_formula_number_tag, + _downgrade_formula_number_to_text, + ) + + +def optimize_flash_formula_number_blocks(model_list: Iterable[list[Block]]) -> None: + """按统一相邻规则处理Hybrid flash的VLM公式编号块。""" + for page_model_list in model_list: + optimized_blocks = _optimize_formula_number_sequence( + page_model_list or [], + lambda block: block.get("type") == BlockType.FORMULA_NUMBER, + lambda block: block.get("type") == BlockType.EQUATION, + _append_flash_formula_number_tag, + _downgrade_formula_number_to_text, + ) + page_model_list[:] = optimized_blocks diff --git a/mineru/backend/utils/para_block_utils.py b/mineru/backend/utils/para_block_utils.py index d66f2dac..315ce3e6 100644 --- a/mineru/backend/utils/para_block_utils.py +++ b/mineru/backend/utils/para_block_utils.py @@ -2,6 +2,7 @@ import copy from mineru.utils.enum_class import BlockType, SplitFlag +from mineru.utils.span_block_fix import is_vertical_text_block_by_spans LINE_STOP_FLAG = ('.', '!', '?', '。', '！', '？', ')', '）', '"', '”', ':', '：', ';', '；') @@ -43,7 +44,11 @@ def build_para_blocks_from_preproc(pdf_info_list): page_info["para_blocks"] = copy.deepcopy(page_info.get("preproc_blocks", [])) -def merge_para_text_blocks(pdf_info_list, auto_merge_by_det=False): +def merge_para_text_blocks( + pdf_info_list, + auto_merge_by_det=False, + auto_merge_vertical_by_det=False, +): ordered_blocks = [] for page_info in pdf_info_list: page_idx = page_info.get("page_idx") @@ -62,6 +67,7 @@ def merge_para_text_blocks(pdf_info_list, auto_merge_by_det=False): current_page_idx, current_block, auto_merge_by_det, + auto_merge_vertical_by_det, ) elif current_type == BlockType.LIST: if not current_block.get("blocks"): @@ -79,6 +85,7 @@ def _merge_current_text_block( current_page_idx, current_block, auto_merge_by_det, + auto_merge_vertical_by_det, ): """处理当前 text block 的 merge_prev 候选合并和 Hybrid det 自动合并。""" previous_block = None @@ -107,6 +114,7 @@ def _merge_current_text_block( if not can_auto_merge_text_blocks( current_block, previous_text_block, + allow_vertical_blocks=auto_merge_vertical_by_det, ): previous_block = None @@ -148,6 +156,7 @@ def can_auto_merge_text_blocks( current_block, previous_block, allow_single_line_blocks=False, + allow_vertical_blocks=False, ): """按段落首尾文本和行几何规则判断 text 是否可合并。""" current_lines = current_block.get("lines", []) @@ -162,6 +171,20 @@ def can_auto_merge_text_blocks( ): return False + if ( + allow_vertical_blocks + and _is_vertical_text_block_by_lines(current_metric_lines) + and _is_vertical_text_block_by_lines(previous_metric_lines) + ): + return _can_auto_merge_vertical_text_blocks( + current_block, + previous_block, + current_lines, + previous_lines, + current_metric_lines, + previous_metric_lines, + ) + first_metric_line = current_metric_lines[0] last_metric_line = previous_metric_lines[-1] first_line_height = _line_height(first_metric_line) @@ -339,6 +362,14 @@ def _line_height(line): return bbox[3] - bbox[1] +def _line_width(line): + """计算行或纵排列的宽度，供纵排几何规则使用。""" + bbox = line.get("bbox") + if not bbox: + return 0 + return bbox[2] - bbox[0] + + def _build_bbox_fs(block, lines): if lines: return [ @@ -379,6 +410,56 @@ def _has_mergeable_block_bbox_relation(current_block, previous_block): return current_block["bbox"][1] < previous_block["bbox"][3] +def _is_vertical_text_block_by_lines(lines): + """使用当前几何行中的 spans 判断文本块是否为纵排。""" + spans = [ + span + for line in lines + for span in line.get("spans", []) + ] + return is_vertical_text_block_by_spans(spans) + + +def _can_auto_merge_vertical_text_blocks( + current_block, + previous_block, + current_lines, + previous_lines, + current_metric_lines, + previous_metric_lines, +): + """复刻 pipeline 纵排文本块合并规则，几何优先使用 OCR det 行。""" + first_metric_line = current_metric_lines[0] + last_metric_line = previous_metric_lines[-1] + first_line_width = _line_width(first_metric_line) + last_line_width = _line_width(last_metric_line) + if first_line_width <= 0 or last_line_width <= 0: + return False + + current_bbox_fs = _build_bbox_fs(current_block, current_metric_lines) + previous_bbox_fs = _build_bbox_fs(previous_block, previous_metric_lines) + if abs(current_bbox_fs[1] - first_metric_line["bbox"][1]) >= first_line_width / 2: + return False + if abs(previous_bbox_fs[3] - last_metric_line["bbox"][3]) >= last_line_width: + return False + + first_content = _first_non_empty_content(current_lines) + last_content = _last_non_empty_content(previous_lines) + if not first_content or not last_content: + return False + if last_content.endswith(LINE_STOP_FLAG): + return False + if first_content[0].isdigit() or first_content[0].isupper(): + return False + + current_metric_height = current_bbox_fs[3] - current_bbox_fs[1] + previous_metric_height = previous_bbox_fs[3] - previous_bbox_fs[1] + min_metric_height = min(current_metric_height, previous_metric_height) + if min_metric_height <= 0: + return False + return abs(current_metric_height - previous_metric_height) < min_metric_height + + def _cleanup_block_internal_metadata(block): """递归清理只供 finalize 内部流程使用的临时字段。""" for metadata_key in INTERNAL_BLOCK_METADATA_KEYS: diff --git a/mineru/cli/api_request.py b/mineru/cli/api_request.py index a90cb8d1..ddc82b8f 100644 --- a/mineru/cli/api_request.py +++ b/mineru/cli/api_request.py @@ -4,6 +4,11 @@ from typing import Annotated, Optional from fastapi import File, Form, HTTPException, Request, UploadFile +from mineru.cli.backend_options import ( + BACKEND_SCHEMA_EXTRA, + DEFAULT_BACKEND, + validate_backend as validate_public_backend, +) from mineru.cli.public_http_client_policy import validate_public_http_client_request ALLOWED_PARSE_METHODS = {"auto", "txt", "ocr"} @@ -51,6 +56,14 @@ def validate_parse_method(parse_method: str) -> str: return parse_method +def validate_parse_backend(backend: str) -> str: + """校验公开 API 允许的解析后端，避免旧入口名进入下游执行链路。""" + try: + return validate_public_backend(backend) + except ValueError as exc: + raise HTTPException(status_code=400, detail=str(exc)) from exc + + async def parse_request_form( request: Request, files: Annotated[ @@ -89,12 +102,15 @@ async def parse_request_form( Form( description="""The backend for parsing: - pipeline: More general, supports multiple languages, hallucination-free. -- vlm-auto-engine: High accuracy via local computing power, supports Chinese and English documents only. +- vlm-engine: High accuracy via local computing power, supports Chinese and English documents only. - vlm-http-client: High accuracy via remote computing power(client suitable for openai-compatible servers), supports Chinese and English documents only. -- hybrid-auto-engine: Next-generation high accuracy solution via local computing power, supports multiple languages. -- hybrid-http-client: High accuracy via remote computing power but requires a little local computing power(client suitable for openai-compatible servers), supports multiple languages.""", +- hybrid-engine: Next-generation high accuracy solution via local computing power, supports multiple languages. +- hybrid-flash-engine: Hybrid flash mode via local computing power, supports multiple languages. +- hybrid-http-client: High accuracy via remote computing power but requires a little local computing power(client suitable for openai-compatible servers), supports multiple languages. +- hybrid-flash-http-client: Hybrid flash mode via remote computing power(client suitable for openai-compatible servers), supports multiple languages.""", + json_schema_extra=BACKEND_SCHEMA_EXTRA, ), - ] = "hybrid-auto-engine", + ] = DEFAULT_BACKEND, parse_method: Annotated[ str, Form( @@ -175,6 +191,7 @@ async def parse_request_form( ] = 99999, ) -> ParseRequestOptions: """解析 API/Router 共用的 multipart 表单，并保持 Swagger 参数同源。""" + backend = validate_parse_backend(backend) validate_public_http_client_request( public_bind_exposed=bool( getattr(request.app.state, "public_bind_exposed", False) diff --git a/mineru/cli/backend_options.py b/mineru/cli/backend_options.py new file mode 100644 index 00000000..5796d562 --- /dev/null +++ b/mineru/cli/backend_options.py @@ -0,0 +1,52 @@ +# Copyright (c) Opendatalab. All rights reserved. + +BACKEND_PIPELINE = "pipeline" +BACKEND_VLM_ENGINE = "vlm-engine" +BACKEND_HYBRID_ENGINE = "hybrid-engine" +BACKEND_HYBRID_FLASH_ENGINE = "hybrid-flash-engine" +BACKEND_VLM_HTTP_CLIENT = "vlm-http-client" +BACKEND_HYBRID_HTTP_CLIENT = "hybrid-http-client" +BACKEND_HYBRID_FLASH_HTTP_CLIENT = "hybrid-flash-http-client" + +DEFAULT_BACKEND = BACKEND_HYBRID_FLASH_ENGINE + +LOCAL_BACKEND_CHOICES = ( + BACKEND_PIPELINE, + BACKEND_VLM_ENGINE, + BACKEND_HYBRID_ENGINE, + BACKEND_HYBRID_FLASH_ENGINE, +) +HTTP_CLIENT_BACKEND_CHOICES = ( + BACKEND_VLM_HTTP_CLIENT, + BACKEND_HYBRID_HTTP_CLIENT, + BACKEND_HYBRID_FLASH_HTTP_CLIENT, +) +PUBLIC_BACKEND_CHOICES = LOCAL_BACKEND_CHOICES + HTTP_CLIENT_BACKEND_CHOICES +BACKEND_SCHEMA_EXTRA = {"enum": list(PUBLIC_BACKEND_CHOICES)} +LEGACY_BACKEND_ALIASES = { + "vlm-auto-engine": BACKEND_VLM_ENGINE, + "hybrid-auto-engine": BACKEND_HYBRID_ENGINE, + "hybrid-flash-auto-engine": BACKEND_HYBRID_FLASH_ENGINE, +} + + +def get_backend_choices(include_http_client: bool = True) -> list[str]: + """按入口配置返回公开 backend 选项，避免各入口重复维护字符串列表。""" + choices = list(LOCAL_BACKEND_CHOICES) + if include_http_client: + choices.extend(HTTP_CLIENT_BACKEND_CHOICES) + return choices + + +def normalize_backend(backend: str) -> str: + """将旧 backend 别名规范为当前公开名称，并校验最终名称是否合法。""" + normalized_backend = LEGACY_BACKEND_ALIASES.get(backend, backend) + if normalized_backend not in PUBLIC_BACKEND_CHOICES: + allowed_values = ", ".join(PUBLIC_BACKEND_CHOICES) + raise ValueError(f"Invalid backend. Allowed values: {allowed_values}") + return normalized_backend + + +def validate_backend(backend: str) -> str: + """校验公开入口允许的 backend 名称，并返回规范后的后端名称。""" + return normalize_backend(backend) diff --git a/mineru/cli/client.py b/mineru/cli/client.py index c00c68d5..02dcb1bd 100644 --- a/mineru/cli/client.py +++ b/mineru/cli/client.py @@ -18,6 +18,11 @@ from mineru.cli.api_protocol import ( DEFAULT_MAX_CONCURRENT_REQUESTS, DEFAULT_PROCESSING_WINDOW_SIZE, ) +from mineru.cli.backend_options import ( + DEFAULT_BACKEND, + PUBLIC_BACKEND_CHOICES, + normalize_backend, +) from mineru.utils.config_reader import ( get_max_concurrent_requests as read_max_concurrent_requests, ) @@ -85,6 +90,18 @@ SubmitResponse = _api_client.SubmitResponse LocalAPIServer = _api_client.LocalAPIServer +def normalize_backend_option( + ctx: click.Context, + param: click.Parameter, + value: str, +) -> str: + """将 CLI 输入的旧 backend 名称规范为当前公开名称。""" + try: + return normalize_backend(value) + except ValueError as exc: + raise click.BadParameter(str(exc), ctx=ctx, param=param) from exc + + @dataclass(frozen=True) class TaskFailure: task_index: int @@ -1029,24 +1046,20 @@ async def run_orchestrated_cli( "-b", "--backend", "backend", - type=click.Choice( - [ - "pipeline", - "vlm-http-client", - "hybrid-http-client", - "vlm-auto-engine", - "hybrid-auto-engine", - ] - ), - default="hybrid-auto-engine", + type=str, + default=DEFAULT_BACKEND, + callback=normalize_backend_option, + metavar="[" + "|".join(PUBLIC_BACKEND_CHOICES) + "]", help="""\b the backend for parsing pdf: pipeline: More general. - vlm-auto-engine: High accuracy via local computing power. + vlm-engine: High accuracy via local computing power. vlm-http-client: High accuracy via remote computing power(client suitable for openai-compatible servers). - hybrid-auto-engine: Next-generation high accuracy solution via local computing power. + hybrid-engine: Next-generation high accuracy solution via local computing power. + hybrid-flash-engine: Hybrid flash mode via local computing power. hybrid-http-client: High accuracy but requires a little local computing power(client suitable for openai-compatible servers). - Without method specified, hybrid-auto-engine will be used by default.""", + hybrid-flash-http-client: Hybrid flash mode via remote computing power(client suitable for openai-compatible servers). + Without backend specified, hybrid-flash-engine will be used by default.""", ) @click.option( "-l", diff --git a/mineru/cli/common.py b/mineru/cli/common.py index f029dd76..e95496bf 100644 --- a/mineru/cli/common.py +++ b/mineru/cli/common.py @@ -10,6 +10,7 @@ from typing import Sequence from loguru import logger +from mineru.cli.backend_options import normalize_backend from mineru.data.data_reader_writer import FileBasedDataWriter from mineru.utils.draw_bbox import draw_layout_bbox, draw_span_bbox from mineru.utils.engine_utils import get_vlm_engine @@ -69,9 +70,11 @@ def ensure_backend_dependencies(backend: str) -> None: def _load_hybrid_analyze_entrypoint(entrypoint_name: str, backend: str): + """加载统一 hybrid analyze 入口，flash/pro 由调用方通过 mode 控制。""" ensure_backend_dependencies(backend) + module_name = "mineru.backend.hybrid.hybrid_analyze" try: - hybrid_analyze = importlib.import_module("mineru.backend.hybrid.hybrid_analyze") + hybrid_analyze = importlib.import_module(module_name) except (ImportError, ModuleNotFoundError) as exc: raise HybridDependencyError( build_hybrid_dependency_error_message(backend) @@ -515,6 +518,7 @@ def _process_hybrid( f_dump_content_list, f_make_md_mode, server_url=None, + mode="pro", **kwargs, ): hybrid_doc_analyze = _load_hybrid_analyze_entrypoint( @@ -538,6 +542,7 @@ def _process_hybrid( language=lang, inline_formula_enable=inline_formula_enable, server_url=server_url, + mode=mode, **kwargs, ) @@ -571,6 +576,7 @@ async def _async_process_hybrid( f_dump_content_list, f_make_md_mode, server_url=None, + mode="pro", **kwargs, ): aio_hybrid_doc_analyze = _load_hybrid_analyze_entrypoint( @@ -594,6 +600,7 @@ async def _async_process_hybrid( language=lang, inline_formula_enable=inline_formula_enable, server_url=server_url, + mode=mode, **kwargs, ) @@ -684,6 +691,7 @@ def do_parse( client_side_output_generation=False, **kwargs, ): + backend = normalize_backend(backend) need_remove_index = _process_office_doc( output_dir, pdf_file_names=pdf_file_names, @@ -718,10 +726,7 @@ def do_parse( if backend.startswith("vlm-"): backend = backend[4:] - if backend == "vllm-async-engine": - raise Exception("vlm-vllm-async-engine backend is not supported in sync mode, please use vlm-vllm-engine backend") - - if backend == "auto-engine": + if backend == "engine": backend = get_vlm_engine(inference_engine='auto', is_async=False) os.environ['MINERU_VLM_FORMULA_ENABLE'] = str(formula_enable) @@ -737,12 +742,12 @@ def do_parse( elif backend.startswith("hybrid-"): ensure_backend_dependencies(backend) backend = backend[7:] + mode = "flash" if backend.startswith("flash-") else "pro" - if backend == "vllm-async-engine": - raise Exception( - "hybrid-vllm-async-engine backend is not supported in sync mode, please use hybrid-vllm-engine backend") + if mode == "flash": + backend = backend[6:] - if backend == "auto-engine": + if backend == "engine": backend = get_vlm_engine(inference_engine='auto', is_async=False) os.environ['MINERU_VLM_TABLE_ENABLE'] = str(table_enable) @@ -752,7 +757,7 @@ def do_parse( output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, backend, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, - server_url, image_analysis=image_analysis, + server_url, mode=mode, image_analysis=image_analysis, client_side_output_generation=client_side_output_generation, **kwargs, ) @@ -781,6 +786,7 @@ async def aio_do_parse( client_side_output_generation=False, **kwargs, ): + backend = normalize_backend(backend) # Office 解析是同步且可能耗时的操作，异步入口需要放到线程中避免阻塞事件循环。 need_remove_index = await asyncio.to_thread( _process_office_doc, @@ -818,10 +824,7 @@ async def aio_do_parse( if backend.startswith("vlm-"): backend = backend[4:] - if backend == "vllm-engine": - raise Exception("vlm-vllm-engine backend is not supported in async mode, please use vlm-vllm-async-engine backend") - - if backend == "auto-engine": + if backend == "engine": backend = get_vlm_engine(inference_engine='auto', is_async=True) os.environ['MINERU_VLM_FORMULA_ENABLE'] = str(formula_enable) @@ -837,11 +840,12 @@ async def aio_do_parse( elif backend.startswith("hybrid-"): ensure_backend_dependencies(backend) backend = backend[7:] + mode = "flash" if backend.startswith("flash-") else "pro" - if backend == "vllm-engine": - raise Exception("hybrid-vllm-engine backend is not supported in async mode, please use hybrid-vllm-async-engine backend") + if mode == "flash": + backend = backend[6:] - if backend == "auto-engine": + if backend == "engine": backend = get_vlm_engine(inference_engine='auto', is_async=True) os.environ['MINERU_VLM_TABLE_ENABLE'] = str(table_enable) @@ -851,7 +855,7 @@ async def aio_do_parse( output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, backend, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, - server_url, image_analysis=image_analysis, + server_url, mode=mode, image_analysis=image_analysis, client_side_output_generation=client_side_output_generation, **kwargs, ) diff --git a/mineru/cli/gradio_app.py b/mineru/cli/gradio_app.py index e305d69f..33229505 100644 --- a/mineru/cli/gradio_app.py +++ b/mineru/cli/gradio_app.py @@ -38,6 +38,11 @@ from mineru.cli.common import ( read_fn, ) from mineru.cli import api_client as _api_client +from mineru.cli.backend_options import ( + DEFAULT_BACKEND, + HTTP_CLIENT_BACKEND_CHOICES, + LOCAL_BACKEND_CHOICES, +) from mineru.cli.client_side_output import regenerate_client_side_outputs from mineru.cli.output_paths import resolve_parse_dir from mineru.cli.vlm_preload import resolve_gradio_local_api_cli_args @@ -224,15 +229,8 @@ STATUS_QUEUED_ON_SERVER = "Queued on server" STATUS_PROCESSING_ON_SERVER = "Processing on server" STATUS_QUEUED_LOCALLY_PREFIX = "Queued locally:" -BACKEND_CHOICE_DEFINITIONS = [ - "pipeline", - "vlm-auto-engine", - "hybrid-auto-engine", -] -HTTP_CLIENT_BACKEND_CHOICE_DEFINITIONS = [ - "vlm-http-client", - "hybrid-http-client", -] +BACKEND_CHOICE_DEFINITIONS = list(LOCAL_BACKEND_CHOICES) +HTTP_CLIENT_BACKEND_CHOICE_DEFINITIONS = list(HTTP_CLIENT_BACKEND_CHOICES) STATUS_STEP_DEFINITIONS = [ ("status_step_prepare", STATUS_PREPARING_REQUEST), ("status_step_check", STATUS_CHECKING_SERVER), @@ -340,6 +338,21 @@ def is_http_client_backend(backend_choice): return isinstance(backend_choice, str) and backend_choice.endswith("-http-client") +def select_backend_info_key(backend_choice): + """根据解析后端选择说明文案的 i18n key，保证 flash 与普通 hybrid 可分别描述。""" + if not isinstance(backend_choice, str): + return "backend_info_default" + if backend_choice.startswith("vlm"): + return "backend_info_vlm" + if backend_choice == "pipeline": + return "backend_info_pipeline" + if backend_choice.startswith("hybrid-flash"): + return "backend_info_hybrid_flash" + if backend_choice.startswith("hybrid"): + return "backend_info_hybrid" + return "backend_info_default" + + def resolve_status_step_index(status_lines): """根据现有状态日志推断步骤面板中当前应高亮的步骤索引。""" if not status_lines: @@ -1596,6 +1609,7 @@ def main(ctx, "backend_info_vlm": "High-precision parsing via VLM, supports Chinese and English documents only.", "backend_info_pipeline": "Traditional Multi-model pipeline parsing, supports multiple languages, hallucination-free.", "backend_info_hybrid": "High-precision hybrid parsing, supports multiple languages.", + "backend_info_hybrid_flash": "Fast, high-precision hybrid parsing, supports multiple languages.", "backend_info_default": "Select the backend engine for document parsing.", }, zh={ @@ -1664,6 +1678,7 @@ def main(ctx, "backend_info_vlm": "多模态大模型高精度解析，仅支持中英文文档。", "backend_info_pipeline": "传统多模型管道解析，支持多语言，无幻觉。", "backend_info_hybrid": "高精度混合解析，支持多语言。", + "backend_info_hybrid_flash": "高精度快速混合解析，支持多语言。", "backend_info_default": "选择文档解析的后端引擎。", }, ) @@ -1690,14 +1705,7 @@ def main(ctx, return "" def get_backend_info(backend_choice): - if backend_choice.startswith("vlm"): - return i18n("backend_info_vlm") - elif backend_choice == "pipeline": - return i18n("backend_info_pipeline") - elif backend_choice.startswith("hybrid"): - return i18n("backend_info_hybrid") - else: - return i18n("backend_info_default") + return i18n(select_backend_info_key(backend_choice)) # 更新界面函数 def update_interface(backend_choice): @@ -1776,7 +1784,7 @@ def main(ctx, file_types=suffixes, elem_classes=["mineru-upload-file"], ) - preferred_option = "hybrid-auto-engine" + preferred_option = DEFAULT_BACKEND backend = gr.Dropdown( build_backend_choices(http_client_enable, i18n), label=i18n("backend"), diff --git a/mineru/model/ocr/pytorch_paddle.py b/mineru/model/ocr/pytorch_paddle.py index 08db11ed..48743bfe 100644 --- a/mineru/model/ocr/pytorch_paddle.py +++ b/mineru/model/ocr/pytorch_paddle.py @@ -289,6 +289,7 @@ class PytorchPaddleOCR(TextSystem): mfd_res=None, tqdm_enable=False, tqdm_desc="OCR-rec Predict", + tqdm_progress_bar=None, ): assert isinstance(img, (np.ndarray, list, str, bytes)) if isinstance(img, list) and det == True: @@ -338,7 +339,12 @@ class PytorchPaddleOCR(TextSystem): if not isinstance(img, list): img = preprocess_image(img) img = [img] - rec_res, elapse = self.text_recognizer(img, tqdm_enable=tqdm_enable, tqdm_desc=tqdm_desc) + rec_res, elapse = self.text_recognizer( + img, + tqdm_enable=tqdm_enable, + tqdm_desc=tqdm_desc, + tqdm_progress_bar=tqdm_progress_bar, + ) # logger.debug("rec_res num : {}, elapsed : {}".format(len(rec_res), elapse)) ocr_res.append(rec_res) return ocr_res diff --git a/mineru/model/table/cls/mineru_table_ori_cls.py b/mineru/model/table/cls/mineru_table_ori_cls.py index b7002ec9..e31da4fb 100644 --- a/mineru/model/table/cls/mineru_table_ori_cls.py +++ b/mineru/model/table/cls/mineru_table_ori_cls.py @@ -5,6 +5,7 @@ from collections import defaultdict from typing import List, Dict import cv2 import numpy as np +from tqdm import tqdm # 旋转候选门控回到旧规则，先尽量召回疑似旋转表，再由 OCR rec 评分决定最终角度。 @@ -227,19 +228,50 @@ class MineruTableOrientationClsModel: return self._select_rotation_label_by_scores(score_by_label) + @staticmethod + def _set_progress_description(progress_bar, desc: str): + """切换复用进度条的阶段描述，并兼容测试替身和 tqdm 对象。""" + if progress_bar is None: + return + if hasattr(progress_bar, "set_description"): + progress_bar.set_description(desc) + else: + progress_bar.desc = desc + + @staticmethod + def _extend_progress_total(progress_bar, count: int): + """按新增工作量动态扩展总进度，保证最终 total 覆盖 det/score/rec。""" + if progress_bar is None or count <= 0: + return + current_total = progress_bar.total if progress_bar.total is not None else progress_bar.n + progress_bar.total = current_total + count + if hasattr(progress_bar, "refresh"): + progress_bar.refresh() + @classmethod def _collect_portrait_image_groups( cls, imgs: List[Dict], resolution_group_stride: int, ) -> Dict[tuple[int, int], list[Dict]]: - """按归一化分辨率收集竖版表格，横版表格默认保持 0 度跳过后续 OCR。""" + """兼容旧私有入口，实际收集逻辑已不再按表格宽高比过滤。""" + return cls._collect_orientation_image_groups( + imgs, + resolution_group_stride, + ) + + @classmethod + def _collect_orientation_image_groups( + cls, + imgs: List[Dict], + resolution_group_stride: int, + ) -> Dict[tuple[int, int], list[Dict]]: + """按归一化分辨率收集所有有效表格图，旋转判断交给 OCR det/rec 评分。""" resolution_groups = defaultdict(list) for index, img in enumerate(imgs): bgr_img = cls._to_bgr_table_image(img) img_height, img_width = bgr_img.shape[:2] - img_aspect_ratio = img_height / img_width if img_width > 0 else 1.0 - if img_aspect_ratio <= 1.2: + if img_height <= 0 or img_width <= 0: continue group_key = ( @@ -280,8 +312,9 @@ class MineruTableOrientationClsModel: resolution_groups: Dict[tuple[int, int], list[Dict]], det_batch_size: int, resolution_group_stride: int, + progress_bar=None, ) -> list[Dict]: - """对竖版表格批量做 OCR det，并筛选需要进入多角度评分的候选。""" + """对表格批量做 OCR det，并筛选需要进入多角度评分的候选。""" rotated_imgs = [] for _group_key, group_imgs in resolution_groups.items(): batch_images = self._pad_group_images(group_imgs, resolution_group_stride) @@ -293,11 +326,14 @@ class MineruTableOrientationClsModel: for img_info, (dt_boxes, _elapse) in zip(group_imgs, batch_results): if self._is_rotation_candidate_by_det_boxes(dt_boxes): rotated_imgs.append(img_info) + if progress_bar is not None: + progress_bar.update(len(group_imgs)) return rotated_imgs def _build_score_tasks_for_candidates( self, rotated_imgs: list[Dict], + progress_bar=None, ) -> tuple[list[tuple[Dict, list[Dict]]], list[np.ndarray]]: """为所有旋转候选构造三角度评分任务，并汇总成一次 OCR rec 输入。""" img_score_tasks = [] @@ -311,9 +347,17 @@ class MineruTableOrientationClsModel: task["crop_start"] = crop_start task["crop_end"] = crop_end img_score_tasks.append((img_info, tasks)) + if progress_bar is not None: + progress_bar.update(1) return img_score_tasks, all_crop_imgs - def _recognize_orientation_crops(self, all_crop_imgs: list[np.ndarray]) -> list: + def _recognize_orientation_crops( + self, + all_crop_imgs: list[np.ndarray], + tqdm_enable: bool = False, + tqdm_desc: str = "Table orientation", + tqdm_progress_bar=None, + ) -> list: """对所有候选角度 crop 合并执行 OCR rec，返回可按 slice 回填的结果。""" if not all_crop_imgs: return [] @@ -322,19 +366,38 @@ class MineruTableOrientationClsModel: all_crop_imgs, det=False, rec=True, + tqdm_enable=tqdm_enable and tqdm_progress_bar is None, + tqdm_desc=f"{tqdm_desc} rec", + tqdm_progress_bar=tqdm_progress_bar, ) return rec_ocr_res[0] if rec_ocr_res else [] - def _score_rotation_candidates(self, rotated_imgs: list[Dict]) -> Dict[int, str]: + def _score_rotation_candidates( + self, + rotated_imgs: list[Dict], + tqdm_enable: bool = False, + tqdm_desc: str = "Table orientation", + progress_bar=None, + ) -> Dict[int, str]: """批量评分旋转候选，并返回原始表格下标到最终角度标签的映射。""" if not rotated_imgs: return {} label_by_index = {} img_score_tasks, all_crop_imgs = self._build_score_tasks_for_candidates( - rotated_imgs + rotated_imgs, + progress_bar=progress_bar, + ) + self._extend_progress_total(progress_bar, len(all_crop_imgs)) + self._set_progress_description(progress_bar, f"{tqdm_desc} rec") + if progress_bar is not None and hasattr(progress_bar, "refresh"): + progress_bar.refresh() + rec_res = self._recognize_orientation_crops( + all_crop_imgs, + tqdm_enable=tqdm_enable, + tqdm_desc=tqdm_desc, + tqdm_progress_bar=progress_bar, ) - rec_res = self._recognize_orientation_crops(all_crop_imgs) for img_info, tasks in img_score_tasks: score_by_label = self._score_orientation_tasks_with_rec(tasks, rec_res) @@ -347,23 +410,50 @@ class MineruTableOrientationClsModel: self, imgs: List[Dict], det_batch_size: int, + tqdm_enable: bool = False, + tqdm_desc: str = "Table orientation", ) -> List[str]: """ 批量预测传入表格图片的旋转角度，只返回角度，不修改输入图片。 """ RESOLUTION_GROUP_STRIDE = 128 rotate_labels = ["0"] * len(imgs) - resolution_groups = self._collect_portrait_image_groups( + resolution_groups = self._collect_orientation_image_groups( imgs, RESOLUTION_GROUP_STRIDE, ) - rotated_imgs = self._detect_rotation_candidates( - resolution_groups, - det_batch_size, - RESOLUTION_GROUP_STRIDE, - ) - label_by_index = self._score_rotation_candidates(rotated_imgs) - for index, label in label_by_index.items(): - rotate_labels[index] = label + total_images = sum(len(group_imgs) for group_imgs in resolution_groups.values()) + progress_bar = None + if tqdm_enable: + progress_bar = tqdm( + total=total_images, + desc=f"{tqdm_desc} det", + leave=True, + ) + try: + rotated_imgs = self._detect_rotation_candidates( + resolution_groups, + det_batch_size, + RESOLUTION_GROUP_STRIDE, + progress_bar=progress_bar, + ) + self._extend_progress_total(progress_bar, len(rotated_imgs)) + self._set_progress_description(progress_bar, f"{tqdm_desc} score") + if progress_bar is not None and hasattr(progress_bar, "refresh"): + progress_bar.refresh() + label_by_index = self._score_rotation_candidates( + rotated_imgs, + tqdm_enable=tqdm_enable, + tqdm_desc=tqdm_desc, + progress_bar=progress_bar, + ) + for index, label in label_by_index.items(): + rotate_labels[index] = label + finally: + self._set_progress_description(progress_bar, tqdm_desc) + if progress_bar is not None and hasattr(progress_bar, "refresh"): + progress_bar.refresh() + if progress_bar is not None: + progress_bar.close() return rotate_labels diff --git a/mineru/model/utils/tools/infer/predict_rec.py b/mineru/model/utils/tools/infer/predict_rec.py index 8bcd8555..8ace3943 100644 --- a/mineru/model/utils/tools/infer/predict_rec.py +++ b/mineru/model/utils/tools/infer/predict_rec.py @@ -288,7 +288,13 @@ class TextRecognizer(BaseOCRV20): return img - def __call__(self, img_list, tqdm_enable=False, tqdm_desc="OCR-rec Predict"): + def __call__( + self, + img_list, + tqdm_enable=False, + tqdm_desc="OCR-rec Predict", + tqdm_progress_bar=None, + ): img_num = len(img_list) # Calculate the aspect ratio of all text bars width_list = [] @@ -301,8 +307,14 @@ class TextRecognizer(BaseOCRV20): rec_res = [['', 0.0]] * img_num batch_num = self.rec_batch_num elapse = 0 - # for beg_img_no in range(0, img_num, batch_num): - with tqdm(total=img_num, desc=tqdm_desc, disable=not tqdm_enable) as pbar: + # tqdm_progress_bar 由上层复用时，不再创建内部 OCR-rec 进度条。 + pbar = tqdm_progress_bar + should_close_pbar = False + if pbar is None: + pbar = tqdm(total=img_num, desc=tqdm_desc, disable=not tqdm_enable) + should_close_pbar = True + + try: index = 0 for beg_img_no in range(0, img_num, batch_num): end_img_no = min(img_num, beg_img_no + batch_num) @@ -428,6 +440,9 @@ class TextRecognizer(BaseOCRV20): current_batch_size = min(batch_num, img_num - index * batch_num) index += 1 pbar.update(current_batch_size) + finally: + if should_close_pbar: + pbar.close() # Fix NaN values in recognition results for i in range(len(rec_res)): diff --git a/mineru/utils/title_level_postprocess.py b/mineru/utils/title_level_postprocess.py index 76a14dd7..634c8925 100644 --- a/mineru/utils/title_level_postprocess.py +++ b/mineru/utils/title_level_postprocess.py @@ -74,6 +74,7 @@ def finalize_client_side_middle_json(middle_json: dict[str, Any]) -> dict[str, A finalize_middle_json_from_preproc( pdf_info, + hybrid_mode=middle_json.get("_hybrid_mode", "pro"), ) return middle_json diff --git a/pyproject.toml b/pyproject.toml index ed6fd3e0..79cc4dd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ "openai>=1.70.0,<3", "beautifulsoup4>=4.13.5,<5", "magika>=0.6.2,<1.1.0", - "mineru-vl-utils>=1.0.2,<2", + "mineru-vl-utils>=1.0.4,<2", "python-docx>=1.2.0,<2", 'pypptx-with-oxml>=1.0.3,<2', "mammoth>=1.11.0,<2",