diff --git a/docker/china/Dockerfile b/docker/china/Dockerfile index 2be8b628..00c95c0e 100644 --- a/docker/china/Dockerfile +++ b/docker/china/Dockerfile @@ -1,7 +1,9 @@ -# Use DaoCloud mirrored vllm image for China region for gpu with Volta、Turing、Ampere、Ada Lovelace、Hopper、Blackwell architecture (7.0 <= Compute Capability <= 12.0) +# Use DaoCloud mirrored vllm image for China region for gpu with Volta、Turing、Ampere、Ada Lovelace、Hopper、Blackwell architecture (7.0 <= Compute Capability <= 12.1) +# The default base image uses vLLM 0.21.0 with CUDA 13.0. For CUDA 12.9 environments, switch to the commented cu129 image below. # Compute Capability version query (https://developer.nvidia.com/cuda-gpus) # support x86_64 architecture and ARM(AArch64) architecture -FROM docker.m.daocloud.io/vllm/vllm-openai:v0.11.2 +FROM docker.m.daocloud.io/vllm/vllm-openai:v0.21.0 +# FROM docker.m.daocloud.io/vllm/vllm-openai:v0.21.0-cu129 # Install libgl for opencv support & Noto fonts for Chinese characters RUN apt-get update && \ @@ -15,11 +17,11 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Install mineru latest -RUN python3 -m pip install -U 'mineru[core]>=3.0.0' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \ +RUN python3 -m pip install -U 'mineru[core]>=3.2.1' -i https://mirrors.aliyun.com/pypi/simple --break-system-packages && \ python3 -m pip cache purge # Download models and update the configuration file RUN /bin/bash -c "mineru-models-download -s modelscope -m all" # Set the entry point to activate the virtual environment and run the command line tool -ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"] diff --git a/docker/global/Dockerfile b/docker/global/Dockerfile index 5e84399d..f09313ac 100644 --- a/docker/global/Dockerfile +++ b/docker/global/Dockerfile @@ -1,7 +1,9 @@ -# Use the official vllm image for gpu with Volta、Turing、Ampere、Ada Lovelace、Hopper、Blackwell architecture (7.0 <= Compute Capability <= 12.0) +# Use the official vllm image for gpu with Volta、Turing、Ampere、Ada Lovelace、Hopper、Blackwell architecture (7.0 <= Compute Capability <= 12.1) +# The default base image uses vLLM 0.21.0 with CUDA 13.0. For CUDA 12.9 environments, switch to the commented cu129 image below. # Compute Capability version query (https://developer.nvidia.com/cuda-gpus) # support x86_64 architecture and ARM(AArch64) architecture -FROM vllm/vllm-openai:v0.11.2 +FROM vllm/vllm-openai:v0.21.0 +# FROM vllm/vllm-openai:v0.21.0-cu129 # Install libgl for opencv support & Noto fonts for Chinese characters RUN apt-get update && \ @@ -15,11 +17,11 @@ RUN apt-get update && \ rm -rf /var/lib/apt/lists/* # Install mineru latest -RUN python3 -m pip install -U 'mineru[core]>=3.0.0' --break-system-packages && \ +RUN python3 -m pip install -U 'mineru[core]>=3.2.1' --break-system-packages && \ python3 -m pip cache purge # Download models and update the configuration file RUN /bin/bash -c "mineru-models-download -s huggingface -m all" # Set the entry point to activate the virtual environment and run the command line tool -ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"] \ No newline at end of file +ENTRYPOINT ["/bin/bash", "-c", "export MINERU_MODEL_SOURCE=local && exec \"$@\"", "--"] diff --git a/docs/en/quick_start/docker_deployment.md b/docs/en/quick_start/docker_deployment.md index 288e6d49..47848515 100644 --- a/docs/en/quick_start/docker_deployment.md +++ b/docs/en/quick_start/docker_deployment.md @@ -15,13 +15,13 @@ docker build -t mineru:latest -f Dockerfile . ## Docker Description -MinerU's Docker uses `vllm/vllm-openai` as the base image, so it includes the `vllm` inference acceleration framework and necessary dependencies by default. Therefore, on compatible devices, you can directly use `vllm` to accelerate VLM model inference. +MinerU's Docker uses `vllm/vllm-openai` as the base image, so it includes the `vllm` inference acceleration framework and necessary dependencies by default. The current Dockerfile uses `vllm/vllm-openai:v0.21.0` by default for CUDA 13.0-compatible environments. If your environment requires a CUDA 12.9-compatible image, comment out the default `FROM` line at the top of the Dockerfile and enable the commented `vllm/vllm-openai:v0.21.0-cu129` base image instead. Therefore, on compatible devices, you can directly use `vllm` to accelerate VLM model inference. > [!NOTE] > Requirements for using `vllm` to accelerate VLM model inference: > > - Device must have Volta architecture or later graphics cards with 8GB+ available VRAM. -> - The host machine's graphics driver should support CUDA 12.9.1 or higher; You can check the driver version using the `nvidia-smi` command. +> - The host machine's graphics driver must support the CUDA runtime used by the selected base image: the default `v0.21.0` image requires a CUDA 13.0-compatible driver, and `v0.21.0-cu129` requires a CUDA 12.9-compatible driver. You can check the driver version using the `nvidia-smi` command. > - Docker container must have access to the host machine's graphics devices. ## Start Docker Container diff --git a/docs/en/quick_start/extension_modules.md b/docs/en/quick_start/extension_modules.md index 32854b6a..daec8c4d 100644 --- a/docs/en/quick_start/extension_modules.md +++ b/docs/en/quick_start/extension_modules.md @@ -29,7 +29,9 @@ The `vllm` module provides acceleration support for VLM model inference, suitabl uv pip install "mineru[core,vllm]" ``` > [!TIP] -> If exceptions occur during installation of the extra package including vllm, please refer to the [vllm official documentation](https://docs.vllm.ai/en/latest/getting_started/installation/index.html) to try to resolve the issue, or directly use the [Docker](./docker_deployment.md) deployment method. +> - Because the `vllm` extra now allows the 0.21 series, the default installation usually resolves to the newer `vllm` version allowed by the current range. Make sure the host GPU driver supports the CUDA runtime required by the installed `vllm` package; the default path requires a CUDA 13.0-compatible driver. +> - If you need a CUDA 12.9-compatible environment, follow the [vllm official documentation](https://docs.vllm.ai/en/latest/getting_started/installation/index.html) to select the matching CUDA installation path, or use the `vllm/vllm-openai:v0.21.0-cu129` base image from the [Docker](./docker_deployment.md) deployment workflow. +> - If exceptions occur during installation of the extra package including vllm, you can also refer to the [vllm official documentation](https://docs.vllm.ai/en/latest/getting_started/installation/index.html) to troubleshoot them. --- diff --git a/docs/zh/quick_start/docker_deployment.md b/docs/zh/quick_start/docker_deployment.md index 5bb4dba4..5b5aa07c 100644 --- a/docs/zh/quick_start/docker_deployment.md +++ b/docs/zh/quick_start/docker_deployment.md @@ -15,12 +15,12 @@ docker build -t mineru:latest -f Dockerfile . ## Docker说明 -Mineru的docker使用了`vllm/vllm-openai`作为基础镜像,因此在docker中默认集成了`vllm`推理加速框架和必需的依赖环境。因此在满足条件的设备上,您可以直接使用`vllm`加速VLM模型推理。 +Mineru的docker使用了`vllm/vllm-openai`作为基础镜像,因此在docker中默认集成了`vllm`推理加速框架和必需的依赖环境。当前国内 Dockerfile 默认使用`docker.m.daocloud.io/vllm/vllm-openai:v0.21.0`,适用于 CUDA 13.0 兼容环境;如果您的环境需要 CUDA 12.9 兼容镜像,请将 Dockerfile 顶部的默认`FROM`注释掉,并启用注释中的`docker.m.daocloud.io/vllm/vllm-openai:v0.21.0-cu129`基础镜像。因此在满足条件的设备上,您可以直接使用`vllm`加速VLM模型推理。 > [!NOTE] > 使用`vllm`加速VLM模型推理需要满足的条件是: > > - 设备包含Volta及以后架构的显卡,且可用显存大于等于8G。 -> - 物理机的显卡驱动应支持CUDA 12.9.1或更高版本,可通过`nvidia-smi`命令检查驱动版本。 +> - 物理机的显卡驱动应支持所选基础镜像对应的 CUDA 运行时版本:默认`v0.21.0`需要 CUDA 13.0 兼容驱动,`v0.21.0-cu129`需要 CUDA 12.9 兼容驱动。可通过`nvidia-smi`命令检查驱动版本。 > - docker中能够访问物理机的显卡设备。 diff --git a/docs/zh/quick_start/extension_modules.md b/docs/zh/quick_start/extension_modules.md index 3d2cb68a..bc150eb6 100644 --- a/docs/zh/quick_start/extension_modules.md +++ b/docs/zh/quick_start/extension_modules.md @@ -28,7 +28,9 @@ uv pip install "mineru[s3]" uv pip install "mineru[core,vllm]" ``` > [!TIP] -> 如在安装包含`vllm`的扩展包过程中发生异常,请参考 [vllm 官方文档](https://docs.vllm.ai/en/latest/getting_started/installation/index.html) 尝试解决,或直接使用 [Docker](./docker_deployment.md) 方式部署镜像。 +> - 由于`vllm`扩展包已放开到 0.21 系列版本,默认安装通常会选择当前允许范围内更高的`vllm`版本。请确保物理机显卡驱动支持所安装`vllm`包对应的 CUDA 运行时,默认路径需要 CUDA 13.0 兼容驱动。 +> - 如需使用 CUDA 12.9 兼容环境,请参考 [vllm 官方文档](https://docs.vllm.ai/en/latest/getting_started/installation/index.html) 选择对应的 CUDA 安装方式,或直接使用 [Docker](./docker_deployment.md) 中的`vllm/vllm-openai:v0.21.0-cu129`基础镜像。 +> - 如在安装包含`vllm`的扩展包过程中发生异常,也可参考 [vllm 官方文档](https://docs.vllm.ai/en/latest/getting_started/installation/index.html) 尝试解决。 --- diff --git a/mineru/backend/hybrid/hybrid_analyze.py b/mineru/backend/hybrid/hybrid_analyze.py index 20c520a6..da70bbbd 100644 --- a/mineru/backend/hybrid/hybrid_analyze.py +++ b/mineru/backend/hybrid/hybrid_analyze.py @@ -13,6 +13,7 @@ from mineru_vl_utils.structs import BlockType from tqdm import tqdm from mineru.backend.hybrid.hybrid_model_output_to_middle_json import ( + apply_server_side_postprocess, append_page_model_list_to_middle_json, finalize_middle_json, init_middle_json, @@ -722,6 +723,9 @@ def doc_analyze( image_analysis: bool = True, **kwargs, ): + client_side_output_generation = bool( + kwargs.pop("client_side_output_generation", False) + ) if predictor is None: predictor = ModelSingleton().get_model(backend, model_path, server_url, **kwargs) predictor = _maybe_enable_serial_execution(predictor, backend) @@ -833,12 +837,20 @@ def doc_analyze( f"speed: {round(len(model_list) / infer_time, 3)} page/s" ) - finalize_middle_json( - middle_json["pdf_info"], - hybrid_pipeline_model, - _ocr_enable, - _vlm_ocr_enable, - ) + if client_side_output_generation: + apply_server_side_postprocess( + middle_json["pdf_info"], + hybrid_pipeline_model, + _ocr_enable, + _vlm_ocr_enable, + ) + else: + finalize_middle_json( + middle_json["pdf_info"], + hybrid_pipeline_model, + _ocr_enable, + _vlm_ocr_enable, + ) close_pdfium_document(pdf_doc) doc_closed = True clean_memory(device) @@ -861,6 +873,9 @@ async def aio_doc_analyze( image_analysis: bool = True, **kwargs, ): + client_side_output_generation = bool( + kwargs.pop("client_side_output_generation", False) + ) if predictor is None: predictor = await _get_model_async(backend, model_path, server_url, **kwargs) predictor = _maybe_enable_serial_execution(predictor, backend) @@ -973,13 +988,22 @@ async def aio_doc_analyze( f"speed: {round(len(model_list) / infer_time, 3)} page/s" ) - await asyncio.to_thread( - finalize_middle_json, - middle_json["pdf_info"], - hybrid_pipeline_model, - _ocr_enable, - _vlm_ocr_enable, - ) + if client_side_output_generation: + await asyncio.to_thread( + apply_server_side_postprocess, + middle_json["pdf_info"], + hybrid_pipeline_model, + _ocr_enable, + _vlm_ocr_enable, + ) + else: + await asyncio.to_thread( + finalize_middle_json, + middle_json["pdf_info"], + hybrid_pipeline_model, + _ocr_enable, + _vlm_ocr_enable, + ) close_pdfium_document(pdf_doc) doc_closed = True clean_memory(device) diff --git a/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py b/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py index 6e5bd40a..80b75c3f 100644 --- a/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py +++ b/mineru/backend/hybrid/hybrid_model_output_to_middle_json.py @@ -1,18 +1,12 @@ # Copyright (c) Opendatalab. All rights reserved. import os -import time -import numpy as np -from loguru import logger from tqdm import tqdm from mineru.backend.utils.html_image_utils import replace_inline_table_images -from mineru.backend.utils.ocr_det_utils import ( - detect_ocr_boxes_from_padded_crop, - get_ch_lite_ocr_det_model, -) from mineru.backend.utils.para_block_utils import ( + OCR_DET_LINES_KEY, build_para_blocks_from_preproc, cleanup_internal_para_block_metadata, iter_block_spans, @@ -20,20 +14,34 @@ from mineru.backend.utils.para_block_utils import ( ) from mineru.backend.hybrid.hybrid_magic_model import MagicModel from mineru.backend.utils.runtime_utils import cross_page_table_merge -from mineru.utils.config_reader import get_table_enable, get_llm_aided_config +from mineru.utils.config_reader import get_table_enable from mineru.utils.cut_image import cut_image_and_table from mineru.utils.enum_class import ContentType, BlockType from mineru.utils.hash_utils import bytes_md5 from mineru.utils.ocr_utils import OcrConfidence, rotate_vertical_crop_if_needed +from mineru.utils.title_level_postprocess import apply_title_leveling_to_pdf_info from mineru.utils.pdfium_guard import close_pdfium_document, pdfium_guard from mineru.version import __version__ -from mineru.utils.llm_aided import llm_aided_title -title_aided_enable = False -llm_aided_config = get_llm_aided_config() -if llm_aided_config: - title_aided_config = llm_aided_config.get('title_aided', {}) - title_aided_enable = title_aided_config.get('enable', False) + +def _resolve_title_line_avg_height(title_block): + """解析标题平均行高:优先复用 Hybrid OCR det 行提示,再回退到原始行或块高。""" + for lines_key in [OCR_DET_LINES_KEY, "lines"]: + line_heights = [] + for line in title_block.get(lines_key, []): + bbox = line.get("bbox") + if not bbox or len(bbox) < 4: + continue + line_height = bbox[3] - bbox[1] + if line_height > 0: + line_heights.append(line_height) + if line_heights: + return round(sum(line_heights) / len(line_heights)) + + bbox = title_block.get("bbox", [0, 0, 0, 0]) + if len(bbox) >= 4: + return bbox[3] - bbox[1] + return 0 def blocks_to_page_info( @@ -73,30 +81,9 @@ def blocks_to_page_info( phonetic_blocks = magic_model.get_phonetic_blocks() list_blocks = magic_model.get_list_blocks() - # 如果有标题优化需求,计算标题的平均行高 - if title_aided_enable: - if _vlm_ocr_enable: # vlm_ocr导致没有line信息,需要重新det获取平均行高 - ocr_model = get_ch_lite_ocr_det_model() - for title_block in title_blocks: - ocr_det_res, _ = detect_ocr_boxes_from_padded_crop( - title_block.get('bbox'), - page_pil_img, - scale, - ocr_model=ocr_model, - ) - if len(ocr_det_res) > 0: - # 计算所有res的平均高度 - avg_height = np.mean([box[2][1] - box[0][1] for box in ocr_det_res]) - title_block['line_avg_height'] = round(avg_height/scale) - else: # 有line信息,直接计算平均行高 - for title_block in title_blocks: - lines = title_block.get('lines', []) - if lines: - # 使用列表推导式和内置函数,一次性计算平均高度 - avg_height = sum(line['bbox'][3] - line['bbox'][1] for line in lines) / len(lines) - title_block['line_avg_height'] = round(avg_height) - else: - title_block['line_avg_height'] = title_block['bbox'][3] - title_block['bbox'][1] + # 标题平均行高是 finalized middle json 的稳定字段,供服务端和客户端标题分级共用。 + for title_block in title_blocks: + title_block['line_avg_height'] = _resolve_title_line_avg_height(title_block) text_blocks = magic_model.get_text_blocks() interline_equation_blocks = magic_model.get_interline_equation_blocks() @@ -245,10 +232,19 @@ def append_page_model_list_to_middle_json( ) -def finalize_middle_json(pdf_info_list, hybrid_pipeline_model, _ocr_enable, _vlm_ocr_enable): +def apply_server_side_postprocess( + pdf_info_list, + hybrid_pipeline_model, + _ocr_enable, + _vlm_ocr_enable, +): + """执行 Hybrid 只能在服务端完成的 post-OCR,避免客户端依赖 pipeline OCR 模型。""" if not (_vlm_ocr_enable or _ocr_enable): _apply_post_ocr(pdf_info_list, hybrid_pipeline_model) + +def finalize_middle_json_from_preproc(pdf_info_list): + """从 Hybrid preproc_blocks 执行完整 finalize,供服务端完整路径和客户端复用。""" build_para_blocks_from_preproc(pdf_info_list) merge_para_text_blocks( pdf_info_list, @@ -259,15 +255,27 @@ def finalize_middle_json(pdf_info_list, hybrid_pipeline_model, _ocr_enable, _vlm if table_enable: cross_page_table_merge(pdf_info_list) - if title_aided_enable: - llm_aided_title_start_time = time.time() - llm_aided_title(pdf_info_list, title_aided_config) - logger.info(f'llm aided title time: {round(time.time() - llm_aided_title_start_time, 2)}') - + apply_title_leveling_to_pdf_info(pdf_info_list) _normalize_split_title_blocks(pdf_info_list) cleanup_internal_para_block_metadata(pdf_info_list) +def finalize_middle_json( + pdf_info_list, + hybrid_pipeline_model, + _ocr_enable, + _vlm_ocr_enable, +): + """保持旧入口语义:服务端先做必要 post-OCR,再执行完整 finalize。""" + apply_server_side_postprocess( + pdf_info_list, + hybrid_pipeline_model, + _ocr_enable, + _vlm_ocr_enable, + ) + finalize_middle_json_from_preproc(pdf_info_list) + + def result_to_middle_json( model_list, images_list, diff --git a/mineru/backend/pipeline/model_json_to_middle_json.py b/mineru/backend/pipeline/model_json_to_middle_json.py index 7a1446f5..4025af06 100644 --- a/mineru/backend/pipeline/model_json_to_middle_json.py +++ b/mineru/backend/pipeline/model_json_to_middle_json.py @@ -1,20 +1,18 @@ # Copyright (c) Opendatalab. All rights reserved. import copy import os -import time -from loguru import logger from tqdm import tqdm from mineru.backend.utils.html_image_utils import replace_inline_table_images from mineru.backend.utils.runtime_utils import cross_page_table_merge -from mineru.utils.config_reader import get_device, get_llm_aided_config +from mineru.utils.config_reader import get_device from mineru.backend.pipeline.model_init import AtomModelSingleton from mineru.backend.pipeline.para_split import para_split from mineru.utils.char_utils import full_to_half from mineru.utils.cut_image import cut_image_and_table from mineru.utils.enum_class import ContentType, BlockType -from mineru.utils.llm_aided import llm_aided_title +from mineru.utils.title_level_postprocess import apply_title_leveling_to_pdf_info from mineru.utils.model_utils import clean_memory from mineru.backend.pipeline.pipeline_magic_model import MagicModel from mineru.utils.ocr_utils import OcrConfidence, rotate_vertical_crop_if_needed @@ -261,23 +259,27 @@ def _post_block_process(pdf_info_list): block["type"] = BlockType.TEXT -def finalize_middle_json(pdf_info_list, lang=None, ocr_enable=False): - """Apply document-level post processing once all page_info entries are ready.""" +def apply_server_side_postprocess(pdf_info_list, lang=None): + """执行只能在服务端完成的后处理;目前仅包含依赖 OCR 模型的 post-OCR。""" _apply_post_ocr(pdf_info_list, lang=lang) + + +def finalize_middle_json_from_preproc(pdf_info_list): + """从 preproc_blocks 执行确定性 finalize,供服务端完整路径和客户端复用。""" _optimize_formula_number_blocks(pdf_info_list) para_split(pdf_info_list) cross_page_table_merge(pdf_info_list) - - llm_aided_config = get_llm_aided_config() - if llm_aided_config is not None: - title_aided_config = llm_aided_config.get('title_aided', None) - if title_aided_config is not None and title_aided_config.get('enable', False): - llm_aided_title_start_time = time.time() - llm_aided_title(pdf_info_list, title_aided_config) - logger.info(f'llm aided title time: {round(time.time() - llm_aided_title_start_time, 2)}') - + apply_title_leveling_to_pdf_info(pdf_info_list) _post_block_process(pdf_info_list) + +def finalize_middle_json( + pdf_info_list, + lang=None, +): + """Apply document-level post processing once all page_info entries are ready.""" + apply_server_side_postprocess(pdf_info_list, lang=lang) + finalize_middle_json_from_preproc(pdf_info_list) if os.getenv('MINERU_DONOT_CLEAN_MEM') is None and len(pdf_info_list) >= 10: clean_memory(get_device()) @@ -299,7 +301,7 @@ def result_to_middle_json(model_list, images_list, pdf_doc, image_writer, lang=N progress_bar=progress_bar, ) - finalize_middle_json(middle_json["pdf_info"], lang=lang, ocr_enable=ocr_enable) + finalize_middle_json(middle_json["pdf_info"], lang=lang) close_pdfium_document(pdf_doc) return middle_json diff --git a/mineru/backend/pipeline/pipeline_analyze.py b/mineru/backend/pipeline/pipeline_analyze.py index 03a36d05..6d31b798 100644 --- a/mineru/backend/pipeline/pipeline_analyze.py +++ b/mineru/backend/pipeline/pipeline_analyze.py @@ -10,6 +10,7 @@ from tqdm import tqdm from .model_init import MineruPipelineModel, PIPELINE_MODEL_INIT_LOCK from .model_json_to_middle_json import ( + apply_server_side_postprocess, append_batch_results_to_middle_json, finalize_middle_json, init_middle_json, @@ -109,14 +110,23 @@ def _format_doc_slices(batch_slices): ) -def _finalize_processing_window_context(context, on_doc_ready): +def _finalize_processing_window_context( + context, + on_doc_ready, + client_side_output_generation=False, +): if context['closed']: return - finalize_middle_json( - context['middle_json']['pdf_info'], - lang=context['lang'], - ocr_enable=context['ocr_enable'], - ) + if client_side_output_generation: + apply_server_side_postprocess( + context['middle_json']['pdf_info'], + lang=context['lang'], + ) + else: + finalize_middle_json( + context['middle_json']['pdf_info'], + lang=context['lang'], + ) logger.debug( f"Pipeline doc ready: doc{context['doc_index']} pages={context['page_count']}" ) @@ -130,10 +140,18 @@ def _finalize_processing_window_context(context, on_doc_ready): context['closed'] = True -def _emit_zero_page_contexts(doc_contexts, on_doc_ready): +def _emit_zero_page_contexts( + doc_contexts, + on_doc_ready, + client_side_output_generation=False, +): for context in doc_contexts: if context['page_count'] == 0 and not context['closed']: - _finalize_processing_window_context(context, on_doc_ready) + _finalize_processing_window_context( + context, + on_doc_ready, + client_side_output_generation=client_side_output_generation, + ) def doc_analyze_streaming( @@ -144,6 +162,7 @@ def doc_analyze_streaming( parse_method: str = 'auto', formula_enable=True, table_enable=True, + client_side_output_generation=False, ): if not (len(pdf_bytes_list) == len(image_writer_list) == len(lang_list)): raise ValueError("pdf_bytes_list, image_writer_list, and lang_list must have the same length") @@ -174,7 +193,11 @@ def doc_analyze_streaming( ) if total_pages == 0: - _emit_zero_page_contexts(doc_contexts, on_doc_ready) + _emit_zero_page_contexts( + doc_contexts, + on_doc_ready, + client_side_output_generation=client_side_output_generation, + ) return window_size = get_processing_window_size(default=64) @@ -184,7 +207,11 @@ def doc_analyze_streaming( f'total_pages={total_pages}, window_size={window_size}, total_batches={total_batches}' ) - _emit_zero_page_contexts(doc_contexts, on_doc_ready) + _emit_zero_page_contexts( + doc_contexts, + on_doc_ready, + client_side_output_generation=client_side_output_generation, + ) processed_pages = 0 infer_start = time.time() try: @@ -270,7 +297,11 @@ def doc_analyze_streaming( images_list.clear() if context['next_page_idx'] >= context['page_count'] and not context['closed']: - _finalize_processing_window_context(context, on_doc_ready) + _finalize_processing_window_context( + context, + on_doc_ready, + client_side_output_generation=client_side_output_generation, + ) last_append_end_time = time.time() processed_pages += len(batch_images) diff --git a/mineru/backend/utils/para_block_utils.py b/mineru/backend/utils/para_block_utils.py index d0d622b1..d66f2dac 100644 --- a/mineru/backend/utils/para_block_utils.py +++ b/mineru/backend/utils/para_block_utils.py @@ -23,6 +23,10 @@ TEXT_MERGE_TRANSPARENT_TYPES = { BlockType.CODE, } OCR_DET_LINES_KEY = "_ocr_det_lines" +INTERNAL_BLOCK_METADATA_KEYS = { + OCR_DET_LINES_KEY, + "line_avg_height", +} def iter_block_spans(block): @@ -376,7 +380,8 @@ def _has_mergeable_block_bbox_relation(current_block, previous_block): def _cleanup_block_internal_metadata(block): - """递归清理只供 Hybrid 内部段落合并使用的临时字段。""" - block.pop(OCR_DET_LINES_KEY, None) + """递归清理只供 finalize 内部流程使用的临时字段。""" + for metadata_key in INTERNAL_BLOCK_METADATA_KEYS: + block.pop(metadata_key, None) for sub_block in block.get("blocks", []): _cleanup_block_internal_metadata(sub_block) diff --git a/mineru/backend/vlm/model_output_to_middle_json.py b/mineru/backend/vlm/model_output_to_middle_json.py index 76efeb82..e0073505 100644 --- a/mineru/backend/vlm/model_output_to_middle_json.py +++ b/mineru/backend/vlm/model_output_to_middle_json.py @@ -1,9 +1,6 @@ # Copyright (c) Opendatalab. All rights reserved. import os -import time -import numpy as np -from loguru import logger from tqdm import tqdm from mineru.backend.utils.html_image_utils import replace_inline_table_images @@ -14,27 +11,15 @@ from mineru.backend.utils.para_block_utils import ( ) from mineru.backend.utils.runtime_utils import cross_page_table_merge from mineru.backend.vlm.vlm_magic_model import MagicModel -from mineru.utils.config_reader import get_table_enable, get_llm_aided_config +from mineru.utils.config_reader import get_table_enable from mineru.utils.cut_image import cut_image_and_table from mineru.utils.enum_class import ContentType from mineru.utils.hash_utils import bytes_md5 +from mineru.utils.title_level_postprocess import apply_title_leveling_to_pdf_info from mineru.utils.pdfium_guard import close_pdfium_document, pdfium_guard from mineru.version import __version__ -heading_level_import_success = False -llm_aided_config = get_llm_aided_config() -if llm_aided_config: - title_aided_config = llm_aided_config.get('title_aided', {}) - if title_aided_config.get('enable', False): - from mineru.utils.llm_aided import llm_aided_title - from mineru.backend.utils.ocr_det_utils import ( - detect_ocr_boxes_from_padded_crop, - get_ch_lite_ocr_det_model, - ) - heading_level_import_success = True - - def blocks_to_page_info(page_blocks, image_dict, page, image_writer, page_index) -> dict: """将blocks转换为页面信息""" @@ -55,21 +40,6 @@ def blocks_to_page_info(page_blocks, image_dict, page, image_writer, page_index) phonetic_blocks = magic_model.get_phonetic_blocks() list_blocks = magic_model.get_list_blocks() - # 如果有标题优化需求,则对title_blocks截图det - if heading_level_import_success: - ocr_model = get_ch_lite_ocr_det_model() - for title_block in title_blocks: - ocr_det_res, _ = detect_ocr_boxes_from_padded_crop( - title_block.get('bbox'), - page_pil_img, - scale, - ocr_model=ocr_model, - ) - if len(ocr_det_res) > 0: - # 计算所有res的平均高度 - avg_height = np.mean([box[2][1] - box[0][1] for box in ocr_det_res]) - title_block['line_avg_height'] = round(avg_height/scale) - text_blocks = magic_model.get_text_blocks() interline_equation_blocks = magic_model.get_interline_equation_blocks() @@ -130,6 +100,7 @@ def append_page_blocks_to_middle_json( def finalize_middle_json(pdf_info_list): + """从 VLM preproc_blocks 执行完整 finalize,客户端和服务端完整路径共用。""" build_para_blocks_from_preproc(pdf_info_list) merge_para_text_blocks(pdf_info_list) @@ -137,10 +108,7 @@ def finalize_middle_json(pdf_info_list): if table_enable: cross_page_table_merge(pdf_info_list) - if heading_level_import_success: - llm_aided_title_start_time = time.time() - llm_aided_title(pdf_info_list, title_aided_config) - logger.info(f'llm aided title time: {round(time.time() - llm_aided_title_start_time, 2)}') + apply_title_leveling_to_pdf_info(pdf_info_list) cleanup_internal_para_block_metadata(pdf_info_list) diff --git a/mineru/backend/vlm/vlm_analyze.py b/mineru/backend/vlm/vlm_analyze.py index 65be2b69..898538ec 100644 --- a/mineru/backend/vlm/vlm_analyze.py +++ b/mineru/backend/vlm/vlm_analyze.py @@ -430,6 +430,9 @@ def doc_analyze( image_analysis: bool = True, **kwargs, ): + client_side_output_generation = bool( + kwargs.pop("client_side_output_generation", False) + ) if predictor is None: predictor = ModelSingleton().get_model(backend, model_path, server_url, **kwargs) predictor = _maybe_enable_serial_execution(predictor, backend) @@ -507,7 +510,8 @@ def doc_analyze( f"processing-window infer finished, cost: {infer_time}, " f"speed: {round(len(results) / infer_time, 3)} page/s" ) - finalize_middle_json(middle_json["pdf_info"]) + if not client_side_output_generation: + finalize_middle_json(middle_json["pdf_info"]) close_pdfium_document(pdf_doc) doc_closed = True return middle_json, results @@ -526,6 +530,9 @@ async def aio_doc_analyze( image_analysis: bool = True, **kwargs, ): + client_side_output_generation = bool( + kwargs.pop("client_side_output_generation", False) + ) if predictor is None: predictor = await _get_model_async(backend, model_path, server_url, **kwargs) predictor = _maybe_enable_serial_execution(predictor, backend) @@ -602,7 +609,8 @@ async def aio_doc_analyze( f"processing-window infer finished, cost: {infer_time}, " f"speed: {round(len(results) / infer_time, 3)} page/s" ) - await asyncio.to_thread(finalize_middle_json, middle_json["pdf_info"]) + if not client_side_output_generation: + await asyncio.to_thread(finalize_middle_json, middle_json["pdf_info"]) close_pdfium_document(pdf_doc) doc_closed = True return middle_json, results diff --git a/mineru/cli/api_client.py b/mineru/cli/api_client.py index 4f3be448..9beada1f 100644 --- a/mineru/cli/api_client.py +++ b/mineru/cli/api_client.py @@ -819,6 +819,7 @@ def build_parse_request_form_data( return_images: bool, response_format_zip: bool, return_original_file: bool, + client_side_output_generation: bool = False, ) -> dict[str, str | list[str]]: effective_lang_list = list(lang_list) or ["ch"] data: dict[str, str | list[str]] = { @@ -835,6 +836,7 @@ def build_parse_request_form_data( "return_images": str(return_images).lower(), "response_format_zip": str(response_format_zip).lower(), "return_original_file": str(return_original_file).lower(), + "client_side_output_generation": str(client_side_output_generation).lower(), "start_page_id": str(start_page_id), "end_page_id": str(99999 if end_page_id is None else end_page_id), } diff --git a/mineru/cli/api_protocol.py b/mineru/cli/api_protocol.py index dee49f09..25f2e57d 100644 --- a/mineru/cli/api_protocol.py +++ b/mineru/cli/api_protocol.py @@ -1,4 +1,4 @@ # Copyright (c) Opendatalab. All rights reserved. -API_PROTOCOL_VERSION = 1 +API_PROTOCOL_VERSION = 2 DEFAULT_MAX_CONCURRENT_REQUESTS = 3 DEFAULT_PROCESSING_WINDOW_SIZE = 64 diff --git a/mineru/cli/api_request.py b/mineru/cli/api_request.py index fa086750..44f1381b 100644 --- a/mineru/cli/api_request.py +++ b/mineru/cli/api_request.py @@ -33,6 +33,7 @@ class ParseRequestOptions: return_images: bool response_format_zip: bool return_original_file: bool + client_side_output_generation: bool start_page_id: int end_page_id: int @@ -155,6 +156,15 @@ async def parse_request_form( ), ), ] = False, + client_side_output_generation: Annotated[ + bool, + Form( + description=( + "Defer final markdown/content-list generation to the client. " + "When enabled, the server returns staged middle JSON and images." + ), + ), + ] = False, start_page_id: Annotated[ int, Form(description="The starting page for PDF parsing, beginning from 0"), @@ -175,6 +185,13 @@ async def parse_request_form( backend=backend, server_url=server_url, ) + if client_side_output_generation: + return_md = False + return_middle_json = True + return_model_output = False + return_content_list = False + return_images = True + effective_return_original_file = return_original_file and response_format_zip return ParseRequestOptions( files=files, @@ -192,6 +209,7 @@ async def parse_request_form( return_images=return_images, response_format_zip=response_format_zip, return_original_file=effective_return_original_file, + client_side_output_generation=client_side_output_generation, start_page_id=start_page_id, end_page_id=end_page_id, ) diff --git a/mineru/cli/client.py b/mineru/cli/client.py index 42d13a3e..0af1f7d6 100644 --- a/mineru/cli/client.py +++ b/mineru/cli/client.py @@ -38,6 +38,7 @@ from mineru.cli.common import ( uniquify_task_stems, ) from mineru.cli import api_client as _api_client +from mineru.cli.client_side_output import regenerate_client_side_outputs from mineru.cli.output_paths import resolve_parse_dir from mineru.cli.visualization import ( VisualizationJob, @@ -623,7 +624,12 @@ def build_request_form_data( start_page_id: int, end_page_id: Optional[int], image_analysis: bool = True, + client_side_output_generation: bool = False, ) -> dict[str, str | list[str]]: + # 开启客户端输出生成时,服务端只负责返回 middle json、图片和原文件。 + return_md = not client_side_output_generation + return_model_output = not client_side_output_generation + return_content_list = not client_side_output_generation return _api_client.build_parse_request_form_data( lang_list=[lang], backend=backend, @@ -634,13 +640,14 @@ def build_request_form_data( server_url=server_url, start_page_id=start_page_id, end_page_id=end_page_id, - return_md=True, + return_md=return_md, return_middle_json=True, - return_model_output=True, - return_content_list=True, + return_model_output=return_model_output, + return_content_list=return_content_list, return_images=True, response_format_zip=True, return_original_file=True, + client_side_output_generation=client_side_output_generation, ) @@ -769,6 +776,7 @@ async def run_planned_task( form_data: dict[str, str], output_dir: Path, live_renderer: Optional[LiveTaskStatusRenderer] = None, + client_side_output_generation: bool = False, ) -> None: logger.info(format_task_submission_message(planned_task, progress)) submit_response = await submit_task( @@ -802,6 +810,21 @@ async def run_planned_task( safe_extract_zip(zip_path, output_dir) finally: zip_path.unlink(missing_ok=True) + if client_side_output_generation: + for document in planned_task.documents: + # 解压后按现有 parse_dir 结构覆盖重生客户端最终输出产物。 + parse_dir = resolve_parse_dir( + output_dir, + document.stem, + backend, + parse_method, + is_office=document.suffix in office_suffixes, + ) + await asyncio.to_thread( + regenerate_client_side_outputs, + parse_dir, + document.stem, + ) completed_tasks, completed_pages = await mark_task_completed( progress, planned_task.total_pages, @@ -846,6 +869,7 @@ async def run_orchestrated_cli( formula_enable: bool, table_enable: bool, image_analysis: bool = True, + client_side_output_generation: bool = False, extra_cli_args: tuple[str, ...] = (), ) -> None: if start_page_id < 0: @@ -916,6 +940,7 @@ async def run_orchestrated_cli( server_url=server_url, start_page_id=start_page_id, end_page_id=end_page_id, + client_side_output_generation=client_side_output_generation, ) visualization_context = create_visualization_context() failures = await execute_planned_tasks( @@ -932,6 +957,7 @@ async def run_orchestrated_cli( form_data=form_data, output_dir=output_dir, live_renderer=live_renderer, + client_side_output_generation=client_side_output_generation, ), ) if failures: @@ -1099,6 +1125,16 @@ async def run_orchestrated_cli( default=True, help="Enable image/chart analysis for VLM and hybrid backends. Default is True. ", ) +@click.option( + "--client-side-output-generation", + "client_side_output_generation", + is_flag=True, + default=False, + help=( + "Generate markdown and content lists locally from server-returned " + "middle json, images, and original files." + ), +) def main( ctx: click.Context, input_path: Path, @@ -1113,6 +1149,7 @@ def main( formula_enable: bool, table_enable: bool, image_analysis: bool, + client_side_output_generation: bool, ) -> None: asyncio.run( run_orchestrated_cli( @@ -1128,6 +1165,7 @@ def main( formula_enable=formula_enable, table_enable=table_enable, image_analysis=image_analysis, + client_side_output_generation=client_side_output_generation, extra_cli_args=tuple(ctx.args), ) ) diff --git a/mineru/cli/client_side_output.py b/mineru/cli/client_side_output.py new file mode 100644 index 00000000..653d4fb0 --- /dev/null +++ b/mineru/cli/client_side_output.py @@ -0,0 +1,97 @@ +# Copyright (c) Opendatalab. All rights reserved. +from __future__ import annotations + +import json +from pathlib import Path +from typing import Any, Callable + +from mineru.utils.enum_class import MakeMode +from mineru.utils.title_level_postprocess import finalize_client_side_middle_json + + +PDF_BACKENDS = {"pipeline", "vlm", "hybrid"} +SUPPORTED_BACKENDS = {*PDF_BACKENDS, "office"} + + +def _select_union_make(backend: str) -> Callable[[list, str, str], Any]: + """根据 middle json 后端选择对应的 Markdown/content list 渲染函数。""" + if backend == "pipeline": + from mineru.backend.pipeline.pipeline_middle_json_mkcontent import union_make + + return union_make + if backend in {"vlm", "hybrid"}: + from mineru.backend.vlm.vlm_middle_json_mkcontent import union_make + + return union_make + if backend == "office": + from mineru.backend.office.office_middle_json_mkcontent import union_make + + return union_make + + raise ValueError( + f"Unsupported middle json backend for client-side output generation: {backend}" + ) + + +def _write_json(path: Path, payload: Any) -> None: + """按项目现有格式写入 JSON 文件,保持中文内容可读。""" + path.write_text( + json.dumps(payload, ensure_ascii=False, indent=4), + encoding="utf-8", + ) + + +def regenerate_client_side_outputs( + parse_dir: str | Path, + doc_stem: str, +) -> tuple[Path, ...]: + """读取服务端 staged/finalized middle json,并在客户端覆盖生成最终输出产物。""" + parse_dir = Path(parse_dir) + middle_json_path = parse_dir / f"{doc_stem}_middle.json" + markdown_path = parse_dir / f"{doc_stem}.md" + content_list_path = parse_dir / f"{doc_stem}_content_list.json" + content_list_v2_path = parse_dir / f"{doc_stem}_content_list_v2.json" + + if not middle_json_path.exists(): + raise FileNotFoundError(f"Missing middle json file: {middle_json_path}") + + middle_json = json.loads(middle_json_path.read_text(encoding="utf-8")) + if not isinstance(middle_json, dict): + raise ValueError("middle_json must be a dict.") + backend = middle_json.get("_backend") + pdf_info = middle_json.get("pdf_info") + if backend not in SUPPORTED_BACKENDS: + raise ValueError( + f"Unsupported middle json backend for client-side output generation: {backend}" + ) + if not isinstance(pdf_info, list): + raise ValueError("middle_json must contain a list field named pdf_info.") + + if backend in PDF_BACKENDS: + finalize_client_side_middle_json(middle_json) + pdf_info = middle_json["pdf_info"] + + make_func = _select_union_make(backend) + image_dir = "images" + + markdown_path.write_text( + make_func(pdf_info, MakeMode.MM_MD, image_dir), + encoding="utf-8", + ) + _write_json( + content_list_path, + make_func(pdf_info, MakeMode.CONTENT_LIST, image_dir), + ) + _write_json( + content_list_v2_path, + make_func(pdf_info, MakeMode.CONTENT_LIST_V2, image_dir), + ) + if backend in PDF_BACKENDS: + _write_json(middle_json_path, middle_json) + + return ( + middle_json_path, + markdown_path, + content_list_path, + content_list_v2_path, + ) diff --git a/mineru/cli/common.py b/mineru/cli/common.py index 99f9afc4..3d69de8e 100644 --- a/mineru/cli/common.py +++ b/mineru/cli/common.py @@ -316,6 +316,7 @@ def _process_pipeline( f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, + client_side_output_generation=False, ): """处理pipeline后端逻辑""" from mineru.backend.pipeline.pipeline_analyze import doc_analyze_streaming as pipeline_doc_analyze_streaming @@ -366,6 +367,7 @@ def _process_pipeline( parse_method=parse_method, formula_enable=p_formula_enable, table_enable=p_table_enable, + client_side_output_generation=client_side_output_generation, ) for future in output_futures: @@ -638,6 +640,7 @@ def do_parse( start_page_id=0, end_page_id=None, image_analysis=True, + client_side_output_generation=False, **kwargs, ): need_remove_index = _process_office_doc( @@ -667,7 +670,8 @@ def do_parse( output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, table_enable, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, - f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode + f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, + client_side_output_generation=client_side_output_generation, ) else: if backend.startswith("vlm-"): @@ -686,7 +690,8 @@ def do_parse( output_dir, pdf_file_names, pdf_bytes_list, backend, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, - server_url, image_analysis=image_analysis, **kwargs, + server_url, image_analysis=image_analysis, + client_side_output_generation=client_side_output_generation, **kwargs, ) elif backend.startswith("hybrid-"): ensure_backend_dependencies(backend) @@ -706,7 +711,8 @@ def do_parse( output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, backend, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, - server_url, image_analysis=image_analysis, **kwargs, + server_url, image_analysis=image_analysis, + client_side_output_generation=client_side_output_generation, **kwargs, ) @@ -731,6 +737,7 @@ async def aio_do_parse( start_page_id=0, end_page_id=None, image_analysis=True, + client_side_output_generation=False, **kwargs, ): # Office 解析是同步且可能耗时的操作,异步入口需要放到线程中避免阻塞事件循环。 @@ -763,7 +770,8 @@ async def aio_do_parse( output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, table_enable, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, - f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode + f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, + client_side_output_generation=client_side_output_generation, ) else: if backend.startswith("vlm-"): @@ -782,7 +790,8 @@ async def aio_do_parse( output_dir, pdf_file_names, pdf_bytes_list, backend, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, - server_url, image_analysis=image_analysis, **kwargs, + server_url, image_analysis=image_analysis, + client_side_output_generation=client_side_output_generation, **kwargs, ) elif backend.startswith("hybrid-"): ensure_backend_dependencies(backend) @@ -801,7 +810,8 @@ async def aio_do_parse( output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, backend, f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json, f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode, - server_url, image_analysis=image_analysis, **kwargs, + server_url, image_analysis=image_analysis, + client_side_output_generation=client_side_output_generation, **kwargs, ) diff --git a/mineru/cli/fast_api.py b/mineru/cli/fast_api.py index 44a5dcf7..2dc09d44 100644 --- a/mineru/cli/fast_api.py +++ b/mineru/cli/fast_api.py @@ -158,6 +158,7 @@ class AsyncParseTask: return_images: bool response_format_zip: bool return_original_file: bool + client_side_output_generation: bool start_page_id: int end_page_id: int upload_names: list[str] @@ -848,6 +849,11 @@ async def run_parse_job( f_dump_content_list=request_options.return_content_list, start_page_id=request_options.start_page_id, end_page_id=request_options.end_page_id, + client_side_output_generation=getattr( + request_options, + "client_side_output_generation", + False, + ), **config, ) @@ -897,6 +903,7 @@ async def create_async_parse_task( return_images=request_options.return_images, response_format_zip=request_options.response_format_zip, return_original_file=request_options.return_original_file, + client_side_output_generation=request_options.client_side_output_generation, start_page_id=request_options.start_page_id, end_page_id=request_options.end_page_id, upload_names=[upload.original_name for upload in uploads], diff --git a/mineru/cli/gradio_app.py b/mineru/cli/gradio_app.py index 1157c9d6..509a3095 100644 --- a/mineru/cli/gradio_app.py +++ b/mineru/cli/gradio_app.py @@ -38,6 +38,7 @@ from mineru.cli.common import ( read_fn, ) from mineru.cli import api_client as _api_client +from mineru.cli.client_side_output import regenerate_client_side_outputs from mineru.cli.output_paths import resolve_parse_dir from mineru.cli.vlm_preload import resolve_gradio_local_api_cli_args from mineru.cli.visualization import VisualizationJob, run_visualization_job @@ -827,6 +828,11 @@ def is_image_analysis_option_visible(backend): return backend.startswith("vlm") or backend.startswith("hybrid") +def should_use_client_side_output_generation(client_side_output_generation): + """判断当前 Gradio 任务是否需要在客户端生成最终输出。""" + return client_side_output_generation + + def create_gradio_run_paths(file_path, output_root="./output"): run_id = f"{time.strftime('%y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}_{safe_stem(Path(file_path).stem)}" run_root = Path(output_root) / "gradio" / run_id @@ -959,6 +965,7 @@ async def _run_to_markdown_job( backend="pipeline", url=None, api_url=None, + client_side_output_generation=False, status_callback: Callable[[str], None] | None = None, ): if file_path is None: @@ -971,6 +978,9 @@ async def _run_to_markdown_job( normalized_language = normalize_language(language) file_path = str(file_path) file_suffix = Path(file_path).suffix.lower().lstrip('.') + use_client_side_output_generation = should_use_client_side_output_generation( + client_side_output_generation + ) parse_method = resolve_parse_method(file_path, is_ocr, backend) run_root, extract_root, archive_zip_path = create_gradio_run_paths(file_path) run_root.mkdir(parents=True, exist_ok=True) @@ -985,13 +995,14 @@ async def _run_to_markdown_job( server_url=url, start_page_id=0, end_page_id=end_pages - 1, - return_md=True, + return_md=not use_client_side_output_generation, return_middle_json=True, - return_model_output=True, - return_content_list=True, + return_model_output=not use_client_side_output_generation, + return_content_list=not use_client_side_output_generation, return_images=True, response_format_zip=True, return_original_file=True, + client_side_output_generation=use_client_side_output_generation, ) upload_assets = [ _api_client.UploadAsset( @@ -1062,6 +1073,10 @@ async def _run_to_markdown_job( parse_method, allow_office_fallback=True, ) + emit_status(STATUS_PROCESSING_OUTPUT) + if use_client_side_output_generation: + await asyncio.to_thread(regenerate_client_side_outputs, local_md_dir, file_name) + preview_pdf_path = maybe_generate_local_preview( extract_root=extract_root, file_name=file_name, @@ -1070,7 +1085,6 @@ async def _run_to_markdown_job( parse_method=parse_method, ) - emit_status(STATUS_PROCESSING_OUTPUT) zip_archive_success = compress_directory_to_zip(local_md_dir, archive_zip_path) if zip_archive_success == 0: logger.info('Compression successful') @@ -1101,6 +1115,7 @@ async def stream_to_markdown( backend="pipeline", url=None, api_url=None, + client_side_output_generation=False, ): status_state = StatusPanelState() job_task: asyncio.Task | None = None @@ -1130,6 +1145,7 @@ async def stream_to_markdown( backend=backend, url=url, api_url=api_url, + client_side_output_generation=client_side_output_generation, status_callback=enqueue_status, ) ) @@ -1489,6 +1505,13 @@ def update_doc_show(file_path): help="Preload the local VLM model when gradio starts a local mineru-api service.", default=False, ) +@click.option( + '--client-side-output-generation', + 'client_side_output_generation', + type=bool, + help="Generate markdown and content lists locally from server-returned middle json.", + default=False, +) @click.option( '--latex-delimiters-type', 'latex_delimiters_type', @@ -1501,7 +1524,8 @@ def main(ctx, example_enable, http_client_enable, api_enable, max_convert_pages, - server_name, server_port, api_url, enable_vlm_preload, latex_delimiters_type, **kwargs + server_name, server_port, api_url, enable_vlm_preload, + client_side_output_generation, latex_delimiters_type, **kwargs ): # 创建 i18n 实例,支持中英文 @@ -1731,6 +1755,7 @@ def main(ctx, backend=backend, url=url, api_url=api_url, + client_side_output_generation=client_side_output_generation, ): update = ( render_status_steps_html(update[0], i18n, locale=request_locale), diff --git a/mineru/model/table/rec/unet_table/utils_table_line_rec.py b/mineru/model/table/rec/unet_table/utils_table_line_rec.py index 25fd2154..126a4677 100644 --- a/mineru/model/table/rec/unet_table/utils_table_line_rec.py +++ b/mineru/model/table/rec/unet_table/utils_table_line_rec.py @@ -4,7 +4,6 @@ from typing import Iterator import cv2 import numpy as np -from scipy.spatial import distance as dist class ConnectedComponent: @@ -232,7 +231,7 @@ def _order_points(pts): left_most = left_most[np.argsort(left_most[:, 1]), :] (tl, bl) = left_most - distance = dist.cdist(tl[np.newaxis], right_most, "euclidean")[0] + distance = np.linalg.norm(right_most - tl, axis=1) (br, tr) = right_most[np.argsort(distance)[::-1], :] return np.array([tl, tr, br, bl], dtype="float32") diff --git a/mineru/utils/llm_aided.py b/mineru/utils/llm_aided.py index aaff369b..9340858e 100644 --- a/mineru/utils/llm_aided.py +++ b/mineru/utils/llm_aided.py @@ -18,18 +18,27 @@ MAX_TITLE_GROUP_WORKERS = 4 def _get_title_line_avg_height(block): - if "line_avg_height" in block: - return block["line_avg_height"] + line_avg_height = block.get("line_avg_height") + if isinstance(line_avg_height, (int, float)) and line_avg_height > 0: + return line_avg_height title_block_line_height_list = [] for line in block.get("lines", []): - bbox = line["bbox"] - title_block_line_height_list.append(int(bbox[3] - bbox[1])) + # 标题行高是 LLM prompt 的几何提示,这里只消费有效 bbox,不回写 middle json。 + bbox = line.get("bbox") + if not bbox or len(bbox) < 4: + continue + line_height = bbox[3] - bbox[1] + if line_height > 0: + title_block_line_height_list.append(int(line_height)) if len(title_block_line_height_list) > 0: return sum(title_block_line_height_list) / len(title_block_line_height_list) - return int(block["bbox"][3] - block["bbox"][1]) + bbox = block.get("bbox") + if bbox and len(bbox) >= 4: + return max(0, int(bbox[3] - bbox[1])) + return 0 def _collect_title_block_refs(page_info_list): diff --git a/mineru/utils/title_level_postprocess.py b/mineru/utils/title_level_postprocess.py new file mode 100644 index 00000000..bb7c1066 --- /dev/null +++ b/mineru/utils/title_level_postprocess.py @@ -0,0 +1,68 @@ +# Copyright (c) Opendatalab. All rights reserved. +from __future__ import annotations + +from copy import deepcopy +from typing import Any + +from mineru.utils.config_reader import get_llm_aided_config +from mineru.utils.llm_aided import llm_aided_title + + +SUPPORTED_PDF_BACKENDS = {"pipeline", "vlm", "hybrid"} + + +def _resolve_title_aided_config() -> dict[str, Any] | None: + """从本地配置解析标题分级开关。""" + llm_aided_config = get_llm_aided_config() + title_aided_config = ( + llm_aided_config.get("title_aided") + if isinstance(llm_aided_config, dict) + else None + ) + if not isinstance(title_aided_config, dict) or not title_aided_config: + return None + if not title_aided_config.get("enable", False): + return None + return deepcopy(title_aided_config) + + +def apply_title_leveling_to_pdf_info(pdf_info: list[dict[str, Any]]): + title_aided_config = _resolve_title_aided_config() + if title_aided_config: + llm_aided_title(pdf_info, title_aided_config) + + +def finalize_client_side_middle_json(middle_json: dict[str, Any]) -> dict[str, Any]: + """根据 staged middle json 的后端类型,在客户端执行完整 finalize。""" + if not isinstance(middle_json, dict): + raise ValueError("middle_json must be a dict.") + + backend = middle_json.get("_backend") + pdf_info = middle_json.get("pdf_info") + + if backend == "pipeline": + from mineru.backend.pipeline.model_json_to_middle_json import ( + finalize_middle_json_from_preproc, + ) + + finalize_middle_json_from_preproc( + pdf_info, + ) + elif backend == "vlm": + from mineru.backend.vlm.model_output_to_middle_json import ( + finalize_middle_json, + ) + + finalize_middle_json( + pdf_info, + ) + elif backend == "hybrid": + from mineru.backend.hybrid.hybrid_model_output_to_middle_json import ( + finalize_middle_json_from_preproc, + ) + + finalize_middle_json_from_preproc( + pdf_info, + ) + + return middle_json diff --git a/pyproject.toml b/pyproject.toml index e1ddc34f..b8b05b91 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,7 @@ vlm = [ "accelerate>=1.5.1", ] vllm = [ - "vllm>=0.10.1.1,<0.12", + "vllm>=0.10.1.1,<0.22.0", ] lmdeploy = [ "lmdeploy>=0.10.2,<0.12",