opensource/MinerU
3
0
Fork 0
mirror of https://github.com/opendatalab/MinerU.git synced 2026-10-29 19:07:28 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: update language specification in OCR options for pipeline backend

Browse Source
This commit is contained in:
myhloli
2026-06-09 17:16:15 +08:00
parent 41931b861c
commit adf0866218
7 changed files with 18 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/en/usage/cli_tools.md
+1 -1
View File
@@ -16,7 +16,7 @@ Options:
Parsing backend (default: hybrid-engine)
--effort [medium|high] Hybrid parsing effort (default: medium)
-l, --lang [ch|ch_server|ch_lite|en|korean|japan|chinese_cht|ta|te|ka|th|el|latin|arabic|east_slavic|cyrillic|devanagari]
Specify document language (improves OCR accuracy, pipeline and hybrid* backend only)
Specify document language (improves OCR accuracy, pipeline backend only)
-u, --url TEXT OpenAI-compatible backend URL passed through to the server when using http-client
-s, --start INTEGER Starting page number for parsing (0-based)
-e, --end INTEGER Ending page number for parsing (0-based)
docs/zh/usage/cli_tools.md
+1 -1
View File
@@ -16,7 +16,7 @@ Options:
解析后端（默认为 hybrid-engine）
--effort [medium|high] Hybrid 解析强度（默认：medium）
-l, --lang [ch|ch_server|ch_lite|en|korean|japan|chinese_cht|ta|te|ka|th|el|latin|arabic|east_slavic|cyrillic|devanagari]
指定文档语言（可提升 OCR 准确率，仅用于 pipeline 与 hybrid* 后端）
指定文档语言（可提升 OCR 准确率，仅用于 pipeline 后端）
-u, --url TEXT 当使用 http-client 时，传给服务端后端的 OpenAI 兼容地址
-s, --start INTEGER 开始解析的页码（从 0 开始）
-e, --end INTEGER 结束解析的页码（从 0 开始）
mineru/backend/hybrid/hybrid_analyze.py
+3 -10
View File
@@ -672,7 +672,6 @@ def _predict_layout_for_title_split(
def _predict_layout_for_window(
images_pil_list,
language,
inline_formula_enable,
batch_ratio,
vlm_ocr_enable,
@@ -680,7 +679,6 @@ def _predict_layout_for_window(
"""为单个处理窗口执行一次 pipeline layout，并返回可复用的小模型实例。"""
hybrid_model_singleton = HybridModelSingleton()
hybrid_pipeline_model = hybrid_model_singleton.get_model(
lang=language,
formula_enable=inline_formula_enable and not vlm_ocr_enable,
)
images_layout_res = _predict_layout_for_title_split(
@@ -954,7 +952,7 @@ def get_batch_ratio(device):
return batch_ratio
def _should_enable_vlm_ocr(ocr_enable: bool, language: str, inline_formula_enable: bool) -> bool:
def _should_enable_vlm_ocr(ocr_enable: bool, inline_formula_enable: bool) -> bool:
"""判断是否启用VLM OCR"""
force_enable = os.getenv("MINERU_FORCE_VLM_OCR_ENABLE", "0").lower() in ("1", "true", "yes")
if force_enable:
@@ -963,7 +961,6 @@ def _should_enable_vlm_ocr(ocr_enable: bool, language: str, inline_formula_enabl
force_pipeline = os.getenv("MINERU_HYBRID_FORCE_PIPELINE_ENABLE", "0").lower() in ("1", "true", "yes")
return (
ocr_enable
and language in ["ch", "en"]
and inline_formula_enable
and not force_pipeline
)
@@ -985,7 +982,6 @@ def doc_analyze(
predictor: MinerUClient | None = None,
backend="transformers",
parse_method: str = 'auto',
language: str = 'ch',
inline_formula_enable: bool = True,
model_path: str | None = None,
server_url: str | None = None,
@@ -1003,7 +999,7 @@ def doc_analyze(
device = get_device()
_ocr_enable = ocr_classify(pdf_bytes, parse_method=parse_method)
_vlm_ocr_enable = _should_enable_vlm_ocr(_ocr_enable, language, inline_formula_enable)
_vlm_ocr_enable = _should_enable_vlm_ocr(_ocr_enable, inline_formula_enable)
pdf_doc = open_pdfium_document(pdfium.PdfDocument, pdf_bytes)
middle_json = init_middle_json(
@@ -1053,7 +1049,6 @@ def doc_analyze(
)
images_layout_res, hybrid_pipeline_model = _predict_layout_for_window(
images_pil_list,
language,
inline_formula_enable,
batch_ratio,
_vlm_ocr_enable,
@@ -1201,7 +1196,6 @@ async def aio_doc_analyze(
predictor: MinerUClient | None = None,
backend="transformers",
parse_method: str = 'auto',
language: str = 'ch',
inline_formula_enable: bool = True,
model_path: str | None = None,
server_url: str | None = None,
@@ -1219,7 +1213,7 @@ async def aio_doc_analyze(
device = get_device()
_ocr_enable = ocr_classify(pdf_bytes, parse_method=parse_method)
_vlm_ocr_enable = _should_enable_vlm_ocr(_ocr_enable, language, inline_formula_enable)
_vlm_ocr_enable = _should_enable_vlm_ocr(_ocr_enable, inline_formula_enable)
pdf_doc = open_pdfium_document(pdfium.PdfDocument, pdf_bytes)
middle_json = init_middle_json(
@@ -1269,7 +1263,6 @@ async def aio_doc_analyze(
images_layout_res, hybrid_pipeline_model = await asyncio.to_thread(
_predict_layout_for_window,
images_pil_list,
language,
inline_formula_enable,
batch_ratio,
_vlm_ocr_enable,
mineru/cli/api_request.py
+1 -1
View File
@@ -88,7 +88,7 @@ async def parse_request_form(
lang_list: Annotated[
list[str],
Form(
description="""(Adapted only for pipeline and hybrid backend)Input the languages in the pdf to improve OCR accuracy.Options:
description="""(Adapted for pipeline backend only) Input the languages in the pdf to improve OCR accuracy. Options:
- ch: Chinese, English, Chinese Traditional.
- ch_lite: Chinese, English, Chinese Traditional, Japanese.
- ch_server: Chinese, English, Chinese Traditional, Japanese.
mineru/cli/client.py
+1 -1
View File
@@ -1121,7 +1121,7 @@ async def run_orchestrated_cli(
help="""
Input the languages in the pdf (if known) to improve OCR accuracy.
Without languages specified, 'ch' will be used by default.
Adapted only for the case where the backend is set to 'pipeline' and 'hybrid-*'.
Specify document language (improves OCR accuracy, pipeline backend only).
""",
)
@click.option(
mineru/cli/common.py
+4 -8
View File
@@ -509,7 +509,6 @@ def _process_hybrid(
output_dir,
pdf_file_names,
pdf_bytes_list,
h_lang_list,
parse_method,
inline_formula_enable,
backend,
@@ -533,7 +532,7 @@ def _process_hybrid(
if not backend.endswith("client"):
server_url = None
for idx, (pdf_bytes, lang) in enumerate(zip(pdf_bytes_list, h_lang_list)):
for idx, pdf_bytes in enumerate(pdf_bytes_list):
pdf_file_name = pdf_file_names[idx]
local_image_dir, local_md_dir = prepare_env(output_dir, pdf_file_name, f"hybrid_{parse_method}")
image_writer, md_writer = FileBasedDataWriter(local_image_dir), FileBasedDataWriter(local_md_dir)
@@ -543,7 +542,6 @@ def _process_hybrid(
image_writer=image_writer,
backend=backend,
parse_method=parse_method,
language=lang,
inline_formula_enable=inline_formula_enable,
server_url=server_url,
effort=validate_effort(effort),
@@ -567,7 +565,6 @@ async def _async_process_hybrid(
output_dir,
pdf_file_names,
pdf_bytes_list,
h_lang_list,
parse_method,
inline_formula_enable,
backend,
@@ -591,7 +588,7 @@ async def _async_process_hybrid(
if not backend.endswith("client"):
server_url = None
for idx, (pdf_bytes, lang) in enumerate(zip(pdf_bytes_list, h_lang_list)):
for idx, pdf_bytes in enumerate(pdf_bytes_list):
pdf_file_name = pdf_file_names[idx]
local_image_dir, local_md_dir = prepare_env(output_dir, pdf_file_name, f"hybrid_{parse_method}")
image_writer, md_writer = FileBasedDataWriter(local_image_dir), FileBasedDataWriter(local_md_dir)
@@ -601,7 +598,6 @@ async def _async_process_hybrid(
image_writer=image_writer,
backend=backend,
parse_method=parse_method,
language=lang,
inline_formula_enable=inline_formula_enable,
server_url=server_url,
effort=validate_effort(effort),
@@ -755,7 +751,7 @@ def do_parse(
os.environ['MINERU_VLM_FORMULA_ENABLE'] = "true"
_process_hybrid(
output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, backend,
output_dir, pdf_file_names, pdf_bytes_list, parse_method, formula_enable, backend,
f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json,
f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode,
server_url, effort=effort, image_analysis=image_analysis,
@@ -850,7 +846,7 @@ async def aio_do_parse(
os.environ['MINERU_VLM_FORMULA_ENABLE'] = "true"
await _async_process_hybrid(
output_dir, pdf_file_names, pdf_bytes_list, p_lang_list, parse_method, formula_enable, backend,
output_dir, pdf_file_names, pdf_bytes_list, parse_method, formula_enable, backend,
f_draw_layout_bbox, f_draw_span_bbox, f_dump_md, f_dump_middle_json,
f_dump_model_output, f_dump_orig_pdf, f_dump_content_list, f_make_md_mode,
server_url, effort=effort, image_analysis=image_analysis,
mineru/cli/gradio_app.py
+7 -5
View File
@@ -846,6 +846,11 @@ def is_image_analysis_option_visible(backend):
return backend.startswith("vlm") or backend.startswith("hybrid")
def is_ocr_options_visible(backend):
"""判断 OCR 语言和强制 OCR 选项是否展示；Hybrid/VLM 已不需要用户指定语言。"""
return backend == "pipeline"
def should_use_client_side_output_generation(client_side_output_generation):
"""判断当前 Gradio 任务是否需要在客户端生成最终输出。"""
return client_side_output_generation
@@ -1725,10 +1730,7 @@ def main(ctx,
image_analysis_update = gr.update(visible=is_image_analysis_option_visible(backend_choice))
effort_update = gr.update(visible=is_effort_option_visible(backend_choice))
client_options_update = gr.update(visible=is_http_client_backend(backend_choice))
if "vlm" in backend_choice:
ocr_options_update = gr.update(visible=False)
else:
ocr_options_update = gr.update(visible=True)
ocr_options_update = gr.update(visible=is_ocr_options_visible(backend_choice))
return client_options_update, ocr_options_update, formula_label_update, backend_info_update, image_analysis_update, effort_update
@@ -1921,7 +1923,7 @@ def main(ctx,
visible=is_effort_option_visible(preferred_option),
info=i18n("hybrid_effort_info"),
)
with gr.Group() as ocr_options:
with gr.Group(visible=is_ocr_options_visible(preferred_option)) as ocr_options:
language = gr.Dropdown(
all_lang,
label=i18n("ocr_language"),