diff --git a/mineru/backend/hybrid/hybrid_analyze.py b/mineru/backend/hybrid/hybrid_analyze.py index 34cb066f..96434e7a 100644 --- a/mineru/backend/hybrid/hybrid_analyze.py +++ b/mineru/backend/hybrid/hybrid_analyze.py @@ -72,13 +72,19 @@ LAYOUT_TITLE_SPLIT_OVERLAP_THRESHOLD = 0.8 not_extract_list = [item.value for item in NotExtractType] HYBRID_OCR_DET_TEXT_TYPES = set(not_extract_list) +HYBRID_VLM_OCR_DET_TEXT_TYPES = { + MineruBlockType.TEXT, + MineruBlockType.TITLE, + MineruBlockType.DOC_TITLE, + MineruBlockType.PARAGRAPH_TITLE, +} HYBRID_ANALYZE_EFFORTS = {"medium", "high"} INLINE_FORMULA_CONTAINER_LABELS = {"table", "image", "chart", "display_formula"} MEDIUM_EFFORT_LAYOUT_LABEL_TO_VLM_TYPE = { "abstract": BlockType.TEXT, "algorithm": BlockType.CODE, "aside_text": BlockType.ASIDE_TEXT, - "content": BlockType.TEXT, + "content": BlockType.INDEX, "doc_title": BlockType.TITLE, "footer": BlockType.FOOTER, "footer_image": BlockType.FOOTER, @@ -126,9 +132,10 @@ def _apply_medium_visual_sub_type(block, label: str | None): block["sub_type"] = "seal" -def _is_hybrid_ocr_det_candidate(block): +def _is_hybrid_ocr_det_candidate(block, candidate_types=None): """判断 Hybrid 文本类块是否需要 OCR det 生成行级视觉信息。""" - return (block.get("type") or block.get("label")) in HYBRID_OCR_DET_TEXT_TYPES + candidate_types = candidate_types or HYBRID_OCR_DET_TEXT_TYPES + return (block.get("type") or block.get("label")) in candidate_types def ocr_classify(pdf_bytes, parse_method: str = 'auto',) -> bool: # 确定OCR设置 @@ -146,6 +153,8 @@ def ocr_det( model_list, mfd_res, batch_ratio: int = 1, + *, + candidate_types=None, ): mask_formula_for_ocr_det = get_ocr_det_mask_inline_formula_enable(True) @@ -169,7 +178,7 @@ def ocr_det( ocr_res_list.append([]) img_height, img_width = np_image.shape[:2] for res in page_results: - if not _is_hybrid_ocr_det_candidate(res): + if not _is_hybrid_ocr_det_candidate(res, candidate_types): continue x0 = max(0, int(res['bbox'][0] * img_width)) y0 = max(0, int(res['bbox'][1] * img_height)) @@ -220,7 +229,7 @@ def ocr_det( ocr_res_list.append([]) img_height, img_width = np_image.shape[:2] for res in page_results: - if not _is_hybrid_ocr_det_candidate(res): + if not _is_hybrid_ocr_det_candidate(res, candidate_types): continue x0 = max(0, int(res['bbox'][0] * img_width)) y0 = max(0, int(res['bbox'][1] * img_height)) @@ -773,6 +782,7 @@ def _apply_vlm_ocr_det_sidecars_for_window( model_list, formula_mask_inputs, batch_ratio=batch_ratio, + candidate_types=HYBRID_VLM_OCR_DET_TEXT_TYPES, ) _normalize_bbox(inline_formula_list, ocr_res_list, images_pil_list) model_list[:] = _merge_page_sidecar_items( diff --git a/mineru/backend/hybrid/hybrid_magic_model.py b/mineru/backend/hybrid/hybrid_magic_model.py index 1d963f74..3b786db5 100644 --- a/mineru/backend/hybrid/hybrid_magic_model.py +++ b/mineru/backend/hybrid/hybrid_magic_model.py @@ -27,15 +27,11 @@ not_extract_list = [item.value for item in NotExtractType] + [ BlockType.PARAGRAPH_TITLE, ] OCR_DET_LINES_KEY = "_ocr_det_lines" -OCR_DET_LINE_BLOCK_TYPES = set(not_extract_list) | { - BlockType.LIST, - BlockType.INDEX, - BlockType.ABSTRACT, - BlockType.ASIDE_TEXT, - BlockType.PHONETIC, - BlockType.CHART_CAPTION, - BlockType.CHART_FOOTNOTE, - BlockType.CODE_FOOTNOTE, +OCR_DET_LINE_BLOCK_TYPES = { + BlockType.TEXT, + BlockType.TITLE, + BlockType.DOC_TITLE, + BlockType.PARAGRAPH_TITLE, } @@ -139,6 +135,7 @@ class MagicModel: "aside_text", "page_footnote", "list", + "index", ]: span_type = ContentType.TEXT elif block_type in ["image_caption", "table_caption", "code_caption"]: @@ -322,6 +319,10 @@ class MagicModel: block = fix_text_block(block) _copy_raw_text_block_metadata(raw_block_type, block_info, block) + if block["type"] == BlockType.INDEX: + # index 仅用于 Hybrid medium content 强制走 VLM-OCR，输出前统一还原为正文。 + block["type"] = BlockType.TEXT + blocks.append(block) fallback_inline_caption_fragments(blocks, VISUAL_MAIN_TYPES) diff --git a/mineru/utils/pdf_classify.py b/mineru/utils/pdf_classify.py index d56f567f..8e23c915 100644 --- a/mineru/utils/pdf_classify.py +++ b/mineru/utils/pdf_classify.py @@ -35,6 +35,41 @@ ASCII_PUNCT_RUN_MIN_LENGTH = 4 SUSPICIOUS_ASCII_PUNCT_MIN_TEXT_CHARS = 100 SUSPICIOUS_ASCII_PUNCT_RATIO_THRESHOLD = 0.25 SUSPICIOUS_ASCII_PUNCT_RUN_RATIO_THRESHOLD = 0.10 +SUSPICIOUS_CROSS_SCRIPT_MIN_TEXT_CHARS = 300 +SUSPICIOUS_CROSS_SCRIPT_MIN_CJK_CHARS = 100 +SUSPICIOUS_CROSS_SCRIPT_COUNT_THRESHOLD = 120 +SUSPICIOUS_CROSS_SCRIPT_RATIO_THRESHOLD = 0.18 +SUSPICIOUS_CROSS_SCRIPT_MIN_SCRIPT_COUNT = 3 +SUSPICIOUS_CROSS_SCRIPT_SCRIPT_MIN_CHARS = 5 +SUSPICIOUS_CROSS_SCRIPT_RANGES = ( + (0x0400, 0x052F, "Cyrillic"), + (0x0600, 0x06FF, "Arabic"), + (0x0700, 0x074F, "Syriac"), + (0x0750, 0x077F, "Arabic Supplement"), + (0x0780, 0x07BF, "Thaana"), + (0x07C0, 0x07FF, "NKo"), + (0x0800, 0x083F, "Samaritan"), + (0x0840, 0x085F, "Mandaic"), + (0x0860, 0x086F, "Syriac Supplement"), + (0x0870, 0x089F, "Arabic Extended-B"), + (0x0900, 0x097F, "Devanagari"), + (0x0C80, 0x0CFF, "Kannada"), + (0x1000, 0x109F, "Myanmar"), + (0x1100, 0x11FF, "Hangul Jamo"), + (0x1200, 0x137F, "Ethiopic"), + (0x13A0, 0x13FF, "Cherokee"), + (0x1400, 0x167F, "Canadian Syllabics"), + (0x1800, 0x18AF, "Mongolian"), + (0x1A20, 0x1AAF, "Tai Tham"), + (0x2C00, 0x2C5F, "Glagolitic"), + (0xA000, 0xA48F, "Yi"), +) +CJK_TEXT_RANGES = ( + (0x3400, 0x4DBF), + (0x4E00, 0x9FFF), + (0xF900, 0xFAFF), + (0x20000, 0x2EBEF), +) _ALLOWED_CONTROL_CODES = {9, 10, 13} _PRIVATE_USE_AREA_START = 0xE000 @@ -133,6 +168,20 @@ def classify(pdf_bytes): ): return "ocr" + cross_script_signal = _get_cross_script_text_signal_from_samples( + text_samples + ) + if cross_script_signal["triggered"]: + logger.debug( + "Classify PDF as OCR due to suspicious cross-script text: " + f"chars={cross_script_signal['total_chars']}, " + f"cjk={cross_script_signal['cjk_chars']}, " + f"suspicious={cross_script_signal['suspicious_chars']}, " + f"ratio={cross_script_signal['suspicious_ratio']:.4f}, " + f"scripts={cross_script_signal['top_scripts']}" + ) + return "ocr" + u72xx_signal = _get_u72xx_text_signal_from_samples(text_samples) if ( u72xx_signal["u72xx_count"] @@ -473,6 +522,74 @@ def _get_cid_font_usage_signal_from_samples(text_samples, cid_font_signal): return best_signal +def _is_cjk_text_char(char: str) -> bool: + """判断字符是否属于中文文档中可接受的 CJK 文字范围。""" + unicode_code = ord(char) + return any(start <= unicode_code <= end for start, end in CJK_TEXT_RANGES) + + +def _get_cross_script_name(char: str) -> str | None: + """识别中文文档乱码中常见的跨脚本字符块名称。""" + unicode_code = ord(char) + for start, end, script_name in SUSPICIOUS_CROSS_SCRIPT_RANGES: + if start <= unicode_code <= end: + return script_name + return None + + +def _get_cross_script_text_signal_from_samples(text_samples): + """统计中文文档文本层中大比例跨脚本混入信号，用于识别合法 Unicode 错码。""" + total_chars = 0 + cjk_chars = 0 + suspicious_chars = 0 + script_counts = {} + + for text_sample in text_samples: + for char in text_sample["cleaned_text"]: + total_chars += 1 + if _is_cjk_text_char(char): + cjk_chars += 1 + + script_name = _get_cross_script_name(char) + if script_name is None: + continue + + suspicious_chars += 1 + script_counts[script_name] = script_counts.get(script_name, 0) + 1 + + suspicious_ratio = 0.0 + if total_chars > 0: + suspicious_ratio = suspicious_chars / total_chars + + dense_script_count = sum( + 1 + for count in script_counts.values() + if count >= SUSPICIOUS_CROSS_SCRIPT_SCRIPT_MIN_CHARS + ) + top_scripts = sorted( + script_counts.items(), + key=lambda item: (-item[1], item[0]), + )[:5] + triggered = ( + total_chars >= SUSPICIOUS_CROSS_SCRIPT_MIN_TEXT_CHARS + and cjk_chars >= SUSPICIOUS_CROSS_SCRIPT_MIN_CJK_CHARS + and suspicious_chars >= SUSPICIOUS_CROSS_SCRIPT_COUNT_THRESHOLD + and suspicious_ratio >= SUSPICIOUS_CROSS_SCRIPT_RATIO_THRESHOLD + and dense_script_count >= SUSPICIOUS_CROSS_SCRIPT_MIN_SCRIPT_COUNT + ) + + return { + "triggered": triggered, + "total_chars": total_chars, + "cjk_chars": cjk_chars, + "suspicious_chars": suspicious_chars, + "suspicious_ratio": suspicious_ratio, + "script_counts": script_counts, + "top_scripts": top_scripts, + "dense_script_count": dense_script_count, + } + + def _get_u72xx_text_signal_from_samples(text_samples): """基于已缓存的抽样页文本统计扣除常用字后的 U+7280-U+72DF 字符占比。""" cjk_chars = 0 diff --git a/pyproject.toml b/pyproject.toml index 79cc4dd7..ab0fe9e1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ dependencies = [ "openai>=1.70.0,<3", "beautifulsoup4>=4.13.5,<5", "magika>=0.6.2,<1.1.0", - "mineru-vl-utils>=1.0.4,<2", + "mineru-vl-utils>=1.0.5,<2", "python-docx>=1.2.0,<2", 'pypptx-with-oxml>=1.0.3,<2', "mammoth>=1.11.0,<2",