diff --git a/mineru/utils/span_pre_proc.py b/mineru/utils/span_pre_proc.py index 80fd375a..e5afc43b 100644 --- a/mineru/utils/span_pre_proc.py +++ b/mineru/utils/span_pre_proc.py @@ -66,10 +66,7 @@ def txt_spans_extract(pdf_page, spans, pil_img, scale, all_bboxes, all_discarded textpage=textpage, page_char_count=page_char_count, ) - page_all_chars = [ - char for char in page_chars['chars'] - if _is_supported_rotation(char['rotation']) - ] + page_all_chars = _get_chars_for_span_fill(page_chars) # 计算所有sapn的高度的中位数 span_height_list = [] @@ -140,6 +137,77 @@ def _is_supported_rotation(rotation) -> bool: return any(abs(rotation_degrees - angle) < 0.1 for angle in [0, 90, 180, 270]) +def _get_char_fill_key(char): + """生成字符回填判定 key,优先使用 pdftext 提供的页内 char_idx。""" + char_idx = char.get('char_idx') + if char_idx is not None: + return ('char_idx', char_idx) + return ('object_id', id(char)) + + +def _iter_line_chars(line): + """按 pdftext line/span 结构展开字符,兼容缺少 chars 字段的异常 span。""" + for span in line.get('spans', []): + for char in span.get('chars', []): + yield char + + +def _is_visible_standard_rotation_char(char) -> bool: + """判断字符是否是可见的标准方向正文字符,避免换行控制符误放行水印。""" + text = char.get('char', '') + if not text or text.isspace() or text in {'\r', '\n'}: + return False + + bbox = char.get('bbox') + if not bbox: + return False + + x0, y0, x1, y1 = [float(v) for v in bbox] + return ( + x1 > x0 + and y1 > y0 + and _is_supported_rotation(char.get('rotation', 0)) + ) + + +def _get_chars_for_span_fill(page_chars): + """选择允许参与 span 回填的字符,保留正文内仿斜体并过滤整行斜向水印。""" + all_chars = page_chars['chars'] + fill_char_keys = { + _get_char_fill_key(char) + for char in all_chars + if _is_supported_rotation(char.get('rotation', 0)) + } + + rotated_chars = [ + char for char in all_chars + if not _is_supported_rotation(char.get('rotation', 0)) + ] + if not rotated_chars: + return [ + char for char in all_chars + if _get_char_fill_key(char) in fill_char_keys + ] + + for line in get_lines_from_chars(all_chars): + if not _is_supported_rotation(line.get('rotation', 0)): + continue + + line_chars = list(_iter_line_chars(line)) + if not any(_is_visible_standard_rotation_char(char) for char in line_chars): + continue + + # 标准方向正文行内的局部旋转字符通常是仿斜体强调,需要允许回填。 + for char in line_chars: + if not _is_supported_rotation(char.get('rotation', 0)): + fill_char_keys.add(_get_char_fill_key(char)) + + return [ + char for char in all_chars + if _get_char_fill_key(char) in fill_char_keys + ] + + def _prepare_post_ocr_spans(need_ocr_spans, spans, pil_img, scale): if len(need_ocr_spans) == 0: return spans