From b433a6c17dbf854250cf59f68319e5f8afafa8eb Mon Sep 17 00:00:00 2001 From: myhloli Date: Wed, 20 May 2026 17:06:51 +0800 Subject: [PATCH] refactor: remove seal type handling and update visual block processing for images --- docs/en/reference/output_files.md | 4 +- docs/zh/reference/output_files.md | 4 +- .../pipeline/model_json_to_middle_json.py | 1 - .../backend/pipeline/pipeline_magic_model.py | 33 ++++- .../pipeline_middle_json_mkcontent.py | 127 +++++++++--------- mineru/utils/draw_bbox.py | 6 +- mineru/utils/enum_class.py | 3 - 7 files changed, 93 insertions(+), 85 deletions(-) diff --git a/docs/en/reference/output_files.md b/docs/en/reference/output_files.md index fd21ec7b..fd6dfe4c 100644 --- a/docs/en/reference/output_files.md +++ b/docs/en/reference/output_files.md @@ -306,7 +306,6 @@ This is a simplified version of `middle.json` that stores all readable content b | `chart` | Chart | | `text` | Text/Title | | `equation` | Interline formula | -| `seal` | Seal | | `code` | Code block / algorithm block | | `list` | List / reference list | | `header` / `footer` / `page_number` / `aside_text` / `page_footnote` | Page auxiliary blocks | @@ -327,6 +326,7 @@ Text levels are distinguished through the `text_level` field: - `code` entries use `sub_type` to distinguish `code` and `algorithm`, and may include fields such as `code_body`, `code_caption`, and `code_footnote`. - `list` entries may use `sub_type` to distinguish ordinary lists from reference-style lists. - `image` / `chart` entries may include an optional `sub_type` field to carry the visual subtype through downstream outputs. +- Seal content is represented as an `image` entry with `sub_type: "seal"`. ##### Sample Data @@ -423,7 +423,7 @@ Text levels are distinguished through the `text_level` field: | `title` | Title block with `title_content` and `level` | | `paragraph` | Paragraph block with `paragraph_content` | | `equation_interline` | Interline formula with `math_content` and `math_type` | -| `image` / `table` / `chart` / `seal` | Visual blocks with image paths, captions, and related structured fields | +| `image` / `table` / `chart` | Visual blocks with image paths, captions, and related structured fields. Seal content uses `image` with `sub_type: "seal"` | | `code` | Code block with `code_content`, `code_caption`, `code_footnote`, and `code_language` | | `algorithm` | Algorithm block with `algorithm_content`, `algorithm_caption`, and `algorithm_footnote` | | `list` / `index` | List and index blocks with `list_items` | diff --git a/docs/zh/reference/output_files.md b/docs/zh/reference/output_files.md index 1b24ff00..fb5000c2 100644 --- a/docs/zh/reference/output_files.md +++ b/docs/zh/reference/output_files.md @@ -306,7 +306,6 @@ | `chart` | 图表 | | `text` | 文本/标题 | | `equation` | 行间公式 | -| `seal` | 印章 | | `code` | 代码块 / 算法块 | | `list` | 列表 / 参考文献列表 | | `header` / `footer` / `page_number` / `aside_text` / `page_footnote` | 页面辅助块 | @@ -327,6 +326,7 @@ - `code` 类型会通过 `sub_type` 区分 `code` 和 `algorithm`，并可包含 `code_body`、`code_caption`、`code_footnote` 等字段。 - `list` 类型可通过 `sub_type` 区分普通列表和参考文献列表。 - `image` / `chart` 类型可包含可选 `sub_type` 字段，用于透传视觉子类型。 +- 印章内容通过 `sub_type: "seal"` 的 `image` 类型表示。 ##### 示例数据 @@ -423,7 +423,7 @@ | `title` | 标题块，包含 `title_content` 与 `level` | | `paragraph` | 段落块，包含 `paragraph_content` | | `equation_interline` | 行间公式，包含 `math_content`、`math_type` | -| `image` / `table` / `chart` / `seal` | 视觉类块，包含图片路径、说明文字等结构化字段 | +| `image` / `table` / `chart` | 视觉类块，包含图片路径、说明文字等结构化字段；印章使用 `sub_type: "seal"` 的 `image` 表示 | | `code` | 代码块，包含 `code_content`、`code_caption`、`code_footnote`、`code_language` | | `algorithm` | 算法块，包含 `algorithm_content`、`algorithm_caption`、`algorithm_footnote` | | `list` / `index` | 列表与索引，包含 `list_items` | diff --git a/mineru/backend/pipeline/model_json_to_middle_json.py b/mineru/backend/pipeline/model_json_to_middle_json.py index d99a207d..7a1446f5 100644 --- a/mineru/backend/pipeline/model_json_to_middle_json.py +++ b/mineru/backend/pipeline/model_json_to_middle_json.py @@ -50,7 +50,6 @@ def page_model_info_to_page_info(page_model_info, image_dict, page, image_writer ContentType.IMAGE, ContentType.TABLE, ContentType.CHART, - ContentType.SEAL, ContentType.INTERLINE_EQUATION ]: span = cut_image_and_table(span, page_pil_img, page_img_md5, page_index, image_writer, scale=scale) diff --git a/mineru/backend/pipeline/pipeline_magic_model.py b/mineru/backend/pipeline/pipeline_magic_model.py index ba105c0c..fa82e223 100644 --- a/mineru/backend/pipeline/pipeline_magic_model.py +++ b/mineru/backend/pipeline/pipeline_magic_model.py @@ -38,7 +38,7 @@ class MagicModel: "number": BlockType.PAGE_NUMBER, "paragraph_title": BlockType.PARAGRAPH_TITLE, "reference_content": BlockType.REF_TEXT, - "seal": BlockType.SEAL, + "seal": BlockType.IMAGE, "table": BlockType.TABLE, "text": BlockType.TEXT, "vertical_text": BlockType.VERTICAL_TEXT, @@ -117,6 +117,8 @@ class MagicModel: index=block_index, score=block_score, ) + if self.__is_seal_layout_block(layout_det): + block["sub_type"] = "seal" self.page_blocks.append(block) self.page_blocks.sort(key=lambda x: x["index"]) @@ -183,6 +185,20 @@ class MagicModel: def __is_ocr_text_block(layout_det: dict) -> bool: return layout_det.get("label") == "ocr_text" + @staticmethod + def __is_seal_layout_block(layout_det: dict) -> bool: + """判断原始 layout 是否为印章，输出层会将其规范为 image 子类型。""" + return layout_det.get("label") == "seal" + + @staticmethod + def __normalize_seal_text(content): + """将 seal OCR 的列表或字符串结果规范为 VLM 一致的多行字符串。""" + if isinstance(content, list): + return "\n".join(str(item) for item in content if str(item).strip()) + if isinstance(content, str): + return content.strip() + return "" + def __build_return_blocks(self): self.preproc_blocks = [] self.discarded_blocks = [] @@ -237,29 +253,28 @@ class MagicModel: span_type = ContentType.CHART elif block["type"] in [BlockType.INTERLINE_EQUATION]: span_type = ContentType.INTERLINE_EQUATION - elif block["type"] in [BlockType.SEAL]: - span_type = ContentType.SEAL if span_type in [ ContentType.IMAGE, ContentType.TABLE, ContentType.CHART, ContentType.INTERLINE_EQUATION, - ContentType.SEAL ]: span = { "bbox": block["bbox"], "type": span_type, } + if span_type == ContentType.IMAGE and block.get("sub_type") == "seal": + seal_text = self.__normalize_seal_text(block.get("text")) + if seal_text: + span["content"] = seal_text + block.pop("text", None) if span_type == ContentType.TABLE: span["html"] = block.get("html", "") block.pop("html", None) if span_type == ContentType.INTERLINE_EQUATION: span["content"] = block.get("latex", "") block.pop("latex", None) - if span_type == ContentType.SEAL: - span["content"] = block.get("text") - block.pop("text", None) self.all_image_spans.append(span) # 构造line对象 @@ -408,6 +423,8 @@ class MagicModel: mapping = self.VISUAL_TYPE_MAPPING[original_block_type] body_block = self.__make_child_block(block, mapping["body"]) + if original_block_type in [BlockType.IMAGE, BlockType.CHART]: + body_block.pop("sub_type", None) captions = sorted( [ self.__make_child_block(caption, mapping["caption"]) @@ -444,6 +461,8 @@ class MagicModel: "index": block["index"], "score": block.get("score"), } + if original_block_type in [BlockType.IMAGE, BlockType.CHART] and block.get("sub_type"): + two_layer_block["sub_type"] = block["sub_type"] # 对blocks按index排序 two_layer_block["blocks"].sort(key=lambda x: x["index"]) rebuilt_page_blocks.append(two_layer_block) diff --git a/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py b/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py index 66006412..e59e3e17 100644 --- a/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py +++ b/mineru/backend/pipeline/pipeline_middle_json_mkcontent.py @@ -41,13 +41,6 @@ def make_blocks_to_markdown(paras_of_layout, para_text = merge_para_with_text(para_block) else: para_text = f"![]({img_buket_path}/{para_block['lines'][0]['spans'][0]['image_path']})" - elif para_type == BlockType.SEAL: - if len(para_block['lines']) == 0 or len(para_block['lines'][0]['spans']) == 0: - continue - para_text = f"![]({img_buket_path}/{para_block['lines'][0]['spans'][0]['image_path']})" - if para_block['lines'][0]['spans'][0].get('content', []): - content = " ".join(para_block['lines'][0]['spans'][0]['content']) - para_text += f" \n{content}" elif para_type == BlockType.IMAGE: if mode == MakeMode.NLP_MD: continue @@ -82,7 +75,7 @@ def merge_visual_blocks_to_markdown(para_block, img_buket_path=''): for block in get_blocks_in_index_order(para_block.get('blocks', [])): render_block = _inherit_parent_code_render_metadata(block, para_block) - rendered_segments.extend(render_visual_block_segments(render_block, img_buket_path)) + rendered_segments.extend(render_visual_block_segments(render_block, img_buket_path, para_block)) para_text = '' prev_segment_kind = None @@ -128,7 +121,7 @@ def _inherit_parent_code_render_metadata(block, parent_block): return render_block -def render_visual_block_segments(block, img_buket_path=''): +def render_visual_block_segments(block, img_buket_path='', para_block=None): # 将单个视觉子 block 渲染成一个或多个 segment。 # 文本类子块统一输出 markdown_line； # table 的 html 输出为 html_block，供后续决定是否需要空行隔开。 @@ -151,12 +144,23 @@ def render_visual_block_segments(block, img_buket_path=''): return [] if block_type == BlockType.IMAGE_BODY: - return [ - (f"![]({img_buket_path}/{span['image_path']})", 'markdown_line') - for line in block['lines'] - for span in line['spans'] - if span['type'] == ContentType.IMAGE and span.get('image_path', '') - ] + rendered_segments = [] + for line in block['lines']: + for span in line['spans']: + if span['type'] != ContentType.IMAGE: + continue + if span.get('image_path', ''): + rendered_segments.append(( + f"![]({img_buket_path}/{span['image_path']})", + 'markdown_line', + )) + details_block = _build_visual_details_block( + span.get('content', ''), + (para_block or {}).get('sub_type') or 'image content', + ) + if details_block: + rendered_segments.append((details_block, 'details_block')) + return rendered_segments if block_type == BlockType.CHART_BODY: return [ @@ -193,6 +197,8 @@ def get_visual_block_separator(prev_segment_kind, current_segment_kind): # Raw HTML blocks need a blank line after them, otherwise the following # markdown text is still treated as part of the HTML block. return '\n\n' + if prev_segment_kind == 'details_block' or current_segment_kind == 'details_block': + return '\n\n' if current_segment_kind == 'html_block': return '\n' return ' \n' @@ -247,6 +253,36 @@ def _format_embedded_html(html, img_buket_path): return _replace_eq_tags_in_table_html(_prefix_table_img_src(html, img_buket_path)) +def _normalize_visual_content(content): + """将视觉块识别内容统一成字符串，便于 markdown 和结构化输出复用。""" + if isinstance(content, list): + return "\n".join(str(item) for item in content if str(item).strip()) + if isinstance(content, str): + return content.strip() + return '' + + +def _build_visual_details_block(content, summary): + """根据视觉块的识别文本生成 VLM 风格的折叠详情块。""" + normalized_content = _normalize_visual_content(content) + if not normalized_content: + return '' + + return ( + "
\n" + f"{summary}\n\n" + f"{normalized_content}\n" + "
" + ) + + +def _apply_visual_sub_type(para_content, para_block): + """将视觉父块的 sub_type 透传到 content_list 输出顶层。""" + sub_type = para_block.get('sub_type') + if sub_type: + para_content['sub_type'] = sub_type + + def merge_para_with_text(para_block): if _is_fenced_code_block(para_block): code_text = _merge_para_text( @@ -453,27 +489,6 @@ def _build_bbox(para_bbox, page_size): ] -def _get_seal_span(para_block): - for line in para_block.get('lines', []): - for span in line.get('spans', []): - if span.get('type') == ContentType.SEAL: - return span - return None - - -def _get_seal_text(para_block): - seal_span = _get_seal_span(para_block) - if not seal_span: - return '' - - content = seal_span.get('content', '') - if isinstance(content, list): - return ' '.join(str(item) for item in content if str(item).strip()) - if isinstance(content, str): - return content.strip() - return '' - - def _get_ref_text_item_blocks(para_block): return para_block.get('blocks') or [para_block] @@ -510,7 +525,7 @@ def _get_body_data(para_block): if span_type == ContentType.CHART: return span.get('image_path', ''), span.get('content', '') if span_type == ContentType.IMAGE: - return span.get('image_path', ''), '' + return span.get('image_path', ''), _normalize_visual_content(span.get('content', '')) if span_type == ContentType.INTERLINE_EQUATION: return span.get('image_path', ''), span.get('content', '') return '', '' @@ -643,17 +658,14 @@ def make_blocks_to_content_list(para_block, img_buket_path, page_idx, page_size) if para_block['lines'][0]['spans'][0].get('content', ''): para_content['text'] = merge_para_with_text(para_block) para_content['text_format'] = 'latex' - elif para_type == BlockType.SEAL: - seal_span = _get_seal_span(para_block) - if not seal_span: - return None - para_content = { - 'type': ContentType.SEAL, - 'img_path': f"{img_buket_path}/{seal_span.get('image_path', '')}", - 'text': _get_seal_text(para_block), - } elif para_type == BlockType.IMAGE: para_content = {'type': ContentType.IMAGE, 'img_path': '', BlockType.IMAGE_CAPTION: [], BlockType.IMAGE_FOOTNOTE: []} + image_path, image_content = _get_body_data(para_block) + if image_path: + para_content['img_path'] = f"{img_buket_path}/{image_path}" + if image_content: + para_content['content'] = image_content + _apply_visual_sub_type(para_content, para_block) for block in para_block['blocks']: if block['type'] == BlockType.IMAGE_BODY: for line in block['lines']: @@ -799,7 +811,7 @@ def make_blocks_to_content_list_v2(para_block, img_buket_path, page_size): elif para_type == BlockType.IMAGE: image_caption = [] image_footnote = [] - image_path, _ = _get_body_data(para_block) + image_path, image_content = _get_body_data(para_block) for block in para_block.get('blocks', []): if block['type'] == BlockType.IMAGE_CAPTION: image_caption.extend(merge_para_with_text_v2(block)) @@ -813,6 +825,9 @@ def make_blocks_to_content_list_v2(para_block, img_buket_path, page_size): 'image_footnote': image_footnote, }, } + if image_content or para_block.get('sub_type'): + para_content['content']['content'] = image_content + _apply_visual_sub_type(para_content, para_block) elif para_type == BlockType.TABLE: table_caption = [] table_footnote = [] @@ -940,24 +955,6 @@ def make_blocks_to_content_list_v2(para_block, img_buket_path, page_size): 'list_items': list_items, }, } - elif para_type == BlockType.SEAL: - seal_span = _get_seal_span(para_block) - if not seal_span: - return None - seal_text = _get_seal_text(para_block) - para_content = { - 'type': ContentTypeV2.SEAL, - 'content': { - 'image_source': { - 'path': f"{img_buket_path}/{seal_span.get('image_path', '')}", - }, - 'seal_content': ( - [{'type': ContentTypeV2.SPAN_TEXT, 'content': seal_text}] - if seal_text else [] - ), - }, - } - if not para_content: return None diff --git a/mineru/utils/draw_bbox.py b/mineru/utils/draw_bbox.py index 55cc337d..2e484db6 100644 --- a/mineru/utils/draw_bbox.py +++ b/mineru/utils/draw_bbox.py @@ -189,8 +189,6 @@ def draw_layout_bbox(pdf_info, pdf_bytes, out_path, filename): elif nested_block["type"] == BlockType.CHART_FOOTNOTE: bbox = nested_block["bbox"] imgs_footnote.append(bbox) - elif block["type"] == BlockType.SEAL: - imgs_body.append(bbox) elif block["type"] == BlockType.TITLE: titles.append(bbox) elif block["type"] in [BlockType.TEXT, BlockType.REF_TEXT, BlockType.ABSTRACT]: @@ -234,7 +232,6 @@ def draw_layout_bbox(pdf_info, pdf_bytes, out_path, filename): BlockType.INTERLINE_EQUATION, BlockType.LIST, BlockType.INDEX, - BlockType.SEAL, ]: bbox = block["bbox"] page_block_list.append(bbox) @@ -315,7 +312,7 @@ def draw_span_bbox(pdf_info, pdf_bytes, out_path, filename): page_inline_equation_list.append(span['bbox']) elif span['type'] == ContentType.INTERLINE_EQUATION: page_interline_equation_list.append(span['bbox']) - elif span['type'] in [ContentType.IMAGE, ContentType.CHART, ContentType.SEAL]: + elif span['type'] in [ContentType.IMAGE, ContentType.CHART]: page_image_list.append(span['bbox']) elif span['type'] == ContentType.TABLE: page_table_list.append(span['bbox']) @@ -346,7 +343,6 @@ def draw_span_bbox(pdf_info, pdf_bytes, out_path, filename): BlockType.INDEX, BlockType.REF_TEXT, BlockType.ABSTRACT, - BlockType.SEAL, ]: for line in block['lines']: for span in line['spans']: diff --git a/mineru/utils/enum_class.py b/mineru/utils/enum_class.py index 50562bf6..6a588415 100644 --- a/mineru/utils/enum_class.py +++ b/mineru/utils/enum_class.py @@ -44,7 +44,6 @@ class BlockType: DOC_TITLE = "doc_title" PARAGRAPH_TITLE = "paragraph_title" VERTICAL_TEXT = "vertical_text" - SEAL = "seal" HEADER_IMAGE = "header_image" FOOTER_IMAGE = "footer_image" FORMULA_NUMBER = "formula_number" @@ -58,7 +57,6 @@ class ContentType: INLINE_EQUATION = 'inline_equation' EQUATION = 'equation' HYPERLINK = 'hyperlink' - SEAL = 'seal' class ContentTypeV2: @@ -66,7 +64,6 @@ class ContentTypeV2: ALGORITHM = "algorithm" EQUATION_INTERLINE = 'equation_interline' IMAGE = 'image' - SEAL = 'seal' TABLE = 'table' CHART = 'chart' TABLE_SIMPLE = 'simple_table'