diff --git a/magic_pdf/dict2md/mkcontent.py b/magic_pdf/dict2md/mkcontent.py.bak similarity index 100% rename from magic_pdf/dict2md/mkcontent.py rename to magic_pdf/dict2md/mkcontent.py.bak diff --git a/magic_pdf/layout/__init__.py b/magic_pdf/layout.bak/__init__.py similarity index 100% rename from magic_pdf/layout/__init__.py rename to magic_pdf/layout.bak/__init__.py diff --git a/magic_pdf/layout/bbox_sort.py b/magic_pdf/layout.bak/bbox_sort.py similarity index 100% rename from magic_pdf/layout/bbox_sort.py rename to magic_pdf/layout.bak/bbox_sort.py diff --git a/magic_pdf/layout/layout_det_utils.py b/magic_pdf/layout.bak/layout_det_utils.py similarity index 100% rename from magic_pdf/layout/layout_det_utils.py rename to magic_pdf/layout.bak/layout_det_utils.py diff --git a/magic_pdf/layout/layout_sort.py b/magic_pdf/layout.bak/layout_sort.py similarity index 100% rename from magic_pdf/layout/layout_sort.py rename to magic_pdf/layout.bak/layout_sort.py diff --git a/magic_pdf/layout/layout_spiler_recog.py b/magic_pdf/layout.bak/layout_spiler_recog.py similarity index 100% rename from magic_pdf/layout/layout_spiler_recog.py rename to magic_pdf/layout.bak/layout_spiler_recog.py diff --git a/magic_pdf/layout/mcol_sort.py b/magic_pdf/layout.bak/mcol_sort.py similarity index 100% rename from magic_pdf/layout/mcol_sort.py rename to magic_pdf/layout.bak/mcol_sort.py diff --git a/magic_pdf/libs/calc_span_stats.py b/magic_pdf/libs/calc_span_stats.py.bak similarity index 100% rename from magic_pdf/libs/calc_span_stats.py rename to magic_pdf/libs/calc_span_stats.py.bak diff --git a/magic_pdf/libs/detect_language_from_model.py b/magic_pdf/libs/detect_language_from_model.py.bak similarity index 100% rename from magic_pdf/libs/detect_language_from_model.py rename to magic_pdf/libs/detect_language_from_model.py.bak diff --git a/magic_pdf/libs/markdown_utils.py b/magic_pdf/libs/markdown_utils.py index 5708b477..736d37a7 100644 --- a/magic_pdf/libs/markdown_utils.py +++ b/magic_pdf/libs/markdown_utils.py @@ -1,6 +1,4 @@ -import re - - +@DeprecationWarning def escape_special_markdown_char(pymu_blocks): """ 转义正文里对markdown语法有特殊意义的字符 diff --git a/magic_pdf/libs/nlp_utils.py b/magic_pdf/libs/nlp_utils.py.bak similarity index 100% rename from magic_pdf/libs/nlp_utils.py rename to magic_pdf/libs/nlp_utils.py.bak diff --git a/magic_pdf/libs/textbase.py b/magic_pdf/libs/textbase.py.bak similarity index 100% rename from magic_pdf/libs/textbase.py rename to magic_pdf/libs/textbase.py.bak diff --git a/magic_pdf/libs/vis_utils.py b/magic_pdf/libs/vis_utils.py.bak similarity index 100% rename from magic_pdf/libs/vis_utils.py rename to magic_pdf/libs/vis_utils.py.bak diff --git a/magic_pdf/para/block_continuation_processor.py b/magic_pdf/para/block_continuation_processor.py.bak similarity index 100% rename from magic_pdf/para/block_continuation_processor.py rename to magic_pdf/para/block_continuation_processor.py.bak diff --git a/magic_pdf/para/block_termination_processor.py b/magic_pdf/para/block_termination_processor.py.bak similarity index 100% rename from magic_pdf/para/block_termination_processor.py rename to magic_pdf/para/block_termination_processor.py.bak diff --git a/magic_pdf/para/commons.py b/magic_pdf/para/commons.py.bak similarity index 100% rename from magic_pdf/para/commons.py rename to magic_pdf/para/commons.py.bak diff --git a/magic_pdf/para/denoise.py b/magic_pdf/para/denoise.py.bak similarity index 100% rename from magic_pdf/para/denoise.py rename to magic_pdf/para/denoise.py.bak diff --git a/magic_pdf/para/draw.py b/magic_pdf/para/draw.py.bak similarity index 100% rename from magic_pdf/para/draw.py rename to magic_pdf/para/draw.py.bak diff --git a/magic_pdf/para/exceptions.py b/magic_pdf/para/exceptions.py.bak similarity index 100% rename from magic_pdf/para/exceptions.py rename to magic_pdf/para/exceptions.py.bak diff --git a/magic_pdf/para/layout_match_processor.py b/magic_pdf/para/layout_match_processor.py.bak similarity index 100% rename from magic_pdf/para/layout_match_processor.py rename to magic_pdf/para/layout_match_processor.py.bak diff --git a/magic_pdf/para/para_split.py b/magic_pdf/para/para_split.py.bak similarity index 100% rename from magic_pdf/para/para_split.py rename to magic_pdf/para/para_split.py.bak diff --git a/magic_pdf/para/para_split_v2.py b/magic_pdf/para/para_split_v2.py.bak similarity index 100% rename from magic_pdf/para/para_split_v2.py rename to magic_pdf/para/para_split_v2.py.bak diff --git a/magic_pdf/para/raw_processor.py b/magic_pdf/para/raw_processor.py.bak similarity index 100% rename from magic_pdf/para/raw_processor.py rename to magic_pdf/para/raw_processor.py.bak diff --git a/magic_pdf/para/stats.py b/magic_pdf/para/stats.py.bak similarity index 100% rename from magic_pdf/para/stats.py rename to magic_pdf/para/stats.py.bak diff --git a/magic_pdf/para/title_processor.py b/magic_pdf/para/title_processor.py.bak similarity index 100% rename from magic_pdf/para/title_processor.py rename to magic_pdf/para/title_processor.py.bak diff --git a/magic_pdf/pdf_parse_union_core_v2.py b/magic_pdf/pdf_parse_union_core_v2.py index df2c8732..e92752bc 100644 --- a/magic_pdf/pdf_parse_union_core_v2.py +++ b/magic_pdf/pdf_parse_union_core_v2.py @@ -34,13 +34,11 @@ except ImportError: from magic_pdf.model.sub_modules.model_init import AtomModelSingleton from magic_pdf.para.para_split_v3 import para_split -from magic_pdf.pre_proc.citationmarker_remove import remove_citation_marker + from magic_pdf.pre_proc.construct_page_dict import \ ocr_construct_page_component_v2 from magic_pdf.pre_proc.cut_image import ocr_cut_image_and_table -from magic_pdf.pre_proc.equations_replace import ( - combine_chars_to_pymudict, remove_chars_in_text_blocks, - replace_equations_in_textblock) + from magic_pdf.pre_proc.ocr_detect_all_bboxes import \ ocr_prepare_bboxes_for_layout_split_v2 from magic_pdf.pre_proc.ocr_dict_merge import (fill_spans_in_blocks, @@ -49,26 +47,6 @@ from magic_pdf.pre_proc.ocr_dict_merge import (fill_spans_in_blocks, from magic_pdf.pre_proc.ocr_span_list_modify import ( get_qa_need_list_v2, remove_overlaps_low_confidence_spans, remove_overlaps_min_spans) -from magic_pdf.pre_proc.resolve_bbox_conflict import \ - check_useful_block_horizontal_overlap - - -def remove_horizontal_overlap_block_which_smaller(all_bboxes): - useful_blocks = [] - for bbox in all_bboxes: - useful_blocks.append({'bbox': bbox[:4]}) - is_useful_block_horz_overlap, smaller_bbox, bigger_bbox = ( - check_useful_block_horizontal_overlap(useful_blocks) - ) - if is_useful_block_horz_overlap: - logger.warning( - f'skip this page, reason: {DropReason.USEFUL_BLOCK_HOR_OVERLAP}, smaller bbox is {smaller_bbox}, bigger bbox is {bigger_bbox}' - ) # noqa: E501 - for bbox in all_bboxes.copy(): - if smaller_bbox == bbox[:4]: - all_bboxes.remove(bbox) - - return is_useful_block_horz_overlap, all_bboxes def __replace_STX_ETX(text_str: str): @@ -264,39 +242,6 @@ def txt_spans_extract_v2(pdf_page, spans, all_bboxes, all_discarded_blocks, lang return spans -def txt_spans_extract_v1(pdf_page, inline_equations, interline_equations): - text_raw_blocks = pdf_page.get_text('dict', flags=fitz.TEXTFLAGS_TEXT)['blocks'] - char_level_text_blocks = pdf_page.get_text('rawdict', flags=fitz.TEXTFLAGS_TEXT)[ - 'blocks' - ] - text_blocks = combine_chars_to_pymudict(text_raw_blocks, char_level_text_blocks) - text_blocks = replace_equations_in_textblock( - text_blocks, inline_equations, interline_equations - ) - text_blocks = remove_citation_marker(text_blocks) - text_blocks = remove_chars_in_text_blocks(text_blocks) - spans = [] - for v in text_blocks: - for line in v['lines']: - for span in line['spans']: - bbox = span['bbox'] - if float_equal(bbox[0], bbox[2]) or float_equal(bbox[1], bbox[3]): - continue - if span.get('type') not in ( - ContentType.InlineEquation, - ContentType.InterlineEquation, - ): - spans.append( - { - 'bbox': list(span['bbox']), - 'content': __replace_STX_ETX(span['text']), - 'type': ContentType.Text, - 'score': 1.0, - } - ) - return spans - - def replace_text_span(pymu_spans, ocr_spans): return list(filter(lambda x: x['type'] != ContentType.Text, ocr_spans)) + pymu_spans @@ -722,10 +667,6 @@ def parse_page_core( """根据parse_mode，构造spans，主要是文本类的字符填充""" if parse_mode == SupportedPdfParseMethod.TXT: - """之前的公式替换方案""" - # pymu_spans = txt_spans_extract_v1(page_doc, inline_equations, interline_equations) - # spans = replace_text_span(pymu_spans, spans) - """使用新版本的混合ocr方案""" spans = txt_spans_extract_v2(page_doc, spans, all_bboxes, all_discarded_blocks, lang) diff --git a/magic_pdf/post_proc/__init__.py b/magic_pdf/post_proc.bak/__init__.py similarity index 100% rename from magic_pdf/post_proc/__init__.py rename to magic_pdf/post_proc.bak/__init__.py diff --git a/magic_pdf/post_proc/detect_para.py b/magic_pdf/post_proc.bak/detect_para.py.bak similarity index 100% rename from magic_pdf/post_proc/detect_para.py rename to magic_pdf/post_proc.bak/detect_para.py.bak diff --git a/magic_pdf/post_proc/pdf_post_filter.py b/magic_pdf/post_proc.bak/pdf_post_filter.py.bak similarity index 100% rename from magic_pdf/post_proc/pdf_post_filter.py rename to magic_pdf/post_proc.bak/pdf_post_filter.py.bak diff --git a/magic_pdf/post_proc/remove_footnote.py b/magic_pdf/post_proc.bak/remove_footnote.py.bak similarity index 100% rename from magic_pdf/post_proc/remove_footnote.py rename to magic_pdf/post_proc.bak/remove_footnote.py.bak diff --git a/magic_pdf/pre_proc/citationmarker_remove.py b/magic_pdf/pre_proc/citationmarker_remove.py.bak similarity index 100% rename from magic_pdf/pre_proc/citationmarker_remove.py rename to magic_pdf/pre_proc/citationmarker_remove.py.bak diff --git a/magic_pdf/pre_proc/detect_equation.py b/magic_pdf/pre_proc/detect_equation.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_equation.py rename to magic_pdf/pre_proc/detect_equation.py.bak diff --git a/magic_pdf/pre_proc/detect_footer_by_model.py b/magic_pdf/pre_proc/detect_footer_by_model.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_footer_by_model.py rename to magic_pdf/pre_proc/detect_footer_by_model.py.bak diff --git a/magic_pdf/pre_proc/detect_footer_header_by_statistics.py b/magic_pdf/pre_proc/detect_footer_header_by_statistics.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_footer_header_by_statistics.py rename to magic_pdf/pre_proc/detect_footer_header_by_statistics.py.bak diff --git a/magic_pdf/pre_proc/detect_footnote.py b/magic_pdf/pre_proc/detect_footnote.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_footnote.py rename to magic_pdf/pre_proc/detect_footnote.py.bak diff --git a/magic_pdf/pre_proc/detect_header.py b/magic_pdf/pre_proc/detect_header.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_header.py rename to magic_pdf/pre_proc/detect_header.py.bak diff --git a/magic_pdf/pre_proc/detect_images.py b/magic_pdf/pre_proc/detect_images.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_images.py rename to magic_pdf/pre_proc/detect_images.py.bak diff --git a/magic_pdf/pre_proc/detect_page_number.py b/magic_pdf/pre_proc/detect_page_number.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_page_number.py rename to magic_pdf/pre_proc/detect_page_number.py.bak diff --git a/magic_pdf/pre_proc/detect_tables.py b/magic_pdf/pre_proc/detect_tables.py.bak similarity index 100% rename from magic_pdf/pre_proc/detect_tables.py rename to magic_pdf/pre_proc/detect_tables.py.bak diff --git a/magic_pdf/pre_proc/equations_replace.py b/magic_pdf/pre_proc/equations_replace.py.bak similarity index 100% rename from magic_pdf/pre_proc/equations_replace.py rename to magic_pdf/pre_proc/equations_replace.py.bak diff --git a/magic_pdf/pre_proc/fix_image.py b/magic_pdf/pre_proc/fix_image.py.bak similarity index 100% rename from magic_pdf/pre_proc/fix_image.py rename to magic_pdf/pre_proc/fix_image.py.bak diff --git a/magic_pdf/pre_proc/fix_table.py b/magic_pdf/pre_proc/fix_table.py.bak similarity index 100% rename from magic_pdf/pre_proc/fix_table.py rename to magic_pdf/pre_proc/fix_table.py.bak diff --git a/magic_pdf/pre_proc/main_text_font.py b/magic_pdf/pre_proc/main_text_font.py.bak similarity index 100% rename from magic_pdf/pre_proc/main_text_font.py rename to magic_pdf/pre_proc/main_text_font.py.bak diff --git a/magic_pdf/pre_proc/ocr_detect_layout.py b/magic_pdf/pre_proc/ocr_detect_layout.py.bak similarity index 100% rename from magic_pdf/pre_proc/ocr_detect_layout.py rename to magic_pdf/pre_proc/ocr_detect_layout.py.bak diff --git a/magic_pdf/pre_proc/pdf_pre_filter.py b/magic_pdf/pre_proc/pdf_pre_filter.py.bak similarity index 100% rename from magic_pdf/pre_proc/pdf_pre_filter.py rename to magic_pdf/pre_proc/pdf_pre_filter.py.bak diff --git a/magic_pdf/pre_proc/post_layout_split.py b/magic_pdf/pre_proc/post_layout_split.py.bak similarity index 100% rename from magic_pdf/pre_proc/post_layout_split.py rename to magic_pdf/pre_proc/post_layout_split.py.bak diff --git a/magic_pdf/pre_proc/remove_colored_strip_bbox.py b/magic_pdf/pre_proc/remove_colored_strip_bbox.py.bak similarity index 100% rename from magic_pdf/pre_proc/remove_colored_strip_bbox.py rename to magic_pdf/pre_proc/remove_colored_strip_bbox.py.bak diff --git a/magic_pdf/pre_proc/remove_footer_header.py b/magic_pdf/pre_proc/remove_footer_header.py.bak similarity index 100% rename from magic_pdf/pre_proc/remove_footer_header.py rename to magic_pdf/pre_proc/remove_footer_header.py.bak diff --git a/magic_pdf/pre_proc/remove_rotate_bbox.py b/magic_pdf/pre_proc/remove_rotate_bbox.py.bak similarity index 100% rename from magic_pdf/pre_proc/remove_rotate_bbox.py rename to magic_pdf/pre_proc/remove_rotate_bbox.py.bak diff --git a/magic_pdf/pre_proc/resolve_bbox_conflict.py b/magic_pdf/pre_proc/resolve_bbox_conflict.py.bak similarity index 100% rename from magic_pdf/pre_proc/resolve_bbox_conflict.py rename to magic_pdf/pre_proc/resolve_bbox_conflict.py.bak diff --git a/magic_pdf/pre_proc/solve_line_alien.py b/magic_pdf/pre_proc/solve_line_alien.py.bak similarity index 100% rename from magic_pdf/pre_proc/solve_line_alien.py rename to magic_pdf/pre_proc/solve_line_alien.py.bak diff --git a/magic_pdf/pre_proc/statistics.py b/magic_pdf/pre_proc/statistics.py.bak similarity index 100% rename from magic_pdf/pre_proc/statistics.py rename to magic_pdf/pre_proc/statistics.py.bak