From 7c03014c2a121a928defe4158afb9bc389fae86e Mon Sep 17 00:00:00 2001 From: myhloli Date: Fri, 1 Nov 2024 23:28:53 +0800 Subject: [PATCH] fix(ocr_mkcontent): improve content handling for different languages and equation types- Adjust content formatting for Chinese, Japanese, Korean, and Western languages - Implement proper spacing rules around inline equations- Remove unnecessary empty lines in paragraph text --- magic_pdf/dict2md/ocr_mkcontent.py | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/magic_pdf/dict2md/ocr_mkcontent.py b/magic_pdf/dict2md/ocr_mkcontent.py index 49837441..7c40668c 100644 --- a/magic_pdf/dict2md/ocr_mkcontent.py +++ b/magic_pdf/dict2md/ocr_mkcontent.py @@ -141,22 +141,31 @@ def merge_para_with_text(para_block): if span_type == ContentType.Text: content = ocr_escape_special_markdown_char(span['content']) elif span_type == ContentType.InlineEquation: - content = f" ${span['content']}$ " + content = f"${span['content']}$" elif span_type == ContentType.InterlineEquation: content = f"\n$$\n{span['content']}\n$$\n" - if content != '': + if content.strip() != '': langs = ['zh', 'ja', 'ko'] if line_lang in langs: # 遇到一些一个字一个span的文档，这种单字语言判断不准，需要用整行文本判断 - para_text += content # 中文/日语/韩文语境下，content间不需要空格分隔 - elif line_lang == 'en': - # 如果是前一行带有-连字符，那么末尾不应该加空格 - if __is_hyphen_at_line_end(content): - para_text += content[:-1] - else: - para_text += content + ' ' + if span_type in [ContentType.Text, ContentType.InterlineEquation]: + para_text += content # 中文/日语/韩文语境下，content间不需要空格分隔 + elif span_type == ContentType.InlineEquation: + para_text += f" {content} " else: - para_text += content + ' ' # 西方文本语境下 content间需要空格分隔 + if span_type == ContentType.Text: + # 如果是前一行带有-连字符，那么末尾不应该加空格 + if __is_hyphen_at_line_end(content): + para_text += content[:-1] + else: # 西方文本语境下 content间需要空格分隔 + para_text += f"{content.strip()} " + elif span_type == ContentType.InterlineEquation: + para_text += content + elif span_type == ContentType.InlineEquation: + para_text += f"{content} " + else: + continue + return para_text