From d637dab3542fcaeb73a6a3f913ef259fb3ad9710 Mon Sep 17 00:00:00 2001 From: icecraft Date: Fri, 27 Dec 2024 18:14:56 +0800 Subject: [PATCH 1/3] fix: s3 path join method --- .../data/data_reader_writer/multi_bucket_s3.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/magic_pdf/data/data_reader_writer/multi_bucket_s3.py b/magic_pdf/data/data_reader_writer/multi_bucket_s3.py index f54cbd24..525209f0 100644 --- a/magic_pdf/data/data_reader_writer/multi_bucket_s3.py +++ b/magic_pdf/data/data_reader_writer/multi_bucket_s3.py @@ -1,4 +1,4 @@ -import os + from magic_pdf.config.exceptions import InvalidConfig, InvalidParams from magic_pdf.data.data_reader_writer.base import DataReader, DataWriter from magic_pdf.data.io.s3 import S3Reader, S3Writer @@ -22,10 +22,10 @@ class MultiS3Mixin: """ if len(default_prefix) == 0: raise InvalidConfig('default_prefix must be provided') - - arr = default_prefix.strip("/").split("/") + + arr = default_prefix.strip('/').split('/') self.default_bucket = arr[0] - self.default_prefix = "/".join(arr[1:]) + self.default_prefix = '/'.join(arr[1:]) found_default_bucket_config = False for conf in s3_configs: @@ -103,7 +103,8 @@ class MultiBucketS3DataReader(DataReader, MultiS3Mixin): s3_reader = self.__get_s3_client(bucket_name) else: s3_reader = self.__get_s3_client(self.default_bucket) - path = os.path.join(self.default_prefix, path) + if self.default_prefix: + path = self.default_prefix + '/' + path return s3_reader.read_at(path, offset, limit) @@ -139,5 +140,6 @@ class MultiBucketS3DataWriter(DataWriter, MultiS3Mixin): s3_writer = self.__get_s3_client(bucket_name) else: s3_writer = self.__get_s3_client(self.default_bucket) - path = os.path.join(self.default_prefix, path) + if self.default_prefix: + path = self.default_prefix + '/' + path return s3_writer.write(path, data) From c93950dc5da00050aae1e580d1dd7e22723d38f9 Mon Sep 17 00:00:00 2001 From: myhloli Date: Thu, 2 Jan 2025 14:30:11 +0800 Subject: [PATCH 2/3] refactor(pdf_parse): improve character spacing handling in PDF text extraction - Update the logic for inserting spaces between characters- Consider the next character's position instead of the previous one - Adjust the spacing threshold to 25% of the average character width - Ignore spaces at the end of lines to prevent double spaces --- magic_pdf/pdf_parse_union_core_v2.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/magic_pdf/pdf_parse_union_core_v2.py b/magic_pdf/pdf_parse_union_core_v2.py index 32e44b38..5105249a 100644 --- a/magic_pdf/pdf_parse_union_core_v2.py +++ b/magic_pdf/pdf_parse_union_core_v2.py @@ -91,10 +91,13 @@ def chars_to_content(span): content = '' for char in span['chars']: - # 如果下一个char的x0和上一个char的x1距离超过一个字符宽度，则需要在中间插入一个空格 - if char['bbox'][0] - span['chars'][span['chars'].index(char) - 1]['bbox'][2] > char_avg_width: - content += ' ' - content += char['c'] + # 如果下一个char的x0和上一个char的x1距离超过0.25个字符宽度，则需要在中间插入一个空格 + char1 = char + char2 = span['chars'][span['chars'].index(char) + 1] if span['chars'].index(char) + 1 < len(span['chars']) else None + if char2 and char2['bbox'][0] - char1['bbox'][2] > char_avg_width * 0.25 and char['c'] != ' ' and char2['c'] != ' ': + content += f"{char['c']} " + else: + content += char['c'] content = __replace_ligatures(content) span['content'] = __replace_0xfffd(content) From 05109c36416a36361fd76f6e9d364f39c2c4fad7 Mon Sep 17 00:00:00 2001 From: yzz Date: Fri, 3 Jan 2025 15:42:39 +0800 Subject: [PATCH 3/3] fix(web_api): Modify the import path of InferenceResult --- projects/web_api/app.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/projects/web_api/app.py b/projects/web_api/app.py index 62cde60b..6e317012 100644 --- a/projects/web_api/app.py +++ b/projects/web_api/app.py @@ -13,7 +13,7 @@ from magic_pdf.config.enums import SupportedPdfParseMethod from magic_pdf.data.data_reader_writer import FileBasedDataWriter from magic_pdf.data.dataset import PymuDocDataset from magic_pdf.model.doc_analyze_by_custom_model import doc_analyze -from magic_pdf.model.operators import InferenceResult +from magic_pdf.operators.models import InferenceResult model_config.__use_inside_model__ = True