From 113448903aac099e42522dc4e2e2d80d0f0dc45c Mon Sep 17 00:00:00 2001 From: icecraft Date: Mon, 9 Dec 2024 13:58:27 +0800 Subject: [PATCH] fix: unicode decode error --- magic_pdf/data/data_reader_writer/base.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/magic_pdf/data/data_reader_writer/base.py b/magic_pdf/data/data_reader_writer/base.py index 7c9a8e8e..d294b329 100644 --- a/magic_pdf/data/data_reader_writer/base.py +++ b/magic_pdf/data/data_reader_writer/base.py @@ -48,4 +48,16 @@ class DataWriter(ABC): path (str): the target file where to write data (str): the data want to write """ - self.write(path, data.encode()) + + def safe_encode(data: str, method: str): + try: + bit_data = data.encode(encoding=method, errors='replace') + return bit_data, True + except: # noqa + return None, False + + for method in ['utf-8', 'ascii']: + bit_data, flag = safe_encode(data, method) + if flag: + self.write(path, bit_data) + break