From e6fe571c02cfb49bf117d23c16bf3488f145546b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 19 Jan 2020 16:40:53 +0100 Subject: [PATCH 1/2] Fallback to binary download if header lies about content encoding Fixes https://github.com/galaxyproject/galaxy/issues/9238 --- lib/galaxy/datatypes/sniff.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 353462bb510..2dafdfbd8b5 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -64,15 +64,20 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source codecs.lookup(target_encoding) except Exception: target_encoding = util.DEFAULT_ENCODING # utf-8 + try_source_encoding = True while True: chunk = stream.read(CHUNK_SIZE) if not chunk: break - if source_encoding is not None: + if source_encoding is not None and try_source_encoding: # If a source encoding is given we use it to convert to the target encoding - if not isinstance(chunk, text_type): - chunk = chunk.decode(source_encoding, source_error) - os.write(fd, chunk.encode(target_encoding, target_error)) + try: + if not isinstance(chunk, text_type): + chunk = chunk.decode(source_encoding, source_error) + os.write(fd, chunk.encode(target_encoding, target_error)) + except UnicodeDecodeError: + try_source_encoding = False + os.write(fd, chunk) else: # Compressed files must be encoded after they are uncompressed in the upload utility, # while binary files should not be encoded at all. From 4d686edb7f3d50e27c85fc070efe7fc07e13174d Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Sun, 19 Jan 2020 19:59:20 +0100 Subject: [PATCH 2/2] Simplify use_source_encoding logic Thanks Nicola, very nice! Co-Authored-By: Nicola Soranzo --- lib/galaxy/datatypes/sniff.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 2dafdfbd8b5..a3ccb86266f 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -64,19 +64,19 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source codecs.lookup(target_encoding) except Exception: target_encoding = util.DEFAULT_ENCODING # utf-8 - try_source_encoding = True + use_source_encoding = source_encoding is not None while True: chunk = stream.read(CHUNK_SIZE) if not chunk: break - if source_encoding is not None and try_source_encoding: + if use_source_encoding: # If a source encoding is given we use it to convert to the target encoding try: if not isinstance(chunk, text_type): chunk = chunk.decode(source_encoding, source_error) os.write(fd, chunk.encode(target_encoding, target_error)) except UnicodeDecodeError: - try_source_encoding = False + use_source_encoding = False os.write(fd, chunk) else: # Compressed files must be encoded after they are uncompressed in the upload utility,