Fix stream_to_open_named_file for non-utf8 text

We only recode the file if a target encoding is given,
and make the assumption that the file is not binary
or compressed if a target encoding is set.

This is required to upload the imzml test data by url
in https://github.com/galaxyproject/galaxy/issues/7412#issuecomment-469379295
This commit is contained in:
mvdbeek
2019-05-28 16:42:23 +02:00
parent 40196a5950
commit 5bcfa3c5b7
+3 -23
View File
@@ -23,10 +23,7 @@ from six.moves import filter
from six.moves.urllib.request import urlopen
from galaxy import util
from galaxy.util import (
compression_utils,
smart_str
)
from galaxy.util import compression_utils
from galaxy.util.checkers import (
check_binary,
check_bz2,
@@ -63,33 +60,16 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source
"""Writes a stream to the provided file descriptor, returns the file name. Closes file descriptor"""
# signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
CHUNK_SIZE = 1048576
data_checked = False
is_compressed = False
is_binary = False
try:
codecs.lookup(target_encoding)
except Exception:
target_encoding = util.DEFAULT_ENCODING # utf-8
if not source_encoding:
source_encoding = util.DEFAULT_ENCODING # sys.getdefaultencoding() would mimic old behavior (defaults to ascii)
while True:
chunk = stream.read(CHUNK_SIZE)
if not chunk:
break
if not data_checked:
# See if we're uploading a compressed file
try:
# Convert chunk to a bytestring if it is not already.
# Check if the first 2 bytes of the chunk are equal to the
# gzip magic number.
if smart_str(chunk)[:2] == util.gzip_magic:
is_compressed = True
except Exception:
pass
if not is_compressed:
is_binary = util.is_binary(chunk)
data_checked = True
if not is_compressed and not is_binary:
if source_encoding is not None:
# If a source encoding is given we use it to convert to the target encoding
if not isinstance(chunk, text_type):
chunk = chunk.decode(source_encoding, source_error)
os.write(fd, chunk.encode(target_encoding, target_error))