mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-25 00:12:08 +08:00
Fix stream_to_open_named_file for non-utf8 text
We only recode the file if a target encoding is given, and make the assumption that the file is not binary or compressed if a target encoding is set. This is required to upload the imzml test data by url in https://github.com/galaxyproject/galaxy/issues/7412#issuecomment-469379295
This commit is contained in:
@@ -23,10 +23,7 @@ from six.moves import filter
|
||||
from six.moves.urllib.request import urlopen
|
||||
|
||||
from galaxy import util
|
||||
from galaxy.util import (
|
||||
compression_utils,
|
||||
smart_str
|
||||
)
|
||||
from galaxy.util import compression_utils
|
||||
from galaxy.util.checkers import (
|
||||
check_binary,
|
||||
check_bz2,
|
||||
@@ -63,33 +60,16 @@ def stream_to_open_named_file(stream, fd, filename, source_encoding=None, source
|
||||
"""Writes a stream to the provided file descriptor, returns the file name. Closes file descriptor"""
|
||||
# signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
|
||||
CHUNK_SIZE = 1048576
|
||||
data_checked = False
|
||||
is_compressed = False
|
||||
is_binary = False
|
||||
try:
|
||||
codecs.lookup(target_encoding)
|
||||
except Exception:
|
||||
target_encoding = util.DEFAULT_ENCODING # utf-8
|
||||
if not source_encoding:
|
||||
source_encoding = util.DEFAULT_ENCODING # sys.getdefaultencoding() would mimic old behavior (defaults to ascii)
|
||||
while True:
|
||||
chunk = stream.read(CHUNK_SIZE)
|
||||
if not chunk:
|
||||
break
|
||||
if not data_checked:
|
||||
# See if we're uploading a compressed file
|
||||
try:
|
||||
# Convert chunk to a bytestring if it is not already.
|
||||
# Check if the first 2 bytes of the chunk are equal to the
|
||||
# gzip magic number.
|
||||
if smart_str(chunk)[:2] == util.gzip_magic:
|
||||
is_compressed = True
|
||||
except Exception:
|
||||
pass
|
||||
if not is_compressed:
|
||||
is_binary = util.is_binary(chunk)
|
||||
data_checked = True
|
||||
if not is_compressed and not is_binary:
|
||||
if source_encoding is not None:
|
||||
# If a source encoding is given we use it to convert to the target encoding
|
||||
if not isinstance(chunk, text_type):
|
||||
chunk = chunk.decode(source_encoding, source_error)
|
||||
os.write(fd, chunk.encode(target_encoding, target_error))
|
||||
|
||||
Reference in New Issue
Block a user