From d84ab3226e98bae64c1bef829e365ca378856e1f Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 28 Aug 2012 19:23:39 -0400 Subject: [PATCH] Handle non-ascii unicode in data source tools. Add util.is_binary() method that returns true when provided string contains a null byte. --- lib/galaxy/datatypes/sniff.py | 31 +++++++++++----------- lib/galaxy/util/__init__.py | 44 ++++++++++++++++++++++++++++++-- tools/data_source/data_source.py | 3 ++- 3 files changed, 59 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index d7a1f32ca05..4388b03a5b4 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -6,6 +6,7 @@ import registry from galaxy import util from galaxy.datatypes.checkers import * from galaxy.datatypes.binary import unsniffable_binary_formats +from encodings import search_function as encodings_search_function log = logging.getLogger(__name__) @@ -15,7 +16,7 @@ def get_test_fname(fname): full_path = os.path.join(path, 'test', fname) return full_path -def stream_to_open_named_file( stream, fd, filename ): +def stream_to_open_named_file( stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict' ): """Writes a stream to the provided file descriptor, returns the file's name and bool( is_multi_byte ). Closes file descriptor""" #signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better CHUNK_SIZE = 1048576 @@ -23,6 +24,10 @@ def stream_to_open_named_file( stream, fd, filename ): is_compressed = False is_binary = False is_multi_byte = False + if not target_encoding or not encodings_search_function( target_encoding ): + target_encoding = util.DEFAULT_ENCODING #utf-8 + if not source_encoding: + source_encoding = util.DEFAULT_ENCODING #sys.getdefaultencoding() would mimic old behavior (defaults to ascii) while 1: chunk = stream.read( CHUNK_SIZE ) if not chunk: @@ -42,13 +47,12 @@ def stream_to_open_named_file( stream, fd, filename ): chars = chunk[:100] is_multi_byte = util.is_multi_byte( chars ) if not is_multi_byte: - for char in chars: - if ord( char ) > 128: - is_binary = True - break + is_binary = util.is_binary( chunk ) data_checked = True if not is_compressed and not is_binary: - os.write( fd, chunk.encode( "utf-8" ) ) + if not isinstance( chunk, unicode ): + chunk = chunk.decode( source_encoding, source_error ) + os.write( fd, chunk.encode( target_encoding, target_error ) ) else: # Compressed files must be encoded after they are uncompressed in the upload utility, # while binary files should not be encoded at all. @@ -56,10 +60,10 @@ def stream_to_open_named_file( stream, fd, filename ): os.close( fd ) return filename, is_multi_byte -def stream_to_file( stream, suffix='', prefix='', dir=None, text=False ): +def stream_to_file( stream, suffix='', prefix='', dir=None, text=False, **kwd ): """Writes a stream to a temporary file, returns the temporary file's name""" fd, temp_name = tempfile.mkstemp( suffix=suffix, prefix=prefix, dir=dir, text=text ) - return stream_to_open_named_file( stream, fd, temp_name ) + return stream_to_open_named_file( stream, fd, temp_name, **kwd ) def check_newlines( fname, bytes_to_read=52428800 ): """ @@ -305,14 +309,9 @@ def guess_ext( fname, sniff_order=None, is_multi_byte=False ): else: for hdr in headers: for char in hdr: - if len( char ) > 1: - for c in char: - if ord( c ) > 128: - is_binary = True - break - elif ord( char ) > 128: - is_binary = True - break + #old behavior had 'char' possibly having length > 1, + #need to determine when/if this occurs + is_binary = util.is_binary( char ) if is_binary: break if is_binary: diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index cff8f3962e1..2c7817e5353 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -34,6 +34,9 @@ _lock = threading.RLock() gzip_magic = '\037\213' bz2_magic = 'BZh' +DEFAULT_ENCODING = 'utf-8' +NULL_CHAR = '\000' +BINARY_CHARS = [ NULL_CHAR ] from inflection import Inflector, English inflector = Inflector(English) @@ -57,6 +60,32 @@ def is_multi_byte( chars ): return True return False +def is_binary( value, binary_chars=None ): + """ + File is binary if it contains a null-byte by default (e.g. behavior of grep, etc.). + This may fail for utf-16 files, but so would ASCII encoding. + >>> is_binary( string.printable ) + False + >>> is_binary( '\\xce\\x94' ) + False + >>> is_binary( '\\000' ) + True + """ + if binary_chars is None: + binary_chars = BINARY_CHARS + for binary_char in binary_chars: + if binary_char in value: + return True + return False + +def get_charset_from_http_headers( headers, default=None ): + rval = headers.get('content-type', None ) + if rval and 'charset=' in rval: + rval = rval.split('charset=')[-1].split(';')[0].strip() + if rval: + return rval + return default + def synchronized(func): """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator.""" def caller(*params, **kparams): @@ -333,6 +362,17 @@ def roundify(amount, sfs = 2): else: return amount[0:sfs] + '0'*(len(amount) - sfs) +def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ): + """ + Returns a unicode string or None + """ + if isinstance( value, unicode ): + return value + try: + return unicode( value, encoding, error ) + except: + return default + def object_to_string( obj ): return binascii.hexlify( pickle.dumps( obj, 2 ) ) @@ -502,7 +542,7 @@ def stringify_dictionary_keys( in_dict ): def recursively_stringify_dictionary_keys( d ): if isinstance(d, dict): - return dict([(k.encode('utf-8'), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()]) + return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()]) elif isinstance(d, list): return [recursively_stringify_dictionary_keys(x) for x in d] else: @@ -622,7 +662,7 @@ def send_mail( frm, to, subject, body, config ): Sends an email. """ to = listify( to ) - msg = MIMEText( body ) + msg = MIMEText( body.encode( 'ascii', 'replace' ) ) msg[ 'To' ] = ', '.join( to ) msg[ 'From' ] = frm msg[ 'Subject' ] = subject diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index fd85fc2906d..df1137bad05 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -4,6 +4,7 @@ import socket, urllib, sys, os from galaxy import eggs #eggs needs to be imported so that galaxy.util can find docutils egg... from galaxy.util.json import from_json_string, to_json_string +from galaxy.util import get_charset_from_http_headers import galaxy.model # need to import model before sniff to resolve a circular import dependency from galaxy.datatypes import sniff from galaxy.datatypes.registry import Registry @@ -92,7 +93,7 @@ def __main__(): stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) ) #do sniff stream for multi_byte try: - cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename ) + cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename, source_encoding=get_charset_from_http_headers( page.headers ) ) except Exception, e: stop_err( 'Unable to fetch %s:\n%s' % ( cur_URL, e ) )