Handle non-ascii unicode in data source tools. Add util.is_binary() method that returns true when provided string contains a null byte.

lib/galaxy/datatypes/sniff.py

@@ -6,6 +6,7 @@ import registry
 from galaxy import util
 from galaxy.datatypes.checkers import *
 from galaxy.datatypes.binary import unsniffable_binary_formats
+from encodings import search_function as encodings_search_function
 
 log = logging.getLogger(__name__)
 
@@ -15,7 +16,7 @@ def get_test_fname(fname):
     full_path = os.path.join(path, 'test', fname)
     return full_path
 
-def stream_to_open_named_file( stream, fd, filename ):
+def stream_to_open_named_file( stream, fd, filename, source_encoding=None, source_error='strict', target_encoding=None, target_error='strict' ):
     """Writes a stream to the provided file descriptor, returns the file's name and bool( is_multi_byte ). Closes file descriptor"""
     #signature and behavor is somewhat odd, due to backwards compatibility, but this can/should be done better
     CHUNK_SIZE = 1048576
@@ -23,6 +24,10 @@ def stream_to_open_named_file( stream, fd, filename ):
     is_compressed = False
     is_binary = False
     is_multi_byte = False
+    if not target_encoding or not encodings_search_function( target_encoding ):
+        target_encoding = util.DEFAULT_ENCODING #utf-8
+    if not source_encoding:
+        source_encoding = util.DEFAULT_ENCODING #sys.getdefaultencoding() would mimic old behavior (defaults to ascii)
     while 1:
         chunk = stream.read( CHUNK_SIZE )
         if not chunk:
@@ -42,13 +47,12 @@ def stream_to_open_named_file( stream, fd, filename ):
                 chars = chunk[:100]
                 is_multi_byte = util.is_multi_byte( chars )
                 if not is_multi_byte:
-                    for char in chars:
-                        if ord( char ) > 128:
-                            is_binary = True
-                            break
+                    is_binary = util.is_binary( chunk )
             data_checked = True
         if not is_compressed and not is_binary:
-            os.write( fd, chunk.encode( "utf-8" ) )
+            if not isinstance( chunk, unicode ):
+                chunk = chunk.decode( source_encoding, source_error )
+            os.write( fd, chunk.encode( target_encoding, target_error ) )
         else:
             # Compressed files must be encoded after they are uncompressed in the upload utility,
             # while binary files should not be encoded at all.
@@ -56,10 +60,10 @@ def stream_to_open_named_file( stream, fd, filename ):
     os.close( fd )
     return filename, is_multi_byte
 
-def stream_to_file( stream, suffix='', prefix='', dir=None, text=False ):
+def stream_to_file( stream, suffix='', prefix='', dir=None, text=False, **kwd ):
     """Writes a stream to a temporary file, returns the temporary file's name"""
     fd, temp_name = tempfile.mkstemp( suffix=suffix, prefix=prefix, dir=dir, text=text )
-    return stream_to_open_named_file( stream, fd, temp_name )
+    return stream_to_open_named_file( stream, fd, temp_name, **kwd )
 
 def check_newlines( fname, bytes_to_read=52428800 ):
     """
@@ -305,14 +309,9 @@ def guess_ext( fname, sniff_order=None, is_multi_byte=False ):
     else:
         for hdr in headers:
             for char in hdr:
-                if len( char ) > 1:
-                    for c in char:
-                        if ord( c ) > 128:
-                            is_binary = True
-                            break
-                elif ord( char ) > 128:
-                    is_binary = True
-                    break
+                #old behavior had 'char' possibly having length > 1,
+                #need to determine when/if this occurs 
+                is_binary = util.is_binary( char )
                 if is_binary:
                     break
             if is_binary:

lib/galaxy/util/__init__.py

@@ -34,6 +34,9 @@ _lock = threading.RLock()
 
 gzip_magic = '\037\213'
 bz2_magic = 'BZh'
+DEFAULT_ENCODING = 'utf-8'
+NULL_CHAR = '\000'
+BINARY_CHARS = [ NULL_CHAR ]
 
 from inflection import Inflector, English
 inflector = Inflector(English)
@@ -57,6 +60,32 @@ def is_multi_byte( chars ):
             return True
     return False
 
+def is_binary( value, binary_chars=None ):
+    """
+    File is binary if it contains a null-byte by default (e.g. behavior of grep, etc.).
+    This may fail for utf-16 files, but so would ASCII encoding.
+    >>> is_binary( string.printable )
+    False
+    >>> is_binary( '\\xce\\x94' )
+    False
+    >>> is_binary( '\\000' )
+    True
+    """
+    if binary_chars is None:
+        binary_chars = BINARY_CHARS
+    for binary_char in binary_chars:
+        if binary_char in value:
+            return True
+    return False
+
+def get_charset_from_http_headers( headers, default=None ):
+    rval = headers.get('content-type', None )
+    if rval and 'charset=' in rval:
+        rval = rval.split('charset=')[-1].split(';')[0].strip()
+        if rval:
+            return rval
+    return default
+
 def synchronized(func):
     """This wrapper will serialize access to 'func' to a single thread. Use it as a decorator."""
     def caller(*params, **kparams):
@@ -333,6 +362,17 @@ def roundify(amount, sfs = 2):
     else:
         return amount[0:sfs] + '0'*(len(amount) - sfs)
 
+def unicodify( value, encoding=DEFAULT_ENCODING, error='replace', default=None ):
+    """
+    Returns a unicode string or None
+    """
+    if isinstance( value, unicode ):
+        return value
+    try:
+        return unicode( value, encoding, error )
+    except:
+        return default
+
 def object_to_string( obj ):
     return binascii.hexlify( pickle.dumps( obj, 2 ) )
     
@@ -502,7 +542,7 @@ def stringify_dictionary_keys( in_dict ):
 
 def recursively_stringify_dictionary_keys( d ):
     if isinstance(d, dict):
-        return dict([(k.encode('utf-8'), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()])
+        return dict([(k.encode( DEFAULT_ENCODING ), recursively_stringify_dictionary_keys(v)) for k,v in d.iteritems()])
     elif isinstance(d, list):
         return [recursively_stringify_dictionary_keys(x) for x in d]
     else:
@@ -622,7 +662,7 @@ def send_mail( frm, to, subject, body, config ):
     Sends an email.
     """
     to = listify( to )
-    msg = MIMEText( body )
+    msg = MIMEText(  body.encode( 'ascii', 'replace' ) )
     msg[ 'To' ] = ', '.join( to )
     msg[ 'From' ] = frm
     msg[ 'Subject' ] = subject

tools/data_source/data_source.py

@@ -4,6 +4,7 @@
 import socket, urllib, sys, os
 from galaxy import eggs #eggs needs to be imported so that galaxy.util can find docutils egg...
 from galaxy.util.json import from_json_string, to_json_string
+from galaxy.util import get_charset_from_http_headers
 import galaxy.model # need to import model before sniff to resolve a circular import dependency
 from galaxy.datatypes import sniff
 from galaxy.datatypes.registry import Registry
@@ -92,7 +93,7 @@ def __main__():
                 stop_err( 'The size of the data (%d bytes) you have requested exceeds the maximum allowed (%d bytes) on this server.' % ( file_size, max_file_size ) )
         #do sniff stream for multi_byte
         try:
-            cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename )
+            cur_filename, is_multi_byte = sniff.stream_to_open_named_file( page, os.open( cur_filename, os.O_WRONLY | os.O_CREAT ), cur_filename, source_encoding=get_charset_from_http_headers( page.headers ) )
         except Exception, e:
             stop_err( 'Unable to fetch %s:\n%s' % ( cur_URL, e ) )