Avoid corruption of binary files embedded in gzip, bz2 and zip archives in the upload tool.

Add an option in the upload tool to disable the conversion of universal
line endings to Posix line endings.
This is useful for avoiding corruption of uploaded files when a binary
file is contained inside a gzip, bz2 and zip archive.

This fixes bug report: https://trello.com/card/issue-with-uploaded-2bit-gz-files/506338ce32ae458f6d15e4b3/702

lib/galaxy/datatypes/data.py

@@ -498,13 +498,14 @@ class Data( object ):
     def before_setting_metadata( self, dataset ):
         """This function is called on the dataset before metadata is set."""
         dataset.clear_associated_files( metadata_safe = True )
-    def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ):
+    def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ):
         kwds[ 'name' ] = name
         kwds[ 'optional' ] = optional
         kwds[ 'mimetype' ] = mimetype
         kwds[ 'description' ] = description
         kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata
         kwds[ 'is_binary' ] = is_binary
+        kwds[ 'to_posix_lines' ] = to_posix_lines
         kwds[ 'space_to_tab' ] = space_to_tab
         return Bunch( **kwds )
     def add_composite_file( self, name, **kwds ):

lib/galaxy/tools/actions/upload_common.py

@@ -307,6 +307,7 @@ def create_paramfile( trans, uploaded_datasets ):
                          is_binary = is_binary,
                          link_data_only = link_data_only,
                          uuid = uuid_str,
+                         to_posix_lines = uploaded_dataset.to_posix_lines,
                          space_to_tab = uploaded_dataset.space_to_tab,
                          in_place = trans.app.config.external_chown_script is None,
                          path = uploaded_dataset.path )

lib/galaxy/tools/parameters/grouping.py

@@ -254,6 +254,9 @@ class UploadDataset( Group ):
             name = context.get( 'NAME', None )
             info = context.get( 'INFO', None )
             warnings = []
+            to_posix_lines = False
+            if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]:
+                to_posix_lines = True
             space_to_tab = False
             if context.get( 'space_to_tab', None ) not in [ "None", None, False ]:
                 space_to_tab = True
@@ -286,6 +289,7 @@ class UploadDataset( Group ):
                             break
                     if file_bunch.path:
                         break
+            file_bunch.to_posix_lines = to_posix_lines
             file_bunch.space_to_tab = space_to_tab
             return file_bunch, warnings
         def get_filenames( context ):
@@ -295,16 +299,21 @@ class UploadDataset( Group ):
             ftp_files = context['ftp_files']
             name = context.get( 'NAME', None )
             info = context.get( 'INFO', None )
+            to_posix_lines = False
+            if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]:
+                to_posix_lines = True
             space_to_tab = False
             if context.get( 'space_to_tab', None ) not in [ "None", None, False ]:
                 space_to_tab = True
             warnings = []
             file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info )
             if file_bunch.path:
+                file_bunch.to_posix_lines = to_posix_lines
                 file_bunch.space_to_tab = space_to_tab
                 rval.append( file_bunch )
             for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ):
                 if file_bunch.path:
+                    file_bunch.to_posix_lines = to_posix_lines
                     file_bunch.space_to_tab = space_to_tab
                     rval.append( file_bunch )
             # look for files uploaded via FTP
@@ -378,11 +387,13 @@ class UploadDataset( Group ):
                 #replace sniff here with just creating an empty file
                 temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' )
                 dataset.primary_file = temp_name
+                dataset.to_posix_lines = True
                 dataset.space_to_tab = False
             else:
                 file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] )
                 writable_files_offset = 1
                 dataset.primary_file = file_bunch.path
+                dataset.to_posix_lines = file_bunch.to_posix_lines
                 dataset.space_to_tab = file_bunch.space_to_tab
                 dataset.warnings.extend( warnings )
             if dataset.primary_file is None:#remove this before finish, this should create an empty dataset

tools/data_source/upload.py

@@ -264,10 +264,14 @@ def add_file( dataset, registry, json_file, output_path ):
             if link_data_only == 'copy_files':
                 if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]:
                     in_place = False
-                if dataset.space_to_tab:
-                    line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place )
-                else:
-                    line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place )
+                # Convert universal line endings to Posix line endings, but allow the user to turn it off,
+                # so that is becomes possible to upload gzip, bz2 or zip files with binary data without
+                # corrupting the content of those files.
+                if dataset.to_posix_lines:
+                    if dataset.space_to_tab:
+                        line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place )
+                    else:
+                        line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place )
             if dataset.file_type == 'auto':
                 ext = sniff.guess_ext( dataset.path, registry.sniff_order )
             else:

tools/data_source/upload.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 
-<tool name="Upload File" id="upload1" version="1.1.3" workflow_compatible="false">
+<tool name="Upload File" id="upload1" version="1.1.4" workflow_compatible="false">
   <description>
     from your computer  
   </description>
@@ -38,6 +38,9 @@
       </param>
       <param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/> 
       <param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/>
+      <param name="to_posix_lines" type="select" display="checkboxes" multiple="True" label="Convert universal line endings to Posix line endings" help="Turn this option off if you upload a gzip, bz2 or zip archive which contains a binary file."> 
+        <option value="Yes" selected="true">Yes</option>
+      </param>
       <param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."> 
         <option value="Yes">Yes</option>
       </param>