From fb01c3cc96baa56cd5305da0349817ef1f179c8a Mon Sep 17 00:00:00 2001 From: Gert Hulselmans Date: Mon, 27 May 2013 20:55:52 +0200 Subject: [PATCH] Avoid corruption of binary files embedded in gzip, bz2 and zip archives in the upload tool. Add an option in the upload tool to disable the conversion of universal line endings to Posix line endings. This is useful for avoiding corruption of uploaded files when a binary file is contained inside a gzip, bz2 and zip archive. This fixes bug report: https://trello.com/card/issue-with-uploaded-2bit-gz-files/506338ce32ae458f6d15e4b3/702 --- lib/galaxy/datatypes/data.py | 3 ++- lib/galaxy/tools/actions/upload_common.py | 1 + lib/galaxy/tools/parameters/grouping.py | 11 +++++++++++ tools/data_source/upload.py | 12 ++++++++---- tools/data_source/upload.xml | 5 ++++- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index c65605daeb0..0b4090dbabb 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -498,13 +498,14 @@ class Data( object ): def before_setting_metadata( self, dataset ): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata kwds[ 'is_binary' ] = is_binary + kwds[ 'to_posix_lines' ] = to_posix_lines kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index a533d7cb35b..920b75d2ecc 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -307,6 +307,7 @@ def create_paramfile( trans, uploaded_datasets ): is_binary = is_binary, link_data_only = link_data_only, uuid = uuid_str, + to_posix_lines = uploaded_dataset.to_posix_lines, space_to_tab = uploaded_dataset.space_to_tab, in_place = trans.app.config.external_chown_script is None, path = uploaded_dataset.path ) diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index aaaf5fc1c9a..d8dd2376db8 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -254,6 +254,9 @@ class UploadDataset( Group ): name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) warnings = [] + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True @@ -286,6 +289,7 @@ class UploadDataset( Group ): break if file_bunch.path: break + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab return file_bunch, warnings def get_filenames( context ): @@ -295,16 +299,21 @@ class UploadDataset( Group ): ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True warnings = [] file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) # look for files uploaded via FTP @@ -378,11 +387,13 @@ class UploadDataset( Group ): #replace sniff here with just creating an empty file temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' ) dataset.primary_file = temp_name + dataset.to_posix_lines = True dataset.space_to_tab = False else: file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] ) writable_files_offset = 1 dataset.primary_file = file_bunch.path + dataset.to_posix_lines = file_bunch.to_posix_lines dataset.space_to_tab = file_bunch.space_to_tab dataset.warnings.extend( warnings ) if dataset.primary_file is None:#remove this before finish, this should create an empty dataset diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 7e76de6a7f3..69dc849eaf6 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -264,10 +264,14 @@ def add_file( dataset, registry, json_file, output_path ): if link_data_only == 'copy_files': if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]: in_place = False - if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) - else: - line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) + # Convert universal line endings to Posix line endings, but allow the user to turn it off, + # so that is becomes possible to upload gzip, bz2 or zip files with binary data without + # corrupting the content of those files. + if dataset.to_posix_lines: + if dataset.space_to_tab: + line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) + else: + line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) else: diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index de02c4afd36..7bf8e1ac5d7 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ - + from your computer @@ -38,6 +38,9 @@ + + +