diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index c65605daeb0..0b4090dbabb 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -498,13 +498,14 @@ class Data( object ): def before_setting_metadata( self, dataset ): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata kwds[ 'is_binary' ] = is_binary + kwds[ 'to_posix_lines' ] = to_posix_lines kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index a533d7cb35b..920b75d2ecc 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -307,6 +307,7 @@ def create_paramfile( trans, uploaded_datasets ): is_binary = is_binary, link_data_only = link_data_only, uuid = uuid_str, + to_posix_lines = uploaded_dataset.to_posix_lines, space_to_tab = uploaded_dataset.space_to_tab, in_place = trans.app.config.external_chown_script is None, path = uploaded_dataset.path ) diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index aaaf5fc1c9a..d8dd2376db8 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -254,6 +254,9 @@ class UploadDataset( Group ): name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) warnings = [] + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True @@ -286,6 +289,7 @@ class UploadDataset( Group ): break if file_bunch.path: break + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab return file_bunch, warnings def get_filenames( context ): @@ -295,16 +299,21 @@ class UploadDataset( Group ): ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True warnings = [] file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) # look for files uploaded via FTP @@ -378,11 +387,13 @@ class UploadDataset( Group ): #replace sniff here with just creating an empty file temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' ) dataset.primary_file = temp_name + dataset.to_posix_lines = True dataset.space_to_tab = False else: file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] ) writable_files_offset = 1 dataset.primary_file = file_bunch.path + dataset.to_posix_lines = file_bunch.to_posix_lines dataset.space_to_tab = file_bunch.space_to_tab dataset.warnings.extend( warnings ) if dataset.primary_file is None:#remove this before finish, this should create an empty dataset diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 7e76de6a7f3..69dc849eaf6 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -264,10 +264,14 @@ def add_file( dataset, registry, json_file, output_path ): if link_data_only == 'copy_files': if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]: in_place = False - if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) - else: - line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) + # Convert universal line endings to Posix line endings, but allow the user to turn it off, + # so that is becomes possible to upload gzip, bz2 or zip files with binary data without + # corrupting the content of those files. + if dataset.to_posix_lines: + if dataset.space_to_tab: + line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) + else: + line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) else: diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index de02c4afd36..7bf8e1ac5d7 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ - + from your computer @@ -38,6 +38,9 @@ + + +