From fb01c3cc96baa56cd5305da0349817ef1f179c8a Mon Sep 17 00:00:00 2001 From: Gert Hulselmans Date: Mon, 27 May 2013 20:55:52 +0200 Subject: [PATCH 1/8] Avoid corruption of binary files embedded in gzip, bz2 and zip archives in the upload tool. Add an option in the upload tool to disable the conversion of universal line endings to Posix line endings. This is useful for avoiding corruption of uploaded files when a binary file is contained inside a gzip, bz2 and zip archive. This fixes bug report: https://trello.com/card/issue-with-uploaded-2bit-gz-files/506338ce32ae458f6d15e4b3/702 --- lib/galaxy/datatypes/data.py | 3 ++- lib/galaxy/tools/actions/upload_common.py | 1 + lib/galaxy/tools/parameters/grouping.py | 11 +++++++++++ tools/data_source/upload.py | 12 ++++++++---- tools/data_source/upload.xml | 5 ++++- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index c65605daeb0..0b4090dbabb 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -498,13 +498,14 @@ class Data( object ): def before_setting_metadata( self, dataset ): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata kwds[ 'is_binary' ] = is_binary + kwds[ 'to_posix_lines' ] = to_posix_lines kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index a533d7cb35b..920b75d2ecc 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -307,6 +307,7 @@ def create_paramfile( trans, uploaded_datasets ): is_binary = is_binary, link_data_only = link_data_only, uuid = uuid_str, + to_posix_lines = uploaded_dataset.to_posix_lines, space_to_tab = uploaded_dataset.space_to_tab, in_place = trans.app.config.external_chown_script is None, path = uploaded_dataset.path ) diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index aaaf5fc1c9a..d8dd2376db8 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -254,6 +254,9 @@ class UploadDataset( Group ): name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) warnings = [] + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True @@ -286,6 +289,7 @@ class UploadDataset( Group ): break if file_bunch.path: break + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab return file_bunch, warnings def get_filenames( context ): @@ -295,16 +299,21 @@ class UploadDataset( Group ): ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True warnings = [] file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) # look for files uploaded via FTP @@ -378,11 +387,13 @@ class UploadDataset( Group ): #replace sniff here with just creating an empty file temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' ) dataset.primary_file = temp_name + dataset.to_posix_lines = True dataset.space_to_tab = False else: file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] ) writable_files_offset = 1 dataset.primary_file = file_bunch.path + dataset.to_posix_lines = file_bunch.to_posix_lines dataset.space_to_tab = file_bunch.space_to_tab dataset.warnings.extend( warnings ) if dataset.primary_file is None:#remove this before finish, this should create an empty dataset diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 7e76de6a7f3..69dc849eaf6 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -264,10 +264,14 @@ def add_file( dataset, registry, json_file, output_path ): if link_data_only == 'copy_files': if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]: in_place = False - if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) - else: - line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) + # Convert universal line endings to Posix line endings, but allow the user to turn it off, + # so that is becomes possible to upload gzip, bz2 or zip files with binary data without + # corrupting the content of those files. + if dataset.to_posix_lines: + if dataset.space_to_tab: + line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) + else: + line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) else: diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index de02c4afd36..7bf8e1ac5d7 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ - + from your computer @@ -38,6 +38,9 @@ + + + From 92657b38dbfef3c08990da2dc00a944dc4acbb92 Mon Sep 17 00:00:00 2001 From: Aysam Guerler Date: Thu, 16 Jan 2014 14:10:32 -0500 Subject: [PATCH 2/8] Charts: Add configuration model, link all view functions to model events --- .../plugins/visualizations/charts/static/app.js | 8 ++++++-- .../visualizations/charts/static/models/chart.js | 3 ++- .../charts/static/models/config.js | 16 ++++++++++++++++ .../visualizations/charts/static/models/group.js | 15 +++------------ .../visualizations/charts/static/views/chart.js | 13 +++++-------- .../visualizations/charts/static/views/charts.js | 8 ++++---- .../visualizations/charts/static/views/group.js | 14 +++++++------- .../visualizations/charts/static/views/groups.js | 2 +- 8 files changed, 44 insertions(+), 35 deletions(-) create mode 100644 config/plugins/visualizations/charts/static/models/config.js diff --git a/config/plugins/visualizations/charts/static/app.js b/config/plugins/visualizations/charts/static/app.js index 7dc9969fe43..3c95349c926 100644 --- a/config/plugins/visualizations/charts/static/app.js +++ b/config/plugins/visualizations/charts/static/app.js @@ -1,10 +1,10 @@ // dependencies define(['library/portlet', 'library/ui', 'library/utils', 'views/charts', 'views/viewport', 'views/chart', 'views/group', - 'models/datasets', 'models/chart', 'models/charts', 'models/types'], + 'models/config', 'models/datasets', 'models/chart', 'models/charts', 'models/group', 'models/types'], function( Portlet, Ui, Utils, ChartsView, ViewportView, ChartView, GroupView, - Datasets, Chart, Charts, Types + Config, Datasets, Chart, Charts, Group, Types ) { // widget @@ -19,10 +19,14 @@ return Backbone.View.extend( // link galaxy this.modal = parent.Galaxy.modal; + // create configuration model + this.config = new Config(); + // create chart models this.types = new Types(); this.chart = new Chart(); this.charts = new Charts(); + this.group = new Group(); // create dataset handler this.datasets = new Datasets(this); diff --git a/config/plugins/visualizations/charts/static/models/chart.js b/config/plugins/visualizations/charts/static/models/chart.js index c729ab4c5a3..67e5c2fc4d5 100644 --- a/config/plugins/visualizations/charts/static/models/chart.js +++ b/config/plugins/visualizations/charts/static/models/chart.js @@ -22,8 +22,9 @@ return Backbone.Model.extend( // reset reset: function() { - this.clear().set(this.defaults); + this.clear({silent: true}).set(this.defaults); this.groups.reset(); + this.trigger('reset', this); }, // copy diff --git a/config/plugins/visualizations/charts/static/models/config.js b/config/plugins/visualizations/charts/static/models/config.js new file mode 100644 index 00000000000..2ac90f096ac --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/config.js @@ -0,0 +1,16 @@ +// dependencies +define([], function() { + + +// model +return Backbone.Model.extend( +{ + // options + defaults : { + query_limit : 20, + query_pace : 1000, + query_max : 5 + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/group.js b/config/plugins/visualizations/charts/static/models/group.js index 6727bbcb815..445e59cae79 100644 --- a/config/plugins/visualizations/charts/static/models/group.js +++ b/config/plugins/visualizations/charts/static/models/group.js @@ -1,23 +1,14 @@ // dependencies -define(['library/utils'], function(Utils) { - +define([], function() { // model return Backbone.Model.extend( { - // defaults - defaults : { - }, - - // initialize - initialize: function(options) - { - }, - // reset reset: function() { - this.clear().set(this.defaults); + this.clear({silent: true}).set(this.defaults); + this.trigger('reset', this); } }); diff --git a/config/plugins/visualizations/charts/static/views/chart.js b/config/plugins/visualizations/charts/static/views/chart.js index 90ed350976e..02a443974d5 100644 --- a/config/plugins/visualizations/charts/static/views/chart.js +++ b/config/plugins/visualizations/charts/static/views/chart.js @@ -125,6 +125,9 @@ return Backbone.View.extend( this.chart.on('change:type', function(chart) { self.table.value(chart.get('type')); }); + this.chart.on('reset', function(chart) { + self._resetChart(); + }); // collection events this.app.charts.on('add', function(chart) { @@ -140,23 +143,17 @@ return Backbone.View.extend( }); // reset - this.reset(); + this._resetChart(); }, // reset - reset: function() { - this.chart.reset(); + _resetChart: function() { this.chart.set('id', Utils.uuid()); this.chart.set('dataset_id', this.app.options.dataset.id); this.chart.set('type', 'bardiagram'); this.chart.set('title', 'Chart title'); }, - // set chart - setChart: function(new_chart) { - this.chart.copy(new_chart); - }, - // create chart _saveChart: function() { // update chart data diff --git a/config/plugins/visualizations/charts/static/views/charts.js b/config/plugins/visualizations/charts/static/views/charts.js index 884e96b60b2..8e5b6b75c59 100644 --- a/config/plugins/visualizations/charts/static/views/charts.js +++ b/config/plugins/visualizations/charts/static/views/charts.js @@ -16,12 +16,12 @@ return Backbone.View.extend( ondblclick : function(chart_id) { // get chart var chart = self.app.charts.get(chart_id); - + self.app.chart.copy(chart); + // show edit self.$el.hide(); // update model and show create - self.app.chart_view.setChart(chart); self.app.chart_view.$el.show(); }, onchange : function(chart_id) { @@ -41,7 +41,7 @@ return Backbone.View.extend( tooltip: 'Create', onclick: function() { self.$el.hide(); - self.app.chart_view.reset(); + self.app.chart.reset(); self.app.chart_view.$el.show(); } }), @@ -57,11 +57,11 @@ return Backbone.View.extend( // get chart var chart = self.app.charts.get(chart_id); + self.app.chart.copy(chart); // show edit self.$el.hide(); self.app.chart_view.$el.show(); - self.app.chart_view.setChart(chart); } }), 'delete' : new Ui.ButtonIcon({ diff --git a/config/plugins/visualizations/charts/static/views/group.js b/config/plugins/visualizations/charts/static/views/group.js index b5ac01b0c71..d4de13ffc9c 100644 --- a/config/plugins/visualizations/charts/static/views/group.js +++ b/config/plugins/visualizations/charts/static/views/group.js @@ -1,13 +1,10 @@ // dependencies -define(['library/portlet', 'library/table', 'library/ui', 'library/utils', 'models/group'], - function(Portlet, Table, Ui, Utils, Group) { +define(['library/portlet', 'library/table', 'library/ui', 'library/utils'], + function(Portlet, Table, Ui, Utils) { // chart config return Backbone.View.extend( { - // model - group: new Group(), - // columns columns: [], @@ -18,6 +15,7 @@ return Backbone.View.extend( // get current chart object this.chart = this.app.chart; + this.group = this.app.group; // ui elements this.message = new Ui.Message(); @@ -69,6 +67,9 @@ return Backbone.View.extend( this.group.on('change', function() { self._refreshGroup(); }); + this.group.on('reset', function() { + self._resetGroup(); + }); }, // show @@ -77,8 +78,7 @@ return Backbone.View.extend( }, // reset - reset: function() { - this.group.reset(); + _resetGroup: function() { this.group.set('id', Utils.uuid()); this.group.set('label', 'Group label'); }, diff --git a/config/plugins/visualizations/charts/static/views/groups.js b/config/plugins/visualizations/charts/static/views/groups.js index 9674edbfc77..bd155c53288 100644 --- a/config/plugins/visualizations/charts/static/views/groups.js +++ b/config/plugins/visualizations/charts/static/views/groups.js @@ -37,9 +37,9 @@ return Backbone.View.extend( icon : 'fa-plus', tooltip: 'Create', onclick: function() { + self.app.group.reset(); self.app.chart_view.$el.hide(); self.app.group_view.show(); - self.app.group_view.reset(); } }), 'edit' : new Ui.ButtonIcon({ From 78b62295844dfa69c426bceb059be6f67dc7c011 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Thu, 16 Jan 2014 16:02:39 -0500 Subject: [PATCH 3/8] Re-include deprecated repositories when resetting metadata since they can be dependencies of installable repositories. --- .../webapps/galaxy/api/tool_shed_repositories.py | 2 +- lib/galaxy/webapps/tool_shed/api/repositories.py | 4 ++-- lib/tool_shed/util/shed_util_common.py | 13 +++++-------- 3 files changed, 8 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 707ad945733..359faebb27b 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -524,7 +524,7 @@ class ToolShedRepositoriesController( BaseAPIController ): results[ 'repository_status' ].append( message ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + return json.to_json_string( results, sort_keys=True, indent=4 ) @web.expose_api def show( self, trans, id, **kwd ): diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index fd019dc0b50..22905eb4992 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -384,7 +384,7 @@ class RepositoriesController( BaseAPIController ): results = handle_repository( trans, repository, results ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + return json.to_json_string( results, sort_keys=True, indent=4 ) @web.expose_api def reset_metadata_on_repository( self, trans, payload, **kwd ): @@ -423,7 +423,7 @@ class RepositoriesController( BaseAPIController ): results = handle_repository( trans, start_time, repository ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + return json.to_json_string( results, sort_keys=True, indent=4 ) @web.expose_api_anonymous def show( self, trans, id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index a4297981b6e..937b1c012f7 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -877,8 +877,7 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or username = trans.user.username clause_list = [] for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( and_( trans.model.Repository.table.c.deleted == False, - trans.model.Repository.table.c.deprecated == False ) ): + .filter( trans.model.Repository.table.c.deleted == False ): # Always reset metadata on all repositories of type tool_dependency_definition. if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: clause_list.append( trans.model.Repository.table.c.id == repository.id ) @@ -905,14 +904,12 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or else: if order: return trans.sa_session.query( trans.model.Repository ) \ - .filter( and_( trans.model.Repository.table.c.deleted == False, - trans.model.Repository.table.c.deprecated == False ) ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ .order_by( trans.model.Repository.table.c.name, trans.model.Repository.table.c.user_id ) else: return trans.sa_session.query( trans.model.Repository ) \ - .filter( and_( trans.model.Repository.table.c.deleted == False, - trans.model.Repository.table.c.deprecated == False ) ) + .filter( trans.model.Repository.table.c.deleted == False ) else: # We're in Galaxy. if order: @@ -1599,8 +1596,8 @@ def open_repository_files_folder( trans, folder_path ): return folder_contents def pretty_print( dict=None ): - if dict: - return json.to_json_string( dict, sort_keys=True, indent=4 * ' ' ) + if dict is not: + return json.to_json_string( dict, sort_keys=True, indent=4 ) def remove_dir( dir ): """Attempt to remove a directory from disk.""" From 9d49a65e603c924902b1b0c2fe5724f7d937b97b Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Thu, 16 Jan 2014 16:13:47 -0500 Subject: [PATCH 4/8] Forgot None in my last commit --- lib/tool_shed/util/shed_util_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 937b1c012f7..a9e57011037 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -1596,7 +1596,7 @@ def open_repository_files_folder( trans, folder_path ): return folder_contents def pretty_print( dict=None ): - if dict is not: + if dict is not None: return json.to_json_string( dict, sort_keys=True, indent=4 ) def remove_dir( dir ): From fddb4b0ba6f1996a668d687f907c381064b05da8 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Thu, 16 Jan 2014 16:21:41 -0500 Subject: [PATCH 5/8] Use toolbox.get_tool instead of toolbox.tools_by_id.get in tools api show(). --- lib/galaxy/webapps/galaxy/api/tools.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 063d1566b24..959c0608a82 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -53,7 +53,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin, UsesHistoryMix link_details = util.string_as_bool( kwd.get( 'link_details', False ) ) try: id = urllib.unquote_plus( id ) - tool = self.app.toolbox.tools_by_id.get( id, None ) + tool = self.app.toolbox.get_tool( id ) if not tool: trans.response.status = 404 return { 'error': 'tool not found', 'id': id } From 2752bc74de9a15580044cf0fce22199b9ea98d41 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Thu, 16 Jan 2014 16:52:39 -0500 Subject: [PATCH 6/8] Fix for api/tools/show/ when no id is provided. --- lib/galaxy/webapps/galaxy/buildapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 455986968da..c17f3f47f08 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -129,7 +129,7 @@ def app_factory( global_conf, **kwargs ): webapp.mapper.resource( 'role', 'roles', path_prefix='/api' ) webapp.mapper.resource( 'group', 'groups', path_prefix='/api' ) webapp.mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' ) - webapp.mapper.connect( '/api/tools/{id:.*?}', action='show', controller="tools" ) + webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" ) webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' ) webapp.mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) webapp.mapper.resource( 'genome', 'genomes', path_prefix='/api' ) From 7be6b9758a352305bd74db00f6b49fbbe427d19b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 17 Jan 2014 00:03:26 -0600 Subject: [PATCH 7/8] Functionally test various upload options via API. --- test/functional/api/test_tools.py | 33 +++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/test/functional/api/test_tools.py b/test/functional/api/test_tools.py index fdef6d59aa4..c2af9b14375 100644 --- a/test/functional/api/test_tools.py +++ b/test/functional/api/test_tools.py @@ -33,6 +33,27 @@ class ToolsTestCase( api.ApiTestCase ): create_response = self._post( "tools", data=payload ) self._assert_has_keys( create_response.json(), 'outputs' ) + def test_upload_posix_newline_fixes( self ): + windows_content = "1\t2\t3\r4\t5\t6\r" + posix_content = windows_content.replace("\r", "\n") + result_content = self._upload_and_get_content( windows_content ) + self.assertEquals( result_content, posix_content ) + + def test_upload_disable_posix_fix( self ): + windows_content = "1\t2\t3\r4\t5\t6\r" + result_content = self._upload_and_get_content( windows_content, to_posix_lines=None ) + self.assertEquals( result_content, windows_content ) + + def test_upload_tab_to_space( self ): + table = "1 2 3\n4 5 6\n" + result_content = self._upload_and_get_content( table, space_to_tab="Yes" ) + self.assertEquals( result_content, "1\t2\t3\n4\t5\t6\n" ) + + def test_upload_tab_to_space_off_by_default( self ): + table = "1 2 3\n4 5 6\n" + result_content = self._upload_and_get_content( table ) + self.assertEquals( result_content, table ) + def test_run_cat1( self ): history_id = self._new_history() new_dataset = self._new_dataset( history_id ) @@ -52,6 +73,14 @@ class ToolsTestCase( api.ApiTestCase ): self._assert_has_keys( create_response.json(), 'outputs' ) self._wait_for_history( history_id, assert_ok=True ) + def _upload_and_get_content( self, content, **upload_kwds ): + history_id = self._new_history() + new_dataset = self._new_dataset( history_id, content=content, **upload_kwds ) + self._wait_for_history( history_id, assert_ok=True ) + display_response = self._get( "histories/%s/contents/%s/display" % ( history_id, new_dataset[ "id" ] ) ) + self._assert_status_code_is( display_response, 200 ) + return display_response.content + def _new_dataset( self, history_id, content='TestData123', **kwds ): payload = self._upload_payload( history_id, content, **kwds ) run_response = self._post( "tools", data=payload ) @@ -86,6 +115,10 @@ class ToolsTestCase( api.ApiTestCase ): 'dbkey': dbkey, 'file_type': file_type, } + if "to_posix_lines" in kwds: + upload_params[ "files_0|to_posix_lines"] = kwds[ "to_posix_lines" ] + if "space_to_tab" in kwds: + upload_params[ "files_0|space_to_tab" ] = kwds[ "space_to_tab" ] return self._run_tool_payload( tool_id='upload1', inputs=upload_params, From 915948a305e8f24429e67265db2649f68460cdfd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 17 Jan 2014 00:03:26 -0600 Subject: [PATCH 8/8] Hide to_posix_lines in UI by default. Galaxy still completely supports this option on backend - API tests still completely work and will hopefully ensure this functionality continues running. Any Galaxy deployements that wish to enable this option simply have to switch this one param from hidden to select. This is a ugly workaround - but it is a very small workaround - and will hopefully alleviate any potential fears Gert Hulselmans has of Galaxy diverging from his changes. This is only a stop gap until the Galaxy upload UI supports more advanced options. I have previously outlined my concerns with the visual clutter on the upload.xml page - this is why I am hiding it by default. If anyone with commit access disagrees - please by all means backout of this individual changeset and restore the full functionality of pull request 171 (I have a terrible eye for this sort of GUI design stuff). --- tools/data_source/upload.xml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index 7bf8e1ac5d7..a0520d5a225 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -38,7 +38,10 @@ - + +