diff --git a/config/plugins/visualizations/charts/static/app.js b/config/plugins/visualizations/charts/static/app.js index 7dc9969fe43..3c95349c926 100644 --- a/config/plugins/visualizations/charts/static/app.js +++ b/config/plugins/visualizations/charts/static/app.js @@ -1,10 +1,10 @@ // dependencies define(['library/portlet', 'library/ui', 'library/utils', 'views/charts', 'views/viewport', 'views/chart', 'views/group', - 'models/datasets', 'models/chart', 'models/charts', 'models/types'], + 'models/config', 'models/datasets', 'models/chart', 'models/charts', 'models/group', 'models/types'], function( Portlet, Ui, Utils, ChartsView, ViewportView, ChartView, GroupView, - Datasets, Chart, Charts, Types + Config, Datasets, Chart, Charts, Group, Types ) { // widget @@ -19,10 +19,14 @@ return Backbone.View.extend( // link galaxy this.modal = parent.Galaxy.modal; + // create configuration model + this.config = new Config(); + // create chart models this.types = new Types(); this.chart = new Chart(); this.charts = new Charts(); + this.group = new Group(); // create dataset handler this.datasets = new Datasets(this); diff --git a/config/plugins/visualizations/charts/static/models/chart.js b/config/plugins/visualizations/charts/static/models/chart.js index c729ab4c5a3..67e5c2fc4d5 100644 --- a/config/plugins/visualizations/charts/static/models/chart.js +++ b/config/plugins/visualizations/charts/static/models/chart.js @@ -22,8 +22,9 @@ return Backbone.Model.extend( // reset reset: function() { - this.clear().set(this.defaults); + this.clear({silent: true}).set(this.defaults); this.groups.reset(); + this.trigger('reset', this); }, // copy diff --git a/config/plugins/visualizations/charts/static/models/config.js b/config/plugins/visualizations/charts/static/models/config.js new file mode 100644 index 00000000000..2ac90f096ac --- /dev/null +++ b/config/plugins/visualizations/charts/static/models/config.js @@ -0,0 +1,16 @@ +// dependencies +define([], function() { + + +// model +return Backbone.Model.extend( +{ + // options + defaults : { + query_limit : 20, + query_pace : 1000, + query_max : 5 + } +}); + +}); \ No newline at end of file diff --git a/config/plugins/visualizations/charts/static/models/group.js b/config/plugins/visualizations/charts/static/models/group.js index 6727bbcb815..445e59cae79 100644 --- a/config/plugins/visualizations/charts/static/models/group.js +++ b/config/plugins/visualizations/charts/static/models/group.js @@ -1,23 +1,14 @@ // dependencies -define(['library/utils'], function(Utils) { - +define([], function() { // model return Backbone.Model.extend( { - // defaults - defaults : { - }, - - // initialize - initialize: function(options) - { - }, - // reset reset: function() { - this.clear().set(this.defaults); + this.clear({silent: true}).set(this.defaults); + this.trigger('reset', this); } }); diff --git a/config/plugins/visualizations/charts/static/views/chart.js b/config/plugins/visualizations/charts/static/views/chart.js index 90ed350976e..02a443974d5 100644 --- a/config/plugins/visualizations/charts/static/views/chart.js +++ b/config/plugins/visualizations/charts/static/views/chart.js @@ -125,6 +125,9 @@ return Backbone.View.extend( this.chart.on('change:type', function(chart) { self.table.value(chart.get('type')); }); + this.chart.on('reset', function(chart) { + self._resetChart(); + }); // collection events this.app.charts.on('add', function(chart) { @@ -140,23 +143,17 @@ return Backbone.View.extend( }); // reset - this.reset(); + this._resetChart(); }, // reset - reset: function() { - this.chart.reset(); + _resetChart: function() { this.chart.set('id', Utils.uuid()); this.chart.set('dataset_id', this.app.options.dataset.id); this.chart.set('type', 'bardiagram'); this.chart.set('title', 'Chart title'); }, - // set chart - setChart: function(new_chart) { - this.chart.copy(new_chart); - }, - // create chart _saveChart: function() { // update chart data diff --git a/config/plugins/visualizations/charts/static/views/charts.js b/config/plugins/visualizations/charts/static/views/charts.js index 884e96b60b2..8e5b6b75c59 100644 --- a/config/plugins/visualizations/charts/static/views/charts.js +++ b/config/plugins/visualizations/charts/static/views/charts.js @@ -16,12 +16,12 @@ return Backbone.View.extend( ondblclick : function(chart_id) { // get chart var chart = self.app.charts.get(chart_id); - + self.app.chart.copy(chart); + // show edit self.$el.hide(); // update model and show create - self.app.chart_view.setChart(chart); self.app.chart_view.$el.show(); }, onchange : function(chart_id) { @@ -41,7 +41,7 @@ return Backbone.View.extend( tooltip: 'Create', onclick: function() { self.$el.hide(); - self.app.chart_view.reset(); + self.app.chart.reset(); self.app.chart_view.$el.show(); } }), @@ -57,11 +57,11 @@ return Backbone.View.extend( // get chart var chart = self.app.charts.get(chart_id); + self.app.chart.copy(chart); // show edit self.$el.hide(); self.app.chart_view.$el.show(); - self.app.chart_view.setChart(chart); } }), 'delete' : new Ui.ButtonIcon({ diff --git a/config/plugins/visualizations/charts/static/views/group.js b/config/plugins/visualizations/charts/static/views/group.js index b5ac01b0c71..d4de13ffc9c 100644 --- a/config/plugins/visualizations/charts/static/views/group.js +++ b/config/plugins/visualizations/charts/static/views/group.js @@ -1,13 +1,10 @@ // dependencies -define(['library/portlet', 'library/table', 'library/ui', 'library/utils', 'models/group'], - function(Portlet, Table, Ui, Utils, Group) { +define(['library/portlet', 'library/table', 'library/ui', 'library/utils'], + function(Portlet, Table, Ui, Utils) { // chart config return Backbone.View.extend( { - // model - group: new Group(), - // columns columns: [], @@ -18,6 +15,7 @@ return Backbone.View.extend( // get current chart object this.chart = this.app.chart; + this.group = this.app.group; // ui elements this.message = new Ui.Message(); @@ -69,6 +67,9 @@ return Backbone.View.extend( this.group.on('change', function() { self._refreshGroup(); }); + this.group.on('reset', function() { + self._resetGroup(); + }); }, // show @@ -77,8 +78,7 @@ return Backbone.View.extend( }, // reset - reset: function() { - this.group.reset(); + _resetGroup: function() { this.group.set('id', Utils.uuid()); this.group.set('label', 'Group label'); }, diff --git a/config/plugins/visualizations/charts/static/views/groups.js b/config/plugins/visualizations/charts/static/views/groups.js index 9674edbfc77..bd155c53288 100644 --- a/config/plugins/visualizations/charts/static/views/groups.js +++ b/config/plugins/visualizations/charts/static/views/groups.js @@ -37,9 +37,9 @@ return Backbone.View.extend( icon : 'fa-plus', tooltip: 'Create', onclick: function() { + self.app.group.reset(); self.app.chart_view.$el.hide(); self.app.group_view.show(); - self.app.group_view.reset(); } }), 'edit' : new Ui.ButtonIcon({ diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 2acaea99f52..62022dddecb 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -512,13 +512,14 @@ class Data( object ): def before_setting_metadata( self, dataset ): """This function is called on the dataset before metadata is set.""" dataset.clear_associated_files( metadata_safe = True ) - def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ): + def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ): kwds[ 'name' ] = name kwds[ 'optional' ] = optional kwds[ 'mimetype' ] = mimetype kwds[ 'description' ] = description kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata kwds[ 'is_binary' ] = is_binary + kwds[ 'to_posix_lines' ] = to_posix_lines kwds[ 'space_to_tab' ] = space_to_tab return Bunch( **kwds ) def add_composite_file( self, name, **kwds ): diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 403556160cc..e134e63fe22 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -332,6 +332,7 @@ def create_paramfile( trans, uploaded_datasets ): is_binary=is_binary, link_data_only=link_data_only, uuid=uuid_str, + to_posix_lines=uploaded_dataset.to_posix_lines, space_to_tab=uploaded_dataset.space_to_tab, in_place=trans.app.config.external_chown_script is None, path=uploaded_dataset.path ) diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index 609b72f48a3..0111c498249 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -266,6 +266,9 @@ class UploadDataset( Group ): name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) warnings = [] + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True @@ -298,6 +301,7 @@ class UploadDataset( Group ): break if file_bunch.path: break + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab return file_bunch, warnings def get_filenames( context ): @@ -307,16 +311,21 @@ class UploadDataset( Group ): ftp_files = context['ftp_files'] name = context.get( 'NAME', None ) info = context.get( 'INFO', None ) + to_posix_lines = False + if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]: + to_posix_lines = True space_to_tab = False if context.get( 'space_to_tab', None ) not in [ "None", None, False ]: space_to_tab = True warnings = [] file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info ) if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ): if file_bunch.path: + file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab rval.append( file_bunch ) # look for files uploaded via FTP @@ -390,11 +399,13 @@ class UploadDataset( Group ): #replace sniff here with just creating an empty file temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' ) dataset.primary_file = temp_name + dataset.to_posix_lines = True dataset.space_to_tab = False else: file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] ) writable_files_offset = 1 dataset.primary_file = file_bunch.path + dataset.to_posix_lines = file_bunch.to_posix_lines dataset.space_to_tab = file_bunch.space_to_tab dataset.warnings.extend( warnings ) if dataset.primary_file is None:#remove this before finish, this should create an empty dataset diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 707ad945733..359faebb27b 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -524,7 +524,7 @@ class ToolShedRepositoriesController( BaseAPIController ): results[ 'repository_status' ].append( message ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + return json.to_json_string( results, sort_keys=True, indent=4 ) @web.expose_api def show( self, trans, id, **kwd ): diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 063d1566b24..959c0608a82 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -53,7 +53,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin, UsesHistoryMix link_details = util.string_as_bool( kwd.get( 'link_details', False ) ) try: id = urllib.unquote_plus( id ) - tool = self.app.toolbox.tools_by_id.get( id, None ) + tool = self.app.toolbox.get_tool( id ) if not tool: trans.response.status = 404 return { 'error': 'tool not found', 'id': id } diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 455986968da..c17f3f47f08 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -129,7 +129,7 @@ def app_factory( global_conf, **kwargs ): webapp.mapper.resource( 'role', 'roles', path_prefix='/api' ) webapp.mapper.resource( 'group', 'groups', path_prefix='/api' ) webapp.mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' ) - webapp.mapper.connect( '/api/tools/{id:.*?}', action='show', controller="tools" ) + webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" ) webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' ) webapp.mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) webapp.mapper.resource( 'genome', 'genomes', path_prefix='/api' ) diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index fd019dc0b50..22905eb4992 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -384,7 +384,7 @@ class RepositoriesController( BaseAPIController ): results = handle_repository( trans, repository, results ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + return json.to_json_string( results, sort_keys=True, indent=4 ) @web.expose_api def reset_metadata_on_repository( self, trans, payload, **kwd ): @@ -423,7 +423,7 @@ class RepositoriesController( BaseAPIController ): results = handle_repository( trans, start_time, repository ) stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) results[ 'stop_time' ] = stop_time - return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + return json.to_json_string( results, sort_keys=True, indent=4 ) @web.expose_api_anonymous def show( self, trans, id, **kwd ): diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index a4297981b6e..a9e57011037 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -877,8 +877,7 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or username = trans.user.username clause_list = [] for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( and_( trans.model.Repository.table.c.deleted == False, - trans.model.Repository.table.c.deprecated == False ) ): + .filter( trans.model.Repository.table.c.deleted == False ): # Always reset metadata on all repositories of type tool_dependency_definition. if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: clause_list.append( trans.model.Repository.table.c.id == repository.id ) @@ -905,14 +904,12 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or else: if order: return trans.sa_session.query( trans.model.Repository ) \ - .filter( and_( trans.model.Repository.table.c.deleted == False, - trans.model.Repository.table.c.deprecated == False ) ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ .order_by( trans.model.Repository.table.c.name, trans.model.Repository.table.c.user_id ) else: return trans.sa_session.query( trans.model.Repository ) \ - .filter( and_( trans.model.Repository.table.c.deleted == False, - trans.model.Repository.table.c.deprecated == False ) ) + .filter( trans.model.Repository.table.c.deleted == False ) else: # We're in Galaxy. if order: @@ -1599,8 +1596,8 @@ def open_repository_files_folder( trans, folder_path ): return folder_contents def pretty_print( dict=None ): - if dict: - return json.to_json_string( dict, sort_keys=True, indent=4 * ' ' ) + if dict is not None: + return json.to_json_string( dict, sort_keys=True, indent=4 ) def remove_dir( dir ): """Attempt to remove a directory from disk.""" diff --git a/test/functional/api/test_tools.py b/test/functional/api/test_tools.py index fdef6d59aa4..c2af9b14375 100644 --- a/test/functional/api/test_tools.py +++ b/test/functional/api/test_tools.py @@ -33,6 +33,27 @@ class ToolsTestCase( api.ApiTestCase ): create_response = self._post( "tools", data=payload ) self._assert_has_keys( create_response.json(), 'outputs' ) + def test_upload_posix_newline_fixes( self ): + windows_content = "1\t2\t3\r4\t5\t6\r" + posix_content = windows_content.replace("\r", "\n") + result_content = self._upload_and_get_content( windows_content ) + self.assertEquals( result_content, posix_content ) + + def test_upload_disable_posix_fix( self ): + windows_content = "1\t2\t3\r4\t5\t6\r" + result_content = self._upload_and_get_content( windows_content, to_posix_lines=None ) + self.assertEquals( result_content, windows_content ) + + def test_upload_tab_to_space( self ): + table = "1 2 3\n4 5 6\n" + result_content = self._upload_and_get_content( table, space_to_tab="Yes" ) + self.assertEquals( result_content, "1\t2\t3\n4\t5\t6\n" ) + + def test_upload_tab_to_space_off_by_default( self ): + table = "1 2 3\n4 5 6\n" + result_content = self._upload_and_get_content( table ) + self.assertEquals( result_content, table ) + def test_run_cat1( self ): history_id = self._new_history() new_dataset = self._new_dataset( history_id ) @@ -52,6 +73,14 @@ class ToolsTestCase( api.ApiTestCase ): self._assert_has_keys( create_response.json(), 'outputs' ) self._wait_for_history( history_id, assert_ok=True ) + def _upload_and_get_content( self, content, **upload_kwds ): + history_id = self._new_history() + new_dataset = self._new_dataset( history_id, content=content, **upload_kwds ) + self._wait_for_history( history_id, assert_ok=True ) + display_response = self._get( "histories/%s/contents/%s/display" % ( history_id, new_dataset[ "id" ] ) ) + self._assert_status_code_is( display_response, 200 ) + return display_response.content + def _new_dataset( self, history_id, content='TestData123', **kwds ): payload = self._upload_payload( history_id, content, **kwds ) run_response = self._post( "tools", data=payload ) @@ -86,6 +115,10 @@ class ToolsTestCase( api.ApiTestCase ): 'dbkey': dbkey, 'file_type': file_type, } + if "to_posix_lines" in kwds: + upload_params[ "files_0|to_posix_lines"] = kwds[ "to_posix_lines" ] + if "space_to_tab" in kwds: + upload_params[ "files_0|space_to_tab" ] = kwds[ "space_to_tab" ] return self._run_tool_payload( tool_id='upload1', inputs=upload_params, diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 7e76de6a7f3..69dc849eaf6 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -264,10 +264,14 @@ def add_file( dataset, registry, json_file, output_path ): if link_data_only == 'copy_files': if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]: in_place = False - if dataset.space_to_tab: - line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) - else: - line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) + # Convert universal line endings to Posix line endings, but allow the user to turn it off, + # so that is becomes possible to upload gzip, bz2 or zip files with binary data without + # corrupting the content of those files. + if dataset.to_posix_lines: + if dataset.space_to_tab: + line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place ) + else: + line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place ) if dataset.file_type == 'auto': ext = sniff.guess_ext( dataset.path, registry.sniff_order ) else: diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index de02c4afd36..a0520d5a225 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -1,6 +1,6 @@ - + from your computer @@ -38,6 +38,12 @@ + + + +