Merge

config/plugins/visualizations/charts/static/app.js

@@ -1,10 +1,10 @@
 // dependencies
 define(['library/portlet', 'library/ui', 'library/utils',
         'views/charts', 'views/viewport', 'views/chart', 'views/group',
-        'models/datasets', 'models/chart', 'models/charts', 'models/types'],
+        'models/config', 'models/datasets', 'models/chart', 'models/charts', 'models/group', 'models/types'],
         function(   Portlet, Ui, Utils,
                     ChartsView, ViewportView, ChartView, GroupView,
-                    Datasets, Chart, Charts, Types
+                    Config, Datasets, Chart, Charts, Group, Types
                 ) {
 
 // widget
@@ -19,10 +19,14 @@ return Backbone.View.extend(
         // link galaxy
         this.modal = parent.Galaxy.modal;
         
+        // create configuration model
+        this.config = new Config();
+        
         // create chart models
         this.types = new Types();
         this.chart = new Chart();
         this.charts = new Charts();
+        this.group = new Group();
         
         // create dataset handler
         this.datasets = new Datasets(this);

config/plugins/visualizations/charts/static/models/chart.js

@@ -22,8 +22,9 @@ return Backbone.Model.extend(
     // reset
     reset: function()
     {
-        this.clear().set(this.defaults);
+        this.clear({silent: true}).set(this.defaults);
         this.groups.reset();
+        this.trigger('reset', this);
     },
     
     // copy

config/plugins/visualizations/charts/static/models/config.js

@@ -0,0 +1,16 @@
+// dependencies
+define([], function() {
+
+
+// model
+return Backbone.Model.extend(
+{
+    // options
+    defaults : {
+        query_limit     : 20,
+        query_pace      : 1000,
+        query_max       : 5
+    }
+});
+
+});

config/plugins/visualizations/charts/static/models/group.js

@@ -1,23 +1,14 @@
 // dependencies
-define(['library/utils'], function(Utils) {
-
+define([], function() {
 
 // model
 return Backbone.Model.extend(
 {
-    // defaults
-    defaults : {
-    },
-    
-    // initialize
-    initialize: function(options)
-    {
-    },
-    
     // reset
     reset: function()
     {
-        this.clear().set(this.defaults);
+        this.clear({silent: true}).set(this.defaults);
+        this.trigger('reset', this);
     }
 });
 

config/plugins/visualizations/charts/static/views/chart.js

@@ -125,6 +125,9 @@ return Backbone.View.extend(
         this.chart.on('change:type', function(chart) {
             self.table.value(chart.get('type'));
         });
+        this.chart.on('reset', function(chart) {
+            self._resetChart();
+        });
         
         // collection events
         this.app.charts.on('add', function(chart) {
@@ -140,23 +143,17 @@ return Backbone.View.extend(
         });
         
         // reset
-        this.reset();
+        this._resetChart();
     },
 
     // reset
-    reset: function() {
-        this.chart.reset();
+    _resetChart: function() {
         this.chart.set('id', Utils.uuid());
         this.chart.set('dataset_id', this.app.options.dataset.id);
         this.chart.set('type', 'bardiagram');
         this.chart.set('title', 'Chart title');
     },
     
-    // set chart
-    setChart: function(new_chart) {
-        this.chart.copy(new_chart);
-    },
-    
     // create chart
     _saveChart: function() {
         // update chart data

config/plugins/visualizations/charts/static/views/charts.js

@@ -16,12 +16,12 @@ return Backbone.View.extend(
             ondblclick  : function(chart_id) {
                 // get chart
                 var chart = self.app.charts.get(chart_id);
-                                
+                self.app.chart.copy(chart);
+                               
                 // show edit
                 self.$el.hide();
                 
                 // update model and show create
-                self.app.chart_view.setChart(chart);
                 self.app.chart_view.$el.show();
             },
             onchange : function(chart_id) {
@@ -41,7 +41,7 @@ return Backbone.View.extend(
                             tooltip: 'Create',
                             onclick: function() {
                                 self.$el.hide();
-                                self.app.chart_view.reset();
+                                self.app.chart.reset();
                                 self.app.chart_view.$el.show();
                             }
                         }),
@@ -57,11 +57,11 @@ return Backbone.View.extend(
                                 
                                 // get chart
                                 var chart = self.app.charts.get(chart_id);
+                                self.app.chart.copy(chart);
                                 
                                 // show edit
                                 self.$el.hide();
                                 self.app.chart_view.$el.show();
-                                self.app.chart_view.setChart(chart);
                             }
                         }),
                 'delete' : new Ui.ButtonIcon({

config/plugins/visualizations/charts/static/views/group.js

@@ -1,13 +1,10 @@
 // dependencies
-define(['library/portlet', 'library/table', 'library/ui', 'library/utils', 'models/group'],
-        function(Portlet, Table, Ui, Utils, Group) {
+define(['library/portlet', 'library/table', 'library/ui', 'library/utils'],
+        function(Portlet, Table, Ui, Utils) {
 
 // chart config
 return Backbone.View.extend(
 {
-    // model
-    group: new Group(),
-    
     // columns
     columns: [],
     
@@ -18,6 +15,7 @@ return Backbone.View.extend(
     
         // get current chart object
         this.chart = this.app.chart;
+        this.group = this.app.group;
         
         // ui elements
         this.message = new Ui.Message();
@@ -69,6 +67,9 @@ return Backbone.View.extend(
         this.group.on('change', function() {
             self._refreshGroup();
         });
+        this.group.on('reset', function() {
+            self._resetGroup();
+        });
     },
     
     // show
@@ -77,8 +78,7 @@ return Backbone.View.extend(
     },
     
     // reset
-    reset: function() {
-        this.group.reset();
+    _resetGroup: function() {
         this.group.set('id', Utils.uuid());
         this.group.set('label', 'Group label');
     },

config/plugins/visualizations/charts/static/views/groups.js

@@ -37,9 +37,9 @@ return Backbone.View.extend(
                             icon : 'fa-plus',
                             tooltip: 'Create',
                             onclick: function() {
+                                self.app.group.reset();
                                 self.app.chart_view.$el.hide();
                                 self.app.group_view.show();
-                                self.app.group_view.reset();
                             }
                         }),
                 'edit'  : new Ui.ButtonIcon({

lib/galaxy/datatypes/data.py

@@ -512,13 +512,14 @@ class Data( object ):
     def before_setting_metadata( self, dataset ):
         """This function is called on the dataset before metadata is set."""
         dataset.clear_associated_files( metadata_safe = True )
-    def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, space_to_tab = False, **kwds ):
+    def __new_composite_file( self, name, optional = False, mimetype = None, description = None, substitute_name_with_metadata = None, is_binary = False, to_posix_lines = True, space_to_tab = False, **kwds ):
         kwds[ 'name' ] = name
         kwds[ 'optional' ] = optional
         kwds[ 'mimetype' ] = mimetype
         kwds[ 'description' ] = description
         kwds[ 'substitute_name_with_metadata' ] = substitute_name_with_metadata
         kwds[ 'is_binary' ] = is_binary
+        kwds[ 'to_posix_lines' ] = to_posix_lines
         kwds[ 'space_to_tab' ] = space_to_tab
         return Bunch( **kwds )
     def add_composite_file( self, name, **kwds ):

lib/galaxy/tools/actions/upload_common.py

@@ -332,6 +332,7 @@ def create_paramfile( trans, uploaded_datasets ):
                          is_binary=is_binary,
                          link_data_only=link_data_only,
                          uuid=uuid_str,
+                         to_posix_lines=uploaded_dataset.to_posix_lines,
                          space_to_tab=uploaded_dataset.space_to_tab,
                          in_place=trans.app.config.external_chown_script is None,
                          path=uploaded_dataset.path )

lib/galaxy/tools/parameters/grouping.py

@@ -266,6 +266,9 @@ class UploadDataset( Group ):
             name = context.get( 'NAME', None )
             info = context.get( 'INFO', None )
             warnings = []
+            to_posix_lines = False
+            if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]:
+                to_posix_lines = True
             space_to_tab = False
             if context.get( 'space_to_tab', None ) not in [ "None", None, False ]:
                 space_to_tab = True
@@ -298,6 +301,7 @@ class UploadDataset( Group ):
                             break
                     if file_bunch.path:
                         break
+            file_bunch.to_posix_lines = to_posix_lines
             file_bunch.space_to_tab = space_to_tab
             return file_bunch, warnings
         def get_filenames( context ):
@@ -307,16 +311,21 @@ class UploadDataset( Group ):
             ftp_files = context['ftp_files']
             name = context.get( 'NAME', None )
             info = context.get( 'INFO', None )
+            to_posix_lines = False
+            if context.get( 'to_posix_lines', None ) not in [ "None", None, False ]:
+                to_posix_lines = True
             space_to_tab = False
             if context.get( 'space_to_tab', None ) not in [ "None", None, False ]:
                 space_to_tab = True
             warnings = []
             file_bunch = get_data_file_filename( data_file, override_name = name, override_info = info )
             if file_bunch.path:
+                file_bunch.to_posix_lines = to_posix_lines
                 file_bunch.space_to_tab = space_to_tab
                 rval.append( file_bunch )
             for file_bunch in get_url_paste_urls_or_filename( context, override_name = name, override_info = info ):
                 if file_bunch.path:
+                    file_bunch.to_posix_lines = to_posix_lines
                     file_bunch.space_to_tab = space_to_tab
                     rval.append( file_bunch )
             # look for files uploaded via FTP
@@ -390,11 +399,13 @@ class UploadDataset( Group ):
                 #replace sniff here with just creating an empty file
                 temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' )
                 dataset.primary_file = temp_name
+                dataset.to_posix_lines = True
                 dataset.space_to_tab = False
             else:
                 file_bunch, warnings = get_one_filename( groups_incoming[ 0 ] )
                 writable_files_offset = 1
                 dataset.primary_file = file_bunch.path
+                dataset.to_posix_lines = file_bunch.to_posix_lines
                 dataset.space_to_tab = file_bunch.space_to_tab
                 dataset.warnings.extend( warnings )
             if dataset.primary_file is None:#remove this before finish, this should create an empty dataset

lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py

@@ -524,7 +524,7 @@ class ToolShedRepositoriesController( BaseAPIController ):
             results[ 'repository_status' ].append( message )
         stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
         results[ 'stop_time' ] = stop_time
-        return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
+        return json.to_json_string( results, sort_keys=True, indent=4 )
 
     @web.expose_api
     def show( self, trans, id, **kwd ):

lib/galaxy/webapps/galaxy/api/tools.py

@@ -53,7 +53,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin, UsesHistoryMix
         link_details = util.string_as_bool( kwd.get( 'link_details', False ) )
         try:
             id = urllib.unquote_plus( id )
-            tool = self.app.toolbox.tools_by_id.get( id, None )
+            tool = self.app.toolbox.get_tool( id )
             if not tool:
                 trans.response.status = 404
                 return { 'error': 'tool not found', 'id': id }

lib/galaxy/webapps/galaxy/buildapp.py

@@ -129,7 +129,7 @@ def app_factory( global_conf, **kwargs ):
     webapp.mapper.resource( 'role', 'roles', path_prefix='/api' )
     webapp.mapper.resource( 'group', 'groups', path_prefix='/api' )
     webapp.mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' )
-    webapp.mapper.connect( '/api/tools/{id:.*?}', action='show', controller="tools" )
+    webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" )
     webapp.mapper.resource( 'tool', 'tools', path_prefix='/api' )
     webapp.mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' )
     webapp.mapper.resource( 'genome', 'genomes', path_prefix='/api' )

lib/galaxy/webapps/tool_shed/api/repositories.py

@@ -384,7 +384,7 @@ class RepositoriesController( BaseAPIController ):
                 results = handle_repository( trans, repository, results )
         stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
         results[ 'stop_time' ] = stop_time
-        return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
+        return json.to_json_string( results, sort_keys=True, indent=4 )
 
     @web.expose_api
     def reset_metadata_on_repository( self, trans, payload, **kwd ):
@@ -423,7 +423,7 @@ class RepositoriesController( BaseAPIController ):
             results = handle_repository( trans, start_time, repository )
             stop_time = strftime( "%Y-%m-%d %H:%M:%S" )
             results[ 'stop_time' ] = stop_time
-        return json.to_json_string( results, sort_keys=True, indent=4 * ' ' )
+        return json.to_json_string( results, sort_keys=True, indent=4 )
 
     @web.expose_api_anonymous
     def show( self, trans, id, **kwd ):

lib/tool_shed/util/shed_util_common.py

@@ -877,8 +877,7 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or
             username = trans.user.username
             clause_list = []
             for repository in trans.sa_session.query( trans.model.Repository ) \
-                                              .filter( and_( trans.model.Repository.table.c.deleted == False,
-                                                             trans.model.Repository.table.c.deprecated == False ) ):
+                                              .filter( trans.model.Repository.table.c.deleted == False ):
                 # Always reset metadata on all repositories of type tool_dependency_definition.
                 if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION:
                     clause_list.append( trans.model.Repository.table.c.id == repository.id )
@@ -905,14 +904,12 @@ def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, or
         else:
             if order:
                 return trans.sa_session.query( trans.model.Repository ) \
-                                       .filter( and_( trans.model.Repository.table.c.deleted == False,
-                                                      trans.model.Repository.table.c.deprecated == False ) ) \
+                                       .filter( trans.model.Repository.table.c.deleted == False ) \
                                        .order_by( trans.model.Repository.table.c.name,
                                                   trans.model.Repository.table.c.user_id )
             else:
                 return trans.sa_session.query( trans.model.Repository ) \
-                                       .filter( and_( trans.model.Repository.table.c.deleted == False,
-                                                      trans.model.Repository.table.c.deprecated == False ) )
+                                       .filter( trans.model.Repository.table.c.deleted == False )
     else:
         # We're in Galaxy.
         if order:
@@ -1599,8 +1596,8 @@ def open_repository_files_folder( trans, folder_path ):
     return folder_contents
 
 def pretty_print( dict=None ):
-    if dict:
-        return json.to_json_string( dict, sort_keys=True, indent=4 * ' ' )
+    if dict is not None:
+        return json.to_json_string( dict, sort_keys=True, indent=4 )
 
 def remove_dir( dir ):
     """Attempt to remove a directory from disk."""

test/functional/api/test_tools.py

@@ -33,6 +33,27 @@ class ToolsTestCase( api.ApiTestCase ):
         create_response = self._post( "tools", data=payload )
         self._assert_has_keys( create_response.json(), 'outputs' )
 
+    def test_upload_posix_newline_fixes( self ):
+        windows_content = "1\t2\t3\r4\t5\t6\r"
+        posix_content = windows_content.replace("\r", "\n")
+        result_content = self._upload_and_get_content( windows_content )
+        self.assertEquals( result_content, posix_content )
+
+    def test_upload_disable_posix_fix( self ):
+        windows_content = "1\t2\t3\r4\t5\t6\r"
+        result_content = self._upload_and_get_content( windows_content, to_posix_lines=None )
+        self.assertEquals( result_content, windows_content )
+
+    def test_upload_tab_to_space( self ):
+        table = "1 2 3\n4 5 6\n"
+        result_content = self._upload_and_get_content( table, space_to_tab="Yes" )
+        self.assertEquals( result_content, "1\t2\t3\n4\t5\t6\n" )
+
+    def test_upload_tab_to_space_off_by_default( self ):
+        table = "1 2 3\n4 5 6\n"
+        result_content = self._upload_and_get_content( table )
+        self.assertEquals( result_content, table )
+
     def test_run_cat1( self ):
         history_id = self._new_history()
         new_dataset = self._new_dataset( history_id )
@@ -52,6 +73,14 @@ class ToolsTestCase( api.ApiTestCase ):
         self._assert_has_keys( create_response.json(), 'outputs' )
         self._wait_for_history( history_id, assert_ok=True )
 
+    def _upload_and_get_content( self, content, **upload_kwds ):
+        history_id = self._new_history()
+        new_dataset = self._new_dataset( history_id, content=content, **upload_kwds )
+        self._wait_for_history( history_id, assert_ok=True )
+        display_response = self._get( "histories/%s/contents/%s/display" % ( history_id, new_dataset[ "id" ] ) )
+        self._assert_status_code_is( display_response, 200 )
+        return display_response.content
+
     def _new_dataset( self, history_id, content='TestData123', **kwds ):
         payload = self._upload_payload( history_id, content, **kwds )
         run_response = self._post( "tools", data=payload )
@@ -86,6 +115,10 @@ class ToolsTestCase( api.ApiTestCase ):
             'dbkey': dbkey,
             'file_type': file_type,
         }
+        if "to_posix_lines" in kwds:
+            upload_params[ "files_0|to_posix_lines"] = kwds[ "to_posix_lines" ]
+        if "space_to_tab" in kwds:
+            upload_params[ "files_0|space_to_tab" ] = kwds[ "space_to_tab" ]
         return self._run_tool_payload(
             tool_id='upload1',
             inputs=upload_params,

tools/data_source/upload.py

@@ -264,10 +264,14 @@ def add_file( dataset, registry, json_file, output_path ):
             if link_data_only == 'copy_files':
                 if dataset.type in ( 'server_dir', 'path_paste' ) and data_type not in [ 'gzip', 'bz2', 'zip' ]:
                     in_place = False
-                if dataset.space_to_tab:
-                    line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place )
-                else:
-                    line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place )
+                # Convert universal line endings to Posix line endings, but allow the user to turn it off,
+                # so that is becomes possible to upload gzip, bz2 or zip files with binary data without
+                # corrupting the content of those files.
+                if dataset.to_posix_lines:
+                    if dataset.space_to_tab:
+                        line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place )
+                    else:
+                        line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place )
             if dataset.file_type == 'auto':
                 ext = sniff.guess_ext( dataset.path, registry.sniff_order )
             else:

tools/data_source/upload.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0"?>
 
-<tool name="Upload File" id="upload1" version="1.1.3" workflow_compatible="false">
+<tool name="Upload File" id="upload1" version="1.1.4" workflow_compatible="false">
   <description>
     from your computer  
   </description>
@@ -38,6 +38,12 @@
       </param>
       <param name="url_paste" type="text" area="true" size="5x35" label="URL/Text" help="Here you may specify a list of URLs (one per line) or paste the contents of a file."/> 
       <param name="ftp_files" type="ftpfile" label="Files uploaded via FTP"/>
+      <!-- Change the following parameter from hidden to select to enable
+           modifying this option via the Web GUI. See Bitbucket pull request
+           171 for more information. -->
+      <param name="to_posix_lines" type="hidden" display="checkboxes" multiple="True" label="Convert universal line endings to Posix line endings" help="Turn this option off if you upload a gzip, bz2 or zip archive which contains a binary file." value="Yes"> 
+        <option value="Yes" selected="true">Yes</option>
+      </param>
       <param name="space_to_tab" type="select" display="checkboxes" multiple="True" label="Convert spaces to tabs" help="Use this option if you are entering intervals by hand."> 
         <option value="Yes">Yes</option>
       </param>