diff --git a/.ci/flake8_docstrings_include_list.txt b/.ci/flake8_docstrings_include_list.txt new file mode 100644 index 00000000000..1f5e96f7837 --- /dev/null +++ b/.ci/flake8_docstrings_include_list.txt @@ -0,0 +1,2 @@ +lib/galaxy/jobs/metrics +lib/galaxy/exceptions diff --git a/.ci/flake8_lint_include_list.txt b/.ci/flake8_lint_include_list.txt index 68de612b664..735434e27ce 100644 --- a/.ci/flake8_lint_include_list.txt +++ b/.ci/flake8_lint_include_list.txt @@ -516,54 +516,4 @@ test/unit/workflows/test_render.py test/unit/workflows/test_workflow_progress.py test/unit/test_objectstore.py tool_list.py -tools/data_source/fetch.py -tools/data_source/genbank.py -tools/data_source/hbvar_filter.py -tools/data_source/import.py -tools/data_source/microbial_import_code.py -tools/data_source/microbial_import.py -tools/data_source/upload.py -tools/evolution/ -tools/extract/liftOver_wrapper.py -tools/filters/axt_to_concat_fasta.py -tools/filters/axt_to_fasta.py -tools/filters/axt_to_lav_code.py -tools/filters/axt_to_lav.py -tools/filters/bed_to_gff_converter.py -tools/filters/catWrapper.py -tools/filters/convert_characters.py -tools/filters/gff/ -tools/filters/gff_to_bed_converter.py -tools/filters/gtf_to_bedgraph_converter.py -tools/filters/join.py -tools/filters/joinWrapper.py -tools/filters/lav_to_bed_code.py -tools/filters/lav_to_bed.py -tools/filters/mergeCols.py -tools/filters/randomlines.py -tools/filters/random_lines_two_pass.py -tools/filters/secure_hash_message_digest.py -tools/filters/sff_extract.py -tools/filters/sorter.py -tools/filters/trimmer.py -tools/filters/ucsc_gene_bed_to_exon_bed.py -tools/filters/ucsc_gene_bed_to_intron_bed.py -tools/filters/ucsc_gene_table_to_intervals.py -tools/filters/uniq.py -tools/filters/wiggle_to_simple.py -tools/genomespace/ -tools/maf/ -tools/meme/ -tools/metag_tools/ -tools/next_gen_conversion/fastq_conversions.py -tools/next_gen_conversion/fastq_gen_conv.py -tools/next_gen_conversion/solid_to_fastq.py -tools/ngs_simulation/ -tools/phenotype_association/ -tools/plotting/ -tools/solid_tools/ -tools/sr_assembly/ -tools/sr_mapping/ -tools/stats/grouping.py -tools/stats/gsummary.py -tools/visualization/ +tools/ diff --git a/.ci/flake8_wrapper.sh b/.ci/flake8_wrapper.sh index 4924f630a8a..47395c0c32f 100755 --- a/.ci/flake8_wrapper.sh +++ b/.ci/flake8_wrapper.sh @@ -5,4 +5,4 @@ set -e flake8 --exclude `paste -sd, .ci/flake8_blacklist.txt` . # Apply stricter rules for the directories shared with Pulsar -flake8 --ignore= --max-line-length=150 lib/galaxy/jobs/runners/util/ +flake8 --ignore=D --max-line-length=150 lib/galaxy/jobs/runners/util/ diff --git a/.ci/flake8_wrapper_docstrings.sh b/.ci/flake8_wrapper_docstrings.sh new file mode 100755 index 00000000000..4fe943cdf42 --- /dev/null +++ b/.ci/flake8_wrapper_docstrings.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -e + +# D100 - Missing docstring in public module. +# D2XX - Whitespace issues. +# D3XX - Quoting issues. +# D401 - First line should be in imperative mood +# D403 - First word of the first line should be properly capitalized +args="--ignore=D --select=D100,D201,D202,D206,D207,D208,D209,D211,D3,D401,D403" + +# If the first argument is --include, lint the modules expected to pass. If +# the first argument is --exclude, lint all modules the full Galaxy linter lints +# (this will fail). + +if [ "$1" = "--include" ]; +then + flake8 $args `paste .ci/flake8_docstrings_include_list.txt` +else + flake8 $args --exclude `paste -sd, .ci/flake8_blacklist.txt` . +fi diff --git a/.ci/py3_sources.txt b/.ci/py3_sources.txt index 30f9210650e..d9f47f7b999 100644 --- a/.ci/py3_sources.txt +++ b/.ci/py3_sources.txt @@ -72,8 +72,4 @@ scripts/db_shell.py scripts/drmaa_external_runner.py test/ tool_list.py -tools/data_source/ -tools/evolution/ -tools/sr_mapping/ -tools/stats/aggregate_scores_in_intervals.py -tools/visualization/ +tools/ diff --git a/.travis.yml b/.travis.yml index e175bf05b86..a128bb9f751 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,6 +11,7 @@ env: - TOX_ENV=py27-lint-imports - TOX_ENV=py27-lint-imports-include-list - TOX_ENV=validate-test-tools + - TOX_ENV=py27-lint-docstring-include-list matrix: include: diff --git a/client/galaxy/scripts/mvc/tool/tool-form-composite.js b/client/galaxy/scripts/mvc/tool/tool-form-composite.js index e5049d1fbbf..dab85588bb8 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-composite.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-composite.js @@ -77,7 +77,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view' _.each( this.steps, function( step, i ) { _.each( step.output_connections, function( output_connection ) { _.each( self.steps, function( sub_step, j ) { - sub_step.step_id === output_connection.input_step_id && self.links[ i ].push( sub_step ); + sub_step.step_index === output_connection.input_step_index && self.links[ i ].push( sub_step ); }); }); }); @@ -88,7 +88,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view' _.each( self.steps, function( sub_step, j ) { var connections_by_name = {}; _.each( step.output_connections, function( connection ) { - sub_step.step_id === connection.input_step_id && ( connections_by_name[ connection.input_name ] = connection ); + sub_step.step_index === connection.input_step_index && ( connections_by_name[ connection.input_name ] = connection ); }); _.each( self.parms[ j ], function( input, name ) { var connection = connections_by_name[ name ]; @@ -224,34 +224,31 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view' /** Render workflow parameters */ _renderHistory: function() { - this.history_form = null; - if ( !this.model.get( 'history_id' ) ) { - this.history_form = new Form({ - cls : 'ui-portlet-narrow', - title : 'History Options', - inputs : [{ - type : 'conditional', - name : 'new_history', - test_param : { - name : 'check', - label : 'Send results to a new history', - type : 'boolean', - value : 'false', - help : '' - }, - cases : [{ - value : 'true', - inputs : [{ - name : 'name', - label : 'History name', - type : 'text', - value : this.model.get( 'name' ) - }] + this.history_form = new Form({ + cls : 'ui-portlet-narrow', + title : 'History Options', + inputs : [{ + type : 'conditional', + name : 'new_history', + test_param : { + name : 'check', + label : 'Send results to a new history', + type : 'boolean', + value : 'false', + help : '' + }, + cases : [{ + value : 'true', + inputs : [{ + name : 'name', + label : 'History name', + type : 'text', + value : this.model.get( 'name' ) }] }] - }); - this._append( this.$steps, this.history_form.$el ); - } + }] + }); + this._append( this.$steps, this.history_form.$el ); }, /** Render step */ @@ -355,17 +352,26 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view' /** Validate and submit workflow */ _submit: function() { var self = this; + var history_form_data = this.history_form.data.create(); var job_def = { - new_history_name : this.history_form ? this.history_form.data.create()[ 'new_history|name' ] : {}, - replacement_params : this.wp_form ? this.wp_form.data.create() : {}, - inputs : {} + new_history_name : history_form_data[ 'new_history|name' ] ? history_form_data[ 'new_history|name' ] : null, + history_id : !history_form_data[ 'new_history|name' ] ? this.model.get( 'history_id' ) : null, + replacement_params : this.wp_form ? this.wp_form.data.create() : {}, + parameters : {}, + // Tool form will submit flat maps for each parameter + // (e.g. "repeat_0|cond|param": "foo" instead of nested + // data structures). + parameters_normalized : true, + // Tool form always wants a list of invocations back + // so that inputs can be batched. + batch : true }; var validated = true; for ( var i in this.forms ) { var form = this.forms[ i ]; var job_inputs = form.data.create(); var step = self.steps[ i ]; - var step_id = step.step_id; + var step_index = step.step_index; form.trigger( 'reset' ); for ( var job_input_id in job_inputs ) { var input_value = job_inputs[ job_input_id ]; @@ -382,8 +388,8 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view' form.highlight( input_id ); break; } - job_def.inputs[ step_id ] = job_def.inputs[ step_id ] || {}; - job_def.inputs[ step_id ][ job_input_id ] = job_inputs[ job_input_id ]; + job_def.parameters[ step_index ] = job_def.parameters[ step_index ] || {}; + job_def.parameters[ step_index ][ job_input_id ] = job_inputs[ job_input_id ]; } } if ( !validated ) { @@ -397,31 +403,35 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view' Galaxy.emit.debug( 'tool-form-composite::submit()', 'Validation complete.', job_def ); Utils.request({ type : 'POST', - url : Galaxy.root + 'api_internal/workflows/' + this.model.id + '/run', + url : Galaxy.root + 'api/workflows/' + this.model.id + '/invocations', data : job_def, success : function( response ) { Galaxy.emit.debug( 'tool-form-composite::submit', 'Submission successful.', response ); - self.$el.empty().append( self._templateSuccess( response ) ); + self.$el.children().hide(); + self.$el.append( self._templateSuccess( response ) ); self._refreshHistory(); }, error : function( response ) { Galaxy.emit.debug( 'tool-form-composite::submit', 'Submission failed.', response ); + var input_found = false; if ( response && response.err_data ) { for ( var i in self.forms ) { var form = self.forms[ i ]; - var step_related_errors = response.err_data[ form.options.step_id ]; + var step_related_errors = response.err_data[ form.options.step_index ]; if ( step_related_errors ) { var error_messages = form.data.matchResponse( step_related_errors ); for ( var input_id in error_messages ) { form.highlight( input_id, error_messages[ input_id ] ); + input_found = true; break; } } } - } else { + } + if ( !input_found ) { self.modal.show({ - title : 'Job submission failed', - body : self._templateError( response && response.err_msg || job_def ), + title : 'Workflow submission failed', + body : self._templateError( job_def, response && response.err_msg ), buttons : { 'Close' : function() { self.modal.hide(); @@ -471,17 +481,19 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view' /** Templates */ _templateSuccess: function( response ) { - if ( response && response.length > 0 ) { - return $( '
' ).addClass( 'donemessagelarge' ).append( $( '

' ).html( 'Successfully invoked workflow ' + Utils.sanitize( this.model.get( 'name' ) ) + '' + ( response.length > 1 ? ' ' + response.length + ' times' : '' ) + '. Datasets will appear as jobs are created.' ) ); + if ( $.isArray( response ) && response.length > 0 ) { + return $( '

' ).addClass( 'donemessagelarge' ) + .append( $( '

' ).html( 'Successfully invoked workflow ' + Utils.sanitize( this.model.get( 'name' ) ) + '' + ( response.length > 1 ? ' ' + response.length + ' times' : '' ) + '.' ) ) + .append( $( '

' ).append( '' ).text( 'You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from \'running\' to \'finished\' if completed successfully or \'error\' if problems were encountered.' ) ); } else { - return this._templateError( response ); + return this._templateError( response, 'Invalid success response. No invocations found.' ); } }, - _templateError: function( response ) { + _templateError: function( response, err_msg ) { return $( '

' ).addClass( 'errormessagelarge' ) - .append( $( '

' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists.' ) ) - .append( $( '

' ).text( JSON.stringify( response, null, 4 ) ) );
+                                 .append( $( '

' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists. ' + ( JSON.stringify( err_msg ) || '' ) ) ) + .append( $( '

' ).text( JSON.stringify( response, null, 4 ) ) );
         }
     });
     return {
diff --git a/client/galaxy/scripts/mvc/tool/tool-form.js b/client/galaxy/scripts/mvc/tool/tool-form.js
index 693f0617452..a60574f753e 100644
--- a/client/galaxy/scripts/mvc/tool/tool-form.js
+++ b/client/galaxy/scripts/mvc/tool/tool-form.js
@@ -88,7 +88,7 @@ define([ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-modal', 'mvc/tool/tool-form
                     var input_found = false;
                     if ( response && response.err_data ) {
                         var error_messages = self.form.data.matchResponse( response.err_data );
-                        for (var input_id in error_messages) {
+                        for ( var input_id in error_messages ) {
                             self.form.highlight( input_id, error_messages[ input_id ]);
                             input_found = true;
                             break;
@@ -97,7 +97,7 @@ define([ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-modal', 'mvc/tool/tool-form
                     if ( !input_found ) {
                         self.modal.show({
                             title   : 'Job submission failed',
-                            body    : ( response && response.err_msg ) || self._templateError( job_def ),
+                            body    : self._templateError( job_def, response && response.err_msg ),
                             buttons : { 'Close' : function() { self.modal.hide() } }
                         });
                     }
@@ -159,13 +159,13 @@ define([ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-modal', 'mvc/tool/tool-form
                 $message.append( $( '

' ).append( '' ).text( 'You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from \'running\' to \'finished\' if completed successfully or \'error\' if problems were encountered.' ) ); return $message; } else { - return this._templateError( response ); + return this._templateError( response, 'Invalid success response. No jobs found.' ); } }, - _templateError: function( response ) { + _templateError: function( response, err_msg ) { return $( '

' ).addClass( 'errormessagelarge' ) - .append( $( '

' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists.' ) ) + .append( $( '

' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists. ' + ( err_msg || '' ) ) ) .append( $( '

' ).text( JSON.stringify( response, null, 4 ) ) );
         }
     });
diff --git a/client/galaxy/scripts/mvc/ui/ui-modal.js b/client/galaxy/scripts/mvc/ui/ui-modal.js
index 6bc52fc0f09..73adf7ba8c7 100644
--- a/client/galaxy/scripts/mvc/ui/ui-modal.js
+++ b/client/galaxy/scripts/mvc/ui/ui-modal.js
@@ -1,5 +1,8 @@
 define([], function() {
 var View = Backbone.View.extend({
+
+    className: "ui-modal",
+
     // defaults
     optionsDefault  : {
         container        : 'body',
@@ -19,18 +22,8 @@ var View = Backbone.View.extend({
 
     // initialize
     initialize: function( options ) {
-        this.setElement( this._template() );
         this.options = _.defaults( options || {}, this.optionsDefault );
         $( this.options.container ).prepend( this.el );
-
-        // link elements
-        this.$header    = this.$( '.modal-header' );
-        this.$dialog    = this.$( '.modal-dialog' );
-        this.$body      = this.$( '.modal-body' );
-        this.$footer    = this.$( '.modal-footer' );
-        this.$backdrop  = this.$( '.modal-backdrop' );
-        this.$buttons   = this.$( '.buttons' );
-
         // optional render
         options && this.render();
     },
@@ -70,6 +63,16 @@ var View = Backbone.View.extend({
     */
     render: function() {
         var self = this;
+        this.$el.html( this._template() );
+
+        // link elements
+        this.$header    = this.$( '.modal-header' );
+        this.$dialog    = this.$( '.modal-dialog' );
+        this.$body      = this.$( '.modal-body' );
+        this.$footer    = this.$( '.modal-footer' );
+        this.$backdrop  = this.$( '.modal-backdrop' );
+        this.$buttons   = this.$( '.buttons' );
+
         if (this.options.body == 'progress') {
             this.options.body = $(  '
' + '
' + @@ -164,17 +167,15 @@ var View = Backbone.View.extend({ * Returns the modal template */ _template: function() { - return '
' + - '