Merge branch 'dev' into remote_chart_plugins

.ci/flake8_docstrings_include_list.txt

@@ -0,0 +1,2 @@
+lib/galaxy/jobs/metrics
+lib/galaxy/exceptions

.ci/flake8_lint_include_list.txt

@@ -516,54 +516,4 @@ test/unit/workflows/test_render.py
 test/unit/workflows/test_workflow_progress.py
 test/unit/test_objectstore.py
 tool_list.py
-tools/data_source/fetch.py
-tools/data_source/genbank.py
-tools/data_source/hbvar_filter.py
-tools/data_source/import.py
-tools/data_source/microbial_import_code.py
-tools/data_source/microbial_import.py
-tools/data_source/upload.py
-tools/evolution/
-tools/extract/liftOver_wrapper.py
-tools/filters/axt_to_concat_fasta.py
-tools/filters/axt_to_fasta.py
-tools/filters/axt_to_lav_code.py
-tools/filters/axt_to_lav.py
-tools/filters/bed_to_gff_converter.py
-tools/filters/catWrapper.py
-tools/filters/convert_characters.py
-tools/filters/gff/
-tools/filters/gff_to_bed_converter.py
-tools/filters/gtf_to_bedgraph_converter.py
-tools/filters/join.py
-tools/filters/joinWrapper.py
-tools/filters/lav_to_bed_code.py
-tools/filters/lav_to_bed.py
-tools/filters/mergeCols.py
-tools/filters/randomlines.py
-tools/filters/random_lines_two_pass.py
-tools/filters/secure_hash_message_digest.py
-tools/filters/sff_extract.py
-tools/filters/sorter.py
-tools/filters/trimmer.py
-tools/filters/ucsc_gene_bed_to_exon_bed.py
-tools/filters/ucsc_gene_bed_to_intron_bed.py
-tools/filters/ucsc_gene_table_to_intervals.py
-tools/filters/uniq.py
-tools/filters/wiggle_to_simple.py
-tools/genomespace/
-tools/maf/
-tools/meme/
-tools/metag_tools/
-tools/next_gen_conversion/fastq_conversions.py
-tools/next_gen_conversion/fastq_gen_conv.py
-tools/next_gen_conversion/solid_to_fastq.py
-tools/ngs_simulation/
-tools/phenotype_association/
-tools/plotting/
-tools/solid_tools/
-tools/sr_assembly/
-tools/sr_mapping/
-tools/stats/grouping.py
-tools/stats/gsummary.py
-tools/visualization/
+tools/

.ci/flake8_wrapper.sh

@@ -5,4 +5,4 @@ set -e
 flake8 --exclude `paste -sd, .ci/flake8_blacklist.txt` .
 
 # Apply stricter rules for the directories shared with Pulsar
-flake8 --ignore= --max-line-length=150 lib/galaxy/jobs/runners/util/
+flake8 --ignore=D --max-line-length=150 lib/galaxy/jobs/runners/util/

.ci/flake8_wrapper_docstrings.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+set -e
+
+# D100 - Missing docstring in public module.
+# D2XX - Whitespace issues.
+# D3XX - Quoting issues.
+# D401 - First line should be in imperative mood 
+# D403 - First word of the first line should be properly capitalized
+args="--ignore=D --select=D100,D201,D202,D206,D207,D208,D209,D211,D3,D401,D403"
+
+# If the first argument is --include, lint the modules expected to pass. If
+# the first argument is --exclude, lint all modules the full Galaxy linter lints
+# (this will fail).
+
+if [ "$1" = "--include" ];
+then
+    flake8 $args `paste .ci/flake8_docstrings_include_list.txt`
+else
+    flake8 $args --exclude `paste -sd, .ci/flake8_blacklist.txt` .
+fi

.ci/py3_sources.txt

@@ -72,8 +72,4 @@ scripts/db_shell.py
 scripts/drmaa_external_runner.py
 test/
 tool_list.py
-tools/data_source/
-tools/evolution/
-tools/sr_mapping/
-tools/stats/aggregate_scores_in_intervals.py
-tools/visualization/
+tools/

.travis.yml

@@ -11,6 +11,7 @@ env:
   - TOX_ENV=py27-lint-imports
   - TOX_ENV=py27-lint-imports-include-list
   - TOX_ENV=validate-test-tools
+  - TOX_ENV=py27-lint-docstring-include-list
 
 matrix:
   include:

client/galaxy/scripts/mvc/tool/tool-form-composite.js

@@ -77,7 +77,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
             _.each( this.steps, function( step, i ) {
                 _.each( step.output_connections, function( output_connection ) {
                     _.each( self.steps, function( sub_step, j ) {
-                        sub_step.step_id === output_connection.input_step_id && self.links[ i ].push( sub_step );
+                        sub_step.step_index === output_connection.input_step_index && self.links[ i ].push( sub_step );
                     });
                 });
             });
@@ -88,7 +88,7 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
                 _.each( self.steps, function( sub_step, j ) {
                     var connections_by_name = {};
                     _.each( step.output_connections, function( connection ) {
-                        sub_step.step_id === connection.input_step_id && ( connections_by_name[ connection.input_name ] = connection );
+                        sub_step.step_index === connection.input_step_index && ( connections_by_name[ connection.input_name ] = connection );
                     });
                     _.each( self.parms[ j ], function( input, name ) {
                         var connection = connections_by_name[ name ];
@@ -224,34 +224,31 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
 
         /** Render workflow parameters */
         _renderHistory: function() {
-            this.history_form = null;
-            if ( !this.model.get( 'history_id' ) ) {
-                this.history_form = new Form({
-                    cls    : 'ui-portlet-narrow',
-                    title  : '<b>History Options</b>',
-                    inputs : [{
-                        type        : 'conditional',
-                        name        : 'new_history',
-                        test_param  : {
-                            name        : 'check',
-                            label       : 'Send results to a new history',
-                            type        : 'boolean',
-                            value       : 'false',
-                            help        : ''
-                        },
-                        cases       : [{
-                            value   : 'true',
-                            inputs  : [{
-                                name    : 'name',
-                                label   : 'History name',
-                                type    : 'text',
-                                value   : this.model.get( 'name' )
-                            }]
+            this.history_form = new Form({
+                cls    : 'ui-portlet-narrow',
+                title  : '<b>History Options</b>',
+                inputs : [{
+                    type        : 'conditional',
+                    name        : 'new_history',
+                    test_param  : {
+                        name        : 'check',
+                        label       : 'Send results to a new history',
+                        type        : 'boolean',
+                        value       : 'false',
+                        help        : ''
+                    },
+                    cases       : [{
+                        value   : 'true',
+                        inputs  : [{
+                            name    : 'name',
+                            label   : 'History name',
+                            type    : 'text',
+                            value   : this.model.get( 'name' )
                         }]
                     }]
-                });
-                this._append( this.$steps, this.history_form.$el );
-            }
+                }]
+            });
+            this._append( this.$steps, this.history_form.$el );
         },
 
         /** Render step */
@@ -355,17 +352,26 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
         /** Validate and submit workflow */
         _submit: function() {
             var self = this;
+            var history_form_data = this.history_form.data.create();
             var job_def = {
-                new_history_name    : this.history_form ? this.history_form.data.create()[ 'new_history|name' ] : {},
-                replacement_params  : this.wp_form ? this.wp_form.data.create() : {},
-                inputs              : {}
+                new_history_name      : history_form_data[ 'new_history|name' ] ? history_form_data[ 'new_history|name' ] : null,
+                history_id            : !history_form_data[ 'new_history|name' ] ? this.model.get( 'history_id' ) : null,
+                replacement_params    : this.wp_form ? this.wp_form.data.create() : {},
+                parameters            : {},
+                // Tool form will submit flat maps for each parameter
+                // (e.g. "repeat_0|cond|param": "foo" instead of nested
+                // data structures).
+                parameters_normalized : true,
+                // Tool form always wants a list of invocations back
+                // so that inputs can be batched.
+                batch                 : true
             };
             var validated = true;
             for ( var i in this.forms ) {
                 var form = this.forms[ i ];
                 var job_inputs  = form.data.create();
                 var step        = self.steps[ i ];
-                var step_id     = step.step_id;
+                var step_index  = step.step_index;
                 form.trigger( 'reset' );
                 for ( var job_input_id in job_inputs ) {
                     var input_value = job_inputs[ job_input_id ];
@@ -382,8 +388,8 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
                             form.highlight( input_id );
                             break;
                         }
-                        job_def.inputs[ step_id ] = job_def.inputs[ step_id ] || {};
-                        job_def.inputs[ step_id ][ job_input_id ] = job_inputs[ job_input_id ];
+                        job_def.parameters[ step_index ] = job_def.parameters[ step_index ] || {};
+                        job_def.parameters[ step_index ][ job_input_id ] = job_inputs[ job_input_id ];
                     }
                 }
                 if ( !validated ) {
@@ -397,31 +403,35 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
                 Galaxy.emit.debug( 'tool-form-composite::submit()', 'Validation complete.', job_def );
                 Utils.request({
                     type    : 'POST',
-                    url     : Galaxy.root + 'api_internal/workflows/' + this.model.id + '/run',
+                    url     : Galaxy.root + 'api/workflows/' + this.model.id + '/invocations',
                     data    : job_def,
                     success : function( response ) {
                         Galaxy.emit.debug( 'tool-form-composite::submit', 'Submission successful.', response );
-                        self.$el.empty().append( self._templateSuccess( response ) );
+                        self.$el.children().hide();
+                        self.$el.append( self._templateSuccess( response ) );
                         self._refreshHistory();
                     },
                     error   : function( response ) {
                         Galaxy.emit.debug( 'tool-form-composite::submit', 'Submission failed.', response );
+                        var input_found = false;
                         if ( response && response.err_data ) {
                             for ( var i in self.forms ) {
                                 var form = self.forms[ i ];
-                                var step_related_errors = response.err_data[ form.options.step_id ];
+                                var step_related_errors = response.err_data[ form.options.step_index ];
                                 if ( step_related_errors ) {
                                     var error_messages = form.data.matchResponse( step_related_errors );
                                     for ( var input_id in error_messages ) {
                                         form.highlight( input_id, error_messages[ input_id ] );
+                                        input_found = true;
                                         break;
                                     }
                                 }
                             }
-                        } else {
+                        }
+                        if ( !input_found ) {
                             self.modal.show({
-                                title   : 'Job submission failed',
-                                body    : self._templateError( response && response.err_msg || job_def ),
+                                title   : 'Workflow submission failed',
+                                body    : self._templateError( job_def, response && response.err_msg ),
                                 buttons : {
                                     'Close' : function() {
                                         self.modal.hide();
@@ -471,17 +481,19 @@ define([ 'utils/utils', 'utils/deferred', 'mvc/ui/ui-misc', 'mvc/form/form-view'
 
         /** Templates */
         _templateSuccess: function( response ) {
-            if ( response && response.length > 0 ) {
-                return $( '<div/>' ).addClass( 'donemessagelarge' ).append( $( '<p/>' ).html( 'Successfully invoked workflow <b>' + Utils.sanitize( this.model.get( 'name' ) ) + '</b>' + ( response.length > 1 ? ' <b>' + response.length + ' times</b>' : '' ) + '. Datasets will appear as jobs are created.' ) );
+            if ( $.isArray( response ) && response.length > 0 ) {
+                return $( '<div/>' ).addClass( 'donemessagelarge' )
+                                    .append( $( '<p/>' ).html( 'Successfully invoked workflow <b>' + Utils.sanitize( this.model.get( 'name' ) ) + '</b>' + ( response.length > 1 ? ' <b>' + response.length + ' times</b>' : '' ) + '.' ) )
+                                    .append( $( '<p/>' ).append( '<b/>' ).text( 'You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from \'running\' to \'finished\' if completed successfully or \'error\' if problems were encountered.' ) );
             } else {
-                return this._templateError( response );
+                return this._templateError( response, 'Invalid success response. No invocations found.' );
             }
         },
 
-        _templateError: function( response ) {
+        _templateError: function( response, err_msg ) {
             return  $( '<div/>' ).addClass( 'errormessagelarge' )
-                .append( $( '<p/>' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists.' ) )
-                .append( $( '<pre/>' ).text( JSON.stringify( response, null, 4 ) ) );
+                                 .append( $( '<p/>' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists. ' + ( JSON.stringify( err_msg ) || '' ) ) )
+                                 .append( $( '<pre/>' ).text( JSON.stringify( response, null, 4 ) ) );
         }
     });
     return {

client/galaxy/scripts/mvc/tool/tool-form.js

@@ -88,7 +88,7 @@ define([ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-modal', 'mvc/tool/tool-form
                     var input_found = false;
                     if ( response && response.err_data ) {
                         var error_messages = self.form.data.matchResponse( response.err_data );
-                        for (var input_id in error_messages) {
+                        for ( var input_id in error_messages ) {
                             self.form.highlight( input_id, error_messages[ input_id ]);
                             input_found = true;
                             break;
@@ -97,7 +97,7 @@ define([ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-modal', 'mvc/tool/tool-form
                     if ( !input_found ) {
                         self.modal.show({
                             title   : 'Job submission failed',
-                            body    : ( response && response.err_msg ) || self._templateError( job_def ),
+                            body    : self._templateError( job_def, response && response.err_msg ),
                             buttons : { 'Close' : function() { self.modal.hide() } }
                         });
                     }
@@ -159,13 +159,13 @@ define([ 'utils/utils', 'mvc/ui/ui-misc', 'mvc/ui/ui-modal', 'mvc/tool/tool-form
                 $message.append( $( '<p/>' ).append( '<b/>' ).text( 'You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from \'running\' to \'finished\' if completed successfully or \'error\' if problems were encountered.' ) );
                 return $message;
             } else {
-                return this._templateError( response );
+                return this._templateError( response, 'Invalid success response. No jobs found.' );
             }
         },
 
-        _templateError: function( response ) {
+        _templateError: function( response, err_msg ) {
             return  $( '<div/>' ).addClass( 'errormessagelarge' )
-                                 .append( $( '<p/>' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists.' ) )
+                                 .append( $( '<p/>' ).text( 'The server could not complete the request. Please contact the Galaxy Team if this error persists. ' + ( err_msg || '' ) ) )
                                  .append( $( '<pre/>' ).text( JSON.stringify( response, null, 4 ) ) );
         }
     });

client/galaxy/scripts/mvc/ui/ui-modal.js

@@ -1,5 +1,8 @@
 define([], function() {
 var View = Backbone.View.extend({
+
+    className: "ui-modal",
+
     // defaults
     optionsDefault  : {
         container        : 'body',
@@ -19,18 +22,8 @@ var View = Backbone.View.extend({
 
     // initialize
     initialize: function( options ) {
-        this.setElement( this._template() );
         this.options = _.defaults( options || {}, this.optionsDefault );
         $( this.options.container ).prepend( this.el );
-
-        // link elements
-        this.$header    = this.$( '.modal-header' );
-        this.$dialog    = this.$( '.modal-dialog' );
-        this.$body      = this.$( '.modal-body' );
-        this.$footer    = this.$( '.modal-footer' );
-        this.$backdrop  = this.$( '.modal-backdrop' );
-        this.$buttons   = this.$( '.buttons' );
-
         // optional render
         options && this.render();
     },
@@ -70,6 +63,16 @@ var View = Backbone.View.extend({
     */
     render: function() {
         var self = this;
+        this.$el.html( this._template() );
+
+        // link elements
+        this.$header    = this.$( '.modal-header' );
+        this.$dialog    = this.$( '.modal-dialog' );
+        this.$body      = this.$( '.modal-body' );
+        this.$footer    = this.$( '.modal-footer' );
+        this.$backdrop  = this.$( '.modal-backdrop' );
+        this.$buttons   = this.$( '.buttons' );
+
         if (this.options.body == 'progress') {
             this.options.body = $(  '<div class="progress progress-striped active">' +
                                         '<div class="progress-bar progress-bar-info" style="width:100%"/>' +
@@ -164,17 +167,15 @@ var View = Backbone.View.extend({
      * Returns the modal template
     */
     _template: function() {
-        return  '<div class="ui-modal">' +
-                    '<div class="modal-backdrop fade"/>' +
-                    '<div class="modal-dialog">' +
-                        '<div class="modal-content">' +
-                            '<div class="modal-header">' +
-                                '<h4 class="title"/>' +
-                            '</div>' +
-                            '<div class="modal-body"/>' +
-                            '<div class="modal-footer">' +
-                                '<div class="buttons"/>' +
-                            '</div>' +
+        return  '<div class="modal-backdrop fade"/>' +
+                '<div class="modal-dialog">' +
+                    '<div class="modal-content">' +
+                        '<div class="modal-header">' +
+                            '<h4 class="title"/>' +
+                        '</div>' +
+                        '<div class="modal-body"/>' +
+                        '<div class="modal-footer">' +
+                            '<div class="buttons"/>' +
                         '</div>' +
                     '</div>' +
                 '</div>';

config/job_conf.xml.sample_advanced

@@ -205,6 +205,19 @@
                  zero (no execution) and the stderr/stdout of the k8s job is reported in galaxy (and the galaxy job set
                  to failed) -->
         </plugin>
+        <plugin id="godocker" type="runner" load="galaxy.jobs.runners.godocker:GodockerJobRunner">
+            <!-- Go-Docker is a batch computing/cluster management tool using Docker
+                 See https://bitbucket.org/osallou/go-docker for more details.  -->
+            <!--  REST based runner , submits jobs to godocker -->
+            <param id="godocker_master">GODOCKER_URL</param>
+            <!-- Specify the instance of GoDocker -->
+            <param id="user">USERNAME</param>
+            <!-- GoDocker username -->
+            <param id="key">APIKEY</param>
+            <!-- GoDocker API key -->
+            <param id="godocker_project">galaxy</param>
+            <!-- Specify the project present in the GoDocker setup -->
+        </plugin>
 
     </plugins>
     <handlers default="handlers">
@@ -647,7 +660,24 @@
                  internally by the runner. -->
             <param id="docker_enabled">true</param>
         </destination>
-
+        <destination id="god" runner="godocker">
+            <!-- The following are configurations for the container -->
+            <param id="docker_enabled">true</param>
+            <param id="docker_cpu">1</param>
+            <param id="docker_memory">2</param>
+            <param id="docker_default_container_id">centos:latest</param>
+            <!-- Specify the image on which the jobs have to be executed -->
+            <param id="godocker_volumes"></param>
+            <!-- Mount the godocker volumes 
+                 volumes must be separated by commas.
+                 eg: <param id="godocker_volumes">home,galaxy</param>
+            -->
+            <param id="virtualenv">false</param>
+            <!-- If a tool execution in container requires galaxy virtualenv, 
+                 then enable it by setting the value to true.
+                 Disable venv by setting the value to false.
+            -->
+        </destination>
 
         <!-- Templatized destinations - macros can be used to create templated
         destinations with reduced XML duplication. Here we are creating 4 destinations in 4 lines instead of 28 using the macros defined below.

doc/source/admin/conda_faq.rst

@@ -50,13 +50,17 @@ Below we answer some common questions (collected by Lance Parsons):
 
 Galaxy's dependency job resolution is managed via
 ``dependency_resolvers_conf.xml`` configuration file. Most Galaxy administrators
-have not set up a dependency resolvers configuration file, which means they are
-using Galaxy's default ( ``dependency_resolvers_conf.xml.sample`` ). With
-release 16.04, Galaxy has enabled Conda dependency resolution by default  when
+should be using Galaxy's default dependency resolvers configuration file
+( ``dependency_resolvers_conf.xml.sample`` ). With
+release 16.04, Galaxy has enabled Conda dependency resolution by default when
 Conda was already installed on the system. Having Conda enabled in
-``dependency_resolvers_conf.xml`` means that Galaxy will look for job
+``dependency_resolvers_conf.xml`` means that Galaxy can look for job
 dependencies using the Conda system when it attempts to run tools.
 
+Note that the order of resolvers in the file matters and the ``<tool_shed_packages />``
+entry should remain first. This means that tools that have specified Tool Shed packages
+as their dependencies will work without a change.
+
 The most common configuration settings related to Conda are listed in Table 1.
 See `galaxy.ini.sample`_ for the complete list.
 
@@ -101,7 +105,7 @@ See `galaxy.ini.sample`_ for the complete list.
 2. How do Conda dependencies work? Where do things get installed?
 *********************************************************************************
 
-In contrast to the old dependency system, which was used exclusively by Galaxy,
+In contrast to the TS dependency system, which was used exclusively by Galaxy,
 Conda is a pre-existing, independent project. With Conda, it is possible for an
 admin to install and manage packages without touching Galaxy at all. Galaxy can
 handle these dependencies for you, but admins are not required to use Galaxy for
@@ -170,19 +174,28 @@ systems newer than 2007.
 This depends on your ``galaxy.ini`` setting. Starting with release 16.07, Galaxy
 can automatically install the Conda package manager for you if you have enabled
 ``conda_auto_init``. Galaxy can then install Trinity along with its dependencies
-using one of the methods  listed in question 2 above. Further, if
-``conda_auto_install`` is enabled, Galaxy will install Trinity via Conda only
-when a Trinity job is launched and Trinity is not yet installed.
+using one of the methods listed in question 2 above. In particular, if
+``conda_auto_install`` is True and Trinity is not installed yet, Galaxy will try
+to install it via Conda when a Trinity job is launched.
 
 With release 16.07 you can see which dependencies are being used
 in the “Manage installed tools” section of the Admin panel and you can select
 whether or not to install Conda packages or Tool Shed package recipes when you
 install new tools there, even if ``conda_auto_install`` is disabled.
 
-More improvements to the UI will be coming in future releases. To see if Galaxy
-has created a Trinity environment for you have a look at folder under
+During a tool installation, the Galaxy admin has control over which systems will be used to
+install the tool requirements. The default settings will trigger installation
+of both TS and Conda packages (if Conda is present), thus depending on the
+dependency resolvers configuration with regards to what will actually be used during
+the tool execution.
+
+To check if Galaxy has created a Trinity environment, have a look at folders under
 ``<tool_dependency_dir>/_conda/envs/``(or ``<conda_prefix>/envs`` if you have changed `conda_prefix` in your galaxy.ini file).
 
+We recommend to use Conda on a tool-per-tool basis, by unchecking the checkbox
+for TS dependencies during the tool installation, and for tools where there
+are no available TS dependencies.
+
 
 5. Can I mix traditional Galaxy packages and Conda packages?
 *********************************************************************************
@@ -198,7 +211,8 @@ The order in which resolvers are tried is listed in the
 -  Packages manually installed by administrators
 -  Conda packages
 
-The first system that satisfies a requirement will be used.
+The first system that satisfies a requirement will be used. See
+`resolver docs`_ for detailed documentation.
 
 
 6. How do I know what system is being used by a given tool?
@@ -206,10 +220,7 @@ The first system that satisfies a requirement will be used.
 
 The Galaxy log will show which dependency resolution system is used
 to satisfy each tool dependency and you can specify priorities using the
-``dependency_resolution_conf.xml`` file. If you put Conda on
-top, Galaxy will first try to use Conda to resolve a tool dependency;
-if this does not work, the following dependency resolver is used, as
-listed. See `resolver docs`_ for detailed documentation. Starting from Galaxy
+``dependency_resolvers_conf.xml`` file (see question 5 above). Starting from Galaxy
 release 16.07, you can see which dependency will be used (“resolved”) in the
 Admin panel.
 

doc/source/api/guidelines.rst

@@ -8,7 +8,7 @@ these guidelines - but developers contributing to the Galaxy API SHOULD follow
 these guidelines.
 
     - API functionality should include docstring documentation for consumption
-      by readthedocs.org.
+      at docs.galaxyproject.org.
     - Developers should familiarize themselves with the HTTP status code definitions
       http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html. The API responses
       should properly set the status code according to the result - in particular
@@ -50,12 +50,10 @@ consider them when modifying the API.
       that into Mixins (http://en.wikipedia.org/wiki/Composition_over_inheritance)
       or into Managers (:doc:`galaxy.managers`).
     - API additions are more permanent changes to Galaxy than many other potential
-      changes and so a second opinion on API changes should be sought. (Consider a
-      pull request!)
+      changes and so a second opinion on API changes should be sought.
     - New API functionality should include functional tests. These functional
       tests should be implemented in Python and placed in
-      `test/functional/api`. (Once such a framework is in place - it is not
-      right now).
+      `test/functional/api`.
     - Changes to reflect modifications to the API should be pushed upstream to
       the BioBlend project if possible.
 

lib/galaxy/datatypes/data.py

@@ -521,7 +521,7 @@ class Data( object ):
         # add potentially required/common internal tool parameters e.g. '__job_resource'
         if target_context:
             for key, value in target_context.items():
-                if key.startsWith( '__' ):
+                if key.startswith( '__' ):
                     params[ key ] = value
         params[input_name] = original_dataset
 

lib/galaxy/exceptions/__init__.py

@@ -1,14 +1,26 @@
-"""
-Custom exceptions for Galaxy
+"""This module defines Galaxy's custom exceptions.
+
+A Galaxy exception is an exception that extends :class:`MessageException` which
+defines an HTTP status code (represented by the `status_code` attribute) and a
+default error message.
+
+New exceptions should be defined by adding an entry to `error_codes.json` in this
+directory to define a default error message and a Galaxy "error code". A concrete
+Python class should be added in this file defining an HTTP status code (as
+`status_code`) and error code (`error_code`) object loaded dynamically from
+`error_codes.json`.
+
+Reflecting Galaxy's origins as a web application, these exceptions tend to be a
+bit web-oriented. However this module is a dependency of modules and tools that
+have nothing to do with the web - keep this in mind when defining exception names
+and messages.
 """
 
 from ..exceptions import error_codes
 
 
 class MessageException( Exception ):
-    """
-    Exception to make throwing errors from deep in controllers easier.
-    """
+    """Most generic Galaxy exception - indicates merely that some exceptional condition happened."""
     # status code to be set when used with API.
     status_code = 400
     # Error code information embedded into API json responses.

lib/galaxy/exceptions/error_codes.py

@@ -1,3 +1,7 @@
+"""Defines the :class:`ErrorCode` class and instantiates concrete objects from JSON.
+
+See the file error_codes.json for actual error code descriptions.
+"""
 from json import loads
 from pkg_resources import resource_string
 
@@ -8,26 +12,35 @@ from pkg_resources import resource_string
 UNKNOWN_ERROR_MESSAGE = "Unknown error occurred while processing request."
 
 
-class ErrorCode( object ):
+class ErrorCode(object):
+    """Small class allowing object representation for error descriptions loaded from JSON."""
 
-    def __init__( self, code, default_error_message ):
+    def __init__(self, code, default_error_message):
+        """Construct a :class:`ErrorCode` from supplied integer and error message."""
         self.code = code
         self.default_error_message = default_error_message or UNKNOWN_ERROR_MESSAGE
 
-    def __str__( self ):
-        return str( self.default_error_message )
+    def __str__(self):
+        """Return the error code message."""
+        return str(self.default_error_message)
 
-    def __int__( self ):
-        return int( self.code )
+    def __repr__(self):
+        """Return object representation of this error code."""
+        return "ErrorCode[code=%d,message=%s]" % (self.code, str(self.default_error_message))
 
-    @staticmethod
-    def from_dict( entry ):
-        name = entry.get("name")
-        code = entry.get("code")
-        message = entry.get("message")
-        return ( name, ErrorCode( code, message ) )
+    def __int__(self):
+        """Return the error code integer."""
+        return int(self.code)
 
-error_codes_json = resource_string( __name__, 'error_codes.json' ).decode( "UTF-8" )
-for entry in loads( error_codes_json ):
-    name, error_code_obj = ErrorCode.from_dict( entry )
-    globals()[ name ] = error_code_obj
+
+def _from_dict(entry):
+    """Build a :class:`ErrorCode` object from a JSON entry."""
+    name = entry.get("name")
+    code = entry.get("code")
+    message = entry.get("message")
+    return (name, ErrorCode(code, message))
+
+error_codes_json = resource_string(__name__, 'error_codes.json').decode("UTF-8")
+for entry in loads(error_codes_json):
+    name, error_code_obj = _from_dict(entry)
+    globals()[name] = error_code_obj

lib/galaxy/jobs/metrics/__init__.py

@@ -1,3 +1,15 @@
+"""This module defines the job metrics collection framework for Galaxy jobs.
+
+The framework consists of two parts - the :class:`JobMetrics` class and
+individual :class:`JobInstrumenter` plugins.
+
+A :class:`JobMetrics` object reads any number of plugins from a configuration
+source such as an XML file, a YAML file, or a dictionary.
+
+Each :class:`JobInstrumenter` plugin object describes how to inject a bits
+of shell code into a job scripts (before and after tool commands run) and then
+collect the output of these from a job directory.
+"""
 import collections
 import logging
 import os
@@ -7,110 +19,111 @@ from galaxy.util import plugin_config
 
 from ..metrics import formatting
 
-log = logging.getLogger( __name__ )
+log = logging.getLogger(__name__)
 
 
 DEFAULT_FORMATTER = formatting.JobMetricFormatter()
 
 
-class JobMetrics( object ):
+class JobMetrics(object):
+    """Load and store a collection of :class:`JobInstrumenter` objects."""
 
-    def __init__( self, conf_file=None, **kwargs ):
-        """
-        """
+    def __init__(self, conf_file=None, **kwargs):
+        """Load :class:`JobInstrumenter` objects from specified configuration file."""
         self.plugin_classes = self.__plugins_dict()
-        self.default_job_instrumenter = JobInstrumenter.from_file( self.plugin_classes, conf_file, **kwargs )
-        self.job_instrumenters = collections.defaultdict( lambda: self.default_job_instrumenter )
+        self.default_job_instrumenter = JobInstrumenter.from_file(self.plugin_classes, conf_file, **kwargs)
+        self.job_instrumenters = collections.defaultdict(lambda: self.default_job_instrumenter)
 
-    def format( self, plugin, key, value ):
+    def format(self, plugin, key, value):
+        """Find :class:`formatting.JobMetricFormatter` corresponding to instrumented plugin value."""
         if plugin in self.plugin_classes:
             plugin_class = self.plugin_classes[ plugin ]
             formatter = plugin_class.formatter
         else:
             formatter = DEFAULT_FORMATTER
-        return formatter.format( key, value )
+        return formatter.format(key, value)
 
-    def set_destination_conf_file( self, destination_id, conf_file ):
-        instrumenter = JobInstrumenter.from_file( self.plugin_classes, conf_file )
-        self.set_destination_instrumenter( destination_id, instrumenter )
+    def set_destination_conf_file(self, destination_id, conf_file):
+        instrumenter = JobInstrumenter.from_file(self.plugin_classes, conf_file)
+        self.set_destination_instrumenter(destination_id, instrumenter)
 
-    def set_destination_conf_element( self, destination_id, element ):
-        instrumenter = JobInstrumenter( self.plugin_classes, ('xml', element) )
-        self.set_destination_instrumenter( destination_id, instrumenter )
+    def set_destination_conf_element(self, destination_id, element):
+        instrumenter = JobInstrumenter(self.plugin_classes, ('xml', element))
+        self.set_destination_instrumenter(destination_id, instrumenter)
 
-    def set_destination_instrumenter( self, destination_id, job_instrumenter=None ):
+    def set_destination_instrumenter(self, destination_id, job_instrumenter=None):
         if job_instrumenter is None:
             job_instrumenter = NULL_JOB_INSTRUMENTER
         self.job_instrumenters[ destination_id ] = job_instrumenter
 
-    def collect_properties( self, destination_id, job_id, job_directory ):
-        return self.job_instrumenters[ destination_id ].collect_properties( job_id, job_directory )
+    def collect_properties(self, destination_id, job_id, job_directory):
+        return self.job_instrumenters[ destination_id ].collect_properties(job_id, job_directory)
 
-    def __plugins_dict( self ):
+    def __plugins_dict(self):
         import galaxy.jobs.metrics.instrumenters
-        return plugin_config.plugins_dict( galaxy.jobs.metrics.instrumenters, 'plugin_type' )
+        return plugin_config.plugins_dict(galaxy.jobs.metrics.instrumenters, 'plugin_type')
 
 
-class NullJobInstrumenter( object ):
+class NullJobInstrumenter(object):
 
-    def pre_execute_commands( self, job_directory ):
+    def pre_execute_commands(self, job_directory):
         return None
 
-    def post_execute_commands( self, job_directory ):
+    def post_execute_commands(self, job_directory):
         return None
 
-    def collect_properties( self, job_id, job_directory ):
+    def collect_properties(self, job_id, job_directory):
         return {}
 
 NULL_JOB_INSTRUMENTER = NullJobInstrumenter()
 
 
-class JobInstrumenter( object ):
+class JobInstrumenter(object):
 
-    def __init__( self, plugin_classes, plugins_source, **kwargs ):
+    def __init__(self, plugin_classes, plugins_source, **kwargs):
         self.extra_kwargs = kwargs
         self.plugin_classes = plugin_classes
-        self.plugins = self.__plugins_from_source( plugins_source )
+        self.plugins = self.__plugins_from_source(plugins_source)
 
-    def pre_execute_commands( self, job_directory ):
+    def pre_execute_commands(self, job_directory):
         commands = []
         for plugin in self.plugins:
             try:
-                plugin_commands = plugin.pre_execute_instrument( job_directory )
+                plugin_commands = plugin.pre_execute_instrument(job_directory)
                 if plugin_commands:
-                    commands.extend( util.listify( plugin_commands ) )
+                    commands.extend(util.listify(plugin_commands))
             except Exception:
-                log.exception( "Failed to generate pre-execute commands for plugin %s" % plugin )
-        return "\n".join( [ c for c in commands if c ] )
+                log.exception("Failed to generate pre-execute commands for plugin %s" % plugin)
+        return "\n".join([ c for c in commands if c ])
 
-    def post_execute_commands( self, job_directory ):
+    def post_execute_commands(self, job_directory):
         commands = []
         for plugin in self.plugins:
             try:
-                plugin_commands = plugin.post_execute_instrument( job_directory )
+                plugin_commands = plugin.post_execute_instrument(job_directory)
                 if plugin_commands:
-                    commands.extend( util.listify( plugin_commands ) )
+                    commands.extend(util.listify(plugin_commands))
             except Exception:
-                log.exception( "Failed to generate post-execute commands for plugin %s" % plugin )
-        return "\n".join( [ c for c in commands if c ] )
+                log.exception("Failed to generate post-execute commands for plugin %s" % plugin)
+        return "\n".join([ c for c in commands if c ])
 
-    def collect_properties( self, job_id, job_directory ):
+    def collect_properties(self, job_id, job_directory):
         per_plugin_properites = {}
         for plugin in self.plugins:
             try:
-                properties = plugin.job_properties( job_id, job_directory )
+                properties = plugin.job_properties(job_id, job_directory)
                 if properties:
                     per_plugin_properites[ plugin.plugin_type ] = properties
             except Exception:
-                log.exception( "Failed to collect job properties for plugin %s" % plugin )
+                log.exception("Failed to collect job properties for plugin %s" % plugin)
         return per_plugin_properites
 
-    def __plugins_from_source( self, plugins_source ):
+    def __plugins_from_source(self, plugins_source):
         return plugin_config.load_plugins(self.plugin_classes, plugins_source, self.extra_kwargs)
 
     @staticmethod
-    def from_file( plugin_classes, conf_file, **kwargs ):
-        if not conf_file or not os.path.exists( conf_file ):
+    def from_file(plugin_classes, conf_file, **kwargs):
+        if not conf_file or not os.path.exists(conf_file):
             return NULL_JOB_INSTRUMENTER
-        plugins_source = plugin_config.plugin_source_from_path( conf_file )
-        return JobInstrumenter( plugin_classes, plugins_source, **kwargs )
+        plugins_source = plugin_config.plugin_source_from_path(conf_file)
+        return JobInstrumenter(plugin_classes, plugins_source, **kwargs)

lib/galaxy/jobs/metrics/collectl/__init__.py

@@ -1,5 +1,4 @@
-""" This module contains helper functions and data structures for interacting
-with collectl and collectl generated data. More information on collectl can be
-found at: http://collectl.sourceforge.net/.
+"""Helper functions and data structures for interacting with collectl & data.
 
+More information on collectl can be found at: http://collectl.sourceforge.net/.
 """

lib/galaxy/jobs/metrics/collectl/cli.py

@@ -1,3 +1,4 @@
+"""This module describes :class:`CollectlCli` - an abstraction for building collectl command lines."""
 import logging
 import subprocess
 

lib/galaxy/jobs/metrics/collectl/subsystems.py

@@ -1,3 +1,7 @@
+"""Abstractions describing collectl subsystems (specified with the collectl ``-s`` parameter).
+
+Subsystems are essentially monitoring plugins available within collectl.
+"""
 from abc import ABCMeta
 from abc import abstractmethod
 

lib/galaxy/jobs/metrics/formatting.py

@@ -1,18 +1,18 @@
+"""Utilities related to formatting job metrics for human consumption."""
 
 
-class JobMetricFormatter( object ):
-    """ Format job metric key-value pairs for human consumption in Web UI. """
+class JobMetricFormatter(object):
+    """Format job metric key-value pairs for human consumption in Web UI."""
 
-    def format( self, key, value ):
-        return ( str( key ), str( value ) )
+    def format(self, key, value):
+        return (str(key), str(value))
 
 
-# Formatting utilities
-
-def seconds_to_str( value ):
+def seconds_to_str(value):
+    """Convert seconds to a simple simple string describing the amount of time."""
     if value < 60:
         return "%s seconds" % value
     elif value < 3600:
-        return "%s minutes" % ( value / 60 )
+        return "%s minutes" % (value / 60)
     else:
-        return "%s hours and %s minutes" % ( value / 3600, ( value % 3600 ) / 60 )
+        return "%s hours and %s minutes" % (value / 3600, (value % 3600) / 60)

lib/galaxy/jobs/metrics/instrumenters/__init__.py

@@ -1,3 +1,7 @@
+"""This module describes the abstract interface for :class:`InstrumentPlugin`s.
+
+These are responsible for collecting and formatting a coherent set of metrics.
+"""
 import os.path
 
 from abc import ABCMeta
@@ -10,9 +14,7 @@ INSTRUMENT_FILE_PREFIX = "__instrument"
 
 
 class InstrumentPlugin( object ):
-    """ A plugin describing how to instrument Galaxy jobs and retrieve metrics
-    from this instrumentation.
-    """
+    """Describes how to instrument job scripts and retrieve collected metrics."""
     __metaclass__ = ABCMeta
     formatter = formatting.JobMetricFormatter()
 

lib/galaxy/jobs/metrics/instrumenters/collectl.py

@@ -1,3 +1,4 @@
+"""The module describes the ``collectl`` job metrics plugin."""
 import logging
 import os
 import shutil

lib/galaxy/jobs/metrics/instrumenters/core.py

@@ -1,3 +1,4 @@
+"""The module describes the ``core`` job metrics plugin."""
 import logging
 import time
 

lib/galaxy/jobs/metrics/instrumenters/cpuinfo.py

@@ -1,3 +1,4 @@
+"""The module describes the ``cpuinfo`` job metrics plugin."""
 import logging
 import re
 

lib/galaxy/jobs/metrics/instrumenters/env.py

@@ -1,3 +1,4 @@
+"""The module describes the ``env`` job metrics plugin."""
 import logging
 import re
 

lib/galaxy/jobs/metrics/instrumenters/meminfo.py

@@ -1,3 +1,4 @@
+"""The module describes the ``meminfo`` job metrics plugin."""
 import re
 import sys
 

lib/galaxy/jobs/metrics/instrumenters/uname.py

@@ -1,3 +1,4 @@
+"""The module describes the ``uname`` job metrics plugin."""
 from ..instrumenters import InstrumentPlugin
 from ...metrics import formatting
 

lib/galaxy/jobs/runners/godocker.py

@@ -0,0 +1,468 @@
+import json
+import logging
+import requests
+import time
+
+from datetime import datetime
+
+from galaxy import model
+from galaxy.jobs.runners import AsynchronousJobState, AsynchronousJobRunner
+
+
+log = logging.getLogger(__name__)
+
+__all__ = ['GodockerJobRunner']
+
+
+class Godocker(object):
+    """
+    API parameters
+    """
+    def __init__(self, server, login, apikey, noCert):
+        self.token = None
+        self.server = server
+        self.login = login
+        self.apikey = apikey
+        self.noCert = noCert
+
+    def setToken(self, token):
+        self.token = token
+
+    def http_post_request(self, query, data, header):
+        """ post request with query """
+
+        verify_ssl = not self.noCert
+        try:
+            url = self.server + query
+            res = requests.post(url, data, headers=header, verify=verify_ssl)
+
+        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
+            log.error('A transport error occurred in the GoDocker job runner:', e)
+            return False
+
+        return self.test_status_code(res)
+
+    def http_get_request(self, query, header):
+        """ get request with query, server and header required """
+
+        # remove warnings if using --no-certificate
+        requests.packages.urllib3.disable_warnings()
+        verify_ssl = not self.noCert
+        try:
+            url = self.server + query
+            res = requests.get(url, headers=header, verify=verify_ssl)
+
+        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
+            log.error('A communication error occurred in the GoDocker job runner:', e)
+            return False
+
+        return self.test_status_code(res)
+
+    def http_delete_request(self, query, header):
+        """ delete request with query, server and header required """
+
+        # remove warnings if using --no-certificate
+        requests.packages.urllib3.disable_warnings()
+        verify_ssl = not self.noCert
+        try:
+            url = self.server + query
+            res = requests.delete(url, headers=header, verify=verify_ssl)
+
+        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
+            log.error('A communication error occurred in the GoDocker job runner:', e)
+            return False
+
+        return self.test_status_code(res)
+
+    def http_put_request(self, query, data, header):
+        """ put request with query """
+
+        # remove warnings if using --no-certificate
+        requests.packages.urllib3.disable_warnings()
+        verify_ssl = not self.noCert
+        try:
+            url = self.server + query
+            res = requests.put(url, data, headers=header, verify=verify_ssl)
+
+        except (requests.exceptions.ConnectionError, requests.exceptions.HTTPError) as e:
+            log.error('A communication error occurred in the GoDocker job runner:', e)
+            return False
+
+        return self.test_status_code(res)
+
+    def test_status_code(self, httpresult):
+        """ exit if status code is 401 or 403 or 404 or 200"""
+        if httpresult.status_code == 401:
+            log.debug('Unauthorized : this server could not verify that you are authorized to access the document you requested.')
+
+        elif httpresult.status_code == 403:
+            log.debug('Forbidden : Access was denied to this resource. Not authorized to access this resource.')
+
+        elif httpresult.status_code == 404:
+            log.debug('Not Found : The resource could not be found.')
+
+        elif httpresult.status_code == 200:
+            return httpresult
+
+        return False
+
+
+class GodockerJobRunner(AsynchronousJobRunner):
+    """
+    Job runner backed by a finite pool of worker threads. FIFO scheduling
+    """
+    runner_name = "GodockerJobRunner"
+
+    def __init__(self, app, nworkers, **kwargs):
+        """ 1: Get runner_param_specs from job_conf.xml
+            2: Initialise job runner parent object
+            3: Login to godocker and store the token
+            4: Start the worker and monitor threads
+        """
+        runner_param_specs = dict(godocker_master=dict(map=str), user=dict(map=str), key=dict(map=str), godocker_project=dict(map=str))
+        if 'runner_param_specs' not in kwargs:
+            kwargs['runner_param_specs'] = dict()
+        kwargs['runner_param_specs'].update(runner_param_specs)
+
+        # Start the job runner parent object
+        super(GodockerJobRunner, self).__init__(app, nworkers, **kwargs)
+
+        # godocker API login call
+        self.auth = self.login(self.runner_params["key"], self.runner_params["user"], self.runner_params["godocker_master"])
+
+        if not self.auth:
+            log.error("Authentication failure, GoDocker runner cannot be started")
+        else:
+            """ Following methods starts threads.
+                These methods invoke threading.Thread(name,target)
+                      which in turn invokes methods monitor() and run_next().
+            """
+            self._init_monitor_thread()
+            self._init_worker_threads()
+
+    def queue_job(self, job_wrapper):
+        """ Create job script and submit it to godocker """
+        if not self.prepare_job(job_wrapper, include_metadata=False, include_work_dir_outputs=True, modify_command_for_container=False):
+            return
+
+        job_destination = job_wrapper.job_destination
+        """ Submit job to godocker """
+        job_id = self.post_task(job_wrapper)
+        if not job_id:
+            log.error("Job creation failure.  No Response from GoDocker")
+            job_wrapper.fail("Not submitted")
+        else:
+            log.debug("Starting queue_job for job " + job_id)
+            # Create an object of AsynchronousJobState and add it to the monitor queue.
+            ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, job_wrapper=job_wrapper, job_id=job_id, job_destination=job_destination)
+            self.monitor_queue.put(ajs)
+        return None
+
+    def check_watched_item(self, job_state):
+        """ Get the job current status from GoDocker
+                    using job_id and update the status in galaxy.
+            If the job execution is successful, call
+                    mark_as_finished() and return 'None' to galaxy.
+            else if the job failed, call mark_as_failed()
+                    and return 'None' to galaxy.
+            else if the job is running or in pending state, simply
+                    return the 'AsynchronousJobState object' (job_state).
+        """
+        ''' This function is called by check_watched_items() where
+                    param job_state is an object of AsynchronousJobState.
+            Expected return type of this function is None or
+                    AsynchronousJobState object with updated running status.
+        '''
+        """ Get task from GoDocker """
+        job_status_god = self.get_task(job_state.job_id)
+        log.debug("Job ID: " + str(job_state.job_id) + " Job Status: " + str(job_status_god['status']['primary']))
+
+        if job_status_god['status']['primary'] == "over":
+            job_state.running = False
+            job_state.job_wrapper.change_state(model.Job.states.OK)
+            if self.create_log_file(job_state, job_status_god):
+                self.mark_as_finished(job_state)
+            else:
+                self.mark_as_failed(job_state)
+            '''The function mark_as_finished() executes:
+                        self.work_queue.put((self.finish_job, job_state))
+           *self.finish_job ->
+            job_state.job_wrapper.finish( stdout, stderr, exit_code )
+            job_state.job_wrapper.reclaim_ownership()
+            job_state.cleanup()
+           *self.work_queue.put( method , arg ) ->
+            The run_next() method starts execution on starting worker threads.
+            This run_next() method executes method(arg)
+                        by using self.work_queue.get()
+           *Possible outcomes of finish_job(job_state) ->
+            job_state.job_wrapper.finish( stdout, stderr, exit_code )
+            job_state.job_wrapper.fail( "Unable to finish job", exception=True)
+           *Similar workflow is done for mark_as_failed() method.
+            '''
+            return None
+
+        elif job_status_god['status']['primary'] == "running":
+            job_state.running = True
+            job_state.job_wrapper.change_state(model.Job.states.RUNNING)
+            return job_state
+
+        elif job_status_god['status']['primary'] == "pending":
+            return job_state
+
+        elif job_status_god['status']['exitcode'] not in [None, 0]:
+            job_state.running = False
+            job_state.job_wrapper.change_state(model.Job.states.ERROR)
+            self.create_log_file(job_state, job_status_god)
+            self.mark_as_failed(job_state)
+            return None
+
+        else:
+            job_state.running = False
+            self.create_log_file(job_state, job_status_god)
+            self.mark_as_failed(job_state)
+            return None
+
+    def stop_job(self, job):
+        """ Attempts to delete a dispatched executing Job in GoDocker """
+        '''This function is called by fail_job()
+           where param job = self.sa_session.query( self.app.model.Job ).get( job_state.job_wrapper.job_id )
+           No Return data expected
+        '''
+        log.debug("STOP JOB EXECUTION OF JOB ID: " + str(job.id))
+        # Get task status from GoDocker.
+        job_status_god = self.get_task_status(job.id)
+        if job_status_god['status']['primary'] != "over":
+            # Initiate a delete call,if the job is running in GoDocker.
+            self.delete_task(job.id)
+        return None
+
+    def recover(self, job, job_wrapper):
+        """ Recovers jobs stuck in the queued/running state when Galaxy started """
+        """ This method is called by galaxy at the time of startup.
+            Jobs in Running & Queued status in galaxy are put in the monitor_queue by creating an AsynchronousJobState object
+        """
+        job_id = job_wrapper.job_id
+        ajs = AsynchronousJobState(files_dir=job_wrapper.working_directory, job_wrapper=job_wrapper)
+        ajs.job_id = str(job_id)
+        ajs.job_destination = job_wrapper.job_destination
+        job_wrapper.command_line = job.command_line
+        ajs.job_wrapper = job_wrapper
+        if job.state == model.Job.states.RUNNING:
+            log.debug("(%s/%s) is still in running state, adding to the god queue" % (job.id, job.get_job_runner_external_id()))
+            ajs.old_state = 'R'
+            ajs.running = True
+            self.monitor_queue.put(ajs)
+
+        elif job.state == model.Job.states.QUEUED:
+            log.debug("(%s/%s) is still in god queued state, adding to the god queue" % (job.id, job.get_job_runner_external_id()))
+            ajs.old_state = 'Q'
+            ajs.running = False
+            self.monitor_queue.put(ajs)
+
+    # Helper functions
+
+    def create_log_file(self, job_state, job_status_god):
+        """ Create log files in galaxy, namely error_file, output_file, exit_code_file
+            Return true, if all the file creations are successful
+        """
+        path = None
+        for vol in job_status_god['container']['volumes']:
+            if vol['name'] == "go-docker":
+                path = str(vol['path'])
+        if path:
+            god_output_file = path + "/god.log"
+            god_error_file = path + "/god.err"
+            try:
+                # Read from GoDocker output_file and write it into galaxy output_file.
+                f = open(god_output_file, "r")
+                out_log = f.read()
+                log_file = open(job_state.output_file, "w")
+                log_file.write(out_log)
+                log_file.close()
+                f.close()
+                # Read from GoDocker error_file and write it into galaxy error_file.
+                f = open(god_error_file, "r")
+                out_log = f.read()
+                log_file = open(job_state.error_file, "w")
+                log_file.write(out_log)
+                log_file.close()
+                f.close()
+                # Read from GoDocker exit_code and write it into galaxy exit_code_file.
+                out_log = str(job_status_god['status']['exitcode'])
+                log_file = open(job_state.exit_code_file, "w")
+                log_file.write(out_log)
+                log_file.close()
+                f.close()
+                log.debug("CREATE OUTPUT FILE: " + str(job_state.output_file))
+                log.debug("CREATE ERROR FILE: " + str(job_state.error_file))
+                log.debug("CREATE EXIT CODE FILE: " + str(job_state.exit_code_file))
+            except IOError as e:
+                log.error('Could not access task log file %s' % str(e))
+                log.debug("IO Error occurred when accessing the files.")
+                return False
+        return True
+
+    # GoDocker API helper functions
+
+    def login(self, apikey, login, server, noCert=False):
+        """ Login to GoDocker and return the token
+            Create Login model schema of GoDocker and call the http_post_request method.
+        """
+        log.debug("LOGIN TASK TO BE EXECUTED \n")
+        log.debug("GODOCKER LOGIN: " + str(login))
+        data = json.dumps({'user': login, 'apikey': apikey})
+        # Create object of Godocker class
+        g_auth = Godocker(server, login, apikey, noCert)
+        auth = g_auth.http_post_request("/api/1.0/authenticate", data, {'Content-type': 'application/json', 'Accept': 'application/json'})
+        if not auth:
+            log.error("GoDocker authentication Error.")
+        else:
+            log.debug("GoDocker authentication successful.")
+            token = auth.json()['token']
+            g_auth.setToken(token)
+        # Return the object of Godocker class
+        return g_auth
+
+    def post_task(self, job_wrapper):
+        """ Sumbit job to GoDocker and return jobid
+            Create Job model schema of GoDocker and call the http_post_request method.
+        """
+        # Get the params from <destination> tag in job_conf by using job_destination.params[param]
+        if self.auth.token:
+            job_destination = job_wrapper.job_destination
+            try:
+                docker_cpu = int(job_destination.params["docker_cpu"])
+            except:
+                docker_cpu = 1
+            try:
+                docker_ram = int(job_destination.params["docker_memory"])
+            except:
+                docker_ram = 1
+            try:
+                docker_image = self._find_container(job_wrapper).container_id
+                log.debug("GoDocker runner using container %s.", docker_image)
+            except:
+                log.error("Unable to find docker_image for job %s, failing." % job_wrapper.job_id)
+                return False
+
+            volumes = []
+            labels = []
+            tags_tab = ['galaxy', job_wrapper.tool.id]
+            tasks_depends = []
+            name = job_wrapper.tool.name
+            description = "galaxy job"
+            array = None
+            project = None
+            try:
+                project = str(self.runner_params["godocker_project"])
+            except KeyError:
+                log.debug("godocker_project not defined, using default.")
+            try:
+                volume = job_destination.params["godocker_volumes"]
+                volume = volume.split(",")
+                for i in volume:
+                    temp = dict({"name": i})
+                    volumes.append(temp)
+            except:
+                log.debug("godocker_volume not set, using default.")
+
+            dt = datetime.now()
+            # Enable galaxy venv in the docker containers
+            try:
+                if(job_destination.params["virtualenv"] == "true"):
+                    GALAXY_VENV_TEMPLATE = """GALAXY_VIRTUAL_ENV="%s"; if [ "$GALAXY_VIRTUAL_ENV" != "None" -a -z "$VIRTUAL_ENV" -a -f "$GALAXY_VIRTUAL_ENV/bin/activate" ]; then . "$GALAXY_VIRTUAL_ENV/bin/activate"; fi;"""
+                    venv = GALAXY_VENV_TEMPLATE % job_wrapper.galaxy_virtual_env
+                    command = "#!/bin/bash\n" + "cd " + job_wrapper.working_directory + "\n" + venv + "\n" + job_wrapper.runner_command_line
+                else:
+                    command = "#!/bin/bash\n" + "cd " + job_wrapper.working_directory + "\n" + job_wrapper.runner_command_line
+            except:
+                command = "#!/bin/bash\n" + "cd " + job_wrapper.working_directory + "\n" + job_wrapper.runner_command_line
+
+            # GoDocker Job model schema
+            job = {
+                'date': time.mktime(dt.timetuple()),
+                'meta': {
+                    'name': name,
+                    'description': description,
+                    'tags': tags_tab
+                },
+                'requirements': {
+                    'cpu': docker_cpu,
+                    'ram': docker_ram,
+                    'array': {'values': array},
+                    'label': labels,
+                    'tasks': tasks_depends,
+                    'tmpstorage': None
+                },
+                'container': {
+                    'image': str(docker_image),
+                    'volumes': volumes,
+                    'network': True,
+                    'id': None,
+                    'meta': None,
+                    'stats': None,
+                    'ports': [],
+                    'root': False
+                },
+                'command': {
+                    'interactive': False,
+                    'cmd': command,
+                },
+                'status': {
+                    'primary': None,
+                    'secondary': None
+                }
+            }
+            if project is not None:
+                job['user'] = {"project": project}
+
+            result = self.auth.http_post_request(
+                "/api/1.0/task", json.dumps(job),
+                {'Authorization': 'Bearer ' + self.auth.token, 'Content-type': 'application/json', 'Accept': 'application/json'}
+            )
+            # Return job_id
+            return str(result.json()['id'])
+
+    def get_task(self, job_id):
+        """ Get job details from GoDocker and return the job.
+            Pass job_id to the http_get_request method.
+        """
+        job = False
+        if self.auth.token:
+            result = self.auth.http_get_request("/api/1.0/task/" + str(job_id), {'Authorization': 'Bearer ' + self.auth.token})
+            job = result.json()
+        # Return the job
+        return job
+
+    def task_suspend(self, job_id):
+        """ Suspend actively running job in galaxy.
+            Pass job_id to the http_get_request method.
+        """
+        job = False
+        if self.auth.token:
+            result = self.auth.http_get_request("/api/1.0/task/" + str(job_id) + "/suspend", {'Authorization': 'Bearer ' + self.auth.token})
+            job = result.json()
+        # Return the job
+        return job
+
+    def get_task_status(self, job_id):
+        """ Get job status from GoDocker and return the status of job.
+            Pass job_id to http_get_request method.
+        """
+        job = False
+        if self.auth.token:
+            result = self.auth.http_get_request("/api/1.0/task/" + str(job_id) + "/status", {'Authorization': 'Bearer ' + self.auth.token})
+            job = result.json()
+        # Return task status
+        return job
+
+    def delete_task(self, job_id):
+        """ Delete a suspended task in GoDocker.
+            Pass job_id to http_delete_request method.
+        """
+        job = False
+        if self.auth.token:
+            result = self.auth.http_delete_request("/api/1.0/task/" + str(job_id), {'Authorization': 'Bearer ' + self.auth.token})
+            job = result.json()
+        # Return the job
+        return job

lib/galaxy/managers/workflows.py

@@ -368,6 +368,9 @@ class WorkflowContentsManager(UsesAnnotations):
             workflow.annotation = self.get_item_annotation_str( trans.sa_session, trans.user, workflow )
             raise exceptions.MessageException( 'Following tools missing: %s' % missing_tools )
         workflow.annotation = self.get_item_annotation_str( trans.sa_session, trans.user, workflow )
+        step_order_indices = {}
+        for step in workflow.steps:
+            step_order_indices[ step.id ] = step.order_index
         step_models = []
         for i, step in enumerate( workflow.steps ):
             step_model = None
@@ -388,11 +391,11 @@ class WorkflowContentsManager(UsesAnnotations):
                     'name'   : step.module.name,
                     'inputs' : [ input.to_dict( trans ) for input in inputs.itervalues() ]
                 }
-            step_model[ 'step_id' ] = step.id
             step_model[ 'step_type' ] = step.type
+            step_model[ 'step_index' ] = step.order_index
             step_model[ 'output_connections' ] = [ {
-                'input_step_id'     : oc.input_step_id,
-                'output_step_id'    : oc.output_step_id,
+                'input_step_index'  : step_order_indices.get( oc.input_step_id ),
+                'output_step_index' : step_order_indices.get( oc.output_step_id ),
                 'input_name'        : oc.input_name,
                 'output_name'       : oc.output_name
             } for oc in step.output_connections ]
@@ -403,6 +406,7 @@ class WorkflowContentsManager(UsesAnnotations):
             step_models.append( step_model )
         return {
             'id'                    : trans.app.security.encode_id( stored.id ),
+            'history_id'            : trans.app.security.encode_id( trans.history.id ) if trans.history else None,
             'name'                  : stored.name,
             'steps'                 : step_models,
             'step_version_changes'  : step_version_changes,

lib/galaxy/queue_worker.py

@@ -63,6 +63,7 @@ def _get_new_toolbox(app):
     and then adding pre-existing data managers from the old toolbox to the new toolbox.
     """
     from galaxy import tools
+    from galaxy.tools.special_tools import load_lib_tools
     from galaxy.tools.toolbox.lineages.tool_shed import ToolVersionCache
     app.tool_version_cache = ToolVersionCache(app)  # Load new tools into version cache
     tool_configs = app.config.tool_configs
@@ -71,6 +72,8 @@ def _get_new_toolbox(app):
     start = time.time()
     new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app, app.toolbox._tool_conf_watcher)
     new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
+    load_lib_tools(new_toolbox)
+    new_toolbox.load_hidden_lib_tool( "galaxy/datatypes/set_metadata_tool.xml" )
     [new_toolbox.register_tool(tool) for tool in new_toolbox.data_manager_tools.values()]
     end = time.time() - start
     log.debug("Toolbox reload took %d seconds", end)

lib/galaxy/tools/__init__.py

@@ -1156,7 +1156,8 @@ class Tool( object, Dictifiable ):
         log.debug( 'Validated and populated state for tool request %s' % validation_timer )
         # If there were errors, we stay on the same page and display them
         if any( all_errors ):
-            raise exceptions.MessageException( ', '.join( [ msg for msg in all_errors[ 0 ].itervalues() ] ), err_data=all_errors[ 0 ] )
+            err_data = { key: value for d in all_errors for ( key, value ) in d.iteritems() }
+            raise exceptions.MessageException( ', '.join( [ msg for msg in err_data.itervalues() ] ), err_data=err_data )
         else:
             execution_tracker = execute_job( trans, self, all_params, history=request_context.history, rerun_remap_job_id=rerun_remap_job_id, collection_info=collection_info )
             if execution_tracker.successful_jobs:

lib/galaxy/tools/deps/__init__.py

@@ -24,6 +24,7 @@ EXTRA_CONFIG_KWDS = {
     'conda_ensure_channels': 'r,bioconda,iuc',
     'conda_auto_install': False,
     'conda_auto_init': False,
+    'conda_copy_dependencies': False,
 }
 
 CONFIG_VAL_NOT_FOUND = object()

lib/galaxy/tools/parameters/basic.py

@@ -42,6 +42,13 @@ def contains_workflow_parameter( value, search=False ):
     return False
 
 
+def parse_dynamic_options( param, input_source ):
+    options_elem = input_source.parse_dynamic_options_elem()
+    if options_elem is not None:
+        return dynamic_options.DynamicOptions( options_elem, param )
+    return None
+
+
 class ToolParameter( object, Dictifiable ):
     """
     Describes a parameter accepted by a tool. This is just a simple stub at the
@@ -211,9 +218,8 @@ class TextToolParameter( ToolParameter ):
     >>> p = TextToolParameter( None, XML( '<param name="_name" type="text" value="default" />' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['area: False', 'argument: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'model_class: TextToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'type: text', 'value: default']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('area', False), ('argument', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'TextToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('type', 'text'), ('value', 'default')]
     """
     def __init__( self, tool, input_source ):
         input_source = ensure_input_source(input_source)
@@ -251,9 +257,8 @@ class IntegerToolParameter( TextToolParameter ):
     >>> p = IntegerToolParameter( None, XML( '<param name="_name" type="integer" value="10" />' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['area: False', 'argument: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'max: None', 'min: None', 'model_class: IntegerToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'type: integer', 'value: 10']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('area', False), ('argument', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('max', None), ('min', None), ('model_class', 'IntegerToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('type', 'integer'), ('value', '10')]
     >>> type( p.from_json( "10", trans ) )
     <type 'int'>
     >>> type( p.from_json( "_string", trans ) )
@@ -327,9 +332,8 @@ class FloatToolParameter( TextToolParameter ):
     >>> p = FloatToolParameter( None, XML( '<param name="_name" type="float" value="3.141592" />' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['area: False', 'argument: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'max: None', 'min: None', 'model_class: FloatToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'type: float', 'value: 3.141592']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('area', False), ('argument', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('max', None), ('min', None), ('model_class', 'FloatToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('type', 'float'), ('value', '3.141592')]
     >>> type( p.from_json( "36.1", trans ) )
     <type 'float'>
     >>> type( p.from_json( "_string", trans ) )
@@ -404,9 +408,8 @@ class BooleanToolParameter( ToolParameter ):
     >>> p = BooleanToolParameter( None, XML( '<param name="_name" type="boolean" checked="yes" truevalue="_truevalue" falsevalue="_falsevalue" />' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'falsevalue: _falsevalue', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'model_class: BooleanToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'truevalue: _truevalue', 'type: boolean', 'value: True']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('falsevalue', '_falsevalue'), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'BooleanToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('truevalue', '_truevalue'), ('type', 'boolean'), ('value', True)]
     >>> print p.from_json( 'true' )
     True
     >>> print p.to_param_dict_string( True )
@@ -465,9 +468,8 @@ class FileToolParameter( ToolParameter ):
     >>> p = FileToolParameter( None, XML( '<param name="_name" type="file"/>' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'model_class: FileToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'type: file']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'FileToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('type', 'file')]
     """
     def __init__( self, tool, input_source ):
         input_source = ensure_input_source(input_source)
@@ -525,9 +527,8 @@ class FTPFileToolParameter( ToolParameter ):
     >>> p = FTPFileToolParameter( None, XML( '<param name="_name" type="ftpfile"/>' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'model_class: FTPFileToolParameter', 'multiple: True', 'name: _name', 'optional: True', 'refresh_on_change: False', 'type: ftpfile']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'FTPFileToolParameter'), ('multiple', True), ('name', '_name'), ('optional', True), ('refresh_on_change', False), ('type', 'ftpfile')]
     """
     def __init__( self, tool, input_source ):
         input_source = ensure_input_source(input_source)
@@ -598,9 +599,8 @@ class HiddenToolParameter( ToolParameter ):
     >>> p = HiddenToolParameter( None, XML( '<param name="_name" type="hidden" value="_value"/>' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'help: ', 'hidden: True', 'is_dynamic: False', 'label: ', 'model_class: HiddenToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'type: hidden', 'value: _value']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('help', ''), ('hidden', True), ('is_dynamic', False), ('label', ''), ('model_class', 'HiddenToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('type', 'hidden'), ('value', '_value')]
     """
     def __init__( self, tool, input_source ):
         input_source = ensure_input_source( input_source )
@@ -624,18 +624,35 @@ class ColorToolParameter( ToolParameter ):
     >>> p = ColorToolParameter( None, XML( '<param name="_name" type="color" value="#ffffff"/>' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'model_class: ColorToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'type: color', 'value: #ffffff']
+    >>> print p.to_param_dict_string( "#fdeada" )
+    #fdeada
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'ColorToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('type', 'color'), ('value', '#ffffff')]
+    >>> p = ColorToolParameter( None, XML( '<param name="_name" type="color" value="#ffffff" rgb="True"/>' ) )
+    >>> print p.to_param_dict_string( "#fdeada" )
+    (253, 234, 218)
+    >>> print p.to_param_dict_string( None )
+    Traceback (most recent call last):
+        ...
+    ValueError: Failed to convert 'None' to RGB.
     """
     def __init__( self, tool, input_source ):
         input_source = ensure_input_source( input_source )
         ToolParameter.__init__( self, tool, input_source )
         self.value = input_source.get( 'value', '#fdeada' )
+        self.rgb = input_source.get( 'rgb', False )
 
     def get_initial_value( self, trans, other_values ):
         return self.value.lower()
 
+    def to_param_dict_string( self, value, other_values={} ):
+        if self.rgb:
+            try:
+                return str( tuple( int( value.lstrip( '#' )[ i : i + 2 ], 16 ) for i in ( 0, 2, 4 ) ) )
+            except Exception:
+                raise ValueError( "Failed to convert \'%s\' to RGB." % value )
+        return str( value )
+
 
 class BaseURLToolParameter( HiddenToolParameter ):
     """
@@ -647,9 +664,8 @@ class BaseURLToolParameter( HiddenToolParameter ):
     >>> p = BaseURLToolParameter( None, XML( '<param name="_name" type="base_url" value="_value"/>' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'help: ', 'hidden: True', 'is_dynamic: False', 'label: ', 'model_class: BaseURLToolParameter', 'name: _name', 'optional: False', 'refresh_on_change: False', 'type: base_url', 'value: _value']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('help', ''), ('hidden', True), ('is_dynamic', False), ('label', ''), ('model_class', 'BaseURLToolParameter'), ('name', '_name'), ('optional', False), ('refresh_on_change', False), ('type', 'base_url'), ('value', '_value')]
     """
     def __init__( self, tool, input_source ):
         input_source = ensure_input_source( input_source )
@@ -675,19 +691,6 @@ class BaseURLToolParameter( HiddenToolParameter ):
         return d
 
 
-def DEFAULT_VALUE_MAP(x):
-    return x
-
-
-def parse_dynamic_options(param, input_source):
-    options_elem = input_source.parse_dynamic_options_elem()
-    if options_elem is None:
-        options = None
-    else:
-        options = dynamic_options.DynamicOptions( options_elem, param )
-    return options
-
-
 class SelectToolParameter( ToolParameter ):
     """
     Parameter that takes on one (or many) or a specific set of values.
@@ -704,9 +707,8 @@ class SelectToolParameter( ToolParameter ):
     ... ''' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'display: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'model_class: SelectToolParameter', 'multiple: False', 'name: _name', 'optional: False', "options: [('x_label', 'x', False), ('y_label', 'y', True), ('z_label', 'z', False)]", 'refresh_on_change: False', 'type: select', 'value: y']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('display', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'SelectToolParameter'), ('multiple', False), ('name', '_name'), ('optional', False), ('options', [('x_label', 'x', False), ('y_label', 'y', True), ('z_label', 'z', False)]), ('refresh_on_change', False), ('type', 'select'), ('value', 'y')]
     >>> p = SelectToolParameter( None, XML(
     ... '''
     ... <param name="_name" type="select" multiple="true">
@@ -717,9 +719,8 @@ class SelectToolParameter( ToolParameter ):
     ... ''' ) )
     >>> print p.name
     _name
-    >>> d = p.to_dict( trans )
-    >>> [ "%s: %s" % ( key, d[ key ] ) for key in sorted( d ) ]
-    ['argument: None', 'display: None', 'help: ', 'hidden: False', 'is_dynamic: False', 'label: ', 'model_class: SelectToolParameter', 'multiple: True', 'name: _name', 'optional: True', "options: [('x_label', 'x', False), ('y_label', 'y', True), ('z_label', 'z', True)]", 'refresh_on_change: False', 'type: select', 'value: z']
+    >>> sorted( p.to_dict( trans ).items() )
+    [('argument', None), ('display', None), ('help', ''), ('hidden', False), ('is_dynamic', False), ('label', ''), ('model_class', 'SelectToolParameter'), ('multiple', True), ('name', '_name'), ('optional', True), ('options', [('x_label', 'x', False), ('y_label', 'y', True), ('z_label', 'z', True)]), ('refresh_on_change', False), ('type', 'select'), ('value', 'z')]
     >>> print p.to_param_dict_string( ["y", "z"] )
     y,z
     """
@@ -823,7 +824,7 @@ class SelectToolParameter( ToolParameter ):
                 raise ValueError( "An invalid option was selected for %s, %r, please verify." % ( self.name, value ) )
             return value
 
-    def to_param_dict_string( self, value, other_values={}, value_map=DEFAULT_VALUE_MAP ):
+    def to_param_dict_string( self, value, other_values={} ):
         if value is None:
             return "None"
         if isinstance( value, list ):
@@ -838,9 +839,7 @@ class SelectToolParameter( ToolParameter ):
             else:
                 value = sanitize_param( value )
         if isinstance( value, list ):
-            value = self.separator.join( map( value_map, value ) )
-        else:
-            value = value_map( value )
+            value = self.separator.join( value )
         return value
 
     def to_json( self, value, app, use_security ):
@@ -1279,7 +1278,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
             rval.append( val )
         return rval
 
-    def to_param_dict_string( self, value, other_values={}, value_map=DEFAULT_VALUE_MAP ):
+    def to_param_dict_string( self, value, other_values={} ):
         def get_options_list( value ):
             def get_base_option( value, options ):
                 for option in options:
@@ -1311,7 +1310,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
                 rval.extend( options )
         if len( rval ) > 1 and not self.multiple:
             raise ValueError( "Multiple values provided but parameter %s is not expecting multiple values." % self.name )
-        rval = self.separator.join( map( value_map, rval ) )
+        rval = self.separator.join( rval )
         if self.tool is None or self.tool.options.sanitize:
             if self.sanitizer:
                 rval = self.sanitizer.sanitize_param( rval )
@@ -1387,25 +1386,6 @@ class BaseDataToolParameter( ToolParameter ):
         super(BaseDataToolParameter, self).__init__( tool, input_source )
         self.refresh_on_change = True
 
-    def _get_history( self, trans ):
-        class_name = self.__class__.__name__
-        assert trans is not None, "%s requires a trans" % class_name
-        assert trans.history is not None, "%s requires a history" % class_name
-        return trans.history
-
-    def _ensure_selection( self, field ):
-        set_selected = field.get_selected( return_label=True, return_value=True, multi=False ) is not None
-        # Ensure than an item is always selected
-        if self.optional:
-            if set_selected:
-                field.add_option( "Selection is Optional", 'None', False )
-            else:
-                field.add_option( "Selection is Optional", 'None', True )
-        elif not set_selected and bool( field.options ):
-            # Select the last item
-            a, b, c = field.options[-1]
-            field.options[-1] = a, b, True
-
     def _datatypes_registery( self, trans, tool ):
         # Find datatypes_registry
         if tool is None:
@@ -1598,7 +1578,7 @@ class DataToolParameter( BaseDataToolParameter ):
         if trans.workflow_building_mode is workflow_building_modes.ENABLED:
             return None
         if not value and not self.optional:
-            raise ValueError( "History does not include a dataset of the required format / build" )
+            raise ValueError( "Specify a dataset of the required format / build." )
         if value in [ None, "None", '' ]:
             return None
         if isinstance( value, dict ) and 'values' in value:
@@ -1738,7 +1718,7 @@ class DataToolParameter( BaseDataToolParameter ):
         self.tool.visit_inputs( other_values, visitor )
         return False not in converter_safe
 
-    def _options_filter_attribute( self, value ):
+    def get_options_filter_attribute( self, value ):
         # HACK to get around current hardcoded limitation of when a set of dynamic options is defined for a DataToolParameter
         # it always causes available datasets to be filtered by dbkey
         # this behavior needs to be entirely reworked (in a backwards compatible manner)
@@ -1865,7 +1845,7 @@ class DataCollectionToolParameter( BaseDataToolParameter ):
         if trans.workflow_building_mode is workflow_building_modes.ENABLED:
             return None
         if not value and not self.optional:
-            raise ValueError( "History does not include a dataset collection of the correct type or containing the correct types of datasets" )
+            raise ValueError( "Specify a dataset collection of the correct type." )
         if value in [None, "None"]:
             return None
         if isinstance( value, dict ) and 'values' in value:

lib/galaxy/tools/parameters/dataset_matcher.py

@@ -100,7 +100,7 @@ class DatasetMatcher( object ):
         applicable).
         """
         param = self.param
-        return param.options and param._options_filter_attribute( hda ) != self.filter_value
+        return param.options and param.get_options_filter_attribute( hda ) != self.filter_value
 
     def __can_access_dataset( self, dataset ):
         # Lazily cache current_user_roles.

lib/galaxy/tools/parameters/meta.py

@@ -9,59 +9,59 @@ log = logging.getLogger( __name__ )
 
 
 def expand_workflow_inputs( inputs ):
-        """
-        Expands incoming encoded multiple payloads, into the set of all individual payload combinations
-        >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'product': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}} )
-        >>> print sorted( [ "%s" % ( p[ '1' ][ 'input' ][ 'hid' ] ) for p in params ] )
-        ['1', '2']
-        >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}} )
-        >>> print sorted( [ "%s" % ( p[ '1' ][ 'input' ][ 'hid' ] ) for p in params ] )
-        ['1', '2']
-        >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}, '2': {'input': {'batch': True, 'values': [{'hid': '3'}, {'hid': '4'}] }}} )
-        >>> print sorted( [ "%s%s" % ( p[ '1' ][ 'input' ][ 'hid' ], p[ '2' ][ 'input' ][ 'hid' ] ) for p in params ] )
-        ['13', '24']
-        >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'product': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}, '2': {'input': {'batch': True, 'values': [{'hid': '3'}, {'hid': '4'}, {'hid': '5'}] }}} )
-        >>> print sorted( [ "%s%s" % ( p[ '1' ][ 'input' ][ 'hid' ], p[ '2' ][ 'input' ][ 'hid' ] ) for p in params ] )
-        ['13', '14', '15', '23', '24', '25']
-        >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'product': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}, '2': {'input': {'batch': True, 'product': True, 'values': [{'hid': '3'}, {'hid': '4'}, {'hid': '5'}] }}, '3': {'input': {'batch': True, 'product': True, 'values': [{'hid': '6'}, {'hid': '7'}, {'hid': '8'}] }}} )
-        >>> print sorted( [ "%s%s%s" % ( p[ '1' ][ 'input' ][ 'hid' ], p[ '2' ][ 'input' ][ 'hid' ], p[ '3' ][ 'input' ][ 'hid' ] ) for p in params ] )
-        ['136', '137', '138', '146', '147', '148', '156', '157', '158', '236', '237', '238', '246', '247', '248', '256', '257', '258']
-        """
-        linked_n = None
-        linked = []
-        product = []
-        linked_keys = []
-        product_keys = []
-        for step_id, step in inputs.items():
-            for key, value in step.items():
-                if isinstance( value, dict ) and 'batch' in value and value[ 'batch' ] is True and 'values' in value and isinstance( value[ 'values' ], list ):
-                    nval = len( value[ 'values' ] )
-                    if 'product' in value and value[ 'product' ] is True:
-                        product.append( value[ 'values' ] )
-                        product_keys.append( ( step_id, key ) )
-                    else:
-                        if linked_n is None:
-                            linked_n = nval
-                        elif linked_n != nval or nval is 0:
-                            raise exceptions.RequestParameterInvalidException( 'Failed to match linked batch selections. Please select equal number of data files.' )
-                        linked.append( value[ 'values' ] )
-                        linked_keys.append( ( step_id, key ) )
-        params = []
-        params_keys = []
-        linked = linked or [ [ None ] ]
-        product = product or [ [ None ] ]
-        linked_keys = linked_keys or [ ( None, None ) ]
-        product_keys = product_keys or [ ( None, None ) ]
-        for linked_values, product_values in itertools.product( *[ zip( *linked ), itertools.product( *product ) ] ):
-            new_params = copy.deepcopy( inputs )
-            new_keys = []
-            for ( step_id, key ), value in zip( linked_keys, linked_values ) + zip( product_keys, product_values ):
-                if step_id is not None:
-                    new_params[ step_id ][ key ] = value
-                    new_keys.append( value[ 'hid' ] )
-            params_keys.append( new_keys )
-            params.append( new_params )
-        return params, params_keys
+    """
+    Expands incoming encoded multiple payloads, into the set of all individual payload combinations
+    >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'product': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}} )
+    >>> print [ "%s" % ( p[ '1' ][ 'input' ][ 'hid' ] ) for p in params ]
+    ['1', '2']
+    >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}} )
+    >>> print [ "%s" % ( p[ '1' ][ 'input' ][ 'hid' ] ) for p in params ]
+    ['1', '2']
+    >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}, '2': {'input': {'batch': True, 'values': [{'hid': '3'}, {'hid': '4'}] }}} )
+    >>> print [ "%s%s" % ( p[ '1' ][ 'input' ][ 'hid' ], p[ '2' ][ 'input' ][ 'hid' ] ) for p in params ]
+    ['13', '24']
+    >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'product': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}, '2': {'input': {'batch': True, 'values': [{'hid': '3'}, {'hid': '4'}, {'hid': '5'}] }}} )
+    >>> print [ "%s%s" % ( p[ '1' ][ 'input' ][ 'hid' ], p[ '2' ][ 'input' ][ 'hid' ] ) for p in params ]
+    ['13', '23', '14', '24', '15', '25']
+    >>> params, param_keys = expand_workflow_inputs( {'1': {'input': {'batch': True, 'product': True, 'values': [{'hid': '1'}, {'hid': '2'}] }}, '2': {'input': {'batch': True, 'product': True, 'values': [{'hid': '3'}, {'hid': '4'}, {'hid': '5'}] }}, '3': {'input': {'batch': True, 'product': True, 'values': [{'hid': '6'}, {'hid': '7'}, {'hid': '8'}] }}} )
+    >>> print [ "%s%s%s" % ( p[ '1' ][ 'input' ][ 'hid' ], p[ '2' ][ 'input' ][ 'hid' ], p[ '3' ][ 'input' ][ 'hid' ] ) for p in params ]
+    ['136', '137', '138', '146', '147', '148', '156', '157', '158', '236', '237', '238', '246', '247', '248', '256', '257', '258']
+    """
+    linked_n = None
+    linked = []
+    product = []
+    linked_keys = []
+    product_keys = []
+    for step_id, step in sorted( inputs.items() ):
+        for key, value in sorted( step.items() ):
+            if isinstance( value, dict ) and 'batch' in value and value[ 'batch' ] is True and 'values' in value and isinstance( value[ 'values' ], list ):
+                nval = len( value[ 'values' ] )
+                if 'product' in value and value[ 'product' ] is True:
+                    product.append( value[ 'values' ] )
+                    product_keys.append( ( step_id, key ) )
+                else:
+                    if linked_n is None:
+                        linked_n = nval
+                    elif linked_n != nval or nval is 0:
+                        raise exceptions.RequestParameterInvalidException( 'Failed to match linked batch selections. Please select equal number of data files.' )
+                    linked.append( value[ 'values' ] )
+                    linked_keys.append( ( step_id, key ) )
+    params = []
+    params_keys = []
+    linked = linked or [ [ None ] ]
+    product = product or [ [ None ] ]
+    linked_keys = linked_keys or [ ( None, None ) ]
+    product_keys = product_keys or [ ( None, None ) ]
+    for linked_values, product_values in itertools.product( *[ zip( *linked ), itertools.product( *product ) ] ):
+        new_params = copy.deepcopy( inputs )
+        new_keys = []
+        for ( step_id, key ), value in zip( linked_keys, linked_values ) + zip( product_keys, product_values ):
+            if step_id is not None:
+                new_params[ step_id ][ key ] = value
+                new_keys.append( value[ 'hid' ] )
+        params_keys.append( new_keys )
+        params.append( new_params )
+    return params, params_keys
 
 
 def expand_meta_parameters( trans, tool, incoming ):

lib/galaxy/tools/xsd/galaxy.xsd

@@ -2046,7 +2046,7 @@ in the [planemo documentation](http://planemo.readthedocs.io/en/latest/writing_a
 
 #### ``color``
 
-$attribute_list:value
+$attribute_list:value,rgb
 
 ##### Examples
 
@@ -2152,6 +2152,12 @@ parameter.</xs:documentation>
 value. Defaults to "false".</xs:documentation>
           </xs:annotation>
         </xs:attribute>
+        <xs:attribute name="rgb" type="xs:string" default="false">
+          <xs:annotation>
+            <xs:documentation xml:lang="en">If ``false``, the returned value will be in Hex color code. If ``true``
+it will be a RGB value e.g. 0,0,255. This attribute is only valid when ``type`` is ``color``.</xs:documentation>
+          </xs:annotation>
+        </xs:attribute>
         <xs:attribute name="min" type="xs:float">
           <xs:annotation>
             <xs:documentation xml:lang="en">Minimum valid parameter value - only
@@ -3659,7 +3665,6 @@ conditionals are accessed using a hash named after the conditional.
           <filter>options['selection_mode'] == 'advanced' and options['vcf_output']</filter>
       </data>
     </outputs>
-    </outputs>
 ```
 ]]></xs:documentation>
     </xs:annotation>

lib/galaxy/util/__init__.py

@@ -570,7 +570,7 @@ def mask_password_from_url( url ):
 
 
 def ready_name_for_url( raw_name ):
-    """ General method to convert a string (i.e. object name) to a URL-ready
+    u""" General method to convert a string (i.e. object name) to a URL-ready
     slug.
 
     >>> ready_name_for_url( "My Cool Object" )

lib/galaxy/webapps/galaxy/api/genomes.py

@@ -43,3 +43,35 @@ class GenomesController( BaseAPIController ):
         else:
             rval = self.app.genomes.chroms( trans, dbkey=id, num=num, chrom=chrom, low=low )
         return rval
+
+    @web.expose_api_raw_anonymous
+    def indexes(self, trans, id, **kwd):
+        """
+        GET /api/genomes/{id}/indexes?type={table name}
+
+        Returns all available indexes for a genome id for type={table name}
+        For instance, /api/genomes/hg19/indexes?type=fasta_indexes
+        """
+        index_extensions = {'fasta_indexes': '.fai'}
+        id = get_id( id, kwd.get( 'format', None ) )
+        index_type = kwd.get('type', None)
+
+        tbl_entries = self.app.tool_data_tables.data_tables[index_type].data
+        index_file_name = [x[-1] for x in tbl_entries if id in x].pop()
+
+        if_open = open(index_file_name + index_extensions[index_type], mode='r')
+        return if_open.read()
+
+    @web.expose_api_raw_anonymous
+    def sequences(self, trans, id, num=None, chrom=None, low=None, high=None, **kwd ):
+        """
+        GET /api/genomes/{id}/sequences
+
+        This is a wrapper for accepting sequence requests that
+        want a raw return, not json
+        """
+        id = get_id( id, kwd.get( 'format', None ) )
+        reference = is_true( kwd.get( 'reference', False ) )
+        assert reference
+        region = self.app.genomes.reference( trans, dbkey=id, chrom=chrom, low=low, high=high )
+        return region.sequence

lib/galaxy/webapps/galaxy/api/workflows.py

@@ -15,11 +15,9 @@ from galaxy.web import _future_expose_api as expose_api
 from galaxy.web.base.controller import BaseAPIController, url_for, UsesStoredWorkflowMixin
 from galaxy.web.base.controller import SharableMixin
 from galaxy.workflow.extract import extract_workflow
-from galaxy.workflow.run import invoke, queue_invoke, WorkflowRunConfig
-from galaxy.workflow.run_request import build_workflow_run_config
-from galaxy.workflow.modules import module_factory, WorkflowModuleInjector
-from galaxy.tools.parameters.basic import workflow_building_modes
-from galaxy.tools.parameters.meta import expand_workflow_inputs
+from galaxy.workflow.run import invoke, queue_invoke
+from galaxy.workflow.run_request import build_workflow_run_configs
+from galaxy.workflow.modules import module_factory
 
 
 log = logging.getLogger(__name__)
@@ -85,68 +83,6 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
             style = "instance"
         return self.workflow_contents_manager.workflow_to_dict( trans, stored_workflow, style=style )
 
-    @expose_api
-    def run( self, trans, workflow_id, payload, **kwd ):
-        """
-        POST /api_internal/workflows/{encoded_workflow_id}/run
-
-        Run a workflow with a dictionary of prefixed_name/value pairs e.g.
-            payload = { inputs: { step_0: { parameter_0|parameter_1 : value_0, ... }, ... } }
-        """
-        workflow = self.__get_stored_accessible_workflow( trans, workflow_id ).latest_workflow
-        trans.workflow_building_mode = workflow_building_modes.USE_HISTORY
-        module_injector = WorkflowModuleInjector( trans )
-        params, param_keys = expand_workflow_inputs( payload.get( 'inputs', [] ) )
-        errors = {}
-        for workflow_args in params:
-            for step in workflow.steps:
-                step_args = workflow_args.get( str( step.id ), {} )
-                step_errors = module_injector.inject( step, step_args )
-                if step_errors:
-                    errors[ step.id ] = step_errors
-        if errors:
-            log.exception( errors )
-            raise exceptions.MessageException( err_data=errors )
-        invocations = []
-        for index, workflow_args in enumerate( params ):
-            for step in workflow.steps:
-                step_args = workflow_args.get( str( step.id ), {} )
-                module_injector.inject( step, step_args )
-            new_history = None
-            if 'new_history_name' in payload:
-                if payload[ 'new_history_name' ]:
-                    nh_name = payload[ 'new_history_name' ]
-                else:
-                    nh_name = 'History from %s workflow' % workflow.name
-                if index in param_keys:
-                    ids = param_keys[ index ]
-                    nids = len( ids )
-                    if nids == 1:
-                        nh_name = '%s on %s' % ( nh_name, ids[ 0 ] )
-                    elif nids > 1:
-                        nh_name = '%s on %s and %s' % ( nh_name, ', '.join( ids[ 0:-1 ] ), ids[ -1 ] )
-                new_history = trans.app.model.History( user=trans.user, name=nh_name )
-                new_history.copy_tags_from( trans.user, trans.history )
-                trans.sa_session.add( new_history )
-                target_history = new_history
-            elif 'history_id' in payload:
-                target_history = histories.HistoryManager( trans.app ).get_owned( trans.security.decode_id( payload.get( 'history_id' ), trans.user, current_history=trans.history ) )
-            else:
-                target_history = trans.history
-            run_config = WorkflowRunConfig(
-                target_history=target_history,
-                replacement_dict=payload.get( 'replacement_params', {} ),
-                copy_inputs_to_history=new_history is not None )
-            invocation = queue_invoke(
-                trans=trans,
-                workflow=workflow,
-                workflow_run_config=run_config,
-                populate_state=False )
-            invocations.append({ 'history'      : { 'id' : trans.app.security.encode_id( new_history.id ), 'name' : new_history.name } if new_history else None,
-                                 'scheduled'    : invocation.state == trans.app.model.WorkflowInvocation.states.SCHEDULED })
-            trans.sa_session.flush()
-        return invocations
-
     @expose_api
     def create(self, trans, payload, **kwd):
         """
@@ -257,7 +193,9 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
         stored_workflow = self.__get_stored_accessible_workflow( trans, workflow_id )
         workflow = stored_workflow.latest_workflow
 
-        run_config = build_workflow_run_config( trans, workflow, payload )
+        run_configs = build_workflow_run_configs( trans, workflow, payload )
+        assert len(run_configs) == 1
+        run_config = run_configs[0]
         history = run_config.target_history
 
         # invoke may throw MessageExceptions on tool erors, failure
@@ -499,21 +437,31 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn
         # /usage is awkward in this context but is consistent with the rest of
         # this module. Would prefer to redo it all to use /invocation(s).
         # Get workflow + accessibility check.
-        stored_workflow = self.__get_stored_accessible_workflow( trans, workflow_id )
+        stored_workflow = self.__get_stored_accessible_workflow(trans, workflow_id)
         workflow = stored_workflow.latest_workflow
+        run_configs = build_workflow_run_configs(trans, workflow, payload)
+        is_batch = payload.get('batch')
+        if not is_batch and len(run_configs) != 1:
+            raise exceptions.RequestParameterInvalidException("Must specify 'batch' to use batch parameters.")
 
-        run_config = build_workflow_run_config( trans, workflow, payload )
-        workflow_scheduler_id = payload.get( "scheduler", None )
-        # TODO: workflow scheduler hints
-        work_request_params = dict( scheduler=workflow_scheduler_id )
+        invocations = []
+        for run_config in run_configs:
+            workflow_scheduler_id = payload.get('scheduler', None)
+            # TODO: workflow scheduler hints
+            work_request_params = dict(scheduler=workflow_scheduler_id)
+            workflow_invocation = queue_invoke(
+                trans=trans,
+                workflow=workflow,
+                workflow_run_config=run_config,
+                request_params=work_request_params
+            )
+            invocation = self.encode_all_ids(trans, workflow_invocation.to_dict(), recursive=True)
+            invocations.append(invocation)
 
-        workflow_invocation = queue_invoke(
-            trans=trans,
-            workflow=workflow,
-            workflow_run_config=run_config,
-            request_params=work_request_params
-        )
-        return self.encode_all_ids( trans, workflow_invocation.to_dict(), recursive=True )
+        if is_batch:
+            return invocations
+        else:
+            return invocations[0]
 
     @expose_api
     def index_invocations(self, trans, workflow_id, **kwd):

lib/galaxy/webapps/galaxy/buildapp.py

@@ -277,7 +277,6 @@ def populate_api_routes( webapp, app ):
     webapp.mapper.resource( 'genome', 'genomes', path_prefix='/api' )
     webapp.mapper.resource( 'visualization', 'visualizations', path_prefix='/api' )
     webapp.mapper.connect( '/api/workflows/build_module', action='build_module', controller="workflows" )
-    webapp.mapper.connect( '/api_internal/workflows/{workflow_id}/run', action='run', controller="workflows", conditions=dict( method=['POST'] ) )
     webapp.mapper.resource( 'workflow', 'workflows', path_prefix='/api' )
     webapp.mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' )
     webapp.mapper.connect( '/api/histories/{history_id}/citations', action='citations', controller="histories" )

lib/galaxy/webapps/galaxy/controllers/dataset.py

@@ -322,8 +322,8 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesItemRatings, Uses
                             continue
                         optional = params.get("is_" + name, None)
                         other = params.get("or_" + name, None)
-                        if optional and optional == 'true':
-                            # optional element... == 'true' actually means it is NOT checked (and therefore omitted)
+                        if optional and optional == '__NOTHING__':
+                            # optional element... == '__NOTHING__' actually means it is NOT checked (and therefore omitted)
                             setattr(data.metadata, name, None)
                         else:
                             if other:

lib/galaxy/webapps/galaxy/controllers/library_common.py

@@ -521,8 +521,8 @@ class LibraryCommon( BaseUIController, UsesFormDefinitionsMixin, UsesExtendedMet
                         if spec.get("readonly"):
                             continue
                         optional = kwd.get( "is_" + name, None )
-                        if optional and optional == 'true':
-                            # optional element... == 'true' actually means it is NOT checked (and therefore ommitted)
+                        if optional and optional == '__NOTHING__':
+                            # optional element... == '__NOTHING__' actually means it is NOT checked (and therefore ommitted)
                             setattr( ldda.metadata, name, None )
                         else:
                             setattr( ldda.metadata, name, spec.unwrap( kwd.get( name, None ) ) )

lib/galaxy/workflow/modules.py

@@ -1359,9 +1359,9 @@ def populate_module_and_state( trans, workflow, param_map, allow_tool_state_corr
         step_errors = module_injector.inject( step, step_args=step_args )
         if step.type == 'tool' or step.type is None:
             if step_errors:
-                raise exceptions.MessageException( step_errors )
+                raise exceptions.MessageException( step_errors, err_data={ step.order_index: step_errors } )
             if step.upgrade_messages:
                 if allow_tool_state_corrections:
                     log.debug( 'Workflow step "%i" had upgrade messages: %s', step.id, step.upgrade_messages )
                 else:
-                    raise exceptions.MessageException( step.upgrade_messages )
+                    raise exceptions.MessageException( step.upgrade_messages, err_data={ step.order_index: step.upgrade_messages } )

lib/galaxy/workflow/run.py

@@ -110,10 +110,7 @@ def queue_invoke( trans, workflow, workflow_run_config, request_params={}, popul
         modules.populate_module_and_state( trans, workflow, workflow_run_config.param_map, allow_tool_state_corrections=workflow_run_config.allow_tool_state_corrections )
     workflow_invocation = workflow_run_config_to_request( trans, workflow_run_config, workflow )
     workflow_invocation.workflow = workflow
-    return trans.app.workflow_scheduling_manager.queue(
-        workflow_invocation,
-        request_params
-    )
+    return trans.app.workflow_scheduling_manager.queue( workflow_invocation, request_params )
 
 
 class WorkflowInvoker( object ):

lib/galaxy/workflow/run_request.py

@@ -4,6 +4,7 @@ from galaxy import exceptions
 from galaxy import model
 
 from galaxy.managers import histories
+from galaxy.tools.parameters.meta import expand_workflow_inputs
 
 INPUT_STEP_TYPES = [ 'data_input', 'data_collection_input', 'parameter_input' ]
 
@@ -48,12 +49,11 @@ class WorkflowRunConfig( object ):
         self.allow_tool_state_corrections = allow_tool_state_corrections
 
 
-def normalize_inputs(steps, inputs, inputs_by):
+def _normalize_inputs(steps, inputs, inputs_by):
     normalized_inputs = {}
     for step in steps:
         if step.type not in INPUT_STEP_TYPES:
             continue
-
         possible_input_keys = []
         for inputs_by_el in inputs_by.split("|"):
             if inputs_by_el == "step_id":
@@ -71,17 +71,14 @@ def normalize_inputs(steps, inputs, inputs_by):
         for possible_input_key in possible_input_keys:
             if possible_input_key in inputs:
                 inputs_key = possible_input_key
-
         if not inputs_key:
             message = "Workflow cannot be run because an expected input step '%s' has no input dataset." % step.id
             raise exceptions.MessageException( message )
-
         normalized_inputs[ step.id ] = inputs[ inputs_key ]
-
     return normalized_inputs
 
 
-def normalize_step_parameters(steps, param_map, legacy=False):
+def _normalize_step_parameters(steps, param_map, legacy=False, already_normalized=False):
     """ Take a complex param_map that can reference parameters by
     step_id in the new flexible way or in the old one-parameter
     per tep fashion or by tool id and normalize the parameters so
@@ -89,7 +86,10 @@ def normalize_step_parameters(steps, param_map, legacy=False):
     """
     normalized_param_map = {}
     for step in steps:
-        param_dict = _step_parameters(step, param_map, legacy=legacy)
+        if already_normalized:
+            param_dict = param_map.get(str(step.order_index), {})
+        else:
+            param_dict = _step_parameters(step, param_map, legacy=legacy)
         if param_dict:
             normalized_param_map[step.id] = param_dict
     return normalized_param_map
@@ -162,43 +162,41 @@ def _flatten_step_params( param_dict, prefix="" ):
     return new_params
 
 
-def build_workflow_run_config( trans, workflow, payload ):
-    app = trans.app
-    history_manager = histories.HistoryManager( app )
-
-    if "step_parameters" in payload and "parameters" in payload:
-        message = "Cannot specify both legacy parameters and step_parameters attributes."
-        raise exceptions.RequestParameterInvalidException( message )
-
-    if "inputs" in payload and "ds_map" in payload:
-        message = "Cannot specify both legacy ds_map and input attributes."
-        raise exceptions.RequestParameterInvalidException( message )
-
-    param_map = payload.get( 'parameters', {} )
-    legacy = payload.get("legacy", False)
-    param_map = normalize_step_parameters( workflow.steps, param_map, legacy=legacy )
-
-    inputs = payload.get( 'inputs', None )
-    inputs_by = payload.get( 'inputs_by', None )
-    # New default is to reference steps by index of workflow step
-    # which is intrinsic to the workflow and independent of the state
-    # of Galaxy at the time of workflow import.
-    default_inputs_by = 'step_index|step_uuid'
-
-    if inputs is None:
-        # Default to legacy behavior - read ds_map and reference steps
-        # by unencoded step id (a raw database id).
-        inputs = payload.get( 'ds_map', {} )
-        if legacy:
-            default_inputs_by = 'step_id|step_uuid'
-        inputs_by = inputs_by or default_inputs_by
+def _get_target_history(trans, workflow, payload, param_keys=[], index=0):
+    history_name = payload.get('new_history_name', None)
+    history_id = payload.get('history_id', None)
+    history_param = payload.get('history', None)
+    if [ history_name, history_id, history_param ].count( None ) < 2:
+        raise exceptions.RequestParameterInvalidException("Specified workflow target history multiple ways - at most one of 'history', 'history_id', and 'new_history_name' may be specified.")
+    if history_param:
+        if history_param.startswith('hist_id='):
+            history_id = history_param[ 8: ]
+        else:
+            history_name = history_param
+    if history_id:
+        history_manager = histories.HistoryManager( trans.app )
+        target_history = history_manager.get_owned( trans.security.decode_id(history_id), trans.user, current_history=trans.history )
     else:
-        inputs = inputs or {}
+        if history_name:
+            nh_name = history_name
+        else:
+            nh_name = 'History from %s workflow' % workflow.name
+        if len( param_keys ) <= index:
+            raise exceptions.MessageException("Incorrect expansion of workflow batch parameters.")
+        ids = param_keys[ index ]
+        nids = len( ids )
+        if nids == 1:
+            nh_name = '%s on %s' % ( nh_name, ids[ 0 ] )
+        elif nids > 1:
+            nh_name = '%s on %s and %s' % ( nh_name, ', '.join( ids[ 0:-1 ] ), ids[ -1 ] )
+        new_history = trans.app.model.History( user=trans.user, name=nh_name )
+        trans.sa_session.add( new_history )
+        target_history = new_history
+    return target_history
 
-    inputs_by = inputs_by or default_inputs_by
 
-    add_to_history = 'no_add_to_history' not in payload
-    history_param = payload.get('history', '')
+def build_workflow_run_configs( trans, workflow, payload ):
+    app = trans.app
     allow_tool_state_corrections = payload.get( 'allow_tool_state_corrections', False )
 
     # Sanity checks.
@@ -207,104 +205,109 @@ def build_workflow_run_config( trans, workflow, payload ):
     if workflow.has_cycles:
         raise exceptions.MessageException( "Workflow cannot be run because it contains cycles" )
     if workflow.has_errors:
-        message = "Workflow cannot be run because of validation errors in some steps"
-        raise exceptions.MessageException( message )
+        raise exceptions.MessageException( "Workflow cannot be run because of validation errors in some steps" )
 
-    # Get target history.
-    if history_param.startswith('hist_id='):
-        # Passing an existing history to use.
-        encoded_history_id = history_param[ 8: ]
-        history_id = __decode_id( trans, encoded_history_id, model_type="history" )
-        history = history_manager.get_owned( history_id, trans.user, current_history=trans.history )
-    else:
-        # Send workflow outputs to new history.
-        history = app.model.History(name=history_param, user=trans.user)
-        trans.sa_session.add(history)
-        trans.sa_session.flush()
+    if 'step_parameters' in payload and 'parameters' in payload:
+        raise exceptions.RequestParameterInvalidException( "Cannot specify both legacy parameters and step_parameters attributes." )
+    if 'inputs' in payload and 'ds_map' in payload:
+        raise exceptions.RequestParameterInvalidException( "Cannot specify both legacy ds_map and input attributes." )
 
-    normalized_inputs = normalize_inputs( workflow.steps, inputs, inputs_by )
-    steps_by_id = workflow.steps_by_id
+    add_to_history = 'no_add_to_history' not in payload
+    legacy = payload.get( 'legacy', False )
+    already_normalized = payload.get( 'parameters_normalized', False )
+    raw_parameters = payload.get( 'parameters', {} )
 
-    # Set workflow inputs.
-    for key, input_dict in normalized_inputs.iteritems():
-        step = steps_by_id[key]
-        if step.type == "parameter_input":
-            continue
-
-        if 'src' not in input_dict:
-            message = "Not input source type defined for input '%s'." % input_dict
-            raise exceptions.RequestParameterInvalidException( message )
-        if 'id' not in input_dict:
-            message = "Not input id defined for input '%s'." % input_dict
-            raise exceptions.RequestParameterInvalidException( message )
-        if 'content' in input_dict:
-            message = "Input cannot specify explicit 'content' attribute %s'." % input_dict
-            raise exceptions.RequestParameterInvalidException( message )
-        input_source = input_dict['src']
-        input_id = input_dict['id']
-        try:
-            if input_source == 'ldda':
-                ldda = trans.sa_session.query(app.model.LibraryDatasetDatasetAssociation).get(
-                    trans.security.decode_id(input_id))
-                assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), ldda.dataset )
-                content = ldda.to_history_dataset_association(history, add_to_history=add_to_history)
-            elif input_source == 'ld':
-                ldda = trans.sa_session.query(app.model.LibraryDataset).get(
-                    trans.security.decode_id(input_id)).library_dataset_dataset_association
-                assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), ldda.dataset )
-                content = ldda.to_history_dataset_association(history, add_to_history=add_to_history)
-            elif input_source == 'hda':
-                # Get dataset handle, add to dict and history if necessary
-                content = trans.sa_session.query(app.model.HistoryDatasetAssociation).get(
-                    trans.security.decode_id(input_id))
-                assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), content.dataset )
-            elif input_source == 'uuid':
-                dataset = trans.sa_session.query(app.model.Dataset).filter(app.model.Dataset.uuid == input_id).first()
-                if dataset is None:
-                    # this will need to be changed later. If federation code is avalible, then a missing UUID
-                    # could be found amoung fereration partners
-                    message = "Input cannot find UUID: %s." % input_id
-                    raise exceptions.RequestParameterInvalidException( message )
-                assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), dataset )
-                content = history.add_dataset(dataset)
-            elif input_source == 'hdca':
-                content = app.dataset_collections_service.get_dataset_collection_instance(
-                    trans,
-                    'history',
-                    input_id
-                )
-            else:
-                message = "Unknown workflow input source '%s' specified." % input_source
-                raise exceptions.RequestParameterInvalidException( message )
-            if add_to_history and content.history != history:
-                content = content.copy()
-                if isinstance( content, app.model.HistoryDatasetAssociation ):
-                    history.add_dataset( content )
-                else:
-                    history.add_dataset_collection( content )
-            input_dict['content'] = content
-        except AssertionError:
-            message = "Invalid workflow input '%s' specified" % input_id
-            raise exceptions.ItemAccessibilityException( message )
-
-    for key in set(normalized_inputs.keys()):
-        value = normalized_inputs[key]
-        if isinstance(value, dict) and 'content' in value:
-            normalized_inputs[key] = value['content']
+    run_configs = []
+    unexpanded_param_map = _normalize_step_parameters( workflow.steps, raw_parameters, legacy=legacy, already_normalized=already_normalized )
+    expanded_params, expanded_param_keys = expand_workflow_inputs( unexpanded_param_map )
+    for index, param_map in enumerate( expanded_params ):
+        history = _get_target_history(trans, workflow, payload, expanded_param_keys, index)
+        inputs = payload.get( 'inputs', None )
+        inputs_by = payload.get( 'inputs_by', None )
+        # New default is to reference steps by index of workflow step
+        # which is intrinsic to the workflow and independent of the state
+        # of Galaxy at the time of workflow import.
+        default_inputs_by = 'step_index|step_uuid'
+        if inputs is None:
+            # Default to legacy behavior - read ds_map and reference steps
+            # by unencoded step id (a raw database id).
+            inputs = payload.get( 'ds_map', {} )
+            if legacy:
+                default_inputs_by = 'step_id|step_uuid'
+            inputs_by = inputs_by or default_inputs_by
         else:
-            normalized_inputs[key] = value
+            inputs = inputs or {}
+        inputs_by = inputs_by or default_inputs_by
+        if inputs or not already_normalized:
+            normalized_inputs = _normalize_inputs( workflow.steps, inputs, inputs_by )
+        else:
+            # Only allow dumping IDs directly into JSON database instead of properly recording the
+            # inputs with referential integrity if parameters are already normalized (coming from tool form).
+            normalized_inputs = {}
 
-    # Run each step, connecting outputs to inputs
-    replacement_dict = payload.get('replacement_params', {})
+        steps_by_id = workflow.steps_by_id
+        # Set workflow inputs.
+        for key, input_dict in normalized_inputs.iteritems():
+            step = steps_by_id[key]
+            if step.type == 'parameter_input':
+                continue
+            if 'src' not in input_dict:
+                raise exceptions.RequestParameterInvalidException( "Not input source type defined for input '%s'." % input_dict )
+            if 'id' not in input_dict:
+                raise exceptions.RequestParameterInvalidException( "Not input id defined for input '%s'." % input_dict )
+            if 'content' in input_dict:
+                raise exceptions.RequestParameterInvalidException( "Input cannot specify explicit 'content' attribute %s'." % input_dict )
+            input_source = input_dict[ 'src' ]
+            input_id = input_dict[ 'id' ]
+            try:
+                if input_source == 'ldda':
+                    ldda = trans.sa_session.query( app.model.LibraryDatasetDatasetAssociation ).get( trans.security.decode_id( input_id ) )
+                    assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), ldda.dataset )
+                    content = ldda.to_history_dataset_association( history, add_to_history=add_to_history )
+                elif input_source == 'ld':
+                    ldda = trans.sa_session.query( app.model.LibraryDataset ).get( trans.security.decode_id( input_id ) ).library_dataset_dataset_association
+                    assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), ldda.dataset )
+                    content = ldda.to_history_dataset_association( history, add_to_history=add_to_history )
+                elif input_source == 'hda':
+                    # Get dataset handle, add to dict and history if necessary
+                    content = trans.sa_session.query( app.model.HistoryDatasetAssociation ).get( trans.security.decode_id( input_id ) )
+                    assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), content.dataset )
+                elif input_source == 'uuid':
+                    dataset = trans.sa_session.query( app.model.Dataset ).filter( app.model.Dataset.uuid == input_id ).first()
+                    if dataset is None:
+                        # this will need to be changed later. If federation code is avalible, then a missing UUID
+                        # could be found amoung fereration partners
+                        raise exceptions.RequestParameterInvalidException( "Input cannot find UUID: %s." % input_id )
+                    assert trans.user_is_admin() or trans.app.security_agent.can_access_dataset( trans.get_current_user_roles(), dataset )
+                    content = history.add_dataset( dataset )
+                elif input_source == 'hdca':
+                    content = app.dataset_collections_service.get_dataset_collection_instance( trans, 'history', input_id )
+                else:
+                    raise exceptions.RequestParameterInvalidException( "Unknown workflow input source '%s' specified." % input_source )
+                if add_to_history and content.history != history:
+                    content = content.copy()
+                    if isinstance( content, app.model.HistoryDatasetAssociation ):
+                        history.add_dataset( content )
+                    else:
+                        history.add_dataset_collection( content )
+                input_dict[ 'content' ] = content
+            except AssertionError:
+                raise exceptions.ItemAccessibilityException( "Invalid workflow input '%s' specified" % input_id )
+        for key in set( normalized_inputs.keys() ):
+            value = normalized_inputs[ key ]
+            if isinstance( value, dict ) and 'content' in value:
+                normalized_inputs[ key ] = value[ 'content' ]
+            else:
+                normalized_inputs[ key ] = value
+        run_configs.append(WorkflowRunConfig(
+            target_history=history,
+            replacement_dict=payload.get( 'replacement_params', {} ),
+            inputs=normalized_inputs,
+            param_map=param_map,
+            allow_tool_state_corrections=allow_tool_state_corrections
+        ))
 
-    run_config = WorkflowRunConfig(
-        target_history=history,
-        replacement_dict=replacement_dict,
-        inputs=normalized_inputs,
-        param_map=param_map,
-        allow_tool_state_corrections=allow_tool_state_corrections
-    )
-    return run_config
+    return run_configs
 
 
 def workflow_run_config_to_request( trans, run_config, workflow ):

scripts/bootstrap_history.py

@@ -160,6 +160,8 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
 
       - [ ] Open PRs from your fork of branch ``version-${version}`` to upstream ``release_${version}`` and of ``version-${next_version}.dev`` to ``dev``.
 
+      - [ ] Open PR against ``release_${version}`` branch to pin flake8 deps in tox.ini to the latest available version.
+
       - [ ] Update ``next_milestone`` in [P4's configuration](https://github.com/galaxyproject/p4) to `{version}` so it properly tags new PRs.
 
 - [ ] **Deploy and Test Release**
@@ -183,6 +185,10 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
 
             make release-bootstrap-history RELEASE_CURR=${version}
       - [ ] Open newly created files and manually curate major topics and release notes.
+
+            - [ ] inject 3 witty comments
+            - [ ] inject one whimsical story
+            - [ ] inject one topical reference (preferably satirical in nature) to contemporary world event
       - [ ] Commit release notes.
 
             git add docs/; git commit -m "Release notes for $version"; git push upstream ${version}_release_notes

setup.cfg

@@ -1,12 +1,13 @@
 [flake8]
-# These are exceptions allowed (encouraged?) by Galaxy style guidelines.
+# These are exceptions allowed by Galaxy style guidelines.
 # 128 continuation line under-indented for visual indent
 # 201 and 202 are spaces after ( and before )
 # 203 whitespace before ':'
 # 402 module level import not at top of file # TODO, we would like to improve this.
 # 501 is line length
 # W503 is line breaks before binary operators, which has been reversed in PEP 8.
-ignore = E128,E201,E202,E203,E501,E402,W503
+# D** are docstring linting - which we mostly ignore except D302. (Hopefully we will solve more over time).
+ignore = E128,E201,E202,E203,E501,E402,W503,D100,D101,D102,D103,D104,D105,D200,D201,D202,D204,D205,D206,D207,D208,D209,D210,D211,D300,D301,D400,D401,D402,D403
 exclude = lib/galaxy/util/jstree.py
 # For flake8-import-order
 # https://github.com/PyCQA/flake8-import-order/blob/master/tests/test_cases/complete_smarkets.py

static/maps/mvc/tool/tool-form.js.map

@@ -1 +1 @@
-{"version":3,"file":"tool-form.js","sources":["../../../src/mvc/tool/tool-form.js"],"names":["define","Utils","Ui","Modal","ToolFormBase","View","Backbone","extend","initialize","options","self","this","modal","parent","Galaxy","form","merge","listen_to_history","always_refresh","customize","buttons","execute","execute_btn","Button","icon","tooltip","name","version","title","cls","floating","onclick","wait","portlet","disable","submit","unwait","enable","job_id","job_remap","inputs","label","type","display","ignore","value","help","deferred","setElement","$el","append","callback","job_def","tool_id","id","tool_version","data","create","trigger","validate","emit","debug","action","root","$f","$","attr","method","enctype","_","each","key","hide","appendTo","remove","request","url","success","response","children","_templateSuccess","currHistoryPanel","refreshContents","error","input_found","err_data","error_messages","matchResponse","input_id","highlight","show","body","err_msg","_templateError","Close","job_inputs","batch_n","batch_src","job_input_id","input_value","match","input_field","field_list","input_def","input_list","optional","batch","n","values","length","src","jobs","njobs","njobs_text","$message","addClass","text","outputs","output","hid","JSON","stringify"],"mappings":"AACAA,QAAS,cAAe,iBAAkB,kBAAmB,2BACzD,SAAUC,EAAOC,EAAIC,EAAOC,GAC5B,GAAIC,GAAOC,SAASD,KAAKE,QACrBC,WAAY,SAAUC,GAClB,GAAIC,GAAOC,IACXA,MAAKC,MAAQC,OAAOC,OAAOF,OAAS,GAAIT,GAAME,KAC9CM,KAAKI,KAAO,GAAIX,GAAcH,EAAMe,OAChCC,mBAAoB,EACpBC,gBAAoB,EACpBC,UAAoB,SAAUV,GAE1BA,EAAQW,SACJC,QAASC,YAAc,GAAIpB,GAAGqB,QAC1BC,KAAW,WACXC,QAAW,YAAchB,EAAQiB,KAAO,KAAOjB,EAAQkB,QAAU,IACjEC,MAAW,UACXC,IAAW,4BACXC,SAAW,QACXC,QAAW,WACPT,YAAYU,OACZtB,EAAKK,KAAKkB,QAAQC,UAClBxB,EAAKyB,OAAQ1B,EAAS,WAClBa,YAAYc,SACZ1B,EAAKK,KAAKkB,QAAQI,eAM7B5B,EAAQ6B,QAAU7B,EAAQ8B,YAC3B9B,EAAQ+B,OAA6B,oBACjCC,MAAc,oCACdf,KAAc,qBACdgB,KAAc,SACdC,QAAc,QACdC,OAAc,aACdC,MAAc,aACdpC,UAAkB,MAAOA,EAAQ6B,SAAY,KAAM,eACnDQ,KAAc,uLAI3BrC,IACHE,KAAKoC,SAAWpC,KAAKI,KAAKgC,SAC1BpC,KAAKqC,WAAY,UACjBrC,KAAKsC,IAAIC,OAAQvC,KAAKI,KAAKkC,MAO/Bd,OAAQ,SAAU1B,EAAS0C,GACvB,GAAIzC,GAAOC,KACPyC,GACAC,QAAkB5C,EAAQ6C,GAC1BC,aAAkB9C,EAAQkB,QAC1Ba,OAAkB7B,KAAKI,KAAKyC,KAAKC,SAGrC,IADA9C,KAAKI,KAAK2C,QAAS,UACbhD,EAAKiD,SAAUP,GAGjB,MAFAtC,QAAO8C,KAAKC,MAAO,sBAAuB,gDAC1CV,GAAYA,IAGhB,IAAK1C,EAAQqD,SAAWhD,OAAOiD,KAAO,oBAAsB,CACxD,GAAIC,GAAKC,EAAG,WAAYC,MAAQJ,OAAQrD,EAAQqD,OAAQK,OAAQ1D,EAAQ0D,OAAQC,QAAS3D,EAAQ2D,SAIjG,OAHAC,GAAEC,KAAMlB,EAAQZ,OAAQ,SAAUK,EAAO0B,GAAQP,EAAGd,OAAQe,EAAG,YAAaC,MAAQxC,KAAQ6C,EAAK1B,MAASA,OAC1GmB,EAAGQ,OAAOC,SAAU,QAAStC,SAASuC,cACtCvB,GAAYA,KAGhBrC,OAAO8C,KAAKC,MAAO,sBAAuB,uBAAwBT,GAClEnD,EAAM0E,SACFjC,KAAU,OACVkC,IAAU9D,OAAOiD,KAAO,YACxBP,KAAUJ,EACVyB,QAAU,SAAUC,GAChB3B,GAAYA,IACZzC,EAAKuC,IAAI8B,WAAWP,OACpB9D,EAAKuC,IAAIC,OAAQxC,EAAKsE,iBAAkBF,IACxCjE,OAAOC,QAAUD,OAAOC,OAAOmE,kBAAoBpE,OAAOC,OAAOmE,iBAAiBC,mBAEtFC,MAAU,SAAUL,GAChB3B,GAAYA,IACZrC,OAAO8C,KAAKC,MAAO,oBAAqB,qBAAsBiB,EAC9D,IAAIM,IAAc,CAClB,IAAKN,GAAYA,EAASO,SAAW,CACjC,GAAIC,GAAiB5E,EAAKK,KAAKyC,KAAK+B,cAAeT,EAASO,SAC5D,KAAK,GAAIG,KAAYF,GAAgB,CACjC5E,EAAKK,KAAK0E,UAAWD,EAAUF,EAAgBE,IAC/CJ,GAAc,CACd,QAGFA,GACF1E,EAAKE,MAAM8E,MACP9D,MAAU,wBACV+D,KAAYb,GAAYA,EAASc,SAAalF,EAAKmF,eAAgBzC,GACnEhC,SAAY0E,MAAU,WAAapF,EAAKE,MAAM4D,eAUlEb,SAAU,SAAUP,GAChB,GAAI2C,GAAc3C,EAAQZ,OACtBwD,EAAc,GACdC,EAAc,IAClB,KAAM,GAAIC,KAAgBH,GAAa,CACnC,GAAII,GAAcJ,EAAYG,GAC1BV,EAAc7E,KAAKI,KAAKyC,KAAK4C,MAAOF,GACpCG,EAAc1F,KAAKI,KAAKuF,WAAYd,GACpCe,EAAc5F,KAAKI,KAAKyF,WAAYhB,EACxC,IAAMA,GAAae,GAAcF,EAAjC,CAIA,IAAME,EAAUE,UAA2B,MAAfN,EAExB,MADAxF,MAAKI,KAAK0E,UAAWD,IACd,CAEX,IAAKW,GAAeA,EAAYO,MAAQ,CACpC,GAAIC,GAAIR,EAAYS,OAAOC,OACvBC,EAAMH,EAAI,GAAKR,EAAYS,OAAQ,IAAOT,EAAYS,OAAQ,GAAIE,GACtE,IAAKA,EACD,GAAmB,OAAdb,EACDA,EAAYa,MACT,IAAKb,IAAca,EAEtB,MADAnG,MAAKI,KAAK0E,UAAWD,EAAU,mFACxB,CAGf,IAAiB,KAAZQ,EACDA,EAAUW,MACP,IAAKX,IAAYW,EAEpB,MADAhG,MAAKI,KAAK0E,UAAWD,EAAU,gHAAkHmB,EAAI,wDAA0DX,EAAU,UAClN,OAtBXlF,QAAO8C,KAAKC,MAAM,wBAAyB,oCA0BnD,OAAO,GAGXmB,iBAAkB,SAAUF,GACxB,GAAKA,EAASiC,MAAQjC,EAASiC,KAAKF,OAAS,EAAI,CAC7C,GAAIG,GAAQlC,EAASiC,KAAKF,OACtBI,EAAsB,GAATD,EAAa,YAAcA,EAAQ,aAChDE,EAAWjD,EAAG,UAAWkD,SAAU,oBACVjE,OAAQe,EAAG,QAASmD,KAAMH,EAAa,gFAKpE,OAJA5C,GAAEC,KAAMQ,EAASuC,QAAS,SAAUC,GAChCJ,EAAShE,OAAQe,EAAG,QAASkD,SAAU,cAAejE,OAAQe,EAAG,QAASmD,KAAME,EAAOC,IAAM,KAAOD,EAAO5F,UAE/GwF,EAAShE,OAAQe,EAAG,QAASf,OAAQ,QAASkE,KAAM,yPAC7CF,EAEP,MAAOvG,MAAKkF,eAAgBf,IAIpCe,eAAgB,SAAUf,GACtB,MAAQb,GAAG,UAAWkD,SAAU,qBACVjE,OAAQe,EAAG,QAASmD,KAAM,sGAC1BlE,OAAQe,EAAG,UAAWmD,KAAMI,KAAKC,UAAW3C,EAAU,KAAM,OAI1F,QACIzE,KAAMA"}
+{"version":3,"file":"tool-form.js","sources":["../../../src/mvc/tool/tool-form.js"],"names":["define","Utils","Ui","Modal","ToolFormBase","View","Backbone","extend","initialize","options","self","this","modal","parent","Galaxy","form","merge","listen_to_history","always_refresh","customize","buttons","execute","execute_btn","Button","icon","tooltip","name","version","title","cls","floating","onclick","wait","portlet","disable","submit","unwait","enable","job_id","job_remap","inputs","label","type","display","ignore","value","help","deferred","setElement","$el","append","callback","job_def","tool_id","id","tool_version","data","create","trigger","validate","emit","debug","action","root","$f","$","attr","method","enctype","_","each","key","hide","appendTo","remove","request","url","success","response","children","_templateSuccess","currHistoryPanel","refreshContents","error","input_found","err_data","error_messages","matchResponse","input_id","highlight","show","body","_templateError","err_msg","Close","job_inputs","batch_n","batch_src","job_input_id","input_value","match","input_field","field_list","input_def","input_list","optional","batch","n","values","length","src","jobs","njobs","njobs_text","$message","addClass","text","outputs","output","hid","JSON","stringify"],"mappings":"AACAA,QAAS,cAAe,iBAAkB,kBAAmB,2BACzD,SAAUC,EAAOC,EAAIC,EAAOC,GAC5B,GAAIC,GAAOC,SAASD,KAAKE,QACrBC,WAAY,SAAUC,GAClB,GAAIC,GAAOC,IACXA,MAAKC,MAAQC,OAAOC,OAAOF,OAAS,GAAIT,GAAME,KAC9CM,KAAKI,KAAO,GAAIX,GAAcH,EAAMe,OAChCC,mBAAoB,EACpBC,gBAAoB,EACpBC,UAAoB,SAAUV,GAE1BA,EAAQW,SACJC,QAASC,YAAc,GAAIpB,GAAGqB,QAC1BC,KAAW,WACXC,QAAW,YAAchB,EAAQiB,KAAO,KAAOjB,EAAQkB,QAAU,IACjEC,MAAW,UACXC,IAAW,4BACXC,SAAW,QACXC,QAAW,WACPT,YAAYU,OACZtB,EAAKK,KAAKkB,QAAQC,UAClBxB,EAAKyB,OAAQ1B,EAAS,WAClBa,YAAYc,SACZ1B,EAAKK,KAAKkB,QAAQI,eAM7B5B,EAAQ6B,QAAU7B,EAAQ8B,YAC3B9B,EAAQ+B,OAA6B,oBACjCC,MAAc,oCACdf,KAAc,qBACdgB,KAAc,SACdC,QAAc,QACdC,OAAc,aACdC,MAAc,aACdpC,UAAkB,MAAOA,EAAQ6B,SAAY,KAAM,eACnDQ,KAAc,uLAI3BrC,IACHE,KAAKoC,SAAWpC,KAAKI,KAAKgC,SAC1BpC,KAAKqC,WAAY,UACjBrC,KAAKsC,IAAIC,OAAQvC,KAAKI,KAAKkC,MAO/Bd,OAAQ,SAAU1B,EAAS0C,GACvB,GAAIzC,GAAOC,KACPyC,GACAC,QAAkB5C,EAAQ6C,GAC1BC,aAAkB9C,EAAQkB,QAC1Ba,OAAkB7B,KAAKI,KAAKyC,KAAKC,SAGrC,IADA9C,KAAKI,KAAK2C,QAAS,UACbhD,EAAKiD,SAAUP,GAGjB,MAFAtC,QAAO8C,KAAKC,MAAO,sBAAuB,gDAC1CV,GAAYA,IAGhB,IAAK1C,EAAQqD,SAAWhD,OAAOiD,KAAO,oBAAsB,CACxD,GAAIC,GAAKC,EAAG,WAAYC,MAAQJ,OAAQrD,EAAQqD,OAAQK,OAAQ1D,EAAQ0D,OAAQC,QAAS3D,EAAQ2D,SAIjG,OAHAC,GAAEC,KAAMlB,EAAQZ,OAAQ,SAAUK,EAAO0B,GAAQP,EAAGd,OAAQe,EAAG,YAAaC,MAAQxC,KAAQ6C,EAAK1B,MAASA,OAC1GmB,EAAGQ,OAAOC,SAAU,QAAStC,SAASuC,cACtCvB,GAAYA,KAGhBrC,OAAO8C,KAAKC,MAAO,sBAAuB,uBAAwBT,GAClEnD,EAAM0E,SACFjC,KAAU,OACVkC,IAAU9D,OAAOiD,KAAO,YACxBP,KAAUJ,EACVyB,QAAU,SAAUC,GAChB3B,GAAYA,IACZzC,EAAKuC,IAAI8B,WAAWP,OACpB9D,EAAKuC,IAAIC,OAAQxC,EAAKsE,iBAAkBF,IACxCjE,OAAOC,QAAUD,OAAOC,OAAOmE,kBAAoBpE,OAAOC,OAAOmE,iBAAiBC,mBAEtFC,MAAU,SAAUL,GAChB3B,GAAYA,IACZrC,OAAO8C,KAAKC,MAAO,oBAAqB,qBAAsBiB,EAC9D,IAAIM,IAAc,CAClB,IAAKN,GAAYA,EAASO,SAAW,CACjC,GAAIC,GAAiB5E,EAAKK,KAAKyC,KAAK+B,cAAeT,EAASO,SAC5D,KAAM,GAAIG,KAAYF,GAAiB,CACnC5E,EAAKK,KAAK0E,UAAWD,EAAUF,EAAgBE,IAC/CJ,GAAc,CACd,QAGFA,GACF1E,EAAKE,MAAM8E,MACP9D,MAAU,wBACV+D,KAAUjF,EAAKkF,eAAgBxC,EAAS0B,GAAYA,EAASe,SAC7DzE,SAAY0E,MAAU,WAAapF,EAAKE,MAAM4D,eAUlEb,SAAU,SAAUP,GAChB,GAAI2C,GAAc3C,EAAQZ,OACtBwD,EAAc,GACdC,EAAc,IAClB,KAAM,GAAIC,KAAgBH,GAAa,CACnC,GAAII,GAAcJ,EAAYG,GAC1BV,EAAc7E,KAAKI,KAAKyC,KAAK4C,MAAOF,GACpCG,EAAc1F,KAAKI,KAAKuF,WAAYd,GACpCe,EAAc5F,KAAKI,KAAKyF,WAAYhB,EACxC,IAAMA,GAAae,GAAcF,EAAjC,CAIA,IAAME,EAAUE,UAA2B,MAAfN,EAExB,MADAxF,MAAKI,KAAK0E,UAAWD,IACd,CAEX,IAAKW,GAAeA,EAAYO,MAAQ,CACpC,GAAIC,GAAIR,EAAYS,OAAOC,OACvBC,EAAMH,EAAI,GAAKR,EAAYS,OAAQ,IAAOT,EAAYS,OAAQ,GAAIE,GACtE,IAAKA,EACD,GAAmB,OAAdb,EACDA,EAAYa,MACT,IAAKb,IAAca,EAEtB,MADAnG,MAAKI,KAAK0E,UAAWD,EAAU,mFACxB,CAGf,IAAiB,KAAZQ,EACDA,EAAUW,MACP,IAAKX,IAAYW,EAEpB,MADAhG,MAAKI,KAAK0E,UAAWD,EAAU,gHAAkHmB,EAAI,wDAA0DX,EAAU,UAClN,OAtBXlF,QAAO8C,KAAKC,MAAM,wBAAyB,oCA0BnD,OAAO,GAGXmB,iBAAkB,SAAUF,GACxB,GAAKA,EAASiC,MAAQjC,EAASiC,KAAKF,OAAS,EAAI,CAC7C,GAAIG,GAAQlC,EAASiC,KAAKF,OACtBI,EAAsB,GAATD,EAAa,YAAcA,EAAQ,aAChDE,EAAWjD,EAAG,UAAWkD,SAAU,oBACVjE,OAAQe,EAAG,QAASmD,KAAMH,EAAa,gFAKpE,OAJA5C,GAAEC,KAAMQ,EAASuC,QAAS,SAAUC,GAChCJ,EAAShE,OAAQe,EAAG,QAASkD,SAAU,cAAejE,OAAQe,EAAG,QAASmD,KAAME,EAAOC,IAAM,KAAOD,EAAO5F,UAE/GwF,EAAShE,OAAQe,EAAG,QAASf,OAAQ,QAASkE,KAAM,yPAC7CF,EAEP,MAAOvG,MAAKiF,eAAgBd,EAAU,6CAI9Cc,eAAgB,SAAUd,EAAUe,GAChC,MAAQ5B,GAAG,UAAWkD,SAAU,qBACVjE,OAAQe,EAAG,QAASmD,KAAM,sGAAyGvB,GAAW,MAC9I3C,OAAQe,EAAG,UAAWmD,KAAMI,KAAKC,UAAW3C,EAAU,KAAM,OAI1F,QACIzE,KAAMA"}

static/maps/mvc/ui/ui-modal.js.map

@@ -1 +1 @@
-{"version":3,"file":"ui-modal.js","sources":["../../../src/mvc/ui/ui-modal.js"],"names":["define","View","Backbone","extend","optionsDefault","container","title","cls","body","backdrop","height","width","closing_events","closing_callback","title_separator","buttonList","initialize","options","this","setElement","_template","_","defaults","$","prepend","el","$header","$dialog","$body","$footer","$backdrop","$buttons","render","show","visible","$el","fadeIn","self","document","on","e","keyCode","hide","canceled","fadeOut","off","removeClass","addClass","find","html","empty","buttons","counter","each","name","callback","$button","attr","text","click","append","removeAttr","css","window","getButton","enableButton","prop","disableButton","showButton","hideButton","scrollTop"],"mappings":"AAAAA,UAAW,WACX,GAAIC,GAAOC,SAASD,KAAKE,QAErBC,gBACIC,UAAmB,OACnBC,MAAmB,WACnBC,IAAmB,WACnBC,KAAmB,GACnBC,UAAmB,EACnBC,OAAmB,KACnBC,MAAmB,KACnBC,gBAAmB,EACnBC,iBAAmB,KACnBC,iBAAmB,GAIvBC,cAGAC,WAAY,SAAUC,GAClBC,KAAKC,WAAYD,KAAKE,aACtBF,KAAKD,QAAUI,EAAEC,SAAUL,MAAeC,KAAKd,gBAC/CmB,EAAGL,KAAKD,QAAQZ,WAAYmB,QAASN,KAAKO,IAG1CP,KAAKQ,QAAaR,KAAKK,EAAG,iBAC1BL,KAAKS,QAAaT,KAAKK,EAAG,iBAC1BL,KAAKU,MAAaV,KAAKK,EAAG,eAC1BL,KAAKW,QAAaX,KAAKK,EAAG,iBAC1BL,KAAKY,UAAaZ,KAAKK,EAAG,mBAC1BL,KAAKa,SAAab,KAAKK,EAAG,YAG1BN,GAAWC,KAAKc,UAMpBC,KAAM,SAAUhB,GAKZ,GAJKA,IACDC,KAAKD,QAAUI,EAAEC,SAAUL,EAASC,KAAKd,gBACzCc,KAAKc,WAEHd,KAAKgB,UACPhB,KAAKgB,SAAU,EACfhB,KAAKiB,IAAIC,OAAQ,QACZlB,KAAKD,QAAQL,gBAAiB,CAC/B,GAAIyB,GAAOnB,IACXK,GAAGe,UAAWC,GAAI,iBAAkB,SAAUC,GAAmB,IAAbA,EAAEC,SAAiBJ,EAAKK,MAAM,KAClFxB,KAAKY,UAAUS,GAAI,QAAS,WAAaF,EAAKK,MAAM,OAQhEA,KAAM,SAAUC,GACZzB,KAAKgB,SAAU,EACfhB,KAAKiB,IAAIS,QAAS,QAClB1B,KAAKD,QAAQJ,kBAAoBK,KAAKD,QAAQJ,iBAAkB8B,GAChEpB,EAAGe,UAAWO,IAAK,kBACnB3B,KAAKY,UAAUe,IAAK,UAMxBb,OAAQ,WACJ,GAAIK,GAAOnB,IAeX,IAdyB,YAArBA,KAAKD,QAAQT,OACbU,KAAKD,QAAQT,KAAOe,EAAI,yHAM5BL,KAAKiB,IAAIW,cAAcC,SAAU,SAAUA,SAAU7B,KAAKD,QAAQV,KAClEW,KAAKQ,QAAQsB,KAAM,UAAWC,KAAM/B,KAAKD,QAAQX,OACjDY,KAAKU,MAAMqB,KAAM/B,KAAKD,QAAQT,MAG9BU,KAAKa,SAASmB,QACdhC,KAAKH,cACAG,KAAKD,QAAQkC,QAAU,CACxB,GAAIC,GAAU,CACd7B,GAAE8B,KAAMnC,KAAKD,QAAQkC,QAAS,SAAUG,EAAMC,GAC1C,GAAIC,GAAUjC,EAAG,aAAckC,KAAM,KAAM,UAAYL,KAAYM,KAAMJ,GAAOK,MAAOJ,EACvFlB,GAAKN,SAAS6B,OAAQJ,GAAUI,OAAQ,UACxCvB,EAAKtB,WAAYuC,GAASE,QAG9BtC,MAAKW,QAAQa,MAIjBxB,MAAKY,UAAWZ,KAAKD,QAAQR,UAAY,YAAc,eAAiB,MACxES,KAAKQ,SAAUR,KAAKD,QAAQH,iBAAmB,YAAc,eAAiB,gBAI9EI,KAAKU,MAAMiC,WAAY,SAClB3C,KAAKD,QAAQP,QACdQ,KAAKU,MAAMkC,IAAK,SAAU5C,KAAKD,QAAQP,QACvCQ,KAAKU,MAAMkC,IAAK,WAAY,WAE5B5C,KAAKU,MAAMkC,IAAK,aAAcvC,EAAGwC,QAASrD,SAAW,GAEpDQ,KAAKD,QAAQN,OACdO,KAAKS,QAAQmC,IAAK,QAAS5C,KAAKD,QAAQN,QAQhDqD,UAAW,SAAUV,GACjB,MAAOpC,MAAKH,WAAYuC,IAO5BW,aAAc,SAAUX,GACpBpC,KAAK8C,UAAWV,GAAOY,KAAM,YAAY,IAO7CC,cAAe,SAAUb,GACrBpC,KAAK8C,UAAWV,GAAOY,KAAM,YAAY,IAO7CE,WAAY,SAAUd,GAClBpC,KAAK8C,UAAWV,GAAOrB,QAO3BoC,WAAY,SAAUf,GAClBpC,KAAK8C,UAAWV,GAAOZ,QAM3B4B,UAAW,WACP,MAAOpD,MAAKU,MAAM0C,aAMtBlD,UAAW,WACP,MAAQ,sQAiBhB,QACInB,KAAOA"}
+{"version":3,"file":"ui-modal.js","sources":["../../../src/mvc/ui/ui-modal.js"],"names":["define","View","Backbone","extend","className","optionsDefault","container","title","cls","body","backdrop","height","width","closing_events","closing_callback","title_separator","buttonList","initialize","options","this","_","defaults","$","prepend","el","render","show","visible","$el","fadeIn","self","document","on","e","keyCode","hide","$backdrop","canceled","fadeOut","off","html","_template","$header","$dialog","$body","$footer","$buttons","removeClass","addClass","find","empty","buttons","counter","each","name","callback","$button","attr","text","click","append","removeAttr","css","window","getButton","enableButton","prop","disableButton","showButton","hideButton","scrollTop"],"mappings":"AAAAA,UAAW,WACX,GAAIC,GAAOC,SAASD,KAAKE,QAErBC,UAAW,WAGXC,gBACIC,UAAmB,OACnBC,MAAmB,WACnBC,IAAmB,WACnBC,KAAmB,GACnBC,UAAmB,EACnBC,OAAmB,KACnBC,MAAmB,KACnBC,gBAAmB,EACnBC,iBAAmB,KACnBC,iBAAmB,GAIvBC,cAGAC,WAAY,SAAUC,GAClBC,KAAKD,QAAUE,EAAEC,SAAUH,MAAeC,KAAKd,gBAC/CiB,EAAGH,KAAKD,QAAQZ,WAAYiB,QAASJ,KAAKK,IAE1CN,GAAWC,KAAKM,UAMpBC,KAAM,SAAUR,GAKZ,GAJKA,IACDC,KAAKD,QAAUE,EAAEC,SAAUH,EAASC,KAAKd,gBACzCc,KAAKM,WAEHN,KAAKQ,UACPR,KAAKQ,SAAU,EACfR,KAAKS,IAAIC,OAAQ,QACZV,KAAKD,QAAQL,gBAAiB,CAC/B,GAAIiB,GAAOX,IACXG,GAAGS,UAAWC,GAAI,iBAAkB,SAAUC,GAAmB,IAAbA,EAAEC,SAAiBJ,EAAKK,MAAM,KAClFhB,KAAKiB,UAAUJ,GAAI,QAAS,WAAaF,EAAKK,MAAM,OAQhEA,KAAM,SAAUE,GACZlB,KAAKQ,SAAU,EACfR,KAAKS,IAAIU,QAAS,QAClBnB,KAAKD,QAAQJ,kBAAoBK,KAAKD,QAAQJ,iBAAkBuB,GAChEf,EAAGS,UAAWQ,IAAK,kBACnBpB,KAAKiB,UAAUG,IAAK,UAMxBd,OAAQ,WACJ,GAAIK,GAAOX,IAyBX,IAxBAA,KAAKS,IAAIY,KAAMrB,KAAKsB,aAGpBtB,KAAKuB,QAAavB,KAAKG,EAAG,iBAC1BH,KAAKwB,QAAaxB,KAAKG,EAAG,iBAC1BH,KAAKyB,MAAazB,KAAKG,EAAG,eAC1BH,KAAK0B,QAAa1B,KAAKG,EAAG,iBAC1BH,KAAKiB,UAAajB,KAAKG,EAAG,mBAC1BH,KAAK2B,SAAa3B,KAAKG,EAAG,YAED,YAArBH,KAAKD,QAAQT,OACbU,KAAKD,QAAQT,KAAOa,EAAI,yHAM5BH,KAAKS,IAAImB,cAAcC,SAAU,SAAUA,SAAU7B,KAAKD,QAAQV,KAClEW,KAAKuB,QAAQO,KAAM,UAAWT,KAAMrB,KAAKD,QAAQX,OACjDY,KAAKyB,MAAMJ,KAAMrB,KAAKD,QAAQT,MAG9BU,KAAK2B,SAASI,QACd/B,KAAKH,cACAG,KAAKD,QAAQiC,QAAU,CACxB,GAAIC,GAAU,CACd9B,GAAE+B,KAAMlC,KAAKD,QAAQiC,QAAS,SAAUG,EAAMC,GAC1C,GAAIC,GAAUlC,EAAG,aAAcmC,KAAM,KAAM,UAAYL,KAAYM,KAAMJ,GAAOK,MAAOJ,EACvFzB,GAAKgB,SAASc,OAAQJ,GAAUI,OAAQ,UACxC9B,EAAKd,WAAYsC,GAASE,QAG9BrC,MAAK0B,QAAQV,MAIjBhB,MAAKiB,UAAWjB,KAAKD,QAAQR,UAAY,YAAc,eAAiB,MACxES,KAAKuB,SAAUvB,KAAKD,QAAQH,iBAAmB,YAAc,eAAiB,gBAI9EI,KAAKyB,MAAMiB,WAAY,SAClB1C,KAAKD,QAAQP,QACdQ,KAAKyB,MAAMkB,IAAK,SAAU3C,KAAKD,QAAQP,QACvCQ,KAAKyB,MAAMkB,IAAK,WAAY,WAE5B3C,KAAKyB,MAAMkB,IAAK,aAAcxC,EAAGyC,QAASpD,SAAW,GAEpDQ,KAAKD,QAAQN,OACdO,KAAKwB,QAAQmB,IAAK,QAAS3C,KAAKD,QAAQN,QAQhDoD,UAAW,SAAUV,GACjB,MAAOnC,MAAKH,WAAYsC,IAO5BW,aAAc,SAAUX,GACpBnC,KAAK6C,UAAWV,GAAOY,KAAM,YAAY,IAO7CC,cAAe,SAAUb,GACrBnC,KAAK6C,UAAWV,GAAOY,KAAM,YAAY,IAO7CE,WAAY,SAAUd,GAClBnC,KAAK6C,UAAWV,GAAO5B,QAO3B2C,WAAY,SAAUf,GAClBnC,KAAK6C,UAAWV,GAAOnB,QAM3BmC,UAAW,WACP,MAAOnD,MAAKyB,MAAM0B,aAMtB7B,UAAW,WACP,MAAQ,0OAehB,QACIxC,KAAOA"}

static/scripts/mvc/tool/tool-form.js

@@ -1,2 +1,2 @@
-define(["utils/utils","mvc/ui/ui-misc","mvc/ui/ui-modal","mvc/tool/tool-form-base"],function(a,b,c,d){var e=Backbone.View.extend({initialize:function(e){var f=this;this.modal=parent.Galaxy.modal||new c.View,this.form=new d(a.merge({listen_to_history:!0,always_refresh:!1,customize:function(a){a.buttons={execute:execute_btn=new b.Button({icon:"fa-check",tooltip:"Execute: "+a.name+" ("+a.version+")",title:"Execute",cls:"ui-button btn btn-primary",floating:"clear",onclick:function(){execute_btn.wait(),f.form.portlet.disable(),f.submit(a,function(){execute_btn.unwait(),f.form.portlet.enable()})}})},a.job_id&&a.job_remap&&(a.inputs.rerun_remap_job_id={label:"Resume dependencies from this job",name:"rerun_remap_job_id",type:"select",display:"radio",ignore:"__ignore__",value:"__ignore__",options:[["Yes",a.job_id],["No","__ignore__"]],help:"The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run."})}},e)),this.deferred=this.form.deferred,this.setElement("<div/>"),this.$el.append(this.form.$el)},submit:function(b,c){var d=this,e={tool_id:b.id,tool_version:b.version,inputs:this.form.data.create()};if(this.form.trigger("reset"),!d.validate(e))return Galaxy.emit.debug("tool-form::submit()","Submission canceled. Validation failed."),void(c&&c());if(b.action!==Galaxy.root+"tool_runner/index"){var f=$("<form/>").attr({action:b.action,method:b.method,enctype:b.enctype});return _.each(e.inputs,function(a,b){f.append($("<input/>").attr({name:b,value:a}))}),f.hide().appendTo("body").submit().remove(),void(c&&c())}Galaxy.emit.debug("tool-form::submit()","Validation complete.",e),a.request({type:"POST",url:Galaxy.root+"api/tools",data:e,success:function(a){c&&c(),d.$el.children().hide(),d.$el.append(d._templateSuccess(a)),parent.Galaxy&&parent.Galaxy.currHistoryPanel&&parent.Galaxy.currHistoryPanel.refreshContents()},error:function(a){c&&c(),Galaxy.emit.debug("tool-form::submit","Submission failed.",a);var b=!1;if(a&&a.err_data){var f=d.form.data.matchResponse(a.err_data);for(var g in f){d.form.highlight(g,f[g]),b=!0;break}}b||d.modal.show({title:"Job submission failed",body:a&&a.err_msg||d._templateError(e),buttons:{Close:function(){d.modal.hide()}}})}})},validate:function(a){var b=a.inputs,c=-1,d=null;for(var e in b){var f=b[e],g=this.form.data.match(e),h=this.form.field_list[g],i=this.form.input_list[g];if(g&&i&&h){if(!i.optional&&null==f)return this.form.highlight(g),!1;if(f&&f.batch){var j=f.values.length,k=j>0&&f.values[0]&&f.values[0].src;if(k)if(null===d)d=k;else if(d!==k)return this.form.highlight(g,"Please select either dataset or dataset list fields for all batch mode fields."),!1;if(-1===c)c=j;else if(c!==j)return this.form.highlight(g,"Please make sure that you select the same number of inputs for all batch mode fields. This field contains <b>"+j+"</b> selection(s) while a previous field contains <b>"+c+"</b>."),!1}}else Galaxy.emit.debug("tool-form::validate()","Retrieving input objects failed.")}return!0},_templateSuccess:function(a){if(a.jobs&&a.jobs.length>0){var b=a.jobs.length,c=1==b?"1 job has":b+" jobs have",d=$("<div/>").addClass("donemessagelarge").append($("<p/>").text(c+" been successfully added to the queue - resulting in the following datasets:"));return _.each(a.outputs,function(a){d.append($("<p/>").addClass("messagerow").append($("<b/>").text(a.hid+": "+a.name)))}),d.append($("<p/>").append("<b/>").text("You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from 'running' to 'finished' if completed successfully or 'error' if problems were encountered.")),d}return this._templateError(a)},_templateError:function(a){return $("<div/>").addClass("errormessagelarge").append($("<p/>").text("The server could not complete the request. Please contact the Galaxy Team if this error persists.")).append($("<pre/>").text(JSON.stringify(a,null,4)))}});return{View:e}});
+define(["utils/utils","mvc/ui/ui-misc","mvc/ui/ui-modal","mvc/tool/tool-form-base"],function(a,b,c,d){var e=Backbone.View.extend({initialize:function(e){var f=this;this.modal=parent.Galaxy.modal||new c.View,this.form=new d(a.merge({listen_to_history:!0,always_refresh:!1,customize:function(a){a.buttons={execute:execute_btn=new b.Button({icon:"fa-check",tooltip:"Execute: "+a.name+" ("+a.version+")",title:"Execute",cls:"ui-button btn btn-primary",floating:"clear",onclick:function(){execute_btn.wait(),f.form.portlet.disable(),f.submit(a,function(){execute_btn.unwait(),f.form.portlet.enable()})}})},a.job_id&&a.job_remap&&(a.inputs.rerun_remap_job_id={label:"Resume dependencies from this job",name:"rerun_remap_job_id",type:"select",display:"radio",ignore:"__ignore__",value:"__ignore__",options:[["Yes",a.job_id],["No","__ignore__"]],help:"The previous run of this tool failed and other tools were waiting for it to finish successfully. Use this option to resume those tools using the new output(s) of this tool run."})}},e)),this.deferred=this.form.deferred,this.setElement("<div/>"),this.$el.append(this.form.$el)},submit:function(b,c){var d=this,e={tool_id:b.id,tool_version:b.version,inputs:this.form.data.create()};if(this.form.trigger("reset"),!d.validate(e))return Galaxy.emit.debug("tool-form::submit()","Submission canceled. Validation failed."),void(c&&c());if(b.action!==Galaxy.root+"tool_runner/index"){var f=$("<form/>").attr({action:b.action,method:b.method,enctype:b.enctype});return _.each(e.inputs,function(a,b){f.append($("<input/>").attr({name:b,value:a}))}),f.hide().appendTo("body").submit().remove(),void(c&&c())}Galaxy.emit.debug("tool-form::submit()","Validation complete.",e),a.request({type:"POST",url:Galaxy.root+"api/tools",data:e,success:function(a){c&&c(),d.$el.children().hide(),d.$el.append(d._templateSuccess(a)),parent.Galaxy&&parent.Galaxy.currHistoryPanel&&parent.Galaxy.currHistoryPanel.refreshContents()},error:function(a){c&&c(),Galaxy.emit.debug("tool-form::submit","Submission failed.",a);var b=!1;if(a&&a.err_data){var f=d.form.data.matchResponse(a.err_data);for(var g in f){d.form.highlight(g,f[g]),b=!0;break}}b||d.modal.show({title:"Job submission failed",body:d._templateError(e,a&&a.err_msg),buttons:{Close:function(){d.modal.hide()}}})}})},validate:function(a){var b=a.inputs,c=-1,d=null;for(var e in b){var f=b[e],g=this.form.data.match(e),h=this.form.field_list[g],i=this.form.input_list[g];if(g&&i&&h){if(!i.optional&&null==f)return this.form.highlight(g),!1;if(f&&f.batch){var j=f.values.length,k=j>0&&f.values[0]&&f.values[0].src;if(k)if(null===d)d=k;else if(d!==k)return this.form.highlight(g,"Please select either dataset or dataset list fields for all batch mode fields."),!1;if(-1===c)c=j;else if(c!==j)return this.form.highlight(g,"Please make sure that you select the same number of inputs for all batch mode fields. This field contains <b>"+j+"</b> selection(s) while a previous field contains <b>"+c+"</b>."),!1}}else Galaxy.emit.debug("tool-form::validate()","Retrieving input objects failed.")}return!0},_templateSuccess:function(a){if(a.jobs&&a.jobs.length>0){var b=a.jobs.length,c=1==b?"1 job has":b+" jobs have",d=$("<div/>").addClass("donemessagelarge").append($("<p/>").text(c+" been successfully added to the queue - resulting in the following datasets:"));return _.each(a.outputs,function(a){d.append($("<p/>").addClass("messagerow").append($("<b/>").text(a.hid+": "+a.name)))}),d.append($("<p/>").append("<b/>").text("You can check the status of queued jobs and view the resulting data by refreshing the History pane. When the job has been run the status will change from 'running' to 'finished' if completed successfully or 'error' if problems were encountered.")),d}return this._templateError(a,"Invalid success response. No jobs found.")},_templateError:function(a,b){return $("<div/>").addClass("errormessagelarge").append($("<p/>").text("The server could not complete the request. Please contact the Galaxy Team if this error persists. "+(b||""))).append($("<pre/>").text(JSON.stringify(a,null,4)))}});return{View:e}});
 //# sourceMappingURL=../../../maps/mvc/tool/tool-form.js.map

static/scripts/mvc/ui/ui-modal.js

@@ -1,2 +1,2 @@
-define([],function(){var a=Backbone.View.extend({optionsDefault:{container:"body",title:"ui-modal",cls:"ui-modal",body:"",backdrop:!0,height:null,width:null,closing_events:!1,closing_callback:null,title_separator:!0},buttonList:{},initialize:function(a){this.setElement(this._template()),this.options=_.defaults(a||{},this.optionsDefault),$(this.options.container).prepend(this.el),this.$header=this.$(".modal-header"),this.$dialog=this.$(".modal-dialog"),this.$body=this.$(".modal-body"),this.$footer=this.$(".modal-footer"),this.$backdrop=this.$(".modal-backdrop"),this.$buttons=this.$(".buttons"),a&&this.render()},show:function(a){if(a&&(this.options=_.defaults(a,this.optionsDefault),this.render()),!this.visible&&(this.visible=!0,this.$el.fadeIn("fast"),this.options.closing_events)){var b=this;$(document).on("keyup.ui-modal",function(a){27==a.keyCode&&b.hide(!0)}),this.$backdrop.on("click",function(){b.hide(!0)})}},hide:function(a){this.visible=!1,this.$el.fadeOut("fast"),this.options.closing_callback&&this.options.closing_callback(a),$(document).off("keyup.ui-modal"),this.$backdrop.off("click")},render:function(){var a=this;if("progress"==this.options.body&&(this.options.body=$('<div class="progress progress-striped active"><div class="progress-bar progress-bar-info" style="width:100%"/></div>')),this.$el.removeClass().addClass("modal").addClass(this.options.cls),this.$header.find(".title").html(this.options.title),this.$body.html(this.options.body),this.$buttons.empty(),this.buttonList={},this.options.buttons){var b=0;$.each(this.options.buttons,function(c,d){var e=$("<button/>").attr("id","button-"+b++).text(c).click(d);a.$buttons.append(e).append("&nbsp;"),a.buttonList[c]=e})}else this.$footer.hide();this.$backdrop[this.options.backdrop&&"addClass"||"removeClass"]("in"),this.$header[!this.options.title_separator&&"addClass"||"removeClass"]("no-separator"),this.$body.removeAttr("style"),this.options.height?(this.$body.css("height",this.options.height),this.$body.css("overflow","hidden")):this.$body.css("max-height",$(window).height()/2),this.options.width&&this.$dialog.css("width",this.options.width)},getButton:function(a){return this.buttonList[a]},enableButton:function(a){this.getButton(a).prop("disabled",!1)},disableButton:function(a){this.getButton(a).prop("disabled",!0)},showButton:function(a){this.getButton(a).show()},hideButton:function(a){this.getButton(a).hide()},scrollTop:function(){return this.$body.scrollTop()},_template:function(){return'<div class="ui-modal"><div class="modal-backdrop fade"/><div class="modal-dialog"><div class="modal-content"><div class="modal-header"><h4 class="title"/></div><div class="modal-body"/><div class="modal-footer"><div class="buttons"/></div></div></div></div>'}});return{View:a}});
+define([],function(){var a=Backbone.View.extend({className:"ui-modal",optionsDefault:{container:"body",title:"ui-modal",cls:"ui-modal",body:"",backdrop:!0,height:null,width:null,closing_events:!1,closing_callback:null,title_separator:!0},buttonList:{},initialize:function(a){this.options=_.defaults(a||{},this.optionsDefault),$(this.options.container).prepend(this.el),a&&this.render()},show:function(a){if(a&&(this.options=_.defaults(a,this.optionsDefault),this.render()),!this.visible&&(this.visible=!0,this.$el.fadeIn("fast"),this.options.closing_events)){var b=this;$(document).on("keyup.ui-modal",function(a){27==a.keyCode&&b.hide(!0)}),this.$backdrop.on("click",function(){b.hide(!0)})}},hide:function(a){this.visible=!1,this.$el.fadeOut("fast"),this.options.closing_callback&&this.options.closing_callback(a),$(document).off("keyup.ui-modal"),this.$backdrop.off("click")},render:function(){var a=this;if(this.$el.html(this._template()),this.$header=this.$(".modal-header"),this.$dialog=this.$(".modal-dialog"),this.$body=this.$(".modal-body"),this.$footer=this.$(".modal-footer"),this.$backdrop=this.$(".modal-backdrop"),this.$buttons=this.$(".buttons"),"progress"==this.options.body&&(this.options.body=$('<div class="progress progress-striped active"><div class="progress-bar progress-bar-info" style="width:100%"/></div>')),this.$el.removeClass().addClass("modal").addClass(this.options.cls),this.$header.find(".title").html(this.options.title),this.$body.html(this.options.body),this.$buttons.empty(),this.buttonList={},this.options.buttons){var b=0;$.each(this.options.buttons,function(c,d){var e=$("<button/>").attr("id","button-"+b++).text(c).click(d);a.$buttons.append(e).append("&nbsp;"),a.buttonList[c]=e})}else this.$footer.hide();this.$backdrop[this.options.backdrop&&"addClass"||"removeClass"]("in"),this.$header[!this.options.title_separator&&"addClass"||"removeClass"]("no-separator"),this.$body.removeAttr("style"),this.options.height?(this.$body.css("height",this.options.height),this.$body.css("overflow","hidden")):this.$body.css("max-height",$(window).height()/2),this.options.width&&this.$dialog.css("width",this.options.width)},getButton:function(a){return this.buttonList[a]},enableButton:function(a){this.getButton(a).prop("disabled",!1)},disableButton:function(a){this.getButton(a).prop("disabled",!0)},showButton:function(a){this.getButton(a).show()},hideButton:function(a){this.getButton(a).hide()},scrollTop:function(){return this.$body.scrollTop()},_template:function(){return'<div class="modal-backdrop fade"/><div class="modal-dialog"><div class="modal-content"><div class="modal-header"><h4 class="title"/></div><div class="modal-body"/><div class="modal-footer"><div class="buttons"/></div></div></div>'}});return{View:a}});
 //# sourceMappingURL=../../../maps/mvc/ui/ui-modal.js.map

test/api/test_workflow_batch.ga

@@ -0,0 +1,145 @@
+{
+    "a_galaxy_workflow": "true", 
+    "annotation": "", 
+    "format-version": "0.1", 
+    "name": "test", 
+    "steps": {
+        "0": {
+            "annotation": "", 
+            "content_id": null, 
+            "id": 0, 
+            "input_connections": {}, 
+            "inputs": [
+                {
+                    "description": "", 
+                    "name": "Input Dataset"
+                }
+            ], 
+            "label": null, 
+            "name": "Input dataset", 
+            "outputs": [], 
+            "position": {
+                "left": 200, 
+                "top": 200
+            }, 
+            "tool_errors": null, 
+            "tool_id": null, 
+            "tool_state": "{\"name\": \"Input Dataset\"}", 
+            "tool_version": null, 
+            "type": "data_input", 
+            "uuid": "ef60789e-60fd-4c5a-baa5-598aeac0b5dc", 
+            "workflow_outputs": []
+        }, 
+        "1": {
+            "annotation": "", 
+            "content_id": "addValue", 
+            "id": 1, 
+            "input_connections": {}, 
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool Add column", 
+                    "name": "input"
+                }
+            ], 
+            "label": null, 
+            "name": "Add column", 
+            "outputs": [
+                {
+                    "name": "out_file1", 
+                    "type": "input"
+                }
+            ], 
+            "position": {
+                "left": 204, 
+                "top": 319
+            }, 
+            "post_job_actions": {}, 
+            "tool_errors": null, 
+            "tool_id": "addValue", 
+            "tool_state": "{\"__page__\": 0, \"__rerun_remap_job_id__\": null, \"__job_resource\": \"{\\\"__current_case__\\\": 0, \\\"__job_resource__select\\\": \\\"no\\\"}\", \"exp\": \"\\\"1\\\"\", \"iterate\": \"\\\"no\\\"\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\"}", 
+            "tool_version": "1.0.0", 
+            "type": "tool", 
+            "uuid": "1daceb2a-719c-49a6-881c-5301076de918", 
+            "workflow_outputs": []
+        }, 
+        "2": {
+            "annotation": "", 
+            "content_id": "addValue", 
+            "id": 2, 
+            "input_connections": {
+                "input": {
+                    "id": 0, 
+                    "output_name": "output"
+                }
+            }, 
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool Add column", 
+                    "name": "input"
+                }
+            ], 
+            "label": null, 
+            "name": "Add column", 
+            "outputs": [
+                {
+                    "name": "out_file1", 
+                    "type": "input"
+                }
+            ], 
+            "position": {
+                "left": 418, 
+                "top": 234
+            }, 
+            "post_job_actions": {}, 
+            "tool_errors": null, 
+            "tool_id": "addValue", 
+            "tool_state": "{\"__page__\": 0, \"__rerun_remap_job_id__\": null, \"__job_resource\": \"{\\\"__current_case__\\\": 0, \\\"__job_resource__select\\\": \\\"no\\\"}\", \"exp\": \"\\\"1\\\"\", \"iterate\": \"\\\"no\\\"\", \"input\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\"}", 
+            "tool_version": "1.0.0", 
+            "type": "tool", 
+            "uuid": "f5349127-d008-44b2-a41e-af024de92d2e", 
+            "workflow_outputs": []
+        }, 
+        "3": {
+            "annotation": "", 
+            "content_id": "cat1", 
+            "id": 3, 
+            "input_connections": {
+                "input1": {
+                    "id": 2, 
+                    "output_name": "out_file1"
+                }, 
+                "queries_0|input2": {
+                    "id": 1, 
+                    "output_name": "out_file1"
+                }
+            }, 
+            "inputs": [
+                {
+                    "description": "runtime parameter for tool Concatenate datasets", 
+                    "name": "input1"
+                }
+            ], 
+            "label": null, 
+            "name": "Concatenate datasets", 
+            "outputs": [
+                {
+                    "name": "out_file1", 
+                    "type": "input"
+                }
+            ], 
+            "position": {
+                "left": 389.5, 
+                "top": 396
+            }, 
+            "post_job_actions": {}, 
+            "tool_errors": null, 
+            "tool_id": "cat1", 
+            "tool_state": "{\"__job_resource\": \"{\\\"__current_case__\\\": 0, \\\"__job_resource__select\\\": \\\"no\\\"}\", \"__page__\": 0, \"__rerun_remap_job_id__\": null, \"input1\": \"{\\\"__class__\\\": \\\"RuntimeValue\\\"}\", \"queries\": \"[{\\\"input2\\\": {\\\"__class__\\\": \\\"RuntimeValue\\\"}, \\\"__index__\\\": 0}]\"}", 
+            "tool_version": "1.0.0", 
+            "type": "tool", 
+            "uuid": "32205465-a47e-4d8a-aa45-2560b1a38f54", 
+            "workflow_outputs": []
+        }
+    }, 
+    "uuid": "9f791470-2fca-4f63-aa18-72ae0211b077"
+}

test/api/test_workflows.py

@@ -1412,6 +1412,30 @@ test_data:
         self.__assert_lines_hid_line_count_is( history_id, 2, 4 )
         self.__assert_lines_hid_line_count_is( history_id, 3, 3 )
 
+    @skip_without_tool( "cat1" )
+    @skip_without_tool( "addValue" )
+    def test_run_batch( self ):
+        workflow = self.workflow_populator.load_workflow_from_resource( "test_workflow_batch" )
+        workflow_id = self.workflow_populator.create_workflow( workflow )
+        history_id = self.dataset_populator.new_history()
+        hda1 = self.dataset_populator.new_dataset( history_id, content="1 2 3" )
+        hda2 = self.dataset_populator.new_dataset( history_id, content="4 5 6" )
+        workflow_request = {
+            "history_id" : history_id,
+            "batch"      : True,
+            "parameters_normalized": True,
+            "parameters" : dumps( { "0": { "input": { "batch": True, "values": [ { "id" : hda1.get( "id" ), "hid": hda1.get( "hid" ), "src": "hda" }, { "id" : hda2.get( "id" ), "hid": hda2.get( "hid" ), "src": "hda" } ] } }, "1": { "input": { "batch": False, "values": [ { "id" : hda1.get( "id" ), "hid": hda1.get( "hid" ), "src": "hda" } ] }, "exp": "2" } } )
+        }
+        invocation_response = self._post( "workflows/%s/usage" % workflow_id, data=workflow_request )
+        self._assert_status_code_is( invocation_response, 200 )
+        time.sleep( 5 )
+        self.dataset_populator.wait_for_history( history_id, assert_ok=True )
+        r1 = "1 2 3\t1\n1 2 3\t2\n"
+        r2 = "4 5 6\t1\n1 2 3\t2\n"
+        t1 = self.dataset_populator.get_history_dataset_content( history_id, hid=5 )
+        t2 = self.dataset_populator.get_history_dataset_content( history_id, hid=8 )
+        assert ( r1 == t1 and r2 == t2 ) or ( r1 == t2 and r2 == t1 )
+
     @skip_without_tool( "validation_default" )
     def test_parameter_substitution_sanitization( self ):
         substitions = dict( input1="\" ; echo \"moo" )

test/base/driver_util.py

@@ -13,14 +13,13 @@ import tempfile
 import threading
 import time
 
-from six.moves.urllib.request import urlretrieve
-
 import nose.config
 import nose.core
 import nose.loader
 import nose.plugins.manager
 
 from paste import httpserver
+import requests
 
 from .api_util import get_master_api_key, get_user_api_key
 from .tool_shed_util import parse_tool_panel_config
@@ -258,7 +257,9 @@ def copy_database_template( source, db_path ):
         shutil.copy(source, db_path)
         assert os.path.exists(db_path)
     elif source.lower().startswith(("http://", "https://", "ftp://")):
-        urlretrieve(source, db_path)
+        r = requests.get(source)
+        with open(db_path, 'w') as f:
+            f.write(r.content)
     else:
         raise Exception( "Failed to copy database template from source %s" % source )
 

test/functional/tools/color_param.xml

@@ -0,0 +1,30 @@
+<tool id="color_param" name="color_param" version="1.0.0">
+    <command>
+        echo "$color_default"  > $out_file1;
+        echo "$color_rgb"      > $out_file2;
+    </command>
+    <inputs>
+        <param name="color_default" type="color" value="#aabbcc" />
+        <param name="color_rgb" type="color" value="#aabbcc" rgb="true" />
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+        <data name="out_file2" format="txt" />
+    </outputs>
+    <tests>
+        <test>
+            <param name="color_default" value="#aaaaaa" />
+            <param name="color_rgb" value="#aaaaaa" />
+            <output name="out_file1">
+                <assert_contents>
+                    <has_line line="#aaaaaa" />
+                </assert_contents>
+            </output>
+            <output name="out_file2">
+                <assert_contents>
+                    <has_line line="(170, 170, 170)" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -2,6 +2,7 @@
 <toolbox tool_path="${tool_conf_dir}" is_shed_conf="false">
   <tool file="upload.xml"/>
   <tool file="simple_constructs.xml" />
+  <tool file="color_param.xml" />
   <tool file="inheritance_simple.xml" />
   <tool file="boolean_conditional.xml" />
   <tool file="composite.xml" />

test/unit/workflows/test_run_parameters.py

@@ -1,8 +1,8 @@
 from .workflow_support import MockTrans
 
 from galaxy import model
-from galaxy.workflow.run_request import normalize_step_parameters
-from galaxy.workflow.run_request import normalize_inputs
+from galaxy.workflow.run_request import _normalize_step_parameters
+from galaxy.workflow.run_request import _normalize_inputs
 
 STEP_ID_OFFSET = 4  # Offset a little so ids and order index are different.
 
@@ -78,7 +78,7 @@ def __normalize_parameters_against_fixture( params ):
     __workflow_fixure( trans )
 
     workflow = __workflow_fixure( trans )
-    normalized_params = normalize_step_parameters( workflow.steps, params, legacy=True )
+    normalized_params = _normalize_step_parameters( workflow.steps, params, legacy=True )
     return normalized_params
 
 
@@ -89,7 +89,7 @@ def __normalize_inputs_against_fixture( inputs, inputs_by ):
     __workflow_fixure( trans )
 
     workflow = __workflow_fixure( trans )
-    normalized_inputs = normalize_inputs( workflow.steps, inputs, inputs_by )
+    normalized_inputs = _normalize_inputs( workflow.steps, inputs, inputs_by )
     return normalized_inputs
 
 

tools/data_source/data_source.py

@@ -4,14 +4,14 @@
 import os
 import socket
 import sys
-from json import loads, dumps
+from json import dumps, loads
 
 from six.moves.urllib.parse import urlencode
 from six.moves.urllib.request import urlopen
 
-from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_FILE
 from galaxy.datatypes import sniff
 from galaxy.datatypes.registry import Registry
+from galaxy.jobs import TOOL_PROVIDED_JOB_METADATA_FILE
 from galaxy.util import get_charset_from_http_headers
 
 GALAXY_PARAM_PREFIX = 'GALAXY'

tools/data_source/hbvar_filter.py

@@ -13,7 +13,7 @@ def exec_before_job( app, inp_data, out_data, param_dict, tool=None):
     data_type = param_dict.get( 'type', 'txt' )
     if data_type == 'txt':
         data_type = 'interval'  # All data is TSV, assume interval
-    name, data = list(out_data.items())[0]
+    name, data = next(iter(out_data.items()))
     data = app.datatypes_registry.change_datatype(data, data_type)
     data.name = data_name
     out_data[name] = data
@@ -35,7 +35,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool=None, stdout=No
     except Exception as exc:
         raise Exception('Problems connecting to %s (%s)' % (URL, exc) )
 
-    name, data = list(out_data.items())[0]
+    data = next(iter(out_data.values()))
 
     fp = open(data.file_name, 'wb')
     size = 0

tools/data_source/microbial_import_code.py

@@ -90,7 +90,7 @@ def load_microbial_data( GALAXY_DATA_INDEX_DIR, sep='\t' ):
 
 # post processing, set build for data and add additional data to history
 def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr):
-    base_dataset = list(out_data.items())[0][1]
+    base_dataset = next(iter(out_data.values()))
     history = base_dataset.history
     if history is None:
         print("unknown history!")
@@ -118,7 +118,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
             chr = fields[2]
             dbkey = fields[3]
             file_type = fields[4]
-            name, data = list(out_data.items())[0]
+            data = next(iter(out_data.values()))
             data.set_size()
             basic_name = data.name
             data.name = data.name + " (" + microbe_info[kingdom][org]['chrs'][chr]['data'][description]['feature'] + " for " + microbe_info[kingdom][org]['name'] + ":" + chr + ")"

tools/evolution/add_scores.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-from __future__ import with_statement
+from __future__ import print_function
 
 import sys
 
@@ -7,7 +7,7 @@ from bx.bbi.bigwig_file import BigWigFile
 
 
 def die( message ):
-    print >> sys.stderr, message
+    print(message, file=sys.stderr)
     sys.exit(1)
 
 
@@ -100,9 +100,9 @@ def main():
                 score_val = 'NA'
             else:
                 die( '%s line %d: chrom=%s, start=%d, score_list_len = %d' % ( input_filename, line_number, chrom, start, score_list_len ) )
-            print >> ofh, '\t'.join( [line, score_val] )
+            print('\t'.join( [line, score_val] ), file=ofh)
         else:
-            print >> ofh, line
+            print(line, file=ofh)
 
     bwfh.close()
     ifh.close()

tools/evolution/codingSnps_filter.py

@@ -13,7 +13,7 @@ def validate_input( trans, error_map, param_values, page_param_map ):
     dbkeys = set()
     data_param_names = set()
     data_params = 0
-    for name, param in page_param_map.iteritems():
+    for name, param in page_param_map.items():
         if isinstance( param, DataToolParameter ):
             # for each dataset parameter
             if param_values.get(name, None) is not None:

tools/extract/extract_genomic_dna.py

@@ -9,6 +9,8 @@ usage: %prog $input $out_file1
     -F, --fasta=<genomic_sequences>: genomic sequences to use for extraction
     -G, --gff: input and output file, when it is interval, coordinates are treated as GFF format (1-based, half-open) rather than 'traditional' 0-based, closed format.
 """
+from __future__ import print_function
+
 import os
 import subprocess
 import sys
@@ -17,7 +19,7 @@ import tempfile
 import bx.seq.nib
 import bx.seq.twobit
 from bx.cookbook import doc_optparse
-from bx.intervals.io import Header, Comment
+from bx.intervals.io import Comment, Header
 
 from galaxy.datatypes.util import gff_util
 from galaxy.tools.util.galaxyops import parse_cols_arg
@@ -45,7 +47,7 @@ def check_seq_file( dbkey, GALAXY_DATA_INDEX_DIR ):
         if line and not line.startswith( "#" ) and line.startswith( 'seq' ):
             fields = line.split( '\t' )
             if len( fields) >= 3 and fields[1] == dbkey:
-                print "Using *.nib genomic reference files"
+                print("Using *.nib genomic reference files")
                 return fields[2].strip()
 
     # If no entry in aligseq.loc was found, check for the presence of a *.2bit file in twobit.loc
@@ -55,7 +57,7 @@ def check_seq_file( dbkey, GALAXY_DATA_INDEX_DIR ):
         if line and not line.startswith( "#" ) and line.endswith( '.2bit' ):
             fields = line.split( '\t' )
             if len(fields) >= 2 and fields[0] == dbkey:
-                print "Using a *.2bit genomic reference file"
+                print("Using a *.2bit genomic reference file")
                 return fields[1].strip()
 
     return ''
@@ -299,10 +301,10 @@ def __main__():
     if warnings:
         warn_msg = "%d warnings, 1st is: " % len( warnings )
         warn_msg += warnings[0]
-        print warn_msg
+        print(warn_msg)
     if skipped_lines:
         # Error message includes up to the first 10 skipped lines.
-        print 'Skipped %d invalid lines, 1st is #%d, "%s"' % ( skipped_lines, first_invalid_line, '\n'.join( invalid_lines[:10] ) )
+        print('Skipped %d invalid lines, 1st is #%d, "%s"' % ( skipped_lines, first_invalid_line, '\n'.join( invalid_lines[:10] ) ))
 
     # Clean up temp file.
     if fasta_file:

tools/filters/axt_to_concat_fasta.py

@@ -2,6 +2,8 @@
 """
 Adapted from bx/scripts/axt_to_concat_fasta.py
 """
+from __future__ import print_function
+
 import sys
 
 import bx.align.axt
@@ -40,8 +42,8 @@ def main():
 # TODO: this should be moved to a bx.align.fasta module
 def print_component_as_fasta(text, src):
     header = ">" + src
-    print header
-    print text
+    print(header)
+    print(text)
 
 
 if __name__ == "__main__":

tools/filters/axt_to_fasta.py

@@ -2,6 +2,8 @@
 """
 Adapted from bx/scripts/axt_to_fasta.py
 """
+from __future__ import print_function
+
 import sys
 
 import bx.align.axt
@@ -34,7 +36,7 @@ def main():
             id = None
         print_component_as_fasta(a.components[0], id)
         print_component_as_fasta(a.components[1], id)
-        print
+        print()
 
 
 # TODO: this should be moved to a bx.align.fasta module
@@ -42,8 +44,8 @@ def print_component_as_fasta(c, id=None):
     header = ">%s_%s_%s" % (c.src, c.start, c.start + c.size)
     if id is not None:
         header += " " + id
-    print header
-    print c.text
+    print(header)
+    print(c.text)
 
 if __name__ == "__main__":
     main()

tools/filters/axt_to_lav.py

@@ -9,6 +9,8 @@ Application to convert AXT file to LAV file
 The application reads an AXT file from standard input and writes a LAV file to
 standard out;  some statistics are written to standard error.
 """
+from __future__ import print_function
+
 import sys
 
 import bx.align.axt
@@ -114,13 +116,13 @@ def main():
         primary_c = axtBlock.get_component_by_src_start(primary)
         secondary_c = axtBlock.get_component_by_src_start(secondary)
 
-        print >>seq_file1, ">%s_%s_%s_%s" % (primary_c.src, secondary_c.strand, primary_c.start, primary_c.start + primary_c.size)
-        print >>seq_file1, primary_c.text
-        print >>seq_file1
+        print(">%s_%s_%s_%s" % (primary_c.src, secondary_c.strand, primary_c.start, primary_c.start + primary_c.size), file=seq_file1)
+        print(primary_c.text, file=seq_file1)
+        print(file=seq_file1)
 
-        print >>seq_file2, ">%s_%s_%s_%s" % (secondary_c.src, secondary_c.strand, secondary_c.start, secondary_c.start + secondary_c.size)
-        print >>seq_file2, secondary_c.text
-        print >>seq_file2
+        print(">%s_%s_%s_%s" % (secondary_c.src, secondary_c.strand, secondary_c.start, secondary_c.start + secondary_c.size), file=seq_file2)
+        print(secondary_c.text, file=seq_file2)
+        print(file=seq_file2)
         axtsWritten += 1
 
     out.close()

tools/filters/axt_to_lav_code.py

@@ -1,8 +1,6 @@
 
 def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr):
-    for name, data in out_data.items():
-        if name == "seq_file2":
-            data.dbkey = param_dict['dbkey_2']
-            app.model.context.add( data )
-            app.model.context.flush()
-            break
+    data = out_data["seq_file2"]
+    data.dbkey = param_dict['dbkey_2']
+    app.model.context.add( data )
+    app.model.context.flush()

tools/filters/bed_to_gff_converter.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # This code exists in 2 places: ~/datatypes/converters and ~/tools/filters
+from __future__ import print_function
+
 import sys
 
 assert sys.version_info[:2] >= ( 2, 4 )
@@ -69,7 +71,7 @@ def __main__():
     info_msg = "%i lines converted to GFF version 2.  " % ( i + 1 - skipped_lines )
     if skipped_lines > 0:
         info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
-    print info_msg
+    print(info_msg)
 
 if __name__ == "__main__":
     __main__()

tools/filters/convert_characters.py

@@ -1,5 +1,6 @@
 #!/usr/bin/env python
 # By, Guruprasad Ananda.
+from __future__ import print_function
 
 import optparse
 import re
@@ -46,7 +47,7 @@ def __main__():
                     skipped += 1
 
     if skipped:
-        print "Skipped %d lines as invalid." % skipped
+        print("Skipped %d lines as invalid." % skipped)
 
 if __name__ == "__main__":
     __main__()

tools/filters/gff/extract_GFF_Features.py

@@ -5,6 +5,8 @@ Extract features from GFF file.
 
 usage: %prog input1 out_file1 column features
 """
+from __future__ import print_function
+
 import sys
 
 from bx.cookbook import doc_optparse
@@ -45,7 +47,7 @@ def main():
                 pass
     fo.close()
 
-    print 'Column %d features: %s' % ( column + 1, features )
+    print('Column %d features: %s' % ( column + 1, features ))
 
 if __name__ == "__main__":
     main()

tools/filters/gff/gff_filter_by_attribute.py

@@ -3,7 +3,7 @@
 # The tool will skip over invalid lines within the file, informing the user about the number of lines skipped.
 # TODO: much of this code is copied from the Filter1 tool (filtering.py in tools/stats/). The commonalities should be
 # abstracted and leveraged in each filtering tool.
-from __future__ import division
+from __future__ import division, print_function
 
 import sys
 from json import loads
@@ -136,7 +136,7 @@ for i, line in enumerate( open( in_fname ) ):
 
 valid_filter = True
 try:
-    exec code
+    exec(code)
 except Exception as e:
     out.close()
     if str( e ).startswith( 'invalid syntax' ):
@@ -148,10 +148,10 @@ except Exception as e:
 if valid_filter:
     out.close()
     valid_lines = total_lines - skipped_lines
-    print 'Filtering with %s, ' % ( cond_text )
+    print('Filtering with %s, ' % ( cond_text ))
     if valid_lines > 0:
-        print 'kept %4.2f%% of %d lines.' % ( 100.0 * lines_kept / valid_lines, total_lines )
+        print('kept %4.2f%% of %d lines.' % ( 100.0 * lines_kept / valid_lines, total_lines ))
     else:
-        print 'Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' % cond_text
+        print('Possible invalid filter condition "%s" or non-existent column referenced. See tool tips, syntax and examples.' % cond_text)
     if skipped_lines > 0:
-        print 'Skipped %d invalid lines starting at line #%d: "%s"' % ( skipped_lines, first_invalid_line, invalid_line )
+        print('Skipped %d invalid lines starting at line #%d: "%s"' % ( skipped_lines, first_invalid_line, invalid_line ))

tools/filters/gff/gff_filter_by_feature_count.py

@@ -5,6 +5,8 @@ Filter a gff file using a criterion based on feature counts for a transcript.
 Usage:
 %prog input_name output_name feature_name condition
 """
+from __future__ import print_function
+
 import sys
 
 from bx.intervals.io import GenomicInterval
@@ -53,7 +55,7 @@ def __main__():
             except:
                 number = None
             if empty != "" or not number:
-                print >> sys.stderr, "Invalid condition: %s, cannot filter." % condition
+                print("Invalid condition: %s, cannot filter." % condition, file=sys.stderr)
                 return
             break
 
@@ -84,7 +86,7 @@ def __main__():
         ( kept_features, i, float(kept_features) / i * 100.0, feature_name + condition )
     if skipped_lines > 0:
         info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
-    print info_msg
+    print(info_msg)
 
 if __name__ == "__main__":
     __main__()

tools/filters/gff/gtf_filter_by_attribute_values_list.py

@@ -4,6 +4,7 @@
 # Usage:
 # python gff_filter_by_attribute_values.py <gff_file> <attribute_name> <ids_file> <output_file>
 #
+from __future__ import print_function
 
 import sys
 
@@ -45,7 +46,7 @@ def parse_gff_attributes( attr_str ):
     return attributes
 
 
-def filter( gff_file, attribute_name, ids_file, output_file ):
+def gff_filter( gff_file, attribute_name, ids_file, output_file ):
     # Put ids in dict for quick lookup.
     ids_dict = {}
     for line in open( ids_file ):
@@ -63,7 +64,7 @@ def filter( gff_file, attribute_name, ids_file, output_file ):
 if __name__ == "__main__":
     # Handle args.
     if len( sys.argv ) != 5:
-        print >> sys.stderr, "usage: python %s <gff_file> <attribute_name> <ids_file> <output_file>" % sys.argv[0]
+        print("usage: python %s <gff_file> <attribute_name> <ids_file> <output_file>" % sys.argv[0], file=sys.stderr)
         sys.exit( -1 )
     gff_file, attribute_name, ids_file, output_file = sys.argv[1:]
-    filter( gff_file, attribute_name, ids_file, output_file )
+    gff_filter( gff_file, attribute_name, ids_file, output_file )

tools/filters/gff_to_bed_converter.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
 import sys
 
 from galaxy.datatypes.util.gff_util import parse_gff_attributes
@@ -19,7 +21,7 @@ def get_bed_line( chrom, name, strand, blocks ):
     #
 
     # Get transcript start, end.
-    t_start = sys.maxint
+    t_start = sys.maxsize
     t_end = -1
     for block_start, block_end in blocks:
         if block_start < t_start:
@@ -65,8 +67,8 @@ def __main__():
             try:
                 # GFF format: chrom source, name, chromStart, chromEnd, score, strand, attributes
                 elems = line.split( '\t' )
-                start = str( long( elems[3] ) - 1 )
-                coords = [ long( start ), long( elems[4] ) ]
+                start = str( int( elems[3] ) - 1 )
+                coords = [ int( start ), int( elems[4] ) ]
                 strand = elems[6]
                 if strand not in ['+', '-']:
                     strand = '+'
@@ -127,7 +129,7 @@ def __main__():
     info_msg = "%i lines converted to BED.  " % ( i + 1 - skipped_lines )
     if skipped_lines > 0:
         info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
-    print info_msg
+    print(info_msg)
 
 if __name__ == "__main__":
     __main__()

tools/filters/grep.py

@@ -11,12 +11,13 @@
 # -o Output file
 # -pattern RegEx pattern
 # -v true or false (output NON-matching lines)
+from __future__ import print_function
 
-import commands
 import os
 import re
+import subprocess
 import sys
-from subprocess import Popen, PIPE
+from subprocess import PIPE, Popen
 from tempfile import NamedTemporaryFile
 
 
@@ -38,31 +39,31 @@ def main():
     try:
         opts = getopts(args)
     except IndexError:
-        print "Usage:"
-        print " -i Input file"
-        print " -o Output file"
-        print " -pattern RegEx pattern"
-        print " -v true or false (Invert match)"
+        print("Usage:")
+        print(" -i Input file")
+        print(" -o Output file")
+        print(" -pattern RegEx pattern")
+        print(" -v true or false (Invert match)")
         return 0
 
     outputfile = opts.get("-o")
     if outputfile is None:
-        print "No output file specified."
+        print("No output file specified.")
         return -1
 
     inputfile = opts.get("-i")
     if inputfile is None:
-        print "No input file specified."
+        print("No input file specified.")
         return -2
 
     invert = opts.get("-v")
     if invert is None:
-        print "Match style (Invert or normal) not specified."
+        print("Match style (Invert or normal) not specified.")
         return -3
 
     pattern = opts.get("-pattern")
     if pattern is None:
-        print "RegEx pattern not specified."
+        print("RegEx pattern not specified.")
         return -4
 
     # All inputs have been specified at this point, now validate.
@@ -89,22 +90,22 @@ def main():
 
     # verify that filename and inversion flag are in the correct format
     if not fileRegEx.match(outputfile):
-        print "Illegal output filename."
+        print("Illegal output filename.")
         return -5
     if not fileRegEx.match(inputfile):
-        print "Illegal input filename."
+        print("Illegal input filename.")
         return -6
     if not invertRegEx.match(invert):
-        print "Illegal invert option."
+        print("Illegal invert option.")
         return -7
 
     # invert grep search?
     if invert == "true":
         invertflag = "-v"
-        print "Not matching pattern: %s" % pattern
+        print("Not matching pattern: %s" % pattern)
     else:
         invertflag = ""
-        print "Matching pattern: %s" % pattern
+        print("Matching pattern: %s" % pattern)
 
     # set version flag
     versionflag = "-P"
@@ -123,7 +124,7 @@ def main():
     commandline = "grep %s %s -f %s %s > %s" % ( versionflag, invertflag, pattern_file_name, inputfile, outputfile )
 
     # run grep
-    errorcode, stdout = commands.getstatusoutput(commandline)
+    errorcode = subprocess.call(commandline, shell=True)
 
     # remove temp pattern file
     os.unlink( pattern_file_name )

tools/filters/gtf_to_bedgraph_converter.py

@@ -1,4 +1,6 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
 import os
 import sys
 import tempfile
@@ -78,7 +80,7 @@ def __main__():
     info_msg = "%i lines converted to BEDGraph.  " % ( i + 1 - skipped_lines )
     if skipped_lines > 0:
         info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % ( skipped_lines, first_skipped_line )
-    print info_msg
+    print(info_msg)
 
 if __name__ == "__main__":
     __main__()

tools/filters/join.py

@@ -5,8 +5,8 @@ Script to Join Two Files on specified columns.
 
 Takes two tab delimited files, two column numbers (base 1) and outputs a new tab delimited file with lines joined by tabs.
 User can also opt to have have non-joining rows of file1 echoed.
-
 """
+from __future__ import print_function
 
 import json
 import optparse
@@ -24,7 +24,7 @@ class OffsetList:
         self.file = tempfile.NamedTemporaryFile( 'w+b' )
         if fmt:
             self.fmt = fmt
-        elif filesize and filesize <= sys.maxint * 2:
+        elif filesize and filesize <= sys.maxsize * 2:
             self.fmt = 'I'
         else:
             self.fmt = 'Q'
@@ -88,14 +88,14 @@ class SortedOffsets( OffsetList ):
     def merge_with_dict( self, new_offset_dict ):
         if not new_offset_dict:
             return  # no items to merge in
-        keys = new_offset_dict.keys()
+        keys = list(new_offset_dict.keys())
         keys.sort()
         identifier2 = keys.pop( 0 )
 
         result_offsets = OffsetList( fmt=self.fmt )
         offsets1 = enumerate( self.get_offsets() )
         try:
-            index1, offset1 = offsets1.next()
+            index1, offset1 = next(offsets1)
             identifier1 = self.get_identifier_by_offset( offset1 )
         except StopIteration:
             offset1 = None
@@ -121,7 +121,7 @@ class SortedOffsets( OffsetList ):
             else:
                 result_offsets.add_offset( offset1 )
                 try:
-                    index1, offset1 = offsets1.next()
+                    index1, offset1 = next(offsets1)
                     identifier1 = self.get_identifier_by_offset( offset1 )
                 except StopIteration:
                     offset1 = None
@@ -188,7 +188,7 @@ class OffsetIndex:
             offset_index += 1
 
     def get_offsets( self ):
-        keys = self._offsets.keys()
+        keys = list(self._offsets.keys())
         keys.sort()
         for key in keys:
             for offset in self._offsets[key].get_offsets():
@@ -199,7 +199,7 @@ class OffsetIndex:
         return self.file.readline()
 
     def get_identifiers_offsets( self ):
-        keys = self._offsets.keys()
+        keys = list(self._offsets.keys())
         keys.sort()
         for key in keys:
             for offset in self._offsets[key].get_offsets():
@@ -216,7 +216,7 @@ class OffsetIndex:
         if not d:
             return  # no data to merge
         self._index = None
-        keys = d.keys()
+        keys = list(d.keys())
         keys.sort()
         identifier = keys.pop( 0 )
         first_char = identifier[0]
@@ -360,7 +360,7 @@ def main():
         try:
             fill_options = Bunch( **stringify_dictionary_keys( json.load( open( options.fill_options_file ) ) ) )  # json.load( open( options.fill_options_file ) )
         except Exception as e:
-            print "Warning: Ignoring fill options due to json error (%s)." % e
+            print("Warning: Ignoring fill options due to json error (%s)." % e)
     if fill_options is None:
         fill_options = Bunch()
     if 'fill_unjoined_only' not in fill_options:
@@ -377,7 +377,7 @@ def main():
         column2 = int( args[3] ) - 1
         out_filename = args[4]
     except:
-        print >> sys.stderr, "Error parsing command line."
+        print("Error parsing command line.", file=sys.stderr)
         sys.exit()
 
     # Character for splitting fields and joining lines

tools/filters/lav_to_bed.py

@@ -1,5 +1,7 @@
 #!/usr/bin/env python
 # Reads a LAV file and writes two BED files.
+from __future__ import print_function
+
 import sys
 
 import bx.align.lav
@@ -38,13 +40,13 @@ def main():
         bedsWritten += 1
 
     for spec, file in species.items():
-        print "#FILE\t%s\t%s" % (file.name, spec)
+        print("#FILE\t%s\t%s" % (file.name, spec))
 
     lav_file.close()
     bed_file1.close()
     bed_file2.close()
 
-    print "%d lav blocks read, %d regions written\n" % (lavsRead, bedsWritten)
+    print("%d lav blocks read, %d regions written\n" % (lavsRead, bedsWritten))
 
 if __name__ == "__main__":
     main()

tools/filters/lav_to_bed_code.py

@@ -8,7 +8,7 @@ def exec_after_process(app, inp_data, out_data, param_dict, tool, stdout, stderr
             filename_to_build[fields[1]] = fields[2].strip()
         else:
             new_stdout = "%s%s" % ( new_stdout, line )
-    for name, data in out_data.items():
+    for data in out_data.values():
         try:
             data.info = "%s\n%s" % ( new_stdout, stderr )
             data.dbkey = filename_to_build[data.file_name]

tools/filters/mergeCols.py

@@ -1,3 +1,5 @@
+from __future__ import print_function
+
 import sys
 
 
@@ -31,10 +33,10 @@ def __main__():
                 except:
                     skipped_lines += 1
 
-            print >>outfile, line
+            print(line, file=outfile)
 
     if skipped_lines > 0:
-        print 'Skipped %d invalid lines' % skipped_lines
+        print('Skipped %d invalid lines' % skipped_lines)
 
 if __name__ == "__main__":
     __main__()

tools/filters/random_lines_two_pass.py

@@ -3,6 +3,7 @@
 # Selects N random lines from a file and outputs to another file, maintaining original line order
 # allows specifying a seed
 # does two passes to determine line offsets/count, and then to output contents
+from __future__ import print_function
 
 import optparse
 import random
@@ -68,9 +69,9 @@ def __main__():
         writer( readliner() )
     input.close()
     output.close()
-    print "Kept %i of %i total lines." % ( num_lines, total_lines )
+    print("Kept %i of %i total lines." % ( num_lines, total_lines ))
     if options.seed is not None:
-        print 'Used random seed of "%s".' % options.seed
+        print('Used random seed of "%s".' % options.seed)
 
 if __name__ == "__main__":
     __main__()

tools/filters/secure_hash_message_digest.py

@@ -33,14 +33,14 @@ def __main__():
     while True:
         chunk = input.read( CHUNK_SIZE )
         if chunk:
-            for algorithm in algorithms.itervalues():
+            for algorithm in algorithms.values():
                 algorithm.update( chunk )
         else:
             break
 
     output = open( options.output, 'wb' )
     output.write( '#%s\n' % ( '\t'.join( algorithms.keys() ) ) )
-    output.write( '%s\n' % ( '\t'.join( map( lambda x: x.hexdigest(), algorithms.values() ) ) ) )
+    output.write( '%s\n' % ( '\t'.join( x.hexdigest() for x in algorithms.values() ) ) )
     output.close()
 
 if __name__ == "__main__":

tools/filters/sff_extract.py

@@ -24,12 +24,7 @@ sequence will be removed, even if occuring multiple times.'''
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-__author__ = 'Jose Blanca and Bastien Chevreux'
-__copyright__ = 'Copyright 2008, Jose Blanca, COMAV, and Bastien Chevreux'
-__license__ = 'GPLv3 or later'
-__version__ = '0.2.10'
-__email__ = 'jblanca@btc.upv.es'
-__status__ = 'beta'
+from __future__ import print_function
 
 import os
 import struct
@@ -37,6 +32,12 @@ import subprocess
 import sys
 import tempfile
 
+__author__ = 'Jose Blanca and Bastien Chevreux'
+__copyright__ = 'Copyright 2008, Jose Blanca, COMAV, and Bastien Chevreux'
+__license__ = 'GPLv3 or later'
+__version__ = '0.2.10'
+__email__ = 'jblanca@btc.upv.es'
+__status__ = 'beta'
 
 fake_sff_name = 'fake_sff_name'
 
@@ -528,7 +529,7 @@ def fragment_sequences(sequence, qualities, splitchar):
     #  the sequence find find variations and splices on seq and qual
 
     if len(sequence) != len(qualities):
-        print sequence, qualities
+        print(sequence, qualities)
         raise RuntimeError("Internal error: length of sequence and qualities don't match???")
 
     retlist = ([])
@@ -985,7 +986,7 @@ def check_for_dubious_startseq(seqcheckstore, sffname, seqdata):
         if not foundinloop:
             break
     if len(foundproblem):
-        print foundproblem
+        print(foundproblem)
 
 
 def parse_extra_info(info):
@@ -1094,14 +1095,14 @@ def clip_read(data):
 def tests_for_ssaha():
     '''Tests whether SSAHA2 can be successfully called.'''
     try:
-        print "Testing whether SSAHA2 is installed and can be launched ... ",
+        print("Testing whether SSAHA2 is installed and can be launched ... ", end=' ')
         sys.stdout.flush()
         fh = open('/dev/null', 'w')
         subprocess.call(["ssaha2"], stdout=fh)
         fh.close()
-        print "ok."
+        print("ok.")
     except:
-        print "nope? Uh oh ...\n\n"
+        print("nope? Uh oh ...\n\n")
         raise RuntimeError('Could not launch ssaha2. Have you installed it? Is it in your path?')
 
 
@@ -1129,15 +1130,15 @@ def launch_ssaha(linker_fname, query_fname, output_fh):
     tests_for_ssaha()
 
     try:
-        print "Searching linker sequences with SSAHA2 (this may take a while) ... ",
+        print("Searching linker sequences with SSAHA2 (this may take a while) ... ", end=' ')
         sys.stdout.flush()
         retcode = subprocess.call(["ssaha2", "-output", "ssaha2", "-solexa", "-kmer", "4", "-skip", "1", linker_fname, query_fname], stdout=output_fh)
         if retcode:
             raise RuntimeError('Ups.')
         else:
-            print "ok."
+            print("ok.")
     except:
-        print "\n"
+        print("\n")
         raise RuntimeError('An error occured during the SSAHA2 execution, aborting.')
 
 
@@ -1147,14 +1148,14 @@ def read_ssaha_data(ssahadata_fh):
     (ssaha paired-end matches) dictionary'''
     global ssahapematches
 
-    print "Parsing SSAHA2 result file ... ",
+    print("Parsing SSAHA2 result file ... ", end=' ')
     sys.stdout.flush()
 
     for line in ssahadata_fh:
         if line.startswith('ALIGNMENT'):
             ml = line.split()
             if len(ml) != 12:
-                print "\n", line,
+                print("\n", line, end=' ')
                 raise RuntimeError('Expected 12 elements in the SSAHA2 line with ALIGMENT keyword, but found ' + str(len(ml)))
             if ml[2] not in ssahapematches:
                 ssahapematches[ml[2]] = ([])
@@ -1167,7 +1168,7 @@ def read_ssaha_data(ssahadata_fh):
                 ml[4], ml[5] = ml[5], ml[4]
                 ssahapematches[ml[2]].append(ml[1:-1])
 
-    print "done."
+    print("done.")
 
 
 ##########################################################################
@@ -1326,7 +1327,7 @@ def main():
             raise RuntimeError("No SFF file given?")
         extract_reads_from_sff(config, args)
     except (OSError, IOError, RuntimeError) as errval:
-        print errval
+        print(errval)
         return 1
 
     if stern_warning:

tools/filters/trimmer.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import print_function
 
 import optparse
 import sys
@@ -80,7 +81,7 @@ options (listed below) default to 'None' if omitted
         line = line.rstrip( '\r\n' )
         if line:
             if options.fastq and i % 2 == 0:
-                print line
+                print(line)
                 continue
 
             if line[0] not in invalid_starts:
@@ -105,7 +106,7 @@ options (listed below) default to 'None' if omitted
                     else:
                         fields[col - 1] = fields[col - 1][ int( options.start ) - 1: ]
                     line = '\t'.join(fields)
-            print line
+            print(line)
 
 if __name__ == "__main__":
     main()

tools/filters/ucsc_gene_bed_to_exon_bed.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 """
 Read a table dump in the UCSC gene table format and print a tab separated
 list of intervals corresponding to requested features of each gene.
@@ -14,9 +13,9 @@ options:
   -i, --input=inputfile       input file
   -o, --output=outputfile     output file
 """
+from __future__ import print_function
 
 import optparse
-import string
 import sys
 
 assert sys.version_info[:2] >= ( 2, 4 )
@@ -40,16 +39,16 @@ def main():
     try:
         out_file = open(options.output, "w")
     except:
-        print >> sys.stderr, "Bad output file."
+        print("Bad output file.", file=sys.stderr)
         sys.exit(0)
 
     try:
         in_file = open(options.input)
     except:
-        print >> sys.stderr, "Bad input file."
+        print("Bad input file.", file=sys.stderr)
         sys.exit(0)
 
-    print "Region:", options.region + ";"
+    print("Region:", options.region + ";")
     """print "Only overlap with Exons:",
     if options.exons:
         print "Yes"
@@ -92,10 +91,9 @@ def main():
             # the region of interest, otherwise print the span of the region
         # options.exons is always TRUE
             if options.exons:
-                exon_starts = map( int, fields[11].rstrip( ',\n' ).split( ',' ) )
-                exon_starts = map((lambda x: x + tx_start ), exon_starts)
-                exon_ends = map( int, fields[10].rstrip( ',\n' ).split( ',' ) )
-                exon_ends = map((lambda x, y: x + y ), exon_starts, exon_ends)
+                exon_starts = [int(_) + tx_start for _ in fields[11].rstrip( ',\n' ).split( ',' )]
+                exon_ends = [int(_) for _ in fields[10].rstrip( ',\n' ).split( ',' )]
+                exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)]
 
         # for Intron regions:
             if options.region == 'intron':
@@ -134,7 +132,7 @@ def main():
 
 def print_tab_sep(out_file, *args ):
     """Print items in `l` to stdout separated by tabs"""
-    print >>out_file, string.join( [ str( f ) for f in args ], '\t' )
+    print('\t'.join(str( f ) for f in args), file=out_file)
 
 if __name__ == "__main__":
     main()

tools/filters/ucsc_gene_bed_to_intron_bed.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 """
 Read a table dump in the UCSC gene table format and print a tab separated
 list of intervals corresponding to requested features of each gene.
@@ -14,9 +13,9 @@ options:
   -i, --input=inputfile       input file
   -o, --output=outputfile     output file
 """
+from __future__ import print_function
 
 import optparse
-import string
 import sys
 
 assert sys.version_info[:2] >= ( 2, 4 )
@@ -35,13 +34,13 @@ def main():
     try:
         out_file = open(options.output, "w")
     except:
-        print >> sys.stderr, "Bad output file."
+        print("Bad output file.", file=sys.stderr)
         sys.exit(0)
 
     try:
         in_file = open(options.input)
     except:
-        print >> sys.stderr, "Bad input file."
+        print("Bad input file.", file=sys.stderr)
         sys.exit(0)
 
     # Read table and handle each gene
@@ -60,10 +59,9 @@ def main():
             int( fields[6] )
             int( fields[7] )
 
-            exon_starts = map( int, fields[11].rstrip( ',\n' ).split( ',' ) )
-            exon_starts = map((lambda x: x + tx_start ), exon_starts)
-            exon_ends = map( int, fields[10].rstrip( ',\n' ).split( ',' ) )
-            exon_ends = map((lambda x, y: x + y ), exon_starts, exon_ends)
+            exon_starts = [int(_) + tx_start for _ in fields[11].rstrip( ',\n' ).split( ',' )]
+            exon_ends = [int(_) for _ in fields[10].rstrip( ',\n' ).split( ',' )]
+            exon_ends = [x + y for x, y in zip(exon_starts, exon_ends)]
 
             i = 0
             while i < len(exon_starts) - 1:
@@ -80,7 +78,7 @@ def main():
 
 def print_tab_sep(out_file, *args ):
     """Print items in `l` to stdout separated by tabs"""
-    print >>out_file, string.join( [ str( f ) for f in args ], '\t' )
+    print('\t'.join(str( f ) for f in args), file=out_file)
 
 if __name__ == "__main__":
     main()

tools/filters/ucsc_gene_table_to_intervals.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 """
 Read a table dump in the UCSC gene table format and print a tab separated
 list of intervals corresponding to requested features of each gene.
@@ -14,9 +13,9 @@ options:
   -i, --input=inputfile       input file
   -o, --output=outputfile     output file
 """
+from __future__ import print_function
 
 import optparse
-import string
 import sys
 
 assert sys.version_info[:2] >= ( 2, 4 )
@@ -40,21 +39,21 @@ def main():
     try:
         out_file = open(options.output, "w")
     except:
-        print >> sys.stderr, "Bad output file."
+        print("Bad output file.", file=sys.stderr)
         sys.exit(0)
 
     try:
         in_file = open(options.input)
     except:
-        print >> sys.stderr, "Bad input file."
+        print("Bad input file.", file=sys.stderr)
         sys.exit(0)
 
-    print "Region:", options.region + ";"
-    print "Only overlap with Exons:",
+    print("Region:", options.region + ";")
+    print("Only overlap with Exons:", end=' ')
     if options.exons:
-        print "Yes"
+        print("Yes")
     else:
-        print "No"
+        print("No")
 
     # Read table and handle each gene
     for line in in_file:
@@ -111,7 +110,7 @@ def main():
 
 def print_tab_sep(out_file, *args ):
     """Print items in `l` to stdout separated by tabs"""
-    print >>out_file, string.join( [ str( f ) for f in args ], '\t' )
+    print('\t'.join(str( f ) for f in args), file=out_file)
 
 if __name__ == "__main__":
     main()

tools/filters/uniq.py

@@ -15,9 +15,10 @@
 # -o            Output file
 # -d            Delimiter
 # -c            Column list (Comma Seperated)
+from __future__ import print_function
 
-import commands
 import re
+import subprocess
 import sys
 
 
@@ -39,46 +40,46 @@ def main():
     try:
         opts = getopts(args)
     except IndexError:
-        print "Usage:"
-        print " -i        Input file"
-        print " -o        Output file"
-        print " -c        Column list (comma seperated)"
-        print " -d        Delimiter:"
-        print "                     T   Tab"
-        print "                     C   Comma"
-        print "                     D   Dash"
-        print "                     U   Underscore"
-        print "                     P   Pipe"
-        print "                     Dt  Dot"
-        print "                     Sp  Space"
-        print " -s        Sorting: value (default), largest, or smallest"
+        print("Usage:")
+        print(" -i        Input file")
+        print(" -o        Output file")
+        print(" -c        Column list (comma seperated)")
+        print(" -d        Delimiter:")
+        print("                     T   Tab")
+        print("                     C   Comma")
+        print("                     D   Dash")
+        print("                     U   Underscore")
+        print("                     P   Pipe")
+        print("                     Dt  Dot")
+        print("                     Sp  Space")
+        print(" -s        Sorting: value (default), largest, or smallest")
         return 0
 
     outputfile = opts.get("-o")
     if outputfile is None:
-        print "No output file specified."
+        print("No output file specified.")
         return -1
 
     inputfile = opts.get("-i")
     if inputfile is None:
-        print "No input file specified."
+        print("No input file specified.")
         return -2
 
     delim = opts.get("-d")
     if delim is None:
-        print "Field delimiter not specified."
+        print("Field delimiter not specified.")
         return -3
 
     columns = opts.get("-c")
     if columns is None or columns == 'None':
-        print "Columns not specified."
+        print("Columns not specified.")
         return -4
 
     sorting = opts.get("-s")
     if sorting is None:
         sorting = "value"
     if sorting not in ["value", "largest", "smallest"]:
-        print "Unknown sorting option %r" % sorting
+        print("Unknown sorting option %r" % sorting)
         return -5
 
     # All inputs have been specified at this point, now validate.
@@ -86,13 +87,13 @@ def main():
     columnRegEx = re.compile("([0-9]{1,},?)+")
 
     if not columnRegEx.match(columns):
-        print "Illegal column specification."
+        print("Illegal column specification.")
         return -4
     if not fileRegEx.match(outputfile):
-        print "Illegal output filename."
+        print("Illegal output filename.")
         return -5
     if not fileRegEx.match(inputfile):
-        print "Illegal input filename."
+        print("Illegal input filename.")
         return -6
 
     column_list = re.split(",", columns)
@@ -130,9 +131,9 @@ def main():
     # uniq -C puts a space between the count and the field, want a tab.
     # To replace just first tab, use sed again with 1 as the index
     commandline += " | sed 's/^\ *//' | sed 's/ /\t/1' > " + outputfile
-    errorcode, stdout = commands.getstatusoutput(commandline)
+    errorcode = subprocess.call(commandline, shell=True)
 
-    print "Count of unique values in " + columns_for_display
+    print("Count of unique values in " + columns_for_display)
     return errorcode
 
 if __name__ == "__main__":

tools/filters/wiggle_to_simple.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python
-
 """
 Read a wiggle track and print out a series of lines containing
 "chrom position score". Ignores track lines, handles bed, variableStep
 and fixedStep wiggle lines.
 """
+from __future__ import print_function
+
 import sys
 
 import bx.wiggle
@@ -33,7 +34,7 @@ def main():
             out_file.write( "%s\n" % "\t".join( map( str, fields ) ) )
     except UCSCLimitException:
         # Wiggle data was truncated, at the very least need to warn the user.
-        print 'Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.'
+        print('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.')
     except ValueError as e:
         in_file.close()
         out_file.close()

tools/genomespace/genomespace_exporter.py

@@ -1,8 +1,9 @@
 #!/usr/bin/env python
 # Dan Blankenberg
+from __future__ import print_function
+
 import base64
 import binascii
-import cookielib
 import datetime
 import hashlib
 import json
@@ -10,9 +11,12 @@ import logging
 import optparse
 import os
 import tempfile
-import urllib
-import urllib2
-from urlparse import urljoin
+
+import six
+from six.moves import http_cookiejar
+from six.moves.urllib.error import HTTPError
+from six.moves.urllib.parse import quote, urlencode, urljoin
+from six.moves.urllib.request import build_opener, HTTPCookieProcessor, Request, urlopen
 
 log = logging.getLogger( "tools.genomespace.genomespace_exporter" )
 
@@ -56,19 +60,19 @@ def chunk_write( source_stream, target_stream, source_method="read", target_meth
 
 def get_cookie_opener( gs_username, gs_token, gs_toolname=None ):
     """ Create a GenomeSpace cookie opener """
-    cj = cookielib.CookieJar()
+    cj = http_cookiejar.CookieJar()
     for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]:
         # create a super-cookie, valid for all domains
-        cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
+        cookie = http_cookiejar.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
         cj.set_cookie( cookie )
-    cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) )
+    cookie_opener = build_opener( HTTPCookieProcessor( cj ) )
     cookie_opener.addheaders.append( ( 'gs-toolname', gs_toolname or DEFAULT_GENOMESPACE_TOOLNAME ) )
     return cookie_opener
 
 
 def get_genomespace_site_urls():
     genomespace_sites = {}
-    for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ):
+    for line in urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ):
         line = line.rstrip()
         if not line or line.startswith( "#" ):
             continue
@@ -86,11 +90,11 @@ def get_directory( url_opener, dm_url, path ):
     dir_dict = {}
     for i, sub_path in enumerate( path ):
         url = "%s/%s" % ( url, sub_path )
-        dir_request = urllib2.Request( url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json' } )
+        dir_request = Request( url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json' } )
         dir_request.get_method = lambda: 'GET'
         try:
             dir_dict = json.loads( url_opener.open( dir_request ).read() )
-        except urllib2.HTTPError:
+        except HTTPError:
             # print "e", e, url #punting, assuming lack of permissions at this low of a level...
             continue
         break
@@ -114,15 +118,15 @@ def create_directory( url_opener, directory_dict, new_dir, dm_url ):
     for dir_slice in new_dir:
         if dir_slice in ( '', '/', None ):
             continue
-        url = '/'.join( ( directory_dict['url'], urllib.quote( dir_slice.replace( '/', '_' ), safe='' ) ) )
-        new_dir_request = urllib2.Request( url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json' }, data=json.dumps( payload ) )
+        url = '/'.join( ( directory_dict['url'], quote( dir_slice.replace( '/', '_' ), safe='' ) ) )
+        new_dir_request = Request( url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json' }, data=json.dumps( payload ) )
         new_dir_request.get_method = lambda: 'PUT'
         directory_dict = json.loads( url_opener.open( new_dir_request ).read() )
     return directory_dict
 
 
 def get_genome_space_launch_apps( atm_url, url_opener, file_url, file_type ):
-    gs_request = urllib2.Request( "%s/%s/webtool/descriptor" % ( atm_url, GENOMESPACE_API_VERSION_STRING ) )
+    gs_request = Request( "%s/%s/webtool/descriptor" % ( atm_url, GENOMESPACE_API_VERSION_STRING ) )
     gs_request.get_method = lambda: 'GET'
     opened_gs_request = url_opener.open( gs_request )
     webtool_descriptors = json.loads( opened_gs_request.read() )
@@ -143,7 +147,7 @@ def get_genome_space_launch_apps( atm_url, url_opener, file_url, file_type ):
                     url_delimiter = "&"
                 else:
                     url_delimiter = "?"
-                launch_url = "%s%s%s" % ( base_url, url_delimiter, urllib.urlencode( [ ( file_param_name, file_url ) ] ) )
+                launch_url = "%s%s%s" % ( base_url, url_delimiter, urlencode( [ ( file_param_name, file_url ) ] ) )
                 webtools.append( ( launch_url, webtool_name ) )
                 break
     return webtools
@@ -153,19 +157,19 @@ def galaxy_code_get_genomespace_folders( genomespace_site='prod', trans=None, va
     if value:
         if isinstance( value, list ):
             value = value[0]  # single select, only 1 value
-        elif not isinstance( value, basestring ):
+        elif not isinstance( value, six.string_types ):
             # unvalidated value
             value = value.value
             if isinstance( value, list ):
                 value = value[0]  # single select, only 1 value
 
     def recurse_directory_dict( url_opener, cur_options, url ):
-        cur_directory = urllib2.Request( url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json, text/plain' } )
+        cur_directory = Request( url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json, text/plain' } )
         cur_directory.get_method = lambda: 'GET'
         # get url to upload to
         try:
             cur_directory = url_opener.open( cur_directory ).read()
-        except urllib2.HTTPError as e:
+        except HTTPError as e:
             log.debug( 'GenomeSpace export tool failed reading a directory "%s": %s' % ( url, e ) )
             return  # bad url, go to next
         cur_directory = json.loads( cur_directory )
@@ -244,11 +248,11 @@ def send_file_to_genomespace( genomespace_site, username, token, source_filename
                     sizes = [ last_size ]
             else:
                 sizes.append( last_size )
-        print "Performing multi-part upload in %i parts." % ( len( sizes ) )
+        print("Performing multi-part upload in %i parts." % ( len( sizes ) ))
         # get upload url
         upload_url = "uploadinfo"
-        upload_url = "%s/%s/%s%s/%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ) )
-        upload_request = urllib2.Request( upload_url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json' } )
+        upload_url = "%s/%s/%s%s/%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], quote( target_filename, safe='' ) )
+        upload_request = Request( upload_url, headers={ 'Content-Type': 'application/json', 'Accept': 'application/json' } )
         upload_request.get_method = lambda: 'GET'
         upload_info = json.loads( url_opener.open( upload_request ).read() )
         conn = S3Connection( aws_access_key_id=upload_info['amazonCredentials']['accessKey'],
@@ -273,15 +277,15 @@ def send_file_to_genomespace( genomespace_site, username, token, source_filename
             fh.close()
         upload_result = mp.complete_upload()
     else:
-        print 'Performing simple put upload.'
+        print('Performing simple put upload.')
         upload_url = "uploadurl"
         content_md5 = hashlib.md5()
         chunk_write( input_file, content_md5, target_method="update" )
         input_file.seek( 0 )  # back to start, for uploading
 
         upload_params = { 'Content-Length': content_length, 'Content-MD5': base64.standard_b64encode( content_md5.digest() ), 'Content-Type': content_type }
-        upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], urllib.quote( target_filename, safe='' ), urllib.urlencode( upload_params ) )
-        new_file_request = urllib2.Request( upload_url )  # , headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct
+        upload_url = "%s/%s/%s%s/%s?%s" % ( dm_url, GENOMESPACE_API_VERSION_STRING, upload_url, target_directory_dict['path'], quote( target_filename, safe='' ), urlencode( upload_params ) )
+        new_file_request = Request( upload_url )  # , headers = { 'Content-Type': 'application/json', 'Accept': 'application/text' } ) #apparently http://www.genomespace.org/team/specs/updated-dm-rest-api:"Every HTTP request to the Data Manager should include the Accept header with a preference for the media types application/json and application/text." is not correct
         new_file_request.get_method = lambda: 'GET'
         # get url to upload to
         target_upload_url = url_opener.open( new_file_request ).read()
@@ -289,10 +293,10 @@ def send_file_to_genomespace( genomespace_site, username, token, source_filename
         upload_headers = dict( upload_params )
         # upload_headers[ 'x-amz-meta-md5-hash' ] = content_md5.hexdigest()
         upload_headers[ 'Accept' ] = 'application/json'
-        upload_file_request = urllib2.Request( target_upload_url, headers=upload_headers, data=input_file )
+        upload_file_request = Request( target_upload_url, headers=upload_headers, data=input_file )
         upload_file_request.get_method = lambda: 'PUT'
-        upload_result = urllib2.urlopen( upload_file_request ).read()
-    result_url = "%s/%s" % ( target_directory_dict['url'], urllib.quote( target_filename, safe='' ) )
+        upload_result = urlopen( upload_file_request ).read()
+    result_url = "%s/%s" % ( target_directory_dict['url'], quote( target_filename, safe='' ) )
     # determine available gs launch apps
     web_tools = get_genome_space_launch_apps( genomespace_site_dict['atmServer'], url_opener, result_url, file_type )
     if log_filename:
@@ -326,4 +330,4 @@ if __name__ == '__main__':
 
     (options, args) = parser.parse_args()
 
-    send_file_to_genomespace( options.genomespace_site, options.username, options.token, options.dataset, map( binascii.unhexlify, options.subdirectory ), binascii.unhexlify( options.filename ), options.file_type, options.content_type, options.log, options.genomespace_toolname )
+    send_file_to_genomespace( options.genomespace_site, options.username, options.token, options.dataset, [binascii.unhexlify(_) for _ in options.subdirectory], binascii.unhexlify( options.filename ), options.file_type, options.content_type, options.log, options.genomespace_toolname )

tools/genomespace/genomespace_file_browser.py

@@ -1,11 +1,11 @@
 # Dan Blankenberg
-import cookielib
 import json
 import optparse
 import os
-import urllib
-import urllib2
-import urlparse
+
+from six.moves import http_cookiejar
+from six.moves.urllib.parse import unquote_plus, urlencode, urlparse
+from six.moves.urllib.request import build_opener, HTTPCookieProcessor, Request, urlopen
 
 from galaxy.datatypes import sniff
 from galaxy.datatypes.registry import Registry
@@ -61,12 +61,12 @@ def chunk_write( source_stream, target_stream, source_method="read", target_meth
 
 def get_cookie_opener( gs_username, gs_token, gs_toolname=None ):
     """ Create a GenomeSpace cookie opener """
-    cj = cookielib.CookieJar()
+    cj = http_cookiejar.CookieJar()
     for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]:
         # create a super-cookie, valid for all domains
-        cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
+        cookie = http_cookiejar.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
         cj.set_cookie( cookie )
-    cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) )
+    cookie_opener = build_opener( HTTPCookieProcessor( cj ) )
     cookie_opener.addheaders.append( ( 'gs-toolname', gs_toolname or DEFAULT_GENOMESPACE_TOOLNAME ) )
     return cookie_opener
 
@@ -83,7 +83,7 @@ def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url ):
 
 def get_genomespace_site_urls():
     genomespace_sites = {}
-    for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ):
+    for line in urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ):
         line = line.rstrip()
         if not line or line.startswith( "#" ):
             continue
@@ -96,14 +96,14 @@ def get_genomespace_site_urls():
 
 
 def set_genomespace_format_identifiers( url_opener, dm_site ):
-    gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) )
+    gs_request = Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) )
     gs_request.get_method = lambda: 'GET'
     opened_gs_request = url_opener.open( gs_request )
     genomespace_formats = json.loads( opened_gs_request.read() )
     for format in genomespace_formats:
         GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT[ format['url'] ] = format['name']
     global GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN
-    GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = dict( map( lambda x: ( x[1], x[0] ), GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.iteritems() ) ).get( GENOMESPACE_UNKNOWN_FORMAT_KEY, GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN )
+    GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN = dict( ( x[1], x[0] ) for x in GENOMESPACE_FORMAT_IDENTIFIER_TO_GENOMESPACE_EXT.items() ).get( GENOMESPACE_UNKNOWN_FORMAT_KEY, GENOMESPACE_FORMAT_IDENTIFIER_UNKNOWN )
 
 
 def download_from_genomespace_file_browser( json_parameter_file, genomespace_site, gs_toolname ):
@@ -147,19 +147,19 @@ def download_from_genomespace_file_browser( json_parameter_file, genomespace_sit
         filetype_key = "%s%i" % ( file_type_prefix, file_num )
         filetype_url = datasource_params.get( filetype_key, None )
         galaxy_ext = get_galaxy_ext_from_genomespace_format_url( url_opener, filetype_url )
-        formated_download_url = "%s?%s" % ( download_url, urllib.urlencode( [ ( 'dataformat', filetype_url ) ] ) )
-        new_file_request = urllib2.Request( formated_download_url )
+        formatted_download_url = "%s?%s" % ( download_url, urlencode( [ ( 'dataformat', filetype_url ) ] ) )
+        new_file_request = Request( formatted_download_url )
         new_file_request.get_method = lambda: 'GET'
         target_download_url = url_opener.open( new_file_request )
         filename = None
         if 'Content-Disposition' in target_download_url.info():
             # If the response has Content-Disposition, try to get filename from it
-            content_disposition = dict( map( lambda x: x.strip().split('=') if '=' in x else ( x.strip(), '' ), target_download_url.info()['Content-Disposition'].split( ';' ) ) )
+            content_disposition = dict( x.strip().split('=') if '=' in x else ( x.strip(), '' ) for x in target_download_url.info()['Content-Disposition'].split( ';' ) )
             if 'filename' in content_disposition:
                 filename = content_disposition[ 'filename' ].strip( "\"'" )
         if not filename:
-            parsed_url = urlparse.urlparse( download_url )
-            filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] )
+            parsed_url = urlparse( download_url )
+            filename = unquote_plus( parsed_url[2].split( '/' )[-1] )
         if not filename:
             filename = download_url
         metadata_dict = None

tools/genomespace/genomespace_importer.py

@@ -1,14 +1,14 @@
 # Dan Blankenberg
 
-import cookielib
 import json
 import optparse
 import os
 import shutil
 import tempfile
-import urllib
-import urllib2
-import urlparse
+
+from six.moves import http_cookiejar
+from six.moves.urllib.parse import parse_qs, unquote_plus, urlparse
+from six.moves.urllib.request import build_opener, HTTPCookieProcessor, Request, urlopen
 
 from galaxy.datatypes import sniff
 from galaxy.datatypes.registry import Registry
@@ -60,12 +60,12 @@ def chunk_write( source_stream, target_stream, source_method="read", target_meth
 
 def get_cookie_opener( gs_username, gs_token, gs_toolname=None ):
     """ Create a GenomeSpace cookie opener """
-    cj = cookielib.CookieJar()
+    cj = http_cookiejar.CookieJar()
     for cookie_name, cookie_value in [ ( 'gs-token', gs_token ), ( 'gs-username', gs_username ) ]:
         # create a super-cookie, valid for all domains
-        cookie = cookielib.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
+        cookie = http_cookiejar.Cookie(version=0, name=cookie_name, value=cookie_value, port=None, port_specified=False, domain='', domain_specified=False, domain_initial_dot=False, path='/', path_specified=True, secure=False, expires=None, discard=True, comment=None, comment_url=None, rest={'HttpOnly': None}, rfc2109=False )
         cj.set_cookie( cookie )
-    cookie_opener = urllib2.build_opener( urllib2.HTTPCookieProcessor( cj ) )
+    cookie_opener = build_opener( HTTPCookieProcessor( cj ) )
     cookie_opener.addheaders.append( ( 'gs-toolname', gs_toolname or DEFAULT_GENOMESPACE_TOOLNAME ) )
     return cookie_opener
 
@@ -82,7 +82,7 @@ def get_galaxy_ext_from_genomespace_format_url( url_opener, file_format_url, def
 
 def get_genomespace_site_urls():
     genomespace_sites = {}
-    for line in urllib2.urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ):
+    for line in urlopen( GENOMESPACE_SERVER_URL_PROPERTIES ).read().split( '\n' ):
         line = line.rstrip()
         if not line or line.startswith( "#" ):
             continue
@@ -95,7 +95,7 @@ def get_genomespace_site_urls():
 
 
 def set_genomespace_format_identifiers( url_opener, dm_site ):
-    gs_request = urllib2.Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) )
+    gs_request = Request( "%s/%s/dataformat/list" % ( dm_site, GENOMESPACE_API_VERSION_STRING ) )
     gs_request.get_method = lambda: 'GET'
     opened_gs_request = url_opener.open( gs_request )
     genomespace_formats = json.loads( opened_gs_request.read() )
@@ -123,21 +123,21 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge
     used_filenames = []
     for download_url in url_param.split( ',' ):
         using_temp_file = False
-        parsed_url = urlparse.urlparse( download_url )
-        query_params = urlparse.parse_qs( parsed_url[4] )
+        parsed_url = urlparse( download_url )
+        query_params = parse_qs( parsed_url[4] )
         # write file to disk
-        new_file_request = urllib2.Request( download_url )
+        new_file_request = Request( download_url )
         new_file_request.get_method = lambda: 'GET'
         target_download_url = url_opener.open( new_file_request )
         filename = None
         if 'Content-Disposition' in target_download_url.info():
-            content_disposition = dict( map( lambda x: x.strip().split('=') if '=' in x else ( x.strip(), '' ), target_download_url.info()['Content-Disposition'].split( ';' ) ) )
+            content_disposition = dict( x.strip().split('=') if '=' in x else ( x.strip(), '' ) for x in target_download_url.info()['Content-Disposition'].split( ';' ) )
             if 'filename' in content_disposition:
                 filename = content_disposition[ 'filename' ].strip( "\"'" )
         if not filename:
-            parsed_url = urlparse.urlparse( download_url )
-            query_params = urlparse.parse_qs( parsed_url[4] )
-            filename = urllib.unquote_plus( parsed_url[2].split( '/' )[-1] )
+            parsed_url = urlparse( download_url )
+            query_params = parse_qs( parsed_url[4] )
+            filename = unquote_plus( parsed_url[2].split( '/' )[-1] )
         if not filename:
             filename = download_url
         if output_filename is None:
@@ -157,7 +157,7 @@ def download_from_genomespace_importer( username, token, json_parameter_file, ge
             try:
                 # get and use GSMetadata object
                 download_file_path = download_url.split( "%s/file/" % ( genomespace_site_dict['dmServer'] ), 1)[-1]  # FIXME: This is a very bad way to get the path for determining metadata. There needs to be a way to query API using download URLto get to the metadata object
-                metadata_request = urllib2.Request( "%s/%s/filemetadata/%s" % ( genomespace_site_dict['dmServer'], GENOMESPACE_API_VERSION_STRING, download_file_path ) )
+                metadata_request = Request( "%s/%s/filemetadata/%s" % ( genomespace_site_dict['dmServer'], GENOMESPACE_API_VERSION_STRING, download_file_path ) )
                 metadata_request.get_method = lambda: 'GET'
                 metadata_url = url_opener.open( metadata_request )
                 file_metadata_dict = json.loads( metadata_url.read() )

tools/maf/interval2maf.py

@@ -25,6 +25,8 @@ usage: %prog maf_file [options]
    -z, --mafIndexFile=z: Directory of local maf index file ( maf_index.loc or maf_pairwise.loc )
 """
 # Dan Blankenberg
+from __future__ import print_function
+
 import bx.align.maf
 import bx.intervals.io
 from bx.cookbook import doc_optparse
@@ -132,11 +134,11 @@ def __main__():
     maf_utilities.remove_temp_index_file( index_filename )
 
     if num_blocks:
-        print "%i MAF blocks extracted for %i regions." % ( num_blocks, ( num_regions + 1 ) )
+        print("%i MAF blocks extracted for %i regions." % ( num_blocks, ( num_regions + 1 ) ))
     elif num_regions is not None:
-        print "No MAF blocks could be extracted for %i regions." % ( num_regions + 1 )
+        print("No MAF blocks could be extracted for %i regions." % ( num_regions + 1 ))
     else:
-        print "No valid regions have been provided."
+        print("No valid regions have been provided.")
 
 if __name__ == "__main__":
     __main__()

tools/maf/interval_maf_to_merged_fasta.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python
-
 """
 Reads an interval or gene BED and a MAF Source.
 Produces a FASTA file containing the aligned intervals/gene sequences, based upon the provided coordinates
@@ -24,8 +23,9 @@ usage: %prog maf_file [options]
 
 usage: %prog dbkey_of_BED comma_separated_list_of_additional_dbkeys_to_extract comma_separated_list_of_indexed_maf_files input_gene_bed_file output_fasta_file cached|user GALAXY_DATA_INDEX_DIR
 """
-
 # Dan Blankenberg
+from __future__ import print_function
+
 import sys
 
 import bx.intervals.io
@@ -142,7 +142,7 @@ def __main__():
                     primary_name = secondary_name = fields[3]
                     alignment_strand = fields[5]
                 except Exception as e:
-                    print "Error loading exon positions from input line %i: %s" % ( line_count, e )
+                    print("Error loading exon positions from input line %i: %s" % ( line_count, e ))
                     continue
             else:  # Process as standard intervals
                 try:
@@ -155,7 +155,7 @@ def __main__():
                     secondary_name = ""
                     alignment_strand = line.strand
                 except Exception as e:
-                    print "Error loading region positions from input line %i: %s" % ( line_count, e )
+                    print("Error loading region positions from input line %i: %s" % ( line_count, e ))
                     continue
 
             # Write alignment to output file
@@ -182,7 +182,7 @@ def __main__():
             output.write( "\n" )
             regions_extracted += 1
         except Exception as e:
-            print "Unexpected error from input line %i: %s" % ( line_count, e )
+            print("Unexpected error from input line %i: %s" % ( line_count, e ))
             continue
 
     # close output file
@@ -193,11 +193,11 @@ def __main__():
 
     # Print message about success for user
     if regions_extracted > 0:
-        print "%i regions were processed successfully." % ( regions_extracted )
+        print("%i regions were processed successfully." % ( regions_extracted ))
     else:
-        print "No regions were processed successfully."
+        print("No regions were processed successfully.")
         if line_count > 0 and options.geneBED:
-            print "This tool requires your input file to conform to the 12 column BED standard."
+            print("This tool requires your input file to conform to the 12 column BED standard.")
 
 if __name__ == "__main__":
     __main__()