Merge branch 'dev' into fix_workflow_ui

.ci/flake8_lint_include_list.txt

@@ -264,35 +264,7 @@ lib/galaxy/sample_tracking/__init__.py
 lib/galaxy/sample_tracking/sample.py
 lib/galaxy/security/validate_user_input.py
 lib/galaxy/tags/
-lib/galaxy/tools/actions/
-lib/galaxy/tools/cwl/
-lib/galaxy/tools/data_manager/__init__.py
-lib/galaxy/tools/deps/
-lib/galaxy/tools/exception_handling.py
-lib/galaxy/tools/execute.py
-lib/galaxy/tools/filters/
-lib/galaxy/tools/imp_exp/export_history.py
-lib/galaxy/tools/imp_exp/__init__.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/lint.py
-lib/galaxy/tools/lint_util.py
-lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/loader.py
-lib/galaxy/tools/parameters/dataset_matcher.py
-lib/galaxy/tools/parameters/history_query.py
-lib/galaxy/tools/parameters/__init__.py
-lib/galaxy/tools/parameters/input_translation.py
-lib/galaxy/tools/parameters/sanitize.py
-lib/galaxy/tools/parameters/validation.py
-lib/galaxy/tools/parameters/wrapped_json.py
-lib/galaxy/tools/parameters/wrapped.py
-lib/galaxy/tools/parser/
-lib/galaxy/tools/special_tools.py
-lib/galaxy/tools/test.py
-lib/galaxy/tools/toolbox/
-lib/galaxy/tools/util/galaxyops/
-lib/galaxy/tools/util/__init__.py
-lib/galaxy/tools/verify/
+lib/galaxy/tools/
 lib/galaxy/util/
 lib/galaxy_utils/__init__.py
 lib/galaxy/util/sleeper.py

.ci/jenkins/selenium/run_galaxy.bash

@@ -2,9 +2,12 @@
 
 set -e
 
-sleep 30  # TODO: wait on something instead of just sleeping...
-
-echo `df`
+echo "Waiting for postgres to become available"
+while ! nc -z postgres 5432;
+do
+    sleep 1
+    printf "."
+done
 
 echo "Creating postgres database for Galaxy"
 createdb -w -U postgres -h postgres galaxy

.ci/py3_sources.txt

@@ -40,26 +40,7 @@ lib/galaxy/quota/
 lib/galaxy/sample_tracking/
 lib/galaxy/security/
 lib/galaxy/tags/
-lib/galaxy/tools/actions/
-lib/galaxy/tools/cwl/
-lib/galaxy/tools/deps/
-lib/galaxy/tools/exception_handling.py
-lib/galaxy/tools/execute.py
-lib/galaxy/tools/lint.py
-lib/galaxy/tools/lint_util.py
-lib/galaxy/tools/linters/
-lib/galaxy/tools/loader.py
-lib/galaxy/tools/loader_directory.py
-lib/galaxy/tools/parameters/dataset_matcher.py
-lib/galaxy/tools/parameters/__init__.py
-lib/galaxy/tools/parameters/input_translation.py
-lib/galaxy/tools/parameters/sanitize.py
-lib/galaxy/tools/parameters/validation.py
-lib/galaxy/tools/parameters/wrapped_json.py
-lib/galaxy/tools/parameters/wrapped.py
-lib/galaxy/tools/parser/
-lib/galaxy/tools/test.py
-lib/galaxy/tools/toolbox/
+lib/galaxy/tools/
 lib/galaxy/tours/
 lib/galaxy/util/
 lib/galaxy/visualization/

README.rst

@@ -9,9 +9,13 @@ The latest information about Galaxy is available via `https://galaxyproject.org/
 
 .. image:: https://img.shields.io/badge/chat-irc.freenode.net%23galaxyproject-blue.svg
     :target: https://webchat.freenode.net/?channels=galaxyproject
-    :alt: Chat with us
+    :alt: Chat on irc
+    
+.. image:: https://img.shields.io/badge/chat-gitter-blue.svg
+    :target: https://gitter.im/galaxyproject/Lobby
+    :alt: Chat on gitter
 
-.. image:: https://img.shields.io/badge/docs-release-green.svg
+.. image:: https://img.shields.io/badge/release-documentation-blue.svg
     :target: https://docs.galaxyproject.org/en/master/
     :alt: Release Documentation
 

client/galaxy/scripts/mvc/form/form-view.js

@@ -125,7 +125,7 @@ function( Utils, Portlet, Ui, FormSection, FormData ) {
         _renderForm: function() {
             $( '.tooltip' ).remove();
             var options = this.model.attributes;
-            this.message = new Ui.Message();
+            this.message = new Ui.UnescapedMessage();
             this.section = new FormSection.View( this, { inputs: options.inputs } );
             this.portlet = new Portlet.View({
                 icon            : options.icon,
@@ -145,4 +145,4 @@ function( Utils, Portlet, Ui, FormSection, FormData ) {
             Galaxy.emit.debug( 'form-view::initialize()', 'Completed' );
         }
     });
-});
+});

client/galaxy/scripts/mvc/ui/ui-misc.js

@@ -62,7 +62,7 @@ define(['utils/utils',
                 this.$el.addClass( 'alert' ).addClass( 'alert-' + status );
             }
             if ( this.model.get( 'message' ) ) {
-                this.$el.html( this.model.get( 'message' ) );
+                this.$el.html( this.messageForDisplay() );
                 this.$el[ this.model.get( 'fade' ) ? 'fadeIn' : 'show' ]();
                 this.timeout && window.clearTimeout( this.timeout );
                 if ( !this.model.get( 'persistent' ) ) {
@@ -75,6 +75,15 @@ define(['utils/utils',
                 this.$el.fadeOut();
             }
             return this;
+        },
+        messageForDisplay: function() {
+            return _.escape( this.model.get( 'message' ) );
+        }
+    });
+
+    var UnescapedMessage = Message.extend({
+        messageForDisplay: function() {
+            return this.model.get( 'message' );
         }
     });
 
@@ -157,21 +166,22 @@ define(['utils/utils',
     });
 
     return {
-        Button      : Buttons.ButtonDefault,
-        ButtonIcon  : Buttons.ButtonIcon,
-        ButtonCheck : Buttons.ButtonCheck,
-        ButtonMenu  : Buttons.ButtonMenu,
-        ButtonLink  : Buttons.ButtonLink,
-        Input       : Input,
-        Label       : Label,
-        Message     : Message,
-        Modal       : Modal,
-        RadioButton : Options.RadioButton,
-        Checkbox    : Options.Checkbox,
-        Radio       : Options.Radio,
-        Select      : Select,
-        Hidden      : Hidden,
-        Slider      : Slider,
-        Drilldown   : Drilldown
+        Button           : Buttons.ButtonDefault,
+        ButtonIcon       : Buttons.ButtonIcon,
+        ButtonCheck      : Buttons.ButtonCheck,
+        ButtonMenu       : Buttons.ButtonMenu,
+        ButtonLink       : Buttons.ButtonLink,
+        Input            : Input,
+        Label            : Label,
+        Message          : Message,
+        UnescapedMessage : UnescapedMessage,
+        Modal            : Modal,
+        RadioButton      : Options.RadioButton,
+        Checkbox         : Options.Checkbox,
+        Radio            : Options.Radio,
+        Select           : Select,
+        Hidden           : Hidden,
+        Slider           : Slider,
+        Drilldown        : Drilldown
     }
-});
+});

client/galaxy/scripts/viz/trackster/tracks.js

@@ -1222,18 +1222,23 @@ extend( TracksterView.prototype, DrawableCollection.prototype, {
                     view.reference_track = ref_track;
                 }
                 view.chrom_data = result.chrom_info;
-                var chrom_options = '<option value="">Select Chrom/Contig</option>';
+                
+                view.chrom_select.html('');
+                view.chrom_select.append($('<option value="">Select Chrom/Contig</option>'));
+
                 for (var i = 0, len = view.chrom_data.length; i < len; i++) {
                     var chrom = view.chrom_data[i].chrom;
-                    chrom_options += '<option value="' + chrom + '">' + chrom + '</option>';
+                    var chrom_option  = $("<option>");
+                    chrom_option.text(chrom);
+                    chrom_option.val(chrom);
+                    view.chrom_select.append(chrom_option);
                 }
                 if (result.prev_chroms) {
-                    chrom_options += '<option value="previous">Previous ' + MAX_CHROMS_SELECTABLE + '</option>';
+                    view.chrom_select.append($('<option value="previous">Previous ' + MAX_CHROMS_SELECTABLE + '</option>'));
                 }
                 if (result.next_chroms) {
-                    chrom_options += '<option value="next">Next ' + MAX_CHROMS_SELECTABLE + '</option>';
+                    view.chrom_select.append($('<option value="next">Next ' + MAX_CHROMS_SELECTABLE + '</option>'));
                 }
-                view.chrom_select.html(chrom_options);
                 view.chrom_start_index = result.start_index;
 
                 chrom_data.resolve(result.chrom_info);

config/datatypes_conf.xml.sample

@@ -38,6 +38,7 @@
     <datatype extension="len" type="galaxy.datatypes.chrominfo:ChromInfo" display_in_upload="true">
       <converter file="len_to_linecount.xml" target_datatype="linecount" />
     </datatype>
+    <datatype extension="dmnd" type="galaxy.datatypes.binary:Binary:DMND" subclass="True" display_in_upload="false"/>
     <datatype extension="bigbed" type="galaxy.datatypes.binary:BigBed" mimetype="application/octet-stream" display_in_upload="true">
       <display file="ucsc/bigbed.xml" />
       <display file="igb/bb.xml" />
@@ -87,6 +88,36 @@
     <datatype extension="fastqillumina" type="galaxy.datatypes.sequence:FastqIllumina" display_in_upload="true">
         <converter file="fastq_to_fqtoc.xml" target_datatype="fqtoc"/>
     </datatype>
+    <datatype extension="fastq.gz" type="galaxy.datatypes.sequence:FastqGz" display_in_upload="true">
+        <converter file="fastqgz_to_fastq.xml" target_datatype="fastq"/>
+    </datatype>
+    <datatype extension="fastqsanger.gz" type="galaxy.datatypes.sequence:FastqSangerGz" display_in_upload="true">
+        <converter file="fastqsangergz_to_fastqsanger.xml" target_datatype="fastqsanger"/>
+    </datatype>
+    <datatype extension="fastqsolexa.gz" type="galaxy.datatypes.sequence:FastqSolexaGz" display_in_upload="true">
+        <converter file="fastqsolexagz_to_fastqsolexa.xml" target_datatype="fastqsolexa"/>
+    </datatype>
+    <datatype extension="fastqcssanger.gz" type="galaxy.datatypes.sequence:FastqCSSangerGz" display_in_upload="true">
+        <converter file="fastqcssangergz_to_fastqcssanger.xml" target_datatype="fastqcssanger"/>
+    </datatype>
+    <datatype extension="fastqillumina.gz" type="galaxy.datatypes.sequence:FastqIlluminaGz" display_in_upload="true">
+        <converter file="fastqilluminagz_to_fastqillumina.xml" target_datatype="fastqillumina"/>
+    </datatype>
+    <datatype extension="fastq.bz2" type="galaxy.datatypes.sequence:FastqBz2" display_in_upload="true">
+        <converter file="fastqbz2_to_fastq.xml" target_datatype="fastq"/>
+    </datatype>
+    <datatype extension="fastqsanger.bz2" type="galaxy.datatypes.sequence:FastqSangerBz2" display_in_upload="true">
+        <converter file="fastqsangerbz2_to_fastqsanger.xml" target_datatype="fastqsanger"/>
+    </datatype>
+    <datatype extension="fastqsolexa.bz2" type="galaxy.datatypes.sequence:FastqSolexaBz2" display_in_upload="true">
+        <converter file="fastqsolexabz2_to_fastqsolexa.xml" target_datatype="fastqsolexa"/>
+    </datatype>
+    <datatype extension="fastqcssanger.bz2" type="galaxy.datatypes.sequence:FastqCSSangerBz2" display_in_upload="true">
+        <converter file="fastqcssangerbz2_to_fastqcssanger.xml" target_datatype="fastqcssanger"/>
+    </datatype>
+    <datatype extension="fastqillumina.bz2" type="galaxy.datatypes.sequence:FastqIlluminaBz2" display_in_upload="true">
+        <converter file="fastqilluminabz2_to_fastqillumina.xml" target_datatype="fastqillumina"/>
+    </datatype>
     <datatype extension="fqtoc" type="galaxy.datatypes.sequence:SequenceSplitLocations" display_in_upload="true"/>
     <datatype extension="eland" type="galaxy.datatypes.tabular:Eland" display_in_upload="true"/>
     <datatype extension="elandmulti" type="galaxy.datatypes.tabular:ElandMulti" display_in_upload="true"/>
@@ -606,6 +637,7 @@
     <sniffer type="galaxy.datatypes.binary:Sff"/>
     <sniffer type="galaxy.datatypes.binary:Sra"/>
     <sniffer type="galaxy.datatypes.binary:NetCDF"/>
+    <sniffer type="galaxy.datatypes.binary:DMND" />
     <sniffer type="galaxy.datatypes.triples:Rdf"/>
     <sniffer type="galaxy.datatypes.blast:BlastXml"/>
     <sniffer type="galaxy.datatypes.xml:Phyloxml"/>
@@ -642,6 +674,8 @@
     <!-- TODO: see molecules.py <sniffer type="galaxy.datatypes.molecules:SMILES"/>-->
     <sniffer type="galaxy.datatypes.sequence:Fasta"/>
     <sniffer type="galaxy.datatypes.sequence:Fastq"/>
+    <sniffer type="galaxy.datatypes.sequence:FastqGz"/>
+    <sniffer type="galaxy.datatypes.sequence:FastqBz2"/>
     <sniffer type="galaxy.datatypes.interval:Wiggle"/>
     <sniffer type="galaxy.datatypes.text:Html"/>
     <sniffer type="galaxy.datatypes.images:Pdf"/>

config/galaxy.ini.sample

@@ -210,7 +210,7 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 #conda_auto_install = False
 # Set to True to instruct Galaxy to install Conda from the web automatically
 # if it cannot find a local copy and conda_exec is not configured.
-#conda_auto_init = False
+#conda_auto_init = True
 # You must set this to True if conda_prefix and job_working_directory are not on the same
 # volume, or some conda dependencies will fail to execute at job runtime.
 # Conda will copy packages content instead of creating hardlinks or symlinks.
@@ -909,6 +909,16 @@ use_interactive = True
 # results will appear.
 # tool_search_limit = 20
 
+# Enable/ disable Ngram-search for tools. It makes tool
+# search results tolerant for spelling mistakes in the query
+# by dividing the query into multiple ngrams and search for
+# each ngram
+#tool_enable_ngram_search = False
+
+# Set minimum and maximum sizes of ngrams
+#tool_ngram_minsize = 3
+#tool_ngram_maxsize = 4
+
 # -- Users and Security
 
 # Galaxy encodes various internal values when these values will be output in

config/local_conda_mapping.yml.sample

@@ -0,0 +1,2 @@
+# See $GALAXY_ROOT/lib/galaxy/tools/deps/default_conda_mapping.yml for example mapping - 
+# additional site-specific mappings can be added to config/conda_mapping.yml.

config/plugins/interactive_environments/rstudio/templates/rstudio.mako

@@ -33,7 +33,7 @@ ie_request.launch(
 # through proxy.
 notebook_pubkey_url = ie_request.url_template('${PROXY_URL}/rstudio/auth-public-key')
 notebook_access_url = ie_request.url_template('${PROXY_URL}/rstudio/')
-notebook_login_url =  ie_request.url_template('${PROXY_URL}/rstudio/auth-sign-in')
+notebook_login_url =  ie_request.url_template('${PROXY_URL}/rstudio/auth-do-sign-in')
 
 %>
 <html>

config/plugins/visualizations/charts/templates/charts.mako

@@ -8,7 +8,7 @@
 <html>
     <head>
         <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-        <title>${hda.name} | ${visualization_name}</title>
+        <title>${hda.name | h} | ${visualization_name}</title>
         ${h.js( 'libs/jquery/jquery',
                 'libs/jquery/jquery-ui',
                 'libs/jquery/select2',

config/plugins/visualizations/csg/templates/csg.mako

@@ -7,7 +7,7 @@
 <html>
     <head>
         <!-- CSG Viewer is a web application for 3D shape visualization. -->
-        <title>${hda.name} | ${visualization_name}</title>
+        <title>${hda.name | h} | ${visualization_name}</title>
         ${h.javascript_link( app_root + 'dat.gui.min.js' )}
         ${h.javascript_link( app_root + 'three.min.js' )}
         ${h.javascript_link( app_root + 'Detector.js' )}

config/plugins/visualizations/graphviz/templates/graphviz.mako

@@ -5,7 +5,7 @@
     <meta charset="utf-8">
     <meta name="viewport" content="width=device-width, user-scalable=no, minimum-scale=1.0, maximum-scale=1.0">
 
-    <title>${hda.name} | ${visualization_name}</title>
+    <title>${hda.name | h} | ${visualization_name}</title>
     <%
         root = h.url_for( '/' )
     %>

config/plugins/visualizations/scatterplot/templates/scatterplot.mako

@@ -16,7 +16,7 @@
 <html>
 <head>
 <meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
-<title>${title or default_title} | ${visualization_display_name}</title>
+<title>${title or default_title | h} | ${visualization_display_name}</title>
 
 ## ----------------------------------------------------------------------------
 ${h.css( 'base', 'jquery-ui/smoothness/jquery-ui')}

config/plugins/webhooks/demo/phdcomics/helper/__init__.py

@@ -8,7 +8,7 @@ log = logging.getLogger(__name__)
 
 def main(trans, webhook):
     error = ''
-    data = {}
+    comic_src = ''
 
     try:
         # Third-party dependencies
@@ -16,7 +16,7 @@ def main(trans, webhook):
             from bs4 import BeautifulSoup
         except ImportError as e:
             log.exception(e)
-            return {}
+            return {'success': False, 'error': str(e)}
 
         # Get latest id
         if 'latest_id' not in webhook.config.keys():
@@ -33,11 +33,16 @@ def main(trans, webhook):
         url = 'http://www.phdcomics.com/comics/archive.php?comicid=%d' % \
             random_id
         content = urllib.urlopen(url).read()
-        soap = BeautifulSoup(content, 'html.parser')
-        comics_src = soap.find_all('img', id='comic')[0].attrs.get('src')
-        data = {'src': comics_src}
+        soup = BeautifulSoup(content, 'html.parser')
+        comic_img = soup.find_all('img', id='comic2')
+
+        try:
+            comic_src = comic_img[0].attrs.get('src')
+        except IndexError:
+            pattern = '<img id=comic2 name=comic2 src=([\w:\/\.]+)'
+            comic_src = re.search(pattern, content).group(1)
 
     except Exception as e:
         error = str(e)
 
-    return {'success': not error, 'error': error, 'data': data}
+    return {'success': not error, 'error': error, 'src': comic_src}

config/plugins/webhooks/demo/phdcomics/static/script.js

@@ -16,7 +16,7 @@ $(document).ready(function() {
         imgTemplate: _.template('<img src="<%= src %>"">'),
 
         events: {
-            'click #phdcomics-random': 'getRandomPHDComics'
+            'click #phdcomics-random': 'getRandomComic'
         },
 
         initialize: function() {
@@ -25,31 +25,32 @@ $(document).ready(function() {
 
         render: function() {
             this.$el.html(this.appTemplate());
-            this.phdComicsImg = this.$('#phdcomics-img');
-            this.getRandomPHDComics();
+            this.$comicImg = this.$('#phdcomics-img');
+            this.getRandomComic();
             return this;
         },
 
-        getRandomPHDComics: function() {
+        getRandomComic: function() {
             var me = this,
                 url = galaxyRoot + 'api/webhooks/phdcomics/get_data';
 
-            this.phdComicsImg.html($('<div/>', {id: 'phdcomics-loader'}));
+            this.$comicImg.html($('<div/>', {
+                id: 'phdcomics-loader'
+            }));
+
             $.getJSON(url, function(data) {
                 if (data.success) {
-                    me.phdComics = {src: data.data.src};
-                    me.renderImg();
+                    me.renderImg(data.src);
                 } else {
-                    console.log('[ERROR] "' + url + '":\n' + data.error);
+                    console.error('[ERROR] "' + url + '":\n' + data.error);
                 }
             });
         },
 
-        renderImg: function() {
-            this.phdComicsImg.html(this.imgTemplate({src: this.phdComics.src}));
+        renderImg: function(src) {
+            this.$comicImg.html(this.imgTemplate({src: src}));
         }
     });
 
-    var PHDComicsApp = new PHDComicsAppView;
-
+    new PHDComicsAppView();
 });

doc/schema_template.md

@@ -25,6 +25,7 @@ $tag:tool|inputs|conditional|when://complexType[@name='ConditionalWhen']
 $tag:tool|inputs|param://complexType[@name='Param']
 $tag:tool|inputs|param|validator://complexType[@name='Validator']
 $tag:tool|inputs|param|option://complexType[@name='ParamOption']
+$tag:tool|inputs|param|conversion://complexType[@name='ParamConversion']
 $tag:tool|inputs|param|options://complexType[@name='ParamOptions']
 $tag:tool|inputs|param|options|column://complexType[@name='Column']
 $tag:tool|inputs|param|options|filter://complexType[@name='Filter']

doc/source/admin/webhooks.rst

@@ -126,7 +126,7 @@ Plugin dependencies
 -------------------
 
 Some plugins might have additional dependencies that needs to be installed into the Galaxy environment.
-For example the PhD-Comic plugin requires the library beautifulsoup4. If thses dependencies are not present
+For example the PhD-Comic plugin requires the library beautifulsoup4. If these dependencies are not present
 plugins should deactivate themself and issue an error into the Galaxy log.
 
 To install these additional plugin do the following:

doc/source/api/guidelines.rst

@@ -1,7 +1,7 @@
 API Design Guidelines
 =====================
 
-The following section outlines guidelines related to extending and/or modifing
+The following section outlines guidelines related to extending and/or modifying
 the Galaxy API. The Galaxy API has grown in an ad-hoc fashion over time by
 many contributors and so clients SHOULD NOT expect the API will conform to
 these guidelines - but developers contributing to the Galaxy API SHOULD follow
@@ -37,7 +37,7 @@ these guidelines.
       Various error conditions (once a format has been chosen and framework to
       enforce it in place) should be spelled out in this document.
     - Backward compatibility is important and should be maintained when possible.
-      If changing behavior in a non-backward compatibile way please ensure one
+      If changing behavior in a non-backward compatible way please ensure one
       of the following holds - there is a strong reason to believe no consumers
       depend on a behavior, the behavior is effectively broken, or the API
       method being modified has not been part of a tagged dist release.

lib/galaxy/config.py

@@ -32,6 +32,46 @@ from .version import VERSION_MAJOR
 log = logging.getLogger( __name__ )
 
 
+PATH_DEFAULTS = dict(
+    auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'],
+    data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'],
+    datatypes_config_file=['config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample'],
+    external_service_type_config_file=['config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample'],
+    job_config_file=['config/job_conf.xml', 'job_conf.xml'],
+    tool_destinations_config_file=['config/tool_destinations.yml', 'config/tool_destinations.yml.sample'],
+    job_metrics_config_file=['config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample'],
+    dependency_resolvers_config_file=['config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml'],
+    job_resource_params_file=['config/job_resource_params_conf.xml', 'job_resource_params_conf.xml'],
+    migrated_tools_config=['migrated_tools_conf.xml', 'config/migrated_tools_conf.xml'],
+    object_store_config_file=['config/object_store_conf.xml', 'object_store_conf.xml'],
+    openid_config_file=['config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample'],
+    shed_data_manager_config_file=['shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml'],
+    shed_tool_data_table_config=['shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml'],
+    tool_sheds_config_file=['config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample'],
+    workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
+    modules_mapping_files=['config/environment_modules_mapping.yml', 'config/environment_modules_mapping.yml.sample'],
+    local_conda_mapping_file=['config/local_conda_mapping.yml', 'config/local_conda_mapping.yml.sample'],
+)
+
+PATH_LIST_DEFAULTS = dict(
+    tool_data_table_config_path=['config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample'],
+    # rationale:
+    # [0]: user has explicitly created config/tool_conf.xml but did not
+    #      move their existing shed_tool_conf.xml, don't use
+    #      config/shed_tool_conf.xml, which is probably the empty
+    #      version copied from the sample, or else their shed tools
+    #      will disappear
+    # [1]: user has created config/tool_conf.xml and, having passed
+    #      [0], probably moved their shed_tool_conf.xml as well
+    # [2]: user has done nothing, use the old files
+    # [3]: fresh install
+    tool_config_file=['config/tool_conf.xml,shed_tool_conf.xml',
+                      'config/tool_conf.xml,config/shed_tool_conf.xml',
+                      'tool_conf.xml,shed_tool_conf.xml',
+                      'config/tool_conf.xml.sample,config/shed_tool_conf.xml']
+)
+
+
 def resolve_path( path, root ):
     """If 'path' is relative make absolute by prepending 'root'"""
     if not os.path.isabs( path ):
@@ -39,12 +79,34 @@ def resolve_path( path, root ):
     return path
 
 
+def find_path(kwargs, var, root):
+    """Find a configuration path that may exist at different defaults."""
+    defaults = PATH_DEFAULTS[var]
+
+    if kwargs.get(var, None) is not None:
+        path = kwargs.get(var)
+    else:
+        for default in defaults:
+            if os.path.exists(resolve_path(default, root)):
+                path = default
+                break
+        else:
+            path = defaults[-1]
+
+    return resolve_path(path, root)
+
+
+def find_root(kwargs):
+    root = kwargs.get('root_dir', '.')
+    return root
+
+
 class Configuration( object ):
     deprecated_options = ( 'database_file', )
 
     def __init__( self, **kwargs ):
         self.config_dict = kwargs
-        self.root = kwargs.get( 'root_dir', '.' )
+        self.root = find_root(kwargs)
 
         # Resolve paths of other config files
         self.__parse_config_file_options( kwargs )
@@ -325,24 +387,26 @@ class Configuration( object ):
         self.tool_stub_boost = kwargs.get( "tool_stub_boost", 5 )
         self.tool_help_boost = kwargs.get( "tool_help_boost", 0.5 )
         self.tool_search_limit = kwargs.get( "tool_search_limit", 20 )
+        self.tool_enable_ngram_search = kwargs.get( "tool_enable_ngram_search", False )
+        self.tool_ngram_minsize = kwargs.get( "tool_ngram_minsize", 3 )
+        self.tool_ngram_maxsize = kwargs.get( "tool_ngram_maxsize", 4 )
         # Location for tool dependencies.
-        # Location for tool dependencies.
-        tool_dependency_dir = kwargs.get( "tool_dependency_dir", "database/dependencies" )
-        if tool_dependency_dir.lower() == "none":
-            tool_dependency_dir = None
-
-        if tool_dependency_dir is not None:
-            self.tool_dependency_dir = resolve_path( tool_dependency_dir, self.root )
-            # Setting the following flag to true will ultimately cause tool dependencies
-            # to be located in the shell environment and used by the job that is executing
-            # the tool.
-            self.use_tool_dependencies = True
+        use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
+            parse_dependency_options(kwargs, self.root, self.dependency_resolvers_config_file)
+        self.use_tool_dependencies = use_tool_dependencies
+        self.tool_dependency_dir = tool_dependency_dir
+        self.use_cached_dependency_manager = use_cached_dependency_manager
+        self.tool_dependency_cache_dir = tool_dependency_cache_dir
+        self.precache_dependencies = precache_dependencies
+        # Deployers may either specify a complete list of mapping files or get the default for free and just
+        # specify a local mapping file to adapt and extend the default one.
+        if "conda_mapping_files" in kwargs:
+            self.conda_mapping_files = kwargs["conda_mapping_files"]
         else:
-            self.tool_dependency_dir = None
-            self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file)
-        self.use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False'))
-        self.tool_dependency_cache_dir = kwargs.get( 'tool_dependency_cache_dir', os.path.join(self.tool_dependency_dir, '_cache'))
-        self.precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True'))
+            self.conda_mapping_files = [
+                self.local_conda_mapping_file,
+                os.path.join(self.root, "lib", "galaxy", "tools", "deps", "resolvers", "default_conda_mapping.yml"),
+            ]
 
         self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) )
         containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None )
@@ -555,56 +619,11 @@ class Configuration( object ):
         """
         Backwards compatibility for config files moved to the config/ dir.
         """
-        defaults = dict(
-            auth_config_file=[ 'config/auth_conf.xml', 'config/auth_conf.xml.sample' ],
-            data_manager_config_file=[ 'config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample' ],
-            datatypes_config_file=[ 'config/datatypes_conf.xml', 'datatypes_conf.xml', 'config/datatypes_conf.xml.sample' ],
-            external_service_type_config_file=[ 'config/external_service_types_conf.xml', 'external_service_types_conf.xml', 'config/external_service_types_conf.xml.sample' ],
-            job_config_file=[ 'config/job_conf.xml', 'job_conf.xml' ],
-            tool_destinations_config_file=[ 'config/tool_destinations.yml', 'config/tool_destinations.yml.sample' ],
-            job_metrics_config_file=[ 'config/job_metrics_conf.xml', 'job_metrics_conf.xml', 'config/job_metrics_conf.xml.sample' ],
-            dependency_resolvers_config_file=[ 'config/dependency_resolvers_conf.xml', 'dependency_resolvers_conf.xml' ],
-            job_resource_params_file=[ 'config/job_resource_params_conf.xml', 'job_resource_params_conf.xml' ],
-            migrated_tools_config=[ 'migrated_tools_conf.xml', 'config/migrated_tools_conf.xml' ],
-            object_store_config_file=[ 'config/object_store_conf.xml', 'object_store_conf.xml' ],
-            openid_config_file=[ 'config/openid_conf.xml', 'openid_conf.xml', 'config/openid_conf.xml.sample' ],
-            shed_data_manager_config_file=[ 'shed_data_manager_conf.xml', 'config/shed_data_manager_conf.xml' ],
-            shed_tool_data_table_config=[ 'shed_tool_data_table_conf.xml', 'config/shed_tool_data_table_conf.xml' ],
-            tool_sheds_config_file=[ 'config/tool_sheds_conf.xml', 'tool_sheds_conf.xml', 'config/tool_sheds_conf.xml.sample' ],
-            workflow_schedulers_config_file=['config/workflow_schedulers_conf.xml', 'config/workflow_schedulers_conf.xml.sample'],
-        )
 
-        listify_defaults = dict(
-            tool_data_table_config_path=[ 'config/tool_data_table_conf.xml', 'tool_data_table_conf.xml', 'config/tool_data_table_conf.xml.sample' ],
-            # rationale:
-            # [0]: user has explicitly created config/tool_conf.xml but did not
-            #      move their existing shed_tool_conf.xml, don't use
-            #      config/shed_tool_conf.xml, which is probably the empty
-            #      version copied from the sample, or else their shed tools
-            #      will disappear
-            # [1]: user has created config/tool_conf.xml and, having passed
-            #      [0], probably moved their shed_tool_conf.xml as well
-            # [2]: user has done nothing, use the old files
-            # [3]: fresh install
-            tool_config_file=[ 'config/tool_conf.xml,shed_tool_conf.xml',
-                               'config/tool_conf.xml,config/shed_tool_conf.xml',
-                               'tool_conf.xml,shed_tool_conf.xml',
-                               'config/tool_conf.xml.sample,config/shed_tool_conf.xml' ]
-        )
+        for var in PATH_DEFAULTS:
+            setattr( self, var, find_path( kwargs, var, self.root ) )
 
-        for var, defaults in defaults.items():
-            if kwargs.get( var, None ) is not None:
-                path = kwargs.get( var )
-            else:
-                for default in defaults:
-                    if os.path.exists( resolve_path( default, self.root ) ):
-                        path = default
-                        break
-                else:
-                    path = defaults[-1]
-            setattr( self, var, resolve_path( path, self.root ) )
-
-        for var, defaults in listify_defaults.items():
+        for var, defaults in PATH_LIST_DEFAULTS.items():
             paths = []
             if kwargs.get( var, None ) is not None:
                 paths = listify( kwargs.get( var ) )
@@ -755,6 +774,31 @@ class Configuration( object ):
         return [ parse( v ) for v in allowed_origin_hostnames if v ]
 
 
+def parse_dependency_options(kwargs, root, dependency_resolvers_config_file):
+    # Location for tool dependencies.
+    tool_dependency_dir = kwargs.get("tool_dependency_dir", "database/dependencies")
+    if tool_dependency_dir.lower() == "none":
+        tool_dependency_dir = None
+
+    if tool_dependency_dir is not None:
+        tool_dependency_dir = resolve_path(tool_dependency_dir, root)
+        # Setting the following flag to true will ultimately cause tool dependencies
+        # to be located in the shell environment and used by the job that is executing
+        # the tool.
+        use_tool_dependencies = True
+        tool_dependency_cache_dir = kwargs.get('tool_dependency_cache_dir', os.path.join(tool_dependency_dir, '_cache'))
+        use_cached_dependency_manager = string_as_bool(kwargs.get("use_cached_dependency_manager", 'False'))
+        precache_dependencies = string_as_bool(kwargs.get("precache_dependencies", 'True'))
+    else:
+        tool_dependency_dir = None
+        use_tool_dependencies = os.path.exists(dependency_resolvers_config_file)
+        tool_dependency_cache_dir = None
+        precache_dependencies = False
+        use_cached_dependency_manager = False
+
+    return use_tool_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies
+
+
 def get_database_engine_options( kwargs, model_prefix='' ):
     """
     Allow options for the SQLAlchemy database engine to be passed by using
@@ -784,15 +828,21 @@ def get_database_engine_options( kwargs, model_prefix='' ):
 
 
 def configure_logging( config ):
-    """
-    Allow some basic logging configuration to be read from ini file.
+    """Allow some basic logging configuration to be read from ini file.
+
+    This should be able to consume either a galaxy.config.Configuration object
+    or a simple dictionary of configuration variables.
     """
     # Get root logger
     root = logging.getLogger()
     # PasteScript will have already configured the logger if the
     # 'loggers' section was found in the config file, otherwise we do
     # some simple setup using the 'log_*' values from the config.
-    paste_configures_logging = config.global_conf_parser.has_section( "loggers" )
+    parser = getattr(config, "global_conf_parser", None)
+    if parser:
+        paste_configures_logging = config.global_conf_parser.has_section( "loggers" )
+    else:
+        paste_configures_logging = False
     auto_configure_logging = not paste_configures_logging and string_as_bool( config.get( "auto_configure_logging", "True" ) )
     if auto_configure_logging:
         format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" )
@@ -823,7 +873,7 @@ def configure_logging( config ):
         handler.setFormatter( formatter )
         root.addHandler( handler )
     # If sentry is configured, also log to it
-    if config.sentry_dsn:
+    if getattr(config, "sentry_dsn", None):
         from raven.handlers.logging import SentryHandler
         sentry_handler = SentryHandler( config.sentry_dsn )
         sentry_handler.setLevel( logging.WARN )

lib/galaxy/datatypes/binary.py

@@ -1436,3 +1436,36 @@ class NetCDF( Binary ):
 
 
 Binary.register_sniffable_binary_format("netcdf", "netcdf", NetCDF)
+
+
+class DMND( Binary ):
+    """
+    Class describing an DMND file
+    >>> from galaxy.datatypes.sniff import get_test_fname
+    >>> fname = get_test_fname( 'diamond_db.dmnd' )
+    >>> DMND().sniff( fname )
+    True
+    >>> fname = get_test_fname( 'interval.interval' )
+    >>> DMND().sniff( fname )
+    False
+    """
+    file_ext = "dmnd"
+    edam_format = ""
+
+    def __init__( self, **kwd ):
+        Binary.__init__( self, **kwd )
+        self._magic = binascii.unhexlify("6d18ee15a4f84a02")
+
+    def sniff( self, filename ):
+        # The first 8 bytes of any dmnd file are 0x24af8a415ee186d
+
+        try:
+            header = open( filename, 'rb' ).read(8)
+            if header == self._magic:
+                return True
+            return False
+        except:
+            return False
+
+
+Binary.register_sniffable_binary_format("dmnd", "dmnd", DMND)

lib/galaxy/datatypes/converters/fastqbz2_to_fastq.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqbz2_to_fastq" name="Convert fastq.bz2 files to fastq" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastq.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastq" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqcssangerbz2_to_fastqcssanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqcssangerbz2_to_fastqcssanger" name="Convert fastqcssanger.bz2 files to fastqcssanger" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqcssanger.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqcssanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqcssangergz_to_fastqcssanger" name="Convert fastqcssanger.gz files to fastqcssanger" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqcssanger.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqcssanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqgz_to_fastq" name="Convert fastq.gz files to fastq" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastq.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastq" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqilluminabz2_to_fastqillumina.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqilluminabz2_to_fastqillumina" name="Convert fastqillumina.bz2 files to fastqillumina" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqillumina.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqillumina" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqilluminagz_to_fastqillumina" name="Convert fastqillumina.gz files to fastqillumina" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqillumina.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqillumina" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsangerbz2_to_fastqsanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsangerbz2_to_fastqsanger" name="Convert fastqsanger.bz2 files to fastqsanger" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsanger.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsangergz_to_fastqsanger" name="Convert fastqsanger.gz files to fastqsanger" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsanger.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsolexabz2_to_fastqsolexa.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsolexabz2_to_fastqsolexa" name="Convert fastqsolexa.bz2 files to fastqsolexa" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsolexa.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsolexa" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsolexagz_to_fastqsolexa" name="Convert fastqsolexa.gz files to fastqsolexa" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsolexa.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsolexa" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/data.py

@@ -502,7 +502,7 @@ class Data( object ):
         """Returns ( target_ext, existing converted dataset )"""
         return datatypes_registry.find_conversion_destination_for_dataset_by_extensions( dataset, accepted_formats, **kwd )
 
-    def convert_dataset(self, trans, original_dataset, target_type, return_output=False, visible=True, deps=None, target_context=None):
+    def convert_dataset(self, trans, original_dataset, target_type, return_output=False, visible=True, deps=None, target_context=None, history=None):
         """This function adds a job to the queue to convert a dataset to another type. Returns a message about success/failure."""
         converter = trans.app.datatypes_registry.get_converter_by_target_type( original_dataset.ext, target_type )
 
@@ -525,7 +525,7 @@ class Data( object ):
         params[input_name] = original_dataset
 
         # Run converter, job is dispatched through Queue
-        converted_dataset = converter.execute( trans, incoming=params, set_output_hid=visible )[1]
+        converted_dataset = converter.execute( trans, incoming=params, set_output_hid=visible, history=history )[1]
         if len(params) > 0:
             trans.log_event( "Converter params: %s" % (str(params)), tool_id=converter.id )
         if not visible:

lib/galaxy/datatypes/sequence.py

@@ -2,6 +2,7 @@
 Sequence classes
 """
 
+import bz2
 import gzip
 import json
 import logging
@@ -15,10 +16,14 @@ import bx.align.maf
 
 from galaxy import util
 from galaxy.datatypes import metadata
+from galaxy.datatypes.binary import Binary
 from galaxy.datatypes.metadata import MetadataElement
 from galaxy.datatypes.sniff import get_headers
 from galaxy.util import nice_size
-from galaxy.util.checkers import is_gzip
+from galaxy.util.checkers import (
+    is_bz2,
+    is_gzip
+)
 from galaxy.util.image_util import check_image_type
 
 from . import data
@@ -553,8 +558,8 @@ class csFasta( Sequence ):
         return Sequence.set_meta( self, dataset, **kwd )
 
 
-class Fastq ( Sequence ):
-    """Class representing a generic FASTQ sequence"""
+class BaseFastq ( Sequence ):
+    """Base class for FastQ sequences"""
     edam_format = "format_1930"
     file_ext = "fastq"
 
@@ -571,24 +576,35 @@ class Fastq ( Sequence ):
         data_lines = 0
         sequences = 0
         seq_counter = 0     # blocks should be 4 lines long
-        for line in open( dataset.file_name ):
-            line = line.strip()
-            if line and line.startswith( '#' ) and not data_lines:
-                # We don't count comment lines for sequence data types
-                continue
-            seq_counter += 1
-            data_lines += 1
-            if line and line.startswith( '@' ):
-                if seq_counter >= 4:
-                    # count previous block
-                    # blocks should be 4 lines long
-                    sequences += 1
-                    seq_counter = 1
-        if seq_counter >= 4:
-            # count final block
-            sequences += 1
-        dataset.metadata.data_lines = data_lines
-        dataset.metadata.sequences = sequences
+        compressed_gzip = is_gzip(dataset.file_name)
+        compressed_bzip2 = is_bz2(dataset.file_name)
+        try:
+            if compressed_gzip:
+                in_file = gzip.GzipFile(dataset.file_name)
+            elif compressed_bzip2:
+                in_file = bz2.BZ2File(dataset.file_name)
+            else:
+                in_file = open(dataset.file_name)
+            for line in in_file:
+                line = line.strip()
+                if line and line.startswith( '#' ) and not data_lines:
+                    # We don't count comment lines for sequence data types
+                    continue
+                seq_counter += 1
+                data_lines += 1
+                if line and line.startswith( '@' ):
+                    if seq_counter >= 4:
+                        # count previous block
+                        # blocks should be 4 lines long
+                        sequences += 1
+                        seq_counter = 1
+            if seq_counter >= 4:
+                # count final block
+                sequences += 1
+            dataset.metadata.data_lines = data_lines
+            dataset.metadata.sequences = sequences
+        finally:
+            in_file.close()
 
     def sniff( self, filename ):
         """
@@ -606,6 +622,9 @@ class Fastq ( Sequence ):
         >>> Fastq().sniff( fname )
         True
         """
+        compressed = is_gzip(filename) or is_bz2(filename)
+        if compressed and not isinstance(self, Binary):
+            return False
         headers = get_headers( filename, None )
         bases_regexp = re.compile( "^[NGTAC]*" )
         # check that first block looks like a fastq block
@@ -668,6 +687,12 @@ class Fastq ( Sequence ):
     process_split_file = staticmethod(process_split_file)
 
 
+class Fastq( BaseFastq ):
+    """Class representing a generic FASTQ sequence"""
+    edam_format = "format_1930"
+    file_ext = "fastq"
+
+
 class FastqSanger( Fastq ):
     """Class representing a FASTQ sequence ( the Sanger variant )"""
     edam_format = "format_1932"
@@ -691,6 +716,94 @@ class FastqCSSanger( Fastq ):
     file_ext = "fastqcssanger"
 
 
+class FastqGz ( BaseFastq, Binary ):
+    """Class representing a generic compressed FASTQ sequence"""
+    edam_format = "format_1930"
+    file_ext = "fastq.gz"
+
+
+Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
+
+
+class FastqSangerGz( FastqGz ):
+    """Class representing a compressed FASTQ sequence ( the Sanger variant )"""
+    edam_format = "format_1932"
+    file_ext = "fastqsanger.gz"
+
+
+Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz)
+
+
+class FastqSolexaGz( FastqGz ):
+    """Class representing a compressed FASTQ sequence ( the Solexa variant )"""
+    edam_format = "format_1933"
+    file_ext = "fastqsolexa.gz"
+
+
+Binary.register_sniffable_binary_format("fastqsolexa.gz", "fastqsolexa.gz", FastqSolexaGz)
+
+
+class FastqIlluminaGz( FastqGz ):
+    """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )"""
+    edam_format = "format_1931"
+    file_ext = "fastqillumina.gz"
+
+
+Binary.register_sniffable_binary_format("fastqillumina.gz", "fastqillumina.gz", FastqIlluminaGz)
+
+
+class FastqCSSangerGz( FastqGz ):
+    """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )"""
+    file_ext = "fastqcssanger.gz"
+
+
+Binary.register_sniffable_binary_format("fastqcssanger.gz", "fastqcssanger.gz", FastqCSSangerGz)
+
+
+class FastqBz2 ( BaseFastq, Binary ):
+    """Class representing a generic compressed FASTQ sequence"""
+    edam_format = "format_1930"
+    file_ext = "fastq.gz"
+
+
+Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
+
+
+class FastqSangerBz2( FastqBz2 ):
+    """Class representing a compressed FASTQ sequence ( the Sanger variant )"""
+    edam_format = "format_1932"
+    file_ext = "fastqsanger.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqsanger.bz2", "fastqsanger.bz2", FastqSangerBz2)
+
+
+class FastqSolexaBz2( FastqBz2 ):
+    """Class representing a compressed FASTQ sequence ( the Solexa variant )"""
+    edam_format = "format_1933"
+    file_ext = "fastqsolexa.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqsolexa.bz2", "fastqsolexa.bz2", FastqSolexaBz2)
+
+
+class FastqIlluminaBz2( FastqBz2 ):
+    """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )"""
+    edam_format = "format_1931"
+    file_ext = "fastqillumina.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqillumina.bz2", "fastqillumina.bz2", FastqIlluminaBz2)
+
+
+class FastqCSSangerBz2( FastqBz2 ):
+    """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )"""
+    file_ext = "fastqcssanger.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqcssanger.bz2", "fastqcssanger.bz2", FastqCSSangerBz2)
+
+
 class Maf( Alignment ):
     """Class describing a Maf alignment"""
     edam_format = "format_3008"

lib/galaxy/datatypes/sniff.py

@@ -4,6 +4,7 @@ File format detector
 from __future__ import absolute_import
 
 import gzip
+import bz2
 import logging
 import os
 import re
@@ -18,7 +19,12 @@ from six import text_type
 from galaxy import util
 from galaxy.util import multi_byte
 from galaxy.util import unicodify
-from galaxy.util.checkers import check_binary, check_html, is_gzip
+from galaxy.util.checkers import (
+    check_binary,
+    check_html,
+    is_bz2,
+    is_gzip
+)
 from galaxy.datatypes.binary import Binary
 
 log = logging.getLogger(__name__)
@@ -198,15 +204,26 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ):
     [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
     """
     headers = []
-    for idx, line in enumerate(open(fname)):
-        line = line.rstrip('\n\r')
-        if is_multi_byte:
-            # TODO: fix this - sep is never found in line
-            line = unicodify( line, 'utf-8' )
-            sep = sep.encode( 'utf-8' )
-        headers.append( line.split(sep) )
-        if idx == count:
-            break
+    compressed_gzip = is_gzip(fname)
+    compressed_bzip2 = is_bz2(fname)
+    try:
+        if compressed_gzip:
+            in_file = gzip.GzipFile(fname, 'r')
+        elif compressed_bzip2:
+            in_file = bz2.BZ2File(fname, 'r')
+        else:
+            in_file = open(fname, 'rt')
+        for idx, line in enumerate(in_file):
+            line = line.rstrip('\n\r')
+            if is_multi_byte:
+                # TODO: fix this - sep is never found in line
+                line = unicodify( line, 'utf-8' )
+                sep = sep.encode( 'utf-8' )
+            headers.append( line.split(sep) )
+            if idx == count:
+                break
+    finally:
+        in_file.close()
     return headers
 
 
@@ -356,6 +373,9 @@ def guess_ext( fname, sniff_order, is_multi_byte=False ):
     >>> fname = get_test_fname('1.gg')
     >>> guess_ext(fname, sniff_order)
     'gg'
+    >>> fname = get_test_fname('diamond_db.dmnd')
+    >>> guess_ext(fname, sniff_order)
+    'dmnd'
     """
     file_ext = None
     for datatype in sniff_order:
@@ -472,7 +492,7 @@ def handle_uploaded_dataset_file( filename, datatypes_registry, ext='auto', is_m
 AUTO_DETECT_EXTENSIONS = [ 'auto' ]  # should 'data' also cause auto detect?
 DECOMPRESSION_FUNCTIONS = dict( gzip=gzip.GzipFile )
 COMPRESSION_CHECK_FUNCTIONS = [ ( 'gzip', is_gzip ) ]
-COMPRESSION_DATATYPES = dict( gzip=[ 'bam' ] )
+COMPRESSION_DATATYPES = dict( gzip=[ 'bam', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz', 'fastq.bz2', 'fastqsanger.bz2', 'fastqillumina.bz2', 'fastqsolexa.bz2', 'fastqcssanger.bz2' ] )
 COMPRESSED_EXTENSIONS = []
 for exts in COMPRESSION_DATATYPES.values():
     COMPRESSED_EXTENSIONS.extend( exts )

lib/galaxy/dependencies/pinned-requirements.txt

@@ -27,7 +27,7 @@ Beaker==1.7.0
 dictobj==0.3.1
 nose==1.3.7
 Parsley==1.3
-six==1.9.0
+six==1.10.0
 Whoosh==2.7.4
 testfixtures==4.10.0
 

lib/galaxy/jobs/__init__.py

@@ -243,7 +243,7 @@ class JobConfiguration( object ):
             job_destination['env'] = self.__get_envs(destination)
             destination_resubmits = self.__get_resubmits(destination)
             if destination_resubmits:
-                resubmits = self.default_resubmits
+                resubmits = destination_resubmits
             else:
                 resubmits = self.default_resubmits
             job_destination["resubmit"] = resubmits

lib/galaxy/jobs/actions/post.py

@@ -110,11 +110,12 @@ class RenameDatasetAction(DefaultJobAction):
             #      "replace" option so you can replace a portion of the name,
             #      support multiple #{name} in one rename action...
 
-            while new_name.find("#{") > -1:
+            start_pos = 0
+            while new_name.find("#{", start_pos) > -1:
                 to_be_replaced = ""
                 #  This assumes a single instance of #{variable} will exist
-                start_pos = new_name.find("#{") + 2
-                end_pos = new_name.find("}")
+                start_pos = new_name.find("#{", start_pos) + 2
+                end_pos = new_name.find("}", start_pos)
                 to_be_replaced = new_name[start_pos:end_pos]
                 input_file_var = to_be_replaced
                 #  Pull out the piped controls and store them for later

lib/galaxy/jobs/runners/condor.py

@@ -66,7 +66,7 @@ class CondorJobRunner( AsynchronousJobRunner ):
         container = None
         universe = query_params.get('universe', None)
         if universe and universe.strip().lower() == 'docker':
-            container = self.find_container( job_wrapper )
+            container = self._find_container( job_wrapper )
             if container:
                 # HTCondor needs the image as 'docker_image'
                 query_params.update({'docker_image': container})

lib/galaxy/model/__init__.py

@@ -2046,7 +2046,7 @@ class DatasetInstance( object ):
             depends_list = []
         return dict([ (dep, self.get_converted_dataset(trans, dep)) for dep in depends_list ])
 
-    def get_converted_dataset(self, trans, target_ext, target_context=None):
+    def get_converted_dataset(self, trans, target_ext, target_context=None, history=None):
         """
         Return converted dataset(s) if they exist, along with a dict of dependencies.
         If not converted yet, do so and return None (the first time). If unconvertible, raise exception.
@@ -2085,7 +2085,7 @@ class DatasetInstance( object ):
             raise NoConverterException("A dependency (%s) is missing a converter." % dependency)
         except KeyError:
             pass  # No deps
-        new_dataset = next(iter(self.datatype.convert_dataset( trans, self, target_ext, return_output=True, visible=False, deps=deps, target_context=target_context ).values()))
+        new_dataset = next(iter(self.datatype.convert_dataset( trans, self, target_ext, return_output=True, visible=False, deps=deps, target_context=target_context, history=history ).values()))
         new_dataset.name = self.name
         self.copy_attributes( new_dataset )
         assoc = ImplicitlyConvertedDatasetAssociation( parent=self, file_type=target_ext, dataset=new_dataset, metadata_safe=False )

lib/galaxy/model/orm/scripts.py

@@ -2,9 +2,8 @@
 Code to support database helper scripts (create_db.py, manage_db.py, etc...).
 """
 import logging
-import os.path
 
-from galaxy.util.properties import load_app_properties
+from galaxy.util.properties import find_config_file, load_app_properties
 
 
 log = logging.getLogger( __name__ )
@@ -40,25 +39,21 @@ DATABASE = {
 }
 
 
-def read_config_file_arg( argv, default, old_default ):
+def read_config_file_arg( argv, default, old_default, cwd=None ):
+    config_file = None
     if '-c' in argv:
         pos = argv.index( '-c' )
         argv.pop(pos)
         config_file = argv.pop( pos )
-    else:
-        if not os.path.exists( default ) and os.path.exists( old_default ):
-            config_file = old_default
-        elif os.path.exists( default ):
-            config_file = default
-        else:
-            config_file = default + ".sample"
-    return config_file
+
+    return find_config_file( default, old_default, config_file, cwd=cwd )
 
 
 def get_config( argv, cwd=None ):
     """
     Read sys.argv and parse out repository of migrations and database url.
 
+    >>> import os
     >>> from ConfigParser import SafeConfigParser
     >>> from tempfile import mkdtemp
     >>> config_dir = mkdtemp()
@@ -89,16 +84,11 @@ def get_config( argv, cwd=None ):
 
     default = database_defaults.get( 'config_file', DEFAULT_CONFIG_FILE )
     old_default = database_defaults.get( 'old_config_file' )
-    if cwd is not None:
-        default = os.path.join( cwd, default )
-        old_default = os.path.join( cwd, old_default )
-    config_file = read_config_file_arg( argv, default, old_default )
+    config_file = read_config_file_arg( argv, default, old_default, cwd=cwd )
     repo = database_defaults[ 'repo' ]
     config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX )
     config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' )
     default_sqlite_file = database_defaults[ 'default_sqlite_file' ]
-    if cwd:
-        config_file = os.path.join( cwd, config_file )
 
     properties = load_app_properties( ini_file=config_file, config_prefix=config_override )
 

lib/galaxy/tools/__init__.py

@@ -1,7 +1,6 @@
 """
 Classes encapsulating galaxy tools and tool configuration.
 """
-
 import glob
 import json
 import logging
@@ -10,65 +9,90 @@ import re
 import tarfile
 import tempfile
 import threading
-import urllib
-from datetime import datetime
-
 from cgi import FieldStorage
+from datetime import datetime
 from xml.etree import ElementTree
+
 from mako.template import Template
 from paste import httpexceptions
 from six import string_types
+from six.moves.urllib.parse import unquote_plus
 
-from galaxy.version import VERSION_MAJOR
-from galaxy import model
-from galaxy.managers import histories
+import galaxy.jobs
+import tool_shed.util.repository_util as repository_util
+import tool_shed.util.shed_util_common
+
+from galaxy import (
+    exceptions,
+    model
+)
 from galaxy.datatypes.metadata import JobExternalOutputMetadataWrapper
-from galaxy import exceptions
+from galaxy.managers import histories
 from galaxy.queue_worker import (
     reload_toolbox,
     send_control_task
 )
 from galaxy.tools.actions import DefaultToolAction
-from galaxy.tools.actions.upload import UploadToolAction
-from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.data_manager import DataManagerToolAction
+from galaxy.tools.actions.data_source import DataSourceToolAction
 from galaxy.tools.actions.model_operations import ModelOperationToolAction
-from galaxy.tools.deps import views
-from galaxy.tools.deps import CachedDependencyManager
-from galaxy.tools.parameters import params_to_incoming, check_param, params_from_strings, params_to_strings, visit_input_values
+from galaxy.tools.actions.upload import UploadToolAction
+from galaxy.tools.deps import (
+    CachedDependencyManager,
+    views
+)
+from galaxy.tools.parameters import (
+    check_param,
+    params_from_strings,
+    params_to_incoming,
+    params_to_strings,
+    visit_input_values
+)
 from galaxy.tools.parameters import output_collect
-from galaxy.tools.parameters.basic import (BaseURLToolParameter,
-                                           DataToolParameter, DataCollectionToolParameter, HiddenToolParameter,
-                                           SelectToolParameter, ToolParameter)
+from galaxy.tools.parameters.basic import (
+    BaseURLToolParameter,
+    DataCollectionToolParameter,
+    DataToolParameter,
+    HiddenToolParameter,
+    SelectToolParameter,
+    ToolParameter
+)
 from galaxy.tools.parameters.grouping import Conditional, ConditionalWhen, Repeat, Section, UploadDataset
 from galaxy.tools.parameters.input_translation import ToolInputTranslator
-from galaxy.tools.test import parse_tests
-from galaxy.tools.parser import get_tool_source
-from galaxy.tools.parser.xml import XmlPageSource
-from galaxy.tools.parser import ToolOutputCollectionPart
-from galaxy.tools.toolbox import BaseGalaxyToolBox
-from galaxy.util import rst_to_html, string_as_bool
-from galaxy.util import ExecutionTimer
-from galaxy.util import listify
-from galaxy.util import unicodify
 from galaxy.tools.parameters.meta import expand_meta_parameters
+from galaxy.tools.parser import (
+    get_tool_source,
+    ToolOutputCollectionPart
+)
+from galaxy.tools.parser.xml import XmlPageSource
+from galaxy.tools.test import parse_tests
+from galaxy.tools.toolbox import BaseGalaxyToolBox
+from galaxy.util import (
+    ExecutionTimer,
+    listify,
+    rst_to_html,
+    string_as_bool,
+    unicodify
+)
 from galaxy.util.bunch import Bunch
+from galaxy.util.dictifiable import Dictifiable
 from galaxy.util.expressions import ExpressionContext
 from galaxy.util.json import json_fix
 from galaxy.util.json import safe_loads
 from galaxy.util.odict import odict
 from galaxy.util.template import fill_template
+from galaxy.version import VERSION_MAJOR
 from galaxy.web import url_for
 from galaxy.web.form_builder import SelectField
-from galaxy.util.dictifiable import Dictifiable
 from galaxy.work.context import WorkRequestContext
 from tool_shed.util import common_util
-import tool_shed.util.repository_util as repository_util
-from tool_shed.util import shed_util_common as suc
 
-from .loader import template_macro_params, raw_tool_xml_tree, imported_macro_paths
 from .execute import execute as execute_job
-import galaxy.jobs
+from .loader import (
+    imported_macro_paths,
+    raw_tool_xml_tree,
+    template_macro_params
+)
 
 log = logging.getLogger( __name__ )
 
@@ -174,8 +198,8 @@ class ToolBox( BaseGalaxyToolBox ):
 
     @property
     def all_requirements(self):
-        reqs = [json.dumps(req, sort_keys=True) for _, tool in self.tools() for req in tool.tool_requirements]
-        return [json.loads(req) for req in set(reqs)]
+        reqs = set([req for _, tool in self.tools() for req in tool.tool_requirements])
+        return [r.to_dict() for r in reqs]
 
     @property
     def tools_by_id( self ):
@@ -481,7 +505,7 @@ class Tool( object, Dictifiable ):
                 if job_tool_config.params:
                     # There are job params and this config has params defined
                     for param, value in job_params.items():
-                        if param not in job_tool_config.params or job_tool_config.params[param] != job_params[param]:
+                        if param not in job_tool_config.params or job_tool_config.params[param] != value:
                             break
                     else:
                         # All params match, use this config
@@ -689,12 +713,13 @@ class Tool( object, Dictifiable ):
                         self.hook_map[key] = value
                 file_name = code_elem.get("file")
                 code_path = os.path.join( self.tool_dir, file_name )
-                execfile( code_path, self.code_namespace )
+                with open(code_path) as f:
+                    exec(compile(f.read(), code_path, 'exec'), self.code_namespace)
 
         # User interface hints
         uihints_elem = root.find( "uihints" )
         if uihints_elem is not None:
-            for key, value in uihints_elem.attrib.iteritems():
+            for key, value in uihints_elem.attrib.items():
                 self.uihints[ key ] = value
 
     def __parse_tests(self, tool_source):
@@ -784,12 +809,12 @@ class Tool( object, Dictifiable ):
                 # nginx_upload_path. This logic is handled in the tool_form.mako
                 # template.
                 if self.nginx_upload and self.app.config.nginx_upload_path:
-                    if '?' in urllib.unquote_plus( self.action ):
+                    if '?' in unquote_plus( self.action ):
                         raise Exception( 'URL parameters in a non-default tool action can not be used '
                                          'in conjunction with nginx upload.  Please convert them to '
                                          'hidden POST parameters' )
                     self.action = (self.app.config.nginx_upload_path + '?nginx_redir=',
-                                   urllib.unquote_plus(self.action))
+                                   unquote_plus(self.action))
                 self.target = input_elem.get( "target", self.target )
                 self.method = input_elem.get( "method", self.method )
                 # Parse the actual parameters
@@ -915,7 +940,7 @@ class Tool( object, Dictifiable ):
                     group.test_param.refresh_on_change = True
                     for attr in value_from[1].split( '.' ):
                         group.value_from = getattr( group.value_from, attr )
-                    for case_value, case_inputs in group.value_from( context, group, self ).iteritems():
+                    for case_value, case_inputs in group.value_from( context, group, self ).items():
                         case = ConditionalWhen()
                         case.value = case_value
                         if case_inputs:
@@ -1053,7 +1078,7 @@ class Tool( object, Dictifiable ):
             if self.repository_id and help_text.find( '.. image:: ' ) >= 0:
                 # Handle tool help image display for tools that are contained in repositories in the tool shed or installed into Galaxy.
                 try:
-                    help_text = suc.set_image_paths( self.app, self.repository_id, help_text )
+                    help_text = tool_shed.util.shed_util_common.set_image_paths( self.app, self.repository_id, help_text )
                 except Exception as e:
                     log.exception( "Exception in parse_help, so images may not be properly displayed:\n%s" % str( e ) )
             try:
@@ -1222,8 +1247,8 @@ class Tool( object, Dictifiable ):
         log.debug( 'Validated and populated state for tool request %s' % validation_timer )
         # If there were errors, we stay on the same page and display them
         if any( all_errors ):
-            err_data = { key: value for d in all_errors for ( key, value ) in d.iteritems() }
-            raise exceptions.MessageException( ', '.join( [ msg for msg in err_data.itervalues() ] ), err_data=err_data )
+            err_data = { key: value for d in all_errors for ( key, value ) in d.items() }
+            raise exceptions.MessageException( ', '.join( msg for msg in err_data.values() ), err_data=err_data )
         else:
             execution_tracker = execute_job( trans, self, all_params, history=request_context.history, rerun_remap_job_id=rerun_remap_job_id, collection_info=collection_info )
             if execution_tracker.successful_jobs:
@@ -1253,7 +1278,7 @@ class Tool( object, Dictifiable ):
             message = 'Error executing tool: %s' % str(e)
             return False, message
         if isinstance( out_data, odict ):
-            return job, out_data.items()
+            return job, list(out_data.items())
         else:
             if isinstance( out_data, string_types ):
                 message = out_data
@@ -1304,7 +1329,7 @@ class Tool( object, Dictifiable ):
         does require input.
         """
         args = dict()
-        for key, param in self.inputs.iteritems():
+        for key, param in self.inputs.items():
             # BaseURLToolParameter is now a subclass of HiddenToolParameter, so
             # we must check if param is a BaseURLToolParameter first
             if isinstance( param, BaseURLToolParameter ):
@@ -1397,15 +1422,14 @@ class Tool( object, Dictifiable ):
         """
         Return all requiremens of type package
         """
-        reqs = [req.to_dict() for req in self.requirements if req.type == 'package']
-        return reqs
+        return self.requirements.packages
 
     @property
     def tool_requirements_status(self):
         """
         Return a list of dictionaries for all tool dependencies with their associated status
         """
-        return self._view.get_requirements_status(self.tool_requirements, self.installed_tool_dependencies)
+        return self._view.get_requirements_status({self.id: self.tool_requirements}, self.installed_tool_dependencies)
 
     def build_redirect_url_params( self, param_dict ):
         """
@@ -1758,7 +1782,7 @@ class Tool( object, Dictifiable ):
         # populates model from state
         def populate_model( inputs, state_inputs, group_inputs, other_values=None ):
             other_values = ExpressionContext( state_inputs, other_values )
-            for input_index, input in enumerate( inputs.itervalues() ):
+            for input_index, input in enumerate( inputs.values() ):
                 tool_dict = None
                 group_state = state_inputs.get( input.name, {} )
                 if input.type == 'repeat':
@@ -1854,7 +1878,7 @@ class Tool( object, Dictifiable ):
     # populates state from incoming parameters
     def populate_state( self, request_context, inputs, incoming, state, errors={}, prefix='', context=None ):
         context = ExpressionContext( state, context )
-        for input in inputs.itervalues():
+        for input in inputs.values():
             state[ input.name ] = input.get_initial_value( request_context, context )
             key = prefix + input.name
             group_state = state[ input.name ]
@@ -1864,7 +1888,7 @@ class Tool( object, Dictifiable ):
                 del group_state[:]
                 while True:
                     rep_prefix = '%s_%d' % ( key, rep_index )
-                    if not any( [ incoming_key.startswith( rep_prefix ) for incoming_key in incoming.keys() ] ) and rep_index >= input.min:
+                    if not any( incoming_key.startswith( rep_prefix ) for incoming_key in incoming.keys() ) and rep_index >= input.min:
                         break
                     if rep_index < input.max:
                         new_state = { '__index__' : rep_index }
@@ -1899,7 +1923,7 @@ class Tool( object, Dictifiable ):
                     del group_state[ -1 ]
                 while len( writable_files ) > len( group_state ):
                     new_state = { '__index__' : len( group_state ) }
-                    for upload_item in input.inputs.itervalues():
+                    for upload_item in input.inputs.values():
                         new_state[ upload_item.name ] = upload_item.get_initial_value( request_context, context )
                     group_state.append( new_state )
                 for i, rep_state in enumerate( group_state ):
@@ -2069,7 +2093,7 @@ class OutputParameterJSONTool( Tool ):
 
     def _prepare_json_param_dict( self, param_dict ):
         rval = {}
-        for key, value in param_dict.iteritems():
+        for key, value in param_dict.items():
             if isinstance( value, dict ):
                 rval[ key ] = self._prepare_json_param_dict( value )
             elif isinstance( value, list ):
@@ -2086,7 +2110,7 @@ class OutputParameterJSONTool( Tool ):
         json_params[ 'output_data' ] = []
         json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE )
         json_filename = None
-        for i, ( out_name, data ) in enumerate( out_data.iteritems() ):
+        for i, ( out_name, data ) in enumerate( out_data.items() ):
             # use wrapped dataset to access certain values
             wrapped_data = param_dict.get( out_name )
             # allow multiple files to be created
@@ -2138,7 +2162,7 @@ class DataSourceTool( OutputParameterJSONTool ):
         json_params[ 'output_data' ] = []
         json_params[ 'job_config' ] = dict( GALAXY_DATATYPES_CONF_FILE=param_dict.get( 'GALAXY_DATATYPES_CONF_FILE' ), GALAXY_ROOT_DIR=param_dict.get( 'GALAXY_ROOT_DIR' ), TOOL_PROVIDED_JOB_METADATA_FILE=galaxy.jobs.TOOL_PROVIDED_JOB_METADATA_FILE )
         json_filename = None
-        for i, ( out_name, data ) in enumerate( out_data.iteritems() ):
+        for i, ( out_name, data ) in enumerate( out_data.items() ):
             # use wrapped dataset to access certain values
             wrapped_data = param_dict.get( out_name )
             # allow multiple files to be created
@@ -2191,7 +2215,7 @@ class SetMetadataTool( Tool ):
     requires_setting_metadata = False
 
     def exec_after_process( self, app, inp_data, out_data, param_dict, job=None ):
-        for name, dataset in inp_data.iteritems():
+        for name, dataset in inp_data.items():
             external_metadata = JobExternalOutputMetadataWrapper( job )
             if external_metadata.external_metadata_set_successfully( dataset, app.model.context ):
                 dataset.metadata.from_JSON_dict( external_metadata.get_output_filenames_by_dataset( dataset, app.model.context ).filename_out )
@@ -2249,7 +2273,7 @@ class DataManagerTool( OutputParameterJSONTool ):
         if job and job.state == job.states.ERROR:
             return
         # Job state may now be 'running' instead of previous 'error', but datasets are still set to e.g. error
-        for dataset in out_data.itervalues():
+        for dataset in out_data.values():
             if dataset.state != dataset.states.OK:
                 return
         data_manager_id = job.data_manager_association.data_manager_id
@@ -2384,7 +2408,7 @@ class ZipCollectionTool( DatabaseOperationTool ):
         new_elements["reverse"] = reverse
 
         output_collections.create_collection(
-            self.outputs.values()[0], "output", elements=new_elements
+            next(iter(self.outputs.values())), "output", elements=new_elements
         )
 
 
@@ -2463,7 +2487,7 @@ class MergeCollectionTool( DatabaseOperationTool ):
             new_elements[key] = value.copy()
 
         output_collections.create_collection(
-            self.outputs.values()[0], "output", elements=new_elements
+            next(iter(self.outputs.values())), "output", elements=new_elements
         )
 
 
@@ -2498,7 +2522,7 @@ class FilterFailedDatasetsTool( DatabaseOperationTool ):
                 new_elements[element_identifier] = element.copy()
 
         output_collections.create_collection(
-            self.outputs.values()[0], "output", elements=new_elements
+            next(iter(self.outputs.values())), "output", elements=new_elements
         )
 
 
@@ -2522,7 +2546,7 @@ class FlattenTool( DatabaseOperationTool ):
 
         add_elements(hdca.collection)
         output_collections.create_collection(
-            self.outputs.values()[0], "output", elements=new_elements
+            next(iter(self.outputs.values())), "output", elements=new_elements
         )
 
 

lib/galaxy/tools/actions/__init__.py

@@ -41,7 +41,7 @@ class ToolAction( object ):
 class DefaultToolAction( object ):
     """Default tool action is to run an external command"""
 
-    def collect_input_datasets( self, tool, param_values, trans, current_user_roles=None ):
+    def _collect_input_datasets( self, tool, param_values, trans, history, current_user_roles=None ):
         """
         Collect any dataset inputs from incoming. Returns a mapping from
         parameter name to Dataset instance for each tool parameter that is
@@ -66,7 +66,7 @@ class DefaultToolAction( object ):
                         if converted_dataset:
                             data = converted_dataset
                         else:
-                            data = data.get_converted_dataset( trans, target_ext, target_context=parent )
+                            data = data.get_converted_dataset( trans, target_ext, target_context=parent, history=history )
 
                 if not trans.app.security_agent.can_access_dataset( current_user_roles, data.dataset ):
                     raise Exception( "User does not have permission to use a dataset (%s) provided for input." % data.id )
@@ -189,7 +189,7 @@ class DefaultToolAction( object ):
         # input datasets can process these normally.
         inp_dataset_collections = self.collect_input_dataset_collections( tool, incoming )
         # Collect any input datasets from the incoming parameters
-        inp_data = self.collect_input_datasets( tool, incoming, trans, current_user_roles=current_user_roles )
+        inp_data = self._collect_input_datasets( tool, incoming, trans, history=history, current_user_roles=current_user_roles )
 
         return history, inp_data, inp_dataset_collections
 

lib/galaxy/tools/data/__init__.py

@@ -6,21 +6,20 @@ users to configure data tables for a local Galaxy instance without needing
 to modify the tool configurations.
 """
 
+import hashlib
 import logging
 import os
 import os.path
 import re
 import string
-import hashlib
-
 from glob import glob
 from tempfile import NamedTemporaryFile
-from urllib2 import urlopen
+
+from six.moves.urllib.request import urlopen
 
 from galaxy import util
-from galaxy.util.odict import odict
-
 from galaxy.util.dictifiable import Dictifiable
+from galaxy.util.odict import odict
 
 log = logging.getLogger( __name__ )
 
@@ -158,7 +157,7 @@ class ToolDataTableManager( object ):
             for elem in out_elems:
                 out.write( util.xml_to_string( elem, pretty=True ) )
             out.write( '</tables>\n' )
-        os.chmod( full_path, 0644 )
+        os.chmod( full_path, 0o644 )
 
     def reload_tables( self, table_names=None ):
         """
@@ -166,7 +165,7 @@ class ToolDataTableManager( object ):
         """
         tables = self.get_tables()
         if not table_names:
-            table_names = tables.keys()
+            table_names = list(tables.keys())
         elif not isinstance( table_names, list ):
             table_names = [ table_names ]
         for table_name in table_names:
@@ -349,7 +348,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ):
                 self.filenames[ filename ] = dict( found=found, filename=filename, from_shed_config=from_shed_config, tool_data_path=tool_data_path,
                                                    config_element=config_element, tool_shed_repository=repo_info, errors=errors )
             else:
-                log.debug( "Filename '%s' already exists in filenames (%s), not adding", filename, self.filenames.keys() )
+                log.debug( "Filename '%s' already exists in filenames (%s), not adding", filename, list(self.filenames.keys()) )
             # Remove URL tmp file
             if tmp_file is not None:
                 tmp_file.close()
@@ -357,7 +356,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ):
     def merge_tool_data_table( self, other_table, allow_duplicates=True, persist=False, persist_on_error=False, entry_source=None, **kwd ):
         assert self.columns == other_table.columns, "Merging tabular data tables with non matching columns is not allowed: %s:%s != %s:%s" % ( self.name, self.columns, other_table.name, other_table.columns )
         # merge filename info
-        for filename, info in other_table.filenames.iteritems():
+        for filename, info in other_table.filenames.items():
             if filename not in self.filenames:
                 self.filenames[ filename ] = info
         # save info about table
@@ -473,7 +472,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ):
         rval = []
         for i in range( self.largest_index + 1 ):
             found_column = False
-            for name, index in self.columns.iteritems():
+            for name, index in self.columns.items():
                 if index == i:
                     if not found_column:
                         rval.append( name )
@@ -530,7 +529,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ):
         else:
             source_repo_info = None
         filename = default
-        for name, value in self.filenames.iteritems():
+        for name, value in self.filenames.items():
             repo_info = value.get( 'tool_shed_repository', None )
             if ( not source_repo_info and not repo_info ) or ( source_repo_info and repo_info and source_repo_info == repo_info ):
                 filename = name
@@ -637,7 +636,7 @@ class TabularToolDataTable( ToolDataTable, Dictifiable ):
                     replace = "_"
                 else:
                     replace = " "
-        return map( lambda x: x.replace( separator, replace ), fields )
+        return [x.replace( separator, replace ) for x in fields]
 
     def _deduplicate_data( self ):
         # Remove duplicate entries, without recreating self.data object

lib/galaxy/tools/data_manager/manager.py

@@ -1,20 +1,24 @@
 import errno
 import json
+import logging
 import os
+
 from six import string_types
 
 from galaxy import util
-from galaxy.util.odict import odict
-from galaxy.util.template import fill_template
+from galaxy.queue_worker import (
+    reload_data_managers,
+    send_control_task
+)
 from galaxy.tools.data import TabularToolDataTable
 from galaxy.tools.toolbox.watcher import get_tool_conf_watcher
-from tool_shed.util import common_util
-from tool_shed.util import repository_util
-from galaxy.queue_worker import reload_data_managers
-from galaxy.queue_worker import send_control_task
+from galaxy.util.odict import odict
+from galaxy.util.template import fill_template
+from tool_shed.util import (
+    common_util,
+    repository_util
+)
 
-# set up logger
-import logging
 log = logging.getLogger( __name__ )
 
 SUPPORTED_DATA_TABLE_TYPES = ( TabularToolDataTable )
@@ -112,7 +116,7 @@ class DataManagers( object ):
                 # determine if any data_tables are no longer tracked
                 for data_table_name in data_manager.data_tables.keys():
                     remove_data_table_tracking = True
-                    for other_data_manager in self.data_managers.itervalues():
+                    for other_data_manager in self.data_managers.values():
                         if data_table_name in other_data_manager.data_tables:
                             remove_data_table_tracking = False
                             break
@@ -279,21 +283,21 @@ class DataManager( object ):
         data_manager_dicts = {}
         data_manager_dict = {}
         # TODO: fix this merging below
-        for output_name, output_dataset in out_data.iteritems():
+        for output_name, output_dataset in out_data.items():
             try:
                 output_dict = json.loads( open( output_dataset.file_name ).read() )
             except Exception as e:
                 log.warning( 'Error reading DataManagerTool json for "%s": %s' % ( output_name, e ) )
                 continue
             data_manager_dicts[ output_name ] = output_dict
-            for key, value in output_dict.iteritems():
+            for key, value in output_dict.items():
                 if key not in data_manager_dict:
                     data_manager_dict[ key ] = {}
                 data_manager_dict[ key ].update( value )
             data_manager_dict.update( output_dict )
 
         data_tables_dict = data_manager_dict.get( 'data_tables', {} )
-        for data_table_name in self.data_tables.iterkeys():
+        for data_table_name in self.data_tables.keys():
             data_table_values = data_tables_dict.pop( data_table_name, None )
             if not data_table_values:
                 log.warning( 'No values for data table "%s" were returned by the data manager "%s".' % ( data_table_name, self.id ) )
@@ -307,7 +311,7 @@ class DataManager( object ):
                 continue  # next table name
             output_ref_values = {}
             if data_table_name in self.output_ref_by_data_table:
-                for data_table_column, output_ref in self.output_ref_by_data_table[ data_table_name ].iteritems():
+                for data_table_column, output_ref in self.output_ref_by_data_table[ data_table_name ].items():
                     output_ref_dataset = out_data.get( output_ref, None )
                     assert output_ref_dataset is not None, "Referenced output was not found."
                     output_ref_values[ data_table_column ] = output_ref_dataset
@@ -316,7 +320,7 @@ class DataManager( object ):
                 data_table_values = [ data_table_values ]
             for data_table_row in data_table_values:
                 data_table_value = dict( **data_table_row )  # keep original values here
-                for name, value in data_table_row.iteritems():  # FIXME: need to loop through here based upon order listed in data_manager config
+                for name, value in data_table_row.items():  # FIXME: need to loop through here based upon order listed in data_manager config
                     if name in output_ref_values:
                         self.process_move( data_table_name, name, output_ref_values[ name ].extra_files_path, **data_table_value )
                         data_table_value[ name ] = self.process_value_translation( data_table_name, name, **data_table_value )
@@ -332,13 +336,13 @@ class DataManager( object ):
             for ref_file in out_data.values():
                 util.move_merge( ref_file.extra_files_path, self.data_managers.app.config.galaxy_data_manager_data_path )
             path_column_names = [ 'path' ]
-            for data_table_name, data_table_values in data_tables_dict.iteritems():
+            for data_table_name, data_table_values in data_tables_dict.items():
                 data_table = self.data_managers.app.tool_data_tables.get( data_table_name, None )
                 if not isinstance( data_table_values, list ):
                     data_table_values = [ data_table_values ]
                 for data_table_row in data_table_values:
                     data_table_value = dict( **data_table_row )  # keep original values here
-                    for name, value in data_table_row.iteritems():
+                    for name, value in data_table_row.items():
                         if name in path_column_names:
                             data_table_value[ name ] = os.path.abspath( os.path.join( self.data_managers.app.config.galaxy_data_manager_data_path, value ) )
                     data_table.add_entry( data_table_value, persist=True, entry_source=self )
@@ -346,7 +350,7 @@ class DataManager( object ):
                                       noop_self=True,
                                       kwargs={'table_name': data_table_name} )
         else:
-            for data_table_name, data_table_values in data_tables_dict.iteritems():
+            for data_table_name, data_table_values in data_tables_dict.items():
                 # tool returned extra data table entries, but data table was not declared in data manager
                 # do not add these values, but do provide messages
                 log.warning( 'The data manager "%s" returned an undeclared data table "%s" with new entries "%s". These entries will not be created. Please confirm that an entry for "%s" exists in your "%s" file.' % ( self.id, data_table_name, data_table_values, data_table_name, self.data_managers.filename ) )

lib/galaxy/tools/deps/__init__.py

@@ -14,27 +14,17 @@ from galaxy.util import (
     plugin_config
 )
 
+from .requirements import (
+    ToolRequirement,
+    ToolRequirements
+)
 from .resolvers import NullDependency
-from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
+from .resolvers.conda import CondaDependencyResolver
 from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
 from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
 
 log = logging.getLogger( __name__ )
 
-# TODO: Load these from the plugins. Would require a two step initialization of
-# DependencyManager - where the plugins are loaded first and then the config
-# is parsed and sent through.
-EXTRA_CONFIG_KWDS = {
-    'conda_prefix': None,
-    'conda_exec': None,
-    'conda_debug': None,
-    'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS,
-    'conda_auto_install': False,
-    'conda_auto_init': False,
-    'conda_copy_dependencies': False,
-    'precache_dependencies': True,
-}
-
 CONFIG_VAL_NOT_FOUND = object()
 
 
@@ -43,16 +33,9 @@ def build_dependency_manager( config ):
         dependency_manager_kwds = {
             'default_base_path': config.tool_dependency_dir,
             'conf_file': config.dependency_resolvers_config_file,
+            'app_config': config,
         }
-        for key, default_value in EXTRA_CONFIG_KWDS.items():
-            value = getattr(config, key, CONFIG_VAL_NOT_FOUND)
-            if value is CONFIG_VAL_NOT_FOUND and hasattr(config, "config_dict"):
-                value = config.config_dict.get(key, CONFIG_VAL_NOT_FOUND)
-            if value is CONFIG_VAL_NOT_FOUND:
-                value = default_value
-            dependency_manager_kwds[key] = value
-        if config.use_cached_dependency_manager:
-            dependency_manager_kwds['tool_dependency_cache_dir'] = config.tool_dependency_cache_dir
+        if getattr(config, "use_cached_dependency_manager", False):
             dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
         else:
             dependency_manager = DependencyManager( **dependency_manager_kwds )
@@ -86,7 +69,7 @@ class DependencyManager( object ):
     and should each contain a file 'env.sh' which can be sourced to make the
     dependency available in the current shell environment.
     """
-    def __init__( self, default_base_path, conf_file=None, **extra_config ):
+    def __init__( self, default_base_path, conf_file=None, app_config={} ):
         """
         Create a new dependency manager looking for packages under the paths listed
         in `base_paths`.  The default base path is app.config.tool_dependency_dir.
@@ -95,11 +78,35 @@ class DependencyManager( object ):
             log.warning( "Path '%s' does not exist, ignoring", default_base_path )
         if not os.path.isdir( default_base_path ):
             log.warning( "Path '%s' is not directory, ignoring", default_base_path )
-        self.extra_config = extra_config
+        self.__app_config = app_config
         self.default_base_path = os.path.abspath( default_base_path )
         self.resolver_classes = self.__resolvers_dict()
         self.dependency_resolvers = self.__build_dependency_resolvers( conf_file )
 
+    def get_resolver_option(self, resolver, key, explicit_resolver_options={}):
+        """Look in resolver-specific settings for option and then fallback to global settings.
+        """
+        default = resolver.config_options.get(key)
+        config_prefix = resolver.resolver_type
+        global_key = "%s_%s" % (config_prefix, key)
+        value = explicit_resolver_options.get(key, CONFIG_VAL_NOT_FOUND)
+        if value is CONFIG_VAL_NOT_FOUND:
+            value = self.get_app_option(global_key, default)
+
+        return value
+
+    def get_app_option(self, key, default=None):
+        value = CONFIG_VAL_NOT_FOUND
+        if isinstance(self.__app_config, dict):
+            value = self.__app_config.get(key, CONFIG_VAL_NOT_FOUND)
+        else:
+            value = getattr(self.__app_config, key, CONFIG_VAL_NOT_FOUND)
+        if value is CONFIG_VAL_NOT_FOUND and hasattr(self.__app_config, "config_dict"):
+            value = self.__app_config.config_dict.get(key, CONFIG_VAL_NOT_FOUND)
+        if value is CONFIG_VAL_NOT_FOUND:
+            value = default
+        return value
+
     def dependency_shell_commands( self, requirements, **kwds ):
         requirement_to_dependency = self.requirements_to_dependencies(requirements, **kwds)
         return [dependency.shell_commands(requirement) for requirement, dependency in requirement_to_dependency.items()]
@@ -107,20 +114,62 @@ class DependencyManager( object ):
     def requirements_to_dependencies(self, requirements, **kwds):
         """
         Takes a list of requirements and returns a dictionary
-        with requirements as key and dependencies as value.
+        with requirements as key and dependencies as value caching
+        these on the tool instance if supplied.
         """
-        requirement_to_dependency = OrderedDict()
-        for requirement in requirements:
-            if requirement.type in [ 'package', 'set_environment' ]:
-                dependency = self.find_dep( name=requirement.name,
-                                            version=requirement.version,
-                                            type=requirement.type,
-                                            **kwds )
-                log.debug(dependency.resolver_msg)
-                if dependency.dependency_type:
-                    requirement_to_dependency[requirement] = dependency
+        requirement_to_dependency = self._requirements_to_dependencies_dict(requirements, **kwds)
+
         if 'tool_instance' in kwds:
             kwds['tool_instance'].dependencies = [dep.to_dict() for dep in requirement_to_dependency.values()]
+
+        return requirement_to_dependency
+
+    def _requirements_to_dependencies_dict(self, requirements, **kwds):
+        """Build simple requirements to dependencies dict for resolution."""
+        requirement_to_dependency = OrderedDict()
+        index = kwds.get('index', None)
+        require_exact = kwds.get('exact', False)
+        return_null_dependencies = kwds.get('return_null', False)
+
+        resolvable_requirements = requirements.resolvable
+
+        for i, resolver in enumerate(self.dependency_resolvers):
+            if index is not None and i != index:
+                continue
+
+            if len(requirement_to_dependency) == len(resolvable_requirements):
+                # Shortcut - resolution complete.
+                break
+
+            # Check requirements all at once
+            all_unmet = len(requirement_to_dependency) == 0
+            if all_unmet and hasattr(resolver, "resolve_all"):
+                # TODO: Handle specs.
+                dependencies = resolver.resolve_all(resolvable_requirements, **kwds)
+                if dependencies:
+                    assert len(dependencies) == len(resolvable_requirements)
+                    for requirement, dependency in zip(resolvable_requirements, dependencies):
+                        requirement_to_dependency[requirement] = dependency
+
+                    # Shortcut - resolution complete.
+                    break
+
+            # Check individual requirements
+            for requirement in resolvable_requirements:
+                if requirement in requirement_to_dependency:
+                    continue
+
+                dependency = resolver.resolve( requirement, **kwds )
+                if require_exact and not dependency.exact:
+                    continue
+
+                if not isinstance(dependency, NullDependency):
+                    log.debug(dependency.resolver_msg)
+                    requirement_to_dependency[requirement] = dependency
+                elif return_null_dependencies and (resolver == self.dependency_resolvers[-1] or i == index):
+                    log.debug(dependency.resolver_msg)
+                    requirement_to_dependency[requirement] = dependency
+
         return requirement_to_dependency
 
     def uses_tool_shed_dependencies(self):
@@ -128,17 +177,12 @@ class DependencyManager( object ):
 
     def find_dep( self, name, version=None, type='package', **kwds ):
         log.debug('Find dependency %s version %s' % (name, version))
-        index = kwds.get('index', None)
-        require_exact = kwds.get('exact', False)
-        for i, resolver in enumerate(self.dependency_resolvers):
-            if index is not None and i != index:
-                continue
-            dependency = resolver.resolve( name, version, type, **kwds )
-            if require_exact and not dependency.exact:
-                continue
-            if not isinstance(dependency, NullDependency):
-                return dependency
-        return NullDependency(version=version, name=name)
+        requirements = ToolRequirements([ToolRequirement(name=name, version=version, type=type)])
+        dep_dict = self._requirements_to_dependencies_dict(requirements, **kwds)
+        if len(dep_dict) > 0:
+            return dep_dict.values()[0]
+        else:
+            return NullDependency(name=name, version=version)
 
     def __build_dependency_resolvers( self, conf_file ):
         if not conf_file:
@@ -170,8 +214,9 @@ class DependencyManager( object ):
 
 
 class CachedDependencyManager(DependencyManager):
-    def __init__(self, default_base_path, conf_file=None, **extra_config):
-        super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, **extra_config)
+    def __init__(self, default_base_path, conf_file=None, app_config={}, tool_dependency_cache_dir=None):
+        super(CachedDependencyManager, self).__init__(default_base_path=default_base_path, conf_file=conf_file, app_config=app_config)
+        self.tool_dependency_cache_dir = self.get_app_option("tool_dependency_cache_dir")
 
     def build_cache(self, requirements, **kwds):
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
@@ -200,7 +245,7 @@ class CachedDependencyManager(DependencyManager):
         resolved_dependencies = self.requirements_to_dependencies(requirements, **kwds)
         cacheable_dependencies = [dep for dep in resolved_dependencies.values() if dep.cacheable]
         hashed_dependencies_dir = self.get_hashed_dependencies_path(cacheable_dependencies)
-        if not os.path.exists(hashed_dependencies_dir) and self.extra_config['precache_dependencies']:
+        if not os.path.exists(hashed_dependencies_dir) and self.get_app_option("precache_dependencies", False):
             # Cache not present, try to create it
             self.build_cache(requirements, **kwds)
         if os.path.exists(hashed_dependencies_dir):
@@ -225,4 +270,4 @@ class CachedDependencyManager(DependencyManager):
         :rtype: str
         """
         req_hashes = self.hash_dependencies(resolved_dependencies)
-        return os.path.abspath(os.path.join(self.extra_config['tool_dependency_cache_dir'], req_hashes))
+        return os.path.abspath(os.path.join(self.tool_dependency_cache_dir, req_hashes))

lib/galaxy/tools/deps/conda_util.py

@@ -34,9 +34,9 @@ CONDA_VERSION = "4.2.13"
 
 def conda_link():
     if IS_OS_X:
-        url = "https://repo.continuum.io/miniconda/Miniconda2-4.0.5-MacOSX-x86_64.sh"
+        url = "https://repo.continuum.io/miniconda/Miniconda3-4.2.12-MacOSX-x86_64.sh"
     else:
-        url = "https://repo.continuum.io/miniconda/Miniconda2-4.0.5-Linux-x86_64.sh"
+        url = "https://repo.continuum.io/miniconda/Miniconda3-4.2.12-Linux-x86_64.sh"
     return url
 
 
@@ -358,12 +358,24 @@ def install_conda(conda_context=None):
     fix_version_cmd = "%s install -y -q conda=%s " % (os.path.join(conda_context.conda_prefix, 'bin/conda'), CONDA_VERSION)
     full_command = "%s && %s && %s" % (download_cmd, install_cmd, fix_version_cmd)
     try:
+        log.info("Installing Conda, this may take several minutes.")
         return conda_context.shell_exec(full_command)
     finally:
         if os.path.exists(script_path):
             os.remove(script_path)
 
 
+def install_conda_targets(conda_targets, env_name, conda_context=None):
+    conda_context = _ensure_conda_context(conda_context)
+    conda_context.ensure_channels_configured()
+    create_args = [
+        "--name", env_name,  # enviornment for package
+    ]
+    for conda_target in conda_targets:
+        create_args.append(conda_target.package_specifier)
+    return conda_context.exec_create(create_args)
+
+
 def install_conda_target(conda_target, conda_context=None):
     """ Install specified target into a its own environment.
     """
@@ -376,10 +388,14 @@ def install_conda_target(conda_target, conda_context=None):
     return conda_context.exec_create(create_args)
 
 
-def cleanup_failed_install(conda_target, conda_context=None):
+def cleanup_failed_install_of_environment(env, conda_context=None):
     conda_context = _ensure_conda_context(conda_context)
-    if conda_context.has_env(conda_target.install_environment):
-        conda_context.exec_remove([conda_target.install_environment])
+    if conda_context.has_env(env):
+        conda_context.exec_remove([env])
+
+
+def cleanup_failed_install(conda_target, conda_context=None):
+    cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
 
 
 def best_search_result(conda_target, conda_context=None, channels_override=None):

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -33,7 +33,7 @@ CachedMulledImageMultiTarget.multi_target = True
 
 
 def list_cached_mulled_images(namespace=None):
-    command = build_docker_images_command(truncate=True, sudo_docker=False)
+    command = build_docker_images_command(truncate=True, sudo=False)
     command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
     images_and_versions = check_output(command)
     name_filter = get_filter(namespace)

lib/galaxy/tools/deps/dependencies.py

@@ -1,4 +1,4 @@
-from galaxy.tools.deps.requirements import ToolRequirement
+from galaxy.tools.deps.requirements import ToolRequirements
 from galaxy.util import bunch
 
 
@@ -29,7 +29,7 @@ class DependenciesDescription(object):
             return None
 
         requirements_dicts = as_dict.get('requirements', [])
-        requirements = [ToolRequirement.from_dict(r) for r in requirements_dicts]
+        requirements = ToolRequirements.from_list(requirements_dicts)
         installed_tool_dependencies_dicts = as_dict.get('installed_tool_dependencies', [])
         installed_tool_dependencies = map(DependenciesDescription._toolshed_install_dependency_from_dict, installed_tool_dependencies_dicts)
         return DependenciesDescription(

lib/galaxy/tools/deps/requirements.py

@@ -1,4 +1,11 @@
-from galaxy.util import asbool, xml_text
+import copy
+
+from galaxy.util import (
+    asbool,
+    xml_text,
+)
+from galaxy.util.oset import OrderedSet
+
 
 DEFAULT_REQUIREMENT_TYPE = "package"
 DEFAULT_REQUIREMENT_VERSION = None
@@ -10,23 +17,125 @@ class ToolRequirement( object ):
     run (for example, a program, package, or library).  Requirements can
     optionally assert a specific version.
     """
-    def __init__( self, name=None, type=None, version=None ):
+    def __init__( self, name=None, type=None, version=None, specs=[] ):
         self.name = name
         self.type = type
         self.version = version
+        self.specs = specs
 
     def to_dict( self ):
-        return dict(name=self.name, type=self.type, version=self.version)
+        specs = [s.to_dict() for s in self.specs]
+        return dict(name=self.name, type=self.type, version=self.version, specs=specs)
+
+    def copy( self ):
+        return copy.deepcopy( self )
 
     @staticmethod
     def from_dict( dict ):
         version = dict.get( "version", None )
         name = dict.get("name", None)
         type = dict.get("type", None)
-        return ToolRequirement( name=name, type=type, version=version )
+        specs = [RequirementSpecification.from_dict(s) for s in dict.get("specs", [])]
+        return ToolRequirement( name=name, type=type, version=version, specs=specs )
 
     def __eq__(self, other):
-        return self.name == other.name and self.type == other.type and self.version == other.version
+        return self.name == other.name and self.type == other.type and self.version == other.version and self.specs == other.specs
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash((self.name, self.type, self.version, frozenset(self.specs)))
+
+
+class RequirementSpecification(object):
+    """Refine a requirement using a URI."""
+
+    def __init__(self, uri, version=None):
+        self.uri = uri
+        self.version = version
+
+    @property
+    def specifies_version(self):
+        return self.version is not None
+
+    @property
+    def short_name(self):
+        return self.uri.split("/")[-1]
+
+    def to_dict(self):
+        return dict(uri=self.uri, version=self.version)
+
+    @staticmethod
+    def from_dict(dict):
+        uri = dict.get["uri"]
+        version = dict.get("version", None)
+        return RequirementSpecification(uri=uri, version=version)
+
+    def __eq__(self, other):
+        return self.uri == other.uri and self.version == other.version
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __hash__(self):
+        return hash((self.uri, self.version))
+
+
+class ToolRequirements(object):
+    """
+    Represents all requirements (packages, env vars) needed to run a tool.
+    """
+    def __init__(self, tool_requirements=None):
+        if tool_requirements:
+            if not isinstance(tool_requirements, list):
+                raise ToolRequirementsException('ToolRequirements Constructor expects a list')
+            self.tool_requirements = OrderedSet([r if isinstance(r, ToolRequirement) else ToolRequirement.from_dict(r) for r in tool_requirements])
+        else:
+            self.tool_requirements = OrderedSet()
+
+    @staticmethod
+    def from_list(requirements):
+        return ToolRequirements(requirements)
+
+    @property
+    def resolvable(self):
+        return ToolRequirements([r for r in self.tool_requirements if r.type in {'package', 'set_environment'}])
+
+    @property
+    def packages(self):
+        return ToolRequirements([r for r in self.tool_requirements if r.type == 'package'])
+
+    def to_list(self):
+        return [r.to_dict() for r in self.tool_requirements]
+
+    def append(self, requirement):
+        if not isinstance(requirement, ToolRequirement):
+            requirement = ToolRequirement.from_dict(requirement)
+        self.tool_requirements.add(requirement)
+
+    def __eq__(self, other):
+        return len(self.tool_requirements & other.tool_requirements) == len(self.tool_requirements) == len(other.tool_requirements)
+
+    def __ne__(self, other):
+        return not self.__eq__(other)
+
+    def __iter__(self):
+        for r in self.tool_requirements:
+            yield r
+
+    def __getitem__(self, ii):
+        return list(self.tool_requirements)[ii]
+
+    def __len__(self):
+        return len(self.tool_requirements)
+
+    def __hash__(self):
+        return sum([r.__hash__() for r in self.tool_requirements])
+
+
+class ToolRequirementsException(Exception):
+    pass
 
 
 DEFAULT_CONTAINER_TYPE = "docker"
@@ -73,7 +182,7 @@ class ContainerDescription( object ):
 def parse_requirements_from_dict( root_dict ):
     requirements = root_dict.get("requirements", [])
     containers = root_dict.get("containers", [])
-    return map(ToolRequirement.from_dict, requirements), map(ContainerDescription.from_dict, containers)
+    return ToolRequirements.from_list(requirements), map(ContainerDescription.from_dict, containers)
 
 
 def parse_requirements_from_xml( xml_root ):
@@ -105,7 +214,7 @@ def parse_requirements_from_xml( xml_root ):
     if requirements_elem is not None:
         requirement_elems = requirements_elem.findall( 'requirement' )
 
-    requirements = []
+    requirements = ToolRequirements()
     for requirement_elem in requirement_elems:
         name = xml_text( requirement_elem )
         type = requirement_elem.get( "type", DEFAULT_REQUIREMENT_TYPE )

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -5,6 +5,9 @@ from abc import (
     abstractproperty,
 )
 
+import yaml
+
+from galaxy.util import listify
 from galaxy.util.dictifiable import Dictifiable
 
 from ..requirements import ToolRequirement
@@ -22,10 +25,11 @@ class DependencyResolver(Dictifiable, object):
     # resolution.
     disabled = False
     resolves_simple_dependencies = True
+    config_options = {}
     __metaclass__ = ABCMeta
 
     @abstractmethod
-    def resolve( self, name, version, type, **kwds ):
+    def resolve( self, requirement, **kwds ):
         """Given inputs describing dependency in the abstract yield a Dependency object.
 
         The Dependency object describes various attributes (script, bin,
@@ -37,17 +41,14 @@ class DependencyResolver(Dictifiable, object):
         request version is 'default'.)
         """
 
-    def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds):
-        """ Look in resolver-specific settings for option and then fallback to
-        global settings.
+
+class MultipleDependencyResolver:
+    """Variant of DependencyResolver that can optionally resolve multiple dependencies together."""
+
+    @abstractmethod
+    def resolve_all( self, requirements, **kwds ):
+        """Given multiple requirements yield Dependency objects if and only if they may all be resolved together.
         """
-        global_key = "%s_%s" % (config_prefix, key)
-        if key in kwds:
-            return kwds.get(key)
-        elif global_key in dependency_resolver.extra_config:
-            return dependency_resolver.extra_config.get(global_key)
-        else:
-            return default
 
 
 class ListableDependencyResolver:
@@ -67,6 +68,116 @@ class ListableDependencyResolver:
         return ToolRequirement(name=name, type="package", version=version)
 
 
+class MappableDependencyResolver:
+    """Mix this into a ``DependencyResolver`` to allow mapping files.
+
+    Mapping files allow adapting generic requirements to specific local implementations.
+    """
+
+    def _setup_mapping(self, dependency_manager, **kwds):
+        mapping_files = dependency_manager.get_resolver_option(self, "mapping_files", explicit_resolver_options=kwds)
+        mappings = []
+        if mapping_files:
+            mapping_files = listify(mapping_files)
+            for mapping_file in mapping_files:
+                mappings.extend(MappableDependencyResolver._mapping_file_to_list(mapping_file))
+        self._mappings = mappings
+
+    @staticmethod
+    def _mapping_file_to_list(mapping_file):
+        with open(mapping_file, "r") as f:
+            raw_mapping = yaml.load(f) or []
+            return map(RequirementMapping.from_dict, raw_mapping)
+
+    def _expand_mappings(self, requirement):
+        for mapping in self._mappings:
+            if requirement.name == mapping.from_name:
+                if mapping.from_version is not None and mapping.from_version != requirement.version:
+                    continue
+
+                requirement = requirement.copy()
+                requirement.name = mapping.to_name
+                if mapping.to_version is not None:
+                    requirement.version = mapping.to_version
+
+                break
+
+        return requirement
+
+
+class RequirementMapping(object):
+
+    def __init__(self, from_name, from_version, to_name, to_version):
+        self.from_name = from_name
+        self.from_version = from_version
+        self.to_name = to_name
+        self.to_version = to_version
+
+    @staticmethod
+    def from_dict(raw_mapping):
+        from_raw = raw_mapping.get("from")
+        if isinstance(from_raw, dict):
+            from_name = from_raw.get("name")
+            from_version = str(from_raw.get("version"))
+        else:
+            from_name = from_raw
+            from_version = None
+
+        to_raw = raw_mapping.get("to")
+        if isinstance(to_raw, dict):
+            to_name = to_raw.get("name", from_name)
+            to_version = str(to_raw.get("version"))
+        else:
+            to_name = to_raw
+            to_version = None
+
+        return RequirementMapping(from_name, from_version, to_name, to_version)
+
+
+class SpecificationAwareDependencyResolver:
+    """Mix this into a :class:`DependencyResolver` to implement URI specification matching.
+
+    Allows adapting generic requirements to more specific URIs - to tailor name
+    or version to specified resolution system.
+    """
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def _expand_specs(self, requirement):
+        """Find closest matching specification for discovered resolver and return new concrete requirement."""
+
+
+class SpecificationPatternDependencyResolver:
+    """Implement the :class:`SpecificationAwareDependencyResolver` with a regex pattern."""
+
+    @abstractproperty
+    def _specification_pattern(self):
+        """Pattern of URI to match against."""
+
+    def _find_specification(self, specs):
+        pattern = self._specification_pattern
+        for spec in specs:
+            if pattern.match(spec.uri):
+                return spec
+        return None
+
+    def _expand_specs(self, requirement):
+        name = requirement.name
+        version = requirement.version
+        specs = requirement.specs
+
+        spec = self._find_specification(specs)
+        if spec is not None:
+            name = spec.short_name
+            version = spec.version or version
+
+            requirement = requirement.copy()
+            requirement.name = name
+            requirement.version = version
+
+        return requirement
+
+
 class InstallableDependencyResolver:
     """ Mix this into a ``DependencyResolver`` and implement to indicate
     the dependency resolver can attempt to install new dependencies.

lib/galaxy/tools/deps/resolvers/brewed_tool_shed_packages.py

@@ -29,7 +29,8 @@ class HomebrewToolShedDependencyResolver(
         self._init_homebrew(**kwds)
         self._init_base_path(dependency_manager, **kwds)
 
-    def resolve(self, name, version, type, **kwds):
+    def resolve(self, requirement, **kwds):
+        name, version, type = requirement.name, requirement.version, requirement.type
         if type != "package":
             return NullDependency(version=version, name=name)
         if version is None:

lib/galaxy/tools/deps/resolvers/conda.py

@@ -5,16 +5,20 @@ incompatible changes coming.
 
 import logging
 import os
+import re
 
 import galaxy.tools.deps.installable
 
 from ..conda_util import (
     build_isolated_environment,
     cleanup_failed_install,
+    cleanup_failed_install_of_environment,
     CondaContext,
     CondaTarget,
+    hash_conda_packages,
     install_conda,
     install_conda_target,
+    install_conda_targets,
     installed_conda_targets,
     is_conda_target_installed,
     USE_PATH_EXEC_DEFAULT,
@@ -25,7 +29,10 @@ from ..resolvers import (
     DependencyResolver,
     InstallableDependencyResolver,
     ListableDependencyResolver,
+    MappableDependencyResolver,
+    MultipleDependencyResolver,
     NullDependency,
+    SpecificationPatternDependencyResolver,
 )
 
 
@@ -36,16 +43,27 @@ DEFAULT_ENSURE_CHANNELS = "iuc,bioconda,r,defaults,conda-forge"
 log = logging.getLogger(__name__)
 
 
-class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, InstallableDependencyResolver):
+class CondaDependencyResolver(DependencyResolver, MultipleDependencyResolver, ListableDependencyResolver, InstallableDependencyResolver, SpecificationPatternDependencyResolver, MappableDependencyResolver):
     dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['conda_prefix', 'versionless', 'ensure_channels', 'auto_install']
     resolver_type = "conda"
+    config_options = {
+        'prefix': None,
+        'exec': None,
+        'debug': None,
+        'ensure_channels': DEFAULT_ENSURE_CHANNELS,
+        'auto_install': False,
+        'auto_init': True,
+        'copy_dependencies': False,
+    }
+    _specification_pattern = re.compile(r"https\:\/\/anaconda.org\/\w+\/\w+")
 
     def __init__(self, dependency_manager, **kwds):
+        self._setup_mapping(dependency_manager, **kwds)
         self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
         self.dependency_manager = dependency_manager
 
         def get_option(name):
-            return self._get_config_option(name, dependency_manager, config_prefix="conda", **kwds)
+            return dependency_manager.get_resolver_option(self, name, explicit_resolver_options=kwds)
 
         # Conda context options (these define the environment)
         conda_prefix = get_option("prefix")
@@ -102,7 +120,77 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
     def clean(self, **kwds):
         return self.conda_context.exec_clean()
 
-    def resolve(self, name, version, type, **kwds):
+    def install_all(self, conda_targets):
+        env = self.merged_environment_name(conda_targets)
+        return_code = install_conda_targets(conda_targets, env, conda_context=self.conda_context)
+        if return_code != 0:
+            is_installed = False
+        else:
+            # Recheck if installed
+            is_installed = self.conda_context.has_env(env)
+
+        if not is_installed:
+            log.debug("Removing failed conda install of {}".format(str(conda_targets)))
+            cleanup_failed_install_of_environment(env, conda_context=self.conda_context)
+
+        return is_installed
+
+    def resolve_all(self, requirements, **kwds):
+        if len(requirements) == 0:
+            return False
+
+        if not os.path.isdir(self.conda_context.conda_prefix):
+            return False
+
+        for requirement in requirements:
+            if requirement.type != "package":
+                return False
+
+        conda_targets = []
+        for requirement in requirements:
+            requirement = self._expand_requirement(requirement)
+
+            version = requirement.version
+            if self.versionless:
+                version = None
+
+            conda_targets.append(CondaTarget(requirement.name, version=version))
+
+        preserve_python_environment = kwds.get("preserve_python_environment", False)
+
+        env = self.merged_environment_name(conda_targets)
+        dependencies = []
+
+        is_installed = self.conda_context.has_env(env)
+        if not is_installed and (self.auto_install or kwds.get('install', False)):
+            is_installed = self.install_all(conda_targets)
+
+        if is_installed:
+            for requirement in requirements:
+                dependency = MergedCondaDependency(
+                    self.conda_context,
+                    self.conda_context.env_path(env),
+                    exact=not self.versionless or requirement.version is None,
+                    name=requirement.name,
+                    version=requirement.version,
+                    preserve_python_environment=preserve_python_environment,
+                )
+                dependencies.append(dependency)
+
+        return dependencies
+
+    def merged_environment_name(self, conda_targets):
+        if len(conda_targets) > 1:
+            # For continuity with mulled containers this is kind of nice.
+            return "mulled-v1-%s" % hash_conda_packages(conda_targets)
+        else:
+            assert len(conda_targets) == 1
+            return conda_targets[0].install_environment
+
+    def resolve(self, requirement, **kwds):
+        requirement = self._expand_requirement(requirement)
+        name, version, type = requirement.name, requirement.version, requirement.type
+
         # Check for conda just not being there, this way we can enable
         # conda by default and just do nothing in not configured.
         if not os.path.isdir(self.conda_context.conda_prefix):
@@ -123,7 +211,7 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         preserve_python_environment = kwds.get("preserve_python_environment", False)
 
         job_directory = kwds.get("job_directory", None)
-        if not is_installed and self.auto_install and job_directory:
+        if not is_installed and (self.auto_install or kwds.get('install', False)):
             is_installed = self.install_dependency(name=name, version=version, type=type)
 
         if not is_installed:
@@ -150,6 +238,9 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
             preserve_python_environment=preserve_python_environment,
         )
 
+    def _expand_requirement(self, requirement):
+        return self._expand_specs(self._expand_mappings(requirement))
+
     def list_dependencies(self):
         for install_target in installed_conda_targets(self.conda_context):
             name = install_target.package
@@ -193,6 +284,48 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         return self.conda_context.conda_prefix
 
 
+class MergedCondaDependency(Dependency):
+    dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
+    dependency_type = 'conda'
+
+    def __init__(self, conda_context, environment_path, exact, name=None, version=None, preserve_python_environment=False):
+        self.activate = conda_context.activate
+        self.conda_context = conda_context
+        self.environment_path = environment_path
+        self._exact = exact
+        self._name = name
+        self._version = version
+        self.cache_path = None
+        self._preserve_python_environment = preserve_python_environment
+
+    @property
+    def exact(self):
+        return self._exact
+
+    @property
+    def name(self):
+        return self._name
+
+    @property
+    def version(self):
+        return self._version
+
+    def shell_commands(self, requirement):
+        if self._preserve_python_environment:
+            # On explicit testing the only such requirement I am aware of is samtools - and it seems to work
+            # fine with just appending the PATH as done below. Other tools may require additional
+            # variables in the future.
+            return """export PATH=$PATH:'%s/bin' """ % (
+                self.environment_path,
+            )
+        else:
+            return """[ "$CONDA_DEFAULT_ENV" = "%s" ] || . %s '%s' > conda_activate.log 2>&1 """ % (
+                self.environment_path,
+                self.activate,
+                self.environment_path
+            )
+
+
 class CondaDependency(Dependency):
     dict_collection_visible_keys = Dependency.dict_collection_visible_keys + ['environment_path', 'name', 'version']
     dependency_type = 'conda'

lib/galaxy/tools/deps/resolvers/default_conda_mapping.yml

@@ -0,0 +1,5 @@
+- from: R
+  to: r-base
+- from: blast+
+  to: blast
+  

lib/galaxy/tools/deps/resolvers/galaxy_packages.py

@@ -16,6 +16,7 @@ from ..resolvers import (
     Dependency,
     DependencyResolver,
     ListableDependencyResolver,
+    MappableDependencyResolver,
     NullDependency,
 )
 
@@ -66,11 +67,13 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency
         self.versionless = str(kwds.get('versionless', "false")).lower() == "true"
         self._init_base_path( dependency_manager, **kwds )
 
-    def resolve( self, name, version, type, **kwds ):
+    def resolve(self, requirement, **kwds):
         """
         Attempt to find a dependency named `name` at version `version`. If version is None, return the "default" version as determined using a
         symbolic link (if found). Returns a triple of: env_script, base_path, real_version
         """
+        name, version, type = requirement.name, requirement.version, requirement.type
+
         if version is None or self.versionless:
             exact = not self.versionless or version is None
             return self._find_dep_default( name, type=type, exact=exact, **kwds )
@@ -101,9 +104,17 @@ class BaseGalaxyPackageDependencyResolver(DependencyResolver, UsesToolDependency
         return NullDependency(version=version, name=name)
 
 
-class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver):
+class GalaxyPackageDependencyResolver(BaseGalaxyPackageDependencyResolver, ListableDependencyResolver, MappableDependencyResolver):
     resolver_type = "galaxy_packages"
 
+    def __init__(self, dependency_manager, **kwds):
+        super(GalaxyPackageDependencyResolver, self).__init__(dependency_manager, **kwds)
+        self._setup_mapping(dependency_manager, **kwds)
+
+    def resolve(self, requirement, **kwds):
+        requirement = self._expand_mappings(requirement)
+        return super(GalaxyPackageDependencyResolver, self).resolve(requirement, **kwds)
+
     def list_dependencies(self):
         base_path = self.base_path
         for package_name in listdir(base_path):

lib/galaxy/tools/deps/resolvers/homebrew.py

@@ -39,7 +39,9 @@ class HomebrewDependencyResolver(DependencyResolver, UsesHomebrewMixin):
 
         self._init_homebrew(**kwds)
 
-    def resolve(self, name, version, type, **kwds):
+    def resolve(self, requirement, **kwds):
+        name, version, type = requirement.name, requirement.version, requirement.type
+
         if type != "package":
             return NullDependency(version=version, name=name)
 

lib/galaxy/tools/deps/resolvers/modules.py

@@ -13,7 +13,12 @@ from subprocess import PIPE, Popen
 
 from six import StringIO
 
-from ..resolvers import Dependency, DependencyResolver, NullDependency
+from ..resolvers import (
+    Dependency,
+    DependencyResolver,
+    MappableDependencyResolver,
+    NullDependency,
+)
 
 log = logging.getLogger( __name__ )
 
@@ -24,11 +29,12 @@ DEFAULT_MODULE_PREFETCH = "true"
 UNKNOWN_FIND_BY_MESSAGE = "ModuleDependencyResolver does not know how to find modules by [%s], find_by should be one of %s"
 
 
-class ModuleDependencyResolver(DependencyResolver):
+class ModuleDependencyResolver(DependencyResolver, MappableDependencyResolver):
     dict_collection_visible_keys = DependencyResolver.dict_collection_visible_keys + ['base_path', 'modulepath']
     resolver_type = "modules"
 
     def __init__(self, dependency_manager, **kwds):
+        self._setup_mapping(dependency_manager, **kwds)
         self.versionless = _string_as_bool(kwds.get('versionless', 'false'))
         find_by = kwds.get('find_by', 'avail')
         prefetch = _string_as_bool(kwds.get('prefetch', DEFAULT_MODULE_PREFETCH))
@@ -51,7 +57,10 @@ class ModuleDependencyResolver(DependencyResolver):
             module_path = DEFAULT_MODULE_PATH
         return module_path
 
-    def resolve( self, name, version, type, **kwds ):
+    def resolve(self, requirement, **kwds):
+        requirement = self._expand_mappings(requirement)
+        name, version, type = requirement.name, requirement.version, requirement.type
+
         if type != "package":
             return NullDependency(version=version, name=name)
 

lib/galaxy/tools/deps/resolvers/resolver_mixins.py

@@ -41,7 +41,7 @@ class UsesHomebrewMixin:
             return []
 
         names = os.listdir(recipe_base_path)
-        return filter(lambda n: os.path.isdir(os.path.join(recipe_base_path, n)), names)
+        return [n for n in names if os.path.isdir(os.path.join(recipe_base_path, n))]
 
 
 class UsesToolDependencyDirMixin:
@@ -53,14 +53,10 @@ class UsesToolDependencyDirMixin:
 class UsesInstalledRepositoriesMixin:
 
     def _get_installed_dependency( self, name, type, version=None, **kwds ):
-        installed_tool_dependencies = kwds.get("installed_tool_dependencies", [])
-        for installed_tool_dependency in (installed_tool_dependencies or []):
-            name_and_type_equal = installed_tool_dependency.name == name and installed_tool_dependency.type == type
-            if version:
-                if name_and_type_equal and installed_tool_dependency.version == version:
-                    return installed_tool_dependency
-            else:
-                if name_and_type_equal:
+        installed_tool_dependencies = kwds.get("installed_tool_dependencies") or []
+        for installed_tool_dependency in installed_tool_dependencies:
+            if installed_tool_dependency.name == name and installed_tool_dependency.type == type:
+                if not version or installed_tool_dependency.version == version:
                     return installed_tool_dependency
         return None
 

lib/galaxy/tools/deps/views.py

@@ -45,6 +45,19 @@ class DependencyResolversView(object):
     def resolver_dependency(self, index, **kwds):
         return self._dependency(**kwds)
 
+    def show_dependencies(self, tool_requirements_d, installed_tool_dependencies=None):
+        """
+        Resolves dependencies to build a requirements status in the admin panel/API
+        """
+        kwds = {'install': False,
+                'return_null': True,
+                'installed_tool_dependencies': installed_tool_dependencies}
+        dependencies_per_tool = {tool: self._dependency_manager.requirements_to_dependencies(requirements, **kwds) for tool, requirements in tool_requirements_d.items()}
+        return dependencies_per_tool
+
+    def install_dependencies(self, requirements):
+        return self._dependency_manager._requirements_to_dependencies_dict(requirements, **{'install': True})
+
     def install_dependency(self, index=None, **payload):
         """
         Installs dependency using highest priority resolver that supports dependency installation
@@ -125,8 +138,19 @@ class DependencyResolversView(object):
         """
         return [index for index, resolver in enumerate(self._dependency_resolvers) if hasattr(resolver, "install_dependency") and not resolver.disabled ]
 
-    def get_requirements_status(self, requested_requirements, installed_tool_dependencies=None):
-        return [self.manager_dependency(installed_tool_dependencies=installed_tool_dependencies, **req) for req in requested_requirements]
+    def get_requirements_status(self, tool_requirements_d, installed_tool_dependencies=None):
+        dependencies = self.show_dependencies(tool_requirements_d, installed_tool_dependencies)
+        # dependencies is a dict keyed on tool_ids, value is a ToolRequirements object for that tool.
+        # We use the union of resolvable ToolRequirements to get resolved dependencies without duplicates.
+        requirements = [r.resolvable for r in tool_requirements_d.values()]
+        flat_tool_requirements = set().union(*requirements)
+        flat_dependencies = []
+        for requirements_odict in dependencies.values():
+            for requirement in requirements_odict:
+                if requirement in flat_tool_requirements:
+                    flat_dependencies.append(requirements_odict[requirement])
+                    flat_tool_requirements.remove(requirement)
+        return [d.to_dict() for d in flat_dependencies]
 
     def clean(self, index=None, **kwds):
         if index:

lib/galaxy/tools/errors.py

@@ -1,9 +1,14 @@
 """
 Functionality for dealing with tool errors.
 """
-import string
-from galaxy import model, util, web
 import cgi
+import string
+
+from galaxy import (
+    model,
+    util,
+    web
+)
 from galaxy.util import unicodify
 
 error_report_template = """
@@ -54,41 +59,25 @@ ${job_traceback}
 """
 
 error_report_template_html = """
-<html><head></head>
+<html>
     <body>
-<style type="text/css">
-tr:nth-child(even) {background-color: #f2f2f2}
-table{margin: 1em;}
-.mono{font-family: monospace;}
-pre {
-white-space: pre-wrap;
-white-space: -moz-pre-wrap;
-white-space: -pre-wrap;
-white-space: -o-pre-wrap;
-word-wrap: break-word;
-background: #eee;
-border:1px solid black;
-padding:10px;
-}
-</style>
-
 <h1>Galaxy Tool Error Report</h1>
-<span class="sub"><i>from</i> <span class="mono"><a href="${host}">${host}</a></span>
+<span class="sub"><i>from</i> <span style="font-family: monospace;"><a href="${host}">${host}</a></span>
 
 <h3>Error Localization</h3>
-<table>
+<table style="margin:1em">
     <tbody>
         <tr><td>Dataset</td><td>${dataset_id} (${dataset_id_encoded})</td></tr>
-        <tr><td>History</td><td><a href="${history_view_link}">${history_id} (${history_id_encoded})</a></td></tr>
+        <tr style="background-color: #f2f2f2"><td>History</td><td><a href="${history_view_link}">${history_id} (${history_id_encoded})</a></td></tr>
         <tr><td>Failed Job</td><td>${hid}: ${history_item_name} (${hda_id_encoded})</td></tr>
     </tbody>
 </table>
 
 <h3>User Provided Information</h3>
 
-The user <a href="mailto:${email_str}"><span class="mono">${email_str}</span></a> provided the following information:
+The user <a href="mailto:${email_str}"><span style="font-family: monospace;">${email_str}</span></a> provided the following information:
 
-<pre>
+<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
 ${message}
 </pre>
 
@@ -97,12 +86,12 @@ ${message}
 
 Job environment and execution information is available at the job <a href="${hda_show_params_link}">Info Page</a>.
 
-<table>
+<table style="margin:1em">
     <tbody>
         <tr><td>Job ID</td><td>${job_id} (${job_id_encoded})</td></tr>
-        <tr><td>Tool ID</td><td>${job_tool_id}</td></tr>
+        <tr style="background-color: #f2f2f2"><td>Tool ID</td><td>${job_tool_id}</td></tr>
         <tr><td>Tool Version</td><td>${tool_version}</td></tr>
-        <tr><td>Job PID or DRM id</td><td>${job_runner_external_id}</td></tr>
+        <tr style="background-color: #f2f2f2"><td>Job PID or DRM id</td><td>${job_runner_external_id}</td></tr>
         <tr><td>Job Tool Version</td><td>${job_tool_version}</td></tr>
     </tbody>
 </table>
@@ -110,27 +99,27 @@ Job environment and execution information is available at the job <a href="${hda
 <h3>Job Execution and Failure Information</h3>
 
 <h4>Command Line</h4>
-<pre>
+<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
 ${job_command_line}
 </pre>
 
 <h4>stderr</h4>
-<pre>
+<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
 ${job_stderr}
 </pre>
 
 <h4>stdout</h4>
-<pre>
+<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
 ${job_stdout}
 </pre>
 
 <h4>Job Information</h4>
-<pre>
+<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
 ${job_info}
 </pre>
 
 <h4>Job Traceback</h4>
-<pre>
+<pre style="white-space: pre-wrap;background: #eeeeee;border:1px solid black;padding:1em;">
 ${job_traceback}
 </pre>
 

lib/galaxy/tools/evaluation.py

@@ -1,33 +1,42 @@
 import json
+import logging
 import os
 import tempfile
+
 from six import string_types
 
 from galaxy import model
-from galaxy.util.object_wrapper import wrap_with_safe_string
-from galaxy.util.bunch import Bunch
-from galaxy.util.none_like import NoneDataset
-from galaxy.util.template import fill_template
-from galaxy.tools.wrappers import (
-    ToolParameterValueWrapper,
-    DatasetFilenameWrapper,
-    DatasetListWrapper,
-    DatasetCollectionWrapper,
-    SelectToolParameterWrapper,
-    InputValueWrapper,
-    RawObjectWrapper
+from galaxy.jobs.datasets import dataset_path_rewrites
+from galaxy.tools import global_tool_errors
+from galaxy.tools.parameters import (
+    visit_input_values,
+    wrapped_json,
 )
 from galaxy.tools.parameters.basic import (
-    DataToolParameter,
     DataCollectionToolParameter,
+    DataToolParameter,
     SelectToolParameter,
 )
-from galaxy.tools.parameters import wrapped_json, visit_input_values
-from galaxy.tools.parameters.grouping import Conditional, Repeat, Section
-from galaxy.tools import global_tool_errors
-from galaxy.jobs.datasets import dataset_path_rewrites
+from galaxy.tools.parameters.grouping import (
+    Conditional,
+    Repeat,
+    Section
+)
+from galaxy.tools.wrappers import (
+    DatasetCollectionWrapper,
+    DatasetFilenameWrapper,
+    DatasetListWrapper,
+    InputValueWrapper,
+    RawObjectWrapper,
+    SelectToolParameterWrapper,
+    ToolParameterValueWrapper,
+)
+from galaxy.util.bunch import Bunch
+from galaxy.util.none_like import NoneDataset
+from galaxy.util.object_wrapper import wrap_with_safe_string
+from galaxy.util.template import fill_template
 from galaxy.work.context import WorkRequestContext
-import logging
+
 log = logging.getLogger( __name__ )
 
 
@@ -150,7 +159,7 @@ class ToolEvaluator( object ):
             """
             Wraps parameters as neccesary.
             """
-            for input in inputs.itervalues():
+            for input in inputs.values():
                 if isinstance( input, Repeat ):
                     for d in input_values[ input.name ]:
                         do_walk( input.inputs, d )
@@ -256,7 +265,7 @@ class ToolEvaluator( object ):
         #       tools where the inputs don't even get passed through. These
         #       tools (e.g. UCSC) should really be handled in a special way.
         if self.tool.check_values:
-            identifier_key_dict = dict((v, "%s|__identifier__" % k) for k, v in input_datasets.iteritems())  # allows lookup of identifier through HDA.
+            identifier_key_dict = dict((v, "%s|__identifier__" % k) for k, v in input_datasets.items())  # allows lookup of identifier through HDA.
             self.__walk_inputs( self.tool.inputs, param_dict, wrap_input )
 
     def __populate_input_dataset_wrappers(self, param_dict, input_datasets, input_dataset_paths):
@@ -347,7 +356,7 @@ class ToolEvaluator( object ):
             param_dict[name].files_path = os.path.abspath(os.path.join( job_working_directory, "dataset_%s_files" % (hda.dataset.id) ))
             for child in hda.children:
                 param_dict[ "_CHILD___%s___%s" % ( name, child.designation ) ] = DatasetFilenameWrapper( child )
-        for out_name, output in self.tool.outputs.iteritems():
+        for out_name, output in self.tool.outputs.items():
             if out_name not in param_dict and output.filters:
                 # Assume the reason we lack this output is because a filter
                 # failed to pass; for tool writing convienence, provide a
@@ -407,9 +416,9 @@ class ToolEvaluator( object ):
         Note: this method follows the style of the similar populate calls, in that param_dict is modified in-place.
         """
         # chromInfo is a filename, do not sanitize it.
-        skip = [ 'chromInfo' ] + self.tool.template_macro_params.keys()
+        skip = [ 'chromInfo' ] + list(self.tool.template_macro_params.keys())
         if not self.tool or not self.tool.options or self.tool.options.sanitize:
-            for key, value in param_dict.items():
+            for key, value in list(param_dict.items()):
                 if key not in skip:
                     # Remove key so that new wrapped object will occupy key slot
                     del param_dict[key]
@@ -565,7 +574,7 @@ class ToolEvaluator( object ):
         with open( config_filename, "w" ) as f:
             f.write( value )
         # For running jobs as the actual user, ensure the config file is globally readable
-        os.chmod( config_filename, 0644 )
+        os.chmod( config_filename, 0o644 )
 
     def __register_extra_file( self, name, local_config_path ):
         """

lib/galaxy/tools/imp_exp/__init__.py

@@ -342,7 +342,7 @@ class JobExportHistoryArchiveWrapper( object, UsesAnnotations ):
 
         def prepare_metadata( metadata ):
             """ Prepare metatdata for exporting. """
-            for name, value in metadata.items():
+            for name, value in list(metadata.items()):
                 # Metadata files are not needed for export because they can be
                 # regenerated.
                 if isinstance( value, trans.app.model.MetadataFile ):

lib/galaxy/tools/imp_exp/export_history.py

@@ -5,6 +5,7 @@ Export a history to an archive file using attribute files.
 usage: %prog history_attrs dataset_attrs job_attrs out_file
     -G, --gzip: gzip archive file
 """
+from __future__ import print_function
 
 import optparse
 import os
@@ -105,7 +106,7 @@ def main():
 
     # Create archive.
     status = create_archive( history_attrs, dataset_attrs, job_attrs, out_file, gzip )
-    print status
+    print(status)
 
 
 if __name__ == "__main__":

lib/galaxy/tools/imp_exp/unpack_tar_gz_archive.py

@@ -5,16 +5,18 @@ Unpack a tar or tar.gz archive into a directory.
 usage: %prog archive_source dest_dir
     --[url|file] source type, either a URL or a file.
 """
+from __future__ import print_function
 
+import math
+import optparse
 import os
 import sys
-import optparse
 import tarfile
 import tempfile
-import urllib2
-import math
 from base64 import b64decode
 
+from six.moves.urllib.request import urlopen
+
 # Set max size of archive/file that will be handled to be 100 GB. This is
 # arbitrary and should be adjusted as needed.
 MAX_SIZE = 100 * math.pow( 2, 30 )
@@ -25,7 +27,7 @@ def url_to_file( url, dest_file ):
     Transfer a file from a remote URL to a temporary file.
     """
     try:
-        url_reader = urllib2.urlopen( url )
+        url_reader = urlopen( url )
         CHUNK = 10 * 1024  # 10k
         total = 0
         fp = open( dest_file, 'wb')
@@ -40,7 +42,7 @@ def url_to_file( url, dest_file ):
         fp.close()
         return dest_file
     except Exception as e:
-        print "Exception getting file from URL: %s" % e, sys.stderr
+        print("Exception getting file from URL: %s" % e, file=sys.stderr)
         return None
 
 
@@ -99,4 +101,4 @@ if __name__ == "__main__":
     try:
         main(options, args)
     except Exception as e:
-        print "Error unpacking tar/gz archive: %s" % e, sys.stderr
+        print("Error unpacking tar/gz archive: %s" % e, file=sys.stderr)

lib/galaxy/tools/linters/inputs.py

@@ -1,4 +1,6 @@
 """This module contains a linting functions for tool inputs."""
+from galaxy.util import string_as_bool
+
 from ..lint_util import is_datasource
 
 
@@ -36,8 +38,6 @@ def lint_inputs(tool_xml, lint_ctx):
             if any(['value' not in option.attrib for option in select_options]):
                 lint_ctx.error("Option without value")
 
-            select_option_ids = [option.attrib.get('value', None) for option in select_options]
-
             if dynamic_options is None and len(select_options) == 0:
                 message = "No options defined for select [%s]" % param_name
                 lint_ctx.warn(message)
@@ -46,43 +46,49 @@ def lint_inputs(tool_xml, lint_ctx):
 
     conditional_selects = tool_xml.findall("./inputs//conditional")
     for conditional in conditional_selects:
-        booleans = _find_with_attribute(conditional, "param", "type", "boolean")
-        selects = _find_with_attribute(conditional, "param", "type", "select")
-        # Should conditionals ever not have a select?
-        if not len(selects) and not len(booleans):
-            lint_ctx.warn("Conditional without <param type=\"select\" /> or <param type=\"boolean\" />")
+        conditional_name = conditional.get('name')
+        if not conditional_name:
+            lint_ctx.error("Conditional without a name")
+        if conditional.get("value_from"):
+            # Probably only the upload tool use this, no children elements
+            continue
+        first_param = conditional.find("param")
+        if first_param is None:
+            lint_ctx.error("Conditional '%s' has no child <param>" % conditional_name)
+            continue
+        first_param_type = first_param.get('type')
+        if first_param_type not in ['select', 'boolean']:
+            lint_ctx.warn("Conditional '%s' first param should have type=\"select\" /> or type=\"boolean\"" % conditional_name)
             continue
 
-        test_param_optional = False
-        for select in selects:
-            test_param_optional = test_param_optional or (select.attrib.get('optional', None) is not None)
-            select_options = _find_with_attribute(select, 'option', 'value')
-            select_option_ids = [option.attrib.get('value', None) for option in select_options]
-
-        for boolean in booleans:
-            test_param_optional = test_param_optional or (boolean.attrib.get('optional', None) is not None)
-            select_option_ids = [
-                boolean.attrib.get('truevalue', 'true'),
-                boolean.attrib.get('falsevalue', 'false')
+        if first_param_type == 'select':
+            select_options = _find_with_attribute(first_param, 'option', 'value')
+            option_ids = [option.get('value') for option in select_options]
+        else:  # boolean
+            option_ids = [
+                first_param.get('truevalue', 'true'),
+                first_param.get('falsevalue', 'false')
             ]
 
-        if test_param_optional:
-            lint_ctx.warn("Conditional test parameter declares an invalid optional attribute.")
+        if string_as_bool(first_param.get('optional', False)):
+            lint_ctx.warn("Conditional test parameter cannot be optional")
 
         whens = conditional.findall('./when')
-        if any(['value' not in when.attrib for when in whens]):
+        if any('value' not in when.attrib for when in whens):
             lint_ctx.error("When without value")
 
-        when_ids = [w.attrib.get('value', None) for w in whens]
-        when_ids = [i.lower() if i in ["True", "False"] else i for i in when_ids]
+        when_ids = [w.get('value') for w in whens]
 
-        for select_id in select_option_ids:
-            if select_id not in when_ids:
-                lint_ctx.warn("No <when /> block found for select option '%s'" % select_id)
+        for option_id in option_ids:
+            if option_id not in when_ids:
+                lint_ctx.warn("No <when /> block found for %s option '%s' inside conditional '%s'" % (first_param_type, option_id, conditional_name))
 
         for when_id in when_ids:
-            if when_id not in select_option_ids:
-                lint_ctx.warn("No <option /> block found for when block '%s'" % when_id)
+            if when_id not in option_ids:
+                if first_param_type == 'select':
+                    lint_ctx.warn("No <option /> found for when block '%s' inside conditional '%s'" % (when_id, conditional_name))
+                else:
+                    lint_ctx.warn("No truevalue/falsevalue found for when block '%s' inside conditional '%s'" % (when_id, conditional_name))
 
     if datasource:
         for datasource_tag in ('display', 'uihints'):

lib/galaxy/tools/parameters/basic.py

@@ -1,29 +1,38 @@
 """
 Basic tool parameters.
 """
-
 import logging
-import re
 import os
 import os.path
-from six import string_types
+import re
 from xml.etree.ElementTree import XML
 
-from galaxy import util
-from galaxy.util import string_as_bool, sanitize_param, unicodify
-from galaxy.util.expressions import ExpressionContext
-from sanitize import ToolParameterSanitizer
-import validation
-import galaxy.tools.parser
-from ..parser import get_input_source as ensure_input_source
-from ..parameters import history_query
-from ..parameters import dynamic_options
-from .dataset_matcher import DatasetMatcher
-from .dataset_matcher import DatasetCollectionMatcher
-from galaxy.web import url_for
-from galaxy.util.dictifiable import Dictifiable
+from six import string_types
+
 import galaxy.model
+import galaxy.tools.parser
+from galaxy import util
+from galaxy.util import (
+    sanitize_param,
+    string_as_bool,
+    unicodify
+)
 from galaxy.util.bunch import Bunch
+from galaxy.util.dictifiable import Dictifiable
+from galaxy.util.expressions import ExpressionContext
+from galaxy.web import url_for
+
+from . import validation
+from .dataset_matcher import (
+    DatasetCollectionMatcher,
+    DatasetMatcher
+)
+from .sanitize import ToolParameterSanitizer
+from ..parameters import (
+    dynamic_options,
+    history_query
+)
+from ..parser import get_input_source as ensure_input_source
 
 log = logging.getLogger( __name__ )
 
@@ -802,7 +811,7 @@ class SelectToolParameter( ToolParameter ):
     def from_json( self, value, trans, other_values={} ):
         legal_values = self.get_legal_values( trans, other_values )
         workflow_building_mode = trans.workflow_building_mode
-        for context_value in other_values.itervalues():
+        for context_value in other_values.values():
             if isinstance( context_value, RuntimeValue ):
                 workflow_building_mode = True
                 break
@@ -852,7 +861,7 @@ class SelectToolParameter( ToolParameter ):
         if isinstance( value, list ):
             if not self.multiple:
                 raise ValueError( "Multiple values provided but parameter %s is not expecting multiple values." % self.name )
-            value = map( str, value )
+            value = list(map( str, value ))
         else:
             value = str( value )
         if self.tool is None or self.tool.options.sanitize:
@@ -1049,7 +1058,7 @@ class ColumnListParameter( SelectToolParameter ):
                         column2 = column2.strip()
                         if column2:
                             column_list.append( column2 )
-                value = map( ColumnListParameter._strip_c, column_list )
+                value = list(map( ColumnListParameter._strip_c, column_list ))
             else:
                 value = []
         else:
@@ -1101,7 +1110,7 @@ class ColumnListParameter( SelectToolParameter ):
             if column_list is None:
                 column_list = this_column_list
             else:
-                column_list = filter( lambda c: c in this_column_list, column_list )
+                column_list = [c for c in column_list if c in this_column_list]
         return column_list
 
     def get_options( self, trans, other_values ):
@@ -1252,12 +1261,12 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
                 options = self._get_options_from_code( trans=trans, value=value, other_values=other_values )
             else:
                 options = []
-            for filter_key, filter_value in self.filtered.iteritems():
+            for filter_key, filter_value in self.filtered.items():
                 dataset = other_values.get(filter_key)
                 if dataset.__class__.__name__.endswith( "DatasetFilenameWrapper" ):  # this is a bad way to check for this, but problems importing class ( due to circular imports? )
                     dataset = dataset.dataset
                 if dataset:
-                    for meta_key, meta_dict in filter_value.iteritems():
+                    for meta_key, meta_dict in filter_value.items():
                         if hasattr( dataset, 'metadata' ) and hasattr( dataset.metadata, 'spec' ):
                             check_meta_val = dataset.metadata.spec[ meta_key ].param.to_string( dataset.metadata.get( meta_key ) )
                             if check_meta_val in meta_dict:
@@ -1391,7 +1400,7 @@ class DrillDownSelectToolParameter( SelectToolParameter ):
         """
         Get the *names* of the other params this param depends on.
         """
-        return self.filtered.keys()
+        return list(self.filtered.keys())
 
     def to_dict( self, trans, other_values={} ):
         # skip SelectToolParameter (the immediate parent) bc we need to get options in a different way here
@@ -1766,8 +1775,8 @@ class DataToolParameter( BaseDataToolParameter ):
         datatypes_registery = self._datatypes_registery( trans, self.tool )
         all_edam_formats = datatypes_registery.edam_formats if hasattr( datatypes_registery, 'edam_formats' ) else {}
         all_edam_data = datatypes_registery.edam_data if hasattr( datatypes_registery, 'edam_formats' ) else {}
-        edam_formats = map(lambda ext: all_edam_formats.get(ext, None), extensions)
-        edam_data = map(lambda ext: all_edam_data.get(ext, None), extensions)
+        edam_formats = [all_edam_formats.get(ext, None) for ext in extensions]
+        edam_data = [all_edam_data.get(ext, None) for ext in extensions]
 
         d['extensions'] = extensions
         d['edam'] = {'edam_formats': edam_formats, 'edam_data': edam_data}

lib/galaxy/tools/parameters/dynamic_options.py

@@ -2,13 +2,20 @@
 Support for generating the options for a SelectToolParameter dynamically (based
 on the values of other parameters or other aspects of the current state)
 """
-
 import logging
 import os
-import validation
-from galaxy.util import string_as_bool
-from galaxy.model import User, HistoryDatasetAssociation, HistoryDatasetCollectionAssociation
+
+from six import StringIO
+
 import galaxy.tools
+from galaxy.model import (
+    HistoryDatasetAssociation,
+    HistoryDatasetCollectionAssociation,
+    User
+)
+from galaxy.util import string_as_bool
+
+from . import validation
 
 log = logging.getLogger(__name__)
 
@@ -590,10 +597,9 @@ class DynamicOptions( object ):
                 options = self.parse_file_fields( open( path ) )
             else:
                 # Pass just the first megabyte to parse_file_fields.
-                import StringIO
                 log.warning( "Attempting to load options from large file, reading just first megabyte" )
                 contents = open( path, 'r' ).read( 1048576 )
-                options = self.parse_file_fields( StringIO.StringIO( contents ) )
+                options = self.parse_file_fields( StringIO( contents ) )
         elif self.tool_data_table:
             options = self.tool_data_table.get_fields()
         else:

lib/galaxy/tools/parameters/grouping.py

@@ -1,21 +1,26 @@
 """
 Constructs for grouping tool parameters
 """
-
 import logging
-log = logging.getLogger( __name__ )
-
 import os
-import StringIO
 import unicodedata
-from six import text_type
+
+from six import (
+    StringIO,
+    text_type
+)
+
 from galaxy.datatypes import sniff
-from galaxy.util import inflector
-from galaxy.util import relpath
-from galaxy.util import sanitize_for_filename
+from galaxy.util import (
+    inflector,
+    relpath,
+    sanitize_for_filename
+)
 from galaxy.util.bunch import Bunch
-from galaxy.util.expressions import ExpressionContext
 from galaxy.util.dictifiable import Dictifiable
+from galaxy.util.expressions import ExpressionContext
+
+log = logging.getLogger( __name__ )
 
 
 class Group( object, Dictifiable ):
@@ -82,7 +87,7 @@ class Repeat( Group ):
             # Propogate __index__
             if '__index__' in d:
                 rval_dict['__index__'] = d['__index__']
-            for input in self.inputs.itervalues():
+            for input in self.inputs.values():
                 rval_dict[ input.name ] = input.value_to_basic( d[input.name], app )
             rval.append( rval_dict )
         return rval
@@ -96,7 +101,7 @@ class Repeat( Group ):
                 # compatibility)
                 rval_dict['__index__'] = d.get( '__index__', i )
                 # Restore child inputs
-                for input in self.inputs.itervalues():
+                for input in self.inputs.values():
                     if ignore_errors and input.name not in d:
                         # If we do not have a value, and are ignoring errors, we simply
                         # do nothing. There will be no value for the parameter in the
@@ -114,7 +119,7 @@ class Repeat( Group ):
         rval = []
         for i in range( self.default ):
             rval_dict = { '__index__': i}
-            for input in self.inputs.itervalues():
+            for input in self.inputs.values():
                 rval_dict[ input.name ] = input.get_initial_value( trans, context )
             rval.append( rval_dict )
         return rval
@@ -125,7 +130,7 @@ class Repeat( Group ):
         def input_to_dict( input ):
             return input.to_dict( trans )
 
-        repeat_dict[ "inputs" ] = map( input_to_dict, self.inputs.values() )
+        repeat_dict[ "inputs" ] = list(map( input_to_dict, self.inputs.values() ))
         return repeat_dict
 
 
@@ -150,14 +155,14 @@ class Section( Group ):
 
     def value_to_basic( self, value, app ):
         rval = {}
-        for input in self.inputs.itervalues():
+        for input in self.inputs.values():
             rval[ input.name ] = input.value_to_basic( value[input.name], app )
         return rval
 
     def value_from_basic( self, value, app, ignore_errors=False ):
         rval = {}
         try:
-            for input in self.inputs.itervalues():
+            for input in self.inputs.values():
                 if not ignore_errors or input.name in value:
                     rval[ input.name ] = input.value_from_basic( value[ input.name ], app, ignore_errors )
         except Exception as e:
@@ -168,7 +173,7 @@ class Section( Group ):
     def get_initial_value( self, trans, context ):
         rval = {}
         child_context = ExpressionContext( rval, context )
-        for child_input in self.inputs.itervalues():
+        for child_input in self.inputs.values():
             rval[ child_input.name ] = child_input.get_initial_value( trans, child_context )
         return rval
 
@@ -178,7 +183,7 @@ class Section( Group ):
         def input_to_dict( input ):
             return input.to_dict( trans )
 
-        section_dict[ "inputs" ] = map( input_to_dict, self.inputs.values() )
+        section_dict[ "inputs" ] = list(map( input_to_dict, self.inputs.values() ))
         return section_dict
 
 
@@ -232,7 +237,7 @@ class UploadDataset( Group ):
 
     def title_by_index( self, trans, index, context ):
         d_type = self.get_datatype( trans, context )
-        for i, ( composite_name, composite_file ) in enumerate( d_type.writable_files.iteritems() ):
+        for i, ( composite_name, composite_file ) in enumerate( d_type.writable_files.items() ):
             if i == index:
                 rval = composite_name
                 if composite_file.description:
@@ -249,7 +254,7 @@ class UploadDataset( Group ):
             # Propogate __index__
             if '__index__' in d:
                 rval_dict['__index__'] = d['__index__']
-            for input in self.inputs.itervalues():
+            for input in self.inputs.values():
                 rval_dict[ input.name ] = input.value_to_basic( d[input.name], app )
             rval.append( rval_dict )
         return rval
@@ -262,7 +267,7 @@ class UploadDataset( Group ):
             # compatibility)
             rval_dict['__index__'] = d.get( '__index__', i )
             # Restore child inputs
-            for input in self.inputs.itervalues():
+            for input in self.inputs.values():
                 if ignore_errors and input.name not in d:  # this wasn't tested
                     rval_dict[ input.name ] = input.get_initial_value( None, d )
                 else:
@@ -273,10 +278,10 @@ class UploadDataset( Group ):
     def get_initial_value( self, trans, context ):
         d_type = self.get_datatype( trans, context )
         rval = []
-        for i, ( composite_name, composite_file ) in enumerate( d_type.writable_files.iteritems() ):
+        for i, ( composite_name, composite_file ) in enumerate( d_type.writable_files.items() ):
             rval_dict = {}
             rval_dict['__index__'] = i  # create __index__
-            for input in self.inputs.itervalues():
+            for input in self.inputs.values():
                 rval_dict[ input.name ] = input.get_initial_value( trans, context )
             rval.append( rval_dict )
         return rval
@@ -467,8 +472,8 @@ class UploadDataset( Group ):
             dataset.uuid = None
             # load metadata
             files_metadata = context.get( self.metadata_ref, {} )
-            metadata_name_substition_default_dict = dict( [ ( composite_file.substitute_name_with_metadata, d_type.metadata_spec[ composite_file.substitute_name_with_metadata ].default ) for composite_file in d_type.composite_files.values() if composite_file.substitute_name_with_metadata ] )
-            for meta_name, meta_spec in d_type.metadata_spec.iteritems():
+            metadata_name_substition_default_dict = dict( ( composite_file.substitute_name_with_metadata, d_type.metadata_spec[ composite_file.substitute_name_with_metadata ].default ) for composite_file in d_type.composite_files.values() if composite_file.substitute_name_with_metadata )
+            for meta_name, meta_spec in d_type.metadata_spec.items():
                 if meta_spec.set_in_upload:
                     if meta_name in files_metadata:
                         meta_value = files_metadata[ meta_name ]
@@ -478,7 +483,7 @@ class UploadDataset( Group ):
             dataset.precreated_name = dataset.name = self.get_composite_dataset_name( context )
             if dataset.datatype.composite_type == 'auto_primary_file':
                 # replace sniff here with just creating an empty file
-                temp_name, is_multi_byte = sniff.stream_to_file( StringIO.StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' )
+                temp_name, is_multi_byte = sniff.stream_to_file( StringIO( d_type.generate_primary_file( dataset ) ), prefix='upload_auto_primary_file' )
                 dataset.primary_file = temp_name
                 dataset.to_posix_lines = True
                 dataset.space_to_tab = False
@@ -494,7 +499,7 @@ class UploadDataset( Group ):
             keys = [ value.name for value in writable_files.values() ]
             for i, group_incoming in enumerate( groups_incoming[ writable_files_offset : ] ):
                 key = keys[ i + writable_files_offset ]
-                if group_incoming is None and not writable_files[ writable_files.keys()[ keys.index( key ) ] ].optional:
+                if group_incoming is None and not writable_files[ list(writable_files.keys())[ keys.index( key ) ] ].optional:
                     dataset.warnings.append( "A required composite file (%s) was not specified." % ( key ) )
                     dataset.composite_files[ key ] = None
                 else:
@@ -504,7 +509,7 @@ class UploadDataset( Group ):
                         dataset.composite_files[ key ] = file_bunch.__dict__
                     else:
                         dataset.composite_files[ key ] = None
-                        if not writable_files[ writable_files.keys()[ keys.index( key ) ] ].optional:
+                        if not writable_files[ list(writable_files.keys())[ keys.index( key ) ] ].optional:
                             dataset.warnings.append( "A required composite file (%s) was not specified." % ( key ) )
             return [ dataset ]
         else:
@@ -546,7 +551,7 @@ class Conditional( Group ):
         rval = dict()
         rval[ self.test_param.name ] = self.test_param.value_to_basic( value[ self.test_param.name ], app )
         current_case = rval[ '__current_case__' ] = self.get_current_case( value[ self.test_param.name ] )
-        for input in self.cases[ current_case ].inputs.itervalues():
+        for input in self.cases[ current_case ].inputs.values():
             if input.name in value:  # parameter might be absent in unverified workflow
                 rval[ input.name ] = input.value_to_basic( value[ input.name ], app )
         return rval
@@ -557,7 +562,7 @@ class Conditional( Group ):
             rval[ self.test_param.name ] = self.test_param.value_from_basic( value.get( self.test_param.name ), app, ignore_errors )
             current_case = rval[ '__current_case__' ] = self.get_current_case( rval[ self.test_param.name ] )
             # Inputs associated with current case
-            for input in self.cases[ current_case ].inputs.itervalues():
+            for input in self.cases[ current_case ].inputs.values():
                 # If we do not have a value, and are ignoring errors, we simply
                 # do nothing. There will be no value for the parameter in the
                 # conditional's values dictionary.
@@ -581,7 +586,7 @@ class Conditional( Group ):
         rval[ self.test_param.name ] = test_value
         # Fill in state for selected case
         child_context = ExpressionContext( rval, context )
-        for child_input in self.cases[current_case].inputs.itervalues():
+        for child_input in self.cases[current_case].inputs.values():
             rval[ child_input.name ] = child_input.get_initial_value( trans, child_context )
         return rval
 
@@ -591,7 +596,7 @@ class Conditional( Group ):
         def nested_to_dict( input ):
             return input.to_dict( trans )
 
-        cond_dict[ "cases" ] = map( nested_to_dict, self.cases )
+        cond_dict[ "cases" ] = list(map( nested_to_dict, self.cases ))
         cond_dict[ "test_param" ] = nested_to_dict( self.test_param )
         return cond_dict
 
@@ -609,5 +614,5 @@ class ConditionalWhen( object, Dictifiable ):
         def input_to_dict( input ):
             return input.to_dict( trans )
 
-        when_dict[ "inputs" ] = map( input_to_dict, self.inputs.values() )
+        when_dict[ "inputs" ] = list(map( input_to_dict, self.inputs.values() ))
         return when_dict

lib/galaxy/tools/parameters/meta.py

@@ -1,10 +1,14 @@
-from galaxy.util import permutations
-from galaxy import model
-from galaxy import util
-from galaxy import exceptions
-import itertools
 import copy
+import itertools
 import logging
+
+from galaxy import (
+    exceptions,
+    model,
+    util
+)
+from galaxy.util import permutations
+
 log = logging.getLogger( __name__ )
 
 
@@ -52,10 +56,10 @@ def expand_workflow_inputs( inputs ):
     product = product or [ [ None ] ]
     linked_keys = linked_keys or [ ( None, None ) ]
     product_keys = product_keys or [ ( None, None ) ]
-    for linked_values, product_values in itertools.product( *[ zip( *linked ), itertools.product( *product ) ] ):
+    for linked_values, product_values in itertools.product( zip( *linked ), itertools.product( *product ) ):
         new_params = copy.deepcopy( inputs )
         new_keys = []
-        for ( step_id, key ), value in zip( linked_keys, linked_values ) + zip( product_keys, product_values ):
+        for ( step_id, key ), value in list(zip( linked_keys, linked_values )) + list(zip( product_keys, product_values )):
             if step_id is not None:
                 new_params[ step_id ][ key ] = value
                 new_keys.append( value[ 'hid' ] )

lib/galaxy/tools/parameters/output_collect.py

@@ -1,24 +1,25 @@
 """ Code allowing tools to define extra files associated with an output datset.
 """
-import os
-import re
-import operator
 import glob
 import json
+import logging
+import operator
+import os
+import re
 
 from galaxy import jobs
 from galaxy import util
-from galaxy.util import odict
-from galaxy.util import ExecutionTimer
 from galaxy.tools.parser.output_collection_def import (
     DEFAULT_DATASET_COLLECTOR_DESCRIPTION,
     INPUT_DBKEY_TOKEN,
 )
+from galaxy.util import (
+    ExecutionTimer,
+    odict
+)
 
 DATASET_ID_TOKEN = "DATASET_ID"
 
-
-import logging
 log = logging.getLogger( __name__ )
 
 
@@ -105,7 +106,7 @@ class JobContext( object ):
         filenames = self.find_files( collection, dataset_collectors )
 
         element_datasets = []
-        for filename, extra_file_collector in filenames.iteritems():
+        for filename, extra_file_collector in filenames.items():
             create_dataset_timer = ExecutionTimer()
             fields_match = extra_file_collector.match( collection, os.path.basename( filename ) )
             if not fields_match:
@@ -248,7 +249,7 @@ def collect_primary_datasets( tool, output, job_working_directory, input_ext, in
         if 'job_working_directory' in app.config.collect_outputs_from:
             for path, extra_file_collector in walk_over_extra_files( dataset_collectors, job_working_directory, outdata ):
                 filenames[ path ] = extra_file_collector
-        for filename_index, ( filename, extra_file_collector ) in enumerate( filenames.iteritems() ):
+        for filename_index, ( filename, extra_file_collector ) in enumerate( filenames.items() ):
             fields_match = extra_file_collector.match( outdata, os.path.basename( filename ) )
             if not fields_match:
                 # Before I guess pop() would just have thrown an IndexError

lib/galaxy/tools/search/__init__.py

@@ -5,16 +5,24 @@ installed within this Galaxy.
 import logging
 import re
 import tempfile
-
-from galaxy.web.framework.helpers import to_unicode
 from datetime import datetime
 
-from whoosh.filedb.filestore import RamStorage, FileStorage
-from whoosh.fields import KEYWORD, Schema, STORED, TEXT
-from whoosh.scoring import BM25F
-from whoosh.qparser import MultifieldParser
 from whoosh import analysis
+from whoosh.analysis import StandardAnalyzer
+from whoosh.fields import (
+    KEYWORD,
+    Schema,
+    STORED,
+    TEXT
+)
+from whoosh.filedb.filestore import (
+    FileStorage,
+    RamStorage
+)
+from whoosh.qparser import MultifieldParser
+from whoosh.scoring import BM25F
 
+from galaxy.web.framework.helpers import to_unicode
 
 log = logging.getLogger( __name__ )
 
@@ -85,7 +93,7 @@ class ToolBoxSearch( object ):
         stop_time = datetime.now()
         log.debug( 'Toolbox index finished. It took: ' + str(stop_time - start_time) )
 
-    def search( self, q, tool_name_boost, tool_section_boost, tool_description_boost, tool_label_boost, tool_stub_boost, tool_help_boost, tool_search_limit ):
+    def search( self, q, tool_name_boost, tool_section_boost, tool_description_boost, tool_label_boost, tool_stub_boost, tool_help_boost, tool_search_limit, tool_enable_ngram_search, tool_ngram_minsize, tool_ngram_maxsize ):
         """
         Perform search on the in-memory index. Weight in the given boosts.
         """
@@ -105,9 +113,34 @@ class ToolBoxSearch( object ):
         # Hyphens are wildcards in Whoosh causing bad things
         if q.find( '-' ) != -1:
             q = (' ').join( [ token.text for token in self.rex( to_unicode( q ) ) ] )
-        # Perform the search
-        hits = searcher.search( parser.parse( '*' + q + '*' ), limit=float( tool_search_limit ) )
-        return [ hit[ 'id' ] for hit in hits ]
+        # Perform tool search with ngrams if set to true in the config file
+        if ( tool_enable_ngram_search is True or tool_enable_ngram_search == "True" ):
+            hits_with_score = {}
+            token_analyzer = StandardAnalyzer() | analysis.NgramFilter( minsize=int( tool_ngram_minsize ), maxsize=int( tool_ngram_maxsize ) )
+            ngrams = [ token.text for token in token_analyzer( q ) ]
+            for query in ngrams:
+                # Get the tool list with respective scores for each qgram
+                curr_hits = searcher.search( parser.parse( '*' + query + '*' ), limit=float( tool_search_limit ) )
+                for i, curr_hit in enumerate( curr_hits ):
+                    is_present = False
+                    for prev_hit in hits_with_score:
+                        # Check if the tool appears again for the next qgram search
+                        if curr_hit[ 'id' ] == prev_hit:
+                            is_present = True
+                            # Add the current score with the previous one if the
+                            # tool appears again for the next qgram
+                            hits_with_score[ prev_hit ] = curr_hits.score(i) + hits_with_score[ prev_hit ]
+                    # Add the tool if not present to the collection with its score
+                    if not is_present:
+                        hits_with_score[ curr_hit[ 'id' ] ] = curr_hits.score(i)
+            # Sort the results based on aggregated BM25 score in decreasing order of scores
+            hits_with_score = sorted( hits_with_score.items(), key=lambda x: x[1], reverse=True )
+            # Return the tool ids
+            return [ item[0] for item in hits_with_score[ 0:int( tool_search_limit ) ] ]
+        else:
+            # Perform the search
+            hits = searcher.search( parser.parse( '*' + q + '*' ), limit=float( tool_search_limit ) )
+            return [ hit[ 'id' ] for hit in hits ]
 
 
 def _temp_storage(self, name=None):

lib/galaxy/tools/util/galaxyops/__init__.py

@@ -1,16 +1,18 @@
 """Utility functions for galaxyops"""
+from __future__ import print_function
+
 import sys
 
 
 def warn( msg ):
     # TODO: since everything printed to stderr results in job.state = error, we
     # don't need both a warn and a fail...
-    print >> sys.stderr, msg
+    print(msg, file=sys.stderr)
     sys.exit( 1 )
 
 
 def fail( msg ):
-    print >> sys.stderr, msg
+    print(msg, file=sys.stderr)
     sys.exit( 1 )
 
 
@@ -25,15 +27,15 @@ def parse_cols_arg( cols ):
         # looks something like 1,2,3,
         if cols.endswith( ',' ):
             cols += '0'
-        col_list = map( lambda x: int( x ) - 1, cols.split(",") )
+        col_list = [int( x ) - 1 for x in cols.split(",")]
         return col_list
     else:
         return BED_DEFAULT_COLS
 
 
 def default_printer( stream, exc, obj ):
-    print >> stream, "%d: %s" % ( obj.linenum, obj.current_line )
-    print >> stream, "\tError: %s" % ( str(exc) )
+    print("%d: %s" % ( obj.linenum, obj.current_line ), file=stream)
+    print("\tError: %s" % ( str(exc) ), file=stream)
 
 
 def skipped( reader, filedesc="" ):

lib/galaxy/tools/util/maf_utilities.py

@@ -3,18 +3,21 @@
 Provides wrappers and utilities for working with MAF files and alignments.
 """
 # Dan Blankenberg
+from __future__ import print_function
+
 import logging
 import os
+import resource
 import string
 import sys
 import tempfile
+from copy import deepcopy
+from errno import EMFILE
 
 import bx.align.maf
-import bx.intervals
 import bx.interval_index_file
-from errno import EMFILE
-import resource
-from copy import deepcopy
+import bx.intervals
+from six.moves import xrange
 
 assert sys.version_info[:2] >= ( 2, 4 )
 
@@ -51,7 +54,7 @@ def get_species_in_block( block ):
 
 
 def tool_fail( msg="Unknown Error" ):
-    print >> sys.stderr, "Fatal Error: %s" % msg
+    print("Fatal Error: %s" % msg, file=sys.stderr)
     sys.exit()
 
 
@@ -136,7 +139,7 @@ class TempFileHandler( object ):
 class RegionAlignment( object ):
 
     DNA_COMPLEMENT = string.maketrans( "ACGTacgt", "TGCAtgca" )
-    MAX_SEQUENCE_SIZE = sys.maxint  # Maximum length of sequence allowed
+    MAX_SEQUENCE_SIZE = sys.maxsize  # Maximum length of sequence allowed
 
     def __init__( self, size, species=[], temp_file_handler=None ):
         assert size <= self.MAX_SEQUENCE_SIZE, "Maximum length allowed for an individual sequence has been exceeded (%i > %i)." % ( size, self.MAX_SEQUENCE_SIZE )
@@ -161,7 +164,7 @@ class RegionAlignment( object ):
     def get_species_names( self, skip=[] ):
         if not isinstance( skip, list ):
             skip = [skip]
-        names = self.sequences.keys()
+        names = list(self.sequences.keys())
         for name in skip:
             try:
                 names.remove( name )
@@ -314,7 +317,7 @@ def build_maf_index_species_chromosomes( filename, index_species=None ):
         maf_reader = bx.align.maf.Reader( open( filename ) )
         while True:
             pos = maf_reader.file.tell()
-            block = maf_reader.next()
+            block = next(maf_reader)
             if block is None:
                 break
             blocks += 1
@@ -478,7 +481,7 @@ def iter_blocks_split_by_species( block, species=None ):
     empty_block = bx.align.Alignment( score=block.score, attributes=deepcopy( block.attributes ) )  # should we copy attributes?
     empty_block.text_size = block.text_size
     # call recursive function to split into each combo of spec/blocks
-    for value in __split_components_by_species( spec_dict.values(), empty_block ):
+    for value in __split_components_by_species( list(spec_dict.values()), empty_block ):
         sort_block_components_by_block( value, block )  # restore original component order
         yield value
 
@@ -612,10 +615,10 @@ def get_starts_ends_fields_from_gene_bed( line ):
 
     # Calculate and store starts and ends of coding exons
     region_start, region_end = cds_start, cds_end
-    exon_starts = map( int, fields[11].rstrip( ',\n' ).split( ',' ) )
-    exon_starts = map( ( lambda x: x + tx_start ), exon_starts )
-    exon_ends = map( int, fields[10].rstrip( ',' ).split( ',' ) )
-    exon_ends = map( ( lambda x, y: x + y ), exon_starts, exon_ends )
+    exon_starts = list(map( int, fields[11].rstrip( ',\n' ).split( ',' ) ))
+    exon_starts = [x + tx_start for x in exon_starts]
+    exon_ends = list(map( int, fields[10].rstrip( ',' ).split( ',' ) ))
+    exon_ends = [x + y for x, y in zip( exon_starts, exon_ends )]
     for start, end in zip( exon_starts, exon_ends ):
         start = max( start, region_start )
         end = min( end, region_end )
@@ -680,7 +683,7 @@ def remove_temp_index_file( index_filename ):
 
 def get_fasta_header( component, attributes={}, suffix=None ):
     header = ">%s(%s):%i-%i|" % ( component.src, component.strand, component.get_forward_strand_start(), component.get_forward_strand_end() )
-    for key, value in attributes.iteritems():
+    for key, value in attributes.items():
         header = "%s%s=%s|" % ( header, key, value )
     if suffix:
         header = "%s%s" % ( header, suffix )
@@ -714,7 +717,7 @@ def get_attributes_from_fasta_header( header ):
         # fields 0 is not a region coordinate
         pass
     if len( fields ) > 2:
-        for i in xrange( 1, len( fields ) - 1 ):
+        for i in range( 1, len( fields ) - 1 ):
             prop = fields[i].split( '=', 1 )
             if len( prop ) == 2:
                 attributes[ prop[0] ] = prop[1]

lib/galaxy/tools/verify/asserts/__init__.py

@@ -44,7 +44,7 @@ def verify_assertion(data, assertion_description):
 
     assert_function_args = inspect.getargspec(assert_function).args
     args = {}
-    for attribute, value in assertion_description["attributes"].iteritems():
+    for attribute, value in assertion_description["attributes"].items():
         if attribute in assert_function_args:
             args[attribute] = value
 

lib/galaxy/tools/verify/test_data.py

@@ -28,7 +28,7 @@ class TestDataResolver(object):
     def __init__(self, env_var='GALAXY_TEST_FILE_DIR', environ=os.environ):
         file_dirs = environ.get(env_var, None)
         if file_dirs:
-            self.resolvers = map(lambda u: build_resolver(u, environ), LIST_SEP.split(file_dirs))
+            self.resolvers = [build_resolver(u, environ) for u in LIST_SEP.split(file_dirs)]
         else:
             self.resolvers = []
 

lib/galaxy/tools/wrappers.py

@@ -1,13 +1,16 @@
+import logging
 import os
 import pipes
 import tempfile
+
+from six import string_types
+
 from galaxy import exceptions
-from galaxy.util.none_like import NoneDataset
 from galaxy.util import odict
+from galaxy.util.none_like import NoneDataset
 from galaxy.util.object_wrapper import wrap_with_safe_string
 
-from logging import getLogger
-log = getLogger( __name__ )
+log = logging.getLogger( __name__ )
 
 # Fields in .log files corresponding to paths, must have one of the following
 # field names and all such fields are assumed to be paths. This is to allow
@@ -27,8 +30,9 @@ class ToolParameterValueWrapper( object ):
     Base class for object that Wraps a Tool Parameter and Value.
     """
 
-    def __nonzero__( self ):
+    def __bool__( self ):
         return bool( self.value )
+    __nonzero__ = __bool__
 
     def get_display_text( self, quote=True ):
         """
@@ -48,8 +52,9 @@ class RawObjectWrapper( ToolParameterValueWrapper ):
     def __init__( self, obj ):
         self.obj = obj
 
-    def __nonzero__( self ):
+    def __bool__( self ):
         return bool( self.obj )  # FIXME: would it be safe/backwards compatible to rename .obj to .value, so that we can just inherit this method?
+    __nonzero__ = __bool__
 
     def __str__( self ):
         try:
@@ -72,7 +77,7 @@ class InputValueWrapper( ToolParameterValueWrapper ):
         self._other_values = other_values
 
     def __eq__( self, other ):
-        if isinstance( other, basestring ):
+        if isinstance( other, string_types ):
             return str( self ) == other
         elif isinstance( other, int ):
             return int( self ) == other
@@ -144,7 +149,7 @@ class SelectToolParameterWrapper( ToolParameterValueWrapper ):
         self.fields = self.SelectToolParameterFieldWrapper( input, value, other_values, self._path_rewriter )
 
     def __eq__( self, other ):
-        if isinstance( other, basestring ):
+        if isinstance( other, string_types ):
             return str( self ) == other
         else:
             return super( SelectToolParameterWrapper, self ) == other
@@ -193,8 +198,9 @@ class DatasetFilenameWrapper( ToolParameterValueWrapper ):
                 rval = wrap_with_safe_string( rval )
             return rval
 
-        def __nonzero__( self ):
+        def __bool__( self ):
             return self.metadata.__nonzero__()
+        __nonzero__ = __bool__
 
         def __iter__( self ):
             return self.metadata.__iter__()
@@ -206,7 +212,7 @@ class DatasetFilenameWrapper( ToolParameterValueWrapper ):
                 return default
 
         def items( self ):
-            return iter( [ ( k, self.get( k ) ) for k, v in self.metadata.items() ] )
+            return iter( ( k, self.get( k ) ) for k, v in self.metadata.items() )
 
     def __init__( self, dataset, datatypes_registry=None, tool=None, name=None, dataset_path=None, identifier=None ):
         if not dataset:
@@ -276,8 +282,9 @@ class DatasetFilenameWrapper( ToolParameterValueWrapper ):
         else:
             return getattr( self.dataset, key )
 
-    def __nonzero__( self ):
+    def __bool__( self ):
         return bool( self.dataset )
+    __nonzero__ = __bool__
 
 
 class HasDatasets:
@@ -407,7 +414,8 @@ class DatasetCollectionWrapper( ToolParameterValueWrapper, HasDatasets ):
             return [].__iter__()
         return self.__element_instance_list.__iter__()
 
-    def __nonzero__( self ):
+    def __bool__( self ):
         # Fail `#if $param` checks in cheetah is optional input
         # not specified or if resulting collection is empty.
         return self.__input_supplied and bool( self.__element_instance_list )
+    __nonzero__ = __bool__

lib/galaxy/tools/xsd/galaxy.xsd

@@ -332,7 +332,7 @@ Read more about configuring Galaxy to run Docker jobs
     </xs:attribute>
     <xs:attribute name="split_inputs" type="xs:string">
       <xs:annotation>
-        <xs:documentation xml:lang="en">Documentation for split_inputs</xs:documentation>
+        <xs:documentation xml:lang="en">A comma separated list of data inputs to split for job parallelization.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="split_size" type="xs:string">
@@ -345,6 +345,11 @@ Read more about configuring Galaxy to run Docker jobs
         <xs:documentation xml:lang="en">Documentation for split_mode</xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="shared_inputs" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">A comma separated list of data inputs that should not be split for this tool, Galaxy will infer this if not present and so this potentially never needs to be set.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
   </xs:complexType>
   <xs:complexType name="Code">
     <xs:annotation>
@@ -1937,6 +1942,20 @@ rendered on the tool form as a text area instead of a single line text box.
 <param name="foo" type="text" area="True" size="5x25" />
 ```
 
+As of 17.01, ``text`` parameters can also supply a static list of preset
+defaults options. The user **may** be presented with the option to select one of
+these but will be allowed to supply an arbitrary text value.
+
+```xml
+<param name="foo" type="text" value="foo 1">
+    <option value="foo 1">Foo 1 Display</option>
+    <option value="foo 2">Foo 2 Display</option>
+</param>
+```
+
+See [param_text_option.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/param_text_option.xml)
+for a demonstration of this.
+
 $attribute_list:value,size,area:5
 
 #### ``integer`` and ``float``
@@ -2333,6 +2352,7 @@ allow access to Python code to generate options for a select list. See
           <xs:documentation xml:lang="en">Documentation for label</xs:documentation>
         </xs:annotation>
       </xs:element>
+      <xs:element name="conversion" type="ParamConversion" />
       <xs:element name="option" type="ParamOption" />
       <xs:element name="options" type="ParamOptions"/>
       <xs:element name="validator" type="Validator" />
@@ -2633,6 +2653,32 @@ if the corresponding option is selected by default (the default is ``false``).
     </xs:simpleContent>
   </xs:complexType>
 
+  <xs:complexType name="ParamConversion">
+    <xs:annotation>
+      <xs:documentation xml:lang="en"><![CDATA[
+
+A contrived example of a tool that uses this is the test tool
+[explicit_conversion.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/explicit_conversion.xml).
+
+This directive is optionally contained within the ``<param>`` tag when the
+``type`` attribute value is ``data`` and is used to dynamically generated a converted
+dataset for the contained input of the type specified using the ``type`` tag.
+
+]]>
+</xs:documentation>
+    </xs:annotation>
+    <xs:attribute name="name" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Name of cheetah variable to create for converted dataset.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+    <xs:attribute name="type" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">The short extension describing the datatype to convert to - Galaxy must have a datatype converter from the parent input's type to this.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
+  </xs:complexType>
+
 
   <xs:complexType name="ParamOptions">
     <xs:annotation>
@@ -3228,7 +3274,7 @@ to the tool.
 ```
 
 This more advanced example, taken from Mothur's
-[remove.linage.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/mothur/remove.lineage.xml)
+[remove.lineage.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/mothur/remove.lineage.xml)
 tool demonstrates using filters to sort a list and remove duplicate entries.
 
 ```xml
@@ -3278,7 +3324,7 @@ tool demonstrates adding values to an option list using ``filter``s.
 </param>
 ```
 
-While this fragment from maf_to_interval.xml demonstrates removing items.
+While this fragment from [maf_to_interval.xml](https://github.com/galaxyproject/galaxy/blob/dev/tools/maf/maf_to_interval.xml) demonstrates removing items.
 
 ```xml
 <param name="species" type="select" label="Select additional species"
@@ -3343,7 +3389,7 @@ used with ``type`` of ``add_value``).</xs:documentation>
     <xs:attribute name="key" type="xs:string">
       <xs:annotation>
         <xs:documentation xml:lang="en">When ``type`` is ``data_meta``, ``param_value``,
-or ``remove_value`` - this is the name of the metadata key of ref to filter by.</xs:documentation>
+or ``remove_value`` - this is the name of the metadata key to filter by.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
     <xs:attribute name="multiple" type="PermissiveBoolean" default="false">
@@ -3390,6 +3436,13 @@ is the index into the list to add the option to. If not set, the option will be
 added to the end of the list.</xs:documentation>
       </xs:annotation>
     </xs:attribute>
+    <xs:attribute name="meta_ref" type="xs:string">
+      <xs:annotation>
+        <xs:documentation xml:lang="en">Only used when ``type`` is
+``remove_value``. Dataset to look for the value of metadata ``key`` to remove
+from the list.</xs:documentation>
+      </xs:annotation>
+    </xs:attribute>
   </xs:complexType>
   <xs:complexType name="Outputs">
     <xs:annotation>
@@ -4770,7 +4823,7 @@ prepended with the warning ``Warning: Branch A was taken in execution``.
       <xs:documentation xml:lang="en"><![CDATA[See
 [extract_genomic_dna.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/extract_genomic_dna/extract_genomic_dna.xml)
 or the test tool
-[output_action_change_format.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/output_action_change_format.xml)
+[output_format.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/output_format.xml)
 for simple examples of how this tag set is used in a tool. This tag set is
 optionally contained within the ``<data>`` tag set and is the container tag set
 for the following ``<when>`` tag set.]]></xs:documentation>
@@ -4815,7 +4868,7 @@ in the ``<data>`` tag set with ``format="interval"``.
 See
 [extract_genomic_dna.xml](https://github.com/galaxyproject/tools-iuc/blob/master/tools/extract_genomic_dna/extract_genomic_dna.xml)
 or the test tool
-[output_action_change_format.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/output_action_change_format.xml)
+[output_format.xml](https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/output_format.xml)
 for more examples.
 
 ]]></xs:documentation>
@@ -5025,6 +5078,7 @@ and ``bibtex`` are the only supported options.</xs:documentation>
       <xs:enumeration value="unique_value"/>
       <xs:enumeration value="multiple_splitter"/>
       <xs:enumeration value="add_value"/>
+      <xs:enumeration value="remove_value"/>
       <xs:enumeration value="sort_by"/>
     </xs:restriction>
   </xs:simpleType>

lib/galaxy/util/object_wrapper.py

@@ -184,6 +184,7 @@ def wrap_with_safe_string( value, no_wrap_classes=None ):
                 # Set pickle and copy properties
                 copy_reg.pickle( wrapped_class, pickle_safe_object, do_wrap_func )
         return wrapped_class( value, safe_string_wrapper_function=do_wrap_func )
+
     # Determine classes not to wrap
     if no_wrap_classes:
         if not isinstance( no_wrap_classes, ( tuple, list ) ):
@@ -214,7 +215,7 @@ class SafeStringWrapper( object ):
     will still be sanitized, but not wrapped), and e.g. integers will have neither.
     """
     __UNSANITIZED_ATTRIBUTE_NAME__ = 'unsanitized'
-    __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', __UNSANITIZED_ATTRIBUTE_NAME__]
+    __NO_WRAP_NAMES__ = [ '__safe_string_wrapper_function__', '__class__', __UNSANITIZED_ATTRIBUTE_NAME__]
 
     def __new__( cls, *arg, **kwd ):
         # We need to define a __new__ since, we are subclassing from e.g. immutable str, which internally sets data
@@ -284,7 +285,7 @@ class SafeStringWrapper( object ):
     def __getattr__( self, name ):
         if name in SafeStringWrapper.__NO_WRAP_NAMES__:
             # FIXME: is this ever reached?
-            return object.__getattr__( self, name )
+            return object.__getattribute__( self, name )
         return self.__safe_string_wrapper_function__( getattr( self.unsanitized, name ) )
 
     def __setattr__( self, name, value ):

lib/galaxy/util/oset.py

@@ -0,0 +1,62 @@
+"""
+Ordered set implementation from https://code.activestate.com/recipes/576694/
+"""
+import collections
+
+
+class OrderedSet(collections.MutableSet):
+    def __init__(self, iterable=None):
+        self.end = end = []
+        end += [None, end, end]         # sentinel node for doubly linked list
+        self.map = {}                   # key --> [key, prev, next]
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        return len(self.map)
+
+    def __contains__(self, key):
+        return key in self.map
+
+    def add(self, key):
+        if key not in self.map:
+            end = self.end
+            curr = end[1]
+            curr[2] = end[1] = self.map[key] = [key, curr, end]
+
+    def discard(self, key):
+        if key in self.map:
+            key, prev, next = self.map.pop(key)
+            prev[2] = next
+            next[1] = prev
+
+    def __iter__(self):
+        end = self.end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def pop(self, last=True):
+        if not self:
+            raise KeyError('set is empty')
+        key = self.end[1][0] if last else self.end[2][0]
+        self.discard(key)
+        return key
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__,)
+        return '%s(%r)' % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedSet):
+            return len(self) == len(other) and list(self) == list(other)
+        return set(self) == set(other)

lib/galaxy/util/properties.py

@@ -10,6 +10,28 @@ from six import iteritems
 from six.moves.configparser import ConfigParser
 
 
+def find_config_file(default, old_default, explicit, cwd=None):
+    if cwd is not None:
+        default = os.path.join(cwd, default)
+        old_default = os.path.join(cwd, old_default)
+        if explicit is not None:
+            explicit = os.path.join(cwd, explicit)
+
+    if explicit:
+        if os.path.exists(explicit):
+            config_file = explicit
+        else:
+            raise Exception("Problem determining Galaxy's configuration - the specified configuration file cannot be found.")
+    else:
+        if not os.path.exists( default ) and os.path.exists( old_default ):
+            config_file = old_default
+        elif os.path.exists( default ):
+            config_file = default
+        else:
+            config_file = default + ".sample"
+    return config_file
+
+
 def load_app_properties(
     kwds={},
     ini_file=None,
@@ -97,4 +119,4 @@ class NicerConfigParser(ConfigParser):
                 raise
 
 
-__all__ = ('load_app_properties', 'NicerConfigParser')
+__all__ = ('find_config_file', 'load_app_properties', 'NicerConfigParser')

lib/galaxy/version.py

@@ -1,3 +1,3 @@
-VERSION_MAJOR = "17.01"
+VERSION_MAJOR = "17.05"
 VERSION_MINOR = "dev"
 VERSION = VERSION_MAJOR + ('.' + VERSION_MINOR if VERSION_MINOR else '')

lib/galaxy/web/framework/middleware/remoteuser.py

@@ -150,6 +150,7 @@ class RemoteUser( object ):
                 '/user/api_keys',
                 '/user/edit_username',
                 '/user/dbkeys',
+                '/user/logout',
                 '/user/toolbox_filters',
                 '/user/set_default_permissions',
                 '/user/change_communication',

lib/galaxy/webapps/galaxy/api/configuration.py

@@ -5,11 +5,15 @@ and configuration settings.
 
 from galaxy.web import _future_expose_api_anonymous_and_sessionless as expose_api_anonymous_and_sessionless
 from galaxy.web import _future_expose_api as expose_api
+from galaxy.web import _future_expose_api as expose_api_anonymous
 from galaxy.web import require_admin
 from galaxy.web.base.controller import BaseAPIController
-from galaxy.managers import configuration
+from galaxy.managers import configuration, users
 from galaxy.queue_worker import send_control_task
 
+import json
+import os
+
 import logging
 log = logging.getLogger( __name__ )
 
@@ -20,6 +24,19 @@ class ConfigurationController( BaseAPIController ):
         super( ConfigurationController, self ).__init__( app )
         self.config_serializer = configuration.ConfigSerializer( app )
         self.admin_config_serializer = configuration.AdminConfigSerializer( app )
+        self.user_manager = users.UserManager( app )
+
+    @expose_api_anonymous
+    def whoami( self, trans, **kwd ):
+        """
+        GET /api/whoami
+        Return information about the current authenticated user.
+
+        :returns: dictionary with user information
+        :rtype:   dict
+        """
+        current_user = self.user_manager.current_user( trans )
+        return current_user.to_dict()
 
     @expose_api_anonymous_and_sessionless
     def index( self, trans, **kwd ):
@@ -42,7 +59,14 @@ class ConfigurationController( BaseAPIController ):
         :rtype:     dict
         :returns:   dictionary with major version keyed on 'version_major'
         """
-        return {"version_major": self.app.config.version_major }
+        extra = {}
+        try:
+            version_file = os.environ.get("GALAXY_VERSION_JSON_FILE", self.app.container_finder.app_info.galaxy_root_dir + "/version.json")
+            with open(version_file, "r") as f:
+                extra = json.load(f)
+        except Exception:
+            pass
+        return {"version_major": self.app.config.version_major, "extra": extra}
 
     def get_config_dict( self, trans, return_admin=False, view=None, keys=None, default_view='all' ):
         """

lib/galaxy/webapps/galaxy/api/tools.py

@@ -141,11 +141,11 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
             force_rebuild:           If true and chache dir exists, attempts to delete cache dir
         """
         tool = self._get_tool(id)
-        [tool._view.install_dependency(id=None, **req.to_dict()) for req in tool.requirements]
+        tool._view.install_dependencies(tool.requirements)
         if kwds.get('build_dependency_cache'):
             tool.build_dependency_cache(**kwds)
         # TODO: rework resolver install system to log and report what has been done.
-        # _view.install_dependency should return a dict with stdout, stderr and success status
+        # _view.install_dependencies should return a dict with stdout, stderr and success status
         return tool.tool_requirements_status
 
     @expose_api
@@ -236,6 +236,9 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
         tool_stub_boost = self.app.config.get( 'tool_stub_boost', 5 )
         tool_help_boost = self.app.config.get( 'tool_help_boost', 0.5 )
         tool_search_limit = self.app.config.get( 'tool_search_limit', 20 )
+        tool_enable_ngram_search = self.app.config.get( 'tool_enable_ngram_search', False )
+        tool_ngram_minsize = self.app.config.get( 'tool_ngram_minsize', 3 )
+        tool_ngram_maxsize = self.app.config.get( 'tool_ngram_maxsize', 4 )
 
         results = self.app.toolbox_search.search( q=q,
                                                   tool_name_boost=tool_name_boost,
@@ -244,7 +247,10 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ):
                                                   tool_label_boost=tool_label_boost,
                                                   tool_stub_boost=tool_stub_boost,
                                                   tool_help_boost=tool_help_boost,
-                                                  tool_search_limit=tool_search_limit )
+                                                  tool_search_limit=tool_search_limit,
+                                                  tool_enable_ngram_search=tool_enable_ngram_search,
+                                                  tool_ngram_minsize=tool_ngram_minsize,
+                                                  tool_ngram_maxsize=tool_ngram_maxsize )
         return results
 
     @expose_api_anonymous_and_sessionless

lib/galaxy/webapps/galaxy/buildapp.py

@@ -310,6 +310,10 @@ def populate_api_routes( webapp, app ):
     webapp.mapper.connect( "configuration_version",
                            "/api/version", controller="configuration",
                            action="version", conditions=dict( method=[ "GET" ] ) )
+    webapp.mapper.connect( "api_whoami",
+                           "/api/whoami", controller='configuration',
+                           action='whoami',
+                           conditions=dict( method=[ "GET" ] ) )
     webapp.mapper.resource( 'datatype',
                             'datatypes',
                             path_prefix='/api',

lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py

@@ -781,8 +781,8 @@ class AdminToolshed( AdminGalaxy ):
                                                                                 reinstalling=False,
                                                                                 required_repo_info_dicts=None )
         view = views.DependencyResolversView(self.app)
-        requirements = suc.get_unique_requirements_from_repository(repository)
-        requirements_status = view.get_requirements_status(requirements, repository.installed_tool_dependencies)
+        tool_requirements_d = suc.get_requirements_from_repository(repository)
+        requirements_status = view.get_requirements_status(tool_requirements_d, repository.installed_tool_dependencies)
         return trans.fill_template( '/admin/tool_shed_repository/manage_repository.mako',
                                     repository=repository,
                                     description=description,

lib/galaxy/webapps/galaxy/controllers/user.py

@@ -1021,6 +1021,7 @@ class User( BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Creat
                         trans.app.object_store.create( new_len.dataset )
                     except ObjectInvalid:
                         raise Exception( 'Unable to create output dataset: object store is full' )
+
                     trans.sa_session.flush()
                     counter = 0
                     f = open(new_len.file_name, "w")
@@ -1037,9 +1038,16 @@ class User( BaseUIController, UsesFormDefinitionsMixin, CreatesUsersMixin, Creat
                         except ValueError:
                             lines_skipped += 1
                             continue
+
+                        if chrom != escape(chrom):
+                            message = 'Invalid chromosome(s) with HTML detected and skipped'
+                            lines_skipped += 1
+                            continue
+
                         counter += 1
                         f.write("%s\t%s\n" % (chrom, length))
                     f.close()
+
                     build_dict.update( { "len": new_len.id, "count": counter } )
                 else:
                     dataset_id = trans.security.decode_id( dataset_id )

lib/galaxy/webapps/galaxy/controllers/workflow.py

@@ -967,6 +967,10 @@ class WorkflowController( BaseUIController, SharableMixin, UsesStoredWorkflowMix
                 history=history
             )
         else:
+            # If there is just one dataset name selected or one dataset collection, these
+            # come through as string types instead of lists. xref #3247.
+            dataset_names = util.listify(dataset_names)
+            dataset_collection_names = util.listify(dataset_collection_names)
             stored_workflow = extract_workflow(
                 trans,
                 user=user,

lib/galaxy/workflow/modules.py

@@ -566,7 +566,7 @@ class ToolModule( WorkflowModule ):
         self.tool_version = tool_version
         self.tool = trans.app.toolbox.get_tool( tool_id, tool_version=tool_version, exact=exact_tools )
         if self.tool and tool_version and exact_tools and str( self.tool.version ) != str( tool_version ):
-            log.debug( "Exact tool specified during workflow module creation for [%s] but couldn't find correct version [%s]." % (tool_id, tool_version) )
+            log.info( "Exact tool specified during workflow module creation for [%s] but couldn't find correct version [%s]." % (tool_id, tool_version) )
             self.tool = None
         self.post_job_actions = {}
         self.runtime_post_job_actions = {}

lib/tool_shed/galaxy_install/install_manager.py

@@ -905,15 +905,12 @@ class InstallRepositoryManager( object ):
             if 'tools' in metadata and install_resolver_dependencies:
                 self.update_tool_shed_repository_status( tool_shed_repository,
                                                          self.install_model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES )
-                requirements = suc.get_unique_requirements_from_repository(tool_shed_repository)
-                [self._view.install_dependency(id=None, **req) for req in requirements]
+                new_tools = [self.app.toolbox._tools_by_id.get(tool_d['guid'], None) for tool_d in metadata['tools']]
+                new_requirements = set([tool.requirements.packages for tool in new_tools if tool])
+                [self._view.install_dependencies(r) for r in new_requirements]
                 if self.app.config.use_cached_dependency_manager:
-                    cached_requirements = []
-                    for tool_d in metadata['tools']:
-                        tool = self.app.toolbox._tools_by_id.get(tool_d['guid'], None)
-                        if tool and tool.requirements not in cached_requirements:
-                            cached_requirements.append(tool.requirements)
-                            tool.build_dependency_cache()
+                    [self.app.toolbox.dependency_manager.build_cache(r) for r in new_requirements]
+
             if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata:
                 work_dir = tempfile.mkdtemp( prefix="tmp-toolshed-itsr" )
                 # Install tool dependencies.

lib/tool_shed/util/shed_util_common.py

@@ -8,6 +8,8 @@ import string
 import sqlalchemy.orm.exc
 from sqlalchemy import and_, false, true
 
+import galaxy.tools.deps.requirements
+
 from galaxy import util
 from galaxy.util import checkers
 from galaxy.web import url_for
@@ -219,37 +221,18 @@ def get_tool_shed_repo_requirements(app, tool_shed_url, repositories=None, repo_
         valid_tools = json_response[1].get('valid_tools', [])
         if valid_tools:
             tools.extend(valid_tools)
-    return get_unique_requirements_from_tools(tools)
+    return get_requirements_from_tools(tools)
 
 
-def get_unique_requirements_from_tools(tools):
-    requirements = []
-    for tool in tools:
-        if tool['requirements']:
-            requirements.append(tool['requirements'])
-    return get_unique_requirements(requirements)
+def get_requirements_from_tools(tools):
+    return {tool['id']: galaxy.tools.deps.requirements.ToolRequirements.from_list(tool['requirements']) for tool in tools}
 
 
-def get_unique_requirements(requirements):
-    uniq_reqs = dict()
-    for tool_requirements in requirements:
-        for req in tool_requirements:
-            name = req.get("name", None)
-            if not name:
-                continue  # A requirement without a name can't be resolved, so let's skip those
-            version = req.get("version", "versionless")
-            type = req.get("type", None)
-            if not type == "package":
-                continue
-            uniq_reqs["%s_%s" % (name, version)] = {'name': name, 'version': version, 'type': type}
-    return list(uniq_reqs.values())
-
-
-def get_unique_requirements_from_repository(repository):
+def get_requirements_from_repository(repository):
     if not repository.includes_tools:
-        return []
+        return {}
     else:
-        return get_unique_requirements_from_tools(repository.metadata.get('tools', []))
+        return get_requirements_from_tools(repository.metadata.get('tools', []))
 
 
 def get_ctx_rev( app, tool_shed_url, name, owner, changeset_revision ):

run.sh

@@ -14,6 +14,7 @@ then
     . $GALAXY_LOCAL_ENV_FILE
 fi
 
+INITIALIZE_TOOL_DEPENDENCIES=1  # Install Conda if needed.
 # Pop args meant for common_startup.sh
 while :
 do
@@ -22,6 +23,10 @@ do
             common_startup_args="$common_startup_args $1"
             shift
             ;;
+        --skip-tool-dependency-initialization)
+            INITIALIZE_TOOL_DEPENDENCIES=0
+            shift
+            ;;
         --skip-venv)
             skip_venv=1
             common_startup_args="$common_startup_args $1"
@@ -98,6 +103,11 @@ if [ -z "$GALAXY_CONFIG_FILE" ]; then
     export GALAXY_CONFIG_FILE
 fi
 
+if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then
+    # Install Conda environment if needed.
+    python ./scripts/manage_tool_dependencies.py -c "$GALAXY_CONFIG_FILE" init_if_needed
+fi
+
 if [ -n "$GALAXY_RUN_ALL" ]; then
     servers=$(sed -n 's/^\[server:\(.*\)\]/\1/  p' "$GALAXY_CONFIG_FILE" | xargs echo)
     if [ -z "$stop_daemon_arg_set" -a -z "$daemon_or_restart_arg_set" ]; then

run_tests.sh

@@ -214,7 +214,7 @@ ensure_grunt() {
 }
 
 
-DOCKER_DEFAULT_IMAGE='galaxy/testing-base:15.10.3'
+DOCKER_DEFAULT_IMAGE='galaxy/testing-base:17.01.0'
 
 test_script="./scripts/functional_tests.py"
 report_file="run_functional_tests.html"

scripts/manage_tool_dependencies.py

@@ -0,0 +1,87 @@
+import argparse
+import os.path
+import sys
+
+sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
+
+from galaxy.util.properties import find_config_file, load_app_properties
+from galaxy.config import (
+    configure_logging,
+    find_path,
+    find_root,
+    parse_dependency_options,
+)
+from galaxy.tools.deps import CachedDependencyManager, DependencyManager, NullDependencyManager
+
+DESCRIPTION = "Script to manage tool dependencies (with focus on a Conda environments)."
+
+
+def main(argv=None):
+    """Entry point for conversion process."""
+    if argv is None:
+        argv = sys.argv[1:]
+    args = _arg_parser().parse_args(argv)
+    action = args.action
+    action_func = ACTIONS[action]
+    action_func(args)
+
+
+def _init_if_needed(args):
+    kwargs = _app_properties(args)
+
+    # If conda_auto_init is set, simply building the Conda resolver will call handle installation.
+    _build_dependency_manager_no_config(kwargs)
+
+
+def _app_properties(args):
+    config_file = find_config_file("config/galaxy.ini", "universe_wsgi.ini", args.config_file)
+    app_properties = load_app_properties(ini_file=config_file)
+    return app_properties
+
+
+def _arg_parser():
+    parser = argparse.ArgumentParser(description=DESCRIPTION)
+    parser.add_argument('action', metavar='ACTION', type=str,
+                        choices=ACTIONS.keys(),
+                        help='action to perform')
+    parser.add_argument("-c", "--config-file", default=None)
+    return parser
+
+
+def _build_dependency_manager_no_config(kwargs):
+    """Simplified variant of build_dependency_manager from galaxy.tools.deps.
+
+    The canonical factory method requires a full Galaxy configuration object
+    which we do not have available in this script (an optimization).
+    """
+    configure_logging(kwargs)
+    root = find_root(kwargs)
+    dependency_resolvers_config_file = find_path(kwargs, "dependency_resolvers_config_file", root)
+    use_dependencies, tool_dependency_dir, use_cached_dependency_manager, tool_dependency_cache_dir, precache_dependencies = \
+        parse_dependency_options(kwargs, root, dependency_resolvers_config_file)
+
+    if not use_dependencies:
+        dependency_manager = NullDependencyManager()
+    else:
+        dependency_manager_kwds = {
+            'default_base_path': tool_dependency_dir,
+            'conf_file': dependency_resolvers_config_file,
+            'app_config': kwargs,
+        }
+
+        if use_cached_dependency_manager:
+            dependency_manager_kwds['tool_dependency_cache_dir'] = tool_dependency_cache_dir
+            dependency_manager = CachedDependencyManager(**dependency_manager_kwds)
+        else:
+            dependency_manager = DependencyManager( **dependency_manager_kwds )
+
+    return dependency_manager
+
+
+ACTIONS = {
+    "init_if_needed": _init_if_needed,
+}
+
+
+if __name__ == '__main__':
+    main()

static/maps/mvc/form/form-view.js.map

@@ -1 +1 @@
-{"version":3,"file":"form-view.js","sources":["../../../src/mvc/form/form-view.js"],"names":["define","Utils","Portlet","Ui","FormSection","FormData","Backbone","View","extend","initialize","options","this","model","Model","initial_errors","cls","icon","always_refresh","message_status","set","setElement","render","update","new_model","self","data","matchModel","node","input_id","input","input_list","_","isEqual","field","field_list","new_options","indexOf","type","i","opt","length","push","label","value","trigger","Galaxy","emit","debug","wait","active","is_dynamic","unwait","highlight","message","silent","input_element","element_list","error","portlet","expand","$panel","$el","parents","filter","$","css","first","animate","scrollTop","offset","top","errors","error_messages","matchResponse","off","Manager","_renderForm","create","get","attributes","current_check","checksum","on","refresh_on_change","new_check","each","reset","remove","Message","section","inputs","title","operations","buttons","collapsible","collapsed","onchange_title","append","empty","persistent","status"],"mappings":"AAGAA,QAAU,cAAe,oBAAqB,iBAAkB,wBAAyB,sBACzF,SAAUC,EAAOC,EAASC,EAAIC,EAAaC,GACvC,MAAOC,UAASC,KAAKC,QACjBC,WAAY,SAAUC,GAClBC,KAAKC,MAAQ,GAAIN,UAASO,OACtBC,gBAAkB,EAClBC,IAAkB,qBAClBC,KAAkB,KAClBC,gBAAkB,EAClBC,eAAkB,YACnBC,IAAKT,GACRC,KAAKS,WAAY,UACjBT,KAAKU,UAITC,OAAQ,SAAUC,GACd,GAAIC,GAAOb,IACXA,MAAKc,KAAKC,WAAYH,EAAW,SAAUI,EAAMC,GAC7C,GAAIC,GAAQL,EAAKM,WAAYF,EAC7B,IAAKC,GAASA,EAAMnB,UACVqB,EAAEC,QAASH,EAAMnB,QAASiB,EAAKjB,SAAY,CAC7CmB,EAAMnB,QAAUiB,EAAKjB,OACrB,IAAIuB,GAAQT,EAAKU,WAAYN,EAC7B,IAAKK,EAAMX,OAAS,CAChB,GAAIa,KACJ,IAA8E,KAArE,OAAQ,kBAAmB,cAAiBC,QAASP,EAAMQ,MAChEF,EAAcN,EAAMnB,YAEpB,KAAM,GAAI4B,KAAKX,GAAKjB,QAAU,CAC1B,GAAI6B,GAAMZ,EAAKjB,QAAS4B,EACnBC,GAAIC,OAAS,GACdL,EAAYM,MAAQC,MAAOH,EAAK,GAAKI,MAAOJ,EAAK,KAI7DN,EAAMX,OAAQa,GACdF,EAAMW,QAAS,UACfC,OAAOC,KAAKC,MAAO,sBAAuB,wBAA0BnB,QAQxFoB,KAAM,SAAUC,GACZ,IAAM,GAAIX,KAAK3B,MAAKmB,WAAa,CAC7B,GAAIG,GAAQtB,KAAKuB,WAAYI,GACzBT,EAAQlB,KAAKmB,WAAYQ,EACxBT,GAAMqB,YAAcjB,EAAMe,MAAQf,EAAMkB,QACzClB,EAAOgB,EAAS,OAAS,cAMrCG,UAAW,SAAWxB,EAAUyB,EAASC,GACrC,GAAIC,GAAgB5C,KAAK6C,aAAc5B,EACvC,IAAK2B,IACDA,EAAcE,MAAOJ,GAAW,iCAChC1C,KAAK+C,QAAQC,SACbhD,KAAKiC,QAAS,SAAUhB,IAClB0B,GAAS,CACX,GAAIM,GAASjD,KAAKkD,IAAIC,UAAUC,OAAO,WACnC,MAAsE,KAA7D,OAAQ,UAAW3B,QAAS4B,EAAGrD,MAAOsD,IAAK,eACrDC,OACHN,GAAOO,SAAWC,UAAYR,EAAOQ,YAAcb,EAAcM,IAAIQ,SAASC,IAAM,KAAO,OAMvGC,OAAQ,SAAU7D,GAEd,GADAC,KAAKiC,QAAS,SACTlC,GAAWA,EAAQ6D,OAAS,CAC7B,GAAIC,GAAiB7D,KAAKc,KAAKgD,cAAe/D,EAAQ6D,OACtD,KAAM,GAAI3C,KAAYjB,MAAK6C,aAAe,CACtC,CAAY7C,KAAK6C,aAAc5B,GAC1B4C,EAAgB5C,IACjBjB,KAAKyC,UAAWxB,EAAU4C,EAAgB5C,IAAY,MAOtEP,OAAQ,WACJ,GAAIG,GAAOb,IACXA,MAAK+D,IAAI,UACT/D,KAAK+D,IAAI,SAET/D,KAAKuB,cAELvB,KAAKmB,cAELnB,KAAK6C,gBAEL7C,KAAKc,KAAO,GAAIpB,GAASsE,QAAShE,MAClCA,KAAKiE,cACLjE,KAAKc,KAAKoD,SACVlE,KAAKC,MAAMkE,IAAK,mBAAsBnE,KAAK4D,OAAQ5D,KAAKC,MAAMmE,WAE9D,IAAIC,GAAgBrE,KAAKc,KAAKwD,UAc9B,OAbAtE,MAAKuE,GAAI,SAAU,SAAUtD,GACzB,GAAIC,GAAQL,EAAKM,WAAYF,EAC7B,KAAMC,GAASA,EAAMsD,mBAAqB3D,EAAKZ,MAAMkE,IAAK,kBAAqB,CAC3E,GAAIM,GAAY5D,EAAKC,KAAKwD,UACrBG,IAAaJ,IACdA,EAAgBI,EAChB5D,EAAKZ,MAAMkE,IAAK,aAAgBtD,EAAKZ,MAAMkE,IAAK,kBAI5DnE,KAAKuE,GAAG,QAAS,WACbnD,EAAEsD,KAAM7D,EAAKgC,aAAc,SAAUD,GAAkBA,EAAc+B,YAElE3E,MAIXiE,YAAa,WACTZ,EAAG,YAAauB,QAChB,IAAI7E,GAAUC,KAAKC,MAAMmE,UACzBpE,MAAK0C,QAAU,GAAIlD,GAAGqF,QACtB7E,KAAK8E,QAAU,GAAIrF,GAAYG,KAAMI,MAAQ+E,OAAQhF,EAAQgF,SAC7D/E,KAAK+C,QAAU,GAAIxD,GAAQK,MACvBS,KAAkBN,EAAQM,KAC1B2E,MAAkBjF,EAAQiF,MAC1B5E,IAAkBL,EAAQK,IAC1B6E,WAAkBlF,EAAQkF,WAC1BC,QAAkBnF,EAAQmF,QAC1BC,YAAkBpF,EAAQoF,YAC1BC,UAAkBrF,EAAQqF,UAC1BC,eAAkBtF,EAAQsF,iBAE9BrF,KAAK+C,QAAQuC,OAAQtF,KAAK0C,QAAQQ,KAClClD,KAAK+C,QAAQuC,OAAQtF,KAAK8E,QAAQ5B,KAClClD,KAAKkD,IAAIqC,QACTxF,EAAQgF,QAAU/E,KAAKkD,IAAIoC,OAAQtF,KAAK+C,QAAQG,KAChDnD,EAAQ2C,SAAW1C,KAAK0C,QAAQ/B,QAAU6E,YAAY,EAAMC,OAAQ1F,EAAQQ,eAAgBmC,QAAS3C,EAAQ2C,UAC7GR,OAAOC,KAAKC,MAAO,0BAA2B"}
+{"version":3,"file":"form-view.js","sources":["../../../src/mvc/form/form-view.js"],"names":["define","Utils","Portlet","Ui","FormSection","FormData","Backbone","View","extend","initialize","options","this","model","Model","initial_errors","cls","icon","always_refresh","message_status","set","setElement","render","update","new_model","self","data","matchModel","node","input_id","input","input_list","_","isEqual","field","field_list","new_options","indexOf","type","i","opt","length","push","label","value","trigger","Galaxy","emit","debug","wait","active","is_dynamic","unwait","highlight","message","silent","input_element","element_list","error","portlet","expand","$panel","$el","parents","filter","$","css","first","animate","scrollTop","offset","top","errors","error_messages","matchResponse","off","Manager","_renderForm","create","get","attributes","current_check","checksum","on","refresh_on_change","new_check","each","reset","remove","UnescapedMessage","section","inputs","title","operations","buttons","collapsible","collapsed","onchange_title","append","empty","persistent","status"],"mappings":"AAGAA,QAAU,cAAe,oBAAqB,iBAAkB,wBAAyB,sBACzF,SAAUC,EAAOC,EAASC,EAAIC,EAAaC,GACvC,MAAOC,UAASC,KAAKC,QACjBC,WAAY,SAAUC,GAClBC,KAAKC,MAAQ,GAAIN,UAASO,OACtBC,gBAAkB,EAClBC,IAAkB,qBAClBC,KAAkB,KAClBC,gBAAkB,EAClBC,eAAkB,YACnBC,IAAKT,GACRC,KAAKS,WAAY,UACjBT,KAAKU,UAITC,OAAQ,SAAUC,GACd,GAAIC,GAAOb,IACXA,MAAKc,KAAKC,WAAYH,EAAW,SAAUI,EAAMC,GAC7C,GAAIC,GAAQL,EAAKM,WAAYF,EAC7B,IAAKC,GAASA,EAAMnB,UACVqB,EAAEC,QAASH,EAAMnB,QAASiB,EAAKjB,SAAY,CAC7CmB,EAAMnB,QAAUiB,EAAKjB,OACrB,IAAIuB,GAAQT,EAAKU,WAAYN,EAC7B,IAAKK,EAAMX,OAAS,CAChB,GAAIa,KACJ,IAA8E,KAArE,OAAQ,kBAAmB,cAAiBC,QAASP,EAAMQ,MAChEF,EAAcN,EAAMnB,YAEpB,KAAM,GAAI4B,KAAKX,GAAKjB,QAAU,CAC1B,GAAI6B,GAAMZ,EAAKjB,QAAS4B,EACnBC,GAAIC,OAAS,GACdL,EAAYM,MAAQC,MAAOH,EAAK,GAAKI,MAAOJ,EAAK,KAI7DN,EAAMX,OAAQa,GACdF,EAAMW,QAAS,UACfC,OAAOC,KAAKC,MAAO,sBAAuB,wBAA0BnB,QAQxFoB,KAAM,SAAUC,GACZ,IAAM,GAAIX,KAAK3B,MAAKmB,WAAa,CAC7B,GAAIG,GAAQtB,KAAKuB,WAAYI,GACzBT,EAAQlB,KAAKmB,WAAYQ,EACxBT,GAAMqB,YAAcjB,EAAMe,MAAQf,EAAMkB,QACzClB,EAAOgB,EAAS,OAAS,cAMrCG,UAAW,SAAWxB,EAAUyB,EAASC,GACrC,GAAIC,GAAgB5C,KAAK6C,aAAc5B,EACvC,IAAK2B,IACDA,EAAcE,MAAOJ,GAAW,iCAChC1C,KAAK+C,QAAQC,SACbhD,KAAKiC,QAAS,SAAUhB,IAClB0B,GAAS,CACX,GAAIM,GAASjD,KAAKkD,IAAIC,UAAUC,OAAO,WACnC,MAAsE,KAA7D,OAAQ,UAAW3B,QAAS4B,EAAGrD,MAAOsD,IAAK,eACrDC,OACHN,GAAOO,SAAWC,UAAYR,EAAOQ,YAAcb,EAAcM,IAAIQ,SAASC,IAAM,KAAO,OAMvGC,OAAQ,SAAU7D,GAEd,GADAC,KAAKiC,QAAS,SACTlC,GAAWA,EAAQ6D,OAAS,CAC7B,GAAIC,GAAiB7D,KAAKc,KAAKgD,cAAe/D,EAAQ6D,OACtD,KAAM,GAAI3C,KAAYjB,MAAK6C,aAAe,CACtC,CAAY7C,KAAK6C,aAAc5B,GAC1B4C,EAAgB5C,IACjBjB,KAAKyC,UAAWxB,EAAU4C,EAAgB5C,IAAY,MAOtEP,OAAQ,WACJ,GAAIG,GAAOb,IACXA,MAAK+D,IAAI,UACT/D,KAAK+D,IAAI,SAET/D,KAAKuB,cAELvB,KAAKmB,cAELnB,KAAK6C,gBAEL7C,KAAKc,KAAO,GAAIpB,GAASsE,QAAShE,MAClCA,KAAKiE,cACLjE,KAAKc,KAAKoD,SACVlE,KAAKC,MAAMkE,IAAK,mBAAsBnE,KAAK4D,OAAQ5D,KAAKC,MAAMmE,WAE9D,IAAIC,GAAgBrE,KAAKc,KAAKwD,UAc9B,OAbAtE,MAAKuE,GAAI,SAAU,SAAUtD,GACzB,GAAIC,GAAQL,EAAKM,WAAYF,EAC7B,KAAMC,GAASA,EAAMsD,mBAAqB3D,EAAKZ,MAAMkE,IAAK,kBAAqB,CAC3E,GAAIM,GAAY5D,EAAKC,KAAKwD,UACrBG,IAAaJ,IACdA,EAAgBI,EAChB5D,EAAKZ,MAAMkE,IAAK,aAAgBtD,EAAKZ,MAAMkE,IAAK,kBAI5DnE,KAAKuE,GAAG,QAAS,WACbnD,EAAEsD,KAAM7D,EAAKgC,aAAc,SAAUD,GAAkBA,EAAc+B,YAElE3E,MAIXiE,YAAa,WACTZ,EAAG,YAAauB,QAChB,IAAI7E,GAAUC,KAAKC,MAAMmE,UACzBpE,MAAK0C,QAAU,GAAIlD,GAAGqF,iBACtB7E,KAAK8E,QAAU,GAAIrF,GAAYG,KAAMI,MAAQ+E,OAAQhF,EAAQgF,SAC7D/E,KAAK+C,QAAU,GAAIxD,GAAQK,MACvBS,KAAkBN,EAAQM,KAC1B2E,MAAkBjF,EAAQiF,MAC1B5E,IAAkBL,EAAQK,IAC1B6E,WAAkBlF,EAAQkF,WAC1BC,QAAkBnF,EAAQmF,QAC1BC,YAAkBpF,EAAQoF,YAC1BC,UAAkBrF,EAAQqF,UAC1BC,eAAkBtF,EAAQsF,iBAE9BrF,KAAK+C,QAAQuC,OAAQtF,KAAK0C,QAAQQ,KAClClD,KAAK+C,QAAQuC,OAAQtF,KAAK8E,QAAQ5B,KAClClD,KAAKkD,IAAIqC,QACTxF,EAAQgF,QAAU/E,KAAKkD,IAAIoC,OAAQtF,KAAK+C,QAAQG,KAChDnD,EAAQ2C,SAAW1C,KAAK0C,QAAQ/B,QAAU6E,YAAY,EAAMC,OAAQ1F,EAAQQ,eAAgBmC,QAAS3C,EAAQ2C,UAC7GR,OAAOC,KAAKC,MAAO,0BAA2B"}

static/maps/mvc/ui/ui-misc.js.map

@@ -1 +1,5 @@
-{"version":3,"file":"ui-misc.js","sources":["../../../src/mvc/ui/ui-misc.js"],"names":["define","Utils","Select","Slider","Options","Drilldown","Buttons","Modal","Label","Backbone","View","extend","tagName","initialize","options","this","model","Model","setElement","$","listenTo","render","title","new_title","set","value","get","$el","removeClass","addClass","html","Message","message","status","cls","persistent","fade","update","timeout","window","clearTimeout","self","setTimeout","fadeOut","Input","type","placeholder","disabled","readonly","visible","area","color","style","events","input","new_val","undefined","attr","id","css","datalist","isArray","length","autocomplete","source","request","response","change","_onchange","val","_","each","attr_name","removeAttr","Hidden","append","$info","$hidden","show","hide","Button","ButtonDefault","ButtonIcon","ButtonCheck","ButtonMenu","ButtonLink","RadioButton","Checkbox","Radio"],"mappings":"AAGAA,QAAQ,cACJ,2BACA,mBACA,oBACA,sBACA,oBACA,mBACA,SAAUC,EAAOC,EAAQC,EAAQC,EAASC,EAAWC,EAASC,GAG9D,GAAIC,GAAQC,SAASC,KAAKC,QACtBC,QAAS,QACTC,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,MAAOH,GAC7DC,KAAKH,QAAUE,EAAQF,SAAWG,KAAKH,QACvCG,KAAKG,WAAYC,EAAG,IAAMJ,KAAKH,QAAU,OACzCG,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETC,MAAO,SAAUC,GACbR,KAAKC,MAAMQ,IAAK,QAASD,IAE7BE,MAAO,WACH,MAAOV,MAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WAKJ,MAJAN,MAAKY,IAAIC,cACAC,SAAU,YACVA,SAAUd,KAAKC,MAAMU,IAAK,QAC1BI,KAAMf,KAAKC,MAAMU,IAAK,UACxBX,QAKXgB,EAAUtB,SAASC,KAAKC,QACxBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,OAClDe,QAAc,KACdC,OAAc,OACdC,IAAc,GACdC,YAAc,EACdC,MAAc,IACfZ,IAAKV,GACRC,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETgB,OAAQ,SAAUvB,GACdC,KAAKC,MAAMQ,IAAKV,IAEpBO,OAAQ,WACJN,KAAKY,IAAIC,cAAcC,SAAU,cAAeA,SAAUd,KAAKC,MAAMU,IAAK,OAC1E,IAAIO,GAASlB,KAAKC,MAAMU,IAAK,SAQ7B,IAPKX,KAAKC,MAAMU,IAAK,SACjBX,KAAKY,IAAIE,UAAsB,WAAVI,GAAuB,QACb,UAAVA,GAAsB,SACtBA,GAAW,gBAEhClB,KAAKY,IAAIE,SAAU,SAAUA,SAAU,SAAWI,GAEjDlB,KAAKC,MAAMU,IAAK,YAIjB,GAHAX,KAAKY,IAAIG,KAAMf,KAAKC,MAAMU,IAAK,YAC/BX,KAAKY,IAAKZ,KAAKC,MAAMU,IAAK,QAAW,SAAW,UAChDX,KAAKuB,SAAWC,OAAOC,aAAczB,KAAKuB,UACpCvB,KAAKC,MAAMU,IAAK,cAAiB,CACnC,GAAIe,GAAO1B,IACXA,MAAKuB,QAAUC,OAAOG,WAAY,WAC9BD,EAAKzB,MAAMQ,IAAK,UAAW,KAC5B,UAGPT,MAAKY,IAAIgB,SAEb,OAAO5B,SAKX6B,EAAQnC,SAASC,KAAKC,QACtBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,OAClD4B,KAAkB,OAClBC,YAAkB,GAClBC,UAAkB,EAClBC,UAAkB,EAClBC,SAAkB,EAClBf,IAAkB,GAClBgB,MAAkB,EAClBC,MAAkB,KAClBC,MAAkB,OACnB5B,IAAKV,GACRC,KAAKH,QAAUG,KAAKC,MAAMU,IAAK,QAAW,WAAa,QACvDX,KAAKG,WAAYC,EAAG,IAAMJ,KAAKH,QAAU,OACzCG,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETgC,QACIC,MAAS,aAEb7B,MAAO,SAAU8B,GAEb,MADYC,UAAZD,GAAyBxC,KAAKC,MAAMQ,IAAK,QAA4B,gBAAZ+B,GAAuBA,EAAU,IACnFxC,KAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WACJ,GAAIoB,GAAO1B,IACXA,MAAKY,IAAIC,cACAC,SAAU,MAAQd,KAAKH,SACvBiB,SAAUd,KAAKC,MAAMU,IAAK,QAC1BG,SAAUd,KAAKC,MAAMU,IAAK,UAC1B+B,KAAM,KAAM1C,KAAKC,MAAM0C,IACvBD,KAAM,OAAQ1C,KAAKC,MAAMU,IAAK,SAC9B+B,KAAM,cAAe1C,KAAKC,MAAMU,IAAK,gBACrCiC,IAAK,QAAS5C,KAAKC,MAAMU,IAAK,UAAa,IAC3CiC,IAAK,eAAgB5C,KAAKC,MAAMU,IAAK,UAAa,GAC3D,IAAIkC,GAAW7C,KAAKC,MAAMU,IAAK,WAY/B,OAXKP,GAAE0C,QAASD,IAAcA,EAASE,OAAS,GAC5C/C,KAAKY,IAAIoC,cAAgBC,OAAS,SAAUC,EAASC,GAAaA,EAAUzB,EAAKzB,MAAMU,IAAK,cACnEyC,OAAS,WAAa1B,EAAK2B,eAEnDrD,KAAKC,MAAMU,IAAK,WAAcX,KAAKY,IAAI0C,OACxCtD,KAAKY,IAAI0C,IAAKtD,KAAKC,MAAMU,IAAK,UAElC4C,EAAEC,MAAQ,WAAY,YAAc,SAAUC,GAC1C/B,EAAKzB,MAAMU,IAAK8C,GAAc/B,EAAKd,IAAI8B,KAAMe,GAAW,GAAS/B,EAAKd,IAAI8C,WAAYD,KAE1FzD,KAAKY,IAAKZ,KAAKC,MAAMU,IAAK,WAAc,OAAS,UAC1CX,MAEXqD,UAAW,WACPrD,KAAKU,MAAOV,KAAKY,IAAI0C,OACrBtD,KAAKC,MAAMU,IAAK,aAAgBX,KAAKC,MAAMU,IAAK,YAAcX,KAAKC,MAAMU,IAAK,aAKlFgD,EAASjE,SAASC,KAAKC,QACvBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,MAAOH,GAC7DC,KAAKG,WAAYC,EAAI,UAAWwD,OAAQ5D,KAAK6D,MAAQzD,EAAG,WACxBwD,OAAQ5D,KAAK8D,QAAU1D,EAAG,YAC1DJ,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETI,MAAO,SAAU8B,GAEb,MADYC,UAAZD,GAAyBxC,KAAKC,MAAMQ,IAAK,QAAS+B,GAC3CxC,KAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WAIJ,MAHAN,MAAKY,IAAI8B,KAAM,KAAM1C,KAAKC,MAAM0C,IAChC3C,KAAK8D,QAAQR,IAAKtD,KAAKC,MAAMU,IAAK,UAClCX,KAAKC,MAAMU,IAAK,QAAWX,KAAK6D,MAAME,OAAOhD,KAAMf,KAAKC,MAAMU,IAAK,SAAaX,KAAK6D,MAAMG,OACpFhE,OAIf,QACIiE,OAAc1E,EAAQ2E,cACtBC,WAAc5E,EAAQ4E,WACtBC,YAAc7E,EAAQ6E,YACtBC,WAAc9E,EAAQ8E,WACtBC,WAAc/E,EAAQ+E,WACtBzC,MAAcA,EACdpC,MAAcA,EACduB,QAAcA,EACdxB,MAAcA,EACd+E,YAAclF,EAAQkF,YACtBC,SAAcnF,EAAQmF,SACtBC,MAAcpF,EAAQoF,MACtBtF,OAAcA,EACdwE,OAAcA,EACdvE,OAAcA,EACdE,UAAcA"}
+<<<<<<< HEAD
+{"version":3,"file":"ui-misc.js","sources":["../../../src/mvc/ui/ui-misc.js"],"names":["define","Utils","Select","Slider","Options","Drilldown","Buttons","Modal","Label","Backbone","View","extend","tagName","initialize","options","this","model","Model","setElement","$","listenTo","render","title","new_title","set","value","get","$el","removeClass","addClass","html","Message","message","status","cls","persistent","fade","update","timeout","window","clearTimeout","self","setTimeout","fadeOut","Input","type","placeholder","disabled","readonly","visible","area","color","style","events","input","new_val","undefined","attr","id","css","datalist","isArray","length","autocomplete","source","request","response","change","_onchange","val","_","each","attr_name","removeAttr","Hidden","append","$info","$hidden","show","hide","Button","ButtonDefault","ButtonIcon","ButtonCheck","ButtonMenu","ButtonLink","RadioButton","Checkbox","Radio"],"mappings":"AAGAA,QAAQ,cACJ,2BACA,mBACA,oBACA,sBACA,oBACA,mBACA,SAAUC,EAAOC,EAAQC,EAAQC,EAASC,EAAWC,EAASC,GAG9D,GAAIC,GAAQC,SAASC,KAAKC,QACtBC,QAAS,QACTC,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,MAAOH,GAC7DC,KAAKH,QAAUE,EAAQF,SAAWG,KAAKH,QACvCG,KAAKG,WAAYC,EAAG,IAAMJ,KAAKH,QAAU,OACzCG,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETC,MAAO,SAAUC,GACbR,KAAKC,MAAMQ,IAAK,QAASD,IAE7BE,MAAO,WACH,MAAOV,MAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WAKJ,MAJAN,MAAKY,IAAIC,cACAC,SAAU,YACVA,SAAUd,KAAKC,MAAMU,IAAK,QAC1BI,KAAMf,KAAKC,MAAMU,IAAK,UACxBX,QAKXgB,EAAUtB,SAASC,KAAKC,QACxBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,OAClDe,QAAc,KACdC,OAAc,OACdC,IAAc,GACdC,YAAc,EACdC,MAAc,IACfZ,IAAKV,GACRC,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETgB,OAAQ,SAAUvB,GACdC,KAAKC,MAAMQ,IAAKV,IAEpBO,OAAQ,WACJN,KAAKY,IAAIC,cAAcC,SAAU,cAAeA,SAAUd,KAAKC,MAAMU,IAAK,OAC1E,IAAIO,GAASlB,KAAKC,MAAMU,IAAK,SAQ7B,IAPKX,KAAKC,MAAMU,IAAK,SACjBX,KAAKY,IAAIE,UAAsB,WAAVI,GAAuB,QACb,UAAVA,GAAsB,SACtBA,GAAW,gBAEhClB,KAAKY,IAAIE,SAAU,SAAUA,SAAU,SAAWI,GAEjDlB,KAAKC,MAAMU,IAAK,YAIjB,GAHAX,KAAKY,IAAIG,KAAMf,KAAKC,MAAMU,IAAK,YAC/BX,KAAKY,IAAKZ,KAAKC,MAAMU,IAAK,QAAW,SAAW,UAChDX,KAAKuB,SAAWC,OAAOC,aAAczB,KAAKuB,UACpCvB,KAAKC,MAAMU,IAAK,cAAiB,CACnC,GAAIe,GAAO1B,IACXA,MAAKuB,QAAUC,OAAOG,WAAY,WAC9BD,EAAKzB,MAAMQ,IAAK,UAAW,KAC5B,UAGPT,MAAKY,IAAIgB,SAEb,OAAO5B,SAKX6B,EAAQnC,SAASC,KAAKC,QACtBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,OAClD4B,KAAkB,OAClBC,YAAkB,GAClBC,UAAkB,EAClBC,UAAkB,EAClBC,SAAkB,EAClBf,IAAkB,GAClBgB,MAAkB,EAClBC,MAAkB,KAClBC,MAAkB,OACnB5B,IAAKV,GACRC,KAAKH,QAAUG,KAAKC,MAAMU,IAAK,QAAW,WAAa,QACvDX,KAAKG,WAAYC,EAAG,IAAMJ,KAAKH,QAAU,OACzCG,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETgC,QACIC,MAAS,aAEb7B,MAAO,SAAU8B,GAEb,MADYC,UAAZD,GAAyBxC,KAAKC,MAAMQ,IAAK,QAA4B,gBAAZ+B,GAAuBA,EAAU,IACnFxC,KAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WACJ,GAAIoB,GAAO1B,IACXA,MAAKY,IAAIC,cACAC,SAAU,MAAQd,KAAKH,SACvBiB,SAAUd,KAAKC,MAAMU,IAAK,QAC1BG,SAAUd,KAAKC,MAAMU,IAAK,UAC1B+B,KAAM,KAAM1C,KAAKC,MAAM0C,IACvBD,KAAM,OAAQ1C,KAAKC,MAAMU,IAAK,SAC9B+B,KAAM,cAAe1C,KAAKC,MAAMU,IAAK,gBACrCiC,IAAK,QAAS5C,KAAKC,MAAMU,IAAK,UAAa,IAC3CiC,IAAK,eAAgB5C,KAAKC,MAAMU,IAAK,UAAa,GAC3D,IAAIkC,GAAW7C,KAAKC,MAAMU,IAAK,WAY/B,OAXKP,GAAE0C,QAASD,IAAcA,EAASE,OAAS,GAC5C/C,KAAKY,IAAIoC,cAAgBC,OAAS,SAAUC,EAASC,GAAaA,EAAUzB,EAAKzB,MAAMU,IAAK,cACnEyC,OAAS,WAAa1B,EAAK2B,eAEnDrD,KAAKC,MAAMU,IAAK,WAAcX,KAAKY,IAAI0C,OACxCtD,KAAKY,IAAI0C,IAAKtD,KAAKC,MAAMU,IAAK,UAElC4C,EAAEC,MAAQ,WAAY,YAAc,SAAUC,GAC1C/B,EAAKzB,MAAMU,IAAK8C,GAAc/B,EAAKd,IAAI8B,KAAMe,GAAW,GAAS/B,EAAKd,IAAI8C,WAAYD,KAE1FzD,KAAKY,IAAKZ,KAAKC,MAAMU,IAAK,WAAc,OAAS,UAC1CX,MAEXqD,UAAW,WACPrD,KAAKU,MAAOV,KAAKY,IAAI0C,OACrBtD,KAAKC,MAAMU,IAAK,aAAgBX,KAAKC,MAAMU,IAAK,YAAcX,KAAKC,MAAMU,IAAK,aAKlFgD,EAASjE,SAASC,KAAKC,QACvBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,MAAOH,GAC7DC,KAAKG,WAAYC,EAAI,UAAWwD,OAAQ5D,KAAK6D,MAAQzD,EAAG,WACxBwD,OAAQ5D,KAAK8D,QAAU1D,EAAG,YAC1DJ,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETI,MAAO,SAAU8B,GAEb,MADYC,UAAZD,GAAyBxC,KAAKC,MAAMQ,IAAK,QAAS+B,GAC3CxC,KAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WAIJ,MAHAN,MAAKY,IAAI8B,KAAM,KAAM1C,KAAKC,MAAM0C,IAChC3C,KAAK8D,QAAQR,IAAKtD,KAAKC,MAAMU,IAAK,UAClCX,KAAKC,MAAMU,IAAK,QAAWX,KAAK6D,MAAME,OAAOhD,KAAMf,KAAKC,MAAMU,IAAK,SAAaX,KAAK6D,MAAMG,OACpFhE,OAIf,QACIiE,OAAc1E,EAAQ2E,cACtBC,WAAc5E,EAAQ4E,WACtBC,YAAc7E,EAAQ6E,YACtBC,WAAc9E,EAAQ8E,WACtBC,WAAc/E,EAAQ+E,WACtBzC,MAAcA,EACdpC,MAAcA,EACduB,QAAcA,EACdxB,MAAcA,EACd+E,YAAclF,EAAQkF,YACtBC,SAAcnF,EAAQmF,SACtBC,MAAcpF,EAAQoF,MACtBtF,OAAcA,EACdwE,OAAcA,EACdvE,OAAcA,EACdE,UAAcA"}
+=======
+{"version":3,"file":"ui-misc.js","sources":["../../../src/mvc/ui/ui-misc.js"],"names":["define","Utils","Select","Slider","Options","Drilldown","Buttons","Modal","Label","Backbone","View","extend","tagName","initialize","options","this","model","Model","setElement","$","listenTo","render","title","new_title","set","value","get","$el","removeClass","addClass","html","Message","message","status","cls","persistent","fade","update","messageForDisplay","timeout","window","clearTimeout","self","setTimeout","fadeOut","_","escape","UnescapedMessage","Input","type","placeholder","disabled","readonly","visible","area","color","style","events","input","new_val","undefined","attr","id","css","val","each","attr_name","removeAttr","_onchange","Hidden","append","$info","$hidden","show","hide","Button","ButtonDefault","ButtonIcon","ButtonCheck","ButtonMenu","ButtonLink","RadioButton","Checkbox","Radio"],"mappings":"AAGAA,QAAQ,cACJ,2BACA,mBACA,oBACA,sBACA,oBACA,mBACA,SAAUC,EAAOC,EAAQC,EAAQC,EAASC,EAAWC,EAASC,GAG9D,GAAIC,GAAQC,SAASC,KAAKC,QACtBC,QAAS,QACTC,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,MAAOH,GAC7DC,KAAKH,QAAUE,EAAQF,SAAWG,KAAKH,QACvCG,KAAKG,WAAYC,EAAG,IAAMJ,KAAKH,QAAU,OACzCG,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETC,MAAO,SAAUC,GACbR,KAAKC,MAAMQ,IAAK,QAASD,IAE7BE,MAAO,WACH,MAAOV,MAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WAKJ,MAJAN,MAAKY,IAAIC,cACAC,SAAU,YACVA,SAAUd,KAAKC,MAAMU,IAAK,QAC1BI,KAAMf,KAAKC,MAAMU,IAAK,UACxBX,QAKXgB,EAAUtB,SAASC,KAAKC,QACxBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,OAClDe,QAAc,KACdC,OAAc,OACdC,IAAc,GACdC,YAAc,EACdC,MAAc,IACfZ,IAAKV,GACRC,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETgB,OAAQ,SAAUvB,GACdC,KAAKC,MAAMQ,IAAKV,IAEpBO,OAAQ,WACJN,KAAKY,IAAIC,cAAcC,SAAU,cAAeA,SAAUd,KAAKC,MAAMU,IAAK,OAC1E,IAAIO,GAASlB,KAAKC,MAAMU,IAAK,SAQ7B,IAPKX,KAAKC,MAAMU,IAAK,SACjBX,KAAKY,IAAIE,UAAsB,WAAVI,GAAuB,QACb,UAAVA,GAAsB,SACtBA,GAAW,gBAEhClB,KAAKY,IAAIE,SAAU,SAAUA,SAAU,SAAWI,GAEjDlB,KAAKC,MAAMU,IAAK,YAIjB,GAHAX,KAAKY,IAAIG,KAAMf,KAAKuB,qBACpBvB,KAAKY,IAAKZ,KAAKC,MAAMU,IAAK,QAAW,SAAW,UAChDX,KAAKwB,SAAWC,OAAOC,aAAc1B,KAAKwB,UACpCxB,KAAKC,MAAMU,IAAK,cAAiB,CACnC,GAAIgB,GAAO3B,IACXA,MAAKwB,QAAUC,OAAOG,WAAY,WAC9BD,EAAK1B,MAAMQ,IAAK,UAAW,KAC5B,UAGPT,MAAKY,IAAIiB,SAEb,OAAO7B,OAEXuB,kBAAmB,WACf,MAAOO,GAAEC,OAAQ/B,KAAKC,MAAMU,IAAK,eAIrCqB,EAAmBhB,EAAQpB,QAC3B2B,kBAAmB,WACf,MAAOvB,MAAKC,MAAMU,IAAK,cAK3BsB,EAAQvC,SAASC,KAAKC,QACtBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,OAClDgC,KAAkB,OAClBC,YAAkB,GAClBC,UAAkB,EAClBC,UAAkB,EAClBC,SAAkB,EAClBnB,IAAkB,GAClBoB,MAAkB,EAClBC,MAAkB,KAClBC,MAAkB,OACnBhC,IAAKV,GACRC,KAAKH,QAAUG,KAAKC,MAAMU,IAAK,QAAW,WAAa,QACvDX,KAAKG,WAAYC,EAAG,IAAMJ,KAAKH,QAAU,OACzCG,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAEToC,QACIC,MAAS,aAEbjC,MAAO,SAAUkC,GAEb,MADYC,UAAZD,GAAyB5C,KAAKC,MAAMQ,IAAK,QAA4B,gBAAZmC,GAAuBA,EAAU,IACnF5C,KAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WACJ,GAAIqB,GAAO3B,IAiBX,OAhBAA,MAAKY,IAAIC,cACAC,SAAU,MAAQd,KAAKH,SACvBiB,SAAUd,KAAKC,MAAMU,IAAK,QAC1BG,SAAUd,KAAKC,MAAMU,IAAK,UAC1BmC,KAAM,KAAM9C,KAAKC,MAAM8C,IACvBD,KAAM,OAAQ9C,KAAKC,MAAMU,IAAK,SAC9BmC,KAAM,cAAe9C,KAAKC,MAAMU,IAAK,gBACrCqC,IAAK,QAAShD,KAAKC,MAAMU,IAAK,UAAa,IAC3CqC,IAAK,eAAgBhD,KAAKC,MAAMU,IAAK,UAAa,IACtDX,KAAKC,MAAMU,IAAK,WAAcX,KAAKY,IAAIqC,OACxCjD,KAAKY,IAAIqC,IAAKjD,KAAKC,MAAMU,IAAK,UAElCmB,EAAEoB,MAAQ,WAAY,YAAc,SAAUC,GAC1CxB,EAAK1B,MAAMU,IAAKwC,GAAcxB,EAAKf,IAAIkC,KAAMK,GAAW,GAASxB,EAAKf,IAAIwC,WAAYD,KAE1FnD,KAAKY,IAAKZ,KAAKC,MAAMU,IAAK,WAAc,OAAS,UAC1CX,MAEXqD,UAAW,WACPrD,KAAKU,MAAOV,KAAKY,IAAIqC,OACrBjD,KAAKC,MAAMU,IAAK,aAAgBX,KAAKC,MAAMU,IAAK,YAAcX,KAAKC,MAAMU,IAAK,aAKlF2C,EAAS5D,SAASC,KAAKC,QACvBE,WAAY,SAAUC,GAClBC,KAAKC,MAAQF,GAAWA,EAAQE,OAAS,GAAIP,UAASQ,MAAOH,GAC7DC,KAAKG,WAAYC,EAAI,UAAWmD,OAAQvD,KAAKwD,MAAQpD,EAAG,WACxBmD,OAAQvD,KAAKyD,QAAUrD,EAAG,YAC1DJ,KAAKK,SAAUL,KAAKC,MAAO,SAAUD,KAAKM,OAAQN,MAClDA,KAAKM,UAETI,MAAO,SAAUkC,GAEb,MADYC,UAAZD,GAAyB5C,KAAKC,MAAMQ,IAAK,QAASmC,GAC3C5C,KAAKC,MAAMU,IAAK,UAE3BL,OAAQ,WAIJ,MAHAN,MAAKY,IAAIkC,KAAM,KAAM9C,KAAKC,MAAM8C,IAChC/C,KAAKyD,QAAQR,IAAKjD,KAAKC,MAAMU,IAAK,UAClCX,KAAKC,MAAMU,IAAK,QAAWX,KAAKwD,MAAME,OAAO3C,KAAMf,KAAKC,MAAMU,IAAK,SAAaX,KAAKwD,MAAMG,OACpF3D,OAIf,QACI4D,OAAmBrE,EAAQsE,cAC3BC,WAAmBvE,EAAQuE,WAC3BC,YAAmBxE,EAAQwE,YAC3BC,WAAmBzE,EAAQyE,WAC3BC,WAAmB1E,EAAQ0E,WAC3BhC,MAAmBA,EACnBxC,MAAmBA,EACnBuB,QAAmBA,EACnBgB,iBAAmBA,EACnBxC,MAAmBA,EACnB0E,YAAmB7E,EAAQ6E,YAC3BC,SAAmB9E,EAAQ8E,SAC3BC,MAAmB/E,EAAQ+E,MAC3BjF,OAAmBA,EACnBmE,OAAmBA,EACnBlE,OAAmBA,EACnBE,UAAmBA"}
+>>>>>>> dev