From 98e1903a481170744d620326a60b9426ab792fa4 Mon Sep 17 00:00:00 2001 From: roalva1 Date: Wed, 25 Feb 2015 17:25:32 +0100 Subject: [PATCH 01/21] Restore function necessary for splitting. --- lib/galaxy/datatypes/sequence.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 98530928512..043d8d25cff 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -203,7 +203,26 @@ class Sequence( data.Text ): return None raise NotImplementedError("Can't split generic sequence files") + def get_split_commands_sequential(is_compressed, input_name, output_name, start_sequence, sequence_count): + """ + Does a brain-dead sequential scan & extract of certain sequences + >>> Sequence.get_split_commands_sequential(True, './input.gz', './output.gz', start_sequence=0, sequence_count=10) + ['zcat "./input.gz" | ( tail -n +1 2> /dev/null) | head -40 | gzip -c > "./output.gz"'] + >>> Sequence.get_split_commands_sequential(False, './input.fastq', './output.fastq', start_sequence=10, sequence_count=10) + ['tail -n +41 "./input.fastq" 2> /dev/null | head -40 > "./output.fastq"'] + """ + start_line = start_sequence * 4 + line_count = sequence_count * 4 + # TODO: verify that tail can handle 64-bit numbers + if is_compressed: + cmd = 'zcat "%s" | ( tail -n +%s 2> /dev/null) | head -%s | gzip -c' % (input_name, start_line+1, line_count) + else: + cmd = 'tail -n +%s "%s" 2> /dev/null | head -%s' % (start_line+1, input_name, line_count) + cmd += ' > "%s"' % output_name + return [cmd] + get_split_commands_sequential = staticmethod(get_split_commands_sequential) + class Alignment( data.Text ): """Class describing an alignment""" From cc64fd7762f6890e739a48061afe43be8f6c8c5c Mon Sep 17 00:00:00 2001 From: roalva1 Date: Thu, 26 Feb 2015 10:00:12 +0100 Subject: [PATCH 02/21] added get_split_commands_with_toc method --- lib/galaxy/datatypes/sequence.py | 71 +++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 043d8d25cff..c02fe5afb8d 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -203,6 +203,75 @@ class Sequence( data.Text ): return None raise NotImplementedError("Can't split generic sequence files") + def get_split_commands_with_toc(input_name, output_name, toc_file, start_sequence, sequence_count): + """ + Uses a Table of Contents dict, parsed from an FQTOC file, to come up with a set of + shell commands that will extract the parts necessary + >>> three_sections=[dict(start=0, end=74, sequences=10), dict(start=74, end=148, sequences=10), dict(start=148, end=148+76, sequences=10)] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=0, sequence_count=10) + ['dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=1, sequence_count=5) + ['(dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +5 2> /dev/null) | head -20 | gzip -c >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=0, sequence_count=20) + ['dd bs=1 skip=0 count=148 if=./input.gz 2> /dev/null >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=5, sequence_count=10) + ['(dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +21 2> /dev/null) | head -20 | gzip -c >> ./output.gz', '(dd bs=1 skip=74 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +1 2> /dev/null) | head -20 | gzip -c >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=10, sequence_count=10) + ['dd bs=1 skip=74 count=74 if=./input.gz 2> /dev/null >> ./output.gz'] + >>> Sequence.get_split_commands_with_toc('./input.gz', './output.gz', dict(sections=three_sections), start_sequence=5, sequence_count=20) + ['(dd bs=1 skip=0 count=74 if=./input.gz 2> /dev/null )| zcat | ( tail -n +21 2> /dev/null) | head -20 | gzip -c >> ./output.gz', 'dd bs=1 skip=74 count=74 if=./input.gz 2> /dev/null >> ./output.gz', '(dd bs=1 skip=148 count=76 if=./input.gz 2> /dev/null )| zcat | ( tail -n +1 2> /dev/null) | head -20 | gzip -c >> ./output.gz'] + """ + sections = toc_file['sections'] + result = [] + + current_sequence = long(0) + i=0 + # skip to the section that contains my starting sequence + while i < len(sections) and start_sequence >= current_sequence + long(sections[i]['sequences']): + current_sequence += long(sections[i]['sequences']) + i += 1 + if i == len(sections): # bad input data! + raise Exception('No FQTOC section contains starting sequence %s' % start_sequence) + + # These two variables act as an accumulator for consecutive entire blocks that + # can be copied verbatim (without decompressing) + start_chunk = long(-1) + end_chunk = long(-1) + copy_chunk_cmd = 'dd bs=1 skip=%s count=%s if=%s 2> /dev/null >> %s' + + while sequence_count > 0 and i < len(sections): + # we need to extract partial data. So, find the byte offsets of the chunks that contain the data we need + # use a combination of dd (to pull just the right sections out) tail (to skip lines) and head (to get the + # right number of lines + sequences = long(sections[i]['sequences']) + skip_sequences = start_sequence-current_sequence + sequences_to_extract = min(sequence_count, sequences-skip_sequences) + start_copy = long(sections[i]['start']) + end_copy = long(sections[i]['end']) + if sequences_to_extract < sequences: + if start_chunk > -1: + result.append(copy_chunk_cmd % (start_chunk, end_chunk-start_chunk, input_name, output_name)) + start_chunk = -1 + # extract, unzip, trim, recompress + result.append('(dd bs=1 skip=%s count=%s if=%s 2> /dev/null )| zcat | ( tail -n +%s 2> /dev/null) | head -%s | gzip -c >> %s' % + (start_copy, end_copy-start_copy, input_name, skip_sequences*4+1, sequences_to_extract*4, output_name)) + else: # whole section - add it to the start_chunk/end_chunk accumulator + if start_chunk == -1: + start_chunk = start_copy + end_chunk = end_copy + sequence_count -= sequences_to_extract + start_sequence += sequences_to_extract + current_sequence += sequences + i += 1 + if start_chunk > -1: + result.append(copy_chunk_cmd % (start_chunk, end_chunk-start_chunk, input_name, output_name)) + + if sequence_count > 0: + raise Exception('%s sequences not found in file' % sequence_count) + + return result + get_split_commands_with_toc = staticmethod(get_split_commands_with_toc) + def get_split_commands_sequential(is_compressed, input_name, output_name, start_sequence, sequence_count): """ Does a brain-dead sequential scan & extract of certain sequences @@ -222,7 +291,7 @@ class Sequence( data.Text ): return [cmd] get_split_commands_sequential = staticmethod(get_split_commands_sequential) - + class Alignment( data.Text ): """Class describing an alignment""" From 226e8265e4f9a7de9b83960af71e46313f81b650 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 27 Feb 2015 11:35:13 -0500 Subject: [PATCH 03/21] Add some pointers for developers and deployers in README.rst regarding the Github move. --- README.rst | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 66c10038352..54e5929e0b2 100644 --- a/README.rst +++ b/README.rst @@ -38,7 +38,7 @@ Not all dependencies are included for the tools provided in the sample https://wiki.galaxyproject.org/Admin/Tools/ToolDependencies Issues ------- +====== Issues can be submitted to trello via the `galaxyproject website `__ and viewed on the `Galaxy @@ -51,6 +51,19 @@ Galaxy Development Galaxy welcomes new development! There is extensive documentation on developing with Galaxy on the `wiki `__. +Source Repository +----------------- + +Galaxy development has moved to `Github +`__. + +**Developers** should develop against the ``dev`` branch in the Github +repository. + +**Deployers** can continue to pull Galaxy changes from Bitbucket without +interruption. Alternatively, deployers can track the ``master`` branch in +Github. + Syncing a Fork -------------- From ec549db3d7ad775ddd86789250ae90ed9cdead15 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Fri, 27 Feb 2015 13:29:05 -0500 Subject: [PATCH 04/21] use proper placeholder instead of input value to prevent various search glitches --- client/galaxy/scripts/templates/tool_search.handlebars | 4 ++-- static/scripts/templates/tool_search.handlebars | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/client/galaxy/scripts/templates/tool_search.handlebars b/client/galaxy/scripts/templates/tool_search.handlebars index bd035f2e92b..995b9ce4af4 100644 --- a/client/galaxy/scripts/templates/tool_search.handlebars +++ b/client/galaxy/scripts/templates/tool_search.handlebars @@ -1,3 +1,3 @@ - + - \ No newline at end of file + diff --git a/static/scripts/templates/tool_search.handlebars b/static/scripts/templates/tool_search.handlebars index bd035f2e92b..995b9ce4af4 100644 --- a/static/scripts/templates/tool_search.handlebars +++ b/static/scripts/templates/tool_search.handlebars @@ -1,3 +1,3 @@ - + - \ No newline at end of file + From f6afc9b2382cd3dd81fdfdec06824fd992f75db4 Mon Sep 17 00:00:00 2001 From: guerler Date: Fri, 27 Feb 2015 13:43:39 -0500 Subject: [PATCH 05/21] ToolForm: Make deep copy of current state in order to be on the safe side --- client/galaxy/scripts/mvc/tools/tools-form-base.js | 2 +- static/scripts/mvc/tools/tools-form-base.js | 2 +- static/scripts/packed/mvc/tools/tools-form-base.js | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/client/galaxy/scripts/mvc/tools/tools-form-base.js b/client/galaxy/scripts/mvc/tools/tools-form-base.js index 499994eca37..cec95b7c452 100644 --- a/client/galaxy/scripts/mvc/tools/tools-form-base.js +++ b/client/galaxy/scripts/mvc/tools/tools-form-base.js @@ -108,7 +108,7 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', var new_state = self.tree.finalize(); if (!_.isEqual(new_state, current_state)) { current_state = new_state; - self._updateModel($.extend({}, current_state)); + self._updateModel($.extend(true, {}, current_state)); } }); }); diff --git a/static/scripts/mvc/tools/tools-form-base.js b/static/scripts/mvc/tools/tools-form-base.js index 499994eca37..cec95b7c452 100644 --- a/static/scripts/mvc/tools/tools-form-base.js +++ b/static/scripts/mvc/tools/tools-form-base.js @@ -108,7 +108,7 @@ define(['utils/utils', 'utils/deferred', 'mvc/ui/ui-portlet', 'mvc/ui/ui-misc', var new_state = self.tree.finalize(); if (!_.isEqual(new_state, current_state)) { current_state = new_state; - self._updateModel($.extend({}, current_state)); + self._updateModel($.extend(true, {}, current_state)); } }); }); diff --git a/static/scripts/packed/mvc/tools/tools-form-base.js b/static/scripts/packed/mvc/tools/tools-form-base.js index 27b7cff032e..290a776da81 100644 --- a/static/scripts/packed/mvc/tools/tools-form-base.js +++ b/static/scripts/packed/mvc/tools/tools-form-base.js @@ -1 +1 @@ -define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-content","mvc/tools/tools-section","mvc/tools/tools-tree"],function(g,h,f,k,i,a,d,c,e,j,b){return Backbone.View.extend({initialize:function(l){this.optionsDefault={is_dynamic:true,narrow:false,initial_errors:false,cls_portlet:"ui-portlet-limited"};this.options=g.merge(l,this.optionsDefault);console.debug(this.options);var m=parent.Galaxy;if(m&&m.modal){this.modal=m.modal}else{this.modal=new k.Modal.View()}if(m&&m.currUser){this.is_admin=m.currUser.get("is_admin")}else{this.is_admin=false}this.container=this.options.container||"body";this.deferred=new h();this.setElement("
");$(this.container).append(this.$el);this.build(this.options)},build:function(n){var l=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.content=new e(this);l.options.inputs=n&&n.inputs;this._renderForm(n);var m=this.tree.finalize();if(n.initial_errors){this._errors(n)}this.on("refresh",function(){l.deferred.reset();l.deferred.execute(function(){var o=l.tree.finalize();if(!_.isEqual(o,m)){m=o;l._updateModel($.extend({},m))}})});this.on("reset",function(){for(var o in this.element_list){this.element_list[o].reset()}})},reciept:function(l){$(this.container).empty();$(this.container).append(l)},highlight:function(m,n,l){var o=this.element_list[m];if(o){o.error(n||"Please verify this parameter.");if(!l){$("html, body").animate({scrollTop:o.$el.offset().top-20},500)}}},_errors:function(n){this.trigger("reset");if(n&&n.errors){var o=this.tree.matchResponse(n.errors);for(var m in this.element_list){var l=this.element_list[m];if(o[m]){this.highlight(m,o[m],true)}}}},_renderForm:function(t){var s=this;this.message=new k.Message();var m=new k.ButtonMenu({icon:"fa-cubes",title:(!t.narrow&&"Versions")||null,tooltip:"Select another tool version"});if(t.versions&&t.versions.length>1){for(var o in t.versions){var q=t.versions[o];if(q!=t.version){m.addMenu({title:"Switch to "+q,version:q,icon:"fa-cube",onclick:function(){s.options.id=s.options.id.replace(s.options.version,this.version);s.options.version=this.version;s.deferred.reset();s.deferred.execute(function(){s._buildModel()})}})}}}else{m.$el.hide()}var p=new k.ButtonMenu({icon:"fa-caret-down",title:(!t.narrow&&"Options")||null,tooltip:"View available options"});if(t.biostar_url){p.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(t.biostar_url+"/p/new/post/")}});p.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(t.biostar_url+"/t/"+t.id+"/")}})}p.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+t.id)}});if(this.is_admin){p.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+t.id+"/download"}})}if(t.requirements&&t.requirements.length>0){p.addMenu({icon:"fa-info-circle",title:"Requirements",tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;s.message.update({persistent:true,message:c.requirements(t),status:"info"})}else{this.visible=false;s.message.update({message:""})}}})}if(this.options.sharable_url){p.addMenu({icon:"fa-external-link",title:"See in Tool Shed",tooltip:"Access the repository",onclick:function(){window.open(s.options.sharable_url)}})}this.section=new j.View(s,{inputs:t.inputs});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new f.View({icon:"fa-wrench",title:""+t.name+" "+t.description+" (Galaxy Tool Version "+t.version+")",cls:this.options.cls_portlet,operations:{menu:p,versions:m},buttons:this.buttons});this.portlet.append(this.message.$el.addClass("ui-margin-top"));this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(t.help!=""){this.$el.append(c.help(t.help))}if(t.citations){var r=$("
");var l=new i.ToolCitationCollection();l.tool_id=t.id;var n=new a.CitationListView({el:r,collection:l});n.render();l.fetch();this.$el.append(r)}if(t.message){this.message.update({persistent:true,status:"warning",message:t.message})}console.debug("tools-form-base::initialize() - Completed.")}})}); \ No newline at end of file +define(["utils/utils","utils/deferred","mvc/ui/ui-portlet","mvc/ui/ui-misc","mvc/citation/citation-model","mvc/citation/citation-view","mvc/tools","mvc/tools/tools-template","mvc/tools/tools-content","mvc/tools/tools-section","mvc/tools/tools-tree"],function(g,h,f,k,i,a,d,c,e,j,b){return Backbone.View.extend({initialize:function(l){this.optionsDefault={is_dynamic:true,narrow:false,initial_errors:false,cls_portlet:"ui-portlet-limited"};this.options=g.merge(l,this.optionsDefault);console.debug(this.options);var m=parent.Galaxy;if(m&&m.modal){this.modal=m.modal}else{this.modal=new k.Modal.View()}if(m&&m.currUser){this.is_admin=m.currUser.get("is_admin")}else{this.is_admin=false}this.container=this.options.container||"body";this.deferred=new h();this.setElement("
");$(this.container).append(this.$el);this.build(this.options)},build:function(n){var l=this;this.off("refresh");this.off("reset");this.field_list={};this.input_list={};this.element_list={};this.tree=new b(this);this.content=new e(this);l.options.inputs=n&&n.inputs;this._renderForm(n);var m=this.tree.finalize();if(n.initial_errors){this._errors(n)}this.on("refresh",function(){l.deferred.reset();l.deferred.execute(function(){var o=l.tree.finalize();if(!_.isEqual(o,m)){m=o;l._updateModel($.extend(true,{},m))}})});this.on("reset",function(){for(var o in this.element_list){this.element_list[o].reset()}})},reciept:function(l){$(this.container).empty();$(this.container).append(l)},highlight:function(m,n,l){var o=this.element_list[m];if(o){o.error(n||"Please verify this parameter.");if(!l){$("html, body").animate({scrollTop:o.$el.offset().top-20},500)}}},_errors:function(n){this.trigger("reset");if(n&&n.errors){var o=this.tree.matchResponse(n.errors);for(var m in this.element_list){var l=this.element_list[m];if(o[m]){this.highlight(m,o[m],true)}}}},_renderForm:function(t){var s=this;this.message=new k.Message();var m=new k.ButtonMenu({icon:"fa-cubes",title:(!t.narrow&&"Versions")||null,tooltip:"Select another tool version"});if(t.versions&&t.versions.length>1){for(var o in t.versions){var q=t.versions[o];if(q!=t.version){m.addMenu({title:"Switch to "+q,version:q,icon:"fa-cube",onclick:function(){s.options.id=s.options.id.replace(s.options.version,this.version);s.options.version=this.version;s.deferred.reset();s.deferred.execute(function(){s._buildModel()})}})}}}else{m.$el.hide()}var p=new k.ButtonMenu({icon:"fa-caret-down",title:(!t.narrow&&"Options")||null,tooltip:"View available options"});if(t.biostar_url){p.addMenu({icon:"fa-question-circle",title:"Question?",tooltip:"Ask a question about this tool (Biostar)",onclick:function(){window.open(t.biostar_url+"/p/new/post/")}});p.addMenu({icon:"fa-search",title:"Search",tooltip:"Search help for this tool (Biostar)",onclick:function(){window.open(t.biostar_url+"/t/"+t.id+"/")}})}p.addMenu({icon:"fa-share",title:"Share",tooltip:"Share this tool",onclick:function(){prompt("Copy to clipboard: Ctrl+C, Enter",window.location.origin+galaxy_config.root+"root?tool_id="+t.id)}});if(this.is_admin){p.addMenu({icon:"fa-download",title:"Download",tooltip:"Download this tool",onclick:function(){window.location.href=galaxy_config.root+"api/tools/"+t.id+"/download"}})}if(t.requirements&&t.requirements.length>0){p.addMenu({icon:"fa-info-circle",title:"Requirements",tooltip:"Display tool requirements",onclick:function(){if(!this.visible){this.visible=true;s.message.update({persistent:true,message:c.requirements(t),status:"info"})}else{this.visible=false;s.message.update({message:""})}}})}if(this.options.sharable_url){p.addMenu({icon:"fa-external-link",title:"See in Tool Shed",tooltip:"Access the repository",onclick:function(){window.open(s.options.sharable_url)}})}this.section=new j.View(s,{inputs:t.inputs});if(this.incompatible){this.$el.hide();$("#tool-form-classic").show();return}this.portlet=new f.View({icon:"fa-wrench",title:""+t.name+" "+t.description+" (Galaxy Tool Version "+t.version+")",cls:this.options.cls_portlet,operations:{menu:p,versions:m},buttons:this.buttons});this.portlet.append(this.message.$el.addClass("ui-margin-top"));this.portlet.append(this.section.$el);this.$el.empty();this.$el.append(this.portlet.$el);if(t.help!=""){this.$el.append(c.help(t.help))}if(t.citations){var r=$("
");var l=new i.ToolCitationCollection();l.tool_id=t.id;var n=new a.CitationListView({el:r,collection:l});n.render();l.fetch();this.$el.append(r)}if(t.message){this.message.update({persistent:true,status:"warning",message:t.message})}console.debug("tools-form-base::initialize() - Completed.")}})}); \ No newline at end of file From 002fef995a7a96627831943e3aa0de0bb02aa805 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Fri, 27 Feb 2015 14:21:22 -0500 Subject: [PATCH 06/21] Fix a typo in datatypes_conf.xml.sample --- config/datatypes_conf.xml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 51a75abd7d4..65bfb9ebdac 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -180,7 +180,7 @@ - + From f73467f19d35c126fca4034c97baeee10490fae1 Mon Sep 17 00:00:00 2001 From: dannon Date: Thu, 26 Feb 2015 19:59:26 -0500 Subject: [PATCH 07/21] Fix server_starttime being in app for reports, toolshed. I will be refactoring this shortly to have a base UniverseApplication from which things inherit. --- lib/galaxy/webapps/reports/app.py | 6 +++++- lib/galaxy/webapps/tool_shed/app.py | 13 ++++++++----- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/webapps/reports/app.py b/lib/galaxy/webapps/reports/app.py index b545aa687ad..8b362adb05c 100644 --- a/lib/galaxy/webapps/reports/app.py +++ b/lib/galaxy/webapps/reports/app.py @@ -1,5 +1,7 @@ -import sys import config +import sys +import time + import galaxy.model from galaxy.web import security @@ -26,6 +28,8 @@ class UniverseApplication( object ): self.targets_mysql = 'mysql' in self.config.database_connection # Security helper self.security = security.SecurityHelper( id_secret=self.config.id_secret ) + # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. + self.server_starttime = int(time.time()) def shutdown( self ): pass diff --git a/lib/galaxy/webapps/tool_shed/app.py b/lib/galaxy/webapps/tool_shed/app.py index ea736b0c9c3..fa61119c2f8 100644 --- a/lib/galaxy/webapps/tool_shed/app.py +++ b/lib/galaxy/webapps/tool_shed/app.py @@ -1,17 +1,18 @@ import config import sys -from galaxy import tools -import galaxy.tools.data -import galaxy.quota +import time import galaxy.datatypes.registry +import galaxy.quota +import galaxy.tools.data import galaxy.webapps.tool_shed.model +from galaxy import tools +from galaxy.managers.tags import CommunityTagManager from galaxy.openid.providers import OpenIDProviders from galaxy.util.dbkeys import GenomeBuilds from galaxy.web import security -from galaxy.managers.tags import CommunityTagManager -from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager import tool_shed.repository_registry import tool_shed.repository_types.registry +from tool_shed.grids.repository_grid_filter_manager import RepositoryGridFilterManager class UniverseApplication( object ): @@ -70,6 +71,8 @@ class UniverseApplication( object ): self.hgweb_config_manager.hgweb_config_dir = self.config.hgweb_config_dir # Initialize the repository registry. self.repository_registry = tool_shed.repository_registry.Registry( self ) + # used for cachebusting -- refactor this into a *SINGLE* UniverseApplication base. + self.server_starttime = int(time.time()) print >> sys.stderr, "Tool shed hgweb.config file is: ", self.hgweb_config_manager.hgweb_config def shutdown( self ): From 1ddafdc566c19fc42841e7745601214c43e307dd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 20:09:16 -0500 Subject: [PATCH 08/21] Reduce use of bare excepts in install_manager.py. --- lib/tool_shed/galaxy_install/install_manager.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index d955ed355e7..5c3bd7301d9 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -710,19 +710,19 @@ class InstallRepositoryManager( object ): installed_tool_shed_repositories = [] try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'has_repository_dependencies'." ) try: includes_tools = repository_revision_dict[ 'includes_tools' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tools'." ) try: includes_tool_dependencies = repository_revision_dict[ 'includes_tool_dependencies' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tool_dependencies'." ) try: includes_tools_for_display_in_tool_panel = repository_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] - except: + except KeyError: raise exceptions.InternalServerError( "Tool shed response missing required parameter 'includes_tools_for_display_in_tool_panel'." ) # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository information. install_repository_dependencies = install_options.get( 'install_repository_dependencies', False ) From 6b9c97b3c465e7994775da0f479b2c664fecbd09 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 20:14:09 -0500 Subject: [PATCH 09/21] Eliminate unused using_api parameter in install_manager. --- lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py | 2 +- lib/tool_shed/galaxy_install/install_manager.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 03d12c9deee..c12d12ef13b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1092,7 +1092,7 @@ class AdminToolshed( AdminGalaxy ): tool_path=tool_path, tool_shed_url=tool_shed_url ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - install_repository_manager.handle_tool_shed_repositories( installation_dict, using_api=False ) + install_repository_manager.handle_tool_shed_repositories( installation_dict ) if created_or_updated_tool_shed_repositories: installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, filtered_repo_info_dicts=filtered_repo_info_dicts, diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 5c3bd7301d9..6aca52963d4 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -623,7 +623,7 @@ class InstallRepositoryManager( object ): # Load proprietary datatype display applications self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - def handle_tool_shed_repositories( self, installation_dict, using_api=False ): + def handle_tool_shed_repositories( self, installation_dict ): # The following installation_dict entries are all required. install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] new_tool_panel_section_label = installation_dict[ 'new_tool_panel_section_label' ] @@ -759,7 +759,7 @@ class InstallRepositoryManager( object ): tool_shed_url=tool_shed_url ) # Create the tool_shed_repository database records and gather additional information for repository installation. created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts = \ - self.handle_tool_shed_repositories( installation_dict, using_api=True ) + self.handle_tool_shed_repositories( installation_dict ) if created_or_updated_tool_shed_repositories: # Build the dictionary of information necessary for installing the repositories. installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, From 95235d54c745856d0c2715f5f2f1298ea1da5fc1 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 20:14:32 -0500 Subject: [PATCH 10/21] Small refactoring of install_manager.py. Break tool dependency dir checking out into its own function to reduce complexity of __install_repositories. --- lib/tool_shed/galaxy_install/install_manager.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 6aca52963d4..26c84faecbd 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -728,11 +728,7 @@ class InstallRepositoryManager( object ): install_repository_dependencies = install_options.get( 'install_repository_dependencies', False ) install_tool_dependencies = install_options.get( 'install_tool_dependencies', False ) if install_tool_dependencies: - if self.app.config.tool_dependency_dir is None: - no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " - no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " - no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " - raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) + self.__assert_can_install_dependencies() new_tool_panel_section_label = install_options.get( 'new_tool_panel_section_label', '' ) shed_tool_conf = install_options.get( 'shed_tool_conf', None ) if shed_tool_conf: @@ -957,6 +953,13 @@ class InstallRepositoryManager( object ): self.install_model.context.add( tool_shed_repository ) self.install_model.context.flush() + def __assert_can_install_dependencies(self): + if self.app.config.tool_dependency_dir is None: + no_tool_dependency_dir_message = "Tool dependencies can be automatically installed only if you set " + no_tool_dependency_dir_message += "the value of your 'tool_dependency_dir' setting in your Galaxy " + no_tool_dependency_dir_message += "configuration file (galaxy.ini) and restart your Galaxy server. " + raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) + def fetch_tool_versions( app, tool_shed_repository ): """ Fetch a data structure describing tool shed versions from the tool shed From 3a16535bbb349435eb321441b05a8c6d39d2d700 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 20:32:03 -0500 Subject: [PATCH 11/21] Make more clear function in admin_toolshed is not entry point. --- .../galaxy/controllers/admin_toolshed.py | 58 +++++++++---------- 1 file changed, 28 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index c12d12ef13b..c824b7f9995 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -637,32 +637,6 @@ class AdminToolshed( AdminGalaxy ): message=message, status=status ) - @web.expose - @web.require_admin - def install_tool_shed_repositories( self, trans, tool_shed_repositories, reinstalling=False, **kwd ): - """Install specified tool shed repositories.""" - shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) - tool_path = kwd[ 'tool_path' ] - install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) - # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. - tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) - repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) - irm = install_manager.InstallRepositoryManager( trans.app ) - for index, tool_shed_repository in enumerate( tool_shed_repositories ): - repo_info_dict = repo_info_dicts[ index ] - tool_panel_section_key = tool_panel_section_keys[ index ] - irm.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=reinstalling ) - tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='monitor_repository_installation', - tool_shed_repository_ids=tsr_ids_for_monitoring ) ) - @web.expose @web.require_admin def manage_repositories( self, trans, **kwd ): @@ -733,10 +707,10 @@ class AdminToolshed( AdminGalaxy ): if repositories_for_installation: decoded_kwd[ 'repo_info_dicts' ] = filtered_repo_info_dicts decoded_kwd[ 'tool_panel_section_keys' ] = filtered_tool_panel_section_keys - self.install_tool_shed_repositories( trans, - repositories_for_installation, - reinstalling=reinstalling, - **decoded_kwd ) + self.__install_tool_shed_repositories( trans, + repositories_for_installation, + reinstalling=reinstalling, + **decoded_kwd ) else: kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' kwd[ 'status' ] = 'error' @@ -2162,3 +2136,27 @@ class AdminToolshed( AdminGalaxy ): metadata=metadata, message=message, status=status ) + + def __install_tool_shed_repositories( self, trans, tool_shed_repositories, reinstalling=False, **kwd ): + """Install specified tool shed repositories.""" + shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) + tool_path = kwd[ 'tool_path' ] + install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) + # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. + tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) + repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) + irm = install_manager.InstallRepositoryManager( trans.app ) + for index, tool_shed_repository in enumerate( tool_shed_repositories ): + repo_info_dict = repo_info_dicts[ index ] + tool_panel_section_key = tool_panel_section_keys[ index ] + irm.install_tool_shed_repository( tool_shed_repository, + repo_info_dict, + tool_panel_section_key, + shed_tool_conf, + tool_path, + install_tool_dependencies, + reinstalling=reinstalling ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + return trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) From fec7841d12cdc9f894856d586d55adf5f01c39f4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 20:47:09 -0500 Subject: [PATCH 12/21] Extract an exception out in install_manager.py. Will make it easier to catch this specific problem in admin_toolshed in subsequent refactoring. --- lib/tool_shed/galaxy_install/install_manager.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 26c84faecbd..fb9a0fc3c22 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -804,7 +804,7 @@ class InstallRepositoryManager( object ): installed_tool_shed_repositories.append( tool_shed_repository ) else: # We're attempting to install more than 1 repository, and all of them have already been installed. - raise exceptions.RequestParameterInvalidException( 'All repositories that you are attempting to install have been previously installed.' ) + raise RepositoriesInstalledException() return installed_tool_shed_repositories def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, @@ -961,6 +961,12 @@ class InstallRepositoryManager( object ): raise exceptions.ConfigDoesNotAllowException( no_tool_dependency_dir_message ) +class RepositoriesInstalledException(exceptions.RequestParameterInvalidException): + + def __init__(self): + super(RepositoriesInstalledException, self).__init__('All repositories that you are attempting to install have been previously installed.') + + def fetch_tool_versions( app, tool_shed_repository ): """ Fetch a data structure describing tool shed versions from the tool shed corresponding to a tool_shed_repository object. From bfdb59b1e65c20d1005f1a4749a574e0e6afc94c Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 21:24:28 -0500 Subject: [PATCH 13/21] Refactor a bunch of repository installation out of controller... ... and into install_manager.py. There is a bunch of duplication between this and the code already in install manager - which is a good indication this is where it belongs - hopefully we can de-duplicate it with subsequent changes. This will also allow more of install_manager's methods to be marked as internal. --- .../galaxy/controllers/admin_toolshed.py | 73 ++++--------------- .../galaxy_install/install_manager.py | 49 +++++++++++++ 2 files changed, 63 insertions(+), 59 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index c824b7f9995..434c8dba70a 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -678,41 +678,20 @@ class AdminToolshed( AdminGalaxy ): reinstalling = util.string_as_bool( kwd.get( 'reinstalling', False ) ) encoded_kwd = kwd[ 'encoded_kwd' ] decoded_kwd = encoding_util.tool_shed_decode( encoded_kwd ) - tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] - tool_panel_section_keys = decoded_kwd[ 'tool_panel_section_keys' ] - repo_info_dicts = decoded_kwd[ 'repo_info_dicts' ] - filtered_repo_info_dicts = [] - filtered_tool_panel_section_keys = [] - repositories_for_installation = [] - # Some repositories may have repository dependencies that are required to be installed before the - # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in the - # required order. - ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - irm.order_components_for_installation( tsr_ids, - repo_info_dicts, - tool_panel_section_keys=tool_panel_section_keys ) - for tsr_id in ordered_tsr_ids: - repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .get( trans.security.decode_id( tsr_id ) ) - if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, - trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - repositories_for_installation.append( repository ) - repo_info_dict, tool_panel_section_key = \ - irm.get_repository_components_for_installation( tsr_id, - ordered_tsr_ids, - ordered_repo_info_dicts, - ordered_tool_panel_section_keys ) - filtered_repo_info_dicts.append( repo_info_dict ) - filtered_tool_panel_section_keys.append( tool_panel_section_key ) - if repositories_for_installation: - decoded_kwd[ 'repo_info_dicts' ] = filtered_repo_info_dicts - decoded_kwd[ 'tool_panel_section_keys' ] = filtered_tool_panel_section_keys - self.__install_tool_shed_repositories( trans, - repositories_for_installation, - reinstalling=reinstalling, - **decoded_kwd ) - else: - kwd[ 'message' ] = 'All selected tool shed repositories are already installed.' + install_tool_dependencies = CheckboxField.is_checked( decoded_kwd.get( 'install_tool_dependencies', '' ) ) + decoded_kwd['install_tool_dependencies'] = install_tool_dependencies + try: + tool_shed_repositories = irm.install_web( + trans=trans, + decoded_kwd=decoded_kwd, + reinstalling=reinstalling, + ) + tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] + trans.response.send_redirect( web.url_for( controller='admin_toolshed', + action='monitor_repository_installation', + tool_shed_repository_ids=tsr_ids_for_monitoring ) ) + except install_manager.RepositoriesInstalledException as e: + kwd[ 'message' ] = e.message kwd[ 'status' ] = 'error' return self.repository_installation_grid( trans, **kwd ) @@ -2136,27 +2115,3 @@ class AdminToolshed( AdminGalaxy ): metadata=metadata, message=message, status=status ) - - def __install_tool_shed_repositories( self, trans, tool_shed_repositories, reinstalling=False, **kwd ): - """Install specified tool shed repositories.""" - shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) - tool_path = kwd[ 'tool_path' ] - install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) - # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. - tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) - repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) - irm = install_manager.InstallRepositoryManager( trans.app ) - for index, tool_shed_repository in enumerate( tool_shed_repositories ): - repo_info_dict = repo_info_dicts[ index ] - tool_panel_section_key = tool_panel_section_keys[ index ] - irm.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=reinstalling ) - tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] - return trans.response.send_redirect( web.url_for( controller='admin_toolshed', - action='monitor_repository_installation', - tool_shed_repository_ids=tsr_ids_for_monitoring ) ) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index fb9a0fc3c22..68575180b94 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -7,6 +7,7 @@ import traceback from galaxy import exceptions from galaxy import eggs +from galaxy import util eggs.require( 'paramiko' ) eggs.require( 'ssh' ) @@ -807,6 +808,54 @@ class InstallRepositoryManager( object ): raise RepositoriesInstalledException() return installed_tool_shed_repositories + def install_web( self, trans, decoded_kwd, reinstalling ): + shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' ) + tool_path = decoded_kwd[ 'tool_path' ] + tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] + tool_panel_section_keys = util.listify( decoded_kwd[ 'tool_panel_section_keys' ] ) + repo_info_dicts = util.listify( decoded_kwd[ 'repo_info_dicts' ] ) + install_tool_dependencies = decoded_kwd['install_tool_dependencies'] + filtered_repo_info_dicts = [] + filtered_tool_panel_section_keys = [] + repositories_for_installation = [] + # Some repositories may have repository dependencies that are required to be installed before the + # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in the + # required order. + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ + self.order_components_for_installation( tsr_ids, + repo_info_dicts, + tool_panel_section_keys=tool_panel_section_keys ) + for tsr_id in ordered_tsr_ids: + repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ + .get( trans.security.decode_id( tsr_id ) ) + if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, + trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + repositories_for_installation.append( repository ) + repo_info_dict, tool_panel_section_key = \ + self.get_repository_components_for_installation( tsr_id, + ordered_tsr_ids, + ordered_repo_info_dicts, + ordered_tool_panel_section_keys ) + filtered_repo_info_dicts.append( repo_info_dict ) + filtered_tool_panel_section_keys.append( tool_panel_section_key ) + + installed_tool_shed_repositories = [] + if repositories_for_installation: + for index, tool_shed_repository in enumerate( repositories_for_installation ): + repo_info_dict = filtered_repo_info_dicts[ index ] + tool_panel_section_key = filtered_tool_panel_section_keys[ index ] + self.install_tool_shed_repository( tool_shed_repository, + repo_info_dict=repo_info_dict, + tool_panel_section_key=tool_panel_section_key, + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + install_tool_dependencies=install_tool_dependencies, + reinstalling=reinstalling ) + installed_tool_shed_repositories.append( tool_shed_repository ) + else: + raise RepositoriesInstalledException() + return installed_tool_shed_repositories + def install_tool_shed_repository( self, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, reinstalling=False ): if tool_panel_section_key: From cbf5c35598db66df7d5234b5758937afacc657b5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 21:59:36 -0500 Subject: [PATCH 14/21] PEP-8 comments in install_manager.py. --- .../galaxy_install/install_manager.py | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 68575180b94..e4d17141f67 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -455,10 +455,12 @@ class InstallRepositoryManager( object ): def get_repository_components_for_installation( self, encoded_tsr_id, encoded_tsr_ids, repo_info_dicts, tool_panel_section_keys ): """ - The received encoded_tsr_ids, repo_info_dicts, and tool_panel_section_keys are 3 lists that - contain associated elements at each location in the list. This method will return the elements - from repo_info_dicts and tool_panel_section_keys associated with the received encoded_tsr_id - by determining its location in the received encoded_tsr_ids list. + The received encoded_tsr_ids, repo_info_dicts, and + tool_panel_section_keys are 3 lists that contain associated elements + at each location in the list. This method will return the elements + from repo_info_dicts and tool_panel_section_keys associated with the + received encoded_tsr_id by determining its location in the received + encoded_tsr_ids list. """ for index, tsr_id in enumerate( encoded_tsr_ids ): if tsr_id == encoded_tsr_id: @@ -775,15 +777,18 @@ class InstallRepositoryManager( object ): tool_panel_section_keys=tool_panel_section_keys, tool_path=tool_path, tool_shed_url=tool_shed_url ) - # Prepare the repositories for installation. Even though this method receives a single combination - # of tool_shed_url, name, owner and changeset_revision, there may be multiple repositories for installation - # at this point because repository dependencies may have added additional repositories for installation - # along with the single specified repository. + # Prepare the repositories for installation. Even though this + # method receives a single combination of tool_shed_url, name, + # owner and changeset_revision, there may be multiple repositories + # for installation at this point because repository dependencies + # may have added additional repositories for installation along + # with the single specified repository. encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = \ self.initiate_repository_installation( installation_dict ) - # Some repositories may have repository dependencies that are required to be installed before the - # dependent repository, so we'll order the list of tsr_ids to ensure all repositories install in - # the required order. + # Some repositories may have repository dependencies that are + # required to be installed before the dependent repository, so + # we'll order the list of tsr_ids to ensure all repositories + # install in the required order. tsr_ids = [ self.app.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) From 5bac3dafced84754f9d3708ddbb0fd41b1df8322 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 21:59:50 -0500 Subject: [PATCH 15/21] Refactor __install_repositories so it looks more like newer install_web. --- lib/tool_shed/galaxy_install/install_manager.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index e4d17141f67..92e0862c529 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -709,8 +709,6 @@ class InstallRepositoryManager( object ): return installed_tool_shed_repositories def __install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): - # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. - installed_tool_shed_repositories = [] try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] except KeyError: @@ -790,6 +788,10 @@ class InstallRepositoryManager( object ): # we'll order the list of tsr_ids to ensure all repositories # install in the required order. tsr_ids = [ self.app.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] + + # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. + installed_tool_shed_repositories = [] + ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) # Install the repositories, keeping track of each one for later display. @@ -808,10 +810,10 @@ class InstallRepositoryManager( object ): install_tool_dependencies, reinstalling=False ) installed_tool_shed_repositories.append( tool_shed_repository ) + return installed_tool_shed_repositories else: # We're attempting to install more than 1 repository, and all of them have already been installed. raise RepositoriesInstalledException() - return installed_tool_shed_repositories def install_web( self, trans, decoded_kwd, reinstalling ): shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' ) From 7488039b88e18e5ddc34cb22ad7d02758879a415 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 26 Feb 2015 22:12:07 -0500 Subject: [PATCH 16/21] Eliminate duplication between ts installing from API and web controller. Favoring the web controller variant slightly - it is known to work better and has test coverage. --- .../galaxy/controllers/admin_toolshed.py | 5 +- .../galaxy_install/install_manager.py | 56 +++++++------------ 2 files changed, 23 insertions(+), 38 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 434c8dba70a..6bdac195694 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -679,10 +679,11 @@ class AdminToolshed( AdminGalaxy ): encoded_kwd = kwd[ 'encoded_kwd' ] decoded_kwd = encoding_util.tool_shed_decode( encoded_kwd ) install_tool_dependencies = CheckboxField.is_checked( decoded_kwd.get( 'install_tool_dependencies', '' ) ) + tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] decoded_kwd['install_tool_dependencies'] = install_tool_dependencies try: - tool_shed_repositories = irm.install_web( - trans=trans, + tool_shed_repositories = irm.install_repositories( + tsr_ids=tsr_ids, decoded_kwd=decoded_kwd, reinstalling=reinstalling, ) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index 92e0862c529..5537d1f2b16 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -702,13 +702,15 @@ class InstallRepositoryManager( object ): name, owner, changeset_revision ) - installed_tool_shed_repositories = self.__install_repositories( tool_shed_url, - repository_revision_dict, - repo_info_dicts, - install_options ) + installed_tool_shed_repositories = self.__initiate_and_install_repositories( + tool_shed_url, + repository_revision_dict, + repo_info_dicts, + install_options + ) return installed_tool_shed_repositories - def __install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): + def __initiate_and_install_repositories( self, tool_shed_url, repository_revision_dict, repo_info_dicts, install_options ): try: has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] except KeyError: @@ -789,36 +791,18 @@ class InstallRepositoryManager( object ): # install in the required order. tsr_ids = [ self.app.security.encode_id( tool_shed_repository.id ) for tool_shed_repository in tool_shed_repositories ] - # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. - installed_tool_shed_repositories = [] + decoded_kwd = dict( + shed_tool_conf=shed_tool_conf, + tool_path=tool_path, + tool_panel_section_keys=tool_panel_section_keys, + repo_info_dicts=repo_info_dicts, + install_tool_dependencies=install_tool_dependencies, + ) + return self.install_repositories(tsr_ids, decoded_kwd, reinstalling=False) - ordered_tsr_ids, ordered_repo_info_dicts, ordered_tool_panel_section_keys = \ - self.order_components_for_installation( tsr_ids, repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) - # Install the repositories, keeping track of each one for later display. - for index, tsr_id in enumerate( ordered_tsr_ids ): - tool_shed_repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ - .get( self.app.security.decode_id( tsr_id ) ) - if tool_shed_repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, - self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: - repo_info_dict = ordered_repo_info_dicts[ index ] - tool_panel_section_key = ordered_tool_panel_section_keys[ index ] - self.install_tool_shed_repository( tool_shed_repository, - repo_info_dict, - tool_panel_section_key, - shed_tool_conf, - tool_path, - install_tool_dependencies, - reinstalling=False ) - installed_tool_shed_repositories.append( tool_shed_repository ) - return installed_tool_shed_repositories - else: - # We're attempting to install more than 1 repository, and all of them have already been installed. - raise RepositoriesInstalledException() - - def install_web( self, trans, decoded_kwd, reinstalling ): + def install_repositories( self, tsr_ids, decoded_kwd, reinstalling ): shed_tool_conf = decoded_kwd.get( 'shed_tool_conf', '' ) tool_path = decoded_kwd[ 'tool_path' ] - tsr_ids = decoded_kwd[ 'tool_shed_repository_ids' ] tool_panel_section_keys = util.listify( decoded_kwd[ 'tool_panel_section_keys' ] ) repo_info_dicts = util.listify( decoded_kwd[ 'repo_info_dicts' ] ) install_tool_dependencies = decoded_kwd['install_tool_dependencies'] @@ -833,10 +817,10 @@ class InstallRepositoryManager( object ): repo_info_dicts, tool_panel_section_keys=tool_panel_section_keys ) for tsr_id in ordered_tsr_ids: - repository = trans.install_model.context.query( trans.install_model.ToolShedRepository ) \ - .get( trans.security.decode_id( tsr_id ) ) - if repository.status in [ trans.install_model.ToolShedRepository.installation_status.NEW, - trans.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: + repository = self.install_model.context.query( self.install_model.ToolShedRepository ) \ + .get( self.app.security.decode_id( tsr_id ) ) + if repository.status in [ self.install_model.ToolShedRepository.installation_status.NEW, + self.install_model.ToolShedRepository.installation_status.UNINSTALLED ]: repositories_for_installation.append( repository ) repo_info_dict, tool_panel_section_key = \ self.get_repository_components_for_installation( tsr_id, From c2378c197ab5381ebb6f1345a38bc92ee75ca6da Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 27 Feb 2015 09:22:14 -0500 Subject: [PATCH 17/21] Use append_section abstraction. --- lib/galaxy/tools/toolbox/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 9e469ba9659..c71ad757b37 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -237,7 +237,7 @@ class AbstractToolBox( object, Dictifiable ): 'version': '', } tool_section = ToolSection( section_dict ) - self._tool_panel[ tool_panel_section_key ] = tool_section + self._tool_panel.append_section( tool_panel_section_key, tool_section ) log.debug( "Loading new tool panel section: %s" % str( tool_section.name ) ) else: tool_section = None From 6c2fc6847643c8f2f7806799993ff83255fb8f5f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 27 Feb 2015 09:22:35 -0500 Subject: [PATCH 18/21] PEP-8 in toolbox/base.py. --- lib/galaxy/tools/toolbox/base.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index c71ad757b37..564653cb396 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -457,9 +457,9 @@ class AbstractToolBox( object, Dictifiable ): if tool_id in self._tools_by_id and not get_all_versions: if tool_version and tool_version in self._tool_versions_by_id[ tool_id ]: return self._tool_versions_by_id[ tool_id ][ tool_version ] - #tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) + # tool_id exactly matches an available tool by id (which is 'old' tool_id or guid) return self._tools_by_id[ tool_id ] - #exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions + # exact tool id match not found, or all versions requested, search for other options, e.g. migrated tools or different versions rval = [] tool_lineage = self._lineage_map.get( tool_id ) if tool_lineage: @@ -469,7 +469,7 @@ class AbstractToolBox( object, Dictifiable ): if lineage_tool: rval.append( lineage_tool ) if not rval: - #still no tool, do a deeper search and try to match by old ids + # still no tool, do a deeper search and try to match by old ids for tool in self._tools_by_id.itervalues(): if tool.old_id == tool_id: rval.append( tool ) @@ -478,14 +478,14 @@ class AbstractToolBox( object, Dictifiable ): return rval else: if tool_version: - #return first tool with matching version + # return first tool with matching version for tool in rval: if tool.version == tool_version: return tool - #No tool matches by version, simply return the first available tool found + # No tool matches by version, simply return the first available tool found return rval[0] - #We now likely have a Toolshed guid passed in, but no supporting database entries - #If the tool exists by exact id and is loaded then provide exact match within a list + # We now likely have a Toolshed guid passed in, but no supporting database entries + # If the tool exists by exact id and is loaded then provide exact match within a list if tool_id in self._tools_by_id: return[ self._tools_by_id[ tool_id ] ] return None @@ -1012,7 +1012,7 @@ class AbstractToolBox( object, Dictifiable ): break if tool_id in self.data_manager_tools: del self.data_manager_tools[ tool_id ] - #TODO: do we need to manually remove from the integrated panel here? + # TODO: do we need to manually remove from the integrated panel here? message = "Removed the tool:
" message += "name: %s
" % tool.name message += "id: %s
" % tool.id From aa83a87d8ac84cd66974716855a63592e3e33c11 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 27 Feb 2015 14:15:26 -0500 Subject: [PATCH 19/21] Move integrated tool panel logic into its own module/mixin. --- lib/galaxy/tools/toolbox/base.py | 96 ++------------------ lib/galaxy/tools/toolbox/integrated_panel.py | 87 ++++++++++++++++++ 2 files changed, 95 insertions(+), 88 deletions(-) create mode 100644 lib/galaxy/tools/toolbox/integrated_panel.py diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 564653cb396..bbd17ffd1a6 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -1,6 +1,5 @@ import os import re -import shutil import string import tarfile import tempfile @@ -23,6 +22,7 @@ from .panel import ToolPanelElements from .panel import ToolSectionLabel from .panel import ToolSection from .panel import panel_item_types +from .integrated_panel import ManagesIntegratedToolPanelMixin from .lineages import LineageMap from .tags import tool_tag_manager @@ -37,22 +37,7 @@ import logging log = logging.getLogger( __name__ ) -INTEGRATED_TOOL_PANEL_DESCRIPTION = """ -This is Galaxy's integrated tool panel and should be modified directly only for -reordering tools inside a section. Each time Galaxy starts up, this file is -synchronized with the various tool config files: tools, sections and labels -added to one of these files, will be added also here in the appropriate place, -while elements removed from the tool config files will be correspondingly -deleted from this file. -To modify locally managed tools (e.g. from tool_conf.xml) modify that file -directly and restart Galaxy. Whenever possible Tool Shed managed tools (e.g. -from shed_tool_conf.xml) should be managed from within the Galaxy interface or -via its API - but if changes are necessary (such as to hide a tool or re-assign -its section) modify that file and restart Galaxy. -""" - - -class AbstractToolBox( object, Dictifiable ): +class AbstractToolBox( object, Dictifiable, ManagesIntegratedToolPanelMixin ): """ Abstract container for managing a ToolPanel - containing tools and workflows optionally in labelled sections. @@ -78,15 +63,8 @@ class AbstractToolBox( object, Dictifiable ): self._index = 0 self.data_manager_tools = odict() self._lineage_map = LineageMap( app ) - # File that contains the XML section and tool tags from all tool panel config files integrated into a - # single file that defines the tool panel layout. This file can be changed by the Galaxy administrator - # (in a way similar to the single tool_conf.xml file in the past) to alter the layout of the tool panel. - self._integrated_tool_panel_config = app.config.integrated_tool_panel_config - # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. - self._integrated_tool_panel = ToolPanelElements() - self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 - if self._integrated_tool_panel_config_has_contents: - self._load_integrated_tool_panel_keys() + # Sets self._integrated_tool_panel and self._integrated_tool_panel_config_has_contents + self._init_integrated_tool_panel( app.config ) # The following refers to the tool_path config setting for backward compatibility. The shed-related # (e.g., shed_tool_conf.xml) files include the tool_path attribute within the tag. self._tool_root_dir = tool_root_dir @@ -98,12 +76,7 @@ class AbstractToolBox( object, Dictifiable ): if self.app.name == 'galaxy' and self._integrated_tool_panel_config_has_contents: # Load self._tool_panel based on the order in self._integrated_tool_panel. self._load_tool_panel() - if app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() + self._save_integrated_tool_panel() def create_tool( self, config_file, repository_id=None, guid=None, **kwds ): raise NotImplementedError() @@ -215,9 +188,8 @@ class AbstractToolBox( object, Dictifiable ): for index, my_shed_tool_conf in enumerate( self._dynamic_tool_confs ): if shed_conf['config_filename'] == my_shed_tool_conf['config_filename']: self._dynamic_tool_confs[ index ] = shed_conf - if integrated_panel_changes and app.config.update_integrated_tool_panel: - # Write the current in-memory version of the integrated_tool_panel.xml file to disk. - self._write_integrated_tool_panel_config_file() + if integrated_panel_changes: + self._save_integrated_tool_panel() app.reindex_tool_search() def get_section( self, section_id, new_label=None, create_if_needed=False ): @@ -400,52 +372,6 @@ class AbstractToolBox( object, Dictifiable ): elif elem.tag == 'label': self._integrated_tool_panel.stub_label( key ) - def _write_integrated_tool_panel_config_file( self ): - """ - Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators - use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. - """ - fd, filename = tempfile.mkstemp() - os.write( fd, '\n' ) - os.write( fd, '\n' ) - os.write( fd, ' \n') - for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): - if item: - if item_type == panel_item_types.TOOL: - os.write( fd, ' \n' % item.id ) - elif item_type == panel_item_types.WORKFLOW: - os.write( fd, ' \n' % item.id ) - elif item_type == panel_item_types.LABEL: - label_id = item.id or '' - label_text = item.text or '' - label_version = item.version or '' - os.write( fd, ' \n' ) - os.close( fd ) - shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) - os.chmod( self._integrated_tool_panel_config, 0644 ) - def get_tool( self, tool_id, tool_version=None, get_all_versions=False, exact=False ): """Attempt to locate a tool in the tool box.""" if tool_version: @@ -777,13 +703,7 @@ class AbstractToolBox( object, Dictifiable ): if async: self._load_tool_panel() - - if self.app.config.update_integrated_tool_panel: - # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. - # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel - # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel - # will be False when things like functional tests are the caller. - self._write_integrated_tool_panel_config_file() + self._save_integrated_tool_panel() return tool.id except Exception: log.exception("Failed to load potential tool %s." % tool_file) diff --git a/lib/galaxy/tools/toolbox/integrated_panel.py b/lib/galaxy/tools/toolbox/integrated_panel.py new file mode 100644 index 00000000000..6aa8ad95e4b --- /dev/null +++ b/lib/galaxy/tools/toolbox/integrated_panel.py @@ -0,0 +1,87 @@ +import os +import shutil +import tempfile + +from .panel import ToolPanelElements +from .panel import panel_item_types + + +INTEGRATED_TOOL_PANEL_DESCRIPTION = """ +This is Galaxy's integrated tool panel and should be modified directly only for +reordering tools inside a section. Each time Galaxy starts up, this file is +synchronized with the various tool config files: tools, sections and labels +added to one of these files, will be added also here in the appropriate place, +while elements removed from the tool config files will be correspondingly +deleted from this file. +To modify locally managed tools (e.g. from tool_conf.xml) modify that file +directly and restart Galaxy. Whenever possible Tool Shed managed tools (e.g. +from shed_tool_conf.xml) should be managed from within the Galaxy interface or +via its API - but if changes are necessary (such as to hide a tool or re-assign +its section) modify that file and restart Galaxy. +""" + + +class ManagesIntegratedToolPanelMixin: + + def _init_integrated_tool_panel(self, config): + self.update_integrated_tool_panel = config.update_integrated_tool_panel + self._integrated_tool_panel_config = config.integrated_tool_panel_config + # In-memory dictionary that defines the layout of the tool_panel.xml file on disk. + self._integrated_tool_panel = ToolPanelElements() + self._integrated_tool_panel_config_has_contents = os.path.exists( self._integrated_tool_panel_config ) and os.stat( self._integrated_tool_panel_config ).st_size > 0 + if self._integrated_tool_panel_config_has_contents: + self._load_integrated_tool_panel_keys() + + def _save_integrated_tool_panel(self): + if self.update_integrated_tool_panel: + # Write the current in-memory integrated_tool_panel to the integrated_tool_panel.xml file. + # This will cover cases where the Galaxy administrator manually edited one or more of the tool panel + # config files, adding or removing locally developed tools or workflows. The value of integrated_tool_panel + # will be False when things like functional tests are the caller. + self._write_integrated_tool_panel_config_file() + + def _write_integrated_tool_panel_config_file( self ): + """ + Write the current in-memory version of the integrated_tool_panel.xml file to disk. Since Galaxy administrators + use this file to manage the tool panel, we'll not use xml_to_string() since it doesn't write XML quite right. + """ + fd, filename = tempfile.mkstemp() + os.write( fd, '\n' ) + os.write( fd, '\n' ) + os.write( fd, ' \n') + for key, item_type, item in self._integrated_tool_panel.panel_items_iter(): + if item: + if item_type == panel_item_types.TOOL: + os.write( fd, ' \n' % item.id ) + elif item_type == panel_item_types.WORKFLOW: + os.write( fd, ' \n' % item.id ) + elif item_type == panel_item_types.LABEL: + label_id = item.id or '' + label_text = item.text or '' + label_version = item.version or '' + os.write( fd, ' \n' ) + os.close( fd ) + shutil.move( filename, os.path.abspath( self._integrated_tool_panel_config ) ) + os.chmod( self._integrated_tool_panel_config, 0644 ) From 9f68fe0792df1917ffb533ba41108eada47f04d1 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 27 Feb 2015 15:50:13 -0500 Subject: [PATCH 20/21] Allow External display applications to use Tool Data Tables. --- lib/galaxy/app.py | 2 +- .../display_applications/application.py | 133 +++++++++++++----- .../display_applications/parameters.py | 2 +- lib/galaxy/datatypes/registry.py | 4 +- .../datatypes/custom_datatype_manager.py | 2 +- .../galaxy_install/install_manager.py | 2 +- .../galaxy_install/tool_migration_manager.py | 2 +- 7 files changed, 106 insertions(+), 41 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index cc1f122e2e3..9aca94837b8 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -88,7 +88,7 @@ class UniverseApplication( object, config.ConfiguresGalaxyMixin ): # Load proprietary datatype converters and display applications. self.installed_repository_manager.load_proprietary_converters_and_display_applications() # Load datatype display applications defined in local datatypes_conf.xml - self.datatypes_registry.load_display_applications() + self.datatypes_registry.load_display_applications( self ) # Load datatype converters defined in local datatypes_conf.xml self.datatypes_registry.load_datatype_converters( self.toolbox ) # Load external metadata tool diff --git a/lib/galaxy/datatypes/display_applications/application.py b/lib/galaxy/datatypes/display_applications/application.py index 74c20e139e2..75ffcf6368f 100644 --- a/lib/galaxy/datatypes/display_applications/application.py +++ b/lib/galaxy/datatypes/display_applications/application.py @@ -80,21 +80,57 @@ class DisplayApplicationLink( object ): return True class DynamicDisplayApplicationBuilder( object ): - @classmethod + def __init__( self, elem, display_application, build_sites ): - rval = [] filename = None + data_table = None if elem.get( 'site_type', None ) is not None: filename = build_sites.get( elem.get( 'site_type' ) ) else: filename = elem.get( 'from_file', None ) - assert filename is not None, 'Filename and id attributes required for dynamic_links' + if filename is None: + data_table_name = elem.get( 'from_data_table', None ) + if data_table_name: + data_table = display_application.app.tool_data_tables.get( data_table_name, None ) + assert data_table is not None, 'Unable to find data table named "%s".' % data_table_name + + assert filename is not None or data_table is not None,'Filename or data Table is required for dynamic_links.' skip_startswith = elem.get( 'skip_startswith', None ) separator = elem.get( 'separator', '\t' ) - id_col = int( elem.get( 'id', None ) ) - name_col = int( elem.get( 'name', id_col ) ) - dynamic_params = {} + id_col = elem.get( 'id', None ) + try: + id_col = int( id_col ) + except: + if data_table: + if id_col is None: + id_col = data_table.columns.get( 'id', None ) + if id_col is None: + id_col = data_table.columns.get( 'value', None ) + try: + id_col = int( id_col ) + except: + # id is set to a string or None, use column by that name if available + id_col = data_table.columns.get( id_col, None ) + id_col = int( id_col ) + name_col = elem.get( 'name', None ) + try: + name_col = int( name_col ) + except: + if data_table: + if name_col is None: + name_col = data_table.columns.get( 'name', None ) + else: + name_col = data_table.columns.get( name_col, None ) + else: + name_col = None + if name_col is None: + name_col = id_col max_col = max( id_col, name_col ) + dynamic_params = {} + if data_table is not None: + max_col = max( [ max_col ] + data_table.columns.values() ) + for key, value in data_table.columns.items(): + dynamic_params[key] = { 'column': value, 'split': False, 'separator': ',' } for dynamic_param in elem.findall( 'dynamic_param' ): name = dynamic_param.get( 'name' ) value = int( dynamic_param.get( 'value' ) ) @@ -102,27 +138,38 @@ class DynamicDisplayApplicationBuilder( object ): param_separator = dynamic_param.get( 'separator', ',' ) max_col = max( max_col, value ) dynamic_params[name] = { 'column': value, 'split': split, 'separator': param_separator } - for line in open( filename ): - if not skip_startswith or not line.startswith( skip_startswith ): - line = line.rstrip( '\n\r' ) - if not line: - continue - fields = line.split( separator ) - if len( fields ) > max_col: - new_elem = deepcopy( elem ) - new_elem.set( 'id', fields[id_col] ) - new_elem.set( 'name', fields[name_col] ) - dynamic_values = {} - for key, attributes in dynamic_params.iteritems(): - value = fields[ attributes[ 'column' ] ] - if attributes['split']: - value = value.split( attributes['separator'] ) - dynamic_values[key] = value - #now populate - rval.append( DisplayApplicationLink.from_elem( new_elem, display_application, other_values = dynamic_values ) ) + if filename: + data_iter = open( filename ) + elif data_table: + version, data_iter = data_table.get_version_fields() + display_application.add_data_table_watch( data_table.name, version ) + links = [] + for line in data_iter: + if isinstance( line, basestring ): + if not skip_startswith or not line.startswith( skip_startswith ): + line = line.rstrip( '\n\r' ) + if not line: + continue + fields = line.split( separator ) else: - log.warning( 'Invalid dynamic display application link specified in %s: "%s"' % ( filename, line ) ) - self.links = rval + continue + else: + fields = line + if len( fields ) > max_col: + new_elem = deepcopy( elem ) + new_elem.set( 'id', fields[id_col] ) + new_elem.set( 'name', fields[name_col] ) + dynamic_values = {} + for key, attributes in dynamic_params.iteritems(): + value = fields[ attributes[ 'column' ] ] + if attributes['split']: + value = value.split( attributes['separator'] ) + dynamic_values[key] = value + #now populate + links.append( DisplayApplicationLink.from_elem( new_elem, display_application, other_values = dynamic_values ) ) + else: + log.warning( 'Invalid dynamic display application link specified in %s: "%s"' % ( filename, line ) ) + self.links = links def __iter__( self ): return iter( self.links ) @@ -166,12 +213,12 @@ class PopulatedDisplayApplicationLink( object ): class DisplayApplication( object ): @classmethod - def from_file( cls, filename, datatypes_registry ): - return cls.from_elem( parse_xml( filename ).getroot(), datatypes_registry, filename=filename ) + def from_file( cls, filename, app ): + return cls.from_elem( parse_xml( filename ).getroot(), app, filename=filename ) @classmethod - def from_elem( cls, elem, datatypes_registry, filename=None ): + def from_elem( cls, elem, app, filename=None ): att_dict = cls._get_attributes_from_elem( elem ) - rval = DisplayApplication( att_dict['id'], att_dict['name'], datatypes_registry, att_dict['version'], filename=filename, elem=elem ) + rval = DisplayApplication( att_dict['id'], att_dict['name'], app, att_dict['version'], filename=filename, elem=elem ) rval._load_links_from_elem( elem ) return rval @classmethod @@ -181,29 +228,32 @@ class DisplayApplication( object ): name = elem.get( 'name', display_id ) version = elem.get( 'version', None ) return dict( id=display_id, name=name, version=version ) - def __init__( self, display_id, name, datatypes_registry, version = None, filename=None, elem=None ): + def __init__( self, display_id, name, app, version = None, filename=None, elem=None ): self.id = display_id self.name = name - self.datatypes_registry = datatypes_registry + self.app = app if version is None: version = "1.0.0" self.version = version self.links = odict() self._filename = filename self._elem = elem + self._data_table_versions = {} def _load_links_from_elem( self, elem ): for link_elem in elem.findall( 'link' ): link = DisplayApplicationLink.from_elem( link_elem, self ) if link: self.links[ link.id ] = link for dynamic_links in elem.findall( 'dynamic_links' ): - for link in DynamicDisplayApplicationBuilder( dynamic_links, self, self.datatypes_registry.build_sites ): + for link in DynamicDisplayApplicationBuilder( dynamic_links, self, self.app.datatypes_registry.build_sites ): self.links[ link.id ] = link def get_link( self, link_name, data, dataset_hash, user_hash, trans, app_kwds ): #returns a link object with data knowledge to generate links + self._check_and_reload() return PopulatedDisplayApplicationLink( self.links[ link_name ], data, dataset_hash, user_hash, trans, app_kwds ) def filter_by_dataset( self, data, trans ): - filtered = DisplayApplication( self.id, self.name, self.datatypes_registry, version = self.version ) + self._check_and_reload() + filtered = DisplayApplication( self.id, self.name, self.app, version = self.version ) for link_name, link_value in self.links.iteritems(): if link_value.filter_by_dataset( data, trans ): filtered.links[link_name] = link_value @@ -222,9 +272,24 @@ class DisplayApplication( object ): # clear old links for key in self.links.keys(): del self.links[key] + #clear data table versions: + for key in self._data_table_versions.keys(): + del self._data_table_versions[ key ] # Set new attributes for key, value in attr_dict.iteritems(): setattr( self, key, value ) # Load new links self._load_links_from_elem( elem ) return self + def add_data_table_watch( self, table_name, version=None ): + self._data_table_versions[ table_name ] = version + def _requires_reload( self ): + for key, value in self._data_table_versions.iteritems(): + table = self.app.tool_data_tables.get( key, None ) + if table and not table.is_current_version( value ): + return True + return False + def _check_and_reload( self ): + if self._requires_reload(): + self.reload() + diff --git a/lib/galaxy/datatypes/display_applications/parameters.py b/lib/galaxy/datatypes/display_applications/parameters.py index b4d382a626c..8105fcac217 100644 --- a/lib/galaxy/datatypes/display_applications/parameters.py +++ b/lib/galaxy/datatypes/display_applications/parameters.py @@ -59,7 +59,7 @@ class DisplayApplicationDataParameter( DisplayApplicationParameter ): @property def formats( self ): if self.extensions: - return tuple( map( type, map( self.link.display_application.datatypes_registry.get_datatype_by_extension, self.extensions ) ) ) + return tuple( map( type, map( self.link.display_application.app.datatypes_registry.get_datatype_by_extension, self.extensions ) ) ) return None def _get_dataset_like_object( self, other_values ): #this returned object has file_name, state, and states attributes equivalent to a DatasetAssociation diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 76a817a5d00..be9704015d7 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -546,7 +546,7 @@ class Registry( object ): else: self.log.exception( "Error loading converter (%s): %s" % ( converter_path, str( e ) ) ) - def load_display_applications( self, installed_repository_dict=None, deactivate=False ): + def load_display_applications( self, app, installed_repository_dict=None, deactivate=False ): """ If deactivate is False, add display applications from self.display_app_containers or self.proprietary_display_app_containers to appropriate datatypes. If deactivate is @@ -570,7 +570,7 @@ class Registry( object ): config_path = os.path.join( self.display_applications_path, display_file ) try: inherit = galaxy.util.string_as_bool( display_app.get( 'inherit', 'False' ) ) - display_app = DisplayApplication.from_file( config_path, self ) + display_app = DisplayApplication.from_file( config_path, app ) if display_app: if display_app.id in self.display_applications: if deactivate: diff --git a/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py b/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py index 8d56d04c540..5d829967d53 100644 --- a/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py +++ b/lib/tool_shed/galaxy_install/datatypes/custom_datatype_manager.py @@ -214,5 +214,5 @@ class CustomDatatypeLoader( object ): def load_installed_display_applications( self, installed_repository_dict, deactivate=False ): """Load or deactivate custom datatype display applications.""" - self.app.datatypes_registry.load_display_applications( installed_repository_dict=installed_repository_dict, + self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=installed_repository_dict, deactivate=deactivate ) diff --git a/lib/tool_shed/galaxy_install/install_manager.py b/lib/tool_shed/galaxy_install/install_manager.py index d955ed355e7..9bfd311d7ff 100644 --- a/lib/tool_shed/galaxy_install/install_manager.py +++ b/lib/tool_shed/galaxy_install/install_manager.py @@ -621,7 +621,7 @@ class InstallRepositoryManager( object ): self.app.datatypes_registry.load_datatype_converters( self.app.toolbox, installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications - self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=repository_dict ) def handle_tool_shed_repositories( self, installation_dict, using_api=False ): # The following installation_dict entries are all required. diff --git a/lib/tool_shed/galaxy_install/tool_migration_manager.py b/lib/tool_shed/galaxy_install/tool_migration_manager.py index f8469fe96f8..e8cbcc1db54 100644 --- a/lib/tool_shed/galaxy_install/tool_migration_manager.py +++ b/lib/tool_shed/galaxy_install/tool_migration_manager.py @@ -523,7 +523,7 @@ class ToolMigrationManager( object ): installed_repository_dict=repository_dict ) if display_path: # Load proprietary datatype display applications - self.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + self.app.datatypes_registry.load_display_applications( self.app, installed_repository_dict=repository_dict ) basic_util.remove_dir( work_dir ) def install_repository( self, repository_elem, tool_shed_repository, install_dependencies, is_repository_dependency=False ): From 1260d07a2892524698b969a6d2218aef7659230d Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 27 Feb 2015 15:50:28 -0500 Subject: [PATCH 21/21] Have IGV external display application load a set of links for available genomes via a tool data table that reads a URL (http://igv.broadinstitute.org/genomes/genomes.txt). The manually specified builds links should now be used to add additional builds, or for alias mapping (e.g. hg_g1k_v37-->b37). --- config/tool_data_table_conf.xml.sample | 5 +++++ display_applications/igv/bam.xml | 15 ++++++++++++++- display_applications/igv/vcf.xml | 15 ++++++++++++++- tool-data/shared/igv/igv_build_sites.txt.sample | 4 ++-- 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/config/tool_data_table_conf.xml.sample b/config/tool_data_table_conf.xml.sample index 910889e8320..82b3079f07c 100644 --- a/config/tool_data_table_conf.xml.sample +++ b/config/tool_data_table_conf.xml.sample @@ -65,4 +65,9 @@ value, path + + + name, url, value + +
diff --git a/display_applications/igv/bam.xml b/display_applications/igv/bam.xml index 1e7058e01c9..275e6aa16ef 100644 --- a/display_applications/igv/bam.xml +++ b/display_applications/igv/bam.xml @@ -93,7 +93,20 @@ #end if - + + + + + + ${ $dataset.dbkey == $value } + + + http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bam_file.qp}&genome=${bam_file.dbkey}&merge=true&name=${qp( $bam_file.name )} + + + + + diff --git a/display_applications/igv/vcf.xml b/display_applications/igv/vcf.xml index a9276745b0c..00321d22e77 100644 --- a/display_applications/igv/vcf.xml +++ b/display_applications/igv/vcf.xml @@ -93,7 +93,20 @@ #end if - + + + + + + ${ $dataset.dbkey == $value } + + + http://www.broadinstitute.org/igv/projects/current/igv.php?sessionURL=${bgzip_file.qp}&genome=$bgzip_file.dbkey&merge=true&name=${qp( $bgzip_file.name )} + + + + + diff --git a/tool-data/shared/igv/igv_build_sites.txt.sample b/tool-data/shared/igv/igv_build_sites.txt.sample index 77d262da4f6..ce3f18f16b6 100644 --- a/tool-data/shared/igv/igv_build_sites.txt.sample +++ b/tool-data/shared/igv/igv_build_sites.txt.sample @@ -1,4 +1,4 @@ #site_id site_name site_url dbkey ivg_build_name -web_link_main web current http://www.broadinstitute.org/igv/projects/current/igv.php hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum +web_link_main web current http://www.broadinstitute.org/igv/projects/current/igv.php hg_g1k_v37 b37 #web_jnlp_1.5 web 1.5 http://www.broadinstitute.org/igvdata/jws/prod hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum -#local_default local http://localhost:60151/load hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum \ No newline at end of file +#local_default local http://localhost:60151/load hg19,hg_g1k_v37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum hg19,b37,hg18,1kg_ref,hg17,hg16,mm9,mm8,mm7,panTro2,rheMac2,rn4,canFam2,bosTau6,bosTau4,bosTau3,susScrofa,galGal3,cavPor3,monDom5,xenTro2,taeGut1,zebrafish,danRer6,danRer7,gasAcu1,Aplysia,Plasmodium_3D7_v2.1,Plasmodium_3D7_v5.5,Plasmodium_6.1,PlasmoDB_7.0,pvivax,GSM552910,sacCer1,sacCer2,sk1,Y55,sacCer62,spombe_709,spombe_1.55,candida,mg8,spur_2.1,spur_2.5,spur_3.0,WS201,ce6,ce4,dm3,dm2,dmel_5.9,dmel_r5.22,dmel_r5.33,tcas_2.0,tcas_3.0,ncrassa_v3,nc10,Glamblia_2.0,me49,tb927,tbgambi,lmjr,anidulans_4.1,NC_009012,U00096.2,NC_000913.2,NC_002655.2,CSavignyi_v2.1,tair8,tair9,tair10,O_Sativa_r6,osativa_6.1,B73,ZmB73_5a,ppatens_1.2,D.discoideum