From 94328b9684244b5d6782b147130901504229d5fc Mon Sep 17 00:00:00 2001 From: ValentinMarcon Date: Mon, 24 Jun 2019 14:08:03 +0200 Subject: [PATCH 01/31] merge dev into toolref --- client/galaxy/scripts/components/xrefs.vue | 76 +++++++++++++++++++ .../galaxy/scripts/mvc/tool/tool-form-base.js | 13 ++++ doc/schema_template.md | 2 + lib/galaxy/config.py | 3 +- lib/galaxy/managers/xrefs.py | 31 ++++++++ lib/galaxy/tool_util/parser/xml.py | 6 ++ lib/galaxy/tool_util/parser/yaml.py | 3 + lib/galaxy/tool_util/xsd/galaxy.xsd | 69 ++++++++++++++--- lib/galaxy/tools/__init__.py | 20 +++++ lib/galaxy/webapps/galaxy/api/tools.py | 8 ++ lib/galaxy/webapps/galaxy/buildapp.py | 1 + 11 files changed, 221 insertions(+), 11 deletions(-) create mode 100644 client/galaxy/scripts/components/xrefs.vue create mode 100644 lib/galaxy/managers/xrefs.py diff --git a/client/galaxy/scripts/components/xrefs.vue b/client/galaxy/scripts/components/xrefs.vue new file mode 100644 index 00000000000..a6f925d6eb8 --- /dev/null +++ b/client/galaxy/scripts/components/xrefs.vue @@ -0,0 +1,76 @@ + + diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js index ceb1a0dd109..c2b47fe70d4 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-base.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js @@ -12,6 +12,7 @@ import Ui from "mvc/ui/ui-misc"; import FormBase from "mvc/form/form-view"; import Webhooks from "mvc/webhooks"; import Citations from "components/Citations.vue"; +import xrefs from "components/xrefs.vue"; import Vue from "vue"; import axios from "axios"; @@ -268,6 +269,7 @@ export default FormBase.extend({ _footer: function() { var options = this.model.attributes; var $el = $("
").append(this._templateHelp(options)); + if (options.citations) { var citationInstance = Vue.extend(Citations); var vm = document.createElement("div"); @@ -279,6 +281,17 @@ export default FormBase.extend({ } }).$mount(vm); } + if (options.xrefs) { + var xrefInstance = Vue.extend(xrefs); + vm = document.createElement("div"); + $el.append(vm); + new xrefInstance({ + propsData: { + id: options.id, + source: "tools" + } + }).$mount(vm); + } return $el; }, diff --git a/doc/schema_template.md b/doc/schema_template.md index a08c3fc12ec..681ca56a5fe 100644 --- a/doc/schema_template.md +++ b/doc/schema_template.md @@ -18,6 +18,8 @@ $tag:tool|description://element[@name='tool']//element[@name='description'] $tag:tool|macros://complexType[@name='Macros'] $tag:tool|edam_topics://complexType[@name='EdamTopics'] $tag:tool|edam_operations://complexType[@name='EdamOperations'] +$tag:tool|xrefs://complexType[@name='xrefs'] +$tag:tool|xrefs|xref://complexType[@name='xref'] $tag:tool|requirements://complexType[@name='Requirements'] $tag:tool|requirements|requirement://complexType[@name='Requirement'] $tag:tool|requirements|container://complexType[@name='Container'] diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index f81a2293e95..6a384b42df9 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -38,7 +38,6 @@ from .version import VERSION_MAJOR log = logging.getLogger(__name__) - PATH_DEFAULTS = dict( auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'], data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'], @@ -1060,11 +1059,13 @@ class ConfiguresGalaxyMixin(object): def _configure_toolbox(self): from galaxy import tools from galaxy.managers.citations import CitationsManager + from galaxy.managers.xrefs import xrefsManager from galaxy.tool_util.deps import containers from galaxy.tool_util.deps.dependencies import AppInfo import galaxy.tools.search self.citations_manager = CitationsManager(self) + self.xrefs_manager = xrefsManager(self) from galaxy.managers.tools import DynamicToolManager self.dynamic_tools_manager = DynamicToolManager(self) diff --git a/lib/galaxy/managers/xrefs.py b/lib/galaxy/managers/xrefs.py new file mode 100644 index 00000000000..917c82b8571 --- /dev/null +++ b/lib/galaxy/managers/xrefs.py @@ -0,0 +1,31 @@ + +class xrefsManager(object): + + def __init__(self, app): + self.app = app + + def parse_xref(self, xref_elem): + return parse_xref(xref_elem, self) + + def _get_tool(self, tool_id): + tool = self.app.toolbox.get_tool(tool_id) + return tool + + +def parse_xref(elem, tool_ref_manager): + """ Parse a xreference entry from the specified XML element.""" + return xref(elem) + + +class Base_xref(object): + def to_dict(self): + return dict( + reftype=self.reftype, + content=self.content, + ) + + +class xref(Base_xref): + def __init__(self, elem): + self.content = elem.text.strip() + self.reftype = elem.attrib.get('type', None) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 24eca5a78a4..f65280298c9 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -95,6 +95,12 @@ class XmlToolSource(ToolSource): return [] return [edam_topic.text for edam_topic in edam_topics.findall("edam_topic")] + def parse_xrefs(self): + xrefs = self.root.find("xrefs") + if xrefs is None: + return [] + return [xref.text for xref in xrefs.findall("xref") if xref.get("type") in {'bio.tools'}] + def parse_description(self): return xml_text(self.root, "description") diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index d13331cd768..76bbdc1e133 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -40,6 +40,9 @@ class YamlToolSource(ToolSource): def parse_edam_topics(self): return self.root_dict.get("edam_topics", []) + def parse_xrefs(self): + return self.root_dict.get("xrefs", []) + def parse_is_multi_byte(self): return self.root_dict.get("is_multi_byte", self.default_is_multi_byte) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index a9c208e62ca..3c753d87899 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -47,7 +47,8 @@ A ``data_source`` tool contains a few more relevant attributes. - + + - - - - - - - - - + + + + + + + + + + + + `` tags. +A tool can refer multiple reference IDs. + +```xml + + + seqtk + + +``` + +]]> + + + + + + + + + Each xrefs element can contain one or +more ``xref`` tag elements - each of which specifies reference information +according to a catalog or an other. + + + + + + + Type of reference - currently ``bio.tools`` +is the only supported options. + + + + + + + + Type of Reference. + + + + + + diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 920dec40203..782a8269c3a 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -808,6 +808,7 @@ class Tool(Dictifiable): self.containers = containers self.citations = self._parse_citations(tool_source) + self.xrefs = self._parse_xrefs(tool_source) # Determine if this tool can be used in workflows self.is_workflow_compatible = self.check_workflow_compatible(tool_source) @@ -1079,6 +1080,24 @@ class Tool(Dictifiable): citations.append(citation) return citations + def _parse_xrefs(self, tool_source): + if not hasattr(tool_source, 'root'): + return [] + root = tool_source.root + xrefs = [] + xrefs_elem = root.find("xrefs") + if xrefs_elem is None: + return xrefs + + for xref_elem in xrefs_elem: + if xref_elem.tag != "xref": + pass + if hasattr(self.app, 'xrefs_manager'): + xref = self.app.xrefs_manager.parse_xref(xref_elem) + if xref: + xrefs.append(xref) + return xrefs + def parse_input_elem(self, page_source, enctypes, context=None): """ Parse a parent element whose children are inputs -- these could be @@ -1989,6 +2008,7 @@ class Tool(Dictifiable): 'id' : self.id, 'help' : tool_help, 'citations' : bool(self.citations), + 'xrefs' : bool(self.xrefs), 'sharable_url' : self.sharable_url, 'message' : tool_message, 'warnings' : tool_warnings, diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index d86a11d5b9b..6374169b0c2 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -403,6 +403,14 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): rval.append(citation.to_dict('bibtex')) return rval + @expose_api_anonymous_and_sessionless + def xrefs(self, trans, id, **kwds): + tool = self._get_tool(id, user=trans.user) + rval = [] + for xref in tool.xrefs: + rval.append(xref.to_dict()) + return rval + @web.legacy_expose_api_raw @web.require_admin def download(self, trans, id, **kwds): diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 8f9825ed845..a2145988d06 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -348,6 +348,7 @@ def populate_api_routes(webapp, app): webapp.mapper.connect('/api/tools/{id:.+?}/test_data', action='test_data', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/diagnostics', action='diagnostics', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/citations', action='citations', controller="tools") + webapp.mapper.connect('/api/tools/{id:.+?}/xrefs', action='xrefs', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/download', action='download', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/requirements', action='requirements', controller="tools") webapp.mapper.connect('/api/tools/{id:.+?}/install_dependencies', action='install_dependencies', controller="tools", conditions=dict(method=["POST"])) From 17807f4ddd9499b71b5be7724fa10085a466f8b8 Mon Sep 17 00:00:00 2001 From: ValentinMarcon Date: Tue, 25 Jun 2019 09:44:55 +0200 Subject: [PATCH 02/31] advices from nsoranzo --- lib/galaxy/tool_util/xsd/galaxy.xsd | 8 +++----- lib/galaxy/webapps/galaxy/api/tools.py | 5 +---- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 3c753d87899..2f4e839d557 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -48,7 +48,7 @@ A ``data_source`` tool contains a few more relevant attributes. - + - Each xrefs element can contain one or -more ``xref`` tag elements - each of which specifies reference information -according to a catalog or an other. - + The ``xref`` element specifies reference +information according to a catalog. diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 6374169b0c2..f62fc1bb619 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -406,10 +406,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): @expose_api_anonymous_and_sessionless def xrefs(self, trans, id, **kwds): tool = self._get_tool(id, user=trans.user) - rval = [] - for xref in tool.xrefs: - rval.append(xref.to_dict()) - return rval + return [_.to_dict() for _ in tool.xrefs] @web.legacy_expose_api_raw @web.require_admin From f9fa46670a2452d6a8f2ebc0e9d2cbefaeba1dde Mon Sep 17 00:00:00 2001 From: ValentinMarcon Date: Tue, 25 Jun 2019 13:33:25 +0200 Subject: [PATCH 03/31] correct space insert --- client/galaxy/scripts/mvc/tool/tool-form-base.js | 1 - 1 file changed, 1 deletion(-) diff --git a/client/galaxy/scripts/mvc/tool/tool-form-base.js b/client/galaxy/scripts/mvc/tool/tool-form-base.js index c2b47fe70d4..b98d5f1929b 100644 --- a/client/galaxy/scripts/mvc/tool/tool-form-base.js +++ b/client/galaxy/scripts/mvc/tool/tool-form-base.js @@ -269,7 +269,6 @@ export default FormBase.extend({ _footer: function() { var options = this.model.attributes; var $el = $("
").append(this._templateHelp(options)); - if (options.citations) { var citationInstance = Vue.extend(Citations); var vm = document.createElement("div"); From 8df9d5f8c0996b21cd42c514c4894469fe29e92c Mon Sep 17 00:00:00 2001 From: ValentinMarcon Date: Tue, 25 Jun 2019 13:34:22 +0200 Subject: [PATCH 04/31] remove xref manager --- lib/galaxy/config.py | 3 +-- lib/galaxy/managers/xrefs.py | 31 ------------------------------- lib/galaxy/tools/__init__.py | 10 +++++----- 3 files changed, 6 insertions(+), 38 deletions(-) delete mode 100644 lib/galaxy/managers/xrefs.py diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 6a384b42df9..f81a2293e95 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -38,6 +38,7 @@ from .version import VERSION_MAJOR log = logging.getLogger(__name__) + PATH_DEFAULTS = dict( auth_config_file=['config/auth_conf.xml', 'config/auth_conf.xml.sample'], data_manager_config_file=['config/data_manager_conf.xml', 'data_manager_conf.xml', 'config/data_manager_conf.xml.sample'], @@ -1059,13 +1060,11 @@ class ConfiguresGalaxyMixin(object): def _configure_toolbox(self): from galaxy import tools from galaxy.managers.citations import CitationsManager - from galaxy.managers.xrefs import xrefsManager from galaxy.tool_util.deps import containers from galaxy.tool_util.deps.dependencies import AppInfo import galaxy.tools.search self.citations_manager = CitationsManager(self) - self.xrefs_manager = xrefsManager(self) from galaxy.managers.tools import DynamicToolManager self.dynamic_tools_manager = DynamicToolManager(self) diff --git a/lib/galaxy/managers/xrefs.py b/lib/galaxy/managers/xrefs.py deleted file mode 100644 index 917c82b8571..00000000000 --- a/lib/galaxy/managers/xrefs.py +++ /dev/null @@ -1,31 +0,0 @@ - -class xrefsManager(object): - - def __init__(self, app): - self.app = app - - def parse_xref(self, xref_elem): - return parse_xref(xref_elem, self) - - def _get_tool(self, tool_id): - tool = self.app.toolbox.get_tool(tool_id) - return tool - - -def parse_xref(elem, tool_ref_manager): - """ Parse a xreference entry from the specified XML element.""" - return xref(elem) - - -class Base_xref(object): - def to_dict(self): - return dict( - reftype=self.reftype, - content=self.content, - ) - - -class xref(Base_xref): - def __init__(self, elem): - self.content = elem.text.strip() - self.reftype = elem.attrib.get('type', None) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 782a8269c3a..349f7c6205f 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1088,14 +1088,14 @@ class Tool(Dictifiable): xrefs_elem = root.find("xrefs") if xrefs_elem is None: return xrefs - for xref_elem in xrefs_elem: if xref_elem.tag != "xref": pass - if hasattr(self.app, 'xrefs_manager'): - xref = self.app.xrefs_manager.parse_xref(xref_elem) - if xref: - xrefs.append(xref) + xref = dict( + reftype=xref_elem.text.strip(), + content=xref_elem.attrib.get('type', None) + ) + xrefs.append(xref) return xrefs def parse_input_elem(self, page_source, enctypes, context=None): From 40e42b852262c8693cddca475e2a73d0ec368711 Mon Sep 17 00:00:00 2001 From: ValentinMarcon Date: Wed, 26 Jun 2019 10:11:16 +0200 Subject: [PATCH 05/31] correct reftype-content invert --- lib/galaxy/tools/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 349f7c6205f..6d70823fa84 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1092,8 +1092,8 @@ class Tool(Dictifiable): if xref_elem.tag != "xref": pass xref = dict( - reftype=xref_elem.text.strip(), - content=xref_elem.attrib.get('type', None) + reftype=xref_elem.attrib.get('type', None), + content=xref_elem.text.strip() ) xrefs.append(xref) return xrefs From 3eacbeb794858fe5fa5fbda40344b5e9f61303ad Mon Sep 17 00:00:00 2001 From: Valentin Marcon Date: Wed, 26 Jun 2019 17:15:30 +0200 Subject: [PATCH 06/31] Update lib/galaxy/tools/__init__.py Co-Authored-By: Nicola Soranzo --- lib/galaxy/tools/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 6d70823fa84..59ca8c21c17 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -808,7 +808,7 @@ class Tool(Dictifiable): self.containers = containers self.citations = self._parse_citations(tool_source) - self.xrefs = self._parse_xrefs(tool_source) + self.xrefs = tool_source.parse_xrefs() # Determine if this tool can be used in workflows self.is_workflow_compatible = self.check_workflow_compatible(tool_source) From 7347e57509dc24b60bf3d6f9d77a59982068939b Mon Sep 17 00:00:00 2001 From: ValentinMarcon Date: Thu, 27 Jun 2019 11:41:31 +0200 Subject: [PATCH 07/31] add parsing xml and yaml test --- test/unit/tools/test_parsing.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/test/unit/tools/test_parsing.py b/test/unit/tools/test_parsing.py index 168bd09f0fd..ca04095367b 100644 --- a/test/unit/tools/test_parsing.py +++ b/test/unit/tools/test_parsing.py @@ -12,6 +12,9 @@ from galaxy.util import galaxy_directory TOOL_XML_1 = """