From 7692e6203663bd3c3a711194a5d53c0526660cdc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 10:01:19 -0400 Subject: [PATCH 01/19] Load EDAM from PyPI if not configured instead of unexpected download. --- lib/galaxy/dependencies/dev-requirements.txt | 1 + .../dependencies/pinned-requirements.txt | 1 + lib/galaxy/tool_util/edam_util.py | 62 +++++++++++++++++++ lib/galaxy/tool_util/toolbox/base.py | 49 ++------------- lib/galaxy/webapps/galaxy/config_schema.yml | 2 +- packages/test.sh | 2 +- packages/tool_util/setup.py | 3 + pyproject.toml | 1 + test/unit/tool_util/test_edam_util.py | 28 +++++++++ 9 files changed, 102 insertions(+), 47 deletions(-) create mode 100644 lib/galaxy/tool_util/edam_util.py create mode 100644 test/unit/tool_util/test_edam_util.py diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index fa0c98b11c3..85b8fdd2ce9 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -60,6 +60,7 @@ docopt==0.6.2 docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") dogpile.cache==1.1.2; python_version >= "3.6" ecdsa==0.14.1; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" +edam-ontology==1.25.2 fabric3==1.14.post1 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0" fastapi==0.65.2; python_version >= "3.6" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index bd4d0587550..d88ba36986c 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -55,6 +55,7 @@ docopt==0.6.2 docutils==0.16; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.5.0") dogpile.cache==1.1.2; python_version >= "3.6" ecdsa==0.14.1; python_version >= "2.6" and python_full_version < "3.0.0" or python_full_version >= "3.3.0" +edam-ontology==1.25.2 fabric3==1.14.post1 fastapi-utils==0.2.1; python_version >= "3.6" and python_version < "4.0" fastapi==0.65.2; python_version >= "3.6" diff --git a/lib/galaxy/tool_util/edam_util.py b/lib/galaxy/tool_util/edam_util.py new file mode 100644 index 00000000000..fb8917e64b0 --- /dev/null +++ b/lib/galaxy/tool_util/edam_util.py @@ -0,0 +1,62 @@ +import io +import os +from typing import Dict, Optional, TextIO + +try: + from edam_ontology.streams import tabular_stream +except ImportError: + tabular_stream = None + +EDAM_PREFIX = 'http://edamontology.org/' +COLUMN_TERM = 0 +COLUMN_LABEL = 1 +COLUMN_PARENTS = 7 + + +def load_edam_tree(path: Optional[str] = None): + if path is not None: + assert os.path.exists(path), f"Failed to load EDAM tabular data at [{path}] path does not exist." + handle = io.open(path, "r") + else: + assert tabular_stream is not None, "Failed to load optional import from edam-onotology package, install using [pip install edam-ontology]." + handle = tabular_stream() + return load_edam_tree_from_tsv_stream(handle) + + +def load_edam_tree_from_tsv_stream(tsv_stream: TextIO): + edam: Dict[str, Dict] = {} + + def _recurse_edam_parents(term, path=None): + if edam[term]['parents'] and len(edam[term]['parents']) > 0: + for parent in edam[term]['parents']: + yield from _recurse_edam_parents(parent, path + [parent]) + else: + yield path + + for line in tsv_stream.readlines(): + fields = line.split('\t') + + term = fields[COLUMN_TERM] + if not term.startswith(EDAM_PREFIX): + continue + + term_id = term[len(EDAM_PREFIX):] + + # Only care about formats and operations + if not (term_id.startswith('operation_') or term_id.startswith('topic_')): + continue + + parents = fields[COLUMN_PARENTS].split('|') + edam[term_id] = { + 'label': fields[COLUMN_LABEL], # preferred label + 'parents': [x[len(EDAM_PREFIX):] for x in parents if x.startswith(EDAM_PREFIX)], + } + + for term in sorted(edam.keys()): + tails = [] + for x in _recurse_edam_parents(term, path=[]): + if x[-2:] not in tails: + tails.append(x[-2:]) + edam[term]['path'] = tails + + return edam diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 4728139c2ff..a42b4208aee 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -3,7 +3,6 @@ import logging import os import string import time -import urllib.request from collections import namedtuple from errno import ENOENT from urllib.parse import urlparse @@ -19,14 +18,13 @@ from galaxy.tool_util.deps import ( build_dependency_manager, NullDependencyManager ) +from galaxy.tool_util.edam_util import load_edam_tree from galaxy.tool_util.loader_directory import looks_like_a_tool from galaxy.util import ( etree, ExecutionTimer, - get_charset_from_http_headers, listify, parse_xml, - stream_to_open_named_file, string_as_bool, unicodify, ) @@ -124,52 +122,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if save_integrated_tool_panel: self._save_integrated_tool_panel() - def _recurse_edam_parents(self, term, path=None): - log.debug(f"term:{term} path:{path} parents:{self.edam[term]['parents']}") - if self.edam[term]['parents'] and len(self.edam[term]['parents']) > 0: - for parent in self.edam[term]['parents']: - yield from self._recurse_edam_parents(parent, path + [parent]) - else: - yield path - def _load_edam(self): if not self.app.config.enable_beta_edam_toolbox: return log.debug('Loading EDAM Terms') - - if not os.path.exists(self.app.config.beta_edam_toolbox_ontology_path): - log.debug('EDAM ontology file not present, downloading') - - page = urllib.request.urlopen('https://edamontology.org/EDAM.tsv') - stream_to_open_named_file(page, os.open(self.app.config.beta_edam_toolbox_ontology_path, os.O_WRONLY | os.O_CREAT), None, source_encoding=get_charset_from_http_headers(page.headers)) - - log.debug('Processing EDAM Terms') - with open(self.app.config.beta_edam_toolbox_ontology_path, 'r') as handle: - log.debug(f'Processing {handle}') - self.edam = {} - for line in handle.readlines(): - fields = line.split('\t') - if not fields[0].startswith('http://edamontology.org/'): - continue - term_id = fields[0][len('http://edamontology.org/'):] - - # Only care about formats and operations - if not (term_id.startswith('operation_') or term_id.startswith('topic_')): - continue - - parents = fields[7].split('|') - self.edam[term_id] = { - 'label': fields[1], # preferred label - 'parents': [x[len('http://edamontology.org/'):] for x in parents if x.startswith('http://edamontology.org/')], - } - log.debug(f'Loaded term {term_id} => {fields[1]}') - - for term in sorted(self.edam.keys()): - tails = [] - for x in self._recurse_edam_parents(term, path=[]): - if x[-2:] not in tails: - tails.append(x[-2:]) - self.edam[term]['path'] = tails + edam_path = self.app.config.beta_edam_toolbox_ontology_path + edam = load_edam_tree(None if not os.path.exists(edam_path) else edam_path) + self.edam = edam def create_tool(self, config_file, tool_shed_repository=None, guid=None, **kwds): raise NotImplementedError() diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index be85ddc24ef..4d2739e3351 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -2454,7 +2454,7 @@ mapping: path_resolves_to: data_dir required: false desc: | - Sets the path to EDAM ontology file. + Sets the path to EDAM ontology file - if the path doesn't exist PyPI package data will be loaded. default_workflow_export_format: type: str diff --git a/packages/test.sh b/packages/test.sh index 033e1d8d202..0c9ddfe5f2f 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -50,7 +50,7 @@ for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do pip install -e '.[template,jstree]' fi if [ "$package_dir" = "tool_util" ]; then - pip install -e '.[mulled]' + pip install -e '.[mulled,edam]' fi if [[ "$run_tests" == "1" ]]; then diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py index 2d3a6a349ad..d7863021cd3 100644 --- a/packages/tool_util/setup.py +++ b/packages/tool_util/setup.py @@ -104,6 +104,9 @@ setup( 'jinja2', 'Whoosh', ], + 'edam': [ + 'edam-ontology', + ], }, license="AFL", zip_safe=False, diff --git a/pyproject.toml b/pyproject.toml index e85ffdbee53..985e6947c69 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ circus = "*" cwltool = "==3.0.20201109103151" dictobj = "*" docutils = "!=0.17, !=0.17.1" +edam-ontology = "*" Fabric3 = "*" fastapi = ">=0.65.2" fastapi-utils = "*" diff --git a/test/unit/tool_util/test_edam_util.py b/test/unit/tool_util/test_edam_util.py new file mode 100644 index 00000000000..7697f547728 --- /dev/null +++ b/test/unit/tool_util/test_edam_util.py @@ -0,0 +1,28 @@ +import tempfile +from typing import Any, Dict + +from edam_ontology.streams import tabular_stream + +from galaxy.tool_util.edam_util import load_edam_tree + + +def test_load_edam_tree(): + tree = load_edam_tree() + _verify_tree(tree) + + +def test_load_edam_tree_from_path(): + tf = tempfile.NamedTemporaryFile("w") + tf.write(tabular_stream().read()) + tf.flush() + tree = load_edam_tree(tf.name) + _verify_tree(tree) + + +def _verify_tree(tree: Dict[str, Any]): + assert tree is not None + assert "operation_0004" in tree + assert "topic_3974" in tree + assert tree["topic_3974"]["label"] == "Epistasis" + assert tree["topic_3974"]["parents"] == ['topic_0622', 'topic_3295'] + assert tree["topic_3974"]["path"] == [['topic_3391', 'topic_0003'], ['topic_3070', 'topic_0003']] From 7b665d81679b033c471817a01c60aa33847f875a Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 10:50:16 -0400 Subject: [PATCH 02/19] test case for get_integrated_section_for_tool method --- test/unit/tools/test_toolbox.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index 13ef949bd0f..ab06eac419b 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -371,6 +371,10 @@ class ToolBoxTestCase(BaseToolBoxTestCase): assert len(section.elems) == 1 assert next(iter(section.elems.values())).id == "github.com/galaxyproject/example/test_tool/0.2" + test_tool = self.toolbox.get_tool("test_tool", tool_version="0.1") + section_pair = self.toolbox.get_integrated_section_for_tool(test_tool) + assert section_pair == ("tid", "TID") + def test_group_tools_out_of_section(self): self._init_tool() self._setup_two_versions_in_config(section=False) From 7aba4216f910c314c002a77bf5ee5bc5cf107afa Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 10:53:30 -0400 Subject: [PATCH 03/19] Some initial typing for toolbox lineage stuff. --- lib/galaxy/tool_util/toolbox/lineages/factory.py | 15 +++++++++------ .../tool_util/toolbox/lineages/interface.py | 12 ++++++------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/lineages/factory.py b/lib/galaxy/tool_util/toolbox/lineages/factory.py index a9f4f98563c..e7053e5c6b3 100644 --- a/lib/galaxy/tool_util/toolbox/lineages/factory.py +++ b/lib/galaxy/tool_util/toolbox/lineages/factory.py @@ -1,3 +1,5 @@ +from typing import Dict, Optional + from galaxy.util.tool_version import remove_version_from_guid from .interface import ToolLineage @@ -7,16 +9,17 @@ class LineageMap: """ def __init__(self, app): - self.lineage_map = {} + self.lineage_map: Dict[str, ToolLineage] = {} self.app = app - def register(self, tool): + def register(self, tool) -> ToolLineage: tool_id = tool.id versionless_tool_id = remove_version_from_guid(tool_id) - lineage = self.lineage_map.get(versionless_tool_id) - if not lineage: + lineage: ToolLineage + if versionless_tool_id not in self.lineage_map: lineage = ToolLineage.from_tool(tool) else: + lineage = self.lineage_map[versionless_tool_id] # A lineage for a tool with the same versionless_tool_id exists, # but this lineage may not have the current tools' version, # so we add tool.version to the lineage @@ -27,7 +30,7 @@ class LineageMap: self.lineage_map[tool_id] = lineage return self.lineage_map[tool_id] - def get(self, tool_id): + def get(self, tool_id) -> Optional[ToolLineage]: """ Get lineage for `tool_id`. @@ -46,7 +49,7 @@ class LineageMap: self.lineage_map[tool_id] = lineage return self.lineage_map.get(tool_id) - def _get_versionless(self, tool_id): + def _get_versionless(self, tool_id) -> Optional[ToolLineage]: versionless_tool_id = remove_version_from_guid(tool_id) return self.lineage_map.get(versionless_tool_id, None) diff --git a/lib/galaxy/tool_util/toolbox/lineages/interface.py b/lib/galaxy/tool_util/toolbox/lineages/interface.py index eb305a25d76..b8f48657bac 100644 --- a/lib/galaxy/tool_util/toolbox/lineages/interface.py +++ b/lib/galaxy/tool_util/toolbox/lineages/interface.py @@ -1,5 +1,5 @@ import threading -from typing import Dict +from typing import Any, Dict, List import packaging.version from sortedcontainers import SortedSet @@ -44,13 +44,13 @@ class ToolLineage: self.tool_versions = SortedSet(key=packaging.version.parse) @property - def tool_ids(self): + def tool_ids(self) -> List[str]: versionless_tool_id = remove_version_from_guid(self.tool_id) tool_id = versionless_tool_id or self.tool_id return [f"{tool_id}/{version}" for version in self.tool_versions] @staticmethod - def from_tool(tool): + def from_tool(tool) -> 'ToolLineage': tool_id = tool.id lineages_by_id = ToolLineage.lineages_by_id with ToolLineage.lock: @@ -60,7 +60,7 @@ class ToolLineage: lineage.register_version(tool.version) return lineage - def register_version(self, tool_version): + def register_version(self, tool_version) -> None: assert tool_version is not None self.tool_versions.add(str(tool_version)) @@ -71,12 +71,12 @@ class ToolLineage: """ return [ToolLineageVersion(tool_id, tool_version) for tool_id, tool_version in zip(self.tool_ids, self.tool_versions)] - def get_version_ids(self, reverse=False): + def get_version_ids(self, reverse=False) -> List[str]: if reverse: return list(reversed(self.tool_ids)) return self.tool_ids - def to_dict(self): + def to_dict(self) -> Dict[str, Any]: return dict( tool_id=self.tool_id, tool_versions=list(self.tool_versions), From 45f7dcb10c9767ab76bb13ae3fbe90591d1f9ef8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 10:57:50 -0400 Subject: [PATCH 04/19] Rename method for clarify it is a general view concern not a merged toolbox concern. --- lib/galaxy/tool_util/toolbox/base.py | 2 +- lib/galaxy/tools/__init__.py | 2 +- test/unit/tools/test_toolbox.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index a42b4208aee..91381d84e45 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -329,7 +329,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): log.debug(f"Loading new tool panel section: {str(tool_section.name)}") return tool_section - def get_integrated_section_for_tool(self, tool): + def get_section_for_tool(self, tool): tool_id = tool.id if tool_id in self._integrated_section_by_tool: diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index e4dcc7b6669..efbb7950443 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -760,7 +760,7 @@ class Tool(Dictifiable): return self.app.job_config.get_destination(self.__get_job_tool_configuration(job_params=job_params).destination) def get_panel_section(self): - return self.app.toolbox.get_integrated_section_for_tool(self) + return self.app.toolbox.get_section_for_tool(self) def allow_user_access(self, user, attempting_access=True): """ diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index ab06eac419b..5728f20d99a 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -372,7 +372,7 @@ class ToolBoxTestCase(BaseToolBoxTestCase): assert next(iter(section.elems.values())).id == "github.com/galaxyproject/example/test_tool/0.2" test_tool = self.toolbox.get_tool("test_tool", tool_version="0.1") - section_pair = self.toolbox.get_integrated_section_for_tool(test_tool) + section_pair = self.toolbox.get_section_for_tool(test_tool) assert section_pair == ("tid", "TID") def test_group_tools_out_of_section(self): From 7f07987175e83021e8ea80e1692b60531b4bd8f8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 11:10:07 -0400 Subject: [PATCH 05/19] Refactor toolbox._integrated_section_by_tool to make it clear tool_panel is responsible for section info. --- lib/galaxy/tool_util/toolbox/base.py | 19 +++++++------------ lib/galaxy/tool_util/toolbox/panel.py | 11 ++++++++++- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 91381d84e45..5a67a94ab82 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -86,7 +86,6 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): self._dynamic_tool_confs = [] self._tools_by_id = {} self._tools_by_uuid = {} - self._integrated_section_by_tool = {} # Tool lineages can contain chains of related tools with different ids # so each will be present once in the above dictionary. The following # dictionary can instead hold multiple tools with different versions. @@ -331,11 +330,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): def get_section_for_tool(self, tool): tool_id = tool.id - - if tool_id in self._integrated_section_by_tool: - return self._integrated_section_by_tool[tool_id] - - return None, None + return self._tool_panel.get_section_for_tool_id(tool_id) def __resolve_tool_path(self, tool_path, config_filename): if not tool_path: @@ -522,7 +517,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): for (term, tool_id, key, val, val_name) in operations[term].values(): section = self._get_section(term, self.edam[term]['label']) self.__add_tool_to_tool_panel(val, section, section=True) - self._integrated_section_by_tool[tool_id] = key, val_name + self._tool_panel.record_section_for_tool_id(tool_id, key, val_name) for term in sorted(topics.keys(), key=lambda x: self._sort_edam_key(x)): if len(topics[term].keys()) == 0: @@ -536,13 +531,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): for (term, tool_id, key, val, val_name) in topics[term].values(): section = self._get_section(term, self.edam[term]['label']) self.__add_tool_to_tool_panel(val, section, section=True) - self._integrated_section_by_tool[tool_id] = key, val_name + self._tool_panel.record_section_for_tool_id(tool_id, key, val_name) section = self._get_section('uncategorized', 'Uncategorized') for (tool_id, key, val, val_name) in uncategorized: print(tool_id, key, val, val_name) self.__add_tool_to_tool_panel(val, section, section=True) - self._integrated_section_by_tool[tool_id] = key, val_name + self._tool_panel.record_section_for_tool_id(tool_id, key, val_name) def _sort_edam_key(self, x): if x in ('operation_0004', 'topic_0003'): @@ -557,7 +552,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): tool_id = key.replace('tool_', '', 1) if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel(val, self._tool_panel, section=False) - self._integrated_section_by_tool[tool_id] = '', '' + self._tool_panel.record_section_for_tool_id(tool_id, '', '') elif item_type == panel_item_types.WORKFLOW: workflow_id = key.replace('workflow_', '', 1) if workflow_id in self._workflows_by_id: @@ -579,7 +574,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): tool_id = section_key.replace('tool_', '', 1) if tool_id in self._tools_by_id: self.__add_tool_to_tool_panel(section_val, section, section=True) - self._integrated_section_by_tool[tool_id] = key, val.name + self._tool_panel.record_section_for_tool_id(tool_id, key, val.name) elif section_item_type == panel_item_types.WORKFLOW: workflow_id = section_key.replace('workflow_', '', 1) if workflow_id in self._workflows_by_id: @@ -943,7 +938,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if section_item_type == panel_item_types.TOOL: if section_item: tool_id = section_key.replace('tool_', '', 1) - self._integrated_section_by_tool[tool_id] = integrated_section.id, integrated_section.name + self._tool_panel.record_section_for_tool_id(tool_id, integrated_section.id, integrated_section.name) if load_panel_dict: self._tool_panel[key] = section diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py index 71d67a78c5e..a33e164a8a2 100644 --- a/lib/galaxy/tool_util/toolbox/panel.py +++ b/lib/galaxy/tool_util/toolbox/panel.py @@ -1,5 +1,5 @@ from abc import abstractmethod - +from typing import Dict, Optional, Tuple from galaxy.util import bunch from galaxy.util.dictifiable import Dictifiable @@ -115,6 +115,15 @@ class ToolPanelElements(odict, HasPanelItems): """ Represents an ordered dictionary of tool entries - abstraction used both by tool panel itself (normal and integrated) and its sections. """ + _section_by_tool: Dict[str, Tuple[str, str]] = {} + + def record_section_for_tool_id(self, tool_id: str, key: str, val: str): + self._section_by_tool[tool_id] = (key, val) + + def get_section_for_tool_id(self, tool_id: str) -> Tuple[Optional[str], Optional[str]]: + if tool_id in self._section_by_tool: + return self._section_by_tool[tool_id] + return (None, None) def update_or_append(self, index, key, value): if key in self or index is None: From e5747b294ae5265da24680289b0734845b0973b5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 11:39:16 -0400 Subject: [PATCH 06/19] Abstraction for replacing tools in tool panel. --- lib/galaxy/tool_util/toolbox/base.py | 10 +--------- lib/galaxy/tool_util/toolbox/panel.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 5a67a94ab82..5ce50f50636 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -1095,15 +1095,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): new_tool.installed_changeset_revision = old_tool.installed_changeset_revision new_tool.old_id = old_tool.old_id # Replace old_tool with new_tool in self._tool_panel - tool_key = f"tool_{tool_id}" - for key, val in self._tool_panel.items(): - if key == tool_key: - self._tool_panel[key] = new_tool - break - elif key.startswith('section'): - if tool_key in val.elems: - self._tool_panel[key].elems[tool_key] = new_tool - break + self._tool_panel.replace_tool_for_id(tool_id, new_tool) # (Re-)Register the reloaded tool, this will handle # _tools_by_id and _tool_versions_by_id self.register_tool(new_tool) diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py index a33e164a8a2..89075ad287a 100644 --- a/lib/galaxy/tool_util/toolbox/panel.py +++ b/lib/galaxy/tool_util/toolbox/panel.py @@ -125,6 +125,17 @@ class ToolPanelElements(odict, HasPanelItems): return self._section_by_tool[tool_id] return (None, None) + def replace_tool_for_id(self, tool_id: str, new_tool) -> None: + tool_key = f"tool_{tool_id}" + for key, val in self.items(): + if key == tool_key: + self[key] = new_tool + break + elif key.startswith('section'): + if tool_key in val.elems: + self[key].elems[tool_key] = new_tool + break + def update_or_append(self, index, key, value): if key in self or index is None: self[key] = value From 2314058fe0c590d2724b12ceae3d6a606efdc2fd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 12:41:09 -0400 Subject: [PATCH 07/19] Reduce duplication around loading EDAM panel, fix log message. --- lib/galaxy/tool_util/toolbox/base.py | 78 ++++++++++------------------ 1 file changed, 28 insertions(+), 50 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 5ce50f50636..e30bf81eb16 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -431,6 +431,19 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if parentTerm in self.edam[term]['parents']: yield term + def _walk_integrated_tools(self): + for key, item_type, val in self._integrated_tool_panel.panel_items_iter(): + if item_type == panel_item_types.TOOL: + tool_id = key.replace('tool_', '', 1) + if tool_id in self._tools_by_id: + yield (tool_id, key, val, val.name) + elif item_type == panel_item_types.SECTION: + for section_key, section_item_type, section_val in val.panel_items_iter(): + if section_item_type == panel_item_types.TOOL: + tool_id = section_key.replace('tool_', '', 1) + if tool_id in self._tools_by_id: + yield (tool_id, key, section_val, val.name) + def _load_tool_panel_edam(self): execution_timer = ExecutionTimer() @@ -453,57 +466,21 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): } uncategorized = [] - for key, item_type, val in self._integrated_tool_panel.panel_items_iter(): - if item_type == panel_item_types.TOOL: - tool_id = key.replace('tool_', '', 1) - if tool_id in self._tools_by_id: - if tool_id in self._tools_by_id: - for term in self._get_edam_sec(val): - if term == 'uncategorized': - uncategorized.append((tool_id, key, val, val.name)) - else: - for path in self.edam[term]['path']: - if len(path) == 1: - t = term - else: - t = path[0] + for tool_id, key, val, val_name in self._walk_integrated_tools(): + for term in self._get_edam_sec(val): + if term == 'uncategorized': + uncategorized.append((tool_id, key, val, val_name)) + else: + for path in self.edam[term]['path']: + if len(path) == 1: + t = term + else: + t = path[0] - if path[0].startswith('operation_'): - operations[t][tool_id] = (term, tool_id, key, val, val.name) - elif path[0].startswith('topic_'): - topics[t][tool_id] = (term, tool_id, key, val, val.name) - # elif item_type == panel_item_types.WORKFLOW: - # workflow_id = key.replace('workflow_', '', 1) - # if workflow_id in self._workflows_by_id: - # workflow = self._workflows_by_id[workflow_id] - # self._tool_panel[key] = workflow - # log.debug(f"Loaded workflow: {workflow_id} {workflow.name}") - elif item_type == panel_item_types.SECTION: - for section_key, section_item_type, section_val in val.panel_items_iter(): - if section_item_type == panel_item_types.TOOL: - tool_id = section_key.replace('tool_', '', 1) - if tool_id in self._tools_by_id: - for term in self._get_edam_sec(section_val): - if term == 'uncategorized': - uncategorized.append((tool_id, key, section_val, val.name)) - else: - for path in self.edam[term]['path']: - if len(path) == 1: - t = term - else: - t = path[0] - - if path[0].startswith('operation_'): - operations[t][tool_id] = (term, tool_id, key, section_val, val.name) - if path[0].startswith('topic_'): - topics[t][tool_id] = (term, tool_id, key, section_val, val.name) - # elif section_item_type == panel_item_types.WORKFLOW: - # workflow_id = section_key.replace('workflow_', '', 1) - # if workflow_id in self._workflows_by_id: - # workflow = self._workflows_by_id[workflow_id] - # section.elems[section_key] = workflow - # log.debug(f"Loaded workflow: {workflow_id} {workflow.name}") - log.debug("Loading tool panel finished %s", execution_timer) + if path[0].startswith('operation_'): + operations[t][tool_id] = (term, tool_id, key, val, val_name) + elif path[0].startswith('topic_'): + topics[t][tool_id] = (term, tool_id, key, val, val_name) for term in sorted(operations.keys(), key=lambda x: self._sort_edam_key(x)): if len(operations[term].keys()) == 0: @@ -538,6 +515,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): print(tool_id, key, val, val_name) self.__add_tool_to_tool_panel(val, section, section=True) self._tool_panel.record_section_for_tool_id(tool_id, key, val_name) + log.debug("Loading EDAM tool panel finished %s", execution_timer) def _sort_edam_key(self, x): if x in ('operation_0004', 'topic_0003'): From cc187a38f334defe155eeb4f314e02aa73622983 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 12:41:37 -0400 Subject: [PATCH 08/19] Abstraction for removing tool from tool panel. --- lib/galaxy/tool_util/toolbox/base.py | 10 +--------- lib/galaxy/tool_util/toolbox/panel.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index e30bf81eb16..09ca91e8f6d 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -1099,15 +1099,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): if tool_cache: tool_cache.expire_tool(tool_id) if remove_from_panel: - tool_key = f"tool_{tool_id}" - for key, val in self._tool_panel.items(): - if key == tool_key: - del self._tool_panel[key] - break - elif key.startswith('section'): - if tool_key in val.elems: - del self._tool_panel[key].elems[tool_key] - break + self._tool_panel.remove_tool(tool_id) if tool_id in self.data_manager_tools: del self.data_manager_tools[tool_id] # TODO: do we need to manually remove from the integrated panel here? diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py index 89075ad287a..5f01e6e5478 100644 --- a/lib/galaxy/tool_util/toolbox/panel.py +++ b/lib/galaxy/tool_util/toolbox/panel.py @@ -136,6 +136,17 @@ class ToolPanelElements(odict, HasPanelItems): self[key].elems[tool_key] = new_tool break + def remove_tool(self, tool_id: str) -> None: + tool_key = f"tool_{tool_id}" + for key, val in self.items(): + if key == tool_key: + del self[key] + break + elif key.startswith('section'): + if tool_key in val.elems: + del self[key].elems[tool_key] + break + def update_or_append(self, index, key, value): if key in self or index is None: self[key] = value From 5b87813620d51b1539440f82ed7abc14f302141b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 12:44:00 -0400 Subject: [PATCH 09/19] Abstractions for root topic and operation in edam. --- lib/galaxy/tool_util/edam_util.py | 6 ++++++ lib/galaxy/tool_util/toolbox/base.py | 12 ++++++++---- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/edam_util.py b/lib/galaxy/tool_util/edam_util.py index fb8917e64b0..6f6f26059ec 100644 --- a/lib/galaxy/tool_util/edam_util.py +++ b/lib/galaxy/tool_util/edam_util.py @@ -12,6 +12,9 @@ COLUMN_TERM = 0 COLUMN_LABEL = 1 COLUMN_PARENTS = 7 +ROOT_OPERATION = 'operation_0004' +ROOT_TOPIC = 'topic_0003' + def load_edam_tree(path: Optional[str] = None): if path is not None: @@ -60,3 +63,6 @@ def load_edam_tree_from_tsv_stream(tsv_stream: TextIO): edam[term]['path'] = tails return edam + + +__all__ = ('load_edam_tree', 'ROOT_OPERATION', 'ROOT_TOPIC') diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 09ca91e8f6d..2c589ce0cd2 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -18,7 +18,11 @@ from galaxy.tool_util.deps import ( build_dependency_manager, NullDependencyManager ) -from galaxy.tool_util.edam_util import load_edam_tree +from galaxy.tool_util.edam_util import ( + load_edam_tree, + ROOT_OPERATION, + ROOT_TOPIC, +) from galaxy.tool_util.loader_directory import looks_like_a_tool from galaxy.util import ( etree, @@ -448,8 +452,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): execution_timer = ExecutionTimer() # Find the children of the top level topics - operations = ['operation_0004'] + list(self._edam_children_of('operation_0004')) - topics = ['topic_0003'] + list(self._edam_children_of('topic_0003')) + operations = [ROOT_OPERATION] + list(self._edam_children_of(ROOT_OPERATION)) + topics = [ROOT_TOPIC] + list(self._edam_children_of(ROOT_TOPIC)) # Sort them (by english label) # operations = sorted(operations, key=lambda x: self.edam[x]['label']) @@ -518,7 +522,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): log.debug("Loading EDAM tool panel finished %s", execution_timer) def _sort_edam_key(self, x): - if x in ('operation_0004', 'topic_0003'): + if x in (ROOT_OPERATION, ROOT_TOPIC): return f"!{x}" else: return self.edam[x]['label'] From a554fdd4b6d2b006184cd8148249a94d2c48d5f9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 13:10:37 -0400 Subject: [PATCH 10/19] Toolbox docstring. --- lib/galaxy/tool_util/toolbox/base.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 2c589ce0cd2..50a75ff820d 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -1115,6 +1115,10 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): return message, status def get_sections(self): + """Return section id and name pairs. + + Only used by build_tool_panel_section_select_field in tool shed client code. + """ for v in self._tool_panel.values(): if isinstance(v, ToolSection): yield (v.id, v.name) From e4d7cb520820b2d88abb42dc9934dec25d1d59ce Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 13:10:51 -0400 Subject: [PATCH 11/19] EDAM info on test tools. --- test/functional/tools/for_workflows/mapper.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/functional/tools/for_workflows/mapper.xml b/test/functional/tools/for_workflows/mapper.xml index 3a6f325d475..8160dd89300 100644 --- a/test/functional/tools/for_workflows/mapper.xml +++ b/test/functional/tools/for_workflows/mapper.xml @@ -1,4 +1,10 @@ + + topic_0102 + + + operation_3198 + From 618fa7713be510440fd7332d31fb45489c2ff678 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 13:55:35 -0400 Subject: [PATCH 12/19] Integration tests for EDAM toolbox. --- lib/galaxy/tool_util/toolbox/base.py | 3 +++ test/integration/test_edam_toolbox.py | 24 ++++++++++++++++++++++++ 2 files changed, 27 insertions(+) create mode 100644 test/integration/test_edam_toolbox.py diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 50a75ff820d..59aea84b652 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -122,6 +122,9 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): self._load_tool_panel_edam() else: self._load_tool_panel() + elif self.app.name == 'galaxy' and self.app.config.enable_beta_edam_toolbox: + # This would have been ignored in the integrated tool panel loading right? + self._load_tool_panel_edam() if save_integrated_tool_panel: self._save_integrated_tool_panel() diff --git a/test/integration/test_edam_toolbox.py b/test/integration/test_edam_toolbox.py new file mode 100644 index 00000000000..66b0ab7503c --- /dev/null +++ b/test/integration/test_edam_toolbox.py @@ -0,0 +1,24 @@ +from galaxy_test.driver import integration_util + + +class EdamToolboxIntegrationTestCase(integration_util.IntegrationTestCase): + + framework_tool_and_types = True + + def setUp(self): + super().setUp() + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["enable_beta_edam_toolbox"] = True + + def test_edam_toolbox(self): + index = self.galaxy_interactor.get("tools", data=dict(in_panel=True)) + index_as_list = index.json() + sections = [x for x in index_as_list if x["model_class"] == "ToolSection"] + section_names = [s["name"] for s in sections] + assert "Mapping" in section_names + mapping_section = [s for s in sections if s["name"] == "Mapping"][0] + mapping_section_elems = mapping_section["elems"] + # make sure our mapper tool was mapped using Edam correctly... + assert [x for x in mapping_section_elems if x["id"] == "mapper"] From 45df5c6c36b9c3be5cd468fcc58d216c83b3cf92 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 4 Aug 2021 15:57:09 -0400 Subject: [PATCH 13/19] Typing for toolbox panel. --- lib/galaxy/tool_util/toolbox/panel.py | 40 +++++++++++++-------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py index 5f01e6e5478..e8cdbf6e733 100644 --- a/lib/galaxy/tool_util/toolbox/panel.py +++ b/lib/galaxy/tool_util/toolbox/panel.py @@ -1,18 +1,17 @@ from abc import abstractmethod +from enum import Enum from typing import Dict, Optional, Tuple -from galaxy.util import bunch from galaxy.util.dictifiable import Dictifiable from galaxy.util.odict import odict from .parser import ensure_tool_conf_item -panel_item_types = bunch.Bunch( - TOOL="TOOL", - WORKFLOW="WORKFLOW", - SECTION="SECTION", - LABEL="LABEL", -) +class panel_item_types(str, Enum): + TOOL = "TOOL" + WORKFLOW = "WORKFLOW" + SECTION = "SECTION" + LABEL = "LABEL" class HasPanelItems: @@ -147,61 +146,62 @@ class ToolPanelElements(odict, HasPanelItems): del self[key].elems[tool_key] break - def update_or_append(self, index, key, value): + def update_or_append(self, index: int, key: str, value) -> None: if key in self or index is None: self[key] = value else: self.insert(index, key, value) - def get_label(self, label): + def get_label(self, label: str) -> Optional[ToolSection]: for element in self.values(): if isinstance(element, ToolSection) and element.name == label: return element + return None - def has_tool_with_id(self, tool_id): + def has_tool_with_id(self, tool_id: str) -> bool: key = f'tool_{tool_id}' return key in self - def replace_tool(self, previous_tool_id, new_tool_id, tool): + def replace_tool(self, previous_tool_id: str, new_tool_id: str, tool) -> None: previous_key = f'tool_{previous_tool_id}' new_key = f'tool_{new_tool_id}' index = self.keys().index(previous_key) del self[previous_key] self.insert(index, new_key, tool) - def index_of_tool_id(self, tool_id): + def index_of_tool_id(self, tool_id: str) -> Optional[int]: query_key = f'tool_{tool_id}' for index, target_key in enumerate(self.keys()): if query_key == target_key: return index return None - def insert_tool(self, index, tool): + def insert_tool(self, index: int, tool) -> None: key = f"tool_{tool.id}" self.insert(index, key, tool) - def get_tool_with_id(self, tool_id): + def get_tool_with_id(self, tool_id: str): key = f"tool_{tool_id}" return self[key] - def append_tool(self, tool): + def append_tool(self, tool) -> None: key = f"tool_{tool.id}" self[key] = tool - def stub_tool(self, key): + def stub_tool(self, key: str) -> None: key = f"tool_{key}" self[key] = None - def stub_workflow(self, key): + def stub_workflow(self, key: str) -> None: key = f'workflow_{key}' self[key] = None - def stub_label(self, key): + def stub_label(self, key: str) -> None: key = f'label_{key}' self[key] = None - def append_section(self, key, section_elems): - self[key] = section_elems + def append_section(self, key: str, section: ToolSection) -> None: + self[key] = section def panel_items(self): return self From b036d3a81c0950d7f1cb529d3d12704df29de8da Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 5 Aug 2021 13:12:28 -0400 Subject: [PATCH 14/19] typing for galaxy.tools.search --- lib/galaxy/tools/search/__init__.py | 40 +++++++++++++++++++---------- 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 3aeb835b984..849139b6d31 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -7,6 +7,7 @@ through the library docs at https://whoosh.readthedocs.io. import logging import os import re +from typing import Dict, List, Tuple, Union from whoosh import ( analysis, @@ -34,8 +35,11 @@ from galaxy.web.framework.helpers import to_unicode log = logging.getLogger(__name__) +CanConvertToFloat = Union[str, int, float] +CanConvertToInt = Union[str, int, float] -def get_or_create_index(index_dir, schema): + +def get_or_create_index(index_dir: str, schema: Schema) -> index.Index: if not os.path.exists(index_dir): os.makedirs(index_dir) if index.exists_in(index_dir): @@ -54,7 +58,7 @@ class ToolBoxSearch: the Whoosh search library. """ - def __init__(self, toolbox, index_dir=None, index_help=True): + def __init__(self, toolbox, index_dir: str, index_help: bool = True): self.schema = Schema(id=ID(stored=True, unique=True), old_id=ID, stub=KEYWORD, @@ -73,10 +77,10 @@ class ToolBoxSearch: # reindexing if the index count is equal to the toolbox reload count. self.index_count = -1 - def _index_setup(self): + def _index_setup(self) -> index.Index: return get_or_create_index(index_dir=self.index_dir, schema=self.schema) - def build_index(self, tool_cache, index_help=True): + def build_index(self, tool_cache, index_help: bool = True) -> None: """ Prepare search index for tools loaded in toolbox. Use `tool_cache` to determine which tools need indexing and which tools should be expired. @@ -119,7 +123,7 @@ class ToolBoxSearch: writer.update_document(**add_doc_kwds) log.debug("Toolbox index finished %s", execution_timer) - def _create_doc(self, tool_id, tool, index_help=True): + def _create_doc(self, tool_id: str, tool, index_help: bool = True) -> Dict[str, str]: # Do not add data managers to the public index if tool.tool_type == 'manage_data': return {} @@ -153,10 +157,20 @@ class ToolBoxSearch: pass return add_doc_kwds - def search(self, q, tool_name_boost, tool_id_boost, tool_section_boost, - tool_description_boost, tool_label_boost, tool_stub_boost, - tool_help_boost, tool_search_limit, tool_enable_ngram_search, - tool_ngram_minsize, tool_ngram_maxsize): + def search( + self, q: str, + tool_name_boost: CanConvertToFloat, + tool_id_boost: CanConvertToFloat, + tool_section_boost: CanConvertToFloat, + tool_description_boost: CanConvertToFloat, + tool_label_boost: CanConvertToFloat, + tool_stub_boost: CanConvertToFloat, + tool_help_boost: CanConvertToFloat, + tool_search_limit: CanConvertToFloat, + tool_enable_ngram_search: bool, + tool_ngram_minsize: CanConvertToInt, + tool_ngram_maxsize: CanConvertToInt + ) -> List[str]: """ Perform search on the in-memory index. Weight in the given boosts. """ @@ -194,13 +208,13 @@ class ToolBoxSearch: hits = self.searcher.search(parsed_query, limit=float(tool_search_limit), sortedby='') return [hit['id'] for hit in hits] - def _search_ngrams(self, cleaned_query, tool_ngram_minsize, tool_ngram_maxsize, tool_search_limit): + def _search_ngrams(self, cleaned_query: str, tool_ngram_minsize: CanConvertToInt, tool_ngram_maxsize: CanConvertToInt, tool_search_limit: CanConvertToFloat) -> List[str]: """ Break tokens into ngrams and search on those instead. This should make searching more resistant to typos and unfinished words. See docs at https://whoosh.readthedocs.io/en/latest/ngrams.html """ - hits_with_score = {} + hits_with_score: Dict[str, float] = {} token_analyzer = StandardAnalyzer() | analysis.NgramFilter(minsize=int(tool_ngram_minsize), maxsize=int(tool_ngram_maxsize)) ngrams = [token.text for token in token_analyzer(cleaned_query)] for query in ngrams: @@ -219,6 +233,6 @@ class ToolBoxSearch: if not is_present: hits_with_score[curr_hit['id']] = curr_hits.score(i) # Sort the results based on aggregated BM25 score in decreasing order of scores - hits_with_score = sorted(hits_with_score.items(), key=lambda x: x[1], reverse=True) + hits_with_score_list: List[Tuple[str, float]] = sorted(hits_with_score.items(), key=lambda x: x[1], reverse=True) # Return the tool ids - return [item[0] for item in hits_with_score[0:int(tool_search_limit)]] + return [item[0] for item in hits_with_score_list[0:int(tool_search_limit)]] From 8f05a055f26aafba9d7e5bb5ad63c056c2d4caba Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 15:16:43 -0400 Subject: [PATCH 15/19] get_or_create_section abstraction in toolbox panel. --- lib/galaxy/tool_util/toolbox/base.py | 9 ++------- lib/galaxy/tool_util/toolbox/panel.py | 8 ++++++++ 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index 59aea84b652..82fac6d8518 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -425,13 +425,8 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): else: yield 'uncategorized' - def _get_section(self, sec_id, sec_nm): - if sec_id not in self._tool_panel: - section = ToolSection({'id': sec_id, 'name': sec_nm, 'version': ''}) - self._tool_panel[sec_id] = section - else: - section = self._tool_panel[sec_id] - return section + def _get_section(self, sec_id: str, sec_nm: str) -> ToolSection: + return self._tool_panel.get_or_create_section(sec_id, sec_nm) def _edam_children_of(self, parentTerm): for term in self.edam.keys(): diff --git a/lib/galaxy/tool_util/toolbox/panel.py b/lib/galaxy/tool_util/toolbox/panel.py index e8cdbf6e733..044ab8aea10 100644 --- a/lib/galaxy/tool_util/toolbox/panel.py +++ b/lib/galaxy/tool_util/toolbox/panel.py @@ -135,6 +135,14 @@ class ToolPanelElements(odict, HasPanelItems): self[key].elems[tool_key] = new_tool break + def get_or_create_section(self, sec_id: str, sec_nm: str) -> ToolSection: + if sec_id not in self: + section = ToolSection({'id': sec_id, 'name': sec_nm, 'version': ''}) + self[sec_id] = section + else: + section = self[sec_id] + return section + def remove_tool(self, tool_id: str) -> None: tool_key = f"tool_{tool_id}" for key, val in self.items(): From 2dddcac8c86043e451939453154aa760745b77d6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Aug 2021 15:15:03 -0400 Subject: [PATCH 16/19] Implement tool panel views - config, toolbox, search, client. --- .../Panels/Buttons/PanelViewButton.vue | 103 ++++++++ .../Panels/Buttons/PanelViewMenuItem.vue | 52 ++++ .../components/Panels/Common/ToolSearch.vue | 6 +- .../Panels/ProviderAwareToolBox.vue | 52 ++++ .../Panels/ProviderAwareToolBoxWorkflow.vue | 72 +++++ client/src/components/Panels/ToolBox.vue | 26 +- .../src/components/Panels/ToolBoxWorkflow.vue | 26 +- .../src/components/Workflow/Editor/Index.vue | 11 +- .../providers/ToolPanelViewProvider.js | 25 ++ client/src/entry/panels/tool-panel.js | 6 +- client/src/store/index.js | 2 + client/src/store/panelStore.js | 55 ++++ lib/galaxy/managers/configuration.py | 2 + lib/galaxy/tool_util/toolbox/base.py | 250 ++++++++---------- lib/galaxy/tool_util/toolbox/panel.py | 43 ++- lib/galaxy/tool_util/toolbox/parser.py | 6 + .../tool_util/toolbox/views/__init__.py | 0 .../tool_util/toolbox/views/definitions.py | 155 +++++++++++ lib/galaxy/tool_util/toolbox/views/edam.py | 168 ++++++++++++ .../tool_util/toolbox/views/interface.py | 75 ++++++ lib/galaxy/tool_util/toolbox/views/sources.py | 52 ++++ lib/galaxy/tool_util/toolbox/views/static.py | 168 ++++++++++++ lib/galaxy/tools/__init__.py | 8 + lib/galaxy/tools/search/__init__.py | 47 +++- lib/galaxy/webapps/galaxy/api/tools.py | 10 +- lib/galaxy/webapps/galaxy/config_schema.yml | 41 ++- lib/galaxy_test/driver/driver_util.py | 14 +- packages/tool_util/requirements.txt | 2 + packages/tool_util/setup.py | 1 + test/integration/panel_views_1/custom_1.yml | 14 + test/integration/panel_views_1/custom_10.yml | 4 + test/integration/panel_views_1/custom_2.yml | 7 + test/integration/panel_views_1/custom_3.yml | 8 + test/integration/panel_views_1/custom_4.yml | 15 ++ test/integration/panel_views_1/custom_5.yml | 9 + test/integration/panel_views_1/custom_6.yml | 13 + test/integration/panel_views_1/custom_7.yml | 13 + test/integration/panel_views_1/custom_8.yml | 9 + test/integration/panel_views_1/custom_9.yml | 7 + test/integration/panel_views_1/filter.yml | 5 + test/integration/test_edam_toolbox.py | 46 +++- test/integration/test_panel_views.py | 178 +++++++++++++ test/unit/config/test_config_values.py | 2 +- .../tool_util/test_toolbox_view_parsing.py | 101 +++++++ .../tool_util/test_toolbox_view_sources.py | 45 ++++ test/unit/unittest_utils/galaxy_mock.py | 5 + 46 files changed, 1774 insertions(+), 185 deletions(-) create mode 100644 client/src/components/Panels/Buttons/PanelViewButton.vue create mode 100644 client/src/components/Panels/Buttons/PanelViewMenuItem.vue create mode 100644 client/src/components/Panels/ProviderAwareToolBox.vue create mode 100644 client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue create mode 100644 client/src/components/providers/ToolPanelViewProvider.js create mode 100644 client/src/store/panelStore.js create mode 100644 lib/galaxy/tool_util/toolbox/views/__init__.py create mode 100644 lib/galaxy/tool_util/toolbox/views/definitions.py create mode 100644 lib/galaxy/tool_util/toolbox/views/edam.py create mode 100644 lib/galaxy/tool_util/toolbox/views/interface.py create mode 100644 lib/galaxy/tool_util/toolbox/views/sources.py create mode 100644 lib/galaxy/tool_util/toolbox/views/static.py create mode 100644 test/integration/panel_views_1/custom_1.yml create mode 100644 test/integration/panel_views_1/custom_10.yml create mode 100644 test/integration/panel_views_1/custom_2.yml create mode 100644 test/integration/panel_views_1/custom_3.yml create mode 100644 test/integration/panel_views_1/custom_4.yml create mode 100644 test/integration/panel_views_1/custom_5.yml create mode 100644 test/integration/panel_views_1/custom_6.yml create mode 100644 test/integration/panel_views_1/custom_7.yml create mode 100644 test/integration/panel_views_1/custom_8.yml create mode 100644 test/integration/panel_views_1/custom_9.yml create mode 100644 test/integration/panel_views_1/filter.yml create mode 100644 test/integration/test_panel_views.py create mode 100644 test/unit/tool_util/test_toolbox_view_parsing.py create mode 100644 test/unit/tool_util/test_toolbox_view_sources.py diff --git a/client/src/components/Panels/Buttons/PanelViewButton.vue b/client/src/components/Panels/Buttons/PanelViewButton.vue new file mode 100644 index 00000000000..84fc12ddc2e --- /dev/null +++ b/client/src/components/Panels/Buttons/PanelViewButton.vue @@ -0,0 +1,103 @@ + + + diff --git a/client/src/components/Panels/Buttons/PanelViewMenuItem.vue b/client/src/components/Panels/Buttons/PanelViewMenuItem.vue new file mode 100644 index 00000000000..7e03eb20efc --- /dev/null +++ b/client/src/components/Panels/Buttons/PanelViewMenuItem.vue @@ -0,0 +1,52 @@ + + + diff --git a/client/src/components/Panels/Common/ToolSearch.vue b/client/src/components/Panels/Common/ToolSearch.vue index 11fe95f8f77..67c12215669 100644 --- a/client/src/components/Panels/Common/ToolSearch.vue +++ b/client/src/components/Panels/Common/ToolSearch.vue @@ -14,6 +14,10 @@ export default { DelayedInput, }, props: { + currentPanelView: { + type: String, + required: true, + }, query: { type: String, default: "", @@ -46,7 +50,7 @@ export default { this.loading = true; axios .get(`${getAppRoot()}api/tools`, { - params: { q }, + params: { q, view: this.currentPanelView }, }) .then((response) => { this.loading = false; diff --git a/client/src/components/Panels/ProviderAwareToolBox.vue b/client/src/components/Panels/ProviderAwareToolBox.vue new file mode 100644 index 00000000000..ad61547a267 --- /dev/null +++ b/client/src/components/Panels/ProviderAwareToolBox.vue @@ -0,0 +1,52 @@ + + + diff --git a/client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue b/client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue new file mode 100644 index 00000000000..2ab248a6097 --- /dev/null +++ b/client/src/components/Panels/ProviderAwareToolBoxWorkflow.vue @@ -0,0 +1,72 @@ + + + diff --git a/client/src/components/Panels/ToolBox.vue b/client/src/components/Panels/ToolBox.vue index fca9bd812ce..d57a03c7b64 100644 --- a/client/src/components/Panels/ToolBox.vue +++ b/client/src/components/Panels/ToolBox.vue @@ -4,12 +4,24 @@
+
Tools
- +
@@ -52,16 +64,17 @@ import ToolSection from "./Common/ToolSection"; import ToolSearch from "./Common/ToolSearch"; import { UploadButton, openGlobalUploadModal } from "components/Upload"; import FavoritesButton from "./Buttons/FavoritesButton"; +import PanelViewButton from "./Buttons/PanelViewButton"; import { filterToolSections, filterTools } from "./utilities"; import { getGalaxyInstance } from "app"; import { getAppRoot } from "onload"; import _l from "utils/localization"; export default { - name: "ToolBox", components: { UploadButton, FavoritesButton, + PanelViewButton, ToolSection, ToolSearch, }, @@ -81,6 +94,12 @@ export default { type: Array, required: true, }, + panelViews: { + type: Object, + }, + currentPanelView: { + type: String, + }, storedWorkflowMenuEntries: { type: Array, required: true, @@ -163,6 +182,9 @@ export default { this.buttonText = this.showSections ? "Hide Sections" : "Show Sections"; this.buttonIcon = this.showSections ? "fa fa-eye-slash" : "fa fa-eye"; }, + updatePanelView(panelView) { + this.$emit("updatePanelView", panelView); + }, }, }; diff --git a/client/src/components/Panels/ToolBoxWorkflow.vue b/client/src/components/Panels/ToolBoxWorkflow.vue index 64e98721273..8847b9d1958 100644 --- a/client/src/components/Panels/ToolBoxWorkflow.vue +++ b/client/src/components/Panels/ToolBoxWorkflow.vue @@ -2,11 +2,24 @@
+
+ +
Tools
- +
@@ -58,12 +71,14 @@ import _l from "utils/localization"; import ToolSection from "./Common/ToolSection"; import ToolSearch from "./Common/ToolSearch"; import { filterToolSections } from "./utilities"; +import PanelViewButton from "./Buttons/PanelViewButton"; export default { name: "ToolBox", components: { ToolSection, ToolSearch, + PanelViewButton, }, data() { return { @@ -76,6 +91,12 @@ export default { type: Array, required: true, }, + panelViews: { + type: Object, + }, + currentPanelView: { + type: String, + }, workflows: { type: Array, required: true, @@ -147,6 +168,9 @@ export default { onInsertWorkflowSteps(workflow) { this.$emit("onInsertWorkflowSteps", workflow.id, workflow.step_count); }, + updatePanelView(panelView) { + this.$emit("updatePanelView", panelView); + }, }, }; diff --git a/client/src/components/Workflow/Editor/Index.vue b/client/src/components/Workflow/Editor/Index.vue index 5093697d162..3789b594661 100644 --- a/client/src/components/Workflow/Editor/Index.vue +++ b/client/src/components/Workflow/Editor/Index.vue @@ -39,8 +39,7 @@