From 213f3a17fce78463d22a0ae6f8720f88cf83a271 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 7 Mar 2022 11:23:08 -0500 Subject: [PATCH] Allow legacy EDAM mappings. --- lib/galaxy/tool_util/ontologies/__init__.py | 0 .../ontologies}/biotools_mappings.tsv | 0 .../ontologies/edam_operation_mappings.tsv | 2 + .../ontologies/edam_topic_mappings.tsv | 1 + .../tool_util/ontologies/ontology_data.py | 108 ++++++++++++++++++ lib/galaxy/tools/__init__.py | 60 +++------- packages/app/MANIFEST.in | 1 - packages/tool_util/MANIFEST.in | 1 + packages/tool_util/setup.py | 1 + test/unit/tool_util/test_ontologies.py | 68 +++++++++++ test/unit/tool_util/test_parsing.py | 40 ++++--- 11 files changed, 220 insertions(+), 62 deletions(-) create mode 100644 lib/galaxy/tool_util/ontologies/__init__.py rename lib/galaxy/{tools => tool_util/ontologies}/biotools_mappings.tsv (100%) create mode 100644 lib/galaxy/tool_util/ontologies/edam_operation_mappings.tsv create mode 100644 lib/galaxy/tool_util/ontologies/edam_topic_mappings.tsv create mode 100644 lib/galaxy/tool_util/ontologies/ontology_data.py create mode 100644 test/unit/tool_util/test_ontologies.py diff --git a/lib/galaxy/tool_util/ontologies/__init__.py b/lib/galaxy/tool_util/ontologies/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/tools/biotools_mappings.tsv b/lib/galaxy/tool_util/ontologies/biotools_mappings.tsv similarity index 100% rename from lib/galaxy/tools/biotools_mappings.tsv rename to lib/galaxy/tool_util/ontologies/biotools_mappings.tsv diff --git a/lib/galaxy/tool_util/ontologies/edam_operation_mappings.tsv b/lib/galaxy/tool_util/ontologies/edam_operation_mappings.tsv new file mode 100644 index 00000000000..bd68054193a --- /dev/null +++ b/lib/galaxy/tool_util/ontologies/edam_operation_mappings.tsv @@ -0,0 +1,2 @@ +# tool-id edam-operation +sort1 operation_3802 diff --git a/lib/galaxy/tool_util/ontologies/edam_topic_mappings.tsv b/lib/galaxy/tool_util/ontologies/edam_topic_mappings.tsv new file mode 100644 index 00000000000..df031e46d87 --- /dev/null +++ b/lib/galaxy/tool_util/ontologies/edam_topic_mappings.tsv @@ -0,0 +1 @@ +# tool-id edam-topic diff --git a/lib/galaxy/tool_util/ontologies/ontology_data.py b/lib/galaxy/tool_util/ontologies/ontology_data.py new file mode 100644 index 00000000000..edd3a308f65 --- /dev/null +++ b/lib/galaxy/tool_util/ontologies/ontology_data.py @@ -0,0 +1,108 @@ +from typing import ( + cast, + Dict, + List, + NamedTuple, + Optional, + Tuple, +) + +from galaxy.tool_util.biotools import BiotoolsMetadataSource +from galaxy.tool_util.parser import ToolSource +from galaxy.util.resources import files + + +def _multi_dict_mapping(content: str) -> Dict[str, List[str]]: + mapping: Dict[str, List[str]] = {} + for x in content.splitlines(): + if x.startswith("#"): + continue + key, value = cast(Tuple[str, str], tuple(x.split("\t"))) + mapping.setdefault(key, []).append(value) + return mapping + + +def _read_ontology_data_text(filename: str) -> str: + return files(PACKAGE).joinpath(filename).read_text() + + +PACKAGE = "galaxy.tool_util.ontologies" +BIOTOOLS_MAPPING_FILENAME = "biotools_mappings.tsv" +EDAM_OPERATION_MAPPING_FILENAME = "edam_operation_mappings.tsv" +EDAM_TOPIC_MAPPING_FILENAME = "edam_topic_mappings.tsv" + +BIOTOOLS_MAPPING_CONTENT = _read_ontology_data_text(BIOTOOLS_MAPPING_FILENAME) +BIOTOOLS_MAPPING: Dict[str, str] = dict( + [ + cast(Tuple[str, str], tuple(x.split("\t"))) + for x in BIOTOOLS_MAPPING_CONTENT.splitlines() + if not x.startswith("#") + ] +) +EDAM_OPERATION_MAPPING_CONTENT = _read_ontology_data_text(EDAM_OPERATION_MAPPING_FILENAME) +EDAM_OPERATION_MAPPING: Dict[str, List[str]] = _multi_dict_mapping(EDAM_OPERATION_MAPPING_CONTENT) + +EDAM_TOPIC_MAPPING_CONTENT = _read_ontology_data_text(EDAM_TOPIC_MAPPING_FILENAME) +EDAM_TOPIC_MAPPING: Dict[str, List[str]] = _multi_dict_mapping(EDAM_TOPIC_MAPPING_CONTENT) + + +class OntologyData(NamedTuple): + xrefs: List[Dict[str, str]] + edam_operations: Optional[List[str]] + edam_topics: Optional[List[str]] + + +def biotools_reference(xrefs): + for xref in xrefs: + if xref["reftype"] == "bio.tools": + return xref["value"] + return None + + +def legacy_biotools_external_reference(all_ids: List[str]) -> Optional[str]: + for tool_id in all_ids: + if tool_id in BIOTOOLS_MAPPING: + return BIOTOOLS_MAPPING[tool_id] + return None + + +def expand_ontology_data( + tool_source: ToolSource, all_ids: List[str], biotools_metadata_source: Optional[BiotoolsMetadataSource] +) -> OntologyData: + xrefs = tool_source.parse_xrefs() + has_biotools_reference = any(x["reftype"] == "bio.tools" for x in xrefs) + if not has_biotools_reference: + legacy_biotools_ref = legacy_biotools_external_reference(all_ids) + if legacy_biotools_ref is not None: + xrefs.append({"value": legacy_biotools_ref, "reftype": "bio.tools"}) + + edam_operations = tool_source.parse_edam_operations() + edam_topics = tool_source.parse_edam_topics() + + for tool_id in all_ids: + if tool_id in EDAM_OPERATION_MAPPING: + edam_operations = EDAM_OPERATION_MAPPING[tool_id] + break + + for tool_id in all_ids: + if tool_id in EDAM_TOPIC_MAPPING: + edam_topics = EDAM_TOPIC_MAPPING[tool_id] + break + + has_missing_data = len(edam_operations) == 0 or len(edam_topics) == 0 + if has_missing_data: + biotools_reference_str = biotools_reference(xrefs) + if biotools_reference_str and biotools_metadata_source: + biotools_entry = biotools_metadata_source.get_biotools_metadata(biotools_reference_str) + if biotools_entry: + edam_info = biotools_entry.edam_info + if len(edam_operations) == 0: + edam_operations = edam_info.edam_operations + if len(edam_topics) == 0: + edam_topics = edam_info.edam_topics + + return OntologyData( + xrefs, + edam_operations, + edam_topics, + ) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 4ab034d2cf7..7999f3be04d 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -48,6 +48,10 @@ from galaxy.tool_util.loader import ( raw_tool_xml_tree, template_macro_params, ) +from galaxy.tool_util.ontologies.ontology_data import ( + biotools_reference, + expand_ontology_data, +) from galaxy.tool_util.output_checker import DETECTED_JOB_STATE from galaxy.tool_util.parser import ( get_tool_source, @@ -118,7 +122,6 @@ from galaxy.util.dictifiable import Dictifiable from galaxy.util.expressions import ExpressionContext from galaxy.util.form_builder import SelectField from galaxy.util.json import safe_loads -from galaxy.util.resources import resource_string from galaxy.util.rules_dsl import RuleSet from galaxy.util.template import ( fill_template, @@ -217,15 +220,6 @@ GALAXY_LIB_TOOLS_VERSIONED = { "winSplitter": packaging.version.parse("1.0.1"), } -BIOTOOLS_MAPPING_CONTENT = resource_string(__package__, "biotools_mappings.tsv") -BIOTOOLS_MAPPING: Dict[str, str] = dict( - [ - cast(Tuple[str, str], tuple(x.split("\t"))) - for x in BIOTOOLS_MAPPING_CONTENT.splitlines() - if not x.startswith("#") - ] -) - REQUIRE_FULL_DIRECTORY = { "includes": [{"path": "**", "path_type": "glob"}], } @@ -1053,31 +1047,14 @@ class Tool(Dictifiable): self.required_files = required_files self.citations = self._parse_citations(tool_source) - xrefs = tool_source.parse_xrefs() - has_biotools_reference = any(x["reftype"] == "bio.tools" for x in xrefs) - if not has_biotools_reference: - legacy_biotools_ref = self.legacy_biotools_external_reference - if legacy_biotools_ref is not None: - xrefs.append({"value": legacy_biotools_ref, "reftype": "bio.tools"}) - self.xrefs = xrefs - - edam_operations = tool_source.parse_edam_operations() - edam_topics = tool_source.parse_edam_topics() - has_missing_data = len(edam_operations) == 0 or len(edam_topics) == 0 - if has_missing_data: - biotools_reference = self.biotools_reference - metadata_source = self.app.biotools_metadata_source - if biotools_reference and metadata_source: - biotools_entry = metadata_source.get_biotools_metadata(biotools_reference) - if biotools_entry: - edam_info = biotools_entry.edam_info - if len(edam_operations) == 0: - edam_operations = edam_info.edam_operations - if len(edam_topics) == 0: - edam_topics = edam_info.edam_topics - - self.edam_operations = edam_operations - self.edam_topics = edam_topics + ontology_data = expand_ontology_data( + tool_source, + self.all_ids, + self.app.biotools_metadata_source, + ) + self.xrefs = ontology_data.xrefs + self.edam_operations = ontology_data.edam_operations + self.edam_topics = ontology_data.edam_topics self.__parse_trackster_conf(tool_source) # Record macro paths so we can reload a tool if any of its macro has changes @@ -1553,24 +1530,13 @@ class Tool(Dictifiable): self.app, self.tool_shed, self.repository_owner, self.repository_name ) - @property - def legacy_biotools_external_reference(self) -> Optional[str]: - """Return a bio.tools ID if any of tool's IDs are BIOTOOLS_MAPPING.""" - for tool_id in self.all_ids: - if tool_id in BIOTOOLS_MAPPING: - return BIOTOOLS_MAPPING[tool_id] - return None - @property def biotools_reference(self) -> Optional[str]: """Return a bio.tools ID if external reference to it is found. If multiple bio.tools references are found, return just the first one. """ - for xref in self.xrefs: - if xref["reftype"] == "bio.tools": - return xref["value"] - return None + return biotools_reference(self.xrefs) @property def help(self): diff --git a/packages/app/MANIFEST.in b/packages/app/MANIFEST.in index d47c145d0e9..fc212e2a6ae 100644 --- a/packages/app/MANIFEST.in +++ b/packages/app/MANIFEST.in @@ -1,3 +1,2 @@ include *.rst *.txt LICENSE include galaxy/jobs/runners/util/job_script/*.sh -include galaxy/tools/*tsv diff --git a/packages/tool_util/MANIFEST.in b/packages/tool_util/MANIFEST.in index a01c749e7ee..262b6a0b7b8 100644 --- a/packages/tool_util/MANIFEST.in +++ b/packages/tool_util/MANIFEST.in @@ -3,3 +3,4 @@ include galaxy/tool_util/deps/mulled/invfile.lua include galaxy/tool_util/deps/resolvers/default_conda_mapping.yml include galaxy/tool_util/verify/test_config.sample.yml include galaxy/tool_util/xsd/* +include galaxy/tool_util/ontologies/*tsv diff --git a/packages/tool_util/setup.py b/packages/tool_util/setup.py index c79d6175352..3ed4d7fc034 100644 --- a/packages/tool_util/setup.py +++ b/packages/tool_util/setup.py @@ -40,6 +40,7 @@ PACKAGES = [ "galaxy.tool_util.deps.resolvers", "galaxy.tool_util.linters", "galaxy.tool_util.locations", + "galaxy.tool_util.ontologies", "galaxy.tool_util.parser", "galaxy.tool_util.toolbox", "galaxy.tool_util.toolbox.filters", diff --git a/test/unit/tool_util/test_ontologies.py b/test/unit/tool_util/test_ontologies.py new file mode 100644 index 00000000000..3991d358c3d --- /dev/null +++ b/test/unit/tool_util/test_ontologies.py @@ -0,0 +1,68 @@ +from galaxy.tool_util.ontologies.ontology_data import expand_ontology_data +from .test_parsing import ( + get_test_tool_source, + TOOL_YAML_1, +) + +TOOL_YAML_2 = """ +name: "Bowtie Mapper" +class: GalaxyTool +id: bowtie +version: 1.0.2 +description: "The Bowtie Mapper" +xrefs: + - type: bio.tools + value: bwa +command: "bowtie --map-the-stuff" +outputs: + out1: + format: bam + from_work_dir: out1.bam +edam_operations: + - operation_0335 +edam_topics: + - topic_0102 +inputs: + - name: input1 + type: integer +""" + +TOOL_YAML_3 = """ +name: "Bowtie Mapper" +class: GalaxyTool +id: sort1 +version: 1.0.2 +description: "The Bowtie Mapper" +xrefs: + - type: bio.tools + value: bwa +command: "bowtie --map-the-stuff" +outputs: + out1: + format: bam + from_work_dir: out1.bam +inputs: + - name: input1 + type: integer +""" + + +def test_parse_edam_empty(): + test_source = get_test_tool_source(source_file_name="testtool.yml", source_contents=TOOL_YAML_1) + ontology_data = expand_ontology_data(test_source, ["bowtie"], None) + assert ontology_data.edam_operations == [] + assert ontology_data.edam_topics == [] + + +def test_parse_edam_direct(): + test_source = get_test_tool_source(source_file_name="testtool.yml", source_contents=TOOL_YAML_2) + ontology_data = expand_ontology_data(test_source, ["bowtie"], None) + assert ontology_data.edam_operations == ["operation_0335"] + assert ontology_data.edam_topics == ["topic_0102"] + + +def test_parse_edam_mapping_operations_legacy(): + test_source = get_test_tool_source(source_file_name="testtool.yml", source_contents=TOOL_YAML_3) + ontology_data = expand_ontology_data(test_source, ["sort1"], None) + assert ontology_data.edam_operations == ["operation_3802"] + assert ontology_data.edam_topics == [] diff --git a/test/unit/tool_util/test_parsing.py b/test/unit/tool_util/test_parsing.py index 09a76d7d6e0..6d3339b23ea 100644 --- a/test/unit/tool_util/test_parsing.py +++ b/test/unit/tool_util/test_parsing.py @@ -193,6 +193,26 @@ outputs: """ +def get_test_tool_source(source_file_name=None, source_contents=None, macro_contents=None, temp_directory=None): + source_directory = temp_directory or tempfile.mkdtemp() + macro_paths = [] + if not os.path.isabs(source_file_name): + path = os.path.join(source_directory, source_file_name) + with open(path, "w") as out: + out.write(source_contents) + if macro_contents: + macro_path = os.path.join(source_directory, "macros.xml") + with open(macro_path, "w") as out: + out.write(macro_contents) + macro_paths = [macro_path] + else: + path = source_file_name + tool_source = get_tool_source(path, macro_paths=macro_paths) + if temp_directory is None: + shutil.rmtree(source_directory) + return tool_source + + class BaseLoaderTestCase(unittest.TestCase): source_file_name: Optional[str] = None source_contents: Optional[str] = None @@ -208,24 +228,16 @@ class BaseLoaderTestCase(unittest.TestCase): return self._get_tool_source() def _get_tool_source(self, source_file_name=None, source_contents=None, macro_contents=None): - macro_path = None if source_file_name is None: source_file_name = self.source_file_name if source_contents is None: source_contents = self.source_contents - if not os.path.isabs(source_file_name): - path = os.path.join(self.temp_directory, source_file_name) - with open(path, "w") as out: - out.write(source_contents) - if macro_contents: - macro_path = os.path.join(self.temp_directory, "macros.xml") - with open(macro_path, "w") as out: - out.write(macro_contents) - - else: - path = source_file_name - tool_source = get_tool_source(path, macro_paths=[macro_path]) - return tool_source + return get_test_tool_source( + source_file_name, + source_contents, + macro_contents, + self.temp_directory, + ) class XmlExpressionLoaderTestCase(BaseLoaderTestCase):