Merge pull request #13413 from jmchilton/legacy_edam

Allow legacy EDAM mappings.

lib/galaxy/tool_util/ontologies/edam_operation_mappings.tsv

@@ -0,0 +1,2 @@
+# tool-id <tab> edam-operation
+sort1	operation_3802

lib/galaxy/tool_util/ontologies/edam_topic_mappings.tsv

@@ -0,0 +1 @@
+# tool-id <tab> edam-topic

lib/galaxy/tool_util/ontologies/ontology_data.py

@@ -0,0 +1,108 @@
+from typing import (
+    cast,
+    Dict,
+    List,
+    NamedTuple,
+    Optional,
+    Tuple,
+)
+
+from galaxy.tool_util.biotools import BiotoolsMetadataSource
+from galaxy.tool_util.parser import ToolSource
+from galaxy.util.resources import files
+
+
+def _multi_dict_mapping(content: str) -> Dict[str, List[str]]:
+    mapping: Dict[str, List[str]] = {}
+    for x in content.splitlines():
+        if x.startswith("#"):
+            continue
+        key, value = cast(Tuple[str, str], tuple(x.split("\t")))
+        mapping.setdefault(key, []).append(value)
+    return mapping
+
+
+def _read_ontology_data_text(filename: str) -> str:
+    return files(PACKAGE).joinpath(filename).read_text()
+
+
+PACKAGE = "galaxy.tool_util.ontologies"
+BIOTOOLS_MAPPING_FILENAME = "biotools_mappings.tsv"
+EDAM_OPERATION_MAPPING_FILENAME = "edam_operation_mappings.tsv"
+EDAM_TOPIC_MAPPING_FILENAME = "edam_topic_mappings.tsv"
+
+BIOTOOLS_MAPPING_CONTENT = _read_ontology_data_text(BIOTOOLS_MAPPING_FILENAME)
+BIOTOOLS_MAPPING: Dict[str, str] = dict(
+    [
+        cast(Tuple[str, str], tuple(x.split("\t")))
+        for x in BIOTOOLS_MAPPING_CONTENT.splitlines()
+        if not x.startswith("#")
+    ]
+)
+EDAM_OPERATION_MAPPING_CONTENT = _read_ontology_data_text(EDAM_OPERATION_MAPPING_FILENAME)
+EDAM_OPERATION_MAPPING: Dict[str, List[str]] = _multi_dict_mapping(EDAM_OPERATION_MAPPING_CONTENT)
+
+EDAM_TOPIC_MAPPING_CONTENT = _read_ontology_data_text(EDAM_TOPIC_MAPPING_FILENAME)
+EDAM_TOPIC_MAPPING: Dict[str, List[str]] = _multi_dict_mapping(EDAM_TOPIC_MAPPING_CONTENT)
+
+
+class OntologyData(NamedTuple):
+    xrefs: List[Dict[str, str]]
+    edam_operations: Optional[List[str]]
+    edam_topics: Optional[List[str]]
+
+
+def biotools_reference(xrefs):
+    for xref in xrefs:
+        if xref["reftype"] == "bio.tools":
+            return xref["value"]
+    return None
+
+
+def legacy_biotools_external_reference(all_ids: List[str]) -> Optional[str]:
+    for tool_id in all_ids:
+        if tool_id in BIOTOOLS_MAPPING:
+            return BIOTOOLS_MAPPING[tool_id]
+    return None
+
+
+def expand_ontology_data(
+    tool_source: ToolSource, all_ids: List[str], biotools_metadata_source: Optional[BiotoolsMetadataSource]
+) -> OntologyData:
+    xrefs = tool_source.parse_xrefs()
+    has_biotools_reference = any(x["reftype"] == "bio.tools" for x in xrefs)
+    if not has_biotools_reference:
+        legacy_biotools_ref = legacy_biotools_external_reference(all_ids)
+        if legacy_biotools_ref is not None:
+            xrefs.append({"value": legacy_biotools_ref, "reftype": "bio.tools"})
+
+    edam_operations = tool_source.parse_edam_operations()
+    edam_topics = tool_source.parse_edam_topics()
+
+    for tool_id in all_ids:
+        if tool_id in EDAM_OPERATION_MAPPING:
+            edam_operations = EDAM_OPERATION_MAPPING[tool_id]
+            break
+
+    for tool_id in all_ids:
+        if tool_id in EDAM_TOPIC_MAPPING:
+            edam_topics = EDAM_TOPIC_MAPPING[tool_id]
+            break
+
+    has_missing_data = len(edam_operations) == 0 or len(edam_topics) == 0
+    if has_missing_data:
+        biotools_reference_str = biotools_reference(xrefs)
+        if biotools_reference_str and biotools_metadata_source:
+            biotools_entry = biotools_metadata_source.get_biotools_metadata(biotools_reference_str)
+            if biotools_entry:
+                edam_info = biotools_entry.edam_info
+                if len(edam_operations) == 0:
+                    edam_operations = edam_info.edam_operations
+                if len(edam_topics) == 0:
+                    edam_topics = edam_info.edam_topics
+
+    return OntologyData(
+        xrefs,
+        edam_operations,
+        edam_topics,
+    )

lib/galaxy/tools/__init__.py

@@ -48,6 +48,10 @@ from galaxy.tool_util.loader import (
     raw_tool_xml_tree,
     template_macro_params,
 )
+from galaxy.tool_util.ontologies.ontology_data import (
+    biotools_reference,
+    expand_ontology_data,
+)
 from galaxy.tool_util.output_checker import DETECTED_JOB_STATE
 from galaxy.tool_util.parser import (
     get_tool_source,
@@ -118,7 +122,6 @@ from galaxy.util.dictifiable import Dictifiable
 from galaxy.util.expressions import ExpressionContext
 from galaxy.util.form_builder import SelectField
 from galaxy.util.json import safe_loads
-from galaxy.util.resources import resource_string
 from galaxy.util.rules_dsl import RuleSet
 from galaxy.util.template import (
     fill_template,
@@ -217,15 +220,6 @@ GALAXY_LIB_TOOLS_VERSIONED = {
     "winSplitter": packaging.version.parse("1.0.1"),
 }
 
-BIOTOOLS_MAPPING_CONTENT = resource_string(__package__, "biotools_mappings.tsv")
-BIOTOOLS_MAPPING: Dict[str, str] = dict(
-    [
-        cast(Tuple[str, str], tuple(x.split("\t")))
-        for x in BIOTOOLS_MAPPING_CONTENT.splitlines()
-        if not x.startswith("#")
-    ]
-)
-
 REQUIRE_FULL_DIRECTORY = {
     "includes": [{"path": "**", "path_type": "glob"}],
 }
@@ -1053,31 +1047,14 @@ class Tool(Dictifiable):
         self.required_files = required_files
 
         self.citations = self._parse_citations(tool_source)
-        xrefs = tool_source.parse_xrefs()
-        has_biotools_reference = any(x["reftype"] == "bio.tools" for x in xrefs)
-        if not has_biotools_reference:
-            legacy_biotools_ref = self.legacy_biotools_external_reference
-            if legacy_biotools_ref is not None:
-                xrefs.append({"value": legacy_biotools_ref, "reftype": "bio.tools"})
-        self.xrefs = xrefs
-
-        edam_operations = tool_source.parse_edam_operations()
-        edam_topics = tool_source.parse_edam_topics()
-        has_missing_data = len(edam_operations) == 0 or len(edam_topics) == 0
-        if has_missing_data:
-            biotools_reference = self.biotools_reference
-            metadata_source = self.app.biotools_metadata_source
-            if biotools_reference and metadata_source:
-                biotools_entry = metadata_source.get_biotools_metadata(biotools_reference)
-                if biotools_entry:
-                    edam_info = biotools_entry.edam_info
-                    if len(edam_operations) == 0:
-                        edam_operations = edam_info.edam_operations
-                    if len(edam_topics) == 0:
-                        edam_topics = edam_info.edam_topics
-
-        self.edam_operations = edam_operations
-        self.edam_topics = edam_topics
+        ontology_data = expand_ontology_data(
+            tool_source,
+            self.all_ids,
+            self.app.biotools_metadata_source,
+        )
+        self.xrefs = ontology_data.xrefs
+        self.edam_operations = ontology_data.edam_operations
+        self.edam_topics = ontology_data.edam_topics
 
         self.__parse_trackster_conf(tool_source)
         # Record macro paths so we can reload a tool if any of its macro has changes
@@ -1553,24 +1530,13 @@ class Tool(Dictifiable):
                 self.app, self.tool_shed, self.repository_owner, self.repository_name
             )
 
-    @property
-    def legacy_biotools_external_reference(self) -> Optional[str]:
-        """Return a bio.tools ID if any of tool's IDs are BIOTOOLS_MAPPING."""
-        for tool_id in self.all_ids:
-            if tool_id in BIOTOOLS_MAPPING:
-                return BIOTOOLS_MAPPING[tool_id]
-        return None
-
     @property
     def biotools_reference(self) -> Optional[str]:
         """Return a bio.tools ID if external reference to it is found.
 
         If multiple bio.tools references are found, return just the first one.
         """
-        for xref in self.xrefs:
-            if xref["reftype"] == "bio.tools":
-                return xref["value"]
-        return None
+        return biotools_reference(self.xrefs)
 
     @property
     def help(self):

packages/app/MANIFEST.in

@@ -1,3 +1,2 @@
 include *.rst *.txt LICENSE
 include galaxy/jobs/runners/util/job_script/*.sh
-include galaxy/tools/*tsv

packages/tool_util/MANIFEST.in

@@ -3,3 +3,4 @@ include galaxy/tool_util/deps/mulled/invfile.lua
 include galaxy/tool_util/deps/resolvers/default_conda_mapping.yml
 include galaxy/tool_util/verify/test_config.sample.yml
 include galaxy/tool_util/xsd/*
+include galaxy/tool_util/ontologies/*tsv

packages/tool_util/setup.py

@@ -40,6 +40,7 @@ PACKAGES = [
     "galaxy.tool_util.deps.resolvers",
     "galaxy.tool_util.linters",
     "galaxy.tool_util.locations",
+    "galaxy.tool_util.ontologies",
     "galaxy.tool_util.parser",
     "galaxy.tool_util.toolbox",
     "galaxy.tool_util.toolbox.filters",

test/unit/tool_util/test_ontologies.py

@@ -0,0 +1,68 @@
+from galaxy.tool_util.ontologies.ontology_data import expand_ontology_data
+from .test_parsing import (
+    get_test_tool_source,
+    TOOL_YAML_1,
+)
+
+TOOL_YAML_2 = """
+name: "Bowtie Mapper"
+class: GalaxyTool
+id: bowtie
+version: 1.0.2
+description: "The Bowtie Mapper"
+xrefs:
+  - type: bio.tools
+    value: bwa
+command: "bowtie --map-the-stuff"
+outputs:
+  out1:
+    format: bam
+    from_work_dir: out1.bam
+edam_operations:
+ - operation_0335
+edam_topics:
+ - topic_0102
+inputs:
+  - name: input1
+    type: integer
+"""
+
+TOOL_YAML_3 = """
+name: "Bowtie Mapper"
+class: GalaxyTool
+id: sort1
+version: 1.0.2
+description: "The Bowtie Mapper"
+xrefs:
+  - type: bio.tools
+    value: bwa
+command: "bowtie --map-the-stuff"
+outputs:
+  out1:
+    format: bam
+    from_work_dir: out1.bam
+inputs:
+  - name: input1
+    type: integer
+"""
+
+
+def test_parse_edam_empty():
+    test_source = get_test_tool_source(source_file_name="testtool.yml", source_contents=TOOL_YAML_1)
+    ontology_data = expand_ontology_data(test_source, ["bowtie"], None)
+    assert ontology_data.edam_operations == []
+    assert ontology_data.edam_topics == []
+
+
+def test_parse_edam_direct():
+    test_source = get_test_tool_source(source_file_name="testtool.yml", source_contents=TOOL_YAML_2)
+    ontology_data = expand_ontology_data(test_source, ["bowtie"], None)
+    assert ontology_data.edam_operations == ["operation_0335"]
+    assert ontology_data.edam_topics == ["topic_0102"]
+
+
+def test_parse_edam_mapping_operations_legacy():
+    test_source = get_test_tool_source(source_file_name="testtool.yml", source_contents=TOOL_YAML_3)
+    ontology_data = expand_ontology_data(test_source, ["sort1"], None)
+    assert ontology_data.edam_operations == ["operation_3802"]
+    assert ontology_data.edam_topics == []

test/unit/tool_util/test_parsing.py

@@ -193,6 +193,26 @@ outputs:
 """
 
 
+def get_test_tool_source(source_file_name=None, source_contents=None, macro_contents=None, temp_directory=None):
+    source_directory = temp_directory or tempfile.mkdtemp()
+    macro_paths = []
+    if not os.path.isabs(source_file_name):
+        path = os.path.join(source_directory, source_file_name)
+        with open(path, "w") as out:
+            out.write(source_contents)
+        if macro_contents:
+            macro_path = os.path.join(source_directory, "macros.xml")
+            with open(macro_path, "w") as out:
+                out.write(macro_contents)
+            macro_paths = [macro_path]
+    else:
+        path = source_file_name
+    tool_source = get_tool_source(path, macro_paths=macro_paths)
+    if temp_directory is None:
+        shutil.rmtree(source_directory)
+    return tool_source
+
+
 class BaseLoaderTestCase(unittest.TestCase):
     source_file_name: Optional[str] = None
     source_contents: Optional[str] = None
@@ -208,24 +228,16 @@ class BaseLoaderTestCase(unittest.TestCase):
         return self._get_tool_source()
 
     def _get_tool_source(self, source_file_name=None, source_contents=None, macro_contents=None):
-        macro_path = None
         if source_file_name is None:
             source_file_name = self.source_file_name
         if source_contents is None:
             source_contents = self.source_contents
-        if not os.path.isabs(source_file_name):
-            path = os.path.join(self.temp_directory, source_file_name)
-            with open(path, "w") as out:
-                out.write(source_contents)
-            if macro_contents:
-                macro_path = os.path.join(self.temp_directory, "macros.xml")
-                with open(macro_path, "w") as out:
-                    out.write(macro_contents)
-
-        else:
-            path = source_file_name
-        tool_source = get_tool_source(path, macro_paths=[macro_path])
-        return tool_source
+        return get_test_tool_source(
+            source_file_name,
+            source_contents,
+            macro_contents,
+            self.temp_directory,
+        )
 
 
 class XmlExpressionLoaderTestCase(BaseLoaderTestCase):