Merge pull request #14464 from ElectronicBlueberry/datatypes-page

Add "Datatypes" Page

client/src/components/AvailableDatatypes/AvailableDatatypes.vue

@@ -0,0 +1,83 @@
+<script setup>
+import { ref } from "vue";
+import { useDetailedDatatypes } from "composables/datatypes";
+import { useFilterObjectArray } from "composables/utils/filter";
+import DelayedInput from "components/Common/DelayedInput";
+
+const filter = ref("");
+const filterFields = ["extension"];
+
+const { datatypes } = useDetailedDatatypes();
+const filteredDatatypes = useFilterObjectArray(datatypes, filter, filterFields);
+
+const fields = [
+    {
+        key: "extension",
+        sortable: true,
+    },
+    {
+        key: "edamFormatLabel",
+        label: "EDAM Format",
+        sortable: true,
+    },
+    {
+        key: "edamDataLabel",
+        label: "EDAM Data",
+        sortable: true,
+    },
+];
+
+const edamLink = (edamIRI) => `https://edamontology.github.io/edam-browser/#${edamIRI}`;
+</script>
+
+<template>
+    <div>
+        <h1>Datatypes</h1>
+        <p>
+            All Datatypes supported by this Galaxy instance. Hover over an item for more information. These extensions
+            can be filtered by in the History, by expanding "search datasets".
+        </p>
+        <DelayedInput placeholder="filter extensions" class="mb-3" :delay="200" @change="(val) => (filter = val)" />
+        <b-table striped small sort-icon-left sort-by="extension" :items="filteredDatatypes" :fields="fields">
+            <template v-slot:cell(extension)="row">
+                <a
+                    v-if="row.item.descriptionUrl"
+                    v-b-tooltip.hover
+                    target="_blank"
+                    :title="row.item.description"
+                    :href="row.item.descriptionUrl">
+                    {{ row.item.extension }}
+                </a>
+                <span v-else v-b-tooltip.hover :title="row.item.description">
+                    {{ row.item.extension }}
+                </span>
+            </template>
+
+            <template v-slot:cell(edamFormatLabel)="row">
+                <a
+                    v-b-tooltip.hover
+                    target="_blank"
+                    :href="edamLink(row.item.edamFormat)"
+                    :title="row.item.edamFormatDefinition">
+                    {{ row.item.edamFormatLabel }}
+                </a>
+            </template>
+
+            <template v-slot:cell(edamDataLabel)="row">
+                <a
+                    v-b-tooltip.hover
+                    target="_blank"
+                    :href="edamLink(row.item.edamData)"
+                    :title="row.item.edamDataDefinition">
+                    {{ row.item.edamDataLabel }}
+                </a>
+            </template>
+        </b-table>
+    </div>
+</template>
+
+<style scoped>
+table {
+    cursor: default;
+}
+</style>

client/src/composables/datatypes.js

@@ -0,0 +1,51 @@
+import { ref } from "vue";
+import axios from "axios";
+import { getAppRoot } from "onload/loadConfig";
+
+/**
+ * Fetches a detailed array of datatypes available on this galaxy instance.
+ * Does not cache the result or use a store.
+ */
+export function useDetailedDatatypes() {
+    const datatypesLoading = ref(true);
+    const datatypes = ref([]);
+
+    async function getDatatypes() {
+        try {
+            const datatypesPromise = axios.get(`${getAppRoot()}api/datatypes?extension_only=false`);
+            const datatypeEDAMFormatsPromise = axios.get(`${getAppRoot()}api/datatypes/edam_formats/detailed`);
+            const datatypeEDAMDataPromise = axios.get(`${getAppRoot()}api/datatypes/edam_data/detailed`);
+
+            const [baseTypes, datatypeEDAMFormats, datatypeEDAMData] = await Promise.all([
+                datatypesPromise,
+                datatypeEDAMFormatsPromise,
+                datatypeEDAMDataPromise,
+            ]);
+
+            datatypes.value = baseTypes.data.map((type, i) => {
+                const typeEDAMFormat = datatypeEDAMFormats.data[type.extension] ?? null;
+                const typeEDAMData = datatypeEDAMData.data[type.extension] ?? null;
+
+                return {
+                    extension: type.extension,
+                    description: type.description,
+                    descriptionUrl: type.description_url,
+                    edamFormat: typeEDAMFormat.prefix_IRI,
+                    edamFormatLabel: typeEDAMFormat.label,
+                    edamFormatDefinition: typeEDAMFormat.definition,
+                    edamData: typeEDAMData.prefix_IRI,
+                    edamDataLabel: typeEDAMData.label,
+                    edamDataDefinition: typeEDAMData.definition,
+                };
+            });
+        } catch (e) {
+            console.error("unable to fetch available datatypes\n", e);
+        } finally {
+            datatypesLoading.value = false;
+        }
+    }
+
+    getDatatypes();
+
+    return { datatypes, datatypesLoading };
+}

client/src/composables/utils/filter.js

@@ -0,0 +1,32 @@
+import { computed, unref } from "vue";
+
+/**
+ * Reactively filter an array of objects, by comparing `filter` to all `fields`.
+ * All parameters can optionally be refs.
+ * @param array array of objects to filter
+ * @param filter string to filter by
+ * @param objectFields string array of fields to filter by on each object
+ */
+export function useFilterObjectArray(array, filter, objectFields) {
+    const filtered = computed(() => {
+        const f = unref(filter).toLowerCase();
+        const arr = unref(array);
+        const fields = unref(objectFields);
+
+        if (f === "") {
+            return arr;
+        } else {
+            return arr.filter((obj) => {
+                for (const field of fields) {
+                    if (obj[field].toLowerCase().includes(f)) {
+                        return true;
+                    }
+                }
+
+                return false;
+            });
+        }
+    });
+
+    return filtered;
+}

client/src/entry/analysis/router.js

@@ -22,6 +22,7 @@ import DatasetAttributes from "components/DatasetInformation/DatasetAttributes";
 import DatasetDetails from "components/DatasetInformation/DatasetDetails";
 import DatasetError from "components/DatasetInformation/DatasetError";
 import DatasetList from "components/Dataset/DatasetList";
+import AvailableDatatypes from "components/AvailableDatatypes/AvailableDatatypes";
 import DisplayStructured from "components/DisplayStructured";
 import FormGeneric from "components/Form/FormGeneric";
 import Grid from "components/Grid/Grid";
@@ -140,6 +141,10 @@ export function getRouter(Galaxy) {
                         component: DatasetError,
                         props: true,
                     },
+                    {
+                        path: "datatypes",
+                        component: AvailableDatatypes,
+                    },
                     {
                         path: "histories/import",
                         component: HistoryImport,

lib/galaxy/datatypes/_schema.py

@@ -98,3 +98,29 @@ class DatatypeConverter(BaseModel):
 
 class DatatypeConverterList(BaseModel):
     __root__: List[DatatypeConverter] = Field(title="List of data type converters", default=[])
+
+
+class DatatypeEDAMDetails(BaseModel):
+    prefix_IRI: str = Field(
+        ...,  # Mark this field as required
+        title="Prefix IRI",
+        description="The EDAM prefixed Resource Identifier",
+        example="format_1782",
+    )
+    label: Optional[str] = Field(
+        title="Label",
+        description="The EDAM label",
+        example="NCBI gene report format",
+    )
+    definition: Optional[str] = Field(
+        title="Definition",
+        description="The EDAM definition",
+        example="Entry (gene) format of the NCBI database.",
+    )
+
+
+class DatatypesEDAMDetailsDict(BaseModel):
+    __root__: Dict[str, DatatypeEDAMDetails] = Field(
+        title="Dict of EDAM details for formats",
+        default={},
+    )

lib/galaxy/datatypes/registry.py

@@ -19,6 +19,7 @@ from typing import (
 import yaml
 
 import galaxy.util
+from galaxy.tool_util.edam_util import load_edam_tree
 from galaxy.util import RW_R__R__
 from galaxy.util.bunch import Bunch
 from . import (
@@ -46,9 +47,20 @@ class ConfigurationError(Exception):
 
 class Registry:
     def __init__(self, config=None):
+        edam_ontology_path = config.get("edam_toolbox_ontology_path", None) if config is not None else None
+
+        edam = load_edam_tree(
+            None if not edam_ontology_path or not os.path.exists(edam_ontology_path) else edam_ontology_path,
+            "format_",
+            "data_",
+            "operation_",
+            "topic_",
+        )
+
         self.log = logging.getLogger(__name__)
         self.log.addHandler(logging.NullHandler())
         self.config = config
+        self.edam = edam
         self.datatypes_by_extension = {}
         self.datatypes_by_suffix_inferences = {}
         self.mimetypes_by_extension = {}

lib/galaxy/managers/datatypes.py

@@ -11,6 +11,7 @@ from galaxy.datatypes._schema import (
     DatatypeConverterList,
     DatatypeDetails,
     DatatypesCombinedMap,
+    DatatypesEDAMDetailsDict,
     DatatypesMap,
 )
 from galaxy.datatypes.data import Data
@@ -89,10 +90,43 @@ def view_converters(datatypes_registry: Registry) -> DatatypeConverterList:
     return parse_obj_as(DatatypeConverterList, converters)
 
 
+def _get_edam_details(datatypes_registry: Registry, edam_ids: Dict[str, str]) -> Dict[str, Dict]:
+    details_dict = {}
+    for (format, edam_iri) in edam_ids.items():
+        edam_details = datatypes_registry.edam.get(edam_iri, {})
+
+        details_dict[format] = {
+            "prefix_IRI": edam_iri,
+            "label": edam_details.get("label", None),
+            "definition": edam_details.get("definition", None),
+        }
+
+    return details_dict
+
+
+def view_edam_formats(
+    datatypes_registry: Registry, detailed: Optional[bool] = False
+) -> Union[Dict[str, str], Dict[str, Dict[str, str]]]:
+    if detailed:
+        return _get_edam_details(datatypes_registry, datatypes_registry.edam_formats)
+    else:
+        return datatypes_registry.edam_formats
+
+
+def view_edam_data(
+    datatypes_registry: Registry, detailed: Optional[bool] = False
+) -> Union[Dict[str, str], Dict[str, Dict[str, str]]]:
+    if detailed:
+        return _get_edam_details(datatypes_registry, datatypes_registry.edam_data)
+    else:
+        return datatypes_registry.edam_data
+
+
 __all__ = (
     "DatatypeConverterList",
     "DatatypeDetails",
     "DatatypesCombinedMap",
+    "DatatypesEDAMDetailsDict",
     "DatatypesMap",
     "view_index",
     "view_mapping",

lib/galaxy/tool_util/edam_util.py

@@ -16,19 +16,19 @@ ROOT_OPERATION = "operation_0004"
 ROOT_TOPIC = "topic_0003"
 
 
-def load_edam_tree(path: Optional[str] = None):
+def load_edam_tree(path: Optional[str] = None, *included_terms: str):
     if path is not None:
         assert os.path.exists(path), f"Failed to load EDAM tabular data at [{path}] path does not exist."
         handle = open(path)
     else:
         assert (
             tabular_stream is not None
-        ), "Failed to load optional import from edam-onotology package, install using [pip install edam-ontology]."
+        ), "Failed to load optional import from edam-ontology package, install using [pip install edam-ontology]."
         handle = tabular_stream()
-    return load_edam_tree_from_tsv_stream(handle)
+    return load_edam_tree_from_tsv_stream(handle, *included_terms)
 
 
-def load_edam_tree_from_tsv_stream(tsv_stream: TextIO):
+def load_edam_tree_from_tsv_stream(tsv_stream: TextIO, *included_terms: str):
     edam: Dict[str, Dict] = {}
 
     def _recurse_edam_parents(term, path=None):
@@ -47,7 +47,7 @@ def load_edam_tree_from_tsv_stream(tsv_stream: TextIO):
                 columns[field] = i
             is_first = False
 
-            defintion_column = columns["http://www.geneontology.org/formats/oboInOwl#hasDefinition"]
+            definition_column = columns["http://www.geneontology.org/formats/oboInOwl#hasDefinition"]
             term_column = columns["Class ID"]
             label_column = columns["Preferred Label"]
             parents_column = columns["Parents"]
@@ -59,14 +59,14 @@ def load_edam_tree_from_tsv_stream(tsv_stream: TextIO):
 
         term_id = term[len(EDAM_PREFIX) :]
 
-        # Only care about formats and operations
-        if not (term_id.startswith("operation_") or term_id.startswith("topic_")):
+        # Only care about included terms
+        if included_terms and not (term_id.startswith(included_terms)):
             continue
 
         parents = fields[parents_column].split("|")
         edam[term_id] = {
             "label": fields[label_column],
-            "definition": fields[defintion_column].strip('"'),
+            "definition": fields[definition_column].strip('"'),
             "parents": [x[len(EDAM_PREFIX) :] for x in parents if x.startswith(EDAM_PREFIX)],
         }
 

lib/galaxy/tool_util/toolbox/base.py

@@ -193,7 +193,7 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
 
         for edam_view in listify(self.app.config.edam_panel_views):
             mode = EdamPanelMode[edam_view]
-            tool_panel_views_list.append(EdamToolPanelView(self.app.config.edam_toolbox_ontology_path, mode=mode))
+            tool_panel_views_list.append(EdamToolPanelView(self.app.datatypes_registry.edam, mode=mode))
 
         if view_sources is not None:
             for definition in view_sources.get_definitions():

lib/galaxy/tool_util/toolbox/views/edam.py

@@ -1,15 +1,12 @@
 import logging
-import os
 from enum import Enum
 from typing import (
     Dict,
     List,
-    Optional,
     Tuple,
 )
 
 from galaxy.tool_util.edam_util import (
-    load_edam_tree,
     ROOT_OPERATION,
     ROOT_TOPIC,
 )
@@ -36,10 +33,7 @@ class EdamPanelMode(str, Enum):
 
 
 class EdamToolPanelView(ToolPanelView):
-    def __init__(self, edam_ontology_path: Optional[str], mode: EdamPanelMode = EdamPanelMode.merged):
-        edam = load_edam_tree(
-            None if not edam_ontology_path or not os.path.exists(edam_ontology_path) else edam_ontology_path
-        )
+    def __init__(self, edam: Dict[str, Dict], mode: EdamPanelMode = EdamPanelMode.merged):
         self.edam = edam
         self.mode = mode
         self.include_topics = mode in [EdamPanelMode.merged, EdamPanelMode.topics]
@@ -162,15 +156,15 @@ class EdamToolPanelView(ToolPanelView):
         if mode == EdamPanelMode.merged:
             model_id = "ontology:edam_merged"
             name = "EDAM Operations and Topics"
-            description = "Tools are grouped using both annotated operation and topic information (if availabled)."
+            description = "Tools are grouped using both annotated operation and topic information (if available)."
         elif mode == EdamPanelMode.operations:
             model_id = "ontology:edam_operations"
             name = "EDAM Operations"
-            description = "Tools are grouped using annotated EDAM operation information (if availabled)."
+            description = "Tools are grouped using annotated EDAM operation information (if available)."
         elif mode == EdamPanelMode.topics:
             model_id = "ontology:edam_topics"
             name = "EDAM Topics"
-            description = "Tools are grouped using annotated EDAM topic information (if availabled)."
+            description = "Tools are grouped using annotated EDAM topic information (if available)."
         else:
             raise AssertionError(f"Invalid EDAM mode encountered {mode}")
         model_class = self.__class__.__name__

lib/galaxy/webapps/galaxy/api/datatypes.py

@@ -3,6 +3,7 @@ API operations allowing clients to determine datatype supported by Galaxy.
 """
 import logging
 from typing import (
+    cast,
     Dict,
     List,
     Optional,
@@ -16,8 +17,11 @@ from galaxy.managers.datatypes import (
     DatatypeConverterList,
     DatatypeDetails,
     DatatypesCombinedMap,
+    DatatypesEDAMDetailsDict,
     DatatypesMap,
     view_converters,
+    view_edam_data,
+    view_edam_formats,
     view_index,
     view_mapping,
     view_sniffers,
@@ -43,6 +47,12 @@ UploadOnlyQueryParam: Optional[bool] = Query(
     description="Whether to return only datatypes which can be uploaded",
 )
 
+IdentifierOnly: Optional[bool] = Query(
+    default=True,
+    title="prefixIRI only",
+    description="Whether to return only the EDAM prefixIRI rather than the EDAM details",
+)
+
 
 @router.cbv
 class FastAPIDatatypes:
@@ -113,7 +123,18 @@ class FastAPIDatatypes:
     )
     async def edam_formats(self) -> Dict[str, str]:
         """Gets a map of datatypes and their corresponding EDAM formats."""
-        return self.datatypes_registry.edam_formats
+        return cast(Dict[str, str], view_edam_formats(self.datatypes_registry))
+
+    @router.get(
+        "/api/datatypes/edam_formats/detailed",
+        summary="Returns a dictionary of datatypes and EDAM format details",
+        response_description="Dictionary of EDAM format details containing the EDAM iri, label, and definition",
+        response_model=DatatypesEDAMDetailsDict,
+    )
+    async def edam_formats_detailed(self):
+        """Gets a map of datatypes and their corresponding EDAM formats.
+        EDAM formats contain the EDAM iri, label, and definition."""
+        return view_edam_formats(self.datatypes_registry, True)
 
     @router.get(
         "/api/datatypes/edam_data",
@@ -122,4 +143,15 @@ class FastAPIDatatypes:
     )
     async def edam_data(self) -> Dict[str, str]:
         """Gets a map of datatypes and their corresponding EDAM data."""
-        return self.datatypes_registry.edam_data
+        return cast(Dict[str, str], view_edam_data(self.datatypes_registry))
+
+    @router.get(
+        "/api/datatypes/edam_data/detailed",
+        summary="Returns a dictionary of datatypes and EDAM data details",
+        response_description="Dictionary of EDAM data details containing the EDAM iri, label, and definition",
+        response_model=DatatypesEDAMDetailsDict,
+    )
+    async def edam_data_detailed(self):
+        """Gets a map of datatypes and their corresponding EDAM data.
+        EDAM data contains the EDAM iri, label, and definition."""
+        return view_edam_data(self.datatypes_registry, True)

lib/galaxy/webapps/galaxy/buildapp.py

@@ -208,6 +208,7 @@ def app_pair(global_conf, load_app_kwds=None, wsgi_preflight=True, **kwargs):
     webapp.add_client_route("/admin/quotas")
     webapp.add_client_route("/admin/form/{form_id}")
     webapp.add_client_route("/admin/api_keys")
+    webapp.add_client_route("/datatypes")
     webapp.add_client_route("/login/start")
     webapp.add_client_route("/login/confirm")
     webapp.add_client_route("/tools/view")

lib/galaxy_test/api/test_datatypes.py

@@ -89,6 +89,13 @@ class DatatypesApiTestCase(ApiTestCase):
         edam_formats = response.json()
         assert isinstance(edam_formats, dict)
         assert edam_formats["ab1"] == "format_3000"
+        response = self._get("datatypes/edam_formats/detailed")
+        self._assert_status_code_is(response, 200)
+        edam_formats = response.json()
+        assert isinstance(edam_formats, dict)
+        assert isinstance(edam_formats["afg"], dict)
+        assert edam_formats["afg"]["prefix_IRI"] == "format_3582"
+        assert edam_formats["afg"]["label"] == "afg"
 
     def test_edam_data(self):
         response = self._get("datatypes/edam_data")
@@ -96,6 +103,12 @@ class DatatypesApiTestCase(ApiTestCase):
         edam_data = response.json()
         assert isinstance(edam_data, dict)
         assert edam_data["ab1"] == "data_0924"
+        response = self._get("datatypes/edam_data/detailed")
+        edam_data = response.json()
+        assert isinstance(edam_data, dict)
+        assert isinstance(edam_data["afg"], dict)
+        assert edam_data["afg"]["prefix_IRI"] == "data_0925"
+        assert edam_data["afg"]["label"] == "Sequence assembly"
 
     def _index_datatypes(self, data=None):
         data = data or {}