Merge pull request #11098 from davelopez/fastapi-migration-datatypes

Add FastAPI routes for datatypes

lib/galaxy/datatypes/_schema.py

@@ -0,0 +1,129 @@
+from typing import (
+    Dict,
+    List,
+    Optional,
+)
+
+from pydantic import (
+    BaseModel,
+    Field,
+    HttpUrl
+)
+
+
+class CompositeFileInfo(BaseModel):
+    name: str = Field(
+        ...,  # Mark this field as required
+        title="Name",
+        description="The name of this composite file"
+    )
+    optional: bool = Field(
+        title="Optional",
+        description=""  # TODO add description
+    )
+    mimetype: Optional[str] = Field(
+        title="MIME type",
+        description="The MIME type of this file"
+    )
+    description: Optional[str] = Field(
+        title="Description",
+        description="Summary description of the purpouse of this file"
+    )
+    substitute_name_with_metadata: Optional[str] = Field(
+        title="Substitute name with metadata",
+        description=""  # TODO add description
+    )
+    is_binary: bool = Field(
+        title="Is binary",
+        description="Whether this file is a binary file"
+    )
+    to_posix_lines: bool = Field(
+        title="To posix lines",
+        description=""  # TODO add description
+    )
+    space_to_tab: bool = Field(
+        title="Spaces to tabulation",
+        description=""  # TODO add description
+    )
+
+
+class DatatypeDetails(BaseModel):
+    extension: str = Field(
+        ...,  # Mark this field as required
+        title="Extension",
+        description="The data type’s Dataset file extension",
+        example="bed"
+    )
+    description: Optional[str] = Field(
+        title="Description",
+        description="A summary description for this data type"
+    )
+    description_url: Optional[HttpUrl] = Field(
+        title="Description URL",
+        description="The URL to a detailed description for this datatype",
+        example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed"
+    )
+    display_in_upload: bool = Field(
+        default=False,
+        title="Display in upload",
+        description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel"
+    )
+    composite_files: Optional[List[CompositeFileInfo]] = Field(
+        default=None,
+        title="Composite files",
+        description="A collection of files composing this data type"
+    )
+
+
+class DatatypesMap(BaseModel):
+    ext_to_class_name: Dict[str, str] = Field(
+        ...,  # Mark this field as required
+        title="Extension Map",
+        description="Dictionary mapping datatype's extensions with implementation classes",
+    )
+    class_to_classes: Dict[str, Dict[str, bool]] = Field(
+        ...,  # Mark this field as required
+        title="Classes Map",
+        description="Dictionary mapping datatype's classes with their base classes",
+    )
+
+
+class DatatypesCombinedMap(BaseModel):
+    datatypes: List[str] = Field(
+        ...,  # Mark this field as required
+        title="Datatypes",
+        description="List of datatypes extensions",
+    )
+    datatypes_mapping: DatatypesMap = Field(
+        ...,  # Mark this field as required
+        title="Datatypes Mapping",
+        description="Dictionaries for mapping datatype's extensions/classes with their implementation classes",
+    )
+
+
+class DatatypeConverter(BaseModel):
+    source: str = Field(
+        ...,  # Mark this field as required
+        title="Source",
+        description="Source type for conversion",
+        example="bam",
+    )
+    target: str = Field(
+        ...,  # Mark this field as required
+        title="Target",
+        description="Target type for conversion",
+        example="bai",
+    )
+    tool_id: str = Field(
+        ...,  # Mark this field as required
+        title="Tool identifier",
+        description="The converter tool identifier",
+        example="CONVERTER_Bam_Bai_0",
+    )
+
+
+class DatatypeConverterList(BaseModel):
+    __root__: List[DatatypeConverter] = Field(
+        title='List of data type converters',
+        default=[]
+    )

lib/galaxy/managers/datatypes.py

@@ -0,0 +1,99 @@
+from typing import (
+    Dict,
+    List,
+    Optional,
+    Union,
+)
+
+from pydantic.tools import parse_obj_as
+
+from galaxy.datatypes._schema import (
+    DatatypeConverterList,
+    DatatypeDetails,
+    DatatypesCombinedMap,
+    DatatypesMap,
+)
+from galaxy.datatypes.data import Data
+from galaxy.datatypes.registry import Registry
+
+
+def view_index(
+    datatypes_registry: Registry,
+    extension_only: Optional[bool] = True,
+    upload_only: Optional[bool] = True
+) -> Union[List[DatatypeDetails], List[str]]:
+    if extension_only:
+        if upload_only:
+            return datatypes_registry.upload_file_formats
+        else:
+            return [ext for ext in datatypes_registry.datatypes_by_extension]
+    else:
+        rval = []
+        for datatype_info_dict in datatypes_registry.datatype_info_dicts:
+            if upload_only and not datatype_info_dict.get('display_in_upload'):
+                continue
+            rval.append(datatype_info_dict)
+        return rval
+
+
+def view_mapping(datatypes_registry: Registry) -> DatatypesMap:
+    ext_to_class_name: Dict[str, str] = {}
+    classes = []
+    for k, v in datatypes_registry.datatypes_by_extension.items():
+        c = v.__class__
+        ext_to_class_name[k] = f"{c.__module__}.{c.__name__}"
+        classes.append(c)
+    class_to_classes: Dict[str, Dict[str, bool]] = {}
+
+    def visit_bases(types, cls):
+        for base in cls.__bases__:
+            if issubclass(base, Data):
+                types.add(f"{base.__module__}.{base.__name__}")
+            visit_bases(types, base)
+
+    for c in classes:
+        n = f"{c.__module__}.{c.__name__}"
+        types = {n}
+        visit_bases(types, c)
+        class_to_classes[n] = {t: True for t in types}
+    return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes)
+
+
+def view_types_and_mapping(
+    datatypes_registry: Registry,
+    extension_only: Optional[bool] = True,
+    upload_only: Optional[bool] = True
+) -> DatatypesCombinedMap:
+    return DatatypesCombinedMap(
+        datatypes=view_index(datatypes_registry, extension_only, upload_only),
+        datatypes_mapping=view_mapping(datatypes_registry)
+    )
+
+
+def view_sniffers(datatypes_registry: Registry) -> List[str]:
+    rval: List[str] = []
+    for sniffer_elem in datatypes_registry.sniffer_elems:
+        datatype = sniffer_elem.get('type')
+        if datatype is not None:
+            rval.append(datatype)
+    return rval
+
+
+def view_converters(datatypes_registry: Registry) -> DatatypeConverterList:
+    converters = []
+    for (source_type, targets) in datatypes_registry.datatype_converters.items():
+        for target_type in targets:
+            converters.append({
+                'source': source_type,
+                'target': target_type,
+                'tool_id': targets[target_type].id,
+            })
+    return parse_obj_as(DatatypeConverterList, converters)
+
+
+def view_edam_formats(datatypes_registry: Registry) -> Dict[str, str]:
+    return datatypes_registry.edam_formats
+
+
+def view_edam_data(datatypes_registry: Registry) -> Dict[str, str]:
+    return datatypes_registry.edam_data

lib/galaxy/webapps/galaxy/api/datatypes.py

@@ -2,14 +2,148 @@
 API operations allowing clients to determine datatype supported by Galaxy.
 """
 import logging
+from typing import (
+    Dict,
+    List,
+    Optional,
+    Union,
+)
 
-from galaxy.datatypes.data import Data
+from fastapi import (
+    Depends,
+    Query,
+)
+from fastapi.routing import APIRouter
+from fastapi_utils.cbv import cbv
+
+from galaxy.app import UniverseApplication
+from galaxy.datatypes.registry import Registry
+from galaxy.managers.datatypes import (
+    DatatypeConverterList,
+    DatatypeDetails,
+    DatatypesCombinedMap,
+    DatatypesMap,
+    view_converters,
+    view_edam_data,
+    view_edam_formats,
+    view_index,
+    view_mapping,
+    view_sniffers,
+    view_types_and_mapping
+)
 from galaxy.util import asbool
 from galaxy.web import expose_api_anonymous_and_sessionless
 from galaxy.webapps.base.controller import BaseAPIController
+from . import get_app
 
 log = logging.getLogger(__name__)
 
+router = APIRouter(tags=['datatypes'])
+
+ExtensionOnlyQueryParam : Optional[bool] = Query(
+    default=True,
+    title="Extension only",
+    description="Whether to return only the datatype's extension rather than the datatype's details",
+)
+
+UploadOnlyQueryParam : Optional[bool] = Query(
+    default=True,
+    title="Upload only",
+    description="Whether to return only datatypes which can be uploaded",
+)
+
+
+def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Registry:
+    return app.datatypes_registry
+
+
+@cbv(router)
+class FastAPIDatatypes:
+    datatypes_registry: Registry = Depends(get_datatypes_registry)
+
+    @router.get(
+        '/api/datatypes',
+        summary="Lists all available data types",
+        response_model=Union[List[DatatypeDetails], List[str]],
+        response_description="List of data types",
+    )
+    async def index(
+        self,
+        extension_only: Optional[bool] = ExtensionOnlyQueryParam,
+        upload_only: Optional[bool] = UploadOnlyQueryParam
+    ) -> Union[List[DatatypeDetails], List[str]]:
+        """Gets the list of all available data types."""
+        return view_index(self.datatypes_registry, extension_only, upload_only)
+
+    @router.get(
+        '/api/datatypes/mapping',
+        summary="Returns mappings for data types and their implementing classes",
+        response_model=DatatypesMap,
+        response_description="Dictionary to map data types with their classes"
+    )
+    async def mapping(self) -> DatatypesMap:
+        """Gets mappings for data types."""
+        return view_mapping(self.datatypes_registry)
+
+    @router.get(
+        '/api/datatypes/types_and_mapping',
+        summary="Returns all the data types extensions and their mappings",
+        response_model=DatatypesCombinedMap,
+        response_description="Dictionary to map data types with their classes"
+    )
+    async def types_and_mapping(
+        self,
+        extension_only: Optional[bool] = ExtensionOnlyQueryParam,
+        upload_only: Optional[bool] = UploadOnlyQueryParam
+    ) -> DatatypesCombinedMap:
+        """Combines the datatype information from (/api/datatypes) and the
+        mapping information from (/api/datatypes/mapping) into a single
+        response."""
+        return DatatypesCombinedMap(
+            datatypes=view_index(self.datatypes_registry, extension_only, upload_only),
+            datatypes_mapping=view_mapping(self.datatypes_registry)
+        )
+
+    @router.get(
+        '/api/datatypes/sniffers',
+        summary="Returns the list of all installed sniffers",
+        response_model=List[str],
+        response_description="List of datatype sniffers"
+    )
+    async def sniffers(self) -> List[str]:
+        """Gets the list of all installed data type sniffers."""
+        return view_sniffers(self.datatypes_registry)
+
+    @router.get(
+        '/api/datatypes/converters',
+        summary="Returns the list of all installed converters",
+        response_model=DatatypeConverterList,
+        response_description="List of all datatype converters"
+    )
+    async def converters(self) -> DatatypeConverterList:
+        """Gets the list of all installed converters."""
+        return view_converters(self.datatypes_registry)
+
+    @router.get(
+        '/api/datatypes/edam_formats',
+        summary="Returns a dictionary/map of datatypes and EDAM formats",
+        response_model=Dict[str, str],
+        response_description="Dictionary/map of datatypes and EDAM formats"
+    )
+    async def edam_formats(self) -> Dict[str, str]:
+        """Gets a map of datatypes and their corresponding EDAM formats."""
+        return self.datatypes_registry.edam_formats
+
+    @router.get(
+        '/api/datatypes/edam_data',
+        summary="Returns a dictionary/map of datatypes and EDAM data",
+        response_model=Dict[str, str],
+        response_description="Dictionary/map of datatypes and EDAM data"
+    )
+    async def edam_data(self) -> Dict[str, str]:
+        """Gets a map of datatypes and their corresponding EDAM data."""
+        return self.datatypes_registry.edam_data
+
 
 class DatatypesController(BaseAPIController):
 
@@ -19,28 +153,9 @@ class DatatypesController(BaseAPIController):
         GET /api/datatypes
         Return an object containing upload datatypes.
         """
-        try:
-            return self._index(trans, **kwd)
-        except Exception as e:
-            log.exception('Could not get datatypes')
-            raise e
-
-    def _index(self, trans, **kwd):
-        datatypes_registry = self._datatypes_registry
         extension_only = asbool(kwd.get('extension_only', True))
         upload_only = asbool(kwd.get('upload_only', True))
-        if extension_only:
-            if upload_only:
-                return datatypes_registry.upload_file_formats
-            else:
-                return [ext for ext in datatypes_registry.datatypes_by_extension]
-        else:
-            rval = []
-            for datatype_info_dict in datatypes_registry.datatype_info_dicts:
-                if not datatype_info_dict.get('display_in_upload') and upload_only:
-                    continue
-                rval.append(datatype_info_dict)
-            return rval
+        return view_index(self._datatypes_registry, extension_only, upload_only)
 
     @expose_api_anonymous_and_sessionless
     def mapping(self, trans, **kwd):
@@ -48,32 +163,7 @@ class DatatypesController(BaseAPIController):
         GET /api/datatypes/mapping
         Return a dictionary of class to class mappings.
         '''
-        try:
-            return self._mapping()
-        except Exception as e:
-            log.exception('Could not get datatype mapping')
-            raise e
-
-    def _mapping(self):
-        ext_to_class_name = dict()
-        classes = []
-        for k, v in self._datatypes_registry.datatypes_by_extension.items():
-            c = v.__class__
-            ext_to_class_name[k] = c.__module__ + "." + c.__name__
-            classes.append(c)
-        class_to_classes = dict()
-
-        def visit_bases(types, cls):
-            for base in cls.__bases__:
-                if issubclass(base, Data):
-                    types.add(base.__module__ + "." + base.__name__)
-                visit_bases(types, base)
-        for c in classes:
-            n = c.__module__ + "." + c.__name__
-            types = {n}
-            visit_bases(types, c)
-            class_to_classes[n] = {t: True for t in types}
-        return dict(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes)
+        return view_mapping(self._datatypes_registry)
 
     @expose_api_anonymous_and_sessionless
     def types_and_mapping(self, trans, **kwd):
@@ -84,10 +174,9 @@ class DatatypesController(BaseAPIController):
         mapping information from (/api/datatypes/mapping) into a single
         response.
         """
-        return {
-            "datatypes": self._index(trans, **kwd),
-            "datatypes_mapping": self._mapping(),
-        }
+        extension_only = asbool(kwd.get('extension_only', True))
+        upload_only = asbool(kwd.get('upload_only', True))
+        return view_types_and_mapping(self._datatypes_registry, extension_only, upload_only)
 
     @expose_api_anonymous_and_sessionless
     def sniffers(self, trans, **kwd):
@@ -95,37 +184,19 @@ class DatatypesController(BaseAPIController):
         GET /api/datatypes/sniffers
         Return a list of sniffers.
         '''
-        try:
-            rval = []
-            for sniffer_elem in self._datatypes_registry.sniffer_elems:
-                datatype = sniffer_elem.get('type')
-                if datatype is not None:
-                    rval.append(datatype)
-            return rval
-        except Exception as e:
-            log.exception('Could not get datatypes')
-            raise e
+        return view_sniffers(self._datatypes_registry)
 
     @expose_api_anonymous_and_sessionless
     def converters(self, trans, **kwd):
-        converters = []
-        for (source_type, targets) in self._datatypes_registry.datatype_converters.items():
-            for target_type in targets:
-                converters.append({
-                    'source': source_type,
-                    'target': target_type,
-                    'tool_id': targets[target_type].id,
-                })
-
-        return converters
+        return view_converters(self._datatypes_registry)
 
     @expose_api_anonymous_and_sessionless
     def edam_formats(self, trans, **kwds):
-        return self._datatypes_registry.edam_formats
+        return view_edam_formats(self._datatypes_registry)
 
     @expose_api_anonymous_and_sessionless
     def edam_data(self, trans, **kwds):
-        return self._datatypes_registry.edam_data
+        return view_edam_data(self._datatypes_registry)
 
     @property
     def _datatypes_registry(self):

lib/galaxy/webapps/galaxy/fast_app.py

@@ -12,6 +12,10 @@ from galaxy.web.framework.decorators import (
 
 # https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-tags
 api_tags_metadata = [
+    {
+        "name": "datatypes",
+        "description": "Operations with supported data types.",
+    },
     {
         "name": "licenses",
         "description": "Operations with [SPDX licenses](https://spdx.org/licenses/).",
@@ -51,12 +55,14 @@ def initialize_fast_app(gx_app):
     add_exception_handler(app)
     wsgi_handler = WSGIMiddleware(gx_app)
     from galaxy.webapps.galaxy.api import (
+        datatypes,
         job_lock,
         jobs,
         licenses,
         roles,
         tours,
     )
+    app.include_router(datatypes.router)
     app.include_router(jobs.router)
     app.include_router(job_lock.router)
     app.include_router(licenses.router)

lib/galaxy_test/api/test_datatypes.py

@@ -84,6 +84,20 @@ class DatatypesApiTestCase(ApiTestCase):
         self._assert_status_code_is(response, 200)
         assert converters == len(response.json())
 
+    def test_edam_formats(self):
+        response = self._get("datatypes/edam_formats")
+        self._assert_status_code_is(response, 200)
+        edam_formats = response.json()
+        assert isinstance(edam_formats, dict)
+        assert edam_formats["ab1"] == "format_3000"
+
+    def test_edam_data(self):
+        response = self._get("datatypes/edam_data")
+        self._assert_status_code_is(response, 200)
+        edam_data = response.json()
+        assert isinstance(edam_data, dict)
+        assert edam_data["ab1"] == "data_0924"
+
     def _index_datatypes(self, data=None):
         data = data or {}
         response = self._get("datatypes", data=data)

packages/data/requirements.txt

@@ -8,6 +8,7 @@ isa-rwval
 parsley
 numpy<=1.16
 pycryptodome
+pydantic
 pysam
 social_auth_core
 SQLAlchemy