diff --git a/lib/galaxy/datatypes/_schema.py b/lib/galaxy/datatypes/_schema.py new file mode 100644 index 00000000000..6efb7cc4565 --- /dev/null +++ b/lib/galaxy/datatypes/_schema.py @@ -0,0 +1,129 @@ +from typing import ( + Dict, + List, + Optional, +) + +from pydantic import ( + BaseModel, + Field, + HttpUrl +) + + +class CompositeFileInfo(BaseModel): + name: str = Field( + ..., # Mark this field as required + title="Name", + description="The name of this composite file" + ) + optional: bool = Field( + title="Optional", + description="" # TODO add description + ) + mimetype: Optional[str] = Field( + title="MIME type", + description="The MIME type of this file" + ) + description: Optional[str] = Field( + title="Description", + description="Summary description of the purpouse of this file" + ) + substitute_name_with_metadata: Optional[str] = Field( + title="Substitute name with metadata", + description="" # TODO add description + ) + is_binary: bool = Field( + title="Is binary", + description="Whether this file is a binary file" + ) + to_posix_lines: bool = Field( + title="To posix lines", + description="" # TODO add description + ) + space_to_tab: bool = Field( + title="Spaces to tabulation", + description="" # TODO add description + ) + + +class DatatypeDetails(BaseModel): + extension: str = Field( + ..., # Mark this field as required + title="Extension", + description="The data type’s Dataset file extension", + example="bed" + ) + description: Optional[str] = Field( + title="Description", + description="A summary description for this data type" + ) + description_url: Optional[HttpUrl] = Field( + title="Description URL", + description="The URL to a detailed description for this datatype", + example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" + ) + display_in_upload: bool = Field( + default=False, + title="Display in upload", + description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" + ) + composite_files: Optional[List[CompositeFileInfo]] = Field( + default=None, + title="Composite files", + description="A collection of files composing this data type" + ) + + +class DatatypesMap(BaseModel): + ext_to_class_name: Dict[str, str] = Field( + ..., # Mark this field as required + title="Extension Map", + description="Dictionary mapping datatype's extensions with implementation classes", + ) + class_to_classes: Dict[str, Dict[str, bool]] = Field( + ..., # Mark this field as required + title="Classes Map", + description="Dictionary mapping datatype's classes with their base classes", + ) + + +class DatatypesCombinedMap(BaseModel): + datatypes: List[str] = Field( + ..., # Mark this field as required + title="Datatypes", + description="List of datatypes extensions", + ) + datatypes_mapping: DatatypesMap = Field( + ..., # Mark this field as required + title="Datatypes Mapping", + description="Dictionaries for mapping datatype's extensions/classes with their implementation classes", + ) + + +class DatatypeConverter(BaseModel): + source: str = Field( + ..., # Mark this field as required + title="Source", + description="Source type for conversion", + example="bam", + ) + target: str = Field( + ..., # Mark this field as required + title="Target", + description="Target type for conversion", + example="bai", + ) + tool_id: str = Field( + ..., # Mark this field as required + title="Tool identifier", + description="The converter tool identifier", + example="CONVERTER_Bam_Bai_0", + ) + + +class DatatypeConverterList(BaseModel): + __root__: List[DatatypeConverter] = Field( + title='List of data type converters', + default=[] + ) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py new file mode 100644 index 00000000000..6913bc32104 --- /dev/null +++ b/lib/galaxy/managers/datatypes.py @@ -0,0 +1,99 @@ +from typing import ( + Dict, + List, + Optional, + Union, +) + +from pydantic.tools import parse_obj_as + +from galaxy.datatypes._schema import ( + DatatypeConverterList, + DatatypeDetails, + DatatypesCombinedMap, + DatatypesMap, +) +from galaxy.datatypes.data import Data +from galaxy.datatypes.registry import Registry + + +def view_index( + datatypes_registry: Registry, + extension_only: Optional[bool] = True, + upload_only: Optional[bool] = True +) -> Union[List[DatatypeDetails], List[str]]: + if extension_only: + if upload_only: + return datatypes_registry.upload_file_formats + else: + return [ext for ext in datatypes_registry.datatypes_by_extension] + else: + rval = [] + for datatype_info_dict in datatypes_registry.datatype_info_dicts: + if upload_only and not datatype_info_dict.get('display_in_upload'): + continue + rval.append(datatype_info_dict) + return rval + + +def view_mapping(datatypes_registry: Registry) -> DatatypesMap: + ext_to_class_name: Dict[str, str] = {} + classes = [] + for k, v in datatypes_registry.datatypes_by_extension.items(): + c = v.__class__ + ext_to_class_name[k] = f"{c.__module__}.{c.__name__}" + classes.append(c) + class_to_classes: Dict[str, Dict[str, bool]] = {} + + def visit_bases(types, cls): + for base in cls.__bases__: + if issubclass(base, Data): + types.add(f"{base.__module__}.{base.__name__}") + visit_bases(types, base) + + for c in classes: + n = f"{c.__module__}.{c.__name__}" + types = {n} + visit_bases(types, c) + class_to_classes[n] = {t: True for t in types} + return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + + +def view_types_and_mapping( + datatypes_registry: Registry, + extension_only: Optional[bool] = True, + upload_only: Optional[bool] = True +) -> DatatypesCombinedMap: + return DatatypesCombinedMap( + datatypes=view_index(datatypes_registry, extension_only, upload_only), + datatypes_mapping=view_mapping(datatypes_registry) + ) + + +def view_sniffers(datatypes_registry: Registry) -> List[str]: + rval: List[str] = [] + for sniffer_elem in datatypes_registry.sniffer_elems: + datatype = sniffer_elem.get('type') + if datatype is not None: + rval.append(datatype) + return rval + + +def view_converters(datatypes_registry: Registry) -> DatatypeConverterList: + converters = [] + for (source_type, targets) in datatypes_registry.datatype_converters.items(): + for target_type in targets: + converters.append({ + 'source': source_type, + 'target': target_type, + 'tool_id': targets[target_type].id, + }) + return parse_obj_as(DatatypeConverterList, converters) + + +def view_edam_formats(datatypes_registry: Registry) -> Dict[str, str]: + return datatypes_registry.edam_formats + + +def view_edam_data(datatypes_registry: Registry) -> Dict[str, str]: + return datatypes_registry.edam_data diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 3f17f881df6..55d748b886b 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -2,14 +2,148 @@ API operations allowing clients to determine datatype supported by Galaxy. """ import logging +from typing import ( + Dict, + List, + Optional, + Union, +) -from galaxy.datatypes.data import Data +from fastapi import ( + Depends, + Query, +) +from fastapi.routing import APIRouter +from fastapi_utils.cbv import cbv + +from galaxy.app import UniverseApplication +from galaxy.datatypes.registry import Registry +from galaxy.managers.datatypes import ( + DatatypeConverterList, + DatatypeDetails, + DatatypesCombinedMap, + DatatypesMap, + view_converters, + view_edam_data, + view_edam_formats, + view_index, + view_mapping, + view_sniffers, + view_types_and_mapping +) from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless from galaxy.webapps.base.controller import BaseAPIController +from . import get_app log = logging.getLogger(__name__) +router = APIRouter(tags=['datatypes']) + +ExtensionOnlyQueryParam : Optional[bool] = Query( + default=True, + title="Extension only", + description="Whether to return only the datatype's extension rather than the datatype's details", +) + +UploadOnlyQueryParam : Optional[bool] = Query( + default=True, + title="Upload only", + description="Whether to return only datatypes which can be uploaded", +) + + +def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Registry: + return app.datatypes_registry + + +@cbv(router) +class FastAPIDatatypes: + datatypes_registry: Registry = Depends(get_datatypes_registry) + + @router.get( + '/api/datatypes', + summary="Lists all available data types", + response_model=Union[List[DatatypeDetails], List[str]], + response_description="List of data types", + ) + async def index( + self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> Union[List[DatatypeDetails], List[str]]: + """Gets the list of all available data types.""" + return view_index(self.datatypes_registry, extension_only, upload_only) + + @router.get( + '/api/datatypes/mapping', + summary="Returns mappings for data types and their implementing classes", + response_model=DatatypesMap, + response_description="Dictionary to map data types with their classes" + ) + async def mapping(self) -> DatatypesMap: + """Gets mappings for data types.""" + return view_mapping(self.datatypes_registry) + + @router.get( + '/api/datatypes/types_and_mapping', + summary="Returns all the data types extensions and their mappings", + response_model=DatatypesCombinedMap, + response_description="Dictionary to map data types with their classes" + ) + async def types_and_mapping( + self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> DatatypesCombinedMap: + """Combines the datatype information from (/api/datatypes) and the + mapping information from (/api/datatypes/mapping) into a single + response.""" + return DatatypesCombinedMap( + datatypes=view_index(self.datatypes_registry, extension_only, upload_only), + datatypes_mapping=view_mapping(self.datatypes_registry) + ) + + @router.get( + '/api/datatypes/sniffers', + summary="Returns the list of all installed sniffers", + response_model=List[str], + response_description="List of datatype sniffers" + ) + async def sniffers(self) -> List[str]: + """Gets the list of all installed data type sniffers.""" + return view_sniffers(self.datatypes_registry) + + @router.get( + '/api/datatypes/converters', + summary="Returns the list of all installed converters", + response_model=DatatypeConverterList, + response_description="List of all datatype converters" + ) + async def converters(self) -> DatatypeConverterList: + """Gets the list of all installed converters.""" + return view_converters(self.datatypes_registry) + + @router.get( + '/api/datatypes/edam_formats', + summary="Returns a dictionary/map of datatypes and EDAM formats", + response_model=Dict[str, str], + response_description="Dictionary/map of datatypes and EDAM formats" + ) + async def edam_formats(self) -> Dict[str, str]: + """Gets a map of datatypes and their corresponding EDAM formats.""" + return self.datatypes_registry.edam_formats + + @router.get( + '/api/datatypes/edam_data', + summary="Returns a dictionary/map of datatypes and EDAM data", + response_model=Dict[str, str], + response_description="Dictionary/map of datatypes and EDAM data" + ) + async def edam_data(self) -> Dict[str, str]: + """Gets a map of datatypes and their corresponding EDAM data.""" + return self.datatypes_registry.edam_data + class DatatypesController(BaseAPIController): @@ -19,28 +153,9 @@ class DatatypesController(BaseAPIController): GET /api/datatypes Return an object containing upload datatypes. """ - try: - return self._index(trans, **kwd) - except Exception as e: - log.exception('Could not get datatypes') - raise e - - def _index(self, trans, **kwd): - datatypes_registry = self._datatypes_registry extension_only = asbool(kwd.get('extension_only', True)) upload_only = asbool(kwd.get('upload_only', True)) - if extension_only: - if upload_only: - return datatypes_registry.upload_file_formats - else: - return [ext for ext in datatypes_registry.datatypes_by_extension] - else: - rval = [] - for datatype_info_dict in datatypes_registry.datatype_info_dicts: - if not datatype_info_dict.get('display_in_upload') and upload_only: - continue - rval.append(datatype_info_dict) - return rval + return view_index(self._datatypes_registry, extension_only, upload_only) @expose_api_anonymous_and_sessionless def mapping(self, trans, **kwd): @@ -48,32 +163,7 @@ class DatatypesController(BaseAPIController): GET /api/datatypes/mapping Return a dictionary of class to class mappings. ''' - try: - return self._mapping() - except Exception as e: - log.exception('Could not get datatype mapping') - raise e - - def _mapping(self): - ext_to_class_name = dict() - classes = [] - for k, v in self._datatypes_registry.datatypes_by_extension.items(): - c = v.__class__ - ext_to_class_name[k] = c.__module__ + "." + c.__name__ - classes.append(c) - class_to_classes = dict() - - def visit_bases(types, cls): - for base in cls.__bases__: - if issubclass(base, Data): - types.add(base.__module__ + "." + base.__name__) - visit_bases(types, base) - for c in classes: - n = c.__module__ + "." + c.__name__ - types = {n} - visit_bases(types, c) - class_to_classes[n] = {t: True for t in types} - return dict(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + return view_mapping(self._datatypes_registry) @expose_api_anonymous_and_sessionless def types_and_mapping(self, trans, **kwd): @@ -84,10 +174,9 @@ class DatatypesController(BaseAPIController): mapping information from (/api/datatypes/mapping) into a single response. """ - return { - "datatypes": self._index(trans, **kwd), - "datatypes_mapping": self._mapping(), - } + extension_only = asbool(kwd.get('extension_only', True)) + upload_only = asbool(kwd.get('upload_only', True)) + return view_types_and_mapping(self._datatypes_registry, extension_only, upload_only) @expose_api_anonymous_and_sessionless def sniffers(self, trans, **kwd): @@ -95,37 +184,19 @@ class DatatypesController(BaseAPIController): GET /api/datatypes/sniffers Return a list of sniffers. ''' - try: - rval = [] - for sniffer_elem in self._datatypes_registry.sniffer_elems: - datatype = sniffer_elem.get('type') - if datatype is not None: - rval.append(datatype) - return rval - except Exception as e: - log.exception('Could not get datatypes') - raise e + return view_sniffers(self._datatypes_registry) @expose_api_anonymous_and_sessionless def converters(self, trans, **kwd): - converters = [] - for (source_type, targets) in self._datatypes_registry.datatype_converters.items(): - for target_type in targets: - converters.append({ - 'source': source_type, - 'target': target_type, - 'tool_id': targets[target_type].id, - }) - - return converters + return view_converters(self._datatypes_registry) @expose_api_anonymous_and_sessionless def edam_formats(self, trans, **kwds): - return self._datatypes_registry.edam_formats + return view_edam_formats(self._datatypes_registry) @expose_api_anonymous_and_sessionless def edam_data(self, trans, **kwds): - return self._datatypes_registry.edam_data + return view_edam_data(self._datatypes_registry) @property def _datatypes_registry(self): diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index 069011c178d..5e46e6f88c6 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -12,6 +12,10 @@ from galaxy.web.framework.decorators import ( # https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-tags api_tags_metadata = [ + { + "name": "datatypes", + "description": "Operations with supported data types.", + }, { "name": "licenses", "description": "Operations with [SPDX licenses](https://spdx.org/licenses/).", @@ -51,12 +55,14 @@ def initialize_fast_app(gx_app): add_exception_handler(app) wsgi_handler = WSGIMiddleware(gx_app) from galaxy.webapps.galaxy.api import ( + datatypes, job_lock, jobs, licenses, roles, tours, ) + app.include_router(datatypes.router) app.include_router(jobs.router) app.include_router(job_lock.router) app.include_router(licenses.router) diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index bc174af2448..3cc25f8eadf 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -84,6 +84,20 @@ class DatatypesApiTestCase(ApiTestCase): self._assert_status_code_is(response, 200) assert converters == len(response.json()) + def test_edam_formats(self): + response = self._get("datatypes/edam_formats") + self._assert_status_code_is(response, 200) + edam_formats = response.json() + assert isinstance(edam_formats, dict) + assert edam_formats["ab1"] == "format_3000" + + def test_edam_data(self): + response = self._get("datatypes/edam_data") + self._assert_status_code_is(response, 200) + edam_data = response.json() + assert isinstance(edam_data, dict) + assert edam_data["ab1"] == "data_0924" + def _index_datatypes(self, data=None): data = data or {} response = self._get("datatypes", data=data) diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 98424dcbef5..85c7facff5b 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -8,6 +8,7 @@ isa-rwval parsley numpy<=1.16 pycryptodome +pydantic pysam social_auth_core SQLAlchemy