From 594518c60846e037991a0230b133832e2f566e81 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 13:53:17 +0100 Subject: [PATCH 01/14] Add missing tests --- lib/galaxy_test/api/test_datatypes.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index bc174af2448..0634412a07c 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -84,6 +84,18 @@ class DatatypesApiTestCase(ApiTestCase): self._assert_status_code_is(response, 200) assert converters == len(response.json()) + def test_edamp_formats(self): + response = self._get("datatypes/edam_formats") + self._assert_status_code_is(response, 200) + edam_formats = response.json() + assert isinstance(edam_formats, dict) + + def test_edam_data(self): + response = self._get("datatypes/edam_data") + self._assert_status_code_is(response, 200) + edam_data = response.json() + assert isinstance(edam_data, dict) + def _index_datatypes(self, data=None): data = data or {} response = self._get("datatypes", data=data) From d968b6beae22006f38bd59ac8ce0dcd6b6980113 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 17:48:11 +0100 Subject: [PATCH 02/14] Add datatypes schemas --- lib/galaxy/datatypes/schema.py | 118 +++++++++++++++++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 lib/galaxy/datatypes/schema.py diff --git a/lib/galaxy/datatypes/schema.py b/lib/galaxy/datatypes/schema.py new file mode 100644 index 00000000000..1e82c249f1a --- /dev/null +++ b/lib/galaxy/datatypes/schema.py @@ -0,0 +1,118 @@ +from typing import Dict, List, Optional + +from pydantic import ( + BaseModel, + Field, + HttpUrl +) + + +class CompositeFileInfo(BaseModel): + name: str = Field( + ..., # Mark this field as required + title="Name", + description="The name of this composite file" + ) + optional: bool = Field( + title="Optional", + description="" # TODO add description + ) + mimetype: Optional[str] = Field( + title="MIME type", + description="The MIME type of this file" + ) + description: Optional[str] = Field( + title="Description", + description="Summary description of the purpouse of this file" + ) + substitute_name_with_metadata: Optional[str] = Field( + title="Substitute name with metadata", + description="" # TODO add description + ) + is_binary: bool = Field( + title="Is binary", + description="Whether this file is a binary file" + ) + to_posix_lines: bool = Field( + title="To posix lines", + description="" # TODO add description + ) + space_to_tab: bool = Field( + title="Spaces to tabulation", + description="" # TODO add description + ) + + +class DatatypeDetails(BaseModel): + extension: str = Field( + ..., # Mark this field as required + title="Extension", + description="The data type’s Dataset file extension", + example="bed" + ) + description: Optional[str] = Field( + title="Description", + description="A summary description for this data type" + ) + description_url: Optional[HttpUrl] = Field( + title="Description URL", + description="The URL to a detailed description for this datatype", + example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" + ) + display_in_upload: bool = Field( + default=False, + title="Display in upload", + description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" + ) + composite_files: Optional[List[CompositeFileInfo]] = Field( + default=None, + title="Composite files", + description="A collection of files composing this data type" + ) + + +class DatatypesMap(BaseModel): + ext_to_class_name: Dict[str, str] = Field( + ..., # Mark this field as required + title="Extension Map", + description="Dictionary mapping datatype's extensions with implementation classes", + ) + class_to_classes: Dict[str, Dict[str, bool]] = Field( + ..., # Mark this field as required + title="Classes Map", + description="Dictionary mapping datatype's classes with their base classes", + ) + + +class DatatypesCombinedMap(BaseModel): + datatypes: List[str] = Field( + ..., # Mark this field as required + title="Datatypes", + description="List of datatypes extensions", + ) + datatypes_mapping: DatatypesMap = Field( + ..., # Mark this field as required + title="Datatypes Mapping", + description="Dictionaries for mapping datatype's extensions/classes with their implementation classes", + ) + + +class DatatypeConverter(BaseModel): + source: str = Field( + ..., # Mark this field as required + title="Source", + description="Source type for conversion", + example="bam", + ) + target: str = Field( + ..., # Mark this field as required + title="Target", + description="Target type for conversion", + example="bai", + ) + tool_id: str = Field( + ..., # Mark this field as required + title="Tool identifier", + description="The converter tool identifier", + example="CONVERTER_Bam_Bai_0", + ) From 82ae2958b707fa50508128e3e2ec6d6fd104067c Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 17:50:40 +0100 Subject: [PATCH 03/14] Add datatypes routes with documentation --- lib/galaxy/webapps/galaxy/api/datatypes.py | 187 +++++++++++++++++++++ 1 file changed, 187 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 3f17f881df6..89f0880b8dc 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -2,14 +2,201 @@ API operations allowing clients to determine datatype supported by Galaxy. """ import logging +from typing import ( + Dict, + List, + Optional, + Union, +) +from fastapi import ( + Depends, + Query, +) +from fastapi.routing import APIRouter +from fastapi_utils.cbv import cbv + + +from galaxy.app import UniverseApplication from galaxy.datatypes.data import Data +from galaxy.datatypes.registry import Registry +from galaxy.datatypes.schema import ( + DatatypeConverter, + DatatypeDetails, + DatatypesCombinedMap, + DatatypesMap, +) from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless from galaxy.webapps.base.controller import BaseAPIController +from . import get_app + log = logging.getLogger(__name__) +router = APIRouter(tags=['datatypes']) + +ExtensionOnlyQueryParam : Optional[bool] = Query( + default=True, + title="Extension only", + description="Whether to return only the datatype's extension rather than the datatype's details", +) + +UploadOnlyQueryParam : Optional[bool] = Query( + default=True, + title="Upload only", + description="Whether to return only datatypes which can be uploaded", +) + + +def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Registry: + return app.datatypes_registry + + +@cbv(router) +class FastAPIDatatypes: + datatypes_registry: Registry = Depends(get_datatypes_registry) + + @router.get('/api/datatypes', + summary="Lists all available data types", + response_model=Union[List[DatatypeDetails], List[str]], + response_description="List of data types") + async def index(self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> Union[List[DatatypeDetails], List[str]]: + """Gets the list of all available data types.""" + try: + return self._index(extension_only, upload_only) + except Exception as e: + log.exception('Could not get datat ypes') + raise e + + @router.get('/api/datatypes/mapping', + summary="Returns mappings for data types and their implementing classes", + response_model=DatatypesMap, + response_description="Dictionary to map data types with their classes") + async def mapping(self) -> DatatypesMap: + """Gets mappings for data types.""" + try: + return self._mapping() + except Exception as e: + log.exception('Could not get data type mapping') + raise e + + @router.get('/api/datatypes/types_and_mapping', + summary="Returns all the data types extensions and their mappings", + response_model=DatatypesCombinedMap, + response_description="Dictionary to map data types with their classes") + async def types_and_mapping(self) -> DatatypesCombinedMap: + """Combines the datatype information from (/api/datatypes) and the + mapping information from (/api/datatypes/mapping) into a single + response.""" + return DatatypesCombinedMap(datatypes=self._index(), datatypes_mapping=self._mapping()) + + @router.get('/api/datatypes/sniffers', + summary="Returns the list of all installed sniffers", + response_model=List[str], + response_description="List of datatype sniffers") + async def sniffers(self) -> List[str]: + """Gets the list of all installed data type sniffers.""" + try: + return self._sniffers() + except Exception as e: + log.exception('Could not get sniffers') + raise e + + @router.get('/api/datatypes/converters', + summary="Returns the list of all installed converters", + response_model=List[DatatypeConverter], + response_description="List of all datatype converters") + async def converters(self) -> List[DatatypeConverter]: + """Gets the list of all installed converters.""" + try: + return self._converters() + except Exception as e: + log.exception('Could not get converters') + raise e + + @router.get('/api/datatypes/edam_formats', + summary="Returns a dictionary/map of datatypes and EDAM formats", + response_model=Dict[str, str], + response_description="Dictionary/map of datatypes and EDAM formats") + async def edam_formats(self) -> Dict[str, str]: + """Gets a map of datatypes and their corresponding EDAM formats.""" + try: + return self.datatypes_registry.edam_formats + except Exception as e: + log.exception('Could not get EDAM format mappings') + raise e + + @router.get('/api/datatypes/edam_data', + summary="Returns a dictionary/map of datatypes and EDAM data", + response_model=Dict[str, str], + response_description="Dictionary/map of datatypes and EDAM data") + async def edam_data(self) -> Dict[str, str]: + """Gets a map of datatypes and their corresponding EDAM data.""" + try: + return self.datatypes_registry.edam_data + except Exception as e: + log.exception('Could not get EDAM data mappings') + raise e + + def _index(self, extension_only: Optional[bool] = True, upload_only: Optional[bool] = True) -> Union[List[DatatypeDetails], List[str]]: + if extension_only: + if upload_only: + return self.datatypes_registry.upload_file_formats + else: + return [ext for ext in self.datatypes_registry.datatypes_by_extension] + else: + rval = [] + for datatype_info_dict in self.datatypes_registry.datatype_info_dicts: + if upload_only and not datatype_info_dict.get('display_in_upload'): + continue + rval.append(datatype_info_dict) + return rval + + def _mapping(self) -> DatatypesMap: + ext_to_class_name: Dict[str, str] = {} + classes = [] + for k, v in self.datatypes_registry.datatypes_by_extension.items(): + c = v.__class__ + ext_to_class_name[k] = f"{c.__module__}.{c.__name__}" + classes.append(c) + class_to_classes: Dict[str, Dict[str, bool]] = {} + + def visit_bases(types, cls): + for base in cls.__bases__: + if issubclass(base, Data): + types.add(f"{base.__module__}.{base.__name__}") + visit_bases(types, base) + + for c in classes: + n = f"{c.__module__}.{c.__name__}" + types = {n} + visit_bases(types, c) + class_to_classes[n] = {t: True for t in types} + return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + + def _sniffers(self) -> List[str]: + rval: List[str] = [] + for sniffer_elem in self.datatypes_registry.sniffer_elems: + datatype = sniffer_elem.get('type') + if datatype is not None: + rval.append(datatype) + return rval + + def _converters(self) -> List[DatatypeConverter]: + converters: List[DatatypeConverter] = [] + for (source_type, targets) in self.datatypes_registry.datatype_converters.items(): + for target_type in targets: + converters.append(DatatypeConverter( + source=source_type, + target=target_type, + tool_id=targets[target_type].id, + )) + return converters + class DatatypesController(BaseAPIController): From 4ac0f6f882dbabb6afb174678cdc3e3be5a8686b Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 17:56:02 +0100 Subject: [PATCH 04/14] Include router in API --- lib/galaxy/webapps/galaxy/fast_app.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/galaxy/webapps/galaxy/fast_app.py b/lib/galaxy/webapps/galaxy/fast_app.py index 64ad1e29329..5fe94519e33 100644 --- a/lib/galaxy/webapps/galaxy/fast_app.py +++ b/lib/galaxy/webapps/galaxy/fast_app.py @@ -12,6 +12,10 @@ from galaxy.web.framework.decorators import ( # https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-tags api_tags_metadata = [ + { + "name": "datatypes", + "description": "Operations with supported data types.", + }, { "name": "licenses", "description": "Operations with [SPDX licenses](https://spdx.org/licenses/).", @@ -47,11 +51,13 @@ def initialize_fast_app(gx_app): add_exception_handler(app) wsgi_handler = WSGIMiddleware(gx_app) from galaxy.webapps.galaxy.api import ( + datatypes, job_lock, jobs, roles, licenses ) + app.include_router(datatypes.router) app.include_router(jobs.router) app.include_router(job_lock.router) app.include_router(roles.router) From 64ef16a781ebb4d14dde0d4181f6f03e4ee28daf Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 18:02:30 +0100 Subject: [PATCH 05/14] Check additional conditions in new tests --- lib/galaxy_test/api/test_datatypes.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index 0634412a07c..e310cf11bbe 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -89,12 +89,14 @@ class DatatypesApiTestCase(ApiTestCase): self._assert_status_code_is(response, 200) edam_formats = response.json() assert isinstance(edam_formats, dict) + assert edam_formats["ab1"] == "format_3000" def test_edam_data(self): response = self._get("datatypes/edam_data") self._assert_status_code_is(response, 200) edam_data = response.json() assert isinstance(edam_data, dict) + assert edam_data["ab1"] == "data_0924" def _index_datatypes(self, data=None): data = data or {} From d6682f0c0c510ea55f6251165276f62d80686629 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Mon, 11 Jan 2021 18:15:25 +0100 Subject: [PATCH 06/14] Fix import linting error --- lib/galaxy/webapps/galaxy/api/datatypes.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 89f0880b8dc..fe896cdb1f8 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -29,7 +29,6 @@ from galaxy.datatypes.schema import ( from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless from galaxy.webapps.base.controller import BaseAPIController - from . import get_app log = logging.getLogger(__name__) From b9399bf051641ccb3a4373f1bbaf51b5152d8512 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 10:49:45 +0100 Subject: [PATCH 07/14] Refactor move datatypes/schemas.py to managers/datatypes.py --- lib/galaxy/{datatypes/schema.py => managers/datatypes.py} | 0 lib/galaxy/webapps/galaxy/api/datatypes.py | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename lib/galaxy/{datatypes/schema.py => managers/datatypes.py} (100%) diff --git a/lib/galaxy/datatypes/schema.py b/lib/galaxy/managers/datatypes.py similarity index 100% rename from lib/galaxy/datatypes/schema.py rename to lib/galaxy/managers/datatypes.py diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index fe896cdb1f8..cb422ad606f 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -20,7 +20,7 @@ from fastapi_utils.cbv import cbv from galaxy.app import UniverseApplication from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry -from galaxy.datatypes.schema import ( +from galaxy.managers.datatypes import ( DatatypeConverter, DatatypeDetails, DatatypesCombinedMap, From 56d5822862d70d0d1dd3219dd27b63e791524a46 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:53:05 +0100 Subject: [PATCH 08/14] Add view functions --- lib/galaxy/managers/datatypes.py | 92 +++++++++++++++++++++++++++++++- 1 file changed, 91 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index 1e82c249f1a..1512e699419 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -1,4 +1,9 @@ -from typing import Dict, List, Optional +from typing import ( + Dict, + List, + Optional, + Union, +) from pydantic import ( BaseModel, @@ -6,6 +11,9 @@ from pydantic import ( HttpUrl ) +from galaxy.datatypes.data import Data +from galaxy.datatypes.registry import Registry + class CompositeFileInfo(BaseModel): name: str = Field( @@ -116,3 +124,85 @@ class DatatypeConverter(BaseModel): description="The converter tool identifier", example="CONVERTER_Bam_Bai_0", ) + + +def view_index( + datatypes_registry: Registry, + extension_only: Optional[bool] = True, + upload_only: Optional[bool] = True +) -> Union[List[DatatypeDetails], List[str]]: + if extension_only: + if upload_only: + return datatypes_registry.upload_file_formats + else: + return [ext for ext in datatypes_registry.datatypes_by_extension] + else: + rval = [] + for datatype_info_dict in datatypes_registry.datatype_info_dicts: + if upload_only and not datatype_info_dict.get('display_in_upload'): + continue + rval.append(datatype_info_dict) + return rval + + +def view_mapping(datatypes_registry: Registry) -> DatatypesMap: + ext_to_class_name: Dict[str, str] = {} + classes = [] + for k, v in datatypes_registry.datatypes_by_extension.items(): + c = v.__class__ + ext_to_class_name[k] = f"{c.__module__}.{c.__name__}" + classes.append(c) + class_to_classes: Dict[str, Dict[str, bool]] = {} + + def visit_bases(types, cls): + for base in cls.__bases__: + if issubclass(base, Data): + types.add(f"{base.__module__}.{base.__name__}") + visit_bases(types, base) + + for c in classes: + n = f"{c.__module__}.{c.__name__}" + types = {n} + visit_bases(types, c) + class_to_classes[n] = {t: True for t in types} + return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + + +def view_types_and_mapping( + datatypes_registry: Registry, + extension_only: Optional[bool] = True, + upload_only: Optional[bool] = True +) -> DatatypesCombinedMap: + return DatatypesCombinedMap( + datatypes=view_index(datatypes_registry, extension_only, upload_only), + datatypes_mapping=view_mapping(datatypes_registry) + ) + + +def view_sniffers(datatypes_registry: Registry) -> List[str]: + rval: List[str] = [] + for sniffer_elem in datatypes_registry.sniffer_elems: + datatype = sniffer_elem.get('type') + if datatype is not None: + rval.append(datatype) + return rval + + +def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: + converters: List[DatatypeConverter] = [] + for (source_type, targets) in datatypes_registry.datatype_converters.items(): + for target_type in targets: + converters.append(DatatypeConverter( + source=source_type, + target=target_type, + tool_id=targets[target_type].id, + )) + return converters + + +def view_edam_formats(datatypes_registry: Registry) -> Dict[str, str]: + return datatypes_registry.edam_formats + + +def view_edam_data(datatypes_registry: Registry) -> Dict[str, str]: + return datatypes_registry.edam_data From 0e3715806eb604a811107a6cc3d2c23ae3b26800 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 13:56:07 +0100 Subject: [PATCH 09/14] Refactor FastAPI and legacy controller to reuse views --- lib/galaxy/webapps/galaxy/api/datatypes.py | 241 ++++++--------------- 1 file changed, 62 insertions(+), 179 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index cb422ad606f..820aadff2c7 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -16,15 +16,18 @@ from fastapi import ( from fastapi.routing import APIRouter from fastapi_utils.cbv import cbv - from galaxy.app import UniverseApplication -from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry from galaxy.managers.datatypes import ( DatatypeConverter, DatatypeDetails, DatatypesCombinedMap, DatatypesMap, + view_converters, view_edam_data, view_edam_formats, + view_index, + view_mapping, + view_sniffers, + view_types_and_mapping ) from galaxy.util import asbool from galaxy.web import expose_api_anonymous_and_sessionless @@ -56,145 +59,88 @@ def get_datatypes_registry(app: UniverseApplication = Depends(get_app)) -> Regis class FastAPIDatatypes: datatypes_registry: Registry = Depends(get_datatypes_registry) - @router.get('/api/datatypes', + @router.get( + '/api/datatypes', summary="Lists all available data types", response_model=Union[List[DatatypeDetails], List[str]], - response_description="List of data types") - async def index(self, - extension_only: Optional[bool] = ExtensionOnlyQueryParam, - upload_only: Optional[bool] = UploadOnlyQueryParam - ) -> Union[List[DatatypeDetails], List[str]]: + response_description="List of data types", + ) + async def index( + self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> Union[List[DatatypeDetails], List[str]]: """Gets the list of all available data types.""" - try: - return self._index(extension_only, upload_only) - except Exception as e: - log.exception('Could not get datat ypes') - raise e + return view_index(self.datatypes_registry, extension_only, upload_only) - @router.get('/api/datatypes/mapping', + @router.get( + '/api/datatypes/mapping', summary="Returns mappings for data types and their implementing classes", response_model=DatatypesMap, - response_description="Dictionary to map data types with their classes") + response_description="Dictionary to map data types with their classes" + ) async def mapping(self) -> DatatypesMap: """Gets mappings for data types.""" - try: - return self._mapping() - except Exception as e: - log.exception('Could not get data type mapping') - raise e + return view_mapping(self.datatypes_registry) - @router.get('/api/datatypes/types_and_mapping', + @router.get( + '/api/datatypes/types_and_mapping', summary="Returns all the data types extensions and their mappings", response_model=DatatypesCombinedMap, - response_description="Dictionary to map data types with their classes") - async def types_and_mapping(self) -> DatatypesCombinedMap: + response_description="Dictionary to map data types with their classes" + ) + async def types_and_mapping( + self, + extension_only: Optional[bool] = ExtensionOnlyQueryParam, + upload_only: Optional[bool] = UploadOnlyQueryParam + ) -> DatatypesCombinedMap: """Combines the datatype information from (/api/datatypes) and the mapping information from (/api/datatypes/mapping) into a single response.""" - return DatatypesCombinedMap(datatypes=self._index(), datatypes_mapping=self._mapping()) + return DatatypesCombinedMap( + datatypes=view_index(self.datatypes_registry, extension_only, upload_only), + datatypes_mapping=view_mapping(self.datatypes_registry) + ) - @router.get('/api/datatypes/sniffers', + @router.get( + '/api/datatypes/sniffers', summary="Returns the list of all installed sniffers", response_model=List[str], - response_description="List of datatype sniffers") + response_description="List of datatype sniffers" + ) async def sniffers(self) -> List[str]: """Gets the list of all installed data type sniffers.""" - try: - return self._sniffers() - except Exception as e: - log.exception('Could not get sniffers') - raise e + return view_sniffers(self.datatypes_registry) - @router.get('/api/datatypes/converters', + @router.get( + '/api/datatypes/converters', summary="Returns the list of all installed converters", response_model=List[DatatypeConverter], - response_description="List of all datatype converters") + response_description="List of all datatype converters" + ) async def converters(self) -> List[DatatypeConverter]: """Gets the list of all installed converters.""" - try: - return self._converters() - except Exception as e: - log.exception('Could not get converters') - raise e + return view_converters(self.datatypes_registry) - @router.get('/api/datatypes/edam_formats', + @router.get( + '/api/datatypes/edam_formats', summary="Returns a dictionary/map of datatypes and EDAM formats", response_model=Dict[str, str], - response_description="Dictionary/map of datatypes and EDAM formats") + response_description="Dictionary/map of datatypes and EDAM formats" + ) async def edam_formats(self) -> Dict[str, str]: """Gets a map of datatypes and their corresponding EDAM formats.""" - try: - return self.datatypes_registry.edam_formats - except Exception as e: - log.exception('Could not get EDAM format mappings') - raise e + return self.datatypes_registry.edam_formats - @router.get('/api/datatypes/edam_data', + @router.get( + '/api/datatypes/edam_data', summary="Returns a dictionary/map of datatypes and EDAM data", response_model=Dict[str, str], - response_description="Dictionary/map of datatypes and EDAM data") + response_description="Dictionary/map of datatypes and EDAM data" + ) async def edam_data(self) -> Dict[str, str]: """Gets a map of datatypes and their corresponding EDAM data.""" - try: - return self.datatypes_registry.edam_data - except Exception as e: - log.exception('Could not get EDAM data mappings') - raise e - - def _index(self, extension_only: Optional[bool] = True, upload_only: Optional[bool] = True) -> Union[List[DatatypeDetails], List[str]]: - if extension_only: - if upload_only: - return self.datatypes_registry.upload_file_formats - else: - return [ext for ext in self.datatypes_registry.datatypes_by_extension] - else: - rval = [] - for datatype_info_dict in self.datatypes_registry.datatype_info_dicts: - if upload_only and not datatype_info_dict.get('display_in_upload'): - continue - rval.append(datatype_info_dict) - return rval - - def _mapping(self) -> DatatypesMap: - ext_to_class_name: Dict[str, str] = {} - classes = [] - for k, v in self.datatypes_registry.datatypes_by_extension.items(): - c = v.__class__ - ext_to_class_name[k] = f"{c.__module__}.{c.__name__}" - classes.append(c) - class_to_classes: Dict[str, Dict[str, bool]] = {} - - def visit_bases(types, cls): - for base in cls.__bases__: - if issubclass(base, Data): - types.add(f"{base.__module__}.{base.__name__}") - visit_bases(types, base) - - for c in classes: - n = f"{c.__module__}.{c.__name__}" - types = {n} - visit_bases(types, c) - class_to_classes[n] = {t: True for t in types} - return DatatypesMap(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) - - def _sniffers(self) -> List[str]: - rval: List[str] = [] - for sniffer_elem in self.datatypes_registry.sniffer_elems: - datatype = sniffer_elem.get('type') - if datatype is not None: - rval.append(datatype) - return rval - - def _converters(self) -> List[DatatypeConverter]: - converters: List[DatatypeConverter] = [] - for (source_type, targets) in self.datatypes_registry.datatype_converters.items(): - for target_type in targets: - converters.append(DatatypeConverter( - source=source_type, - target=target_type, - tool_id=targets[target_type].id, - )) - return converters + return self.datatypes_registry.edam_data class DatatypesController(BaseAPIController): @@ -205,28 +151,9 @@ class DatatypesController(BaseAPIController): GET /api/datatypes Return an object containing upload datatypes. """ - try: - return self._index(trans, **kwd) - except Exception as e: - log.exception('Could not get datatypes') - raise e - - def _index(self, trans, **kwd): - datatypes_registry = self._datatypes_registry extension_only = asbool(kwd.get('extension_only', True)) upload_only = asbool(kwd.get('upload_only', True)) - if extension_only: - if upload_only: - return datatypes_registry.upload_file_formats - else: - return [ext for ext in datatypes_registry.datatypes_by_extension] - else: - rval = [] - for datatype_info_dict in datatypes_registry.datatype_info_dicts: - if not datatype_info_dict.get('display_in_upload') and upload_only: - continue - rval.append(datatype_info_dict) - return rval + return view_index(self._datatypes_registry, extension_only, upload_only) @expose_api_anonymous_and_sessionless def mapping(self, trans, **kwd): @@ -234,32 +161,7 @@ class DatatypesController(BaseAPIController): GET /api/datatypes/mapping Return a dictionary of class to class mappings. ''' - try: - return self._mapping() - except Exception as e: - log.exception('Could not get datatype mapping') - raise e - - def _mapping(self): - ext_to_class_name = dict() - classes = [] - for k, v in self._datatypes_registry.datatypes_by_extension.items(): - c = v.__class__ - ext_to_class_name[k] = c.__module__ + "." + c.__name__ - classes.append(c) - class_to_classes = dict() - - def visit_bases(types, cls): - for base in cls.__bases__: - if issubclass(base, Data): - types.add(base.__module__ + "." + base.__name__) - visit_bases(types, base) - for c in classes: - n = c.__module__ + "." + c.__name__ - types = {n} - visit_bases(types, c) - class_to_classes[n] = {t: True for t in types} - return dict(ext_to_class_name=ext_to_class_name, class_to_classes=class_to_classes) + return view_mapping(self._datatypes_registry) @expose_api_anonymous_and_sessionless def types_and_mapping(self, trans, **kwd): @@ -270,10 +172,9 @@ class DatatypesController(BaseAPIController): mapping information from (/api/datatypes/mapping) into a single response. """ - return { - "datatypes": self._index(trans, **kwd), - "datatypes_mapping": self._mapping(), - } + extension_only = asbool(kwd.get('extension_only', True)) + upload_only = asbool(kwd.get('upload_only', True)) + return view_types_and_mapping(self._datatypes_registry, extension_only, upload_only) @expose_api_anonymous_and_sessionless def sniffers(self, trans, **kwd): @@ -281,37 +182,19 @@ class DatatypesController(BaseAPIController): GET /api/datatypes/sniffers Return a list of sniffers. ''' - try: - rval = [] - for sniffer_elem in self._datatypes_registry.sniffer_elems: - datatype = sniffer_elem.get('type') - if datatype is not None: - rval.append(datatype) - return rval - except Exception as e: - log.exception('Could not get datatypes') - raise e + return view_sniffers(self._datatypes_registry) @expose_api_anonymous_and_sessionless def converters(self, trans, **kwd): - converters = [] - for (source_type, targets) in self._datatypes_registry.datatype_converters.items(): - for target_type in targets: - converters.append({ - 'source': source_type, - 'target': target_type, - 'tool_id': targets[target_type].id, - }) - - return converters + return view_converters(self._datatypes_registry) @expose_api_anonymous_and_sessionless def edam_formats(self, trans, **kwds): - return self._datatypes_registry.edam_formats + return view_edam_formats(self._datatypes_registry) @expose_api_anonymous_and_sessionless def edam_data(self, trans, **kwds): - return self._datatypes_registry.edam_data + return view_edam_data(self._datatypes_registry) @property def _datatypes_registry(self): From d69203e6985c4a517e79aab41b513bd92af1e172 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:38:52 +0100 Subject: [PATCH 10/14] Fix converters response serialization (legacy) --- lib/galaxy/managers/datatypes.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index 1512e699419..97c2bf895ed 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -189,14 +189,14 @@ def view_sniffers(datatypes_registry: Registry) -> List[str]: def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: - converters: List[DatatypeConverter] = [] + converters = [] for (source_type, targets) in datatypes_registry.datatype_converters.items(): for target_type in targets: - converters.append(DatatypeConverter( - source=source_type, - target=target_type, - tool_id=targets[target_type].id, - )) + converters.append({ + 'source': source_type, + 'target': target_type, + 'tool_id': targets[target_type].id, + }) return converters From c518755d74e7c0d7f1b3e4a00e202d68cd1d564a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20L=C3=B3pez?= <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:55:45 +0100 Subject: [PATCH 11/14] Update lib/galaxy_test/api/test_datatypes.py Co-authored-by: Marius van den Beek --- lib/galaxy_test/api/test_datatypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy_test/api/test_datatypes.py b/lib/galaxy_test/api/test_datatypes.py index e310cf11bbe..3cc25f8eadf 100644 --- a/lib/galaxy_test/api/test_datatypes.py +++ b/lib/galaxy_test/api/test_datatypes.py @@ -84,7 +84,7 @@ class DatatypesApiTestCase(ApiTestCase): self._assert_status_code_is(response, 200) assert converters == len(response.json()) - def test_edamp_formats(self): + def test_edam_formats(self): response = self._get("datatypes/edam_formats") self._assert_status_code_is(response, 200) edam_formats = response.json() From 61abc3fcc26cf8cc389521bcc524c5a114926aa4 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Tue, 12 Jan 2021 15:57:56 +0100 Subject: [PATCH 12/14] Fix typing serialization --- lib/galaxy/managers/datatypes.py | 12 ++++++++++-- lib/galaxy/webapps/galaxy/api/datatypes.py | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index 97c2bf895ed..edce2737e17 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -10,6 +10,7 @@ from pydantic import ( Field, HttpUrl ) +from pydantic.tools import parse_obj_as from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry @@ -126,6 +127,13 @@ class DatatypeConverter(BaseModel): ) +class DatatypeConverterList(BaseModel): + __root__: List[DatatypeConverter] = Field( + title='List of data type converters', + default=[] + ) + + def view_index( datatypes_registry: Registry, extension_only: Optional[bool] = True, @@ -188,7 +196,7 @@ def view_sniffers(datatypes_registry: Registry) -> List[str]: return rval -def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: +def view_converters(datatypes_registry: Registry) -> DatatypeConverterList: converters = [] for (source_type, targets) in datatypes_registry.datatype_converters.items(): for target_type in targets: @@ -197,7 +205,7 @@ def view_converters(datatypes_registry: Registry) -> List[DatatypeConverter]: 'target': target_type, 'tool_id': targets[target_type].id, }) - return converters + return parse_obj_as(DatatypeConverterList, converters) def view_edam_formats(datatypes_registry: Registry) -> Dict[str, str]: diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 820aadff2c7..178cf870b2c 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -19,7 +19,7 @@ from fastapi_utils.cbv import cbv from galaxy.app import UniverseApplication from galaxy.datatypes.registry import Registry from galaxy.managers.datatypes import ( - DatatypeConverter, + DatatypeConverterList, DatatypeDetails, DatatypesCombinedMap, DatatypesMap, @@ -115,10 +115,10 @@ class FastAPIDatatypes: @router.get( '/api/datatypes/converters', summary="Returns the list of all installed converters", - response_model=List[DatatypeConverter], + response_model=DatatypeConverterList, response_description="List of all datatype converters" ) - async def converters(self) -> List[DatatypeConverter]: + async def converters(self) -> DatatypeConverterList: """Gets the list of all installed converters.""" return view_converters(self.datatypes_registry) From 02e9f02db9af0118658f1183a2770283e1798f24 Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 13 Jan 2021 14:04:14 +0100 Subject: [PATCH 13/14] Refactor move pydantic model to datatypes/_schema.py --- lib/galaxy/datatypes/_schema.py | 129 +++++++++++++++++++++ lib/galaxy/managers/datatypes.py | 129 +-------------------- lib/galaxy/webapps/galaxy/api/datatypes.py | 4 +- 3 files changed, 138 insertions(+), 124 deletions(-) create mode 100644 lib/galaxy/datatypes/_schema.py diff --git a/lib/galaxy/datatypes/_schema.py b/lib/galaxy/datatypes/_schema.py new file mode 100644 index 00000000000..6efb7cc4565 --- /dev/null +++ b/lib/galaxy/datatypes/_schema.py @@ -0,0 +1,129 @@ +from typing import ( + Dict, + List, + Optional, +) + +from pydantic import ( + BaseModel, + Field, + HttpUrl +) + + +class CompositeFileInfo(BaseModel): + name: str = Field( + ..., # Mark this field as required + title="Name", + description="The name of this composite file" + ) + optional: bool = Field( + title="Optional", + description="" # TODO add description + ) + mimetype: Optional[str] = Field( + title="MIME type", + description="The MIME type of this file" + ) + description: Optional[str] = Field( + title="Description", + description="Summary description of the purpouse of this file" + ) + substitute_name_with_metadata: Optional[str] = Field( + title="Substitute name with metadata", + description="" # TODO add description + ) + is_binary: bool = Field( + title="Is binary", + description="Whether this file is a binary file" + ) + to_posix_lines: bool = Field( + title="To posix lines", + description="" # TODO add description + ) + space_to_tab: bool = Field( + title="Spaces to tabulation", + description="" # TODO add description + ) + + +class DatatypeDetails(BaseModel): + extension: str = Field( + ..., # Mark this field as required + title="Extension", + description="The data type’s Dataset file extension", + example="bed" + ) + description: Optional[str] = Field( + title="Description", + description="A summary description for this data type" + ) + description_url: Optional[HttpUrl] = Field( + title="Description URL", + description="The URL to a detailed description for this datatype", + example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" + ) + display_in_upload: bool = Field( + default=False, + title="Display in upload", + description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" + ) + composite_files: Optional[List[CompositeFileInfo]] = Field( + default=None, + title="Composite files", + description="A collection of files composing this data type" + ) + + +class DatatypesMap(BaseModel): + ext_to_class_name: Dict[str, str] = Field( + ..., # Mark this field as required + title="Extension Map", + description="Dictionary mapping datatype's extensions with implementation classes", + ) + class_to_classes: Dict[str, Dict[str, bool]] = Field( + ..., # Mark this field as required + title="Classes Map", + description="Dictionary mapping datatype's classes with their base classes", + ) + + +class DatatypesCombinedMap(BaseModel): + datatypes: List[str] = Field( + ..., # Mark this field as required + title="Datatypes", + description="List of datatypes extensions", + ) + datatypes_mapping: DatatypesMap = Field( + ..., # Mark this field as required + title="Datatypes Mapping", + description="Dictionaries for mapping datatype's extensions/classes with their implementation classes", + ) + + +class DatatypeConverter(BaseModel): + source: str = Field( + ..., # Mark this field as required + title="Source", + description="Source type for conversion", + example="bam", + ) + target: str = Field( + ..., # Mark this field as required + title="Target", + description="Target type for conversion", + example="bai", + ) + tool_id: str = Field( + ..., # Mark this field as required + title="Tool identifier", + description="The converter tool identifier", + example="CONVERTER_Bam_Bai_0", + ) + + +class DatatypeConverterList(BaseModel): + __root__: List[DatatypeConverter] = Field( + title='List of data type converters', + default=[] + ) diff --git a/lib/galaxy/managers/datatypes.py b/lib/galaxy/managers/datatypes.py index edce2737e17..6913bc32104 100644 --- a/lib/galaxy/managers/datatypes.py +++ b/lib/galaxy/managers/datatypes.py @@ -5,135 +5,18 @@ from typing import ( Union, ) -from pydantic import ( - BaseModel, - Field, - HttpUrl -) from pydantic.tools import parse_obj_as +from galaxy.datatypes._schema import ( + DatatypeConverterList, + DatatypeDetails, + DatatypesCombinedMap, + DatatypesMap, +) from galaxy.datatypes.data import Data from galaxy.datatypes.registry import Registry -class CompositeFileInfo(BaseModel): - name: str = Field( - ..., # Mark this field as required - title="Name", - description="The name of this composite file" - ) - optional: bool = Field( - title="Optional", - description="" # TODO add description - ) - mimetype: Optional[str] = Field( - title="MIME type", - description="The MIME type of this file" - ) - description: Optional[str] = Field( - title="Description", - description="Summary description of the purpouse of this file" - ) - substitute_name_with_metadata: Optional[str] = Field( - title="Substitute name with metadata", - description="" # TODO add description - ) - is_binary: bool = Field( - title="Is binary", - description="Whether this file is a binary file" - ) - to_posix_lines: bool = Field( - title="To posix lines", - description="" # TODO add description - ) - space_to_tab: bool = Field( - title="Spaces to tabulation", - description="" # TODO add description - ) - - -class DatatypeDetails(BaseModel): - extension: str = Field( - ..., # Mark this field as required - title="Extension", - description="The data type’s Dataset file extension", - example="bed" - ) - description: Optional[str] = Field( - title="Description", - description="A summary description for this data type" - ) - description_url: Optional[HttpUrl] = Field( - title="Description URL", - description="The URL to a detailed description for this datatype", - example="https://wiki.galaxyproject.org/Learn/Datatypes#Bed" - ) - display_in_upload: bool = Field( - default=False, - title="Display in upload", - description="If True, the associated file extension will be displayed in the `File Format` select list in the `Upload File from your computer` tool in the `Get Data` tool section of the tool panel" - ) - composite_files: Optional[List[CompositeFileInfo]] = Field( - default=None, - title="Composite files", - description="A collection of files composing this data type" - ) - - -class DatatypesMap(BaseModel): - ext_to_class_name: Dict[str, str] = Field( - ..., # Mark this field as required - title="Extension Map", - description="Dictionary mapping datatype's extensions with implementation classes", - ) - class_to_classes: Dict[str, Dict[str, bool]] = Field( - ..., # Mark this field as required - title="Classes Map", - description="Dictionary mapping datatype's classes with their base classes", - ) - - -class DatatypesCombinedMap(BaseModel): - datatypes: List[str] = Field( - ..., # Mark this field as required - title="Datatypes", - description="List of datatypes extensions", - ) - datatypes_mapping: DatatypesMap = Field( - ..., # Mark this field as required - title="Datatypes Mapping", - description="Dictionaries for mapping datatype's extensions/classes with their implementation classes", - ) - - -class DatatypeConverter(BaseModel): - source: str = Field( - ..., # Mark this field as required - title="Source", - description="Source type for conversion", - example="bam", - ) - target: str = Field( - ..., # Mark this field as required - title="Target", - description="Target type for conversion", - example="bai", - ) - tool_id: str = Field( - ..., # Mark this field as required - title="Tool identifier", - description="The converter tool identifier", - example="CONVERTER_Bam_Bai_0", - ) - - -class DatatypeConverterList(BaseModel): - __root__: List[DatatypeConverter] = Field( - title='List of data type converters', - default=[] - ) - - def view_index( datatypes_registry: Registry, extension_only: Optional[bool] = True, diff --git a/lib/galaxy/webapps/galaxy/api/datatypes.py b/lib/galaxy/webapps/galaxy/api/datatypes.py index 178cf870b2c..55d748b886b 100644 --- a/lib/galaxy/webapps/galaxy/api/datatypes.py +++ b/lib/galaxy/webapps/galaxy/api/datatypes.py @@ -23,7 +23,9 @@ from galaxy.managers.datatypes import ( DatatypeDetails, DatatypesCombinedMap, DatatypesMap, - view_converters, view_edam_data, view_edam_formats, + view_converters, + view_edam_data, + view_edam_formats, view_index, view_mapping, view_sniffers, From e91e754ac8e2bf2d1858eeec01b77c7581fb304a Mon Sep 17 00:00:00 2001 From: davelopez <46503462+davelopez@users.noreply.github.com> Date: Wed, 13 Jan 2021 14:31:16 +0100 Subject: [PATCH 14/14] Add pydantic to the packages/data/requirements.txt --- packages/data/requirements.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/data/requirements.txt b/packages/data/requirements.txt index 98424dcbef5..85c7facff5b 100644 --- a/packages/data/requirements.txt +++ b/packages/data/requirements.txt @@ -8,6 +8,7 @@ isa-rwval parsley numpy<=1.16 pycryptodome +pydantic pysam social_auth_core SQLAlchemy