diff --git a/client/packages/api-client/src/schema/schema.ts b/client/packages/api-client/src/schema/schema.ts index 7e55bfc71d7..96457a65460 100644 --- a/client/packages/api-client/src/schema/schema.ts +++ b/client/packages/api-client/src/schema/schema.ts @@ -13092,6 +13092,7 @@ export interface components { | "dataverse" | "huggingface" | "iiif" + | "mavedb" | "omero" | "ssh"; /** Variables */ @@ -25387,6 +25388,7 @@ export interface components { | "dataverse" | "huggingface" | "iiif" + | "mavedb" | "omero" | "ssh"; /** Uri Root */ diff --git a/client/src/api/fileSources.ts b/client/src/api/fileSources.ts index 008775f999b..ff8320d92a8 100644 --- a/client/src/api/fileSources.ts +++ b/client/src/api/fileSources.ts @@ -32,6 +32,10 @@ export const templateTypes: FileSourceTypesDetail = { icon: faGoogleDrive, message: "This is a repository plugin that connects with the commercial Google Drive service.", }, + mavedb: { + icon: faNetworkWired, + message: "This is a repository plugin that connects with MaveDB score set files.", + }, onedrive: { icon: faCloud, message: "This is a repository plugin that connects with Microsoft OneDrive.", diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 0cb34fb839f..f1aff439037 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -363,6 +363,9 @@ class ConditionalDependencies: def check_iiif_fsspec(self): return "iiif" in self.file_sources + def check_mavedb_fsspec(self): + return "mavedb" in self.file_sources + def strip_comment(line): # lifted from https://github.com/tox-dev/tox/commit/3c6b4f204e89852c4b7536b246a66d20be6d39ec diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 2c45fb5ba80..0e6f82529d7 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -36,6 +36,7 @@ huggingface_hub omero-py #type: omero iiif-fsspec # type: iiif fs-irods # type: irods, depends on python-irodsclient +mavedb-fsspec>=0.1.0 # type: mavedb # Vault backend hvac diff --git a/lib/galaxy/files/sources/mavedb.py b/lib/galaxy/files/sources/mavedb.py new file mode 100644 index 00000000000..cd70ad1ff63 --- /dev/null +++ b/lib/galaxy/files/sources/mavedb.py @@ -0,0 +1,125 @@ +from typing import ( + Optional, + Union, +) + +from galaxy.exceptions import ( + AuthenticationRequired, + MessageException, +) +from galaxy.files.models import ( + AnyRemoteEntry, + FilesSourceRuntimeContext, +) +from galaxy.files.sources._defaults import DEFAULT_SCHEME +from galaxy.files.sources._fsspec import ( + CacheOptionsDictType, + FsspecBaseFileSourceConfiguration, + FsspecBaseFileSourceTemplateConfiguration, + FsspecFilesSource, +) +from galaxy.util.config_templates import TemplateExpansion + +try: + from mavedb_fsspec import MaveDBFileSystem + from mavedb_fsspec.client import DEFAULT_BASE_URL +except ImportError: + MaveDBFileSystem = None + DEFAULT_BASE_URL = "https://api.mavedb.org/api/v1" + + +class MaveDBFileSourceTemplateConfiguration(FsspecBaseFileSourceTemplateConfiguration): + base_url: Union[str, TemplateExpansion] = DEFAULT_BASE_URL + api_key: Union[str, TemplateExpansion, None] = None + timeout: Union[float, TemplateExpansion] = 30.0 + + +class MaveDBFileSourceConfiguration(FsspecBaseFileSourceConfiguration): + base_url: str = DEFAULT_BASE_URL + api_key: Optional[str] = None + timeout: float = 30.0 + + +class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, MaveDBFileSourceConfiguration]): + plugin_type = "mavedb" + required_module = MaveDBFileSystem + required_package = "mavedb-fsspec" + + template_config_class = MaveDBFileSourceTemplateConfiguration + resolved_config_class = MaveDBFileSourceConfiguration + + def _open_fs( + self, + context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration], + cache_options: CacheOptionsDictType, + ) -> "MaveDBFileSystem": + if MaveDBFileSystem is None: + raise self.required_package_exception + + config = context.config + return MaveDBFileSystem( + base_url=config.base_url, + api_key=config.api_key, + timeout=config.timeout, + skip_instance_cache=True, + **cache_options, + ) + + def _list( + self, + context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration], + path="/", + recursive=False, + write_intent: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None, + query: Optional[str] = None, + sort_by: Optional[str] = None, + ) -> tuple[list[AnyRemoteEntry], int]: + collection = path.strip("/") + if recursive or collection not in {"score-sets", "my-score-sets"}: + return super()._list(context, path, recursive, write_intent, limit, offset, query, sort_by) + + try: + cache_options = self._get_cache_options(context.config) + fs = self._open_fs(context, cache_options) + try: + # Use the MaveDB-specific API for server-side pagination and the total result count. + entries, total_count = fs.list_score_sets( + collection=collection, + limit=limit, + offset=offset, + query=query, + ) + finally: + close = getattr(fs, "close", None) + if close is not None: + close() + return [self._info_to_entry(entry, context.config) for entry in entries], total_count + except PermissionError as e: + raise AuthenticationRequired( + f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}." + ) + except Exception as e: + raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e + + def _info_to_entry(self, info: dict, config: MaveDBFileSourceConfiguration) -> AnyRemoteEntry: + entry = super()._info_to_entry(info, config) + display_name = info.get("display_name") + if display_name: + entry.name = display_name + return entry + + def _write_from( + self, + _target_path: str, + _native_path: str, + _context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration], + ): + raise MessageException("MaveDB file sources are read-only and do not support exporting files.") + + def get_scheme(self) -> str: + return self.scheme if self.scheme and self.scheme != DEFAULT_SCHEME else "mavedb" + + +__all__ = ("MaveDBFilesSource",) diff --git a/lib/galaxy/files/templates/examples/production_mavedb.yml b/lib/galaxy/files/templates/examples/production_mavedb.yml new file mode 100644 index 00000000000..b7b99fdbb7a --- /dev/null +++ b/lib/galaxy/files/templates/examples/production_mavedb.yml @@ -0,0 +1,19 @@ +- id: mavedb + version: 0 + name: MaveDB + description: | + [MaveDB](https://www.mavedb.org) provides Multiplexed Assays of Variant Effect datasets. + This file source exposes score set files through the public MaveDB API. + secrets: + api_key: + label: MaveDB API key + optional: true + help: | + Optional MaveDB API key. Public score set files do not require an API key. + A user API key can be generated in the MaveDB settings page for your account + and is needed to access private datasets. + configuration: + type: mavedb + base_url: https://api.mavedb.org/api/v1 + api_key: "{{ secrets.api_key }}" + timeout: 30 diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py index e1e36e80a24..f104e1f24cf 100644 --- a/lib/galaxy/files/templates/models.py +++ b/lib/galaxy/files/templates/models.py @@ -51,6 +51,7 @@ FileSourceTemplateType = Literal[ "dataverse", "huggingface", "iiif", + "mavedb", "omero", "ssh", ] @@ -440,6 +441,22 @@ class IIIFFileSourceConfiguration(StrictModel): manifest_url: str +class MaveDBFileSourceTemplateConfiguration(StrictModel): + type: Literal["mavedb"] + base_url: Union[str, TemplateExpansion] = "https://api.mavedb.org/api/v1" + api_key: Union[str, TemplateExpansion, None] = None + timeout: Union[float, TemplateExpansion] = 30.0 + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class MaveDBFileSourceConfiguration(StrictModel): + type: Literal["mavedb"] + base_url: str = "https://api.mavedb.org/api/v1" + api_key: Optional[str] = None + timeout: float = 30.0 + + class OmeroFileSourceTemplateConfiguration(StrictModel): type: Literal["omero"] username: Union[str, TemplateExpansion] @@ -480,6 +497,7 @@ FileSourceTemplateConfiguration = Annotated[ DataverseFileSourceTemplateConfiguration, HuggingFaceFileSourceTemplateConfiguration, IIIFFileSourceTemplateConfiguration, + MaveDBFileSourceTemplateConfiguration, OmeroFileSourceTemplateConfiguration, SshFileSourceTemplateConfiguration, ], @@ -506,6 +524,7 @@ FileSourceConfiguration = Annotated[ DataverseFileSourceConfiguration, HuggingFaceFileSourceConfiguration, IIIFFileSourceConfiguration, + MaveDBFileSourceConfiguration, OmeroFileSourceConfiguration, SshFileSourceConfiguration, ], @@ -590,6 +609,7 @@ TypesToConfigurationClasses: dict[FileSourceTemplateType, type[FileSourceConfigu "dataverse": DataverseFileSourceConfiguration, "huggingface": HuggingFaceFileSourceConfiguration, "iiif": IIIFFileSourceConfiguration, + "mavedb": MaveDBFileSourceConfiguration, "omero": OmeroFileSourceConfiguration, "ssh": SshFileSourceConfiguration, }