From ecd6bc0549c4678c796f3aa9873628bce588bd21 Mon Sep 17 00:00:00 2001 From: PlushZ Date: Tue, 28 Apr 2026 15:18:19 +1000 Subject: [PATCH 1/7] add filesource for mavedb based on mavedb-fsspec --- lib/galaxy/files/sources/mavedb.py | 114 ++++++++++++++++++ .../templates/examples/production_mavedb.yml | 19 +++ lib/galaxy/files/templates/models.py | 20 +++ 3 files changed, 153 insertions(+) create mode 100644 lib/galaxy/files/sources/mavedb.py create mode 100644 lib/galaxy/files/templates/examples/production_mavedb.yml diff --git a/lib/galaxy/files/sources/mavedb.py b/lib/galaxy/files/sources/mavedb.py new file mode 100644 index 00000000000..f3b28e7a402 --- /dev/null +++ b/lib/galaxy/files/sources/mavedb.py @@ -0,0 +1,114 @@ +from typing import ( + Optional, + Union, +) + +from fsspec import AbstractFileSystem + +from galaxy.exceptions import AuthenticationRequired, MessageException +from galaxy.files.models import AnyRemoteEntry, FilesSourceRuntimeContext +from galaxy.files.sources._defaults import DEFAULT_SCHEME +from galaxy.files.sources._fsspec import ( + CacheOptionsDictType, + FsspecBaseFileSourceConfiguration, + FsspecBaseFileSourceTemplateConfiguration, + FsspecFilesSource, +) +from galaxy.util.config_templates import TemplateExpansion + +try: + from mavedb_fsspec import MaveDBFileSystem + from mavedb_fsspec.client import DEFAULT_BASE_URL +except ImportError: + MaveDBFileSystem = None # type: ignore[misc, assignment] + DEFAULT_BASE_URL = "https://api.mavedb.org/api/v1" + + +class MaveDBFileSourceTemplateConfiguration(FsspecBaseFileSourceTemplateConfiguration): + base_url: Union[str, TemplateExpansion] = DEFAULT_BASE_URL + api_key: Union[str, TemplateExpansion, None] = None + timeout: Union[float, TemplateExpansion] = 30.0 + + +class MaveDBFileSourceConfiguration(FsspecBaseFileSourceConfiguration): + base_url: str = DEFAULT_BASE_URL + api_key: Optional[str] = None + timeout: float = 30.0 + + +class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, MaveDBFileSourceConfiguration]): + plugin_type = "mavedb" + required_module = MaveDBFileSystem + required_package = "mavedb-fsspec" + + template_config_class = MaveDBFileSourceTemplateConfiguration + resolved_config_class = MaveDBFileSourceConfiguration + + def _open_fs( + self, + context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration], + cache_options: CacheOptionsDictType, + ) -> AbstractFileSystem: + if MaveDBFileSystem is None: + raise self.required_package_exception + + config = context.config + return MaveDBFileSystem( + base_url=config.base_url, + api_key=config.api_key, + timeout=config.timeout, + **cache_options, + ) + + def _list( + self, + context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration], + path="/", + recursive=False, + write_intent: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None, + query: Optional[str] = None, + sort_by: Optional[str] = None, + ) -> tuple[list[AnyRemoteEntry], int]: + collection = path.strip("/") + if recursive or collection not in {"score-sets", "my-score-sets"}: + return super()._list(context, path, recursive, write_intent, limit, offset, query, sort_by) + + try: + cache_options = self._get_cache_options(context.config) + fs = self._open_fs(context, cache_options) + entries, total_count = fs.list_score_sets( # type: ignore[attr-defined] + collection=collection, + limit=limit, + offset=offset, + query=query, + ) + return [self._info_to_entry(entry, context.config) for entry in entries], total_count + except PermissionError as e: + raise AuthenticationRequired( + f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}." + ) + except Exception as e: + raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e + + def _info_to_entry(self, info: dict, config: MaveDBFileSourceConfiguration) -> AnyRemoteEntry: + entry = super()._info_to_entry(info, config) + display_name = info.get("display_name") + if display_name: + entry.name = display_name + return entry + + def _write_from( + self, + _target_path: str, + _native_path: str, + _context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration], + ): + raise MessageException("MaveDB file sources are read-only and do not support exporting files.") + + def get_scheme(self) -> str: + return self.scheme if self.scheme and self.scheme != DEFAULT_SCHEME else "mavedb" + + +__all__ = ("MaveDBFilesSource",) diff --git a/lib/galaxy/files/templates/examples/production_mavedb.yml b/lib/galaxy/files/templates/examples/production_mavedb.yml new file mode 100644 index 00000000000..d51e00a17fa --- /dev/null +++ b/lib/galaxy/files/templates/examples/production_mavedb.yml @@ -0,0 +1,19 @@ +- id: mavedb + version: 0 + name: MaveDB + description: | + [MaveDB](https://www.mavedb.org) provides Multiplexed Assays of Variant Effect datasets. + This file source exposes read-only score set files through the public MaveDB API. + secrets: + api_key: + label: MaveDB API key + optional: true + help: | + Optional MaveDB API key. Public score set files do not require an API key. + A user API key can be generated in the MaveDB settings page for your account + and is needed to access private datasets. + configuration: + type: mavedb + base_url: https://api.mavedb.org/api/v1 + api_key: "{{ secrets.api_key }}" + timeout: 30 diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py index e1e36e80a24..f104e1f24cf 100644 --- a/lib/galaxy/files/templates/models.py +++ b/lib/galaxy/files/templates/models.py @@ -51,6 +51,7 @@ FileSourceTemplateType = Literal[ "dataverse", "huggingface", "iiif", + "mavedb", "omero", "ssh", ] @@ -440,6 +441,22 @@ class IIIFFileSourceConfiguration(StrictModel): manifest_url: str +class MaveDBFileSourceTemplateConfiguration(StrictModel): + type: Literal["mavedb"] + base_url: Union[str, TemplateExpansion] = "https://api.mavedb.org/api/v1" + api_key: Union[str, TemplateExpansion, None] = None + timeout: Union[float, TemplateExpansion] = 30.0 + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class MaveDBFileSourceConfiguration(StrictModel): + type: Literal["mavedb"] + base_url: str = "https://api.mavedb.org/api/v1" + api_key: Optional[str] = None + timeout: float = 30.0 + + class OmeroFileSourceTemplateConfiguration(StrictModel): type: Literal["omero"] username: Union[str, TemplateExpansion] @@ -480,6 +497,7 @@ FileSourceTemplateConfiguration = Annotated[ DataverseFileSourceTemplateConfiguration, HuggingFaceFileSourceTemplateConfiguration, IIIFFileSourceTemplateConfiguration, + MaveDBFileSourceTemplateConfiguration, OmeroFileSourceTemplateConfiguration, SshFileSourceTemplateConfiguration, ], @@ -506,6 +524,7 @@ FileSourceConfiguration = Annotated[ DataverseFileSourceConfiguration, HuggingFaceFileSourceConfiguration, IIIFFileSourceConfiguration, + MaveDBFileSourceConfiguration, OmeroFileSourceConfiguration, SshFileSourceConfiguration, ], @@ -590,6 +609,7 @@ TypesToConfigurationClasses: dict[FileSourceTemplateType, type[FileSourceConfigu "dataverse": DataverseFileSourceConfiguration, "huggingface": HuggingFaceFileSourceConfiguration, "iiif": IIIFFileSourceConfiguration, + "mavedb": MaveDBFileSourceConfiguration, "omero": OmeroFileSourceConfiguration, "ssh": SshFileSourceConfiguration, } From 50f3e387a9a8c043433607a0be848528fbeef6cf Mon Sep 17 00:00:00 2001 From: PlushZ Date: Wed, 29 Apr 2026 12:36:49 +1000 Subject: [PATCH 2/7] fix formatting --- lib/galaxy/files/sources/mavedb.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/files/sources/mavedb.py b/lib/galaxy/files/sources/mavedb.py index f3b28e7a402..a5f4dc0aff2 100644 --- a/lib/galaxy/files/sources/mavedb.py +++ b/lib/galaxy/files/sources/mavedb.py @@ -5,8 +5,14 @@ from typing import ( from fsspec import AbstractFileSystem -from galaxy.exceptions import AuthenticationRequired, MessageException -from galaxy.files.models import AnyRemoteEntry, FilesSourceRuntimeContext +from galaxy.exceptions import ( + AuthenticationRequired, + MessageException, +) +from galaxy.files.models import ( + AnyRemoteEntry, + FilesSourceRuntimeContext, +) from galaxy.files.sources._defaults import DEFAULT_SCHEME from galaxy.files.sources._fsspec import ( CacheOptionsDictType, @@ -20,7 +26,7 @@ try: from mavedb_fsspec import MaveDBFileSystem from mavedb_fsspec.client import DEFAULT_BASE_URL except ImportError: - MaveDBFileSystem = None # type: ignore[misc, assignment] + MaveDBFileSystem = None DEFAULT_BASE_URL = "https://api.mavedb.org/api/v1" @@ -78,7 +84,7 @@ class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, try: cache_options = self._get_cache_options(context.config) fs = self._open_fs(context, cache_options) - entries, total_count = fs.list_score_sets( # type: ignore[attr-defined] + entries, total_count = fs.list_score_sets( collection=collection, limit=limit, offset=offset, From 01fe68789bb071cfd78427155c78a79ddf831496 Mon Sep 17 00:00:00 2001 From: PlushZ Date: Wed, 29 Apr 2026 12:54:48 +1000 Subject: [PATCH 3/7] schema changes after make update-client-api-schema --- client/packages/api-client/src/schema/schema.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/packages/api-client/src/schema/schema.ts b/client/packages/api-client/src/schema/schema.ts index a65bbc4cdb5..54a8ff970c0 100644 --- a/client/packages/api-client/src/schema/schema.ts +++ b/client/packages/api-client/src/schema/schema.ts @@ -13092,6 +13092,7 @@ export interface components { | "dataverse" | "huggingface" | "iiif" + | "mavedb" | "omero" | "ssh"; /** Variables */ @@ -25381,6 +25382,7 @@ export interface components { | "dataverse" | "huggingface" | "iiif" + | "mavedb" | "omero" | "ssh"; /** Uri Root */ From 3d2b15ca65fcec24825bcb666ccf846b9eccc10d Mon Sep 17 00:00:00 2001 From: PlushZ Date: Wed, 29 Apr 2026 14:07:54 +1000 Subject: [PATCH 4/7] add mavedb key to templateTypes --- client/src/api/fileSources.ts | 4 ++++ lib/galaxy/files/templates/examples/production_mavedb.yml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/client/src/api/fileSources.ts b/client/src/api/fileSources.ts index 008775f999b..ff8320d92a8 100644 --- a/client/src/api/fileSources.ts +++ b/client/src/api/fileSources.ts @@ -32,6 +32,10 @@ export const templateTypes: FileSourceTypesDetail = { icon: faGoogleDrive, message: "This is a repository plugin that connects with the commercial Google Drive service.", }, + mavedb: { + icon: faNetworkWired, + message: "This is a repository plugin that connects with MaveDB score set files.", + }, onedrive: { icon: faCloud, message: "This is a repository plugin that connects with Microsoft OneDrive.", diff --git a/lib/galaxy/files/templates/examples/production_mavedb.yml b/lib/galaxy/files/templates/examples/production_mavedb.yml index d51e00a17fa..b7b99fdbb7a 100644 --- a/lib/galaxy/files/templates/examples/production_mavedb.yml +++ b/lib/galaxy/files/templates/examples/production_mavedb.yml @@ -3,7 +3,7 @@ name: MaveDB description: | [MaveDB](https://www.mavedb.org) provides Multiplexed Assays of Variant Effect datasets. - This file source exposes read-only score set files through the public MaveDB API. + This file source exposes score set files through the public MaveDB API. secrets: api_key: label: MaveDB API key From f2e4608a763871edbea33f99c185251546784a47 Mon Sep 17 00:00:00 2001 From: PlushZ Date: Thu, 21 May 2026 19:16:12 +1000 Subject: [PATCH 5/7] adapt to changes after mavedb-fsspec review --- lib/galaxy/files/sources/mavedb.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/files/sources/mavedb.py b/lib/galaxy/files/sources/mavedb.py index a5f4dc0aff2..3557c1a03b4 100644 --- a/lib/galaxy/files/sources/mavedb.py +++ b/lib/galaxy/files/sources/mavedb.py @@ -63,6 +63,7 @@ class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, base_url=config.base_url, api_key=config.api_key, timeout=config.timeout, + skip_instance_cache=True, **cache_options, ) @@ -84,12 +85,17 @@ class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, try: cache_options = self._get_cache_options(context.config) fs = self._open_fs(context, cache_options) - entries, total_count = fs.list_score_sets( - collection=collection, - limit=limit, - offset=offset, - query=query, - ) + try: + entries, total_count = fs.list_score_sets( + collection=collection, + limit=limit, + offset=offset, + query=query, + ) + finally: + close = getattr(fs, "close", None) + if close is not None: + close() return [self._info_to_entry(entry, context.config) for entry in entries], total_count except PermissionError as e: raise AuthenticationRequired( From 64077508ec49b45e4260089cadb7f15ab190a93a Mon Sep 17 00:00:00 2001 From: PlushZ Date: Wed, 10 Jun 2026 20:50:00 +0200 Subject: [PATCH 6/7] Install mavedb-fsspec for MaveDB file source --- lib/galaxy/dependencies/__init__.py | 3 +++ lib/galaxy/dependencies/conditional-requirements.txt | 1 + 2 files changed, 4 insertions(+) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 0cb34fb839f..f1aff439037 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -363,6 +363,9 @@ class ConditionalDependencies: def check_iiif_fsspec(self): return "iiif" in self.file_sources + def check_mavedb_fsspec(self): + return "mavedb" in self.file_sources + def strip_comment(line): # lifted from https://github.com/tox-dev/tox/commit/3c6b4f204e89852c4b7536b246a66d20be6d39ec diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 2c45fb5ba80..0e6f82529d7 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -36,6 +36,7 @@ huggingface_hub omero-py #type: omero iiif-fsspec # type: iiif fs-irods # type: irods, depends on python-irodsclient +mavedb-fsspec>=0.1.0 # type: mavedb # Vault backend hvac From d6264e13cbeb97ba58a964dcc83c0647eecb4b09 Mon Sep 17 00:00:00 2001 From: PlushZ Date: Thu, 11 Jun 2026 16:20:51 +0200 Subject: [PATCH 7/7] Clarify MaveDB-specific filesystem listing --- lib/galaxy/files/sources/mavedb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/files/sources/mavedb.py b/lib/galaxy/files/sources/mavedb.py index 3557c1a03b4..cd70ad1ff63 100644 --- a/lib/galaxy/files/sources/mavedb.py +++ b/lib/galaxy/files/sources/mavedb.py @@ -3,8 +3,6 @@ from typing import ( Union, ) -from fsspec import AbstractFileSystem - from galaxy.exceptions import ( AuthenticationRequired, MessageException, @@ -54,7 +52,7 @@ class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, self, context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration], cache_options: CacheOptionsDictType, - ) -> AbstractFileSystem: + ) -> "MaveDBFileSystem": if MaveDBFileSystem is None: raise self.required_package_exception @@ -86,6 +84,7 @@ class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, cache_options = self._get_cache_options(context.config) fs = self._open_fs(context, cache_options) try: + # Use the MaveDB-specific API for server-side pagination and the total result count. entries, total_count = fs.list_score_sets( collection=collection, limit=limit,