Merge pull request #22576 from PlushZ/add-mavedb-filesource

Add MaveDB filesource based on mavedb-fsspec

client/packages/api-client/src/schema/schema.ts

@@ -13092,6 +13092,7 @@ export interface components {
                 | "dataverse"
                 | "huggingface"
                 | "iiif"
+                | "mavedb"
                 | "omero"
                 | "ssh";
             /** Variables */
@@ -25387,6 +25388,7 @@ export interface components {
                 | "dataverse"
                 | "huggingface"
                 | "iiif"
+                | "mavedb"
                 | "omero"
                 | "ssh";
             /** Uri Root */

client/src/api/fileSources.ts

@@ -32,6 +32,10 @@ export const templateTypes: FileSourceTypesDetail = {
         icon: faGoogleDrive,
         message: "This is a  repository plugin that connects with the commercial Google Drive service.",
     },
+    mavedb: {
+        icon: faNetworkWired,
+        message: "This is a repository plugin that connects with MaveDB score set files.",
+    },
     onedrive: {
         icon: faCloud,
         message: "This is a repository plugin that connects with Microsoft OneDrive.",

lib/galaxy/dependencies/__init__.py

@@ -363,6 +363,9 @@ class ConditionalDependencies:
     def check_iiif_fsspec(self):
         return "iiif" in self.file_sources
 
+    def check_mavedb_fsspec(self):
+        return "mavedb" in self.file_sources
+
 
 def strip_comment(line):
     # lifted from https://github.com/tox-dev/tox/commit/3c6b4f204e89852c4b7536b246a66d20be6d39ec

lib/galaxy/dependencies/conditional-requirements.txt

@@ -36,6 +36,7 @@ huggingface_hub
 omero-py #type: omero
 iiif-fsspec # type: iiif
 fs-irods # type: irods, depends on python-irodsclient
+mavedb-fsspec>=0.1.0 # type: mavedb
 
 # Vault backend
 hvac

lib/galaxy/files/sources/mavedb.py

@@ -0,0 +1,125 @@
+from typing import (
+    Optional,
+    Union,
+)
+
+from galaxy.exceptions import (
+    AuthenticationRequired,
+    MessageException,
+)
+from galaxy.files.models import (
+    AnyRemoteEntry,
+    FilesSourceRuntimeContext,
+)
+from galaxy.files.sources._defaults import DEFAULT_SCHEME
+from galaxy.files.sources._fsspec import (
+    CacheOptionsDictType,
+    FsspecBaseFileSourceConfiguration,
+    FsspecBaseFileSourceTemplateConfiguration,
+    FsspecFilesSource,
+)
+from galaxy.util.config_templates import TemplateExpansion
+
+try:
+    from mavedb_fsspec import MaveDBFileSystem
+    from mavedb_fsspec.client import DEFAULT_BASE_URL
+except ImportError:
+    MaveDBFileSystem = None
+    DEFAULT_BASE_URL = "https://api.mavedb.org/api/v1"
+
+
+class MaveDBFileSourceTemplateConfiguration(FsspecBaseFileSourceTemplateConfiguration):
+    base_url: Union[str, TemplateExpansion] = DEFAULT_BASE_URL
+    api_key: Union[str, TemplateExpansion, None] = None
+    timeout: Union[float, TemplateExpansion] = 30.0
+
+
+class MaveDBFileSourceConfiguration(FsspecBaseFileSourceConfiguration):
+    base_url: str = DEFAULT_BASE_URL
+    api_key: Optional[str] = None
+    timeout: float = 30.0
+
+
+class MaveDBFilesSource(FsspecFilesSource[MaveDBFileSourceTemplateConfiguration, MaveDBFileSourceConfiguration]):
+    plugin_type = "mavedb"
+    required_module = MaveDBFileSystem
+    required_package = "mavedb-fsspec"
+
+    template_config_class = MaveDBFileSourceTemplateConfiguration
+    resolved_config_class = MaveDBFileSourceConfiguration
+
+    def _open_fs(
+        self,
+        context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration],
+        cache_options: CacheOptionsDictType,
+    ) -> "MaveDBFileSystem":
+        if MaveDBFileSystem is None:
+            raise self.required_package_exception
+
+        config = context.config
+        return MaveDBFileSystem(
+            base_url=config.base_url,
+            api_key=config.api_key,
+            timeout=config.timeout,
+            skip_instance_cache=True,
+            **cache_options,
+        )
+
+    def _list(
+        self,
+        context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration],
+        path="/",
+        recursive=False,
+        write_intent: bool = False,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        query: Optional[str] = None,
+        sort_by: Optional[str] = None,
+    ) -> tuple[list[AnyRemoteEntry], int]:
+        collection = path.strip("/")
+        if recursive or collection not in {"score-sets", "my-score-sets"}:
+            return super()._list(context, path, recursive, write_intent, limit, offset, query, sort_by)
+
+        try:
+            cache_options = self._get_cache_options(context.config)
+            fs = self._open_fs(context, cache_options)
+            try:
+                # Use the MaveDB-specific API for server-side pagination and the total result count.
+                entries, total_count = fs.list_score_sets(
+                    collection=collection,
+                    limit=limit,
+                    offset=offset,
+                    query=query,
+                )
+            finally:
+                close = getattr(fs, "close", None)
+                if close is not None:
+                    close()
+            return [self._info_to_entry(entry, context.config) for entry in entries], total_count
+        except PermissionError as e:
+            raise AuthenticationRequired(
+                f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}."
+            )
+        except Exception as e:
+            raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e
+
+    def _info_to_entry(self, info: dict, config: MaveDBFileSourceConfiguration) -> AnyRemoteEntry:
+        entry = super()._info_to_entry(info, config)
+        display_name = info.get("display_name")
+        if display_name:
+            entry.name = display_name
+        return entry
+
+    def _write_from(
+        self,
+        _target_path: str,
+        _native_path: str,
+        _context: FilesSourceRuntimeContext[MaveDBFileSourceConfiguration],
+    ):
+        raise MessageException("MaveDB file sources are read-only and do not support exporting files.")
+
+    def get_scheme(self) -> str:
+        return self.scheme if self.scheme and self.scheme != DEFAULT_SCHEME else "mavedb"
+
+
+__all__ = ("MaveDBFilesSource",)

lib/galaxy/files/templates/examples/production_mavedb.yml

@@ -0,0 +1,19 @@
+- id: mavedb
+  version: 0
+  name: MaveDB
+  description: |
+      [MaveDB](https://www.mavedb.org) provides Multiplexed Assays of Variant Effect datasets.
+      This file source exposes score set files through the public MaveDB API.
+  secrets:
+      api_key:
+          label: MaveDB API key
+          optional: true
+          help: |
+              Optional MaveDB API key. Public score set files do not require an API key.
+              A user API key can be generated in the MaveDB settings page for your account
+              and is needed to access private datasets.
+  configuration:
+      type: mavedb
+      base_url: https://api.mavedb.org/api/v1
+      api_key: "{{ secrets.api_key }}"
+      timeout: 30

lib/galaxy/files/templates/models.py

@@ -51,6 +51,7 @@ FileSourceTemplateType = Literal[
     "dataverse",
     "huggingface",
     "iiif",
+    "mavedb",
     "omero",
     "ssh",
 ]
@@ -440,6 +441,22 @@ class IIIFFileSourceConfiguration(StrictModel):
     manifest_url: str
 
 
+class MaveDBFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["mavedb"]
+    base_url: Union[str, TemplateExpansion] = "https://api.mavedb.org/api/v1"
+    api_key: Union[str, TemplateExpansion, None] = None
+    timeout: Union[float, TemplateExpansion] = 30.0
+    template_start: Optional[str] = None
+    template_end: Optional[str] = None
+
+
+class MaveDBFileSourceConfiguration(StrictModel):
+    type: Literal["mavedb"]
+    base_url: str = "https://api.mavedb.org/api/v1"
+    api_key: Optional[str] = None
+    timeout: float = 30.0
+
+
 class OmeroFileSourceTemplateConfiguration(StrictModel):
     type: Literal["omero"]
     username: Union[str, TemplateExpansion]
@@ -480,6 +497,7 @@ FileSourceTemplateConfiguration = Annotated[
         DataverseFileSourceTemplateConfiguration,
         HuggingFaceFileSourceTemplateConfiguration,
         IIIFFileSourceTemplateConfiguration,
+        MaveDBFileSourceTemplateConfiguration,
         OmeroFileSourceTemplateConfiguration,
         SshFileSourceTemplateConfiguration,
     ],
@@ -506,6 +524,7 @@ FileSourceConfiguration = Annotated[
         DataverseFileSourceConfiguration,
         HuggingFaceFileSourceConfiguration,
         IIIFFileSourceConfiguration,
+        MaveDBFileSourceConfiguration,
         OmeroFileSourceConfiguration,
         SshFileSourceConfiguration,
     ],
@@ -590,6 +609,7 @@ TypesToConfigurationClasses: dict[FileSourceTemplateType, type[FileSourceConfigu
     "dataverse": DataverseFileSourceConfiguration,
     "huggingface": HuggingFaceFileSourceConfiguration,
     "iiif": IIIFFileSourceConfiguration,
+    "mavedb": MaveDBFileSourceConfiguration,
     "omero": OmeroFileSourceConfiguration,
     "ssh": SshFileSourceConfiguration,
 }