Merge pull request #22816 from davelopez/26.1_backport_irods_file_source

[26.1] Backport of iRODS file source

client/packages/api-client/src/schema/schema.ts

@@ -13044,6 +13044,7 @@ export interface components {
                 | "s3fs"
                 | "azure"
                 | "azureflat"
+                | "irods"
                 | "onedata"
                 | "webdav"
                 | "dropbox"
@@ -25350,6 +25351,7 @@ export interface components {
                 | "s3fs"
                 | "azure"
                 | "azureflat"
+                | "irods"
                 | "onedata"
                 | "webdav"
                 | "dropbox"

client/src/api/fileSources.ts

@@ -91,6 +91,10 @@ export const templateTypes: FileSourceTypesDetail = {
         icon: faNetworkWired,
         message: "This is a file repository plugin that connects with a remote server over SSH.",
     },
+    irods: {
+        icon: faNetworkWired,
+        message: "This is a file repository plugin that connects with an iRODS server.",
+    },
 };
 
 export const FileSourcesValidFilters = {

lib/galaxy/dependencies/__init__.py

@@ -291,6 +291,9 @@ class ConditionalDependencies:
     def check_rspace_client(self):
         return "rspace" in self.file_sources
 
+    def check_fs_irods(self):
+        return "irods" in self.file_sources
+
     def check_watchdog(self):
         install_set = {"auto", "True", "true", "polling", True}
         return self.config["watch_tools"] in install_set or self.config["watch_tool_data_dir"] in install_set

lib/galaxy/dependencies/conditional-requirements.txt

@@ -35,6 +35,7 @@ adlfs
 huggingface_hub
 omero-py #type: omero
 iiif-fsspec # type: iiif
+fs-irods # type: irods, depends on python-irodsclient
 
 # Vault backend
 hvac

lib/galaxy/files/sources/irods.py

@@ -0,0 +1,165 @@
+import os
+from fnmatch import fnmatch
+from typing import (
+    Optional,
+    Union,
+)
+
+import fs
+import fs.errors
+
+from galaxy.exceptions import (
+    AuthenticationRequired,
+    MessageException,
+)
+from galaxy.files.models import (
+    AnyRemoteEntry,
+    BaseFileSourceConfiguration,
+    BaseFileSourceTemplateConfiguration,
+    FilesSourceRuntimeContext,
+)
+from galaxy.util.config_templates import TemplateExpansion
+from ._pyfilesystem2 import PyFilesystem2FilesSource
+
+try:
+    from fs_irods import iRODSFS
+    from irods.session import iRODSSession
+except ImportError:
+    iRODSFS = None
+    iRODSSession = None
+
+
+class IrodsFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
+    host: Union[str, TemplateExpansion]
+    port: Union[int, TemplateExpansion] = 1247
+    username: Union[str, TemplateExpansion]
+    password: Union[str, TemplateExpansion]
+    zone: Union[str, TemplateExpansion]
+    root: Optional[Union[str, TemplateExpansion]] = None
+    timeout: Union[int, TemplateExpansion] = 30
+    refresh_time: Union[int, TemplateExpansion] = 300
+    client_server_negotiation: Optional[Union[str, TemplateExpansion]] = None
+    client_server_policy: Optional[Union[str, TemplateExpansion]] = None
+    encryption_algorithm: Optional[Union[str, TemplateExpansion]] = None
+    encryption_key_size: Optional[Union[int, TemplateExpansion]] = None
+    encryption_num_hash_rounds: Optional[Union[int, TemplateExpansion]] = None
+    encryption_salt_size: Optional[Union[int, TemplateExpansion]] = None
+    ssl_verify_server: Optional[Union[str, TemplateExpansion]] = None
+    ssl_ca_certificate_file: Optional[Union[str, TemplateExpansion]] = None
+    resource: Optional[Union[str, TemplateExpansion]] = None
+
+
+class IrodsFileSourceConfiguration(BaseFileSourceConfiguration):
+    host: str
+    port: int = 1247
+    username: str
+    password: str
+    zone: str
+    root: Optional[str] = None
+    timeout: int = 30
+    refresh_time: int = 300
+    client_server_negotiation: Optional[str] = None
+    client_server_policy: Optional[str] = None
+    encryption_algorithm: Optional[str] = None
+    encryption_key_size: Optional[int] = None
+    encryption_num_hash_rounds: Optional[int] = None
+    encryption_salt_size: Optional[int] = None
+    ssl_verify_server: Optional[str] = None
+    ssl_ca_certificate_file: Optional[str] = None
+    resource: Optional[str] = None
+
+
+class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfiguration, IrodsFileSourceConfiguration]):
+    plugin_type = "irods"
+    required_module = iRODSFS
+    required_package = "fs-irods"
+
+    template_config_class = IrodsFileSourceTemplateConfiguration
+    resolved_config_class = IrodsFileSourceConfiguration
+
+    def _iter_directory_entries(self, fs_handle, parent_path: str, normalized_query: Optional[str] = None):
+        for raw_name in fs_handle.listdir(parent_path):
+            name = os.path.basename(str(raw_name).rstrip("/"))
+            if not name:
+                continue
+            if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"):
+                continue
+            entry_path = fs.path.join(parent_path, name)
+            info = fs_handle.getinfo(entry_path, namespaces=["details"])
+            yield entry_path, info
+
+    def _list_non_recursive(
+        self,
+        fs_handle,
+        path: str,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        query: Optional[str] = None,
+    ) -> tuple[list[AnyRemoteEntry], int]:
+        normalized_query = query.lower() if query else None
+        entries = []
+        for _, info in self._iter_directory_entries(fs_handle, path, normalized_query):
+            entries.append(self._resource_info_to_dict(path, info))
+        count = len(entries)
+        page = self._to_page(limit, offset)
+        if page is not None:
+            entries = entries[page[0] : page[1]]
+        return entries, count
+
+    def _list(
+        self,
+        context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration],
+        path="/",
+        recursive=False,
+        write_intent: bool = False,
+        limit: Optional[int] = None,
+        offset: Optional[int] = None,
+        query: Optional[str] = None,
+        sort_by: Optional[str] = None,
+    ) -> tuple[list[AnyRemoteEntry], int]:
+        try:
+            with self._open_fs(context) as fs_handle:
+                if recursive:
+                    raise MessageException("Recursive listing is not supported for iRODS file sources.")
+                return self._list_non_recursive(fs_handle, path, limit, offset, query)
+        except fs.errors.PermissionDenied as e:
+            raise AuthenticationRequired(
+                f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}."
+            ) from e
+        except fs.errors.FSError as e:
+            raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e
+
+    def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration]):
+        if iRODSFS is None or iRODSSession is None:
+            raise self.required_package_exception
+
+        config = context.config
+        session_kwargs = {
+            "host": config.host,
+            "port": config.port,
+            "user": config.username,
+            "password": config.password,
+            "zone": config.zone,
+            "refresh_time": config.refresh_time,
+            "client_server_negotiation": config.client_server_negotiation,
+            "client_server_policy": config.client_server_policy,
+            "encryption_algorithm": config.encryption_algorithm,
+            "encryption_key_size": config.encryption_key_size,
+            "encryption_num_hash_rounds": config.encryption_num_hash_rounds,
+            "encryption_salt_size": config.encryption_salt_size,
+            "ssl_verify_server": config.ssl_verify_server,
+            "ssl_ca_certificate_file": config.ssl_ca_certificate_file,
+        }
+        ssl_context = getattr(config, "ssl_context", None)
+        if ssl_context is not None:
+            session_kwargs["ssl_context"] = ssl_context
+
+        session = iRODSSession(**session_kwargs)
+        session.connection_timeout = config.timeout
+        if config.resource:
+            session.default_resource = config.resource
+
+        return iRODSFS(session=session, root=config.root)
+
+
+__all__ = ("IrodsFilesSource",)

lib/galaxy/files/templates/examples/irods.yml

@@ -0,0 +1,52 @@
+- id: irods
+  version: 0
+  name: iRODS
+  description: |
+    Use this template to connect an iRODS collection as a Galaxy file source for
+    importing and exporting datasets.
+
+    You need valid iRODS credentials and the target root collection path.
+  variables:
+    host:
+      label: iRODS host
+      type: string
+      help: Hostname of the iRODS server.
+    port:
+      label: iRODS port
+      type: integer
+      help: Port of the iRODS server.
+      default: 1247
+    zone:
+      label: iRODS zone
+      type: string
+      help: Zone used for authentication (for example teamZone).
+    username:
+      label: iRODS user
+      type: string
+      help: Username used to authenticate against iRODS.
+    subpath:
+      label: Subfolder path
+      type: string
+      optional: true
+      help: Optional subfolder under your personal directory (for example projectName).
+    writable:
+      label: Writable?
+      type: boolean
+      optional: true
+      help: Allow writing datasets back to iRODS.
+      default: false
+  secrets:
+    password:
+      label: Password
+      help: Password used to authenticate against iRODS.
+  configuration:
+    type: irods
+    host: "{{ variables.host }}"
+    port: "{{ variables.port }}"
+    username: "{{ variables.username }}"
+    password: "{{ secrets.password }}"
+    zone: "{{ variables.zone }}"
+    root: "/{{ variables.zone }}/home/{{ variables.username }}{% if variables.get('subpath') %}/{{ variables.get('subpath') | ensure_path_component }}{% endif %}"
+    timeout: 30
+    refresh_time: 300
+    writable: "{{ variables.writable }}"

lib/galaxy/files/templates/models.py

@@ -38,6 +38,7 @@ FileSourceTemplateType = Literal[
     "s3fs",
     "azure",
     "azureflat",
+    "irods",
     "onedata",
     "webdav",
     "dropbox",
@@ -248,6 +249,34 @@ class AzureFlatFileSourceConfiguration(StrictModel):
     writable: bool = False
 
 
+class IrodsFileSourceTemplateConfiguration(StrictModel):
+    type: Literal["irods"]
+    host: Union[str, TemplateExpansion]
+    port: Union[int, TemplateExpansion] = 1247
+    username: Union[str, TemplateExpansion]
+    password: Union[str, TemplateExpansion]
+    zone: Union[str, TemplateExpansion]
+    root: Optional[Union[str, TemplateExpansion]] = None
+    timeout: Union[int, TemplateExpansion] = 30
+    refresh_time: Union[int, TemplateExpansion] = 300
+    writable: Union[bool, TemplateExpansion] = False
+    template_start: Optional[str] = None
+    template_end: Optional[str] = None
+
+
+class IrodsFileSourceConfiguration(StrictModel):
+    type: Literal["irods"]
+    host: str
+    port: int = 1247
+    username: str
+    password: str
+    zone: str
+    root: Optional[str] = None
+    timeout: int = 30
+    refresh_time: int = 300
+    writable: bool = False
+
+
 class OnedataFileSourceTemplateConfiguration(StrictModel):
     type: Literal["onedata"]
     access_token: Union[str, TemplateExpansion]
@@ -438,6 +467,7 @@ FileSourceTemplateConfiguration = Annotated[
         FtpFileSourceTemplateConfiguration,
         AzureFileSourceTemplateConfiguration,
         AzureFlatFileSourceTemplateConfiguration,
+        IrodsFileSourceTemplateConfiguration,
         OnedataFileSourceTemplateConfiguration,
         WebdavFileSourceTemplateConfiguration,
         DropboxFileSourceTemplateConfiguration,
@@ -463,6 +493,7 @@ FileSourceConfiguration = Annotated[
         FtpFileSourceConfiguration,
         AzureFileSourceConfiguration,
         AzureFlatFileSourceConfiguration,
+        IrodsFileSourceConfiguration,
         OnedataFileSourceConfiguration,
         WebdavFileSourceConfiguration,
         DropboxFileSourceConfiguration,
@@ -546,6 +577,7 @@ TypesToConfigurationClasses: dict[FileSourceTemplateType, type[FileSourceConfigu
     "s3fs": S3FSFileSourceConfiguration,
     "azure": AzureFileSourceConfiguration,
     "azureflat": AzureFlatFileSourceConfiguration,
+    "irods": IrodsFileSourceConfiguration,
     "onedata": OnedataFileSourceConfiguration,
     "webdav": WebdavFileSourceConfiguration,
     "dropbox": DropboxFileSourceConfiguration,

test/unit/app/dependencies/test_deps.py

@@ -25,6 +25,7 @@ FILES_SOURCES_CONFIG = """
 - type: webdav
 - type: dropbox
 - type: googledrive
+- type: irods
 """
 JOB_CONF_YAML = """
 runners:
@@ -95,6 +96,7 @@ def test_fs_configured():
         assert cds.check_gdrive_fsspec()
         assert cds.check_dropboxdrivefs()
         assert cds.check_webdav4()
+        assert cds.check_fs_irods()
 
 
 def test_yaml_jobconf_runners():

test/unit/files/test_irods.py

@@ -0,0 +1,171 @@
+import os
+import socket
+
+import pytest
+import yaml
+
+from galaxy.exceptions import MessageException
+from galaxy.files.plugins import FileSourcePluginLoader
+from galaxy.files.sources.irods import IrodsFilesSource
+from ._util import (
+    assert_realizes_contains,
+    configured_file_sources,
+    write_from,
+)
+
+try:
+    from irods.session import iRODSSession
+except ImportError:
+    iRODSSession = None
+
+
+ROUNDTRIP_TEST_FILENAME = "numerical_sort_and_write_back_to_irods_v2.tab"
+
+
+def _get_setting(irods_config: dict, env_name: str, key: str, cast=None):
+    raw = os.environ.get(env_name)
+    if raw is None:
+        raw = irods_config.get(key)
+    if raw is None:
+        pytest.skip(f"Missing iRODS setting '{key}' in file_sources_conf.yml or env {env_name}.")
+    return cast(raw) if cast else raw
+
+
+def _file_sources_config(irods_config: dict) -> dict:
+    return {
+        "host": _get_setting(irods_config, "GALAXY_TEST_IRODS_HOST", "host"),
+        "port": _get_setting(irods_config, "GALAXY_TEST_IRODS_PORT", "port", int),
+        "username": _get_setting(irods_config, "GALAXY_TEST_IRODS_USER", "username"),
+        "password": _get_setting(irods_config, "GALAXY_TEST_IRODS_PASSWORD", "password"),
+        "zone": _get_setting(irods_config, "GALAXY_TEST_IRODS_ZONE", "zone"),
+        "root": _get_setting(irods_config, "GALAXY_TEST_IRODS_ROOT", "root"),
+        "timeout": _get_setting(irods_config, "GALAXY_TEST_IRODS_TIMEOUT", "timeout", int),
+        "refresh_time": _get_setting(irods_config, "GALAXY_TEST_IRODS_REFRESH_TIME", "refresh_time", int),
+    }
+
+
+def _irods_live_settings() -> dict:
+    config_path = os.environ.get(
+        "GALAXY_TEST_FILE_SOURCES_CONFIG",
+        os.path.abspath(
+            os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "config", "file_sources_conf.yml")
+        ),
+    )
+    if not os.path.exists(config_path):
+        pytest.skip(f"No file sources config at {config_path}; set file_sources_conf.yml to enable iRODS live tests.")
+
+    with open(config_path, "rb") as handle:
+        configs = yaml.safe_load(handle) or []
+
+    irods_id = os.environ.get("GALAXY_TEST_IRODS_SOURCE_ID")
+    irods_configs = [c for c in configs if isinstance(c, dict) and c.get("type") == "irods"]
+    if irods_id:
+        irods_configs = [c for c in irods_configs if c.get("id") == irods_id]
+
+    if not irods_configs:
+        pytest.skip("No iRODS file source found in file_sources_conf.yml; configure one to enable live tests.")
+
+    irods_config = irods_configs[0]
+
+    return _file_sources_config(irods_config)
+
+
+def _skip_if_irods_unreachable(host: str, port: int):
+    try:
+        with socket.create_connection((host, port), timeout=1):
+            return
+    except OSError:
+        pytest.skip(
+            f"No reachable iRODS service at {host}:{port}. "
+            "Start your local iRODS Docker stack or override GALAXY_TEST_IRODS_* settings."
+        )
+
+
+def _live_file_source_config(settings: dict, writable: bool = False) -> list[dict]:
+    return [
+        {
+            "type": "irods",
+            "id": "test1",
+            "label": "iRODS Live Test",
+            "doc": "Live iRODS connectivity smoke test",
+            "host": settings["host"],
+            "port": settings["port"],
+            "username": settings["username"],
+            "password": settings["password"],
+            "zone": settings["zone"],
+            "root": settings["root"],
+            "timeout": settings["timeout"],
+            "refresh_time": settings["refresh_time"],
+            "writable": writable,
+        }
+    ]
+
+
+def _cleanup_live_test_artifacts(settings: dict):
+    root = settings["root"].rstrip("/")
+    logical_path = f"{root}/{ROUNDTRIP_TEST_FILENAME}"
+
+    session = iRODSSession(
+        host=settings["host"],
+        port=settings["port"],
+        user=settings["username"],
+        password=settings["password"],
+        zone=settings["zone"],
+        refresh_time=settings["refresh_time"],
+    )
+    session.connection_timeout = settings["timeout"]
+
+    try:
+        if session.data_objects.exists(logical_path):
+            session.data_objects.unlink(logical_path)
+    finally:
+        session.cleanup()
+
+
+def test_irods_plugin_registered():
+    plugin_loader = FileSourcePluginLoader()
+    plugin_class = plugin_loader.get_plugin_type_class("irods")
+    assert plugin_class is IrodsFilesSource
+
+
+def test_irods_live_touch():
+    settings = _irods_live_settings()
+    _skip_if_irods_unreachable(settings["host"], settings["port"])
+    _cleanup_live_test_artifacts(settings)
+
+    file_sources = configured_file_sources(_live_file_source_config(settings, writable=False))
+    file_source_pair = file_sources.get_file_source_path("gxfiles://test1")
+
+    assert file_source_pair.path == "/"
+    entries, count = file_source_pair.file_source.list("/", recursive=False)
+    assert isinstance(entries, list)
+    assert count >= 0
+    _cleanup_live_test_artifacts(settings)
+
+
+def test_irods_live_recursive_list():
+    settings = _irods_live_settings()
+    _skip_if_irods_unreachable(settings["host"], settings["port"])
+    _cleanup_live_test_artifacts(settings)
+
+    file_sources = configured_file_sources(_live_file_source_config(settings, writable=False))
+    file_source_pair = file_sources.get_file_source_path("gxfiles://test1")
+
+    with pytest.raises(MessageException):
+        file_source_pair.file_source.list("/", recursive=True)
+    _cleanup_live_test_artifacts(settings)
+
+
+def test_irods_live_write_and_read_roundtrip():
+    settings = _irods_live_settings()
+    _skip_if_irods_unreachable(settings["host"], settings["port"])
+    _cleanup_live_test_artifacts(settings)
+
+    test_contents = "1\t2\t999\n666\t6\t555\n3\t4\t5\n"
+    target_uri = f"gxfiles://test1/{ROUNDTRIP_TEST_FILENAME}"
+
+    file_sources = configured_file_sources(_live_file_source_config(settings, writable=True))
+    _ = write_from(file_sources, target_uri, test_contents)
+    assert_realizes_contains(file_sources, target_uri, test_contents)
+
+    _cleanup_live_test_artifacts(settings)