From ce624514eb04bb66440fdc8b65685a4c5093a9c2 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Thu, 26 Mar 2026 09:27:33 +0100 Subject: [PATCH 01/17] Add iRODS file source integration with configuration and listing support --- lib/galaxy/files/sources/irods.py | 171 ++++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 lib/galaxy/files/sources/irods.py diff --git a/lib/galaxy/files/sources/irods.py b/lib/galaxy/files/sources/irods.py new file mode 100644 index 00000000000..4e36e0489b9 --- /dev/null +++ b/lib/galaxy/files/sources/irods.py @@ -0,0 +1,171 @@ +import ssl +from fnmatch import fnmatch +import os +from typing import ( + Optional, + Union, +) + +import fs +import fs.errors + +from galaxy.files.models import ( + AnyRemoteEntry, + BaseFileSourceConfiguration, + BaseFileSourceTemplateConfiguration, + FilesSourceRuntimeContext, +) +from galaxy.exceptions import ( + AuthenticationRequired, + MessageException, +) +from galaxy.util.config_templates import TemplateExpansion +from ._pyfilesystem2 import PyFilesystem2FilesSource + +try: + from fs_irods import iRODSFS +except ImportError: + iRODSFS = None + +try: + from irods.session import iRODSSession +except ImportError: + iRODSSession = None + + +class IrodsFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration): + host: Union[str, TemplateExpansion] + port: Union[int, TemplateExpansion] = 1247 + username: Union[str, TemplateExpansion] + password: Union[str, TemplateExpansion] + zone: Union[str, TemplateExpansion] + root: Optional[Union[str, TemplateExpansion]] = None + timeout: Union[int, TemplateExpansion] = 30 + refresh_time: Union[int, TemplateExpansion] = 300 + client_server_negotiation: Optional[Union[str, TemplateExpansion]] = None + client_server_policy: Optional[Union[str, TemplateExpansion]] = None + encryption_algorithm: Optional[Union[str, TemplateExpansion]] = None + encryption_key_size: Optional[Union[int, TemplateExpansion]] = None + encryption_num_hash_rounds: Optional[Union[int, TemplateExpansion]] = None + encryption_salt_size: Optional[Union[int, TemplateExpansion]] = None + ssl_verify_server: Optional[Union[str, TemplateExpansion]] = None + ssl_ca_certificate_file: Optional[Union[str, TemplateExpansion]] = None + resource: Optional[Union[str, TemplateExpansion]] = None + + +class IrodsFileSourceConfiguration(BaseFileSourceConfiguration): + host: str + port: int = 1247 + username: str + password: str + zone: str + root: Optional[str] = None + timeout: int = 30 + refresh_time: int = 300 + client_server_negotiation: Optional[str] = None + client_server_policy: Optional[str] = None + encryption_algorithm: Optional[str] = None + encryption_key_size: Optional[int] = None + encryption_num_hash_rounds: Optional[int] = None + encryption_salt_size: Optional[int] = None + ssl_verify_server: Optional[str] = None + ssl_ca_certificate_file: Optional[str] = None + resource: Optional[str] = None + + +class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfiguration, IrodsFileSourceConfiguration]): + plugin_type = "irods" + required_module = iRODSFS + required_package = "fs-irods" + + template_config_class = IrodsFileSourceTemplateConfiguration + resolved_config_class = IrodsFileSourceConfiguration + + def _normalize_listdir_name(self, value: str) -> str: + # fs-irods currently returns absolute iRODS paths from listdir; convert to simple entry names. + return os.path.basename(str(value).rstrip("/")) + + def _list( + self, + context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration], + path="/", + recursive=False, + write_intent: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None, + query: Optional[str] = None, + sort_by: Optional[str] = None, + ) -> tuple[list[AnyRemoteEntry], int]: + try: + with self._open_fs(context) as h: + if recursive: + result: list[AnyRemoteEntry] = [] + pending = [path] + while pending: + current_path = pending.pop(0) + for raw_name in h.listdir(current_path): + name = self._normalize_listdir_name(raw_name) + if not name: + continue + entry_path = fs.path.join(current_path, name) + info = h.getinfo(entry_path, namespaces=["details"]) + result.append(self._resource_info_to_dict(current_path, info)) + if info.is_dir: + pending.append(entry_path) + return result, len(result) + + normalized_query = query.lower() if query else None + entries = [] + for raw_name in h.listdir(path): + name = self._normalize_listdir_name(raw_name) + if not name: + continue + if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"): + continue + entry_path = fs.path.join(path, name) + info = h.getinfo(entry_path, namespaces=["details"]) + entries.append(self._resource_info_to_dict(path, info)) + + count = len(entries) + page = self._to_page(limit, offset) + if page is not None: + entries = entries[page[0] : page[1]] + return entries, count + except fs.errors.PermissionDenied as e: + raise AuthenticationRequired( + f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}." + ) from e + except fs.errors.FSError as e: + raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e + + def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration]): + if iRODSFS is None or iRODSSession is None: + raise self.required_package_exception + + config = context.config + session_kwargs = { + "host": config.host, + "port": config.port, + "user": config.username, + "password": config.password, + "zone": config.zone, + "refresh_time": config.refresh_time, + "client_server_negotiation": config.client_server_negotiation, + "client_server_policy": config.client_server_policy, + "encryption_algorithm": config.encryption_algorithm, + "encryption_key_size": config.encryption_key_size, + "encryption_num_hash_rounds": config.encryption_num_hash_rounds, + "encryption_salt_size": config.encryption_salt_size, + "ssl_verify_server": config.ssl_verify_server, + "ssl_ca_certificate_file": config.ssl_ca_certificate_file, + "ssl_context": ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH), + } + session = iRODSSession(**session_kwargs) + session.connection_timeout = config.timeout + if config.resource: + session.default_resource = config.resource + + return iRODSFS(session=session, root=config.root) + + +__all__ = ("IrodsFilesSource",) From 995cd8cc1b757a967872f91bf92be4a6815ab648 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Thu, 26 Mar 2026 09:32:10 +0100 Subject: [PATCH 02/17] Add iRODS file source configuration and template classes --- lib/galaxy/files/templates/models.py | 32 ++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py index f3e51324ed5..e1e36e80a24 100644 --- a/lib/galaxy/files/templates/models.py +++ b/lib/galaxy/files/templates/models.py @@ -38,6 +38,7 @@ FileSourceTemplateType = Literal[ "s3fs", "azure", "azureflat", + "irods", "onedata", "webdav", "dropbox", @@ -248,6 +249,34 @@ class AzureFlatFileSourceConfiguration(StrictModel): writable: bool = False +class IrodsFileSourceTemplateConfiguration(StrictModel): + type: Literal["irods"] + host: Union[str, TemplateExpansion] + port: Union[int, TemplateExpansion] = 1247 + username: Union[str, TemplateExpansion] + password: Union[str, TemplateExpansion] + zone: Union[str, TemplateExpansion] + root: Optional[Union[str, TemplateExpansion]] = None + timeout: Union[int, TemplateExpansion] = 30 + refresh_time: Union[int, TemplateExpansion] = 300 + writable: Union[bool, TemplateExpansion] = False + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class IrodsFileSourceConfiguration(StrictModel): + type: Literal["irods"] + host: str + port: int = 1247 + username: str + password: str + zone: str + root: Optional[str] = None + timeout: int = 30 + refresh_time: int = 300 + writable: bool = False + + class OnedataFileSourceTemplateConfiguration(StrictModel): type: Literal["onedata"] access_token: Union[str, TemplateExpansion] @@ -438,6 +467,7 @@ FileSourceTemplateConfiguration = Annotated[ FtpFileSourceTemplateConfiguration, AzureFileSourceTemplateConfiguration, AzureFlatFileSourceTemplateConfiguration, + IrodsFileSourceTemplateConfiguration, OnedataFileSourceTemplateConfiguration, WebdavFileSourceTemplateConfiguration, DropboxFileSourceTemplateConfiguration, @@ -463,6 +493,7 @@ FileSourceConfiguration = Annotated[ FtpFileSourceConfiguration, AzureFileSourceConfiguration, AzureFlatFileSourceConfiguration, + IrodsFileSourceConfiguration, OnedataFileSourceConfiguration, WebdavFileSourceConfiguration, DropboxFileSourceConfiguration, @@ -546,6 +577,7 @@ TypesToConfigurationClasses: dict[FileSourceTemplateType, type[FileSourceConfigu "s3fs": S3FSFileSourceConfiguration, "azure": AzureFileSourceConfiguration, "azureflat": AzureFlatFileSourceConfiguration, + "irods": IrodsFileSourceConfiguration, "onedata": OnedataFileSourceConfiguration, "webdav": WebdavFileSourceConfiguration, "dropbox": DropboxFileSourceConfiguration, From 24f4c07fe4e878e4eb3b1365b395d52cf67c378a Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Thu, 26 Mar 2026 09:43:07 +0100 Subject: [PATCH 03/17] Add iRODS template for file source repository configuration --- lib/galaxy/files/templates/examples/irods.yml | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 lib/galaxy/files/templates/examples/irods.yml diff --git a/lib/galaxy/files/templates/examples/irods.yml b/lib/galaxy/files/templates/examples/irods.yml new file mode 100644 index 00000000000..22b168a32eb --- /dev/null +++ b/lib/galaxy/files/templates/examples/irods.yml @@ -0,0 +1,61 @@ +- id: irods + version: 0 + name: iRODS + description: | + Use this template to connect an iRODS collection as a Galaxy file source for + importing and exporting datasets. + + You need valid iRODS credentials and the target root collection path. + variables: + host: + label: iRODS host + type: string + help: Hostname of the iRODS server. + port: + label: iRODS port + type: integer + help: Port of the iRODS server. + default: 1247 + username: + label: iRODS user + type: string + help: Username used to authenticate against iRODS. + zone: + label: iRODS zone + type: string + help: Zone used for authentication (for example tempZone). + root: + label: Root collection path + type: string + help: | + Collection path exposed in Galaxy, for example /tempZone/home/rods. + timeout: + label: Connection timeout (seconds) + type: integer + help: Timeout in seconds for iRODS connection attempts. + default: 30 + refresh_time: + label: Session refresh time (seconds) + type: integer + help: Seconds between refresh checks for active iRODS sessions. + default: 300 + writable: + label: Writable? + type: boolean + help: Allow writing datasets back to iRODS. + default: false + secrets: + password: + label: Password + help: Password used to authenticate against iRODS. + configuration: + type: irods + host: "{{ variables.host }}" + port: "{{ variables.port }}" + username: "{{ variables.username }}" + password: "{{ secrets.password }}" + zone: "{{ variables.zone }}" + root: "{{ variables.root }}" + timeout: "{{ variables.timeout }}" + refresh_time: "{{ variables.refresh_time }}" + writable: "{{ variables.writable }}" \ No newline at end of file From 8f2afdf8416a70d17d34d3f473f4131785e5d693 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Thu, 26 Mar 2026 17:27:36 +0100 Subject: [PATCH 04/17] Add unit tests for iRODS file source functionality --- test/unit/files/test_irods.py | 232 ++++++++++++++++++++++++++++++++++ 1 file changed, 232 insertions(+) create mode 100644 test/unit/files/test_irods.py diff --git a/test/unit/files/test_irods.py b/test/unit/files/test_irods.py new file mode 100644 index 00000000000..b867088f6d1 --- /dev/null +++ b/test/unit/files/test_irods.py @@ -0,0 +1,232 @@ +import os +import socket +import pytest +import yaml + +from galaxy.files.models import ( + FileSourcePluginsConfig, + FilesSourceRuntimeContext, + UserData, +) +from galaxy.files.plugins import FileSourcePluginLoader +from galaxy.files.sources.irods import IrodsFilesSource +from ._util import ( + assert_realizes_contains, + configured_file_sources, + write_from, +) + +try: + from irods.session import iRODSSession +except ImportError: + iRODSSession = None + + +ROUNDTRIP_TEST_FILENAME = "numerical_sort_and_write_back_to_irods_v2.tab" + + +class _FakeSession: + init_kwargs = None + + def __init__(self, **kwargs): + type(self).init_kwargs = kwargs + self.connection_timeout = None + self.default_resource = None + + +class _FakeIrodsFs: + def __init__(self, session, root=None): + self.session = session + self.root = root + + +def _get_setting(irods_config: dict, env_name: str, key: str, cast=None): + raw = os.environ.get(env_name) + if raw is None: + raw = irods_config.get(key) + if raw is None: + pytest.skip(f"Missing iRODS setting '{key}' in file_sources_conf.yml or env {env_name}.") + return cast(raw) if cast else raw + + +def _file_sources_config(irods_config: dict) -> dict: + return { + "host": _get_setting(irods_config, "GALAXY_TEST_IRODS_HOST", "host"), + "port": _get_setting(irods_config, "GALAXY_TEST_IRODS_PORT", "port", int), + "username": _get_setting(irods_config, "GALAXY_TEST_IRODS_USER", "username"), + "password": _get_setting(irods_config, "GALAXY_TEST_IRODS_PASSWORD", "password"), + "zone": _get_setting(irods_config, "GALAXY_TEST_IRODS_ZONE", "zone"), + "root": _get_setting(irods_config, "GALAXY_TEST_IRODS_ROOT", "root"), + "timeout": _get_setting(irods_config, "GALAXY_TEST_IRODS_TIMEOUT", "timeout", int), + "refresh_time": _get_setting(irods_config, "GALAXY_TEST_IRODS_REFRESH_TIME", "refresh_time", int), + } + + +def _irods_live_settings() -> dict: + config_path = os.environ.get( + "GALAXY_TEST_FILE_SOURCES_CONFIG", + os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "config", "file_sources_conf.yml") + ), + ) + if not os.path.exists(config_path): + pytest.skip(f"No file sources config at {config_path}; set file_sources_conf.yml to enable iRODS live tests.") + + with open(config_path, "rb") as handle: + configs = yaml.safe_load(handle) or [] + + irods_id = os.environ.get("GALAXY_TEST_IRODS_SOURCE_ID") + irods_configs = [c for c in configs if isinstance(c, dict) and c.get("type") == "irods"] + if irods_id: + irods_configs = [c for c in irods_configs if c.get("id") == irods_id] + + if not irods_configs: + pytest.skip("No iRODS file source found in file_sources_conf.yml; configure one to enable live tests.") + + irods_config = irods_configs[0] + + return _file_sources_config(irods_config) + + +def _skip_if_irods_unreachable(host: str, port: int): + try: + with socket.create_connection((host, port), timeout=1): + return + except OSError: + pytest.skip( + f"No reachable iRODS service at {host}:{port}. " + "Start your local iRODS Docker stack or override GALAXY_TEST_IRODS_* settings." + ) + + +def _live_file_source_config(settings: dict, writable: bool = False) -> list[dict]: + return [ + { + "type": "irods", + "id": "test1", + "label": "iRODS Live Test", + "doc": "Live iRODS connectivity smoke test", + "host": settings["host"], + "port": settings["port"], + "username": settings["username"], + "password": settings["password"], + "zone": settings["zone"], + "root": settings["root"], + "timeout": settings["timeout"], + "refresh_time": settings["refresh_time"], + "writable": writable, + } + ] + + +def _cleanup_live_test_artifacts(settings: dict): + root = settings["root"].rstrip("/") + logical_path = f"{root}/{ROUNDTRIP_TEST_FILENAME}" + + session = iRODSSession( + host=settings["host"], + port=settings["port"], + user=settings["username"], + password=settings["password"], + zone=settings["zone"], + refresh_time=settings["refresh_time"], + ) + session.connection_timeout = settings["timeout"] + + try: + if session.data_objects.exists(logical_path): + session.data_objects.unlink(logical_path) + finally: + session.cleanup() + + +def test_irods_plugin_registered(): + plugin_loader = FileSourcePluginLoader() + plugin_class = plugin_loader.get_plugin_type_class("irods") + assert plugin_class is IrodsFilesSource + + +def test_irods_open_fs_builds_session(monkeypatch): + monkeypatch.setattr("galaxy.files.sources.irods.iRODSSession", _FakeSession) + monkeypatch.setattr("galaxy.files.sources.irods.iRODSFS", _FakeIrodsFs) + monkeypatch.setattr(IrodsFilesSource, "required_module", _FakeIrodsFs) + + file_source = IrodsFilesSource( + IrodsFilesSource.build_template_config( + type="irods", + id="test_irods", + file_sources_config=FileSourcePluginsConfig(), + host="irods.example.org", + port=1247, + username="rods", + password="secret", + zone="tempZone", + root="/tempZone/home/rods", + timeout=42, + refresh_time=120, + resource="demoResc", + writable=True, + ) + ) + + resolved_config = file_source._evaluate_template_config(UserData()) + context = FilesSourceRuntimeContext(user_data=UserData(), config=resolved_config) + + fs = file_source._open_fs(context) + init_kwargs = _FakeSession.init_kwargs + + assert isinstance(fs, _FakeIrodsFs) + assert fs.root == "/tempZone/home/rods" + assert init_kwargs is not None + assert init_kwargs["host"] == "irods.example.org" + assert init_kwargs["port"] == 1247 + assert init_kwargs["user"] == "rods" + assert init_kwargs["password"] == "secret" + assert init_kwargs["zone"] == "tempZone" + assert init_kwargs["refresh_time"] == 120 + assert fs.session.connection_timeout == 42 + assert fs.session.default_resource == "demoResc" + + +def test_irods_live_touch(): + settings = _irods_live_settings() + _skip_if_irods_unreachable(settings["host"], settings["port"]) + _cleanup_live_test_artifacts(settings) + + file_sources = configured_file_sources(_live_file_source_config(settings, writable=False)) + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + entries, count = file_source_pair.file_source.list("/", recursive=False) + assert isinstance(entries, list) + assert count >= 0 + _cleanup_live_test_artifacts(settings) + + +def test_irods_live_recursive_list(): + settings = _irods_live_settings() + _skip_if_irods_unreachable(settings["host"], settings["port"]) + _cleanup_live_test_artifacts(settings) + + file_sources = configured_file_sources(_live_file_source_config(settings, writable=False)) + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + entries, count = file_source_pair.file_source.list("/", recursive=True) + assert isinstance(entries, list) + assert count >= 0 + _cleanup_live_test_artifacts(settings) + + +def test_irods_live_write_and_read_roundtrip(): + settings = _irods_live_settings() + _skip_if_irods_unreachable(settings["host"], settings["port"]) + _cleanup_live_test_artifacts(settings) + + test_contents = "1\t2\t999\n666\t6\t555\n3\t4\t5\n" + target_uri = f"gxfiles://test1/{ROUNDTRIP_TEST_FILENAME}" + + file_sources = configured_file_sources(_live_file_source_config(settings, writable=True)) + _ = write_from(file_sources, target_uri, test_contents) + assert_realizes_contains(file_sources, target_uri, test_contents) + + _cleanup_live_test_artifacts(settings) From e2a03550dff0cae9fec46a555169c9f8348bb0b3 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Tue, 31 Mar 2026 14:52:43 +0200 Subject: [PATCH 05/17] Refactor iRODS file source listing methods for improved maintenance --- lib/galaxy/files/sources/irods.py | 78 +++++++++++++++++-------------- 1 file changed, 42 insertions(+), 36 deletions(-) diff --git a/lib/galaxy/files/sources/irods.py b/lib/galaxy/files/sources/irods.py index 4e36e0489b9..d1e2ed07f19 100644 --- a/lib/galaxy/files/sources/irods.py +++ b/lib/galaxy/files/sources/irods.py @@ -81,9 +81,45 @@ class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfigura template_config_class = IrodsFileSourceTemplateConfiguration resolved_config_class = IrodsFileSourceConfiguration - def _normalize_listdir_name(self, value: str) -> str: - # fs-irods currently returns absolute iRODS paths from listdir; convert to simple entry names. - return os.path.basename(str(value).rstrip("/")) + def _iter_directory_entries(self, fs_handle, parent_path: str, normalized_query: Optional[str] = None): + for raw_name in fs_handle.listdir(parent_path): + name = os.path.basename(str(raw_name).rstrip("/")) + if not name: + continue + if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"): + continue + entry_path = fs.path.join(parent_path, name) + info = fs_handle.getinfo(entry_path, namespaces=["details"]) + yield entry_path, info + + def _list_recursive(self, fs_handle, path: str) -> tuple[list[AnyRemoteEntry], int]: + result: list[AnyRemoteEntry] = [] + pending = [path] + while pending: + current_path = pending.pop(0) + for entry_path, info in self._iter_directory_entries(fs_handle, current_path): + result.append(self._resource_info_to_dict(current_path, info)) + if info.is_dir: + pending.append(entry_path) + return result, len(result) + + def _list_non_recursive( + self, + fs_handle, + path: str, + limit: Optional[int] = None, + offset: Optional[int] = None, + query: Optional[str] = None, + ) -> tuple[list[AnyRemoteEntry], int]: + normalized_query = query.lower() if query else None + entries = [] + for _, info in self._iter_directory_entries(fs_handle, path, normalized_query): + entries.append(self._resource_info_to_dict(path, info)) + count = len(entries) + page = self._to_page(limit, offset) + if page is not None: + entries = entries[page[0] : page[1]] + return entries, count def _list( self, @@ -97,40 +133,10 @@ class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfigura sort_by: Optional[str] = None, ) -> tuple[list[AnyRemoteEntry], int]: try: - with self._open_fs(context) as h: + with self._open_fs(context) as fs_handle: if recursive: - result: list[AnyRemoteEntry] = [] - pending = [path] - while pending: - current_path = pending.pop(0) - for raw_name in h.listdir(current_path): - name = self._normalize_listdir_name(raw_name) - if not name: - continue - entry_path = fs.path.join(current_path, name) - info = h.getinfo(entry_path, namespaces=["details"]) - result.append(self._resource_info_to_dict(current_path, info)) - if info.is_dir: - pending.append(entry_path) - return result, len(result) - - normalized_query = query.lower() if query else None - entries = [] - for raw_name in h.listdir(path): - name = self._normalize_listdir_name(raw_name) - if not name: - continue - if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"): - continue - entry_path = fs.path.join(path, name) - info = h.getinfo(entry_path, namespaces=["details"]) - entries.append(self._resource_info_to_dict(path, info)) - - count = len(entries) - page = self._to_page(limit, offset) - if page is not None: - entries = entries[page[0] : page[1]] - return entries, count + return self._list_recursive(fs_handle, path) + return self._list_non_recursive(fs_handle, path, limit, offset, query) except fs.errors.PermissionDenied as e: raise AuthenticationRequired( f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}." From c907c33a919f814e12912693fc08f34dff57d131 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Mon, 18 May 2026 15:19:03 +0200 Subject: [PATCH 06/17] Format by isort and black --- lib/galaxy/files/sources/irods.py | 10 +++++----- test/unit/files/test_irods.py | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/files/sources/irods.py b/lib/galaxy/files/sources/irods.py index d1e2ed07f19..86d28427fe0 100644 --- a/lib/galaxy/files/sources/irods.py +++ b/lib/galaxy/files/sources/irods.py @@ -1,6 +1,6 @@ +import os import ssl from fnmatch import fnmatch -import os from typing import ( Optional, Union, @@ -9,16 +9,16 @@ from typing import ( import fs import fs.errors +from galaxy.exceptions import ( + AuthenticationRequired, + MessageException, +) from galaxy.files.models import ( AnyRemoteEntry, BaseFileSourceConfiguration, BaseFileSourceTemplateConfiguration, FilesSourceRuntimeContext, ) -from galaxy.exceptions import ( - AuthenticationRequired, - MessageException, -) from galaxy.util.config_templates import TemplateExpansion from ._pyfilesystem2 import PyFilesystem2FilesSource diff --git a/test/unit/files/test_irods.py b/test/unit/files/test_irods.py index b867088f6d1..8a6eac4847a 100644 --- a/test/unit/files/test_irods.py +++ b/test/unit/files/test_irods.py @@ -1,5 +1,6 @@ import os import socket + import pytest import yaml From 54c688fd8c2bb4b7a00558d1633c65a5b38cd529 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Tue, 19 May 2026 14:27:52 +0200 Subject: [PATCH 07/17] Add iRODS file source configuration to sample YAML --- .../config/sample/file_sources_conf.yml.sample | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index 5b46c69cb07..0cd93f07fef 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -1,3 +1,19 @@ +- type: irods + id: irods_local_instance + label: Local iRODS + doc: iRODS in local Docker stack + writable: true + + host: 127.0.0.1 + port: 1247 + username: user + password: password + zone: exampleZone + root: /exampleZone/home/user + + timeout: 30 + refresh_time: 300 + - type: dropbox id: dropbox1 label: Dropbox files (configure access in user preferences) From fccf68f2e29366895eacf1adeb2e98c2e9d43b9a Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Fri, 22 May 2026 10:11:43 +0200 Subject: [PATCH 08/17] Remove the setting from the admin-defined file sources sample --- .../config/sample/file_sources_conf.yml.sample | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/lib/galaxy/config/sample/file_sources_conf.yml.sample b/lib/galaxy/config/sample/file_sources_conf.yml.sample index 0cd93f07fef..5b46c69cb07 100644 --- a/lib/galaxy/config/sample/file_sources_conf.yml.sample +++ b/lib/galaxy/config/sample/file_sources_conf.yml.sample @@ -1,19 +1,3 @@ -- type: irods - id: irods_local_instance - label: Local iRODS - doc: iRODS in local Docker stack - writable: true - - host: 127.0.0.1 - port: 1247 - username: user - password: password - zone: exampleZone - root: /exampleZone/home/user - - timeout: 30 - refresh_time: 300 - - type: dropbox id: dropbox1 label: Dropbox files (configure access in user preferences) From dc0e7b1b1f8f1d650a44d4a282385414c672edd3 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Sun, 24 May 2026 12:01:33 +0200 Subject: [PATCH 09/17] Update the way how conditional dependencies are handled --- lib/galaxy/files/sources/irods.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/galaxy/files/sources/irods.py b/lib/galaxy/files/sources/irods.py index 86d28427fe0..8a28c530216 100644 --- a/lib/galaxy/files/sources/irods.py +++ b/lib/galaxy/files/sources/irods.py @@ -24,12 +24,9 @@ from ._pyfilesystem2 import PyFilesystem2FilesSource try: from fs_irods import iRODSFS -except ImportError: - iRODSFS = None - -try: from irods.session import iRODSSession except ImportError: + iRODSFS = None iRODSSession = None From ddd7f8aa12efe0226fa37dbcd04723bde7c1d0b9 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Sun, 24 May 2026 12:04:23 +0200 Subject: [PATCH 10/17] Refactor the iRODS YAML configuration to improve optional path handling and remove the connection time setting for the user --- lib/galaxy/files/templates/examples/irods.yml | 33 +++++++------------ 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/lib/galaxy/files/templates/examples/irods.yml b/lib/galaxy/files/templates/examples/irods.yml index 22b168a32eb..de088c60dc1 100644 --- a/lib/galaxy/files/templates/examples/irods.yml +++ b/lib/galaxy/files/templates/examples/irods.yml @@ -16,32 +16,23 @@ type: integer help: Port of the iRODS server. default: 1247 + zone: + label: iRODS zone + type: string + help: Zone used for authentication (for example teamZone). username: label: iRODS user type: string help: Username used to authenticate against iRODS. - zone: - label: iRODS zone + subpath: + label: Subfolder path type: string - help: Zone used for authentication (for example tempZone). - root: - label: Root collection path - type: string - help: | - Collection path exposed in Galaxy, for example /tempZone/home/rods. - timeout: - label: Connection timeout (seconds) - type: integer - help: Timeout in seconds for iRODS connection attempts. - default: 30 - refresh_time: - label: Session refresh time (seconds) - type: integer - help: Seconds between refresh checks for active iRODS sessions. - default: 300 + optional: true + help: Optional subfolder under your personal directory (for example projectName). writable: label: Writable? type: boolean + optional: true help: Allow writing datasets back to iRODS. default: false secrets: @@ -55,7 +46,7 @@ username: "{{ variables.username }}" password: "{{ secrets.password }}" zone: "{{ variables.zone }}" - root: "{{ variables.root }}" - timeout: "{{ variables.timeout }}" - refresh_time: "{{ variables.refresh_time }}" + root: "/{{ variables.zone }}/home/{{ variables.username }}{% if variables.get('subpath') %}/{{ variables.get('subpath') | ensure_path_component }}{% endif %}" + timeout: 30 + refresh_time: 300 writable: "{{ variables.writable }}" \ No newline at end of file From 9c01e04557780e6373f03bd432591c6803c95090 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Mon, 25 May 2026 13:47:14 +0200 Subject: [PATCH 11/17] Make the SSLContext optional and align it with the SSL configuration setting in python-irodsclient --- lib/galaxy/files/sources/irods.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/files/sources/irods.py b/lib/galaxy/files/sources/irods.py index 8a28c530216..778479a89bd 100644 --- a/lib/galaxy/files/sources/irods.py +++ b/lib/galaxy/files/sources/irods.py @@ -1,5 +1,4 @@ import os -import ssl from fnmatch import fnmatch from typing import ( Optional, @@ -161,8 +160,11 @@ class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfigura "encryption_salt_size": config.encryption_salt_size, "ssl_verify_server": config.ssl_verify_server, "ssl_ca_certificate_file": config.ssl_ca_certificate_file, - "ssl_context": ssl.create_default_context(purpose=ssl.Purpose.SERVER_AUTH), } + ssl_context = getattr(config, "ssl_context", None) + if ssl_context is not None: + session_kwargs["ssl_context"] = ssl_context + session = iRODSSession(**session_kwargs) session.connection_timeout = config.timeout if config.resource: From 770913e6435e881e81a9e857d91d3d5eb7ffb69d Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Mon, 25 May 2026 16:24:16 +0200 Subject: [PATCH 12/17] Make recursive requests explicitly unsupported to prevent unbounded traversal --- lib/galaxy/files/sources/irods.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/lib/galaxy/files/sources/irods.py b/lib/galaxy/files/sources/irods.py index 778479a89bd..78d649284db 100644 --- a/lib/galaxy/files/sources/irods.py +++ b/lib/galaxy/files/sources/irods.py @@ -88,17 +88,6 @@ class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfigura info = fs_handle.getinfo(entry_path, namespaces=["details"]) yield entry_path, info - def _list_recursive(self, fs_handle, path: str) -> tuple[list[AnyRemoteEntry], int]: - result: list[AnyRemoteEntry] = [] - pending = [path] - while pending: - current_path = pending.pop(0) - for entry_path, info in self._iter_directory_entries(fs_handle, current_path): - result.append(self._resource_info_to_dict(current_path, info)) - if info.is_dir: - pending.append(entry_path) - return result, len(result) - def _list_non_recursive( self, fs_handle, @@ -131,7 +120,7 @@ class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfigura try: with self._open_fs(context) as fs_handle: if recursive: - return self._list_recursive(fs_handle, path) + raise MessageException("Recursive listing is not supported for iRODS file sources.") return self._list_non_recursive(fs_handle, path, limit, offset, query) except fs.errors.PermissionDenied as e: raise AuthenticationRequired( From 02ced4cb21b1c68646a71af4ef0f8f36c664c300 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Tue, 26 May 2026 09:41:49 +0200 Subject: [PATCH 13/17] Remove mock testing and update test to raise MessageException on recursive list --- test/unit/files/test_irods.py | 63 ++--------------------------------- 1 file changed, 3 insertions(+), 60 deletions(-) diff --git a/test/unit/files/test_irods.py b/test/unit/files/test_irods.py index 8a6eac4847a..e7b21e88562 100644 --- a/test/unit/files/test_irods.py +++ b/test/unit/files/test_irods.py @@ -4,6 +4,7 @@ import socket import pytest import yaml +from galaxy.exceptions import MessageException from galaxy.files.models import ( FileSourcePluginsConfig, FilesSourceRuntimeContext, @@ -26,21 +27,6 @@ except ImportError: ROUNDTRIP_TEST_FILENAME = "numerical_sort_and_write_back_to_irods_v2.tab" -class _FakeSession: - init_kwargs = None - - def __init__(self, **kwargs): - type(self).init_kwargs = kwargs - self.connection_timeout = None - self.default_resource = None - - -class _FakeIrodsFs: - def __init__(self, session, root=None): - self.session = session - self.root = root - - def _get_setting(irods_config: dict, env_name: str, key: str, cast=None): raw = os.environ.get(env_name) if raw is None: @@ -147,48 +133,6 @@ def test_irods_plugin_registered(): assert plugin_class is IrodsFilesSource -def test_irods_open_fs_builds_session(monkeypatch): - monkeypatch.setattr("galaxy.files.sources.irods.iRODSSession", _FakeSession) - monkeypatch.setattr("galaxy.files.sources.irods.iRODSFS", _FakeIrodsFs) - monkeypatch.setattr(IrodsFilesSource, "required_module", _FakeIrodsFs) - - file_source = IrodsFilesSource( - IrodsFilesSource.build_template_config( - type="irods", - id="test_irods", - file_sources_config=FileSourcePluginsConfig(), - host="irods.example.org", - port=1247, - username="rods", - password="secret", - zone="tempZone", - root="/tempZone/home/rods", - timeout=42, - refresh_time=120, - resource="demoResc", - writable=True, - ) - ) - - resolved_config = file_source._evaluate_template_config(UserData()) - context = FilesSourceRuntimeContext(user_data=UserData(), config=resolved_config) - - fs = file_source._open_fs(context) - init_kwargs = _FakeSession.init_kwargs - - assert isinstance(fs, _FakeIrodsFs) - assert fs.root == "/tempZone/home/rods" - assert init_kwargs is not None - assert init_kwargs["host"] == "irods.example.org" - assert init_kwargs["port"] == 1247 - assert init_kwargs["user"] == "rods" - assert init_kwargs["password"] == "secret" - assert init_kwargs["zone"] == "tempZone" - assert init_kwargs["refresh_time"] == 120 - assert fs.session.connection_timeout == 42 - assert fs.session.default_resource == "demoResc" - - def test_irods_live_touch(): settings = _irods_live_settings() _skip_if_irods_unreachable(settings["host"], settings["port"]) @@ -212,9 +156,8 @@ def test_irods_live_recursive_list(): file_sources = configured_file_sources(_live_file_source_config(settings, writable=False)) file_source_pair = file_sources.get_file_source_path("gxfiles://test1") - entries, count = file_source_pair.file_source.list("/", recursive=True) - assert isinstance(entries, list) - assert count >= 0 + with pytest.raises(MessageException): + file_source_pair.file_source.list("/", recursive=True) _cleanup_live_test_artifacts(settings) From cd3df9012f9e4deac3270b77cdf65c46a8147315 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Tue, 26 May 2026 23:29:28 +0200 Subject: [PATCH 14/17] Fix the OpenAPI linting --- client/packages/api-client/src/schema/schema.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/client/packages/api-client/src/schema/schema.ts b/client/packages/api-client/src/schema/schema.ts index 1ec8b2f8c34..ddc2fb8bb09 100644 --- a/client/packages/api-client/src/schema/schema.ts +++ b/client/packages/api-client/src/schema/schema.ts @@ -13044,6 +13044,7 @@ export interface components { | "s3fs" | "azure" | "azureflat" + | "irods" | "onedata" | "webdav" | "dropbox" @@ -25350,6 +25351,7 @@ export interface components { | "s3fs" | "azure" | "azureflat" + | "irods" | "onedata" | "webdav" | "dropbox" From a4c25be83f1cba4d29f8de961e7cef5f6cb21249 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Thu, 28 May 2026 14:03:48 +0200 Subject: [PATCH 15/17] Add check for iRODS file source and update conditional requirements --- lib/galaxy/dependencies/__init__.py | 3 +++ lib/galaxy/dependencies/conditional-requirements.txt | 1 + test/unit/app/dependencies/test_deps.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 7153f44105f..b28bf8af973 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -291,6 +291,9 @@ class ConditionalDependencies: def check_rspace_client(self): return "rspace" in self.file_sources + def check_fs_irods(self): + return "irods" in self.file_sources + def check_watchdog(self): install_set = {"auto", "True", "true", "polling", True} return self.config["watch_tools"] in install_set or self.config["watch_tool_data_dir"] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 8d84205bfd2..2c45fb5ba80 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -35,6 +35,7 @@ adlfs huggingface_hub omero-py #type: omero iiif-fsspec # type: iiif +fs-irods # type: irods, depends on python-irodsclient # Vault backend hvac diff --git a/test/unit/app/dependencies/test_deps.py b/test/unit/app/dependencies/test_deps.py index 7e2f8efa03a..f36e1b929f0 100644 --- a/test/unit/app/dependencies/test_deps.py +++ b/test/unit/app/dependencies/test_deps.py @@ -25,6 +25,7 @@ FILES_SOURCES_CONFIG = """ - type: webdav - type: dropbox - type: googledrive +- type: irods """ JOB_CONF_YAML = """ runners: @@ -95,6 +96,7 @@ def test_fs_configured(): assert cds.check_gdrive_fsspec() assert cds.check_dropboxdrivefs() assert cds.check_webdav4() + assert cds.check_fs_irods() def test_yaml_jobconf_runners(): From 8eccf2577a4b0bd39bcf4d97d04cea616e432dae Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Thu, 28 May 2026 14:10:41 +0200 Subject: [PATCH 16/17] Remove unused imports from test_irods.py --- test/unit/files/test_irods.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/test/unit/files/test_irods.py b/test/unit/files/test_irods.py index e7b21e88562..8396dc108e0 100644 --- a/test/unit/files/test_irods.py +++ b/test/unit/files/test_irods.py @@ -5,11 +5,6 @@ import pytest import yaml from galaxy.exceptions import MessageException -from galaxy.files.models import ( - FileSourcePluginsConfig, - FilesSourceRuntimeContext, - UserData, -) from galaxy.files.plugins import FileSourcePluginLoader from galaxy.files.sources.irods import IrodsFilesSource from ._util import ( From 1fd2a3fad48d8a3fa91560c8d38488110670beb1 Mon Sep 17 00:00:00 2001 From: Jenny Kuo <118450954+fairytalesbykcc@users.noreply.github.com> Date: Thu, 28 May 2026 16:12:13 +0200 Subject: [PATCH 17/17] Add iRODS file source type to templateTypes --- client/src/api/fileSources.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/client/src/api/fileSources.ts b/client/src/api/fileSources.ts index e7121c3a270..008775f999b 100644 --- a/client/src/api/fileSources.ts +++ b/client/src/api/fileSources.ts @@ -91,6 +91,10 @@ export const templateTypes: FileSourceTypesDetail = { icon: faNetworkWired, message: "This is a file repository plugin that connects with a remote server over SSH.", }, + irods: { + icon: faNetworkWired, + message: "This is a file repository plugin that connects with an iRODS server.", + }, }; export const FileSourcesValidFilters = {