diff --git a/client/packages/api-client/src/schema/schema.ts b/client/packages/api-client/src/schema/schema.ts index 1ec8b2f8c34..ddc2fb8bb09 100644 --- a/client/packages/api-client/src/schema/schema.ts +++ b/client/packages/api-client/src/schema/schema.ts @@ -13044,6 +13044,7 @@ export interface components { | "s3fs" | "azure" | "azureflat" + | "irods" | "onedata" | "webdav" | "dropbox" @@ -25350,6 +25351,7 @@ export interface components { | "s3fs" | "azure" | "azureflat" + | "irods" | "onedata" | "webdav" | "dropbox" diff --git a/client/src/api/fileSources.ts b/client/src/api/fileSources.ts index e7121c3a270..008775f999b 100644 --- a/client/src/api/fileSources.ts +++ b/client/src/api/fileSources.ts @@ -91,6 +91,10 @@ export const templateTypes: FileSourceTypesDetail = { icon: faNetworkWired, message: "This is a file repository plugin that connects with a remote server over SSH.", }, + irods: { + icon: faNetworkWired, + message: "This is a file repository plugin that connects with an iRODS server.", + }, }; export const FileSourcesValidFilters = { diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 7153f44105f..b28bf8af973 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -291,6 +291,9 @@ class ConditionalDependencies: def check_rspace_client(self): return "rspace" in self.file_sources + def check_fs_irods(self): + return "irods" in self.file_sources + def check_watchdog(self): install_set = {"auto", "True", "true", "polling", True} return self.config["watch_tools"] in install_set or self.config["watch_tool_data_dir"] in install_set diff --git a/lib/galaxy/dependencies/conditional-requirements.txt b/lib/galaxy/dependencies/conditional-requirements.txt index 8d84205bfd2..2c45fb5ba80 100644 --- a/lib/galaxy/dependencies/conditional-requirements.txt +++ b/lib/galaxy/dependencies/conditional-requirements.txt @@ -35,6 +35,7 @@ adlfs huggingface_hub omero-py #type: omero iiif-fsspec # type: iiif +fs-irods # type: irods, depends on python-irodsclient # Vault backend hvac diff --git a/lib/galaxy/files/sources/irods.py b/lib/galaxy/files/sources/irods.py new file mode 100644 index 00000000000..78d649284db --- /dev/null +++ b/lib/galaxy/files/sources/irods.py @@ -0,0 +1,165 @@ +import os +from fnmatch import fnmatch +from typing import ( + Optional, + Union, +) + +import fs +import fs.errors + +from galaxy.exceptions import ( + AuthenticationRequired, + MessageException, +) +from galaxy.files.models import ( + AnyRemoteEntry, + BaseFileSourceConfiguration, + BaseFileSourceTemplateConfiguration, + FilesSourceRuntimeContext, +) +from galaxy.util.config_templates import TemplateExpansion +from ._pyfilesystem2 import PyFilesystem2FilesSource + +try: + from fs_irods import iRODSFS + from irods.session import iRODSSession +except ImportError: + iRODSFS = None + iRODSSession = None + + +class IrodsFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration): + host: Union[str, TemplateExpansion] + port: Union[int, TemplateExpansion] = 1247 + username: Union[str, TemplateExpansion] + password: Union[str, TemplateExpansion] + zone: Union[str, TemplateExpansion] + root: Optional[Union[str, TemplateExpansion]] = None + timeout: Union[int, TemplateExpansion] = 30 + refresh_time: Union[int, TemplateExpansion] = 300 + client_server_negotiation: Optional[Union[str, TemplateExpansion]] = None + client_server_policy: Optional[Union[str, TemplateExpansion]] = None + encryption_algorithm: Optional[Union[str, TemplateExpansion]] = None + encryption_key_size: Optional[Union[int, TemplateExpansion]] = None + encryption_num_hash_rounds: Optional[Union[int, TemplateExpansion]] = None + encryption_salt_size: Optional[Union[int, TemplateExpansion]] = None + ssl_verify_server: Optional[Union[str, TemplateExpansion]] = None + ssl_ca_certificate_file: Optional[Union[str, TemplateExpansion]] = None + resource: Optional[Union[str, TemplateExpansion]] = None + + +class IrodsFileSourceConfiguration(BaseFileSourceConfiguration): + host: str + port: int = 1247 + username: str + password: str + zone: str + root: Optional[str] = None + timeout: int = 30 + refresh_time: int = 300 + client_server_negotiation: Optional[str] = None + client_server_policy: Optional[str] = None + encryption_algorithm: Optional[str] = None + encryption_key_size: Optional[int] = None + encryption_num_hash_rounds: Optional[int] = None + encryption_salt_size: Optional[int] = None + ssl_verify_server: Optional[str] = None + ssl_ca_certificate_file: Optional[str] = None + resource: Optional[str] = None + + +class IrodsFilesSource(PyFilesystem2FilesSource[IrodsFileSourceTemplateConfiguration, IrodsFileSourceConfiguration]): + plugin_type = "irods" + required_module = iRODSFS + required_package = "fs-irods" + + template_config_class = IrodsFileSourceTemplateConfiguration + resolved_config_class = IrodsFileSourceConfiguration + + def _iter_directory_entries(self, fs_handle, parent_path: str, normalized_query: Optional[str] = None): + for raw_name in fs_handle.listdir(parent_path): + name = os.path.basename(str(raw_name).rstrip("/")) + if not name: + continue + if normalized_query and not fnmatch(name.lower(), f"*{normalized_query}*"): + continue + entry_path = fs.path.join(parent_path, name) + info = fs_handle.getinfo(entry_path, namespaces=["details"]) + yield entry_path, info + + def _list_non_recursive( + self, + fs_handle, + path: str, + limit: Optional[int] = None, + offset: Optional[int] = None, + query: Optional[str] = None, + ) -> tuple[list[AnyRemoteEntry], int]: + normalized_query = query.lower() if query else None + entries = [] + for _, info in self._iter_directory_entries(fs_handle, path, normalized_query): + entries.append(self._resource_info_to_dict(path, info)) + count = len(entries) + page = self._to_page(limit, offset) + if page is not None: + entries = entries[page[0] : page[1]] + return entries, count + + def _list( + self, + context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration], + path="/", + recursive=False, + write_intent: bool = False, + limit: Optional[int] = None, + offset: Optional[int] = None, + query: Optional[str] = None, + sort_by: Optional[str] = None, + ) -> tuple[list[AnyRemoteEntry], int]: + try: + with self._open_fs(context) as fs_handle: + if recursive: + raise MessageException("Recursive listing is not supported for iRODS file sources.") + return self._list_non_recursive(fs_handle, path, limit, offset, query) + except fs.errors.PermissionDenied as e: + raise AuthenticationRequired( + f"Permission Denied. Reason: {e}. Please check your credentials in your preferences for {self.label}." + ) from e + except fs.errors.FSError as e: + raise MessageException(f"Problem listing file source path {path}. Reason: {e}") from e + + def _open_fs(self, context: FilesSourceRuntimeContext[IrodsFileSourceConfiguration]): + if iRODSFS is None or iRODSSession is None: + raise self.required_package_exception + + config = context.config + session_kwargs = { + "host": config.host, + "port": config.port, + "user": config.username, + "password": config.password, + "zone": config.zone, + "refresh_time": config.refresh_time, + "client_server_negotiation": config.client_server_negotiation, + "client_server_policy": config.client_server_policy, + "encryption_algorithm": config.encryption_algorithm, + "encryption_key_size": config.encryption_key_size, + "encryption_num_hash_rounds": config.encryption_num_hash_rounds, + "encryption_salt_size": config.encryption_salt_size, + "ssl_verify_server": config.ssl_verify_server, + "ssl_ca_certificate_file": config.ssl_ca_certificate_file, + } + ssl_context = getattr(config, "ssl_context", None) + if ssl_context is not None: + session_kwargs["ssl_context"] = ssl_context + + session = iRODSSession(**session_kwargs) + session.connection_timeout = config.timeout + if config.resource: + session.default_resource = config.resource + + return iRODSFS(session=session, root=config.root) + + +__all__ = ("IrodsFilesSource",) diff --git a/lib/galaxy/files/templates/examples/irods.yml b/lib/galaxy/files/templates/examples/irods.yml new file mode 100644 index 00000000000..de088c60dc1 --- /dev/null +++ b/lib/galaxy/files/templates/examples/irods.yml @@ -0,0 +1,52 @@ +- id: irods + version: 0 + name: iRODS + description: | + Use this template to connect an iRODS collection as a Galaxy file source for + importing and exporting datasets. + + You need valid iRODS credentials and the target root collection path. + variables: + host: + label: iRODS host + type: string + help: Hostname of the iRODS server. + port: + label: iRODS port + type: integer + help: Port of the iRODS server. + default: 1247 + zone: + label: iRODS zone + type: string + help: Zone used for authentication (for example teamZone). + username: + label: iRODS user + type: string + help: Username used to authenticate against iRODS. + subpath: + label: Subfolder path + type: string + optional: true + help: Optional subfolder under your personal directory (for example projectName). + writable: + label: Writable? + type: boolean + optional: true + help: Allow writing datasets back to iRODS. + default: false + secrets: + password: + label: Password + help: Password used to authenticate against iRODS. + configuration: + type: irods + host: "{{ variables.host }}" + port: "{{ variables.port }}" + username: "{{ variables.username }}" + password: "{{ secrets.password }}" + zone: "{{ variables.zone }}" + root: "/{{ variables.zone }}/home/{{ variables.username }}{% if variables.get('subpath') %}/{{ variables.get('subpath') | ensure_path_component }}{% endif %}" + timeout: 30 + refresh_time: 300 + writable: "{{ variables.writable }}" \ No newline at end of file diff --git a/lib/galaxy/files/templates/models.py b/lib/galaxy/files/templates/models.py index f3e51324ed5..e1e36e80a24 100644 --- a/lib/galaxy/files/templates/models.py +++ b/lib/galaxy/files/templates/models.py @@ -38,6 +38,7 @@ FileSourceTemplateType = Literal[ "s3fs", "azure", "azureflat", + "irods", "onedata", "webdav", "dropbox", @@ -248,6 +249,34 @@ class AzureFlatFileSourceConfiguration(StrictModel): writable: bool = False +class IrodsFileSourceTemplateConfiguration(StrictModel): + type: Literal["irods"] + host: Union[str, TemplateExpansion] + port: Union[int, TemplateExpansion] = 1247 + username: Union[str, TemplateExpansion] + password: Union[str, TemplateExpansion] + zone: Union[str, TemplateExpansion] + root: Optional[Union[str, TemplateExpansion]] = None + timeout: Union[int, TemplateExpansion] = 30 + refresh_time: Union[int, TemplateExpansion] = 300 + writable: Union[bool, TemplateExpansion] = False + template_start: Optional[str] = None + template_end: Optional[str] = None + + +class IrodsFileSourceConfiguration(StrictModel): + type: Literal["irods"] + host: str + port: int = 1247 + username: str + password: str + zone: str + root: Optional[str] = None + timeout: int = 30 + refresh_time: int = 300 + writable: bool = False + + class OnedataFileSourceTemplateConfiguration(StrictModel): type: Literal["onedata"] access_token: Union[str, TemplateExpansion] @@ -438,6 +467,7 @@ FileSourceTemplateConfiguration = Annotated[ FtpFileSourceTemplateConfiguration, AzureFileSourceTemplateConfiguration, AzureFlatFileSourceTemplateConfiguration, + IrodsFileSourceTemplateConfiguration, OnedataFileSourceTemplateConfiguration, WebdavFileSourceTemplateConfiguration, DropboxFileSourceTemplateConfiguration, @@ -463,6 +493,7 @@ FileSourceConfiguration = Annotated[ FtpFileSourceConfiguration, AzureFileSourceConfiguration, AzureFlatFileSourceConfiguration, + IrodsFileSourceConfiguration, OnedataFileSourceConfiguration, WebdavFileSourceConfiguration, DropboxFileSourceConfiguration, @@ -546,6 +577,7 @@ TypesToConfigurationClasses: dict[FileSourceTemplateType, type[FileSourceConfigu "s3fs": S3FSFileSourceConfiguration, "azure": AzureFileSourceConfiguration, "azureflat": AzureFlatFileSourceConfiguration, + "irods": IrodsFileSourceConfiguration, "onedata": OnedataFileSourceConfiguration, "webdav": WebdavFileSourceConfiguration, "dropbox": DropboxFileSourceConfiguration, diff --git a/test/unit/app/dependencies/test_deps.py b/test/unit/app/dependencies/test_deps.py index 7e2f8efa03a..f36e1b929f0 100644 --- a/test/unit/app/dependencies/test_deps.py +++ b/test/unit/app/dependencies/test_deps.py @@ -25,6 +25,7 @@ FILES_SOURCES_CONFIG = """ - type: webdav - type: dropbox - type: googledrive +- type: irods """ JOB_CONF_YAML = """ runners: @@ -95,6 +96,7 @@ def test_fs_configured(): assert cds.check_gdrive_fsspec() assert cds.check_dropboxdrivefs() assert cds.check_webdav4() + assert cds.check_fs_irods() def test_yaml_jobconf_runners(): diff --git a/test/unit/files/test_irods.py b/test/unit/files/test_irods.py new file mode 100644 index 00000000000..8396dc108e0 --- /dev/null +++ b/test/unit/files/test_irods.py @@ -0,0 +1,171 @@ +import os +import socket + +import pytest +import yaml + +from galaxy.exceptions import MessageException +from galaxy.files.plugins import FileSourcePluginLoader +from galaxy.files.sources.irods import IrodsFilesSource +from ._util import ( + assert_realizes_contains, + configured_file_sources, + write_from, +) + +try: + from irods.session import iRODSSession +except ImportError: + iRODSSession = None + + +ROUNDTRIP_TEST_FILENAME = "numerical_sort_and_write_back_to_irods_v2.tab" + + +def _get_setting(irods_config: dict, env_name: str, key: str, cast=None): + raw = os.environ.get(env_name) + if raw is None: + raw = irods_config.get(key) + if raw is None: + pytest.skip(f"Missing iRODS setting '{key}' in file_sources_conf.yml or env {env_name}.") + return cast(raw) if cast else raw + + +def _file_sources_config(irods_config: dict) -> dict: + return { + "host": _get_setting(irods_config, "GALAXY_TEST_IRODS_HOST", "host"), + "port": _get_setting(irods_config, "GALAXY_TEST_IRODS_PORT", "port", int), + "username": _get_setting(irods_config, "GALAXY_TEST_IRODS_USER", "username"), + "password": _get_setting(irods_config, "GALAXY_TEST_IRODS_PASSWORD", "password"), + "zone": _get_setting(irods_config, "GALAXY_TEST_IRODS_ZONE", "zone"), + "root": _get_setting(irods_config, "GALAXY_TEST_IRODS_ROOT", "root"), + "timeout": _get_setting(irods_config, "GALAXY_TEST_IRODS_TIMEOUT", "timeout", int), + "refresh_time": _get_setting(irods_config, "GALAXY_TEST_IRODS_REFRESH_TIME", "refresh_time", int), + } + + +def _irods_live_settings() -> dict: + config_path = os.environ.get( + "GALAXY_TEST_FILE_SOURCES_CONFIG", + os.path.abspath( + os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, "config", "file_sources_conf.yml") + ), + ) + if not os.path.exists(config_path): + pytest.skip(f"No file sources config at {config_path}; set file_sources_conf.yml to enable iRODS live tests.") + + with open(config_path, "rb") as handle: + configs = yaml.safe_load(handle) or [] + + irods_id = os.environ.get("GALAXY_TEST_IRODS_SOURCE_ID") + irods_configs = [c for c in configs if isinstance(c, dict) and c.get("type") == "irods"] + if irods_id: + irods_configs = [c for c in irods_configs if c.get("id") == irods_id] + + if not irods_configs: + pytest.skip("No iRODS file source found in file_sources_conf.yml; configure one to enable live tests.") + + irods_config = irods_configs[0] + + return _file_sources_config(irods_config) + + +def _skip_if_irods_unreachable(host: str, port: int): + try: + with socket.create_connection((host, port), timeout=1): + return + except OSError: + pytest.skip( + f"No reachable iRODS service at {host}:{port}. " + "Start your local iRODS Docker stack or override GALAXY_TEST_IRODS_* settings." + ) + + +def _live_file_source_config(settings: dict, writable: bool = False) -> list[dict]: + return [ + { + "type": "irods", + "id": "test1", + "label": "iRODS Live Test", + "doc": "Live iRODS connectivity smoke test", + "host": settings["host"], + "port": settings["port"], + "username": settings["username"], + "password": settings["password"], + "zone": settings["zone"], + "root": settings["root"], + "timeout": settings["timeout"], + "refresh_time": settings["refresh_time"], + "writable": writable, + } + ] + + +def _cleanup_live_test_artifacts(settings: dict): + root = settings["root"].rstrip("/") + logical_path = f"{root}/{ROUNDTRIP_TEST_FILENAME}" + + session = iRODSSession( + host=settings["host"], + port=settings["port"], + user=settings["username"], + password=settings["password"], + zone=settings["zone"], + refresh_time=settings["refresh_time"], + ) + session.connection_timeout = settings["timeout"] + + try: + if session.data_objects.exists(logical_path): + session.data_objects.unlink(logical_path) + finally: + session.cleanup() + + +def test_irods_plugin_registered(): + plugin_loader = FileSourcePluginLoader() + plugin_class = plugin_loader.get_plugin_type_class("irods") + assert plugin_class is IrodsFilesSource + + +def test_irods_live_touch(): + settings = _irods_live_settings() + _skip_if_irods_unreachable(settings["host"], settings["port"]) + _cleanup_live_test_artifacts(settings) + + file_sources = configured_file_sources(_live_file_source_config(settings, writable=False)) + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + assert file_source_pair.path == "/" + entries, count = file_source_pair.file_source.list("/", recursive=False) + assert isinstance(entries, list) + assert count >= 0 + _cleanup_live_test_artifacts(settings) + + +def test_irods_live_recursive_list(): + settings = _irods_live_settings() + _skip_if_irods_unreachable(settings["host"], settings["port"]) + _cleanup_live_test_artifacts(settings) + + file_sources = configured_file_sources(_live_file_source_config(settings, writable=False)) + file_source_pair = file_sources.get_file_source_path("gxfiles://test1") + + with pytest.raises(MessageException): + file_source_pair.file_source.list("/", recursive=True) + _cleanup_live_test_artifacts(settings) + + +def test_irods_live_write_and_read_roundtrip(): + settings = _irods_live_settings() + _skip_if_irods_unreachable(settings["host"], settings["port"]) + _cleanup_live_test_artifacts(settings) + + test_contents = "1\t2\t999\n666\t6\t555\n3\t4\t5\n" + target_uri = f"gxfiles://test1/{ROUNDTRIP_TEST_FILENAME}" + + file_sources = configured_file_sources(_live_file_source_config(settings, writable=True)) + _ = write_from(file_sources, target_uri, test_contents) + assert_realizes_contains(file_sources, target_uri, test_contents) + + _cleanup_live_test_artifacts(settings)