From 534161268734ec9ef966d3103ce3d60da4762dcc Mon Sep 17 00:00:00 2001 From: PlushZ Date: Thu, 2 Apr 2026 18:25:08 +1100 Subject: [PATCH] Add OneDrive file source --- doc/source/admin/data.md | 48 +++++ lib/galaxy/files/sources/onedrive.py | 14 ++ .../app/managers/test_user_file_sources.py | 111 ++++++++++- test/unit/files/test_onedrive.py | 177 ++++++++++++++++++ 4 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 test/unit/files/test_onedrive.py diff --git a/doc/source/admin/data.md b/doc/source/admin/data.md index fe973dcf75f..ae2449274c4 100644 --- a/doc/source/admin/data.md +++ b/doc/source/admin/data.md @@ -626,6 +626,54 @@ a production Galaxy instance but Dropbox operates on a different scale. For more information on what Dropbox considers a "development" app versus a "production" app - checkout the [Dropbox documentation](https://www.dropbox.com/developers/reference/developer-guide#production-approval). +#### OneDrive + +Once you have OAuth 2.0 client credentials from Microsoft Entra (called `oauth2_client_id` +and `oauth2_client_secret` here), the following configuration can be used to enable +OneDrive for your Galaxy instance. + +```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_onedrive.yml +:language: yaml +``` + +To use this template, make the credentials available to Galaxy's web and job handler +processes using the environment variables `GALAXY_ONEDRIVE_CLIENT_ID` and +`GALAXY_ONEDRIVE_CLIENT_SECRET`. Jobs themselves do not need these values and should +not receive them. + +The current OneDrive implementation targets Microsoft Graph special/approot and uses the +`Files.ReadWrite.AppFolder` delegated scope. This means Galaxy can browse, download, +upload, and create folders inside the application's dedicated OneDrive app folder +(`Apps/`). Supporting full-drive access would require code changes +in addition to broader scopes such as `Files.ReadWrite` or `Files.ReadWrite.All` and +configuring yml template with `oauth2_scope: "offline_access Files.ReadWrite.All"` + +To configure Microsoft Entra for this file source: + +1. Create an app registration for your Galaxy instance in Microsoft Entra. +2. Add a web redirect URI pointing to your Galaxy instance with `oauth2_callback` + appended to the root URL. For local development, this is typically + `http://localhost:8080/oauth2_callback`. +3. Configure supported account types for the accounts you intend to support. If you + want to support both work/school accounts and personal Microsoft accounts, set + the audience to "Any Entra ID tenant + Personal Microsoft accounts". +4. Add delegated Microsoft Graph permissions: `Files.ReadWrite.AppFolder`, + `offline_access` (to ensure Galaxy can obtain refresh tokens for long-lived access). +5. Create a client secret and provide its value to Galaxy via + `GALAXY_ONEDRIVE_CLIENT_SECRET` (remember, this value can be seen only once after creation). +6. Go to Overview, find "Application (client) ID" and provide its value to Galaxy + via `GALAXY_ONEDRIVE_CLIENT_ID` + +The OAuth2 endpoints Galaxy uses for this template are the Microsoft identity platform's +v2 endpoints under the `common` tenant. If you register an application that also supports +personal Microsoft accounts, ensure the manifest is consistent with that audience. In +particular, the app manifest should use `AzureADandPersonalMicrosoftAccount` as the +`signInAudience` and `2` as the `requestedAccessTokenVersion`. + +This first implementation currently uses Microsoft Graph's simple upload endpoint and +does not yet implement resumable uploads for very large files, server-side pagination, +or server-side search/sorting. + ## Playing Nicer with Ansible Many large instances of Galaxy are configured with Ansible and much of the existing administrator diff --git a/lib/galaxy/files/sources/onedrive.py b/lib/galaxy/files/sources/onedrive.py index 05076548504..db1da53ad88 100644 --- a/lib/galaxy/files/sources/onedrive.py +++ b/lib/galaxy/files/sources/onedrive.py @@ -81,7 +81,14 @@ class OneDriveFilesSource( return f"{api_base}/special/approot" def _children_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str: +<<<<<<< HEAD return f"{self._item_url(config, path)}/children" +======= + item_url = self._item_url(config, path) + if path.strip("/"): + return f"{item_url}:/children" + return f"{item_url}/children" +>>>>>>> 4132e5d794 (Add OneDrive file source) def _content_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str: return f"{self._item_url(config, path)}:/content" if path.strip("/") else f"{self._item_url(config, path)}/content" @@ -189,7 +196,14 @@ class OneDriveFilesSource( def _create_entry( self, entry_data: EntryData, context: FilesSourceRuntimeContext[OneDriveFilesSourceConfiguration] ) -> Entry: +<<<<<<< HEAD parent_path = getattr(entry_data, "path", "/") +======= + parent_path = getattr(entry_data, "path", None) + if parent_path is None: + target = getattr(entry_data, "target", "/") + parent_path = self.to_relative_path(target) +>>>>>>> 4132e5d794 (Add OneDrive file source) payload = { "name": entry_data.name, "folder": {}, diff --git a/test/unit/app/managers/test_user_file_sources.py b/test/unit/app/managers/test_user_file_sources.py index c0aec084be3..83e172f5c82 100644 --- a/test/unit/app/managers/test_user_file_sources.py +++ b/test/unit/app/managers/test_user_file_sources.py @@ -23,7 +23,10 @@ from galaxy.exceptions import ( RequestParameterMissingException, ) from galaxy.files import FileSourcesUserContext -from galaxy.files.sources import dropbox +from galaxy.files.sources import ( + dropbox, + onedrive, +) from galaxy.files.templates import ConfiguredFileSourceTemplates from galaxy.files.templates.examples import get_example from galaxy.managers._config_templates import prepare_environment_from_root @@ -334,6 +337,90 @@ class TestFileSourcesTestCase(BaseTestCase): assert not status.connection.is_not_ok assert pyfilesystem_fs_init_kwd["access_token"] == "my_test_access_token" + def test_onedrive_oauth2_flow(self, tmp_path, monkeypatch): + json = { + "refresh_token": "my_test_refresh_token", + } + + def mock_get_token_from_code_raw( + code, + client_pair, + config, + redirect_uri, + ): + return MockResponse(json) + + monkeypatch.setattr(config_templates, "get_token_from_code_raw", mock_get_token_from_code_raw) + + self._init_onedrive_env(tmp_path, monkeypatch) + + authorize_url = self.manager.template_oauth2(self.trans, "onedrive", 0).authorize_url + from urllib.parse import ( + parse_qs, + urlparse, + ) + + parse_result = urlparse(authorize_url) + assert parse_result.hostname == "login.microsoftonline.com" + assert parse_result.path == "/common/oauth2/v2.0/authorize" + query_params = parse_qs(parse_result.query) + assert query_params["scope"][0] == "offline_access Files.ReadWrite.AppFolder" + assert "state" in query_params + state_param = query_params["state"] + state = OAuth2State.decode(state_param[0]) + assert state.route == "file_source_instances/onedrive/0" + redirect_url = self.manager.handle_authorization_code( + self.trans, + "moocow", + state, + ) + parse_result = urlparse(redirect_url) + query_params = parse_qs(parse_result.query) + assert "uuid" in query_params + uuid = query_params["uuid"][0] + + user_vault = self.trans.user_vault + config_secret_key = UserFileSource.vault_key_from_uuid(uuid, "_oauth2_refresh_token", None) + assert user_vault.read_secret(config_secret_key) + + def test_onedrive_oauth2_access_token_injection_during_verify(self, tmp_path, monkeypatch): + self._init_onedrive_env(tmp_path, monkeypatch) + + uuid = uuid4().hex + user_vault = self.trans.user_vault + config_secret_key = UserFileSource.vault_key_from_uuid(uuid, "_oauth2_refresh_token", None) + user_vault.write_secret(config_secret_key, "test_refresh_token") + create_payload = CreateInstancePayload( + name=SIMPLE_FILE_SOURCE_NAME, + description=SIMPLE_FILE_SOURCE_DESCRIPTION, + template_id="onedrive", + template_version=0, + variables={}, + secrets={}, + uuid=uuid, + ) + self._create_instance(create_payload) + json = { + "access_token": "my_test_access_token", + } + observed_headers = {} + + def mock_get_token_from_refresh_raw(refresh_token, client_pair, config): + return MockResponse(json) + + def mock_request(method, url, headers=None, timeout=None, **kwargs): + observed_headers.update(headers or {}) + return OneDriveMockResponse(json_data={"value": []}) + + monkeypatch.setattr(config_templates, "get_token_from_refresh_raw", mock_get_token_from_refresh_raw) + monkeypatch.setattr(onedrive.requests, "request", mock_request) + status = self.manager.plugin_status(self.trans, create_payload) + assert status.oauth2_access_token_generation + assert not status.oauth2_access_token_generation.is_not_ok + assert status.connection + assert not status.connection.is_not_ok + assert observed_headers["Authorization"] == "Bearer my_test_access_token" + def test_report_oauth2_access_token_generation_failure(self, tmp_path, monkeypatch): self._init_dropbox_env(tmp_path, monkeypatch) @@ -855,6 +942,13 @@ class TestFileSourcesTestCase(BaseTestCase): monkeypatch.setenv("GALAXY_DROPBOX_APP_CLIENT_ID", "mock_client_id") monkeypatch.setenv("GALAXY_DROPBOX_APP_CLIENT_SECRET", "mock_client_secret") + def _init_onedrive_env(self, tmp_path, monkeypatch): + self.init_user_in_database() + self._init_managers(tmp_path, safe_load(get_example("production_onedrive.yml"))) + + monkeypatch.setenv("GALAXY_ONEDRIVE_CLIENT_ID", "mock_client_id") + monkeypatch.setenv("GALAXY_ONEDRIVE_CLIENT_SECRET", "mock_client_secret") + def _create_user_file_source(self, template_id="home_directory") -> UserFileSourceModel: create_payload = CreateInstancePayload( name=SIMPLE_FILE_SOURCE_NAME, @@ -943,3 +1037,18 @@ class MockExceptionResponse: def raise_for_status(self): raise HTTPError(self._exception_msg, self._exception_msg, response=None) + + +class OneDriveMockResponse: + + def __init__(self, status_code=200, json_data=None, text=""): + self.status_code = status_code + self._json_data = json_data or {} + self.text = text + + @property + def ok(self): + return 200 <= self.status_code < 300 + + def json(self): + return self._json_data diff --git a/test/unit/files/test_onedrive.py b/test/unit/files/test_onedrive.py new file mode 100644 index 00000000000..9aad56196f6 --- /dev/null +++ b/test/unit/files/test_onedrive.py @@ -0,0 +1,177 @@ +from pathlib import Path + +import pytest + +from galaxy.exceptions import ( + AuthenticationRequired, + MessageException, +) +from galaxy.files.models import ( + EntryData, + FileSourcePluginsConfig, +) +from galaxy.files.sources.onedrive import OneDriveFilesSource + + +class MockResponse: + def __init__(self, status_code=200, json_data=None, text="", content_chunks=None): + self.status_code = status_code + self._json_data = json_data or {} + self.text = text + self._content_chunks = content_chunks or [] + + @property + def ok(self): + return 200 <= self.status_code < 300 + + def json(self): + return self._json_data + + def iter_content(self, chunk_size=1024 * 1024): + yield from self._content_chunks + + +def _plugin(): + template = OneDriveFilesSource.build_template_config( + id="test1", + type="onedrive", + label="OneDrive", + doc="Test OneDrive file source", + writable=True, + access_token="test_access_token", + file_sources_config=FileSourcePluginsConfig(), + ) + return OneDriveFilesSource(template) + + +def test_list_root(monkeypatch): + plugin = _plugin() + observed = {} + + def mock_request(method, url, headers=None, timeout=None, **kwargs): + observed["method"] = method + observed["url"] = url + observed["headers"] = headers + return MockResponse( + json_data={ + "value": [ + {"name": "subdir", "folder": {}, "size": 0}, + {"name": "a.txt", "size": 12, "lastModifiedDateTime": "2026-04-05T12:00:00Z"}, + ] + } + ) + + monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request) + + entries, count = plugin.list("/") + + assert count == 2 + assert observed["method"] == "GET" + assert observed["url"] == "https://graph.microsoft.com/v1.0/me/drive/special/approot/children" + assert observed["headers"]["Authorization"] == "Bearer test_access_token" + assert entries[0].path == "/subdir" + assert entries[0].uri == "gxfiles://test1/subdir" + assert entries[1].path == "/a.txt" + assert entries[1].uri == "gxfiles://test1/a.txt" + + +def test_list_nested_folder(monkeypatch): + plugin = _plugin() + observed = {} + + def mock_request(method, url, headers=None, timeout=None, **kwargs): + observed["url"] = url + return MockResponse(json_data={"value": [{"name": "b.txt", "size": 4}]}) + + monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request) + + entries, count = plugin.list("/level1/level 2") + + assert count == 1 + assert observed["url"] == "https://graph.microsoft.com/v1.0/me/drive/special/approot:/level1/level%202:/children" + assert entries[0].path == "/level1/level 2/b.txt" + + +def test_realize_to_downloads_content(monkeypatch, tmp_path: Path): + plugin = _plugin() + target = tmp_path / "downloaded.txt" + + def mock_request(method, url, headers=None, timeout=None, stream=None, **kwargs): + assert method == "GET" + assert stream is True + assert url == "https://graph.microsoft.com/v1.0/me/drive/special/approot:/dir/file.txt:/content" + return MockResponse(content_chunks=[b"hello ", b"world"]) + + monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request) + + plugin.realize_to("/dir/file.txt", str(target)) + + assert target.read_text("utf-8") == "hello world" + + +def test_write_from_uploads_content(monkeypatch, tmp_path: Path): + plugin = _plugin() + source = tmp_path / "upload.txt" + source.write_text("payload", "utf-8") + observed = {} + + def mock_put(url, headers=None, data=None, timeout=None): + observed["url"] = url + observed["headers"] = headers + observed["data"] = data.read() + return MockResponse(json_data={"id": "item1"}) + + monkeypatch.setattr("galaxy.files.sources.onedrive.requests.put", mock_put) + + actual_uri = plugin.write_from("/nested/upload.txt", str(source)) + + assert actual_uri == "gxfiles://test1/nested/upload.txt" + assert observed["url"] == "https://graph.microsoft.com/v1.0/me/drive/special/approot:/nested/upload.txt:/content" + assert observed["headers"]["Authorization"] == "Bearer test_access_token" + assert observed["data"] == b"payload" + + +def test_create_entry_creates_directory(monkeypatch): + plugin = _plugin() + observed = {} + + def mock_post(url, json=None, headers=None, timeout=None): + observed["url"] = url + observed["json"] = json + observed["headers"] = headers + return MockResponse(json_data={"name": "newdir", "webUrl": "https://example.org/newdir"}) + + monkeypatch.setattr("galaxy.files.sources.onedrive.requests.post", mock_post) + + entry = plugin.create_entry(EntryData(name="newdir", target="gxfiles://test1/parent")) + + assert observed["url"] == "https://graph.microsoft.com/v1.0/me/drive/special/approot:/parent:/children" + assert observed["json"]["name"] == "newdir" + assert entry.uri == "gxfiles://test1/parent/newdir" + assert entry.external_link == "https://example.org/newdir" + + +def test_list_authentication_error(monkeypatch): + plugin = _plugin() + + def mock_request(method, url, headers=None, timeout=None, **kwargs): + return MockResponse(status_code=401, json_data={"error": {"message": "Unauthorized"}}) + + monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request) + + with pytest.raises(AuthenticationRequired): + plugin.list("/") + + +def test_write_reports_api_error(monkeypatch, tmp_path: Path): + plugin = _plugin() + source = tmp_path / "upload.txt" + source.write_text("payload", "utf-8") + + def mock_put(url, headers=None, data=None, timeout=None): + return MockResponse(status_code=400, json_data={"error": {"message": "Bad request"}}) + + monkeypatch.setattr("galaxy.files.sources.onedrive.requests.put", mock_put) + + with pytest.raises(MessageException, match="Bad request"): + plugin.write_from("/upload.txt", str(source))