Merge pull request #22367 from PlushZ/add-onedrive-filesource

Add OneDrive file source
This commit is contained in:
David López
2026-04-09 14:25:23 +02:00
committed by GitHub
13 changed files with 510 additions and 1 deletions
+4
View File
@@ -32,6 +32,10 @@ export const templateTypes: FileSourceTypesDetail = {
icon: faGoogleDrive,
message: "This is a repository plugin that connects with the commercial Google Drive service.",
},
onedrive: {
icon: faCloud,
message: "This is a repository plugin that connects with Microsoft OneDrive.",
},
onedata: {
icon: faNetworkWired,
message: "This is a repository plugin based on the Onedata service.",
+2
View File
@@ -12839,6 +12839,7 @@ export interface components {
| "webdav"
| "dropbox"
| "googledrive"
| "onedrive"
| "elabftw"
| "inveniordm"
| "zenodo"
@@ -24477,6 +24478,7 @@ export interface components {
| "webdav"
| "dropbox"
| "googledrive"
| "onedrive"
| "elabftw"
| "inveniordm"
| "zenodo"
@@ -18,7 +18,7 @@ interface Props {
uuid?: string;
}
const OAUTH2_TYPES = ["dropbox", "googledrive"];
const OAUTH2_TYPES = ["dropbox", "googledrive", "onedrive"];
const fileSourceTemplatesStore = useFileSourceTemplatesStore();
fileSourceTemplatesStore.fetchTemplates();
+82
View File
@@ -626,6 +626,88 @@ a production Galaxy instance but Dropbox operates on a different scale.
For more information on what Dropbox considers a "development" app versus a "production"
app - checkout the [Dropbox documentation](https://www.dropbox.com/developers/reference/developer-guide#production-approval).
#### OneDrive
Once you have OAuth 2.0 client credentials from Microsoft Entra (called `oauth2_client_id`
and `oauth2_client_secret` here), the following configurations can be used to enable
OneDrive for your Galaxy instance.
```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_onedrive.yml
:language: yaml
```
or
```{literalinclude} ../../../lib/galaxy/files/templates/examples/production_onedrive_full.yml
:language: yaml
```
To use one of these templates, make the credentials available to Galaxy's web and job handler
processes using the environment variables `GALAXY_ONEDRIVE_CLIENT_ID` and
`GALAXY_ONEDRIVE_CLIENT_SECRET`. Jobs themselves do not need these values and should
not receive them.
If your Galaxy instance has Vault configured, you can use this Vault-backed variant instead:
```{literalinclude} ../../../lib/galaxy/files/templates/examples/onedrive_client_secrets_in_vault.yml
:language: yaml
```
The current OneDrive implementation supports two drive modes:
- `drive_mode: appfolder`
This is the default and targets Microsoft Graph `special/approot`. Galaxy can
browse, download, upload, and create folders inside the application's dedicated
OneDrive app folder (`Apps/<Application Name>`). This mode should be paired
with delegated permission `Files.ReadWrite.AppFolder`.
- `drive_mode: full`
This targets the user's full OneDrive root (`/me/drive/root`) instead of the
application folder. This mode requires broader delegated Microsoft Graph
permissions such as `Files.ReadWrite`.
To configure Microsoft Entra app for this file source:
1. Sign in to [Microsoft Azure](https://portal.azure.com/). Go to `Microsoft Entra ID` and open
`App Registrations`.
2. Select `New registration`.
3. Enter a recognizable application name for Galaxy, for example `Galaxy OneDrive`.
4. Under `Supported account types`, choose the audience that matches your deployment.
If Galaxy users may connect both organizational Microsoft accounts and personal
Microsoft accounts, select `Any Entra ID tenant + Personal Microsoft accounts`.
5. Under `Redirect URI`, choose platform type `Web` and enter your Galaxy callback URL:
`<your galaxy root>/oauth2_callback`.
For example, if Galaxy is available at `https://usegalaxy.eu`, use
`https://usegalaxy.eu/oauth2_callback`.
For local development this is often `http://localhost:8080/oauth2_callback`.
6. Create the registration and open the app's `Overview` page.
Copy the `Application (client) ID` and expose it to Galaxy as
`GALAXY_ONEDRIVE_CLIENT_ID`.
7. Open `Certificates & secrets > Client secrets`, create a new client secret,
and copy the generated secret value immediately.
Expose that value to Galaxy as `GALAXY_ONEDRIVE_CLIENT_SECRET`.
Microsoft only shows the full secret value once.
8. Open `API permissions` and add Microsoft Graph delegated permissions.
For the default app-folder configuration, add `Files.ReadWrite.AppFolder`.
Also add `offline_access` so Galaxy can obtain refresh tokens for long-lived access.
9. If your deployment uses `drive_mode: full` instead of the default `appfolder`,
add delegated permission `Files.ReadWrite` instead of `Files.ReadWrite.AppFolder`.
This must match the scope requested in the Galaxy template.
After this setup, users connect their own OneDrive accounts through Galaxy's OAuth2
flow. The client ID and client secret identify your Galaxy application to Microsoft,
but file access is performed with per-user delegated access and refresh tokens.
To configure full-drive access instead of the default app-folder mode, you need to
change both the Galaxy yml config template and the Microsoft Entra app registration.
In Galaxy yml config, set `drive_mode: full` and request a broader OAuth scope such as
`oauth2_scope: "offline_access Files.ReadWrite"`. In Microsoft Entra, grant the
matching delegated Microsoft Graph permission (`Files.ReadWrite` instead of
`Files.ReadWrite.AppFolder`). If only the Microsoft permission is widened and
`drive_mode` remains `appfolder`, Galaxy will continue to operate only inside the
application folder.
This implementation currently uses Microsoft Graph's simple upload endpoint and
does not yet implement resumable uploads for very large files, server-side pagination,
or server-side search/sorting.
## Playing Nicer with Ansible
Many large instances of Galaxy are configured with Ansible and much of the existing administrator
+212
View File
@@ -0,0 +1,212 @@
from __future__ import annotations
from typing import (
Annotated,
Literal,
Optional,
Union,
)
from urllib.parse import quote
import requests
from pydantic import (
AliasChoices,
Field,
)
from galaxy.exceptions import (
AuthenticationRequired,
MessageException,
RequestParameterInvalidException,
)
from galaxy.files.models import (
AnyRemoteEntry,
BaseFileSourceConfiguration,
BaseFileSourceTemplateConfiguration,
Entry,
EntryData,
FilesSourceRuntimeContext,
RemoteDirectory,
RemoteFile,
)
from galaxy.util.config_templates import TemplateExpansion
from . import BaseFilesSource
AccessTokenField = Field(
...,
title="Access Token",
description="The OAuth2 access token for Microsoft Graph.",
validation_alias=AliasChoices("oauth2_access_token", "accessToken", "access_token"),
)
DriveMode = Literal["appfolder", "full"]
class OneDriveFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
access_token: Annotated[Union[str, TemplateExpansion], AccessTokenField]
drive_api_base: Union[str, TemplateExpansion] = "https://graph.microsoft.com/v1.0/me/drive"
drive_mode: Union[DriveMode, TemplateExpansion] = "appfolder"
class OneDriveFilesSourceConfiguration(BaseFileSourceConfiguration):
access_token: Annotated[str, AccessTokenField]
drive_api_base: str = "https://graph.microsoft.com/v1.0/me/drive"
drive_mode: DriveMode = "appfolder"
class OneDriveFilesSource(BaseFilesSource[OneDriveFileSourceTemplateConfiguration, OneDriveFilesSourceConfiguration]):
plugin_type = "onedrive"
template_config_class = OneDriveFileSourceTemplateConfiguration
resolved_config_class = OneDriveFilesSourceConfiguration
def _headers(self, config: OneDriveFilesSourceConfiguration) -> dict[str, str]:
return {
"Authorization": f"Bearer {config.access_token}",
}
def _encoded_path(self, path: str) -> str:
normalized = path.strip("/")
if not normalized:
return ""
return "/".join(quote(component, safe="") for component in normalized.split("/"))
def _root_url(self, config: OneDriveFilesSourceConfiguration) -> str:
api_base = config.drive_api_base.rstrip("/")
if config.drive_mode == "full":
return f"{api_base}/root"
return f"{api_base}/special/approot"
def _item_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str:
root_url = self._root_url(config)
encoded_path = self._encoded_path(path)
if encoded_path:
return f"{root_url}:/{encoded_path}"
return root_url
def _children_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str:
item_url = self._item_url(config, path)
if path.strip("/"):
return f"{item_url}:/children"
return f"{item_url}/children"
def _content_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str:
return (
f"{self._item_url(config, path)}:/content" if path.strip("/") else f"{self._item_url(config, path)}/content"
)
def _request(
self,
method: str,
url: str,
context: FilesSourceRuntimeContext[OneDriveFilesSourceConfiguration],
timeout: int = 30,
**kwargs,
) -> requests.Response:
try:
response = requests.request(method, url, headers=self._headers(context.config), timeout=timeout, **kwargs)
except requests.RequestException as exc:
raise MessageException(f"Error connecting to OneDrive. Reason: {exc}") from exc
if response.status_code in {401, 403}:
raise AuthenticationRequired(
"Permission denied while accessing OneDrive. Check the Microsoft app registration, granted scopes, and the stored user authorization."
)
if response.status_code == 404:
raise RequestParameterInvalidException(f"Path not found in OneDrive: {url}")
if not response.ok:
try:
payload = response.json()
message = payload.get("error", {}).get("message", response.text)
except Exception:
message = response.text
raise MessageException(f"Error communicating with OneDrive. Reason: {message}")
return response
def _entry_from_item(
self,
item: dict,
parent_path: str,
) -> AnyRemoteEntry:
relative_parent = parent_path.rstrip("/")
relative_path = f"{relative_parent}/{item['name']}".replace("//", "/")
if not relative_path.startswith("/"):
relative_path = f"/{relative_path}"
uri = self.uri_from_path(relative_path)
if "folder" in item:
return RemoteDirectory(
name=item["name"],
uri=uri,
path=relative_path,
)
return RemoteFile(
name=item["name"],
uri=uri,
path=relative_path,
size=item.get("size", 0),
ctime=item.get("lastModifiedDateTime"),
)
def _list(
self,
context: FilesSourceRuntimeContext[OneDriveFilesSourceConfiguration],
path: str = "/",
recursive: bool = False,
write_intent: bool = False,
limit: Optional[int] = None,
offset: Optional[int] = None,
query: Optional[str] = None,
sort_by: Optional[str] = None,
) -> tuple[list[AnyRemoteEntry], int]:
response = self._request("GET", self._children_url(context.config, path), context)
items = response.json().get("value", [])
entries = [self._entry_from_item(item, path) for item in items]
return entries, len(entries)
def _realize_to(
self, source_path: str, native_path: str, context: FilesSourceRuntimeContext[OneDriveFilesSourceConfiguration]
):
response = self._request("GET", self._content_url(context.config, source_path), context, stream=True)
with open(native_path, "wb") as out:
for chunk in response.iter_content(chunk_size=1024 * 1024):
if chunk:
out.write(chunk)
def _write_from(
self, target_path: str, native_path: str, context: FilesSourceRuntimeContext[OneDriveFilesSourceConfiguration]
) -> str:
upload_url = self._content_url(context.config, target_path)
with open(native_path, "rb") as handle:
self._request(
"PUT",
upload_url,
context,
data=handle,
timeout=300,
)
return self.uri_from_path(target_path)
def _create_entry(
self, entry_data: EntryData, context: FilesSourceRuntimeContext[OneDriveFilesSourceConfiguration]
) -> Entry:
parent_path = getattr(entry_data, "path", None)
if parent_path is None:
target = getattr(entry_data, "target", "/")
parent_path = self.to_relative_path(target)
payload = {
"name": entry_data.name,
"folder": {},
"@microsoft.graph.conflictBehavior": "fail",
}
response = self._request(
"POST",
self._children_url(context.config, parent_path),
context,
json=payload,
)
item = response.json()
path = self._entry_from_item(item, parent_path).path
return Entry(name=item["name"], uri=self.uri_from_path(path), external_link=item.get("webUrl"))
__all__ = ("OneDriveFilesSource",)
@@ -0,0 +1,16 @@
- id: onedrive
name: OneDrive
description: Connect to your Microsoft OneDrive app folder to download and upload files.
configuration:
type: onedrive
oauth2_client_id: "{{ environment.oauth2_client_id }}"
oauth2_client_secret: "{{ environment.oauth2_client_secret }}"
drive_mode: appfolder
writable: true
environment:
oauth2_client_id:
type: secret
vault_key: "onedrive_file_source/client_id"
oauth2_client_secret:
type: secret
vault_key: "onedrive_file_source/client_secret"
@@ -0,0 +1,16 @@
- id: onedrive
name: OneDrive
description: Connect to the Galaxy folder in your Microsoft OneDrive. Galaxy will only access its own app folder, not the rest of your OneDrive.
configuration:
type: onedrive
oauth2_client_id: "{{ environment.oauth2_client_id }}"
oauth2_client_secret: "{{ environment.oauth2_client_secret }}"
drive_mode: appfolder
writable: true
environment:
oauth2_client_id:
type: variable
variable: GALAXY_ONEDRIVE_CLIENT_ID
oauth2_client_secret:
type: variable
variable: GALAXY_ONEDRIVE_CLIENT_SECRET
@@ -0,0 +1,19 @@
- id: onedrive_full
name: OneDrive
description: |
Connect to your full Microsoft OneDrive.
You will be asked to grant Galaxy permission to access files across your OneDrive, not just the Galaxy app folder.
configuration:
type: onedrive
oauth2_client_id: "{{ environment.oauth2_client_id }}"
oauth2_client_secret: "{{ environment.oauth2_client_secret }}"
oauth2_scope: "offline_access Files.ReadWrite"
drive_mode: full
writable: true
environment:
oauth2_client_id:
type: variable
variable: GALAXY_ONEDRIVE_CLIENT_ID
oauth2_client_secret:
type: variable
variable: GALAXY_ONEDRIVE_CLIENT_SECRET
+29
View File
@@ -41,6 +41,7 @@ FileSourceTemplateType = Literal[
"webdav",
"dropbox",
"googledrive",
"onedrive",
"elabftw",
"inveniordm",
"zenodo",
@@ -113,6 +114,25 @@ class GoogleDriveFileSourceConfiguration(OAuth2FileSourceConfiguration, StrictMo
oauth2_access_token: str
class OneDriveFileSourceTemplateConfiguration(OAuth2TemplateConfiguration, StrictModel):
type: Literal["onedrive"]
writable: Union[bool, TemplateExpansion] = False
oauth2_client_id: Union[str, TemplateExpansion]
oauth2_client_secret: Union[str, TemplateExpansion]
# Microsoft Graph app-folder scope keeps access limited to Apps/<Application Name>.
oauth2_scope: Optional[Union[str, TemplateExpansion]] = None
drive_mode: Union[Literal["appfolder", "full"], TemplateExpansion] = "appfolder"
template_start: Optional[str] = None
template_end: Optional[str] = None
class OneDriveFileSourceConfiguration(OAuth2FileSourceConfiguration, StrictModel):
type: Literal["onedrive"]
writable: bool = False
oauth2_access_token: str
drive_mode: Literal["appfolder", "full"] = "appfolder"
class S3FSFileSourceTemplateConfiguration(StrictModel):
type: Literal["s3fs"]
endpoint_url: Optional[Union[str, TemplateExpansion]] = None
@@ -364,6 +384,7 @@ FileSourceTemplateConfiguration = Annotated[
WebdavFileSourceTemplateConfiguration,
DropboxFileSourceTemplateConfiguration,
GoogleDriveFileSourceTemplateConfiguration,
OneDriveFileSourceTemplateConfiguration,
eLabFTWFileSourceTemplateConfiguration,
InvenioFileSourceTemplateConfiguration,
ZenodoFileSourceTemplateConfiguration,
@@ -386,6 +407,7 @@ FileSourceConfiguration = Annotated[
WebdavFileSourceConfiguration,
DropboxFileSourceConfiguration,
GoogleDriveFileSourceConfiguration,
OneDriveFileSourceConfiguration,
eLabFTWFileSourceConfiguration,
InvenioFileSourceConfiguration,
ZenodoFileSourceConfiguration,
@@ -466,6 +488,7 @@ TypesToConfigurationClasses: dict[FileSourceTemplateType, type[FileSourceConfigu
"webdav": WebdavFileSourceConfiguration,
"dropbox": DropboxFileSourceConfiguration,
"googledrive": GoogleDriveFileSourceConfiguration,
"onedrive": OneDriveFileSourceConfiguration,
"elabftw": eLabFTWFileSourceConfiguration,
"inveniordm": InvenioFileSourceConfiguration,
"zenodo": ZenodoFileSourceConfiguration,
@@ -488,6 +511,12 @@ OAUTH2_CONFIGURED_SOURCES: ConfiguredOAuth2Sources = {
token_url="https://oauth2.googleapis.com/token",
scope="https://www.googleapis.com/auth/drive.file",
),
"onedrive": OAuth2Configuration(
authorize_url="https://login.microsoftonline.com/common/oauth2/v2.0/authorize",
token_url="https://login.microsoftonline.com/common/oauth2/v2.0/token",
authorize_params={},
scope="offline_access Files.ReadWrite.AppFolder",
),
}
@@ -334,6 +334,90 @@ class TestFileSourcesTestCase(BaseTestCase):
assert not status.connection.is_not_ok
assert pyfilesystem_fs_init_kwd["access_token"] == "my_test_access_token"
def test_onedrive_oauth2_flow(self, tmp_path, monkeypatch):
json = {
"refresh_token": "my_test_refresh_token",
}
def mock_get_token_from_code_raw(
code,
client_pair,
config,
redirect_uri,
):
return MockResponse(json)
monkeypatch.setattr(config_templates, "get_token_from_code_raw", mock_get_token_from_code_raw)
self._init_onedrive_env(tmp_path, monkeypatch)
authorize_url = self.manager.template_oauth2(self.trans, "onedrive", 0).authorize_url
from urllib.parse import (
parse_qs,
urlparse,
)
parse_result = urlparse(authorize_url)
assert parse_result.hostname == "login.microsoftonline.com"
assert parse_result.path == "/common/oauth2/v2.0/authorize"
query_params = parse_qs(parse_result.query)
assert query_params["scope"][0] == "offline_access Files.ReadWrite.AppFolder"
assert "state" in query_params
state_param = query_params["state"]
state = OAuth2State.decode(state_param[0])
assert state.route == "file_source_instances/onedrive/0"
redirect_url = self.manager.handle_authorization_code(
self.trans,
"moocow",
state,
)
parse_result = urlparse(redirect_url)
query_params = parse_qs(parse_result.query)
assert "uuid" in query_params
uuid = query_params["uuid"][0]
user_vault = self.trans.user_vault
config_secret_key = UserFileSource.vault_key_from_uuid(uuid, "_oauth2_refresh_token", None)
assert user_vault.read_secret(config_secret_key)
def test_onedrive_oauth2_access_token_injection_during_verify(self, tmp_path, monkeypatch):
self._init_onedrive_env(tmp_path, monkeypatch)
uuid = uuid4().hex
user_vault = self.trans.user_vault
config_secret_key = UserFileSource.vault_key_from_uuid(uuid, "_oauth2_refresh_token", None)
user_vault.write_secret(config_secret_key, "test_refresh_token")
create_payload = CreateInstancePayload(
name=SIMPLE_FILE_SOURCE_NAME,
description=SIMPLE_FILE_SOURCE_DESCRIPTION,
template_id="onedrive",
template_version=0,
variables={},
secrets={},
uuid=uuid,
)
self._create_instance(create_payload)
json = {
"access_token": "my_test_access_token",
}
observed_headers: dict[str, str] = {}
def mock_get_token_from_refresh_raw(refresh_token, client_pair, config):
return MockResponse(json)
def mock_request(method, url, headers=None, timeout=None, **kwargs):
observed_headers.update(headers or {})
return OneDriveMockResponse(json_data={"value": []})
monkeypatch.setattr(config_templates, "get_token_from_refresh_raw", mock_get_token_from_refresh_raw)
monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request)
status = self.manager.plugin_status(self.trans, create_payload)
assert status.oauth2_access_token_generation
assert not status.oauth2_access_token_generation.is_not_ok
assert status.connection
assert not status.connection.is_not_ok
assert observed_headers["Authorization"] == "Bearer my_test_access_token"
def test_report_oauth2_access_token_generation_failure(self, tmp_path, monkeypatch):
self._init_dropbox_env(tmp_path, monkeypatch)
@@ -855,6 +939,13 @@ class TestFileSourcesTestCase(BaseTestCase):
monkeypatch.setenv("GALAXY_DROPBOX_APP_CLIENT_ID", "mock_client_id")
monkeypatch.setenv("GALAXY_DROPBOX_APP_CLIENT_SECRET", "mock_client_secret")
def _init_onedrive_env(self, tmp_path, monkeypatch):
self.init_user_in_database()
self._init_managers(tmp_path, safe_load(get_example("production_onedrive.yml")))
monkeypatch.setenv("GALAXY_ONEDRIVE_CLIENT_ID", "mock_client_id")
monkeypatch.setenv("GALAXY_ONEDRIVE_CLIENT_SECRET", "mock_client_secret")
def _create_user_file_source(self, template_id="home_directory") -> UserFileSourceModel:
create_payload = CreateInstancePayload(
name=SIMPLE_FILE_SOURCE_NAME,
@@ -943,3 +1034,18 @@ class MockExceptionResponse:
def raise_for_status(self):
raise HTTPError(self._exception_msg, self._exception_msg, response=None)
class OneDriveMockResponse:
def __init__(self, status_code=200, json_data=None, text=""):
self.status_code = status_code
self._json_data = json_data or {}
self.text = text
@property
def ok(self):
return 200 <= self.status_code < 300
def json(self):
return self._json_data
+1
View File
@@ -75,6 +75,7 @@ def user_context_fixture(user_ftp_dir=None, role_names=None, group_names=None, i
"googledrive|client_secret": os.environ.get("GALAXY_TEST_GOOGLE_DRIVE_CLIENT_SECRET"),
"googledrive|access_token": os.environ.get("GALAXY_TEST_GOOGLE_DRIVE_ACCESS_TOKEN"),
"googledrive|refresh_token": os.environ.get("GALAXY_TEST_GOOGLE_DRIVE_REFRESH_TOKEN"),
"onedrive|access_token": os.environ.get("GALAXY_TEST_ONEDRIVE_ACCESS_TOKEN"),
"googlecloudstorage|project": os.environ.get("GALAXY_TEST_GCS_PROJECT"),
"googlecloudstorage|bucket_name": os.environ.get("GALAXY_TEST_GCS_BUCKET"),
"googlecloudstorage|client_id": os.environ.get("GALAXY_TEST_GCS_CLIENT_ID"),
@@ -0,0 +1,4 @@
- type: onedrive
id: test1
doc: Test access to a OneDrive account.
accessToken: ${user.preferences['onedrive|access_token']}
+18
View File
@@ -0,0 +1,18 @@
import os
import pytest
from ._util import assert_simple_file_realize
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
FILE_SOURCES_CONF = os.path.join(SCRIPT_DIRECTORY, "onedrive_file_sources_conf.yml")
skip_if_no_onedrive_access_token = pytest.mark.skipif(
not os.environ.get("GALAXY_TEST_ONEDRIVE_ACCESS_TOKEN"),
reason="GALAXY_TEST_ONEDRIVE_ACCESS_TOKEN not set",
)
@skip_if_no_onedrive_access_token
def test_file_source():
assert_simple_file_realize(FILE_SOURCES_CONF)