mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
add full-access drive mode, template and tests
This commit is contained in:
+27
-11
@@ -641,12 +641,17 @@ processes using the environment variables `GALAXY_ONEDRIVE_CLIENT_ID` and
|
||||
`GALAXY_ONEDRIVE_CLIENT_SECRET`. Jobs themselves do not need these values and should
|
||||
not receive them.
|
||||
|
||||
The current OneDrive implementation targets Microsoft Graph special/approot and uses the
|
||||
`Files.ReadWrite.AppFolder` delegated scope. This means Galaxy can browse, download,
|
||||
upload, and create folders inside the application's dedicated OneDrive app folder
|
||||
(`Apps/<Application Name>`). Supporting full-drive access would require code changes
|
||||
in addition to broader scopes such as `Files.ReadWrite` or `Files.ReadWrite.All` and
|
||||
configuring yml template with `oauth2_scope: "offline_access Files.ReadWrite.All"`
|
||||
The current OneDrive implementation supports two drive modes:
|
||||
|
||||
- `drive_mode: appfolder`
|
||||
This is the default and targets Microsoft Graph `special/approot`. Galaxy can
|
||||
browse, download, upload, and create folders inside the application's dedicated
|
||||
OneDrive app folder (`Apps/<Application Name>`). This mode should be paired
|
||||
with delegated permission `Files.ReadWrite.AppFolder`.
|
||||
- `drive_mode: full`
|
||||
This targets the user's full OneDrive root (`/me/drive/root`) instead of the
|
||||
application folder. This mode requires broader delegated Microsoft Graph
|
||||
permissions such as `Files.ReadWrite`.
|
||||
|
||||
To configure Microsoft Entra for this file source:
|
||||
|
||||
@@ -657,20 +662,31 @@ To configure Microsoft Entra for this file source:
|
||||
3. Configure supported account types for the accounts you intend to support. If you
|
||||
want to support both work/school accounts and personal Microsoft accounts, set
|
||||
the audience to "Any Entra ID tenant + Personal Microsoft accounts".
|
||||
4. Add delegated Microsoft Graph permissions: `Files.ReadWrite.AppFolder`,
|
||||
`offline_access` (to ensure Galaxy can obtain refresh tokens for long-lived access).
|
||||
5. Create a client secret and provide its value to Galaxy via
|
||||
4. Add delegated Microsoft Graph permissions:
|
||||
For the default app-folder setup, add permission `Files.ReadWrite.AppFolder`.
|
||||
To ensure Galaxy can obtain refresh tokens for long-lived access add permission
|
||||
`offline_access`.
|
||||
6. Create a client secret and provide its value to Galaxy via
|
||||
`GALAXY_ONEDRIVE_CLIENT_SECRET` (remember, this value can be seen only once after creation).
|
||||
6. Go to Overview, find "Application (client) ID" and provide its value to Galaxy
|
||||
7. Go to Overview, find "Application (client) ID" and provide its value to Galaxy
|
||||
via `GALAXY_ONEDRIVE_CLIENT_ID`
|
||||
|
||||
To configure full-drive access instead of the default app-folder mode, you need to
|
||||
change both the Galaxy yml config template and the Microsoft Entra app registration.
|
||||
In Galaxy yml config, set `drive_mode: full` and request a broader OAuth scope such as
|
||||
`oauth2_scope: "offline_access Files.ReadWrite"`. In Microsoft Entra, grant the
|
||||
matching delegated Microsoft Graph permission (`Files.ReadWrite` instead of
|
||||
`Files.ReadWrite.AppFolder`). If only the Microsoft permission is widened and
|
||||
`drive_mode` remains `appfolder`, Galaxy will continue to operate only inside the
|
||||
application folder.
|
||||
|
||||
The OAuth2 endpoints Galaxy uses for this template are the Microsoft identity platform's
|
||||
v2 endpoints under the `common` tenant. If you register an application that also supports
|
||||
personal Microsoft accounts, ensure the manifest is consistent with that audience. In
|
||||
particular, the app manifest should use `AzureADandPersonalMicrosoftAccount` as the
|
||||
`signInAudience` and `2` as the `requestedAccessTokenVersion`.
|
||||
|
||||
This first implementation currently uses Microsoft Graph's simple upload endpoint and
|
||||
This implementation currently uses Microsoft Graph's simple upload endpoint and
|
||||
does not yet implement resumable uploads for very large files, server-side pagination,
|
||||
or server-side search/sorting.
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
from typing import (
|
||||
Annotated,
|
||||
Literal,
|
||||
Optional,
|
||||
Union,
|
||||
)
|
||||
@@ -38,20 +39,22 @@ AccessTokenField = Field(
|
||||
validation_alias=AliasChoices("oauth2_access_token", "accessToken", "access_token"),
|
||||
)
|
||||
|
||||
DriveMode = Literal["appfolder", "full"]
|
||||
|
||||
|
||||
class OneDriveFileSourceTemplateConfiguration(BaseFileSourceTemplateConfiguration):
|
||||
access_token: Annotated[Union[str, TemplateExpansion], AccessTokenField]
|
||||
drive_api_base: Union[str, TemplateExpansion] = "https://graph.microsoft.com/v1.0/me/drive"
|
||||
drive_mode: Union[DriveMode, TemplateExpansion] = "appfolder"
|
||||
|
||||
|
||||
class OneDriveFilesSourceConfiguration(BaseFileSourceConfiguration):
|
||||
access_token: Annotated[str, AccessTokenField]
|
||||
drive_api_base: str = "https://graph.microsoft.com/v1.0/me/drive"
|
||||
drive_mode: DriveMode = "appfolder"
|
||||
|
||||
|
||||
class OneDriveFilesSource(
|
||||
BaseFilesSource[OneDriveFileSourceTemplateConfiguration, OneDriveFilesSourceConfiguration]
|
||||
):
|
||||
class OneDriveFilesSource(BaseFilesSource[OneDriveFileSourceTemplateConfiguration, OneDriveFilesSourceConfiguration]):
|
||||
plugin_type = "onedrive"
|
||||
|
||||
template_config_class = OneDriveFileSourceTemplateConfiguration
|
||||
@@ -73,12 +76,18 @@ class OneDriveFilesSource(
|
||||
return ""
|
||||
return "/".join(quote(component, safe="") for component in normalized.split("/"))
|
||||
|
||||
def _item_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str:
|
||||
def _root_url(self, config: OneDriveFilesSourceConfiguration) -> str:
|
||||
api_base = config.drive_api_base.rstrip("/")
|
||||
if config.drive_mode == "full":
|
||||
return f"{api_base}/root"
|
||||
return f"{api_base}/special/approot"
|
||||
|
||||
def _item_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str:
|
||||
root_url = self._root_url(config)
|
||||
encoded_path = self._encoded_path(path)
|
||||
if encoded_path:
|
||||
return f"{api_base}/special/approot:/{encoded_path}"
|
||||
return f"{api_base}/special/approot"
|
||||
return f"{root_url}:/{encoded_path}"
|
||||
return root_url
|
||||
|
||||
def _children_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str:
|
||||
item_url = self._item_url(config, path)
|
||||
@@ -87,7 +96,9 @@ class OneDriveFilesSource(
|
||||
return f"{item_url}/children"
|
||||
|
||||
def _content_url(self, config: OneDriveFilesSourceConfiguration, path: str) -> str:
|
||||
return f"{self._item_url(config, path)}:/content" if path.strip("/") else f"{self._item_url(config, path)}/content"
|
||||
return (
|
||||
f"{self._item_url(config, path)}:/content" if path.strip("/") else f"{self._item_url(config, path)}/content"
|
||||
)
|
||||
|
||||
def _request(
|
||||
self,
|
||||
@@ -172,7 +183,10 @@ class OneDriveFilesSource(
|
||||
with open(native_path, "rb") as handle:
|
||||
response = requests.put(
|
||||
upload_url,
|
||||
headers={"Authorization": f"Bearer {context.config.access_token}", "Content-Type": "application/octet-stream"},
|
||||
headers={
|
||||
"Authorization": f"Bearer {context.config.access_token}",
|
||||
"Content-Type": "application/octet-stream",
|
||||
},
|
||||
data=handle,
|
||||
timeout=300,
|
||||
)
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
type: onedrive
|
||||
oauth2_client_id: "{{ environment.oauth2_client_id }}"
|
||||
oauth2_client_secret: "{{ environment.oauth2_client_secret }}"
|
||||
drive_mode: appfolder
|
||||
writable: true
|
||||
environment:
|
||||
oauth2_client_id:
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
- id: onedrive
|
||||
name: OneDrive
|
||||
description: Connect to your Microsoft OneDrive app folder to download and upload files.
|
||||
configuration:
|
||||
type: onedrive
|
||||
oauth2_client_id: "{{ environment.oauth2_client_id }}"
|
||||
oauth2_client_secret: "{{ environment.oauth2_client_secret }}"
|
||||
oauth2_scope: "offline_access Files.ReadWrite"
|
||||
drive_mode: full
|
||||
writable: true
|
||||
environment:
|
||||
oauth2_client_id:
|
||||
type: variable
|
||||
variable: GALAXY_ONEDRIVE_CLIENT_ID
|
||||
oauth2_client_secret:
|
||||
type: variable
|
||||
variable: GALAXY_ONEDRIVE_CLIENT_SECRET
|
||||
@@ -121,6 +121,7 @@ class OneDriveFileSourceTemplateConfiguration(OAuth2TemplateConfiguration, Stric
|
||||
oauth2_client_secret: Union[str, TemplateExpansion]
|
||||
# Microsoft Graph app-folder scope keeps access limited to Apps/<Application Name>.
|
||||
oauth2_scope: Optional[Union[str, TemplateExpansion]] = None
|
||||
drive_mode: Union[Literal["appfolder", "full"], TemplateExpansion] = "appfolder"
|
||||
template_start: Optional[str] = None
|
||||
template_end: Optional[str] = None
|
||||
|
||||
@@ -129,6 +130,7 @@ class OneDriveFileSourceConfiguration(OAuth2FileSourceConfiguration, StrictModel
|
||||
type: Literal["onedrive"]
|
||||
writable: bool = False
|
||||
oauth2_access_token: str
|
||||
drive_mode: Literal["appfolder", "full"] = "appfolder"
|
||||
|
||||
|
||||
class S3FSFileSourceTemplateConfiguration(StrictModel):
|
||||
|
||||
@@ -23,10 +23,7 @@ from galaxy.exceptions import (
|
||||
RequestParameterMissingException,
|
||||
)
|
||||
from galaxy.files import FileSourcesUserContext
|
||||
from galaxy.files.sources import (
|
||||
dropbox,
|
||||
onedrive,
|
||||
)
|
||||
from galaxy.files.sources import dropbox
|
||||
from galaxy.files.templates import ConfiguredFileSourceTemplates
|
||||
from galaxy.files.templates.examples import get_example
|
||||
from galaxy.managers._config_templates import prepare_environment_from_root
|
||||
@@ -403,7 +400,7 @@ class TestFileSourcesTestCase(BaseTestCase):
|
||||
json = {
|
||||
"access_token": "my_test_access_token",
|
||||
}
|
||||
observed_headers = {}
|
||||
observed_headers: dict[str, str] = {}
|
||||
|
||||
def mock_get_token_from_refresh_raw(refresh_token, client_pair, config):
|
||||
return MockResponse(json)
|
||||
@@ -413,7 +410,7 @@ class TestFileSourcesTestCase(BaseTestCase):
|
||||
return OneDriveMockResponse(json_data={"value": []})
|
||||
|
||||
monkeypatch.setattr(config_templates, "get_token_from_refresh_raw", mock_get_token_from_refresh_raw)
|
||||
monkeypatch.setattr(onedrive.requests, "request", mock_request)
|
||||
monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request)
|
||||
status = self.manager.plugin_status(self.trans, create_payload)
|
||||
assert status.oauth2_access_token_generation
|
||||
assert not status.oauth2_access_token_generation.is_not_ok
|
||||
|
||||
@@ -44,6 +44,20 @@ def _plugin():
|
||||
return OneDriveFilesSource(template)
|
||||
|
||||
|
||||
def _plugin_full():
|
||||
template = OneDriveFilesSource.build_template_config(
|
||||
id="test1",
|
||||
type="onedrive",
|
||||
label="OneDrive",
|
||||
doc="Test OneDrive file source",
|
||||
writable=True,
|
||||
access_token="test_access_token",
|
||||
drive_mode="full",
|
||||
file_sources_config=FileSourcePluginsConfig(),
|
||||
)
|
||||
return OneDriveFilesSource(template)
|
||||
|
||||
|
||||
def test_list_root(monkeypatch):
|
||||
plugin = _plugin()
|
||||
observed = {}
|
||||
@@ -75,6 +89,23 @@ def test_list_root(monkeypatch):
|
||||
assert entries[1].uri == "gxfiles://test1/a.txt"
|
||||
|
||||
|
||||
def test_list_root_full_drive_mode(monkeypatch):
|
||||
plugin = _plugin_full()
|
||||
observed = {}
|
||||
|
||||
def mock_request(method, url, headers=None, timeout=None, **kwargs):
|
||||
observed["url"] = url
|
||||
return MockResponse(json_data={"value": []})
|
||||
|
||||
monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request)
|
||||
|
||||
entries, count = plugin.list("/")
|
||||
|
||||
assert count == 0
|
||||
assert entries == []
|
||||
assert observed["url"] == "https://graph.microsoft.com/v1.0/me/drive/root/children"
|
||||
|
||||
|
||||
def test_list_nested_folder(monkeypatch):
|
||||
plugin = _plugin()
|
||||
observed = {}
|
||||
@@ -92,6 +123,23 @@ def test_list_nested_folder(monkeypatch):
|
||||
assert entries[0].path == "/level1/level 2/b.txt"
|
||||
|
||||
|
||||
def test_list_nested_folder_full_drive_mode(monkeypatch):
|
||||
plugin = _plugin_full()
|
||||
observed = {}
|
||||
|
||||
def mock_request(method, url, headers=None, timeout=None, **kwargs):
|
||||
observed["url"] = url
|
||||
return MockResponse(json_data={"value": [{"name": "b.txt", "size": 4}]})
|
||||
|
||||
monkeypatch.setattr("galaxy.files.sources.onedrive.requests.request", mock_request)
|
||||
|
||||
entries, count = plugin.list("/level1/level 2")
|
||||
|
||||
assert count == 1
|
||||
assert observed["url"] == "https://graph.microsoft.com/v1.0/me/drive/root:/level1/level%202:/children"
|
||||
assert entries[0].path == "/level1/level 2/b.txt"
|
||||
|
||||
|
||||
def test_realize_to_downloads_content(monkeypatch, tmp_path: Path):
|
||||
plugin = _plugin()
|
||||
target = tmp_path / "downloaded.txt"
|
||||
|
||||
Reference in New Issue
Block a user