From 600f59b0eed7906f044bf0bc6f2095edd8dc139b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 12 Oct 2021 12:28:59 +0200 Subject: [PATCH] Use client-logic to run upload tool It is pretty cool to use hooks and an alternative tool submission endpoint, but this is less invasive. We'd have to do this anyway for cases where the upload is already on the server and the pre-finish hook doesn't fire. --- client/src/utils/uploadbox.js | 20 ++++++- doc/source/admin/nginx.md | 46 +++++++++------ lib/galaxy/dependencies/dev-requirements.txt | 2 +- .../dependencies/pinned-requirements.txt | 2 +- lib/galaxy/webapps/galaxy/api/uploads.py | 56 ++----------------- lib/galaxy/webapps/galaxy/buildapp.py | 5 +- lib/galaxy_test/api/test_tools_upload.py | 12 +++- scripts/resumable_upload.py | 52 ++++++++++++----- 8 files changed, 105 insertions(+), 90 deletions(-) diff --git a/client/src/utils/uploadbox.js b/client/src/utils/uploadbox.js index f4464b947ae..968a3c20d46 100644 --- a/client/src/utils/uploadbox.js +++ b/client/src/utils/uploadbox.js @@ -116,10 +116,28 @@ import * as tus from "tus-js-client"; }, onSuccess: function () { console.log( - `Download ${upload.file.name} from ${upload.url} took ${ + `Upload of ${upload.file.name} to ${upload.url} took ${ (performance.now() - startTime) / 1000 } seconds` ); + const toolInputs = JSON.parse(data.payload.inputs); + toolInputs["files_0|file_data"] = { + session_id: upload.url.split("/").at(-1), + name: file.name, + }; + data.payload.inputs = JSON.stringify(toolInputs); + $.ajax({ + url: `${getAppRoot()}api/tools`, + method: "POST", + data: data.payload, + success: (tool_response) => { + cnf.success(tool_response); + }, + error: (tool_response) => { + var err_msg = tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg; + cnf.error(err_msg || cnf.error_tool); + }, + }); cnf.success({}); }, }); diff --git a/doc/source/admin/nginx.md b/doc/source/admin/nginx.md index 43b047748de..b268b2abde5 100644 --- a/doc/source/admin/nginx.md +++ b/doc/source/admin/nginx.md @@ -283,21 +283,20 @@ group as shown above. Galaxy includes a WSGI middleware that implements a tus server for which no configuration is needed. However the middleware ties up resources on the Galaxy server process, and uploads will be interrupted while Galaxy restarts. A more efficient alternative is to run an external server that implements the tus protocol. -Any tus server that can send the pre-finish hook to a URL will work. Here we will use [tusd](https://github.com/tus/tusd). +Any tus server will work. Here we will use [tusd](https://github.com/tus/tusd). Binaries can be downloaded from https://github.com/tus/tusd/releases/. In this example we will set up tusd to: - listen on port 1080 on localhost (`-host localhost -port 1080`) - store uploads in database/tmp (replace this with the value of new_file_path in your galaxy.yml config) (`-upload-dir=/database/tmp`) -- send an event via http to /api/upload/_resumable_upload when an upload has finished (`-hooks-http=/api/upload/_resumable_upload`) +- send an event via http to /api/upload/hooks to ensure the user is logged in (`-hooks-http=/api/upload/hooks`) - forward authentication headers in that event (`-hooks-http-forward-headers=X-Api-Key,sessioncookie`) -- and enable only the pre-finish event hook (`-hooks-enabled-events=pre-finish`) The complete command is thus (replace `` with your Galaxy URL and `` with the path to your Galaxy installation): ```sh -tusd -host localhost -port 1080 -upload-dir=/database/tmp -hooks-http=/api/upload/_resumable_upload -hooks-http-forward-headers=X-Api-Key,sessioncookie --hooks-enabled-events=pre-finish +tusd -host localhost -port 1080 -upload-dir=/database/tmp -hooks-http=/api/upload/hooks -hooks-http-forward-headers=X-Api-Key,sessioncookie ``` We now need to set up nginx to proxy requests to /api/upload/resumable_upload to our tusd server. @@ -326,20 +325,31 @@ If you serve Galaxy at a prefix exchange `/api/upload/resumable_upload` with `/p After reloading the nginx configuration you can verify that this configuration works correctly by uploading a file to Galaxy. Make sure the tusd server logs the request. It should look similar to the following ```log -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="POST" path="" requestId="" -[tusd] 2021/10/10 10:54:11 event="UploadCreated" id="92ad5d17d0ab0d865991bfef79bbb54d" size="2033" url="http://localhost:1080/files/92ad5d17d0ab0d865991bfef79bbb54d" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="201" method="POST" path="" requestId="" -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="200" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="PATCH" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="ChunkWriteStart" id="92ad5d17d0ab0d865991bfef79bbb54d" maxSize="2033" offset="0" -[tusd] 2021/10/10 10:54:11 event="ChunkWriteComplete" id="92ad5d17d0ab0d865991bfef79bbb54d" bytesWritten="2033" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="204" method="PATCH" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="UploadFinished" id="92ad5d17d0ab0d865991bfef79bbb54d" size="2033" -[tusd] 2021/10/10 10:54:11 event="HookInvocationStart" type="post-finish" id="92ad5d17d0ab0d865991bfef79bbb54d" -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="200" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:12 event="HookInvocationFinish" type="post-finish" id="92ad5d17d0ab0d865991bfef79bbb54d" +[tusd] 2021/10/12 13:30:14 Using '/Users/mvandenb/src/galaxy/database/tmp' as directory storage. +[tusd] 2021/10/12 13:30:14 Using 0.00MB as maximum size. +[tusd] 2021/10/12 13:30:14 Using 'http://localhost:8000/api/upload/hooks' as the endpoint for hooks +[tusd] 2021/10/12 13:30:14 Enabled hook events: pre-create, post-create, post-receive, post-terminate, post-finish +[tusd] 2021/10/12 13:30:14 Using localhost:1080 as address to listen. +[tusd] 2021/10/12 13:30:14 Using /files/ as the base path. +[tusd] 2021/10/12 13:30:14 Using /metrics as the metrics path. +[tusd] 2021/10/12 13:30:14 Supported tus extensions: creation,creation-with-upload,termination,concatenation,creation-defer-length +[tusd] 2021/10/12 13:30:14 You can now upload files to: http://localhost:1080/files/ +[tusd] 2021/10/12 13:30:59 event="RequestIncoming" method="POST" path="" requestId="" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="pre-create" id="" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="pre-create" id="" +[tusd] 2021/10/12 13:30:59 event="UploadCreated" id="b1b16fdf8cd76eb0dc4f86d492424949" size="3670032" url="http://localhost:1080/files/b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="ResponseOutgoing" status="201" method="POST" path="" requestId="" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="post-create" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="post-create" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="RequestIncoming" method="PATCH" path="b1b16fdf8cd76eb0dc4f86d492424949" requestId="" +[tusd] 2021/10/12 13:30:59 event="ChunkWriteStart" id="b1b16fdf8cd76eb0dc4f86d492424949" maxSize="3670032" offset="0" +[tusd] 2021/10/12 13:30:59 event="ChunkWriteComplete" id="b1b16fdf8cd76eb0dc4f86d492424949" bytesWritten="3670032" +[tusd] 2021/10/12 13:30:59 event="ResponseOutgoing" status="204" method="PATCH" path="b1b16fdf8cd76eb0dc4f86d492424949" requestId="" +[tusd] 2021/10/12 13:30:59 event="UploadFinished" id="b1b16fdf8cd76eb0dc4f86d492424949" size="3670032" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="post-finish" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="post-receive" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="post-receive" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="post-finish" id="b1b16fdf8cd76eb0dc4f86d492424949" ``` Note that the tusd server does not need to run on the same host that serves Galaxy. diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index b999ae6e3a5..f024d619e1e 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -243,7 +243,7 @@ toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or pytho tornado==6.1; python_version >= "3.5" tqdm==4.60.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" tuspy==0.2.5 -tuswsgi==0.5.3 +tuswsgi==0.5.4 twill==3.0 typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 47fe8ed8ef6..63758f85615 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -195,7 +195,7 @@ tenacity==7.0.0 tifffile==2020.9.3; python_version >= "3.6" tornado==6.1; python_version >= "3.5" tqdm==4.60.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" -tuswsgi==0.5.3 +tuswsgi==0.5.4 typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.1 diff --git a/lib/galaxy/webapps/galaxy/api/uploads.py b/lib/galaxy/webapps/galaxy/api/uploads.py index bf9fef309ee..43d0b6208ce 100644 --- a/lib/galaxy/webapps/galaxy/api/uploads.py +++ b/lib/galaxy/webapps/galaxy/api/uploads.py @@ -1,13 +1,12 @@ """ API operations for uploaded files in storage. """ -import json import logging import os import re from galaxy import exceptions -from galaxy.web import ( +from galaxy.web.framework.decorators import ( expose_api_raw_anonymous, legacy_expose_api_anonymous, ) @@ -23,57 +22,12 @@ class UploadsAPIController(BaseGalaxyAPIController): READ_CHUNK_SIZE = 2 ** 16 @expose_api_raw_anonymous - def tus(self, trans, payload, session_id=None, **kwargs): + def hooks(self, trans, **kwds): """ - PATCH /api/upload/resumable_upload/{filename} - POST /api/upload/_resumable_upload + Exposed as POST /api/upload/hooks and /api/upload/resumable_upload """ - if isinstance(payload, str): - # WSGI middleware - with open(f"{payload}.info") as info: - payload = json.load(info) - metadata = payload['upload_metadata'] - size = payload['upload_length'] - else: - # tusd server hook - metadata = payload['Upload']['MetaData'] - session_id = payload['Upload']['ID'] - size = payload['Upload']['Size'] - trans.response.headers['upload-offset'] = size - filename = metadata.get('filename', 'Uploaded dataset') - dbkey = metadata.get('dbkey', '?') - history_id = metadata.get('history_id') - if not history_id and not trans.session: - raise exceptions.RequestParameterMissingException("history_id or galaxy session required") - ext = 'auto' - for key in ['ext', 'file_type', 'extension']: - if key in metadata: - ext = metadata[key] - break - _create = trans.webapp.api_controllers['tools']._create - if 'inputs' in metadata: - # Do we want to just dump the "normal" tool payload here, or have some more user-friendly API, or both ? - # Going with both for now since I don't want to redo the client-side of this for now. - # This is the 'normal' payload approach - inputs = json.loads(metadata['inputs']) - inputs['files_0|file_data'] = {"session_id": session_id, "name": inputs["files_0|NAME"]} - tool_id = metadata['tool_id'] - else: - # This feels a bit more user-friendly ? - tool_id = 'upload1' - inputs = { - "file_count": 1, - "dbkey": dbkey, - "file_type": "auto", - "files_0|type": "upload_dataset", - "files_0|NAME": filename, - "files_0|to_posix_lines": "Yes", - "files_0|dbkey": dbkey, - "files_0|file_type": ext, - "files_0|file_data": {"session_id": session_id, "name": filename}} - tool_payload = {'tool_id': tool_id, 'inputs': inputs, 'history_id': history_id} - _create(trans, tool_payload) - trans.response.status = 204 + # Internal endpoint, only purpose is to authenticate user, but may grow additional functionality in the future + return None @legacy_expose_api_anonymous def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 2134a34b31f..09a9496189e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -385,8 +385,9 @@ def populate_api_routes(webapp, app): webapp.mapper.resource('form', 'forms', path_prefix='/api') webapp.mapper.resource('role', 'roles', path_prefix='/api') webapp.mapper.resource('upload', 'uploads', path_prefix='/api') - webapp.mapper.connect('/api/upload/resumable_upload/{session_id}', controller="uploads", action="tus", conditions=dict(method=["PATCH"])) - webapp.mapper.connect('/api/upload/_resumable_upload', controller="uploads", action="tus", conditions=dict(method=["POST"])) + webapp.mapper.connect('/api/upload/resumable_upload/{session_id}', controller="uploads", action="hooks", conditions=dict(method=['PATCH'])) + webapp.mapper.connect('/api/upload/resumable_upload', controller="uploads", action="hooks") + webapp.mapper.connect('/api/upload/hooks', controller="uploads", action="hooks", conditions=dict(method=["POST"])) webapp.mapper.connect('/api/ftp_files', controller='remote_files') webapp.mapper.connect('/api/remote_files', action='index', controller='remote_files', conditions=dict(method=["GET"])) webapp.mapper.connect('/api/remote_files/plugins', action='plugins', controller='remote_files', conditions=dict(method=["GET"])) diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py index 70f9cf29d1e..90824fd3719 100644 --- a/lib/galaxy_test/api/test_tools_upload.py +++ b/lib/galaxy_test/api/test_tools_upload.py @@ -920,10 +920,18 @@ class ToolsUploadTestCase(ApiTestCase): my_client = client.TusClient(url, headers={'x-api-key': api_key}) # Upload a file to a tus server. - my_client.uploader(path, metadata=metadata).upload() + uploader = my_client.uploader(path, metadata=metadata) + uploader.upload() + return uploader.url.rsplit('/', 1)[1] with self.dataset_populator.test_history() as history_id: - upload_file(url=f"{self.url}/api/upload/resumable_upload", path=TestDataResolver().get_filename("1.fastqsanger.gz"), api_key=self.galaxy_interactor.api_key, history_id=history_id) + session_id = upload_file(url=f"{self.url}/api/upload/resumable_upload", path=TestDataResolver().get_filename("1.fastqsanger.gz"), api_key=self.galaxy_interactor.api_key, history_id=history_id) + upload_params = { + 'files_0|NAME': '1.fastqsanger.gz', + 'files_0|file_data': json.dumps({'session_id': session_id, 'name': '1.fastqsanger.gz'}), + 'file_type': 'fastqsanger.gz', + } + self.dataset_populator.run_tool('upload1', history_id=history_id, inputs=upload_params) self.dataset_populator.wait_for_history(history_id, assert_ok=True) dataset = self.dataset_populator.get_history_dataset_details(history_id) assert dataset['name'] == '1.fastqsanger.gz' diff --git a/scripts/resumable_upload.py b/scripts/resumable_upload.py index 2a29020564f..38135a154c4 100644 --- a/scripts/resumable_upload.py +++ b/scripts/resumable_upload.py @@ -1,9 +1,15 @@ +import json import os -import aiotus -import asyncclick as click +import click +import requests +from tusclient import client +from tusclient.storage import filestorage + UPLOAD_ENDPOINT = '/api/upload/resumable_upload' +TOOLS_ENDPOINT = '/api/tools' +CHUNK_SIZE = 10 ** 7 @click.command() @@ -13,25 +19,43 @@ UPLOAD_ENDPOINT = '/api/upload/resumable_upload' @click.option('--file_type', default="auto", type=str, help="Galaxy file type to use") @click.option('--dbkey', default="?", type=str, help="Genome Build for dataset") @click.option('--filename', type=str, help="Filename to use in Galaxy history, if different from path") +@click.option('--storage', type=click.Path(), required=False, help="Store URLs to resume here") @click.argument('path', type=click.Path()) -async def upload_file(url, path, api_key, history_id, file_type='auto', dbkey='?', filename=None): +def upload_file(url, path, api_key, history_id, file_type='auto', dbkey='?', filename=None, storage=None): + headers = {'x-api-key': api_key} + my_client = client.TusClient(f"{url}{UPLOAD_ENDPOINT}", headers=headers) filename = filename or os.path.basename(path) metadata = { - 'filename': filename.encode(), - 'history_id': history_id.encode(), - 'file_type': file_type.encode(), - 'dbkey': dbkey.encode(), + 'filename': filename, + 'history_id': history_id, + 'file_type': file_type, + 'dbkey': dbkey, } - headers = {'x-api-key': api_key} # Upload a file to a tus server. - with open(path, "rb") as f: - location = await aiotus.upload(f"{url}{UPLOAD_ENDPOINT}", f, metadata, headers=headers) - # 'location' is the URL where the file was uploaded to. + if storage: + storage = filestorage.FileStorage(storage) + uploader = my_client.uploader(path, metadata=metadata, url_storage=storage) + uploader.chunk_size = CHUNK_SIZE + uploader.upload() - # Read back the metadata from the server. - metadata = await aiotus.metadata(location, headers=headers) - print(metadata) + # Extract session from created upload URL + session_id = uploader.url.rsplit('/', 1)[1] + # This feels a bit more user-friendly ? + tool_id = 'upload1' + inputs = { + "file_count": 1, + "dbkey": dbkey, + "file_type": "auto", + "files_0|type": "upload_dataset", + "files_0|NAME": filename, + "files_0|to_posix_lines": "Yes", + "files_0|dbkey": dbkey, + "files_0|file_type": file_type, + "files_0|file_data": {"session_id": session_id, "name": filename}} + tool_payload = {'tool_id': tool_id, 'inputs': inputs, 'history_id': history_id} + response = requests.post(f"{url}{TOOLS_ENDPOINT}", data=json.dumps(tool_payload), headers=headers) + response.raise_for_status() if __name__ == '__main__':