diff --git a/client/src/utils/uploadbox.js b/client/src/utils/uploadbox.js index f4464b947ae..968a3c20d46 100644 --- a/client/src/utils/uploadbox.js +++ b/client/src/utils/uploadbox.js @@ -116,10 +116,28 @@ import * as tus from "tus-js-client"; }, onSuccess: function () { console.log( - `Download ${upload.file.name} from ${upload.url} took ${ + `Upload of ${upload.file.name} to ${upload.url} took ${ (performance.now() - startTime) / 1000 } seconds` ); + const toolInputs = JSON.parse(data.payload.inputs); + toolInputs["files_0|file_data"] = { + session_id: upload.url.split("/").at(-1), + name: file.name, + }; + data.payload.inputs = JSON.stringify(toolInputs); + $.ajax({ + url: `${getAppRoot()}api/tools`, + method: "POST", + data: data.payload, + success: (tool_response) => { + cnf.success(tool_response); + }, + error: (tool_response) => { + var err_msg = tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg; + cnf.error(err_msg || cnf.error_tool); + }, + }); cnf.success({}); }, }); diff --git a/doc/source/admin/nginx.md b/doc/source/admin/nginx.md index 43b047748de..b268b2abde5 100644 --- a/doc/source/admin/nginx.md +++ b/doc/source/admin/nginx.md @@ -283,21 +283,20 @@ group as shown above. Galaxy includes a WSGI middleware that implements a tus server for which no configuration is needed. However the middleware ties up resources on the Galaxy server process, and uploads will be interrupted while Galaxy restarts. A more efficient alternative is to run an external server that implements the tus protocol. -Any tus server that can send the pre-finish hook to a URL will work. Here we will use [tusd](https://github.com/tus/tusd). +Any tus server will work. Here we will use [tusd](https://github.com/tus/tusd). Binaries can be downloaded from https://github.com/tus/tusd/releases/. In this example we will set up tusd to: - listen on port 1080 on localhost (`-host localhost -port 1080`) - store uploads in database/tmp (replace this with the value of new_file_path in your galaxy.yml config) (`-upload-dir=/database/tmp`) -- send an event via http to /api/upload/_resumable_upload when an upload has finished (`-hooks-http=/api/upload/_resumable_upload`) +- send an event via http to /api/upload/hooks to ensure the user is logged in (`-hooks-http=/api/upload/hooks`) - forward authentication headers in that event (`-hooks-http-forward-headers=X-Api-Key,sessioncookie`) -- and enable only the pre-finish event hook (`-hooks-enabled-events=pre-finish`) The complete command is thus (replace `` with your Galaxy URL and `` with the path to your Galaxy installation): ```sh -tusd -host localhost -port 1080 -upload-dir=/database/tmp -hooks-http=/api/upload/_resumable_upload -hooks-http-forward-headers=X-Api-Key,sessioncookie --hooks-enabled-events=pre-finish +tusd -host localhost -port 1080 -upload-dir=/database/tmp -hooks-http=/api/upload/hooks -hooks-http-forward-headers=X-Api-Key,sessioncookie ``` We now need to set up nginx to proxy requests to /api/upload/resumable_upload to our tusd server. @@ -326,20 +325,31 @@ If you serve Galaxy at a prefix exchange `/api/upload/resumable_upload` with `/p After reloading the nginx configuration you can verify that this configuration works correctly by uploading a file to Galaxy. Make sure the tusd server logs the request. It should look similar to the following ```log -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="POST" path="" requestId="" -[tusd] 2021/10/10 10:54:11 event="UploadCreated" id="92ad5d17d0ab0d865991bfef79bbb54d" size="2033" url="http://localhost:1080/files/92ad5d17d0ab0d865991bfef79bbb54d" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="201" method="POST" path="" requestId="" -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="200" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="PATCH" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="ChunkWriteStart" id="92ad5d17d0ab0d865991bfef79bbb54d" maxSize="2033" offset="0" -[tusd] 2021/10/10 10:54:11 event="ChunkWriteComplete" id="92ad5d17d0ab0d865991bfef79bbb54d" bytesWritten="2033" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="204" method="PATCH" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="UploadFinished" id="92ad5d17d0ab0d865991bfef79bbb54d" size="2033" -[tusd] 2021/10/10 10:54:11 event="HookInvocationStart" type="post-finish" id="92ad5d17d0ab0d865991bfef79bbb54d" -[tusd] 2021/10/10 10:54:11 event="RequestIncoming" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:11 event="ResponseOutgoing" status="200" method="HEAD" path="92ad5d17d0ab0d865991bfef79bbb54d" requestId="" -[tusd] 2021/10/10 10:54:12 event="HookInvocationFinish" type="post-finish" id="92ad5d17d0ab0d865991bfef79bbb54d" +[tusd] 2021/10/12 13:30:14 Using '/Users/mvandenb/src/galaxy/database/tmp' as directory storage. +[tusd] 2021/10/12 13:30:14 Using 0.00MB as maximum size. +[tusd] 2021/10/12 13:30:14 Using 'http://localhost:8000/api/upload/hooks' as the endpoint for hooks +[tusd] 2021/10/12 13:30:14 Enabled hook events: pre-create, post-create, post-receive, post-terminate, post-finish +[tusd] 2021/10/12 13:30:14 Using localhost:1080 as address to listen. +[tusd] 2021/10/12 13:30:14 Using /files/ as the base path. +[tusd] 2021/10/12 13:30:14 Using /metrics as the metrics path. +[tusd] 2021/10/12 13:30:14 Supported tus extensions: creation,creation-with-upload,termination,concatenation,creation-defer-length +[tusd] 2021/10/12 13:30:14 You can now upload files to: http://localhost:1080/files/ +[tusd] 2021/10/12 13:30:59 event="RequestIncoming" method="POST" path="" requestId="" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="pre-create" id="" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="pre-create" id="" +[tusd] 2021/10/12 13:30:59 event="UploadCreated" id="b1b16fdf8cd76eb0dc4f86d492424949" size="3670032" url="http://localhost:1080/files/b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="ResponseOutgoing" status="201" method="POST" path="" requestId="" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="post-create" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="post-create" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="RequestIncoming" method="PATCH" path="b1b16fdf8cd76eb0dc4f86d492424949" requestId="" +[tusd] 2021/10/12 13:30:59 event="ChunkWriteStart" id="b1b16fdf8cd76eb0dc4f86d492424949" maxSize="3670032" offset="0" +[tusd] 2021/10/12 13:30:59 event="ChunkWriteComplete" id="b1b16fdf8cd76eb0dc4f86d492424949" bytesWritten="3670032" +[tusd] 2021/10/12 13:30:59 event="ResponseOutgoing" status="204" method="PATCH" path="b1b16fdf8cd76eb0dc4f86d492424949" requestId="" +[tusd] 2021/10/12 13:30:59 event="UploadFinished" id="b1b16fdf8cd76eb0dc4f86d492424949" size="3670032" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="post-finish" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationStart" type="post-receive" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="post-receive" id="b1b16fdf8cd76eb0dc4f86d492424949" +[tusd] 2021/10/12 13:30:59 event="HookInvocationFinish" type="post-finish" id="b1b16fdf8cd76eb0dc4f86d492424949" ``` Note that the tusd server does not need to run on the same host that serves Galaxy. diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index b999ae6e3a5..f024d619e1e 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -243,7 +243,7 @@ toml==0.10.2; python_version >= "3.6" and python_full_version < "3.0.0" or pytho tornado==6.1; python_version >= "3.5" tqdm==4.60.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" tuspy==0.2.5 -tuswsgi==0.5.3 +tuswsgi==0.5.4 twill==3.0 typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 47fe8ed8ef6..63758f85615 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -195,7 +195,7 @@ tenacity==7.0.0 tifffile==2020.9.3; python_version >= "3.6" tornado==6.1; python_version >= "3.5" tqdm==4.60.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0" -tuswsgi==0.5.3 +tuswsgi==0.5.4 typing-extensions==3.7.4.3; python_version >= "3.6" and python_version < "3.8" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.1 diff --git a/lib/galaxy/webapps/galaxy/api/uploads.py b/lib/galaxy/webapps/galaxy/api/uploads.py index bf9fef309ee..43d0b6208ce 100644 --- a/lib/galaxy/webapps/galaxy/api/uploads.py +++ b/lib/galaxy/webapps/galaxy/api/uploads.py @@ -1,13 +1,12 @@ """ API operations for uploaded files in storage. """ -import json import logging import os import re from galaxy import exceptions -from galaxy.web import ( +from galaxy.web.framework.decorators import ( expose_api_raw_anonymous, legacy_expose_api_anonymous, ) @@ -23,57 +22,12 @@ class UploadsAPIController(BaseGalaxyAPIController): READ_CHUNK_SIZE = 2 ** 16 @expose_api_raw_anonymous - def tus(self, trans, payload, session_id=None, **kwargs): + def hooks(self, trans, **kwds): """ - PATCH /api/upload/resumable_upload/{filename} - POST /api/upload/_resumable_upload + Exposed as POST /api/upload/hooks and /api/upload/resumable_upload """ - if isinstance(payload, str): - # WSGI middleware - with open(f"{payload}.info") as info: - payload = json.load(info) - metadata = payload['upload_metadata'] - size = payload['upload_length'] - else: - # tusd server hook - metadata = payload['Upload']['MetaData'] - session_id = payload['Upload']['ID'] - size = payload['Upload']['Size'] - trans.response.headers['upload-offset'] = size - filename = metadata.get('filename', 'Uploaded dataset') - dbkey = metadata.get('dbkey', '?') - history_id = metadata.get('history_id') - if not history_id and not trans.session: - raise exceptions.RequestParameterMissingException("history_id or galaxy session required") - ext = 'auto' - for key in ['ext', 'file_type', 'extension']: - if key in metadata: - ext = metadata[key] - break - _create = trans.webapp.api_controllers['tools']._create - if 'inputs' in metadata: - # Do we want to just dump the "normal" tool payload here, or have some more user-friendly API, or both ? - # Going with both for now since I don't want to redo the client-side of this for now. - # This is the 'normal' payload approach - inputs = json.loads(metadata['inputs']) - inputs['files_0|file_data'] = {"session_id": session_id, "name": inputs["files_0|NAME"]} - tool_id = metadata['tool_id'] - else: - # This feels a bit more user-friendly ? - tool_id = 'upload1' - inputs = { - "file_count": 1, - "dbkey": dbkey, - "file_type": "auto", - "files_0|type": "upload_dataset", - "files_0|NAME": filename, - "files_0|to_posix_lines": "Yes", - "files_0|dbkey": dbkey, - "files_0|file_type": ext, - "files_0|file_data": {"session_id": session_id, "name": filename}} - tool_payload = {'tool_id': tool_id, 'inputs': inputs, 'history_id': history_id} - _create(trans, tool_payload) - trans.response.status = 204 + # Internal endpoint, only purpose is to authenticate user, but may grow additional functionality in the future + return None @legacy_expose_api_anonymous def index(self, trans, **kwd): diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 2134a34b31f..09a9496189e 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -385,8 +385,9 @@ def populate_api_routes(webapp, app): webapp.mapper.resource('form', 'forms', path_prefix='/api') webapp.mapper.resource('role', 'roles', path_prefix='/api') webapp.mapper.resource('upload', 'uploads', path_prefix='/api') - webapp.mapper.connect('/api/upload/resumable_upload/{session_id}', controller="uploads", action="tus", conditions=dict(method=["PATCH"])) - webapp.mapper.connect('/api/upload/_resumable_upload', controller="uploads", action="tus", conditions=dict(method=["POST"])) + webapp.mapper.connect('/api/upload/resumable_upload/{session_id}', controller="uploads", action="hooks", conditions=dict(method=['PATCH'])) + webapp.mapper.connect('/api/upload/resumable_upload', controller="uploads", action="hooks") + webapp.mapper.connect('/api/upload/hooks', controller="uploads", action="hooks", conditions=dict(method=["POST"])) webapp.mapper.connect('/api/ftp_files', controller='remote_files') webapp.mapper.connect('/api/remote_files', action='index', controller='remote_files', conditions=dict(method=["GET"])) webapp.mapper.connect('/api/remote_files/plugins', action='plugins', controller='remote_files', conditions=dict(method=["GET"])) diff --git a/lib/galaxy_test/api/test_tools_upload.py b/lib/galaxy_test/api/test_tools_upload.py index 70f9cf29d1e..90824fd3719 100644 --- a/lib/galaxy_test/api/test_tools_upload.py +++ b/lib/galaxy_test/api/test_tools_upload.py @@ -920,10 +920,18 @@ class ToolsUploadTestCase(ApiTestCase): my_client = client.TusClient(url, headers={'x-api-key': api_key}) # Upload a file to a tus server. - my_client.uploader(path, metadata=metadata).upload() + uploader = my_client.uploader(path, metadata=metadata) + uploader.upload() + return uploader.url.rsplit('/', 1)[1] with self.dataset_populator.test_history() as history_id: - upload_file(url=f"{self.url}/api/upload/resumable_upload", path=TestDataResolver().get_filename("1.fastqsanger.gz"), api_key=self.galaxy_interactor.api_key, history_id=history_id) + session_id = upload_file(url=f"{self.url}/api/upload/resumable_upload", path=TestDataResolver().get_filename("1.fastqsanger.gz"), api_key=self.galaxy_interactor.api_key, history_id=history_id) + upload_params = { + 'files_0|NAME': '1.fastqsanger.gz', + 'files_0|file_data': json.dumps({'session_id': session_id, 'name': '1.fastqsanger.gz'}), + 'file_type': 'fastqsanger.gz', + } + self.dataset_populator.run_tool('upload1', history_id=history_id, inputs=upload_params) self.dataset_populator.wait_for_history(history_id, assert_ok=True) dataset = self.dataset_populator.get_history_dataset_details(history_id) assert dataset['name'] == '1.fastqsanger.gz' diff --git a/scripts/resumable_upload.py b/scripts/resumable_upload.py index 2a29020564f..38135a154c4 100644 --- a/scripts/resumable_upload.py +++ b/scripts/resumable_upload.py @@ -1,9 +1,15 @@ +import json import os -import aiotus -import asyncclick as click +import click +import requests +from tusclient import client +from tusclient.storage import filestorage + UPLOAD_ENDPOINT = '/api/upload/resumable_upload' +TOOLS_ENDPOINT = '/api/tools' +CHUNK_SIZE = 10 ** 7 @click.command() @@ -13,25 +19,43 @@ UPLOAD_ENDPOINT = '/api/upload/resumable_upload' @click.option('--file_type', default="auto", type=str, help="Galaxy file type to use") @click.option('--dbkey', default="?", type=str, help="Genome Build for dataset") @click.option('--filename', type=str, help="Filename to use in Galaxy history, if different from path") +@click.option('--storage', type=click.Path(), required=False, help="Store URLs to resume here") @click.argument('path', type=click.Path()) -async def upload_file(url, path, api_key, history_id, file_type='auto', dbkey='?', filename=None): +def upload_file(url, path, api_key, history_id, file_type='auto', dbkey='?', filename=None, storage=None): + headers = {'x-api-key': api_key} + my_client = client.TusClient(f"{url}{UPLOAD_ENDPOINT}", headers=headers) filename = filename or os.path.basename(path) metadata = { - 'filename': filename.encode(), - 'history_id': history_id.encode(), - 'file_type': file_type.encode(), - 'dbkey': dbkey.encode(), + 'filename': filename, + 'history_id': history_id, + 'file_type': file_type, + 'dbkey': dbkey, } - headers = {'x-api-key': api_key} # Upload a file to a tus server. - with open(path, "rb") as f: - location = await aiotus.upload(f"{url}{UPLOAD_ENDPOINT}", f, metadata, headers=headers) - # 'location' is the URL where the file was uploaded to. + if storage: + storage = filestorage.FileStorage(storage) + uploader = my_client.uploader(path, metadata=metadata, url_storage=storage) + uploader.chunk_size = CHUNK_SIZE + uploader.upload() - # Read back the metadata from the server. - metadata = await aiotus.metadata(location, headers=headers) - print(metadata) + # Extract session from created upload URL + session_id = uploader.url.rsplit('/', 1)[1] + # This feels a bit more user-friendly ? + tool_id = 'upload1' + inputs = { + "file_count": 1, + "dbkey": dbkey, + "file_type": "auto", + "files_0|type": "upload_dataset", + "files_0|NAME": filename, + "files_0|to_posix_lines": "Yes", + "files_0|dbkey": dbkey, + "files_0|file_type": file_type, + "files_0|file_data": {"session_id": session_id, "name": filename}} + tool_payload = {'tool_id': tool_id, 'inputs': inputs, 'history_id': history_id} + response = requests.post(f"{url}{TOOLS_ENDPOINT}", data=json.dumps(tool_payload), headers=headers) + response.raise_for_status() if __name__ == '__main__':