diff --git a/client/galaxy/scripts/apps/panels/tool-panel.js b/client/galaxy/scripts/apps/panels/tool-panel.js index 0a2cead12b9..c7ba4027c95 100644 --- a/client/galaxy/scripts/apps/panels/tool-panel.js +++ b/client/galaxy/scripts/apps/panels/tool-panel.js @@ -2,6 +2,7 @@ import Tools from "mvc/tool/tools"; import Upload from "mvc/upload/upload-view"; import _l from "utils/localization"; import ToolForm from "mvc/tool/tool-form-composite"; +import _ from "libs/underscore"; var ToolPanel = Backbone.View.extend({ initialize: function(page, options) { @@ -28,7 +29,8 @@ var ToolPanel = Backbone.View.extend({ // add upload modal this.upload_button = new Upload({ - nginx_upload_path: config.nginx_upload_path, + upload_path: config.nginx_upload_path || `${Galaxy.root}api/tools`, + chunk_upload_size: config.chunk_upload_size, ftp_upload_site: config.ftp_upload_site, default_genome: config.default_genome, default_extension: config.default_extension @@ -74,70 +76,54 @@ var ToolPanel = Backbone.View.extend({ /** build a link to one tool */ _templateTool: function(tool) { - return [ - '
', - '', - tool.title, - "", - "
" - ].join(""); + return `
+ + ${tool.title} + +
`; }, /** build a link to 'All Workflows' */ _templateAllWorkflow: function(tool) { - return [ - '
', - // global - '', - tool.title, - "", - "
" - ].join(""); + return `
+ + ${tool.title} + +
`; }, /** build links to workflows in toolpanel */ _templateWorkflowLink: function(wf) { - return [ - '
', - `', - wf.title, - "", - "
" - ].join(""); + return `
+ + ${_.escape(wf.title)} + +
`; }, /** override to include inital menu dom and workflow section */ _template: function() { - return [ - '
', - '", - '
', - '
', - '
', - "", - _l("Workflows"), - "", - "
", - '
', - '
', - "
", - "
" - ].join(""); + return `
+ +
+
+
+ + ${_l("Workflows")} + +
+
+
+
+
`; }, toString: function() { diff --git a/client/galaxy/scripts/mvc/grid/grid-template.js b/client/galaxy/scripts/mvc/grid/grid-template.js index 0bb8c2ba7d6..63e407e16ac 100644 --- a/client/galaxy/scripts/mvc/grid/grid-template.js +++ b/client/galaxy/scripts/mvc/grid/grid-template.js @@ -93,7 +93,9 @@ export default { if (column.visible) { tmpl += ``; if (column.sortable) { - tmpl += `${column.label}`; + tmpl += `${ + column.label + }`; } else { tmpl += column.label; } diff --git a/client/galaxy/scripts/mvc/upload/collection/collection-view.js b/client/galaxy/scripts/mvc/upload/collection/collection-view.js index 817bcaef646..b8744dcbe69 100644 --- a/client/galaxy/scripts/mvc/upload/collection/collection-view.js +++ b/client/galaxy/scripts/mvc/upload/collection/collection-view.js @@ -116,7 +116,7 @@ export default Backbone.View.extend({ // file upload this.uploadbox = this.$(".upload-box").uploadbox({ - url: this.app.options.nginx_upload_path, + url: this.app.options.upload_path, announce: function(index, file) { self._eventAnnounce(index, file); }, diff --git a/client/galaxy/scripts/mvc/upload/composite/composite-view.js b/client/galaxy/scripts/mvc/upload/composite/composite-view.js index 928de5382ab..830dce6cb5f 100644 --- a/client/galaxy/scripts/mvc/upload/composite/composite-view.js +++ b/client/galaxy/scripts/mvc/upload/composite/composite-view.js @@ -133,7 +133,7 @@ export default Backbone.View.extend({ }); }); $.uploadpost({ - url: this.app.options.nginx_upload_path, + url: this.app.options.upload_path, data: this.app.toData(this.collection.filter()), success: function(message) { self._eventSuccess(message); diff --git a/client/galaxy/scripts/mvc/upload/default/default-row.js b/client/galaxy/scripts/mvc/upload/default/default-row.js index 4f6d5c5fe3b..054c2cba775 100644 --- a/client/galaxy/scripts/mvc/upload/default/default-row.js +++ b/client/galaxy/scripts/mvc/upload/default/default-row.js @@ -11,6 +11,7 @@ export default Backbone.View.extend({ init: "upload-icon-button fa fa-trash-o", queued: "upload-icon fa fa-spinner fa-spin", running: "upload-icon fa fa-spinner fa-spin", + warning: "upload-icon fa fa-spinner fa-spin", success: "upload-icon-button fa fa-check", error: "upload-icon-button fa fa-exclamation-triangle" }, @@ -183,7 +184,7 @@ export default Backbone.View.extend({ _refreshInfo: function() { var info = this.model.get("info"); if (info) { - this.$info_text.html(`Failed: ${info}`).show(); + this.$info_text.html(`Warning: ${info}`).show(); } else { this.$info_text.hide(); } @@ -213,13 +214,17 @@ export default Backbone.View.extend({ this.select_genome.disable(); this.select_extension.disable(); } + this.$info_progress.show(); + this.$el.removeClass().addClass("upload-row"); if (status == "success") { this.$el.addClass("success"); this.$percentage.html("100%"); - } - if (status == "error") { + } else if (status == "error") { this.$el.addClass("danger"); this.$info_progress.hide(); + } else if (status == "warning") { + this.$el.addClass("warning"); + this.$info_progress.hide(); } }, diff --git a/client/galaxy/scripts/mvc/upload/default/default-view.js b/client/galaxy/scripts/mvc/upload/default/default-view.js index 741b44a55ab..ede97d8b757 100644 --- a/client/galaxy/scripts/mvc/upload/default/default-view.js +++ b/client/galaxy/scripts/mvc/upload/default/default-view.js @@ -104,7 +104,7 @@ export default Backbone.View.extend({ // file upload this.uploadbox = this.$uploadbox.uploadbox({ - url: this.app.options.nginx_upload_path, + url: this.app.options.upload_path, announce: function(index, file) { self._eventAnnounce(index, file); }, @@ -120,6 +120,9 @@ export default Backbone.View.extend({ error: function(index, message) { self._eventError(index, message); }, + warning: function(index, message) { + self._eventWarning(index, message); + }, complete: function() { self._eventComplete(); }, @@ -247,7 +250,7 @@ export default Backbone.View.extend({ /** Progress */ _eventProgress: function(index, percentage) { var it = this.collection.get(index); - it.set("percentage", percentage); + it.set({ percentage: percentage, status: "running", info: "" }); this.ui_button.model.set("percentage", this._uploadPercentage(percentage, it.get("file_size"))); }, @@ -263,6 +266,12 @@ export default Backbone.View.extend({ Galaxy.currHistoryPanel.refreshContents(); }, + /** Warning */ + _eventWarning: function(index, message) { + var it = this.collection.get(index); + it.set({ status: "warning", info: message }); + }, + /** Error */ _eventError: function(index, message) { var it = this.collection.get(index); @@ -363,7 +372,10 @@ export default Backbone.View.extend({ this._uploadFtp(); // queue remaining files - this.uploadbox.start(); + this.uploadbox.start({ + id: Galaxy.user.id, + chunk_upload_size: this.app.options.chunk_upload_size + }); this.render(); } }, @@ -430,7 +442,7 @@ export default Backbone.View.extend({ if (list.length > 0) { $.uploadpost({ data: this.app.toData(list), - url: this.app.options.nginx_upload_path, + url: this.app.options.upload_path, success: function(message) { _.each(list, model => { self._eventSuccess(model.id); @@ -452,37 +464,37 @@ export default Backbone.View.extend({ /** Template */ _template: function() { - return ( - '
' + - '
' + - '
' + - "
" + - '
' + - '
Drop files here
' + - '' + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "" + - "
" + - '
' + - 'Type (set all):' + - '' + - ' ' + - 'Genome (set all):' + - '' + - "
" + - '
' + - "
" - ); + return `
+
+
+
+
+
+ Drop files here +
+ + + + + + + + + + + + + +
+
+ Type (set all): + + + Genome (set all): + +
+
+
`; } }); diff --git a/client/galaxy/scripts/mvc/upload/upload-view.js b/client/galaxy/scripts/mvc/upload/upload-view.js index 14272498fff..5f4684f53e7 100644 --- a/client/galaxy/scripts/mvc/upload/upload-view.js +++ b/client/galaxy/scripts/mvc/upload/upload-view.js @@ -9,7 +9,6 @@ import UploadViewComposite from "mvc/upload/composite/composite-view"; import UploadViewCollection from "mvc/upload/collection/collection-view"; export default Backbone.View.extend({ options: { - nginx_upload_path: "", ftp_upload_site: "n/a", default_genome: "?", default_extension: "auto", diff --git a/client/galaxy/scripts/utils/uploadbox.js b/client/galaxy/scripts/utils/uploadbox.js index bbd2a1c0590..44faa6e7025 100644 --- a/client/galaxy/scripts/utils/uploadbox.js +++ b/client/galaxy/scripts/utils/uploadbox.js @@ -6,81 +6,29 @@ jQuery.event.props.push("dataTransfer"); /** - Posts file data to the API + xhr request helper */ - $.uploadpost = config => { - // parse options + var _uploadrequest = config => { var cnf = $.extend( - {}, { - data: {}, - success: function() {}, - error: function() {}, - progress: function() {}, - url: null, - maxfilesize: 2048, - error_filesize: "File exceeds 2GB. Please use a FTP client.", error_default: "Please make sure the file is available.", error_server: "Upload request failed.", - error_login: "Uploads require you to log in." + error_login: "Uploads require you to log in.", + error_retry: "Waiting for server to resume..." }, config ); - - // link data - var data = cnf.data; - - // check errors - if (data.error_message) { - cnf.error(data.error_message); - return; - } - - // construct form data - var form = new FormData(); - for (var key in data.payload) { - form.append(key, data.payload[key]); - } - - // add files to submission - var sizes = 0; - for (var key in data.files) { - var d = data.files[key]; - form.append(d.name, d.file, d.file.name); - sizes += d.file.size; - } - - // check file size, unless it's an ftp file - if (sizes > 1048576 * cnf.maxfilesize) { - cnf.error(cnf.error_filesize); - return; - } - - // prepare request + console.debug(cnf); var xhr = new XMLHttpRequest(); xhr.open("POST", cnf.url, true); - xhr.setRequestHeader("Accept", "application/json"); xhr.setRequestHeader("Cache-Control", "no-cache"); xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest"); - - // captures state changes + xhr.setRequestHeader("Accept", "application/json"); xhr.onreadystatechange = () => { - // check for request completed, server connection closed if (xhr.readyState == xhr.DONE) { - // parse response - var response = null; - var extra_info = ""; - if (xhr.responseText) { - try { - response = jQuery.parseJSON(xhr.responseText); - extra_info = response.err_msg; - } catch (e) { - response = xhr.responseText; - extra_info = response; - } - } - // pass any error to the error option - if (xhr.status < 200 || xhr.status > 299) { + if ([502, 0].indexOf(xhr.status) !== -1 && cnf.warning) { + cnf.warning(cnf.error_retry); + } else if (xhr.status < 200 || xhr.status > 299) { var text = xhr.statusText; if (xhr.status == 403) { text = cnf.error_login; @@ -89,27 +37,191 @@ } else if (!text) { text = cnf.error_default; } - cnf.error(`${text} (${xhr.status}). ${extra_info}`); + cnf.error(`${text} (${xhr.status})`); } else { + var response = null; + if (xhr.responseText) { + try { + response = jQuery.parseJSON(xhr.responseText); + } catch (e) { + response = xhr.responseText; + } + } cnf.success(response); } } }; + xhr.upload.addEventListener("progress", cnf.progress, false); + xhr.send(cnf.data); + }; - // prepare upload progress - xhr.upload.addEventListener( - "progress", - e => { + /** + Posts chunked files to the API. + */ + $.uploadchunk = function(config) { + // parse options + var cnf = $.extend( + {}, + { + data: {}, + success: () => {}, + error: () => {}, + warning: () => {}, + progress: () => {}, + attempts: 70000, + timeout: 5000, + url: null, + error_file: "File not provided.", + error_attempt: "Maximum number of attempts reached.", + error_tool: "Tool submission failed." + }, + config + ); + + // initial validation + var data = cnf.data; + if (data.error_message) { + cnf.error(data.error_message); + return; + } + var file_data = data.files && data.files[0]; + if (!file_data) { + cnf.error(cnf.error_file); + return; + } + var file = file_data.file; + var attempts = cnf.attempts; + var session_id = `${cnf.session.id}-${new Date().valueOf()}-${file.size}`; + var chunk_size = cnf.session.chunk_upload_size; + console.debug(`Starting chunked uploads [size=${chunk_size}].`); + + // chunk processing helper + function process(start) { + start = start || 0; + var slicer = file.mozSlice || file.webkitSlice || file.slice; + if (!slicer) { + cnf.error("Browser does not support chunked uploads."); + return; + } + var end = Math.min(start + chunk_size, file.size); + var size = file.size; + console.debug(`Submitting chunk at ${start} bytes...`); + var form = new FormData(); + form.append("session_id", session_id); + form.append("session_start", start); + form.append("session_chunk", slicer.bind(file)(start, end)); + _uploadrequest({ + url: `${Galaxy.root}api/uploads`, + data: form, + success: upload_response => { + var new_start = start + chunk_size; + if (new_start < size) { + attempts = cnf.attempts; + process(new_start); + } else { + console.debug("Upload completed."); + data.payload.inputs = JSON.parse(data.payload.inputs); + data.payload.inputs["files_0|file_data"] = { + session_id: session_id, + name: file.name + }; + data.payload.inputs = JSON.stringify(data.payload.inputs); + $.ajax({ + url: `${Galaxy.root}api/tools`, + method: "POST", + data: data.payload, + success: tool_response => { + cnf.success(tool_response); + }, + error: tool_response => { + var err_msg = + tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg; + cnf.error(err_msg || cnf.error_tool); + } + }); + } + }, + warning: upload_response => { + if (--attempts > 0) { + console.debug("Retrying last chunk..."); + cnf.warning(upload_response); + setTimeout(() => process(start), cnf.timeout); + } else { + console.debug(cnf.error_attempt); + cnf.error(cnf.error_attempt); + } + }, + error: upload_response => { + console.debug(upload_response); + cnf.error(upload_response); + }, + progress: e => { + if (e.lengthComputable) { + cnf.progress(Math.min(Math.round((start + e.loaded) * 100 / file.size), 100)); + } + } + }); + } + + // initiate processing queue for chunks + process(); + }; + + /** + Posts multiple files without chunking to the API. + */ + $.uploadpost = function(config) { + var cnf = $.extend( + {}, + { + data: {}, + success: () => {}, + error: () => {}, + progress: () => {}, + url: null, + maxfilesize: 1048576 * 2048, + error_filesize: "File exceeds 2GB. Please use a FTP client." + }, + config + ); + var data = cnf.data; + if (data.error_message) { + cnf.error(data.error_message); + return; + } + + // construct form data + var form = new FormData(); + for (let key in data.payload) { + form.append(key, data.payload[key]); + } + + // add files to submission + var sizes = 0; + for (let key in data.files) { + var d = data.files[key]; + form.append(d.name, d.file, d.file.name); + sizes += d.file.size; + } + + // check file size, unless it's an ftp file + if (sizes > cnf.maxfilesize) { + cnf.error(cnf.error_filesize); + return; + } + + // submit request + _uploadrequest({ + url: cnf.url, + data: form, + success: cnf.success, + error: cnf.error, + progress: e => { if (e.lengthComputable) { cnf.progress(Math.round(e.loaded * 100 / e.total)); } - }, - false - ); - - // send request - Galaxy.emit.debug("uploadbox::uploadpost()", "Posting following data.", cnf); - xhr.send(form); + } + }); }; /** @@ -121,9 +233,9 @@ var opts = $.extend( {}, { - ondragover: function() {}, - ondragleave: function() {}, - onchange: function() {}, + ondragover: () => {}, + ondragleave: () => {}, + onchange: () => {}, multiple: false }, options @@ -132,7 +244,7 @@ // append hidden upload field var $input = $(``); el.append( - $input.change(function(e) { + $input.change(e => { opts.onchange(e.target.files); $(this).val(""); }) @@ -157,7 +269,7 @@ // exports return { - dialog: function() { + dialog: () => { $input.trigger("click"); } }; @@ -171,16 +283,17 @@ var opts = $.extend( {}, { - dragover: function() {}, - dragleave: function() {}, - announce: function(d) {}, - initialize: function(d) {}, - progress: function(d, m) {}, - success: function(d, m) {}, - error: function(d, m) { + dragover: () => {}, + dragleave: () => {}, + announce: d => {}, + initialize: d => {}, + progress: (d, m) => {}, + success: (d, m) => {}, + warning: (d, m) => {}, + error: (d, m) => { alert(m); }, - complete: function() {} + complete: () => {} }, options ); @@ -188,6 +301,9 @@ // file queue var queue = {}; + // session options + var session = null; + // queue index/length counter var queue_index = 0; var queue_length = 0; @@ -199,7 +315,10 @@ // element var uploadinput = $(this).uploadinput({ multiple: true, - onchange: function(files) { + onchange: files => { + _.each(files, file => { + file.chunk_mode = true; + }); add(files); }, ondragover: options.ondragover, @@ -252,7 +371,7 @@ // get an identifier from the queue var index = -1; - for (var key in queue) { + for (let key in queue) { index = key; break; } @@ -264,18 +383,32 @@ remove(index); // create and submit data - $.uploadpost({ + var submitter = $.uploadpost; + if ( + file.chunk_mode && + session && + session.id && + session.chunk_upload_size && + session.chunk_upload_size > 0 + ) { + submitter = $.uploadchunk; + } + submitter({ url: opts.url, data: opts.initialize(index), - success: function(message) { + session: session, + success: message => { opts.success(index, message); process(); }, - error: function(message) { + warning: message => { + opts.warning(index, message); + }, + error: message => { opts.error(index, message); process(); }, - progress: function(percentage) { + progress: percentage => { opts.progress(index, percentage); } }); @@ -298,7 +431,8 @@ } // initiate upload process - function start() { + function start(_session) { + session = _session; if (!queue_running) { queue_running = true; process(); diff --git a/config/galaxy.yml.sample b/config/galaxy.yml.sample index 6112840f1d7..6951fc2762e 100644 --- a/config/galaxy.yml.sample +++ b/config/galaxy.yml.sample @@ -812,6 +812,11 @@ galaxy: # documentation for the corresponding nginx configuration. #nginx_upload_job_files_path: false + # Galaxy can upload user files in chunks without using nginx. Enable + # the chunk uploader by specifying a chunk size larger than 0. The + # chunk size is specified in bytes (default: 100MB). + #chunk_upload_size: 104857600 + # Have Galaxy manage dynamic proxy component for routing requests to # other services based on Galaxy's session cookie. It will attempt to # do this by default though you do need to install node+npm and do an diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index adb91a41fff..c77b764a514 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -1643,6 +1643,18 @@ :Type: bool +~~~~~~~~~~~~~~~~~~~~~ +``chunk_upload_size`` +~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Galaxy can upload user files in chunks without using nginx. Enable + the chunk uploader by specifying a chunk size larger than 0. The + chunk size is specified in bytes (default: 100MB). +:Default: ``104857600`` +:Type: int + + ~~~~~~~~~~~~~~~~~~~~~~~~ ``dynamic_proxy_manage`` ~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index bd0ba551975..26ea0bc56d9 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -436,6 +436,7 @@ class Configuration(object): self.user_library_import_check_permissions = string_as_bool(kwargs.get('user_library_import_check_permissions', False)) self.user_library_import_dir_auto_creation = string_as_bool(kwargs.get('user_library_import_dir_auto_creation', False)) if self.user_library_import_dir else False # Searching data libraries + self.chunk_upload_size = int(kwargs.get('chunk_upload_size', 104857600)) self.ftp_upload_dir = kwargs.get('ftp_upload_dir', None) self.ftp_upload_dir_identifier = kwargs.get('ftp_upload_dir_identifier', 'email') # attribute on user - email, username, id, etc... self.ftp_upload_dir_template = kwargs.get('ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep) diff --git a/lib/galaxy/managers/configuration.py b/lib/galaxy/managers/configuration.py index 57f1dc889f6..e0f313c443f 100644 --- a/lib/galaxy/managers/configuration.py +++ b/lib/galaxy/managers/configuration.py @@ -73,7 +73,8 @@ class ConfigSerializer(base.ModelSerializer): # TODO: is there no 'correct' way to get an api url? controller='api', action='tools' is a hack # at any rate: the following works with path_prefix but is still brittle # TODO: change this to (more generic) upload_path and incorporate config.nginx_upload_path into building it - 'nginx_upload_path' : lambda i, k, **c: getattr(i, k, False) or self.url_for('/api/tools'), + 'nginx_upload_path' : lambda i, k, **c: getattr(i, k, False), + 'chunk_upload_size' : _defaults_to(104857600), 'ftp_upload_dir' : _defaults_to(None), 'ftp_upload_site' : _defaults_to(None), 'version_major' : _defaults_to(None), diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 719174aba27..888b2206787 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -918,8 +918,7 @@ class Tool(Dictifiable): # If we have an nginx upload, save the action as a tuple instead of # a string. The actual action needs to get url_for run to add any # prefixes, and we want to avoid adding the prefix to the - # nginx_upload_path. This logic is handled in the tool_form.mako - # template. + # nginx_upload_path. if self.nginx_upload and self.app.config.nginx_upload_path: if '?' in unquote_plus(self.action): raise Exception('URL parameters in a non-default tool action can not be used ' diff --git a/lib/galaxy/tools/parameters/basic.py b/lib/galaxy/tools/parameters/basic.py index e3655429751..7265d48a344 100644 --- a/lib/galaxy/tools/parameters/basic.py +++ b/lib/galaxy/tools/parameters/basic.py @@ -521,11 +521,19 @@ class FileToolParameter(ToolParameter): # Middleware or proxies may encode files in special ways (TODO: this # should be pluggable) if type(value) == dict: - upload_store = trans.app.config.nginx_upload_store - assert upload_store, "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it." - # Check that the file is in the right location - local_filename = os.path.abspath(value['path']) - assert local_filename.startswith(upload_store), "Filename provided by nginx (%s) is not in correct directory (%s)." % (local_filename, upload_store) + if 'session_id' in value: + # handle api upload + session_id = value["session_id"] + upload_store = trans.app.config.new_file_path + if re.match('^[\w-]+$', session_id) is None: + raise ValueError("Invald session id format.") + local_filename = os.path.abspath(os.path.join(upload_store, session_id)) + else: + # handle nginx upload + upload_store = trans.app.config.nginx_upload_store + assert upload_store, "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it." + local_filename = os.path.abspath(value['path']) + assert local_filename.startswith(upload_store), "Filename provided by nginx (%s) is not in correct directory (%s)." % (local_filename, upload_store) value = dict(filename=value["name"], local_filename=local_filename) return value diff --git a/lib/galaxy/web/framework/decorators.py b/lib/galaxy/web/framework/decorators.py index ef23ed0a127..9ce0ea099c2 100644 --- a/lib/galaxy/web/framework/decorators.py +++ b/lib/galaxy/web/framework/decorators.py @@ -134,7 +134,7 @@ def expose_api(func, to_json=True, user_required=True): trans.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store" # Perform api_run_as processing, possibly changing identity - if 'payload' in kwargs and 'run_as' in kwargs['payload']: + if 'payload' in kwargs and isinstance(kwargs['payload'], dict) and 'run_as' in kwargs['payload']: if not trans.user_can_do_run_as(): error_message = 'User does not have permissions to run jobs as another user' return error diff --git a/lib/galaxy/webapps/galaxy/api/uploads.py b/lib/galaxy/webapps/galaxy/api/uploads.py new file mode 100644 index 00000000000..bedf6daf061 --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/uploads.py @@ -0,0 +1,53 @@ +""" +API operations for uploaded files in storage. +""" +import logging +import os +import re + +from galaxy.exceptions import MessageException, NotImplemented +from galaxy.web import expose_api_anonymous +from galaxy.web.base.controller import BaseAPIController + +log = logging.getLogger(__name__) + + +class UploadsAPIController(BaseAPIController): + + READ_CHUNK_SIZE = 2 ** 16 + + @expose_api_anonymous + def index(self, trans, **kwd): + raise NotImplemented("Listing uploads is not implemented.") + + @expose_api_anonymous + def create(self, trans, payload, **kwd): + """ + POST /api/uploads/ + """ + session_id = payload.get("session_id") + session_start = payload.get("session_start") + session_chunk = payload.get("session_chunk") + if re.match('^[\w-]+$', session_id) is None: + raise MessageException("Requires a session id.") + if session_start is None: + raise MessageException("Requires a session start.") + if not hasattr(session_chunk, "file"): + raise MessageException("Requires a session chunk.") + target_file = os.path.join(trans.app.config.new_file_path, session_id) + target_size = 0 + if os.path.exists(target_file): + target_size = os.path.getsize(target_file) + if session_start != target_size: + raise MessageException("Incorrect session start.") + chunk_size = os.fstat(session_chunk.file.fileno()).st_size + if chunk_size > trans.app.config.chunk_upload_size: + raise MessageException("Invalid chunk size.") + with open(target_file, "a") as f: + while True: + read_chunk = session_chunk.file.read(self.READ_CHUNK_SIZE) + if not read_chunk: + break + f.write(read_chunk) + session_chunk.file.close() + return {"message": "Successful."} diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index eda00a26ee4..f303fbfa92a 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -256,6 +256,7 @@ def populate_api_routes(webapp, app): webapp.mapper.resource('dataset_collection', 'dataset_collections', path_prefix='/api/') webapp.mapper.resource('form', 'forms', path_prefix='/api') webapp.mapper.resource('role', 'roles', path_prefix='/api') + webapp.mapper.resource('upload', 'uploads', path_prefix='/api') webapp.mapper.connect('/api/ftp_files', controller='remote_files') webapp.mapper.resource('remote_file', 'remote_files', path_prefix='/api') webapp.mapper.resource('group', 'groups', path_prefix='/api') diff --git a/lib/galaxy/webapps/galaxy/config_schema.yml b/lib/galaxy/webapps/galaxy/config_schema.yml index 97819d70d2b..5679ffb01f1 100644 --- a/lib/galaxy/webapps/galaxy/config_schema.yml +++ b/lib/galaxy/webapps/galaxy/config_schema.yml @@ -1239,6 +1239,15 @@ mapping: operations on the remote end. See the Galaxy nginx documentation for the corresponding nginx configuration. + chunk_upload_size: + type: int + default: 104857600 + required: false + desc: | + Galaxy can upload user files in chunks without using nginx. Enable the chunk + uploader by specifying a chunk size larger than 0. The chunk size is specified + in bytes (default: 100MB). + dynamic_proxy_manage: type: bool default: true diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index ca727ea6889..ec732b7066b 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -108,7 +108,6 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): js_options = self._get_js_options(trans) config = js_options['config'] config.update(self._get_extended_config(trans)) - return self.template(trans, 'analysis', options=js_options) @web.expose diff --git a/test/base/driver_util.py b/test/base/driver_util.py index 7c31e7df7ee..47e8ee1ae1c 100644 --- a/test/base/driver_util.py +++ b/test/base/driver_util.py @@ -191,6 +191,7 @@ def setup_galaxy_config( api_allow_run_as='test@bx.psu.edu', auto_configure_logging=logging_config_file is None, check_migrate_tools=False, + chunk_upload_size=100, conda_auto_init=conda_auto_init, conda_auto_install=conda_auto_install, cleanup_job='onsuccess',