opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'chunk_uploads' into dev

Browse Source
This commit is contained in:
Dannon Baker
2018-03-14 09:10:42 -04:00
parent 5f73d7fbbd fcd00166e2
commit c6993caabf
20 changed files with 426 additions and 199 deletions
Download Patch File Download Diff File Expand all files Collapse all files
client/galaxy/scripts/apps/panels/tool-panel.js
+39 -53
View File
@@ -2,6 +2,7 @@ import Tools from "mvc/tool/tools";
import Upload from "mvc/upload/upload-view";
import _l from "utils/localization";
import ToolForm from "mvc/tool/tool-form-composite";
import _ from "libs/underscore";
var ToolPanel = Backbone.View.extend({
initialize: function(page, options) {
@@ -28,7 +29,8 @@ var ToolPanel = Backbone.View.extend({
// add upload modal
this.upload_button = new Upload({
nginx_upload_path: config.nginx_upload_path,
upload_path: config.nginx_upload_path || `${Galaxy.root}api/tools`,
chunk_upload_size: config.chunk_upload_size,
ftp_upload_site: config.ftp_upload_site,
default_genome: config.default_genome,
default_extension: config.default_extension
@@ -74,70 +76,54 @@ var ToolPanel = Backbone.View.extend({
/** build a link to one tool */
_templateTool: function(tool) {
return [
'<div class="toolTitle">',
'<a href="',
this.root,
tool.href,
'" target="galaxy_main">',
tool.title,
"</a>",
"</div>"
].join("");
return `<div class="toolTitle">
<a href="${Galaxy.root}${tool.href}" target="galaxy_main">
${tool.title}
</a>
</div>`;
},
/** build a link to 'All Workflows' */
_templateAllWorkflow: function(tool) {
return [
'<div class="toolTitle">',
// global
'<a href="',
Galaxy.root,
tool.href,
'">',
tool.title,
"</a>",
"</div>"
].join("");
return `<div class="toolTitle">
<a href="${Galaxy.root}${tool.href}">
${tool.title}
</a>
</div>`;
},
/** build links to workflows in toolpanel */
_templateWorkflowLink: function(wf) {
return [
'<div class="toolTitle">',
`<a class="${wf.cls} " href="`,
Galaxy.root,
wf.href,
'">',
wf.title,
"</a>",
"</div>"
].join("");
return `<div class="toolTitle">
<a class="${wf.cls}" href="${Galaxy.root}${wf.href}">
${_.escape(wf.title)}
</a>
</div>`;
},
/** override to include inital menu dom and workflow section */
_template: function() {
return [
'<div class="toolMenuContainer">',
'<div class="toolMenu" style="display: none">',
'<div id="search-no-results" style="display: none; padding-top: 5px">',
"<em><strong>",
_l("Search did not match any tools."),
"</strong></em>",
"</div>",
"</div>",
'<div class="toolSectionPad"/>',
'<div class="toolSectionPad"/>',
'<div class="toolSectionTitle" id="title_XXinternalXXworkflow">',
"<span>",
_l("Workflows"),
"</span>",
"</div>",
'<div id="internal-workflows" class="toolSectionBody">',
'<div class="toolSectionBg"/>',
"</div>",
"</div>"
].join("");
return `<div class="toolMenuContainer">
<div class="toolMenu" style="display: none">
<div id="search-no-results" style="display: none; padding-top: 5px">
<em>
<strong>
${_l("Search did not match any tools.")}
</strong>
</em>
</div>
</div>
<div class="toolSectionPad"/>
<div class="toolSectionPad"/>
<div class="toolSectionTitle" id="title_XXinternalXXworkflow">
<span>
${_l("Workflows")}
</span>
</div>
<div id="internal-workflows" class="toolSectionBody">
<div class="toolSectionBg"/>
</div>
</div>`;
},
toString: function() {
client/galaxy/scripts/mvc/grid/grid-template.js
+3 -1
View File
@@ -93,7 +93,9 @@ export default {
if (column.visible) {
tmpl += `<th id="${column.key}-header">`;
if (column.sortable) {
tmpl += `<a href="javascript:void(0)" class="sort-link" sort_key="${column.key}">${column.label}</a>`;
tmpl += `<a href="javascript:void(0)" class="sort-link" sort_key="${column.key}">${
column.label
}</a>`;
} else {
tmpl += column.label;
}
client/galaxy/scripts/mvc/upload/collection/collection-view.js
+1 -1
View File
@@ -116,7 +116,7 @@ export default Backbone.View.extend({
// file upload
this.uploadbox = this.$(".upload-box").uploadbox({
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
announce: function(index, file) {
self._eventAnnounce(index, file);
},
client/galaxy/scripts/mvc/upload/composite/composite-view.js
+1 -1
View File
@@ -133,7 +133,7 @@ export default Backbone.View.extend({
});
});
$.uploadpost({
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
data: this.app.toData(this.collection.filter()),
success: function(message) {
self._eventSuccess(message);
client/galaxy/scripts/mvc/upload/default/default-row.js
+8 -3
View File
@@ -11,6 +11,7 @@ export default Backbone.View.extend({
init: "upload-icon-button fa fa-trash-o",
queued: "upload-icon fa fa-spinner fa-spin",
running: "upload-icon fa fa-spinner fa-spin",
warning: "upload-icon fa fa-spinner fa-spin",
success: "upload-icon-button fa fa-check",
error: "upload-icon-button fa fa-exclamation-triangle"
},
@@ -183,7 +184,7 @@ export default Backbone.View.extend({
_refreshInfo: function() {
var info = this.model.get("info");
if (info) {
this.$info_text.html(`<strong>Failed: </strong>${info}`).show();
this.$info_text.html(`<strong>Warning: </strong>${info}`).show();
} else {
this.$info_text.hide();
}
@@ -213,13 +214,17 @@ export default Backbone.View.extend({
this.select_genome.disable();
this.select_extension.disable();
}
this.$info_progress.show();
this.$el.removeClass().addClass("upload-row");
if (status == "success") {
this.$el.addClass("success");
this.$percentage.html("100%");
}
if (status == "error") {
} else if (status == "error") {
this.$el.addClass("danger");
this.$info_progress.hide();
} else if (status == "warning") {
this.$el.addClass("warning");
this.$info_progress.hide();
}
},
client/galaxy/scripts/mvc/upload/default/default-view.js
+48 -36
View File
@@ -104,7 +104,7 @@ export default Backbone.View.extend({
// file upload
this.uploadbox = this.$uploadbox.uploadbox({
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
announce: function(index, file) {
self._eventAnnounce(index, file);
},
@@ -120,6 +120,9 @@ export default Backbone.View.extend({
error: function(index, message) {
self._eventError(index, message);
},
warning: function(index, message) {
self._eventWarning(index, message);
},
complete: function() {
self._eventComplete();
},
@@ -247,7 +250,7 @@ export default Backbone.View.extend({
/** Progress */
_eventProgress: function(index, percentage) {
var it = this.collection.get(index);
it.set("percentage", percentage);
it.set({ percentage: percentage, status: "running", info: "" });
this.ui_button.model.set("percentage", this._uploadPercentage(percentage, it.get("file_size")));
},
@@ -263,6 +266,12 @@ export default Backbone.View.extend({
Galaxy.currHistoryPanel.refreshContents();
},
/** Warning */
_eventWarning: function(index, message) {
var it = this.collection.get(index);
it.set({ status: "warning", info: message });
},
/** Error */
_eventError: function(index, message) {
var it = this.collection.get(index);
@@ -363,7 +372,10 @@ export default Backbone.View.extend({
this._uploadFtp();
// queue remaining files
this.uploadbox.start();
this.uploadbox.start({
id: Galaxy.user.id,
chunk_upload_size: this.app.options.chunk_upload_size
});
this.render();
}
},
@@ -430,7 +442,7 @@ export default Backbone.View.extend({
if (list.length > 0) {
$.uploadpost({
data: this.app.toData(list),
url: this.app.options.nginx_upload_path,
url: this.app.options.upload_path,
success: function(message) {
_.each(list, model => {
self._eventSuccess(model.id);
@@ -452,37 +464,37 @@ export default Backbone.View.extend({
/** Template */
_template: function() {
return (
'<div class="upload-view-default">' +
'<div class="upload-top">' +
'<h6 class="upload-top-info"/>' +
"</div>" +
'<div class="upload-box">' +
'<div class="upload-helper"><i class="fa fa-files-o"/>Drop files here</div>' +
'<table class="upload-table ui-table-striped" style="display: none;">' +
"<thead>" +
"<tr>" +
"<th>Name</th>" +
"<th>Size</th>" +
"<th>Type</th>" +
"<th>Genome</th>" +
"<th>Settings</th>" +
"<th>Status</th>" +
"<th/>" +
"</tr>" +
"</thead>" +
"<tbody/>" +
"</table>" +
"</div>" +
'<div class="upload-footer">' +
'<span class="upload-footer-title">Type (set all):</span>' +
'<span class="upload-footer-extension"/>' +
'<span class="upload-footer-extension-info upload-icon-button fa fa-search"/> ' +
'<span class="upload-footer-title">Genome (set all):</span>' +
'<span class="upload-footer-genome"/>' +
"</div>" +
'<div class="upload-buttons"/>' +
"</div>"
);
return `<div class="upload-view-default">
<div class="upload-top">
<h6 class="upload-top-info"/>
</div>
<div class="upload-box">
<div class="upload-helper">
<i class="fa fa-files-o"/>Drop files here
</div>
<table class="upload-table ui-table-striped" style="display: none;">
<thead>
<tr>
<th>Name</th>
<th>Size</th>
<th>Type</th>
<th>Genome</th>
<th>Settings</th>
<th>Status</th>
<th/>
</tr>
</thead>
<tbody/>
</table>
</div>
<div class="upload-footer">
<span class="upload-footer-title">Type (set all):</span>
<span class="upload-footer-extension"/>
<span class="upload-footer-extension-info upload-icon-button fa fa-search"/>
<span class="upload-footer-title">Genome (set all):</span>
<span class="upload-footer-genome"/>
</div>
<div class="upload-buttons"/>
</div>`;
}
});
client/galaxy/scripts/mvc/upload/upload-view.js
-1
View File
@@ -9,7 +9,6 @@ import UploadViewComposite from "mvc/upload/composite/composite-view";
import UploadViewCollection from "mvc/upload/collection/collection-view";
export default Backbone.View.extend({
options: {
nginx_upload_path: "",
ftp_upload_site: "n/a",
default_genome: "?",
default_extension: "auto",
client/galaxy/scripts/utils/uploadbox.js
+227 -93
View File
@@ -6,81 +6,29 @@
jQuery.event.props.push("dataTransfer");
/**
Posts file data to the API
xhr request helper
*/
$.uploadpost = config => {
// parse options
var _uploadrequest = config => {
var cnf = $.extend(
{},
{
data: {},
success: function() {},
error: function() {},
progress: function() {},
url: null,
maxfilesize: 2048,
error_filesize: "File exceeds 2GB. Please use a FTP client.",
error_default: "Please make sure the file is available.",
error_server: "Upload request failed.",
error_login: "Uploads require you to log in."
error_login: "Uploads require you to log in.",
error_retry: "Waiting for server to resume..."
},
config
);
// link data
var data = cnf.data;
// check errors
if (data.error_message) {
cnf.error(data.error_message);
return;
}
// construct form data
var form = new FormData();
for (var key in data.payload) {
form.append(key, data.payload[key]);
}
// add files to submission
var sizes = 0;
for (var key in data.files) {
var d = data.files[key];
form.append(d.name, d.file, d.file.name);
sizes += d.file.size;
}
// check file size, unless it's an ftp file
if (sizes > 1048576 * cnf.maxfilesize) {
cnf.error(cnf.error_filesize);
return;
}
// prepare request
console.debug(cnf);
var xhr = new XMLHttpRequest();
xhr.open("POST", cnf.url, true);
xhr.setRequestHeader("Accept", "application/json");
xhr.setRequestHeader("Cache-Control", "no-cache");
xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest");
// captures state changes
xhr.setRequestHeader("Accept", "application/json");
xhr.onreadystatechange = () => {
// check for request completed, server connection closed
if (xhr.readyState == xhr.DONE) {
// parse response
var response = null;
var extra_info = "";
if (xhr.responseText) {
try {
response = jQuery.parseJSON(xhr.responseText);
extra_info = response.err_msg;
} catch (e) {
response = xhr.responseText;
extra_info = response;
}
}
// pass any error to the error option
if (xhr.status < 200 || xhr.status > 299) {
if ([502, 0].indexOf(xhr.status) !== -1 && cnf.warning) {
cnf.warning(cnf.error_retry);
} else if (xhr.status < 200 || xhr.status > 299) {
var text = xhr.statusText;
if (xhr.status == 403) {
text = cnf.error_login;
@@ -89,27 +37,191 @@
} else if (!text) {
text = cnf.error_default;
}
cnf.error(`${text} (${xhr.status}). ${extra_info}`);
cnf.error(`${text} (${xhr.status})`);
} else {
var response = null;
if (xhr.responseText) {
try {
response = jQuery.parseJSON(xhr.responseText);
} catch (e) {
response = xhr.responseText;
}
}
cnf.success(response);
}
}
};
xhr.upload.addEventListener("progress", cnf.progress, false);
xhr.send(cnf.data);
};
// prepare upload progress
xhr.upload.addEventListener(
"progress",
e => {
/**
Posts chunked files to the API.
*/
$.uploadchunk = function(config) {
// parse options
var cnf = $.extend(
{},
{
data: {},
success: () => {},
error: () => {},
warning: () => {},
progress: () => {},
attempts: 70000,
timeout: 5000,
url: null,
error_file: "File not provided.",
error_attempt: "Maximum number of attempts reached.",
error_tool: "Tool submission failed."
},
config
);
// initial validation
var data = cnf.data;
if (data.error_message) {
cnf.error(data.error_message);
return;
}
var file_data = data.files && data.files[0];
if (!file_data) {
cnf.error(cnf.error_file);
return;
}
var file = file_data.file;
var attempts = cnf.attempts;
var session_id = `${cnf.session.id}-${new Date().valueOf()}-${file.size}`;
var chunk_size = cnf.session.chunk_upload_size;
console.debug(`Starting chunked uploads [size=${chunk_size}].`);
// chunk processing helper
function process(start) {
start = start || 0;
var slicer = file.mozSlice || file.webkitSlice || file.slice;
if (!slicer) {
cnf.error("Browser does not support chunked uploads.");
return;
}
var end = Math.min(start + chunk_size, file.size);
var size = file.size;
console.debug(`Submitting chunk at ${start} bytes...`);
var form = new FormData();
form.append("session_id", session_id);
form.append("session_start", start);
form.append("session_chunk", slicer.bind(file)(start, end));
_uploadrequest({
url: `${Galaxy.root}api/uploads`,
data: form,
success: upload_response => {
var new_start = start + chunk_size;
if (new_start < size) {
attempts = cnf.attempts;
process(new_start);
} else {
console.debug("Upload completed.");
data.payload.inputs = JSON.parse(data.payload.inputs);
data.payload.inputs["files_0|file_data"] = {
session_id: session_id,
name: file.name
};
data.payload.inputs = JSON.stringify(data.payload.inputs);
$.ajax({
url: `${Galaxy.root}api/tools`,
method: "POST",
data: data.payload,
success: tool_response => {
cnf.success(tool_response);
},
error: tool_response => {
var err_msg =
tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg;
cnf.error(err_msg || cnf.error_tool);
}
});
}
},
warning: upload_response => {
if (--attempts > 0) {
console.debug("Retrying last chunk...");
cnf.warning(upload_response);
setTimeout(() => process(start), cnf.timeout);
} else {
console.debug(cnf.error_attempt);
cnf.error(cnf.error_attempt);
}
},
error: upload_response => {
console.debug(upload_response);
cnf.error(upload_response);
},
progress: e => {
if (e.lengthComputable) {
cnf.progress(Math.min(Math.round((start + e.loaded) * 100 / file.size), 100));
}
}
});
}
// initiate processing queue for chunks
process();
};
/**
Posts multiple files without chunking to the API.
*/
$.uploadpost = function(config) {
var cnf = $.extend(
{},
{
data: {},
success: () => {},
error: () => {},
progress: () => {},
url: null,
maxfilesize: 1048576 * 2048,
error_filesize: "File exceeds 2GB. Please use a FTP client."
},
config
);
var data = cnf.data;
if (data.error_message) {
cnf.error(data.error_message);
return;
}
// construct form data
var form = new FormData();
for (let key in data.payload) {
form.append(key, data.payload[key]);
}
// add files to submission
var sizes = 0;
for (let key in data.files) {
var d = data.files[key];
form.append(d.name, d.file, d.file.name);
sizes += d.file.size;
}
// check file size, unless it's an ftp file
if (sizes > cnf.maxfilesize) {
cnf.error(cnf.error_filesize);
return;
}
// submit request
_uploadrequest({
url: cnf.url,
data: form,
success: cnf.success,
error: cnf.error,
progress: e => {
if (e.lengthComputable) {
cnf.progress(Math.round(e.loaded * 100 / e.total));
}
},
false
);
// send request
Galaxy.emit.debug("uploadbox::uploadpost()", "Posting following data.", cnf);
xhr.send(form);
}
});
};
/**
@@ -121,9 +233,9 @@
var opts = $.extend(
{},
{
ondragover: function() {},
ondragleave: function() {},
onchange: function() {},
ondragover: () => {},
ondragleave: () => {},
onchange: () => {},
multiple: false
},
options
@@ -132,7 +244,7 @@
// append hidden upload field
var $input = $(`<input type="file" style="display: none" ${(opts.multiple && "multiple") || ""}/>`);
el.append(
$input.change(function(e) {
$input.change(e => {
opts.onchange(e.target.files);
$(this).val("");
})
@@ -157,7 +269,7 @@
// exports
return {
dialog: function() {
dialog: () => {
$input.trigger("click");
}
};
@@ -171,16 +283,17 @@
var opts = $.extend(
{},
{
dragover: function() {},
dragleave: function() {},
announce: function(d) {},
initialize: function(d) {},
progress: function(d, m) {},
success: function(d, m) {},
error: function(d, m) {
dragover: () => {},
dragleave: () => {},
announce: d => {},
initialize: d => {},
progress: (d, m) => {},
success: (d, m) => {},
warning: (d, m) => {},
error: (d, m) => {
alert(m);
},
complete: function() {}
complete: () => {}
},
options
);
@@ -188,6 +301,9 @@
// file queue
var queue = {};
// session options
var session = null;
// queue index/length counter
var queue_index = 0;
var queue_length = 0;
@@ -199,7 +315,10 @@
// element
var uploadinput = $(this).uploadinput({
multiple: true,
onchange: function(files) {
onchange: files => {
_.each(files, file => {
file.chunk_mode = true;
});
add(files);
},
ondragover: options.ondragover,
@@ -252,7 +371,7 @@
// get an identifier from the queue
var index = -1;
for (var key in queue) {
for (let key in queue) {
index = key;
break;
}
@@ -264,18 +383,32 @@
remove(index);
// create and submit data
$.uploadpost({
var submitter = $.uploadpost;
if (
file.chunk_mode &&
session &&
session.id &&
session.chunk_upload_size &&
session.chunk_upload_size > 0
) {
submitter = $.uploadchunk;
}
submitter({
url: opts.url,
data: opts.initialize(index),
success: function(message) {
session: session,
success: message => {
opts.success(index, message);
process();
},
error: function(message) {
warning: message => {
opts.warning(index, message);
},
error: message => {
opts.error(index, message);
process();
},
progress: function(percentage) {
progress: percentage => {
opts.progress(index, percentage);
}
});
@@ -298,7 +431,8 @@
}
// initiate upload process
function start() {
function start(_session) {
session = _session;
if (!queue_running) {
queue_running = true;
process();
config/galaxy.yml.sample
+5
View File
@@ -812,6 +812,11 @@ galaxy:
# documentation for the corresponding nginx configuration.
#nginx_upload_job_files_path: false
# Galaxy can upload user files in chunks without using nginx. Enable
# the chunk uploader by specifying a chunk size larger than 0. The
# chunk size is specified in bytes (default: 100MB).
#chunk_upload_size: 104857600
# Have Galaxy manage dynamic proxy component for routing requests to
# other services based on Galaxy's session cookie. It will attempt to
# do this by default though you do need to install node+npm and do an
doc/source/admin/galaxy_options.rst
+12
View File
@@ -1643,6 +1643,18 @@
:Type: bool
~~~~~~~~~~~~~~~~~~~~~
``chunk_upload_size``
~~~~~~~~~~~~~~~~~~~~~
:Description:
Galaxy can upload user files in chunks without using nginx. Enable
the chunk uploader by specifying a chunk size larger than 0. The
chunk size is specified in bytes (default: 100MB).
:Default: ``104857600``
:Type: int
~~~~~~~~~~~~~~~~~~~~~~~~
``dynamic_proxy_manage``
~~~~~~~~~~~~~~~~~~~~~~~~
lib/galaxy/config.py
+1
View File
@@ -436,6 +436,7 @@ class Configuration(object):
self.user_library_import_check_permissions = string_as_bool(kwargs.get('user_library_import_check_permissions', False))
self.user_library_import_dir_auto_creation = string_as_bool(kwargs.get('user_library_import_dir_auto_creation', False)) if self.user_library_import_dir else False
# Searching data libraries
self.chunk_upload_size = int(kwargs.get('chunk_upload_size', 104857600))
self.ftp_upload_dir = kwargs.get('ftp_upload_dir', None)
self.ftp_upload_dir_identifier = kwargs.get('ftp_upload_dir_identifier', 'email') # attribute on user - email, username, id, etc...
self.ftp_upload_dir_template = kwargs.get('ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep)
lib/galaxy/managers/configuration.py
+2 -1
View File
@@ -73,7 +73,8 @@ class ConfigSerializer(base.ModelSerializer):
# TODO: is there no 'correct' way to get an api url? controller='api', action='tools' is a hack
# at any rate: the following works with path_prefix but is still brittle
# TODO: change this to (more generic) upload_path and incorporate config.nginx_upload_path into building it
'nginx_upload_path' : lambda i, k, **c: getattr(i, k, False) or self.url_for('/api/tools'),
'nginx_upload_path' : lambda i, k, **c: getattr(i, k, False),
'chunk_upload_size' : _defaults_to(104857600),
'ftp_upload_dir' : _defaults_to(None),
'ftp_upload_site' : _defaults_to(None),
'version_major' : _defaults_to(None),
lib/galaxy/tools/__init__.py
+1 -2
View File
@@ -918,8 +918,7 @@ class Tool(Dictifiable):
# If we have an nginx upload, save the action as a tuple instead of
# a string. The actual action needs to get url_for run to add any
# prefixes, and we want to avoid adding the prefix to the
# nginx_upload_path. This logic is handled in the tool_form.mako
# template.
# nginx_upload_path.
if self.nginx_upload and self.app.config.nginx_upload_path:
if '?' in unquote_plus(self.action):
raise Exception('URL parameters in a non-default tool action can not be used '
lib/galaxy/tools/parameters/basic.py
+13 -5
View File
@@ -521,11 +521,19 @@ class FileToolParameter(ToolParameter):
# Middleware or proxies may encode files in special ways (TODO: this
# should be pluggable)
if type(value) == dict:
upload_store = trans.app.config.nginx_upload_store
assert upload_store, "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it."
# Check that the file is in the right location
local_filename = os.path.abspath(value['path'])
assert local_filename.startswith(upload_store), "Filename provided by nginx (%s) is not in correct directory (%s)." % (local_filename, upload_store)
if 'session_id' in value:
# handle api upload
session_id = value["session_id"]
upload_store = trans.app.config.new_file_path
if re.match('^[\w-]+$', session_id) is None:
raise ValueError("Invald session id format.")
local_filename = os.path.abspath(os.path.join(upload_store, session_id))
else:
# handle nginx upload
upload_store = trans.app.config.nginx_upload_store
assert upload_store, "Request appears to have been processed by nginx_upload_module but Galaxy is not configured to recognize it."
local_filename = os.path.abspath(value['path'])
assert local_filename.startswith(upload_store), "Filename provided by nginx (%s) is not in correct directory (%s)." % (local_filename, upload_store)
value = dict(filename=value["name"], local_filename=local_filename)
return value
lib/galaxy/web/framework/decorators.py
+1 -1
View File
@@ -134,7 +134,7 @@ def expose_api(func, to_json=True, user_required=True):
trans.response.headers['Cache-Control'] = "max-age=0,no-cache,no-store"
# Perform api_run_as processing, possibly changing identity
if 'payload' in kwargs and 'run_as' in kwargs['payload']:
if 'payload' in kwargs and isinstance(kwargs['payload'], dict) and 'run_as' in kwargs['payload']:
if not trans.user_can_do_run_as():
error_message = 'User does not have permissions to run jobs as another user'
return error
lib/galaxy/webapps/galaxy/api/uploads.py
+53
View File
@@ -0,0 +1,53 @@
"""
API operations for uploaded files in storage.
"""
import logging
import os
import re
from galaxy.exceptions import MessageException, NotImplemented
from galaxy.web import expose_api_anonymous
from galaxy.web.base.controller import BaseAPIController
log = logging.getLogger(__name__)
class UploadsAPIController(BaseAPIController):
READ_CHUNK_SIZE = 2 ** 16
@expose_api_anonymous
def index(self, trans, **kwd):
raise NotImplemented("Listing uploads is not implemented.")
@expose_api_anonymous
def create(self, trans, payload, **kwd):
"""
POST /api/uploads/
"""
session_id = payload.get("session_id")
session_start = payload.get("session_start")
session_chunk = payload.get("session_chunk")
if re.match('^[\w-]+$', session_id) is None:
raise MessageException("Requires a session id.")
if session_start is None:
raise MessageException("Requires a session start.")
if not hasattr(session_chunk, "file"):
raise MessageException("Requires a session chunk.")
target_file = os.path.join(trans.app.config.new_file_path, session_id)
target_size = 0
if os.path.exists(target_file):
target_size = os.path.getsize(target_file)
if session_start != target_size:
raise MessageException("Incorrect session start.")
chunk_size = os.fstat(session_chunk.file.fileno()).st_size
if chunk_size > trans.app.config.chunk_upload_size:
raise MessageException("Invalid chunk size.")
with open(target_file, "a") as f:
while True:
read_chunk = session_chunk.file.read(self.READ_CHUNK_SIZE)
if not read_chunk:
break
f.write(read_chunk)
session_chunk.file.close()
return {"message": "Successful."}
lib/galaxy/webapps/galaxy/buildapp.py
+1
View File
@@ -256,6 +256,7 @@ def populate_api_routes(webapp, app):
webapp.mapper.resource('dataset_collection', 'dataset_collections', path_prefix='/api/')
webapp.mapper.resource('form', 'forms', path_prefix='/api')
webapp.mapper.resource('role', 'roles', path_prefix='/api')
webapp.mapper.resource('upload', 'uploads', path_prefix='/api')
webapp.mapper.connect('/api/ftp_files', controller='remote_files')
webapp.mapper.resource('remote_file', 'remote_files', path_prefix='/api')
webapp.mapper.resource('group', 'groups', path_prefix='/api')
lib/galaxy/webapps/galaxy/config_schema.yml
+9
View File
@@ -1239,6 +1239,15 @@ mapping:
operations on the remote end. See the Galaxy nginx documentation for the
corresponding nginx configuration.
chunk_upload_size:
type: int
default: 104857600
required: false
desc: |
Galaxy can upload user files in chunks without using nginx. Enable the chunk
uploader by specifying a chunk size larger than 0. The chunk size is specified
in bytes (default: 100MB).
dynamic_proxy_manage:
type: bool
default: true
lib/galaxy/webapps/galaxy/controllers/root.py
-1
View File
@@ -108,7 +108,6 @@ class RootController(controller.JSAppLauncher, UsesAnnotations):
js_options = self._get_js_options(trans)
config = js_options['config']
config.update(self._get_extended_config(trans))
return self.template(trans, 'analysis', options=js_options)
@web.expose
test/base/driver_util.py
+1
View File
@@ -191,6 +191,7 @@ def setup_galaxy_config(
api_allow_run_as='test@bx.psu.edu',
auto_configure_logging=logging_config_file is None,
check_migrate_tools=False,
chunk_upload_size=100,
conda_auto_init=conda_auto_init,
conda_auto_install=conda_auto_install,
cleanup_job='onsuccess',