mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-10-24 12:13:33 +08:00
Drop legacy upload1 / uploadpost handling
This commit is contained in:
mvdbeek
@@ -121,6 +121,7 @@ import _ from "underscore";
|
||||
import { getGalaxyInstance } from "app";
|
||||
import UploadRow from "mvc/upload/collection/collection-row";
|
||||
import UploadBoxMixin from "./UploadBoxMixin";
|
||||
import { uploadModelsToPayload } from "./helpers";
|
||||
import { BButton } from "bootstrap-vue";
|
||||
|
||||
export default {
|
||||
@@ -178,7 +179,7 @@ export default {
|
||||
this._eventAnnounce(index, file);
|
||||
},
|
||||
initialize: (index) => {
|
||||
return this.app.toData([this.collection.get(index)], this.history_id);
|
||||
return uploadModelsToPayload([this.collection.get(index)], this.history_id);
|
||||
},
|
||||
progress: (index, percentage) => {
|
||||
this._eventProgress(index, percentage);
|
||||
|
||||
@@ -61,6 +61,7 @@ import $ from "jquery";
|
||||
import { getGalaxyInstance } from "app";
|
||||
import UploadRow from "mvc/upload/composite/composite-row";
|
||||
import UploadBoxMixin from "./UploadBoxMixin";
|
||||
import { uploadModelsToPayload } from "./helpers";
|
||||
|
||||
export default {
|
||||
mixins: [UploadBoxMixin],
|
||||
@@ -139,9 +140,9 @@ export default {
|
||||
extension: this.extension,
|
||||
});
|
||||
});
|
||||
$.uploadpost({
|
||||
$.uploadchunk({
|
||||
url: this.app.uploadPath,
|
||||
data: this.app.toFileUploadData(this.collection.filter()),
|
||||
data: uploadModelsToPayload(this.collection.filter(), this.history_id, true),
|
||||
success: (message) => {
|
||||
this._eventSuccess(message);
|
||||
},
|
||||
|
||||
@@ -114,6 +114,7 @@ import _l from "utils/localization";
|
||||
import _ from "underscore";
|
||||
import UploadRow from "mvc/upload/default/default-row";
|
||||
import UploadBoxMixin from "./UploadBoxMixin";
|
||||
import { uploadModelsToPayload } from "./helpers";
|
||||
import { BButton } from "bootstrap-vue";
|
||||
|
||||
export default {
|
||||
@@ -175,7 +176,7 @@ export default {
|
||||
this._eventAnnounce(index, file);
|
||||
},
|
||||
initialize: (index) => {
|
||||
return this.app.toData([this.collection.get(index)], this.history_id);
|
||||
return uploadModelsToPayload([this.collection.get(index)], this.history_id);
|
||||
},
|
||||
progress: (index, percentage) => {
|
||||
this._eventProgress(index, percentage);
|
||||
|
||||
@@ -5,6 +5,7 @@ import Popover from "mvc/ui/ui-popover";
|
||||
import UploadExtension from "mvc/upload/upload-extension";
|
||||
import UploadModel from "mvc/upload/upload-model";
|
||||
import UploadWrapper from "./UploadWrapper";
|
||||
import { defaultNewFileName, uploadModelsToPayload } from "./helpers";
|
||||
import { getGalaxyInstance } from "app";
|
||||
import UploadFtp from "mvc/upload/upload-ftp";
|
||||
import LazyLimited from "mvc/lazy/lazy-limited";
|
||||
@@ -14,7 +15,6 @@ import { getAppRoot } from "onload";
|
||||
import axios from "axios";
|
||||
|
||||
const localize = _l;
|
||||
export const defaultNewFileName = "New File";
|
||||
|
||||
export default {
|
||||
components: {
|
||||
@@ -118,7 +118,7 @@ export default {
|
||||
}
|
||||
});
|
||||
if (list.length > 0) {
|
||||
const data = this.app.toFetchData(list, this.history_id);
|
||||
const data = uploadModelsToPayload(list, this.history_id);
|
||||
axios
|
||||
.post(`${getAppRoot()}api/tools/fetch`, data)
|
||||
.then((message) => {
|
||||
|
||||
@@ -34,6 +34,7 @@ import Collection from "./Collection";
|
||||
import Default from "./Default";
|
||||
import RulesInput from "./RulesInput";
|
||||
import LoadingSpan from "components/LoadingSpan";
|
||||
import { uploadModelsToPayload } from "./helpers";
|
||||
import { BTabs, BTab } from "bootstrap-vue";
|
||||
import { commonProps } from "./helpers";
|
||||
|
||||
@@ -60,7 +61,6 @@ export default {
|
||||
extensionsSet: false,
|
||||
datatypesMapper: null,
|
||||
datatypesMapperReady: true,
|
||||
URI_PREFIXES: ["http", "https", "ftp", "file", "gxfiles", "gximport", "gxuserimport", "gxftp"],
|
||||
};
|
||||
},
|
||||
created() {
|
||||
@@ -174,143 +174,12 @@ export default {
|
||||
currentFtp: function () {
|
||||
return this.currentUserId && this.ftpUploadSite;
|
||||
},
|
||||
toData: function (items, history_id) {
|
||||
const data = {
|
||||
fetchRequest: null,
|
||||
uploadRequest: null,
|
||||
};
|
||||
if (items && items.length > 0) {
|
||||
const split = this.preprocess(items);
|
||||
if (split.urls.length > 0) {
|
||||
data.fetchRequest = this.toFetchData(split.urls, history_id);
|
||||
} else {
|
||||
data.uploadRequest = this.toFileUploadData(split.files, history_id);
|
||||
}
|
||||
}
|
||||
return data;
|
||||
},
|
||||
preprocess: function (items) {
|
||||
const data = {
|
||||
urls: [],
|
||||
files: [],
|
||||
};
|
||||
for (var index in items) {
|
||||
var it = items[index];
|
||||
if (it.get("file_mode") != "new" || !this.itemIsURL(it)) {
|
||||
data.files.push(it);
|
||||
} else {
|
||||
data.urls.push(it);
|
||||
}
|
||||
}
|
||||
return data;
|
||||
},
|
||||
itemIsURL: function (item) {
|
||||
return this.URI_PREFIXES.some((prefix) => item.get("url_paste").startsWith(prefix));
|
||||
},
|
||||
/**
|
||||
* Package API data from array of models
|
||||
* Package API data from array of backbone models
|
||||
* @param{Array} items - Upload items/rows filtered from a collection
|
||||
*/
|
||||
toFileUploadData: function (items, history_id) {
|
||||
// create dictionary for data submission
|
||||
var data = {
|
||||
payload: {
|
||||
tool_id: "upload1",
|
||||
history_id: history_id || this.currentHistoryId,
|
||||
inputs: {},
|
||||
},
|
||||
files: [],
|
||||
error_message: null,
|
||||
};
|
||||
// add upload tools input data
|
||||
if (items && items.length > 0) {
|
||||
var inputs = {
|
||||
file_count: items.length,
|
||||
dbkey: items[0].get("genome", "?"),
|
||||
// sometimes extension set to "" in automated testing after first upload of
|
||||
// a session. https://github.com/galaxyproject/galaxy/issues/5169
|
||||
file_type: items[0].get("extension") || "auto",
|
||||
};
|
||||
for (var index in items) {
|
||||
var it = items[index];
|
||||
it.set("status", "running");
|
||||
if (it.get("file_size") > 0) {
|
||||
var prefix = `files_${index}|`;
|
||||
inputs[`${prefix}type`] = "upload_dataset";
|
||||
if (it.get("file_name") != "New File") {
|
||||
inputs[`${prefix}NAME`] = it.get("file_name");
|
||||
}
|
||||
inputs[`${prefix}space_to_tab`] = (it.get("space_to_tab") && "Yes") || null;
|
||||
inputs[`${prefix}to_posix_lines`] = (it.get("to_posix_lines") && "Yes") || null;
|
||||
inputs[`${prefix}dbkey`] = it.get("genome", null);
|
||||
inputs[`${prefix}file_type`] = it.get("extension", null);
|
||||
let uri;
|
||||
let how;
|
||||
switch (it.get("file_mode")) {
|
||||
case "new":
|
||||
inputs[`${prefix}url_paste`] = it.get("url_paste");
|
||||
break;
|
||||
case "ftp":
|
||||
uri = it.get("file_path");
|
||||
how = "ftp_files";
|
||||
if (uri.indexOf("://") >= 0) {
|
||||
how = "url_paste";
|
||||
}
|
||||
inputs[`${prefix}${how}`] = uri;
|
||||
break;
|
||||
case "local":
|
||||
data.files.push({
|
||||
name: `${prefix}file_data`,
|
||||
file: it.get("file_data"),
|
||||
});
|
||||
}
|
||||
} else if (it.get("optional")) {
|
||||
continue;
|
||||
} else {
|
||||
data.error_message = "Upload content incomplete.";
|
||||
it.set("status", "error");
|
||||
it.set("info", data.error_message);
|
||||
break;
|
||||
}
|
||||
}
|
||||
data.payload.inputs = JSON.stringify(inputs);
|
||||
}
|
||||
return data;
|
||||
},
|
||||
toFetchData: function (items, history_id) {
|
||||
var data = {
|
||||
history_id: history_id,
|
||||
space_to_tab: items[0].get("space_to_tab"),
|
||||
to_posix_lines: items[0].get("to_posix_lines"),
|
||||
targets: [
|
||||
{
|
||||
destination: { type: "hdas" },
|
||||
elements: [],
|
||||
name: "",
|
||||
},
|
||||
],
|
||||
auto_decompress: true,
|
||||
};
|
||||
|
||||
items.forEach((item) => {
|
||||
let urls;
|
||||
if (item.get("file_mode") == "ftp") {
|
||||
urls = [item.get("file_uri") || item.get("file_path")];
|
||||
} else {
|
||||
urls = item.get("url_paste").split("\n");
|
||||
}
|
||||
urls.forEach((url) => {
|
||||
if (url != "") {
|
||||
data.targets[0].elements.push({
|
||||
url: url.trim(),
|
||||
src: "url",
|
||||
dbkey: item.get("genome", "?"),
|
||||
ext: item.get("extension", "auto"),
|
||||
});
|
||||
}
|
||||
});
|
||||
});
|
||||
return data;
|
||||
toData: function (items, history_id, composite = false) {
|
||||
return uploadModelsToPayload(items, history_id, composite);
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
@@ -1,4 +1,83 @@
|
||||
import UploadUtils from "mvc/upload/upload-utils";
|
||||
export const defaultNewFileName = "New File";
|
||||
|
||||
const URI_PREFIXES = ["http", "https", "ftp", "file", "gxfiles", "gximport", "gxuserimport", "gxftp"];
|
||||
function itemIsUrl(item) {
|
||||
return URI_PREFIXES.some((prefix) => item.get("url_paste").startsWith(prefix));
|
||||
}
|
||||
|
||||
export function uploadModelsToPayload(items, history_id, composite = false) {
|
||||
const files = [];
|
||||
const elements = items
|
||||
.map((item) => {
|
||||
if (item.get("optional")) {
|
||||
return null;
|
||||
}
|
||||
let src;
|
||||
let pasteContent = null;
|
||||
let fileName = item.get("file_name");
|
||||
if (fileName === defaultNewFileName) {
|
||||
fileName = null;
|
||||
}
|
||||
const url = (item.get("file_uri") || item.get("file_path") || item.get("url_paste")).trim();
|
||||
switch (item.get("file_mode")) {
|
||||
case "new":
|
||||
if (itemIsUrl(item)) {
|
||||
src = "url";
|
||||
} else {
|
||||
pasteContent = item.get("url_paste");
|
||||
src = "pasted";
|
||||
}
|
||||
break;
|
||||
case "ftp":
|
||||
if (url.indexOf("://") >= 0) {
|
||||
src = "url";
|
||||
}
|
||||
break;
|
||||
case "local":
|
||||
src = "files";
|
||||
files.push(item.get("file_data"));
|
||||
break;
|
||||
default:
|
||||
console.error("Unknown file_mode", item);
|
||||
}
|
||||
return {
|
||||
src: src,
|
||||
url: url,
|
||||
paste_content: pasteContent,
|
||||
name: fileName,
|
||||
dbkey: item.get("genome", "?"),
|
||||
ext: item.get("extension", "auto"),
|
||||
space_to_tab: item.get("space_to_tab"),
|
||||
to_posix_lines: item.get("to_posix_lines"),
|
||||
};
|
||||
})
|
||||
.filter((item) => item);
|
||||
|
||||
const target = {
|
||||
destination: { type: "hdas" },
|
||||
elements: elements,
|
||||
};
|
||||
if (composite) {
|
||||
const compositeItems = [
|
||||
{
|
||||
src: "composite",
|
||||
ext: elements[0].ext,
|
||||
composite: {
|
||||
items: target.elements,
|
||||
},
|
||||
},
|
||||
];
|
||||
delete target["elements"];
|
||||
target["items"] = compositeItems;
|
||||
}
|
||||
return {
|
||||
history_id: history_id,
|
||||
targets: [target],
|
||||
auto_decompress: true,
|
||||
files: files,
|
||||
};
|
||||
}
|
||||
|
||||
export const commonProps = {
|
||||
uploadPath: {
|
||||
|
||||
@@ -5,6 +5,7 @@ import DataDialog from "components/DataDialog/DataDialog.vue";
|
||||
import { FilesDialog } from "components/FilesDialog";
|
||||
import DatasetCollectionDialog from "components/SelectionDialog/DatasetCollectionDialog.vue";
|
||||
import { mountUploadModal } from "components/Upload";
|
||||
import { uploadModelsToPayload } from "components/Upload/helpers";
|
||||
import { getGalaxyInstance } from "app";
|
||||
import { getAppRoot } from "onload/loadConfig";
|
||||
|
||||
@@ -92,8 +93,8 @@ export function create(options) {
|
||||
return options.history_id;
|
||||
}
|
||||
getHistory().then((history_id) => {
|
||||
$.uploadpost({
|
||||
url: `${getAppRoot()}api/tools`,
|
||||
$.uploadchunk({
|
||||
url: `${getAppRoot()}api/tools/fetch`,
|
||||
success: (response) => {
|
||||
if (history_panel) {
|
||||
history_panel.refreshContents();
|
||||
@@ -104,19 +105,7 @@ export function create(options) {
|
||||
},
|
||||
error: options.error,
|
||||
data: {
|
||||
payload: {
|
||||
tool_id: "upload1",
|
||||
history_id: history_id,
|
||||
inputs: JSON.stringify({
|
||||
"files_0|type": "upload_dataset",
|
||||
"files_0|NAME": options.file_name,
|
||||
"files_0|space_to_tab": options.space_to_tab ? "Yes" : null,
|
||||
"files_0|to_posix_lines": options.to_posix_lines ? "Yes" : null,
|
||||
"files_0|dbkey": options.genome || "?",
|
||||
"files_0|file_type": options.extension || "auto",
|
||||
"files_0|url_paste": options.url_paste,
|
||||
}),
|
||||
},
|
||||
payload: uploadModelsToPayload([options], history_id),
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
+23
-174
@@ -6,61 +6,23 @@ import _ from "underscore";
|
||||
import jQuery from "jquery";
|
||||
import { getAppRoot } from "onload/loadConfig";
|
||||
import * as tus from "tus-js-client";
|
||||
import axios from "axios";
|
||||
|
||||
function submitPayload(payload, cnf) {
|
||||
axios
|
||||
.post(`${getAppRoot()}api/tools/fetch`, payload)
|
||||
.then((response) => {
|
||||
cnf.success(response.data);
|
||||
})
|
||||
.catch((error) => {
|
||||
cnf.error(error.response.data.err_msg);
|
||||
});
|
||||
}
|
||||
|
||||
(($) => {
|
||||
// add event properties
|
||||
jQuery.event.props.push("dataTransfer");
|
||||
|
||||
/**
|
||||
xhr request helper
|
||||
*/
|
||||
var _uploadrequest = (config) => {
|
||||
var cnf = $.extend(
|
||||
{
|
||||
error_default: "Please make sure the file is available.",
|
||||
error_server: "Upload request failed.",
|
||||
error_login: "Uploads require you to log in.",
|
||||
error_retry: "Waiting for server to resume...",
|
||||
},
|
||||
config
|
||||
);
|
||||
console.debug(cnf);
|
||||
var xhr = new XMLHttpRequest();
|
||||
xhr.open("POST", cnf.url, true);
|
||||
xhr.setRequestHeader("Cache-Control", "no-cache");
|
||||
xhr.setRequestHeader("X-Requested-With", "XMLHttpRequest");
|
||||
xhr.setRequestHeader("Accept", "application/json");
|
||||
xhr.onreadystatechange = () => {
|
||||
if (xhr.readyState == xhr.DONE) {
|
||||
if ([502, 0].indexOf(xhr.status) !== -1 && cnf.warning) {
|
||||
cnf.warning(cnf.error_retry);
|
||||
} else if (xhr.status < 200 || xhr.status > 299) {
|
||||
var text = xhr.statusText;
|
||||
if (xhr.status == 403) {
|
||||
text = cnf.error_login;
|
||||
} else if (xhr.status == 0) {
|
||||
text = cnf.error_server;
|
||||
} else if (!text) {
|
||||
text = cnf.error_default;
|
||||
}
|
||||
cnf.error(`${text} (${xhr.status})`);
|
||||
} else {
|
||||
var response = null;
|
||||
if (xhr.responseText) {
|
||||
try {
|
||||
response = jQuery.parseJSON(xhr.responseText);
|
||||
} catch (e) {
|
||||
response = xhr.responseText;
|
||||
}
|
||||
}
|
||||
cnf.success(response);
|
||||
}
|
||||
}
|
||||
};
|
||||
xhr.upload.addEventListener("progress", cnf.progress, false);
|
||||
xhr.send(cnf.data);
|
||||
};
|
||||
|
||||
/**
|
||||
Posts chunked files to the API.
|
||||
*/
|
||||
@@ -91,13 +53,12 @@ import * as tus from "tus-js-client";
|
||||
cnf.error(data.error_message);
|
||||
return;
|
||||
}
|
||||
var file_data = data.files && data.files[0];
|
||||
if (!file_data) {
|
||||
cnf.error(cnf.error_file);
|
||||
return;
|
||||
const file = data.files && data.files[0];
|
||||
if (!file) {
|
||||
return submitPayload(data, cnf);
|
||||
}
|
||||
console.log("file is", file);
|
||||
const startTime = performance.now();
|
||||
const file = file_data.file;
|
||||
const tusEndpoint = `${getAppRoot()}api/upload/resumable_upload/`;
|
||||
const chunkSize = cnf.chunkSize;
|
||||
console.debug(`Starting chunked uploads [size=${chunkSize}].`);
|
||||
@@ -121,25 +82,13 @@ import * as tus from "tus-js-client";
|
||||
(performance.now() - startTime) / 1000
|
||||
} seconds`
|
||||
);
|
||||
|
||||
const toolInputs = JSON.parse(data.payload.inputs);
|
||||
toolInputs["files_0|file_data"] = {
|
||||
// drop files from data
|
||||
delete data["files"];
|
||||
data["files_0|file_data"] = {
|
||||
session_id: upload.url.split("/").at(-1),
|
||||
name: file.name,
|
||||
name: upload.file.name,
|
||||
};
|
||||
data.payload.inputs = JSON.stringify(toolInputs);
|
||||
$.ajax({
|
||||
url: `${getAppRoot()}api/tools`,
|
||||
method: "POST",
|
||||
data: data.payload,
|
||||
success: (tool_response) => {
|
||||
cnf.success(tool_response);
|
||||
},
|
||||
error: (tool_response) => {
|
||||
var err_msg = tool_response && tool_response.responseJSON && tool_response.responseJSON.err_msg;
|
||||
cnf.error(err_msg || cnf.error_tool);
|
||||
},
|
||||
});
|
||||
submitPayload(data, cnf);
|
||||
},
|
||||
});
|
||||
// Check if there are any previous uploads to continue.
|
||||
@@ -155,98 +104,6 @@ import * as tus from "tus-js-client";
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
Posts multiple files without chunking to the API.
|
||||
*/
|
||||
$.uploadpost = function (config) {
|
||||
var cnf = $.extend(
|
||||
{},
|
||||
{
|
||||
data: {},
|
||||
success: () => {},
|
||||
error: () => {},
|
||||
progress: () => {},
|
||||
url: null,
|
||||
maxfilesize: 1048576 * 2048,
|
||||
error_filesize: "File exceeds 2GB. Please use a FTP client.",
|
||||
},
|
||||
config
|
||||
);
|
||||
var data = cnf.data;
|
||||
if (data.error_message) {
|
||||
cnf.error(data.error_message);
|
||||
return;
|
||||
}
|
||||
|
||||
// construct form data
|
||||
var form = new FormData();
|
||||
for (const key in data.payload) {
|
||||
form.append(key, data.payload[key]);
|
||||
}
|
||||
|
||||
// add files to submission
|
||||
var sizes = 0;
|
||||
for (const key in data.files) {
|
||||
var d = data.files[key];
|
||||
form.append(d.name, d.file, d.file.name);
|
||||
sizes += d.file.size;
|
||||
}
|
||||
|
||||
// check file size, unless it's an ftp file
|
||||
if (sizes > cnf.maxfilesize) {
|
||||
cnf.error(cnf.error_filesize);
|
||||
return;
|
||||
}
|
||||
|
||||
// submit request
|
||||
_uploadrequest({
|
||||
url: cnf.url,
|
||||
data: form,
|
||||
success: cnf.success,
|
||||
error: cnf.error,
|
||||
progress: (e) => {
|
||||
if (e.lengthComputable) {
|
||||
cnf.progress(Math.round((e.loaded * 100) / e.total));
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
/**
|
||||
*
|
||||
* @param {*} config
|
||||
*/
|
||||
$.datafetchpost = function (config) {
|
||||
var cnf = $.extend(
|
||||
{},
|
||||
{
|
||||
data: {},
|
||||
success: () => {},
|
||||
error: () => {},
|
||||
progress: () => {},
|
||||
url: null,
|
||||
},
|
||||
config
|
||||
);
|
||||
var data = cnf.data;
|
||||
if (data.error_message) {
|
||||
cnf.error(data.error_message);
|
||||
return;
|
||||
}
|
||||
|
||||
// submit request
|
||||
_uploadrequest({
|
||||
url: cnf.url,
|
||||
data: JSON.stringify(cnf.data),
|
||||
success: cnf.success,
|
||||
error: cnf.error,
|
||||
progress: (e) => {
|
||||
if (e.lengthComputable) {
|
||||
cnf.progress(Math.round((e.loaded * 100) / e.total));
|
||||
}
|
||||
},
|
||||
});
|
||||
};
|
||||
|
||||
/**
|
||||
Handles the upload events drag/drop etc.
|
||||
*/
|
||||
@@ -398,24 +255,16 @@ import * as tus from "tus-js-client";
|
||||
break;
|
||||
}
|
||||
|
||||
// get current file from queue
|
||||
var file = queue[index];
|
||||
|
||||
// remove from queue
|
||||
remove(index);
|
||||
|
||||
// create and submit data
|
||||
var submitter = $.uploadpost;
|
||||
var submitter = $.uploadchunk;
|
||||
var requestData = opts.initialize(index);
|
||||
if (file.chunk_mode && opts.chunkSize > 0) {
|
||||
submitter = $.uploadchunk;
|
||||
} else if (requestData.fetchRequest) {
|
||||
submitter = $.datafetchpost;
|
||||
}
|
||||
|
||||
submitter({
|
||||
url: opts.initUrl(index),
|
||||
data: requestData.fetchRequest ? requestData.fetchRequest : requestData.uploadRequest,
|
||||
data: requestData,
|
||||
success: (message) => {
|
||||
opts.success(index, message);
|
||||
process();
|
||||
|
||||
Reference in New Issue
Block a user