From abc31492ddff770b3762fff70f3ff2199d7393de Mon Sep 17 00:00:00 2001 From: guerler Date: Thu, 10 Aug 2023 12:55:21 +0200 Subject: [PATCH] Fix ftp batch submission handler, adjust selenium tests --- client/src/components/Upload/Default.vue | 2 +- client/src/utils/navigation/navigation.yml | 43 ++++++------- client/src/utils/upload-queue.js | 61 +++++++++---------- client/src/utils/upload-submit.js | 6 +- .../test_upload_file_sources.py | 2 +- test/integration_selenium/test_upload_ftp.py | 25 ++++---- 6 files changed, 68 insertions(+), 71 deletions(-) diff --git a/client/src/components/Upload/Default.vue b/client/src/components/Upload/Default.vue index 86c4549ea43..763b406936b 100644 --- a/client/src/components/Upload/Default.vue +++ b/client/src/components/Upload/Default.vue @@ -269,7 +269,7 @@ function eventStart() { }); emit("progress", 0, "success"); counterRunning.value = counterAnnounce.value; - queue.start(true); + queue.start(); } } diff --git a/client/src/utils/navigation/navigation.yml b/client/src/utils/navigation/navigation.yml index 2a3dac48432..7d8d50aa33e 100644 --- a/client/src/utils/navigation/navigation.yml +++ b/client/src/utils/navigation/navigation.yml @@ -1032,35 +1032,32 @@ upload: close: 'div#composite div.upload-buttons button' selectors: - tab: '#tab-title-link-${tab}' - ftp_add: '#btn-ftp' - ftp_popup: '.upload-ftp-body' - ftp_items: '.upload-ftp-row' - ftp_close: '.popover-header .popover-close' - row: '#upload-row-${n}' - settings_button: '#upload-row-${n} .upload-settings span' - title: '#upload-row-${n} .upload-title' - paste_content: '#upload-row-${n} .upload-text-content' - settings: '.upload-settings-table' - setting_space_to_tab: '.upload-space-to-tab' - setting_deferred: '.upload-deferred' - start: '#activity-upload' - start_uploading: '#btn-start' - create: '#btn-new' + build_btn: '#rule-based #btn-build' close: '#btn-close' - rule_source_content: 'textarea.upload-rule-source-content' - rule_select_data_type: '.rule-data-type' + create: '#btn-new' + file_dialog: '#btn-remote-files' + file_dialog_ok: '.file-dialog-modal-ok' + file_source_selector: + type: xpath + selector: '//span[contains(@title, "${path}")]' + paste_content: '#upload-row-${n} .upload-text-content' + paste_new: .upload-paste + row: '#upload-row-${n}' rule_dataset_dialog: '[data-description="rules dataset dialog"]' rule_dataset_selector: selector: '.selection-dialog-modal' rule_dataset_selector_row: selector: '.selection-dialog-modal [aria-rowindex="${rowindex}"]' - build_btn: '#rule-based #btn-build' - file_source_selector: - type: xpath - selector: '//span[contains(@title, "${path}")]' - file_dialog_ok: '.file-dialog-modal-ok' - paste_new: .upload-paste + rule_select_data_type: '.rule-data-type' + rule_source_content: 'textarea.upload-rule-source-content' + tab: '#tab-title-link-${tab}' + title: '#upload-row-${n} .upload-title' + settings: '.upload-settings-table' + settings_button: '#upload-row-${n} .upload-settings span' + setting_deferred: '.upload-deferred' + setting_space_to_tab: '.upload-space-to-tab' + start: '#activity-upload' + start_uploading: '#btn-start' new_user_welcome: selectors: diff --git a/client/src/utils/upload-queue.js b/client/src/utils/upload-queue.js index a03cbe4e15c..09b305fbf38 100644 --- a/client/src/utils/upload-queue.js +++ b/client/src/utils/upload-queue.js @@ -1,12 +1,8 @@ /* galaxy upload utilities - requires FormData and XMLHttpRequest */ - -import axios from "axios"; -import { getAppRoot } from "onload/loadConfig"; - import { uploadPayload } from "@/utils/upload-payload.js"; -import { uploadSubmit } from "@/utils/upload-submit.js"; +import { sendPayload, uploadSubmit } from "@/utils/upload-submit.js"; export class UploadQueue { constructor(options) { @@ -70,34 +66,10 @@ export class UploadQueue { } // Initiate upload process - start(ftpBatch = false) { - if (ftpBatch) { - // package ftp files separately, and remove them from queue - const list = []; - Object.entries(this.queue).forEach(([key, model]) => { - if (model.status === "queued" && model.fileMode === "ftp") { - this.queue.remove(model.id); - list.push(this.opts.get(key)); - } - }); - if (list.length > 0) { - const data = uploadPayload(list, this.opts.historyId); - axios - .post(`${getAppRoot()}api/tools/fetch`, data) - .then((message) => { - list.forEach((model) => { - this.opts.success(model.id, message); - }); - }) - .catch((message) => { - list.forEach((model) => { - this.opts.error(model.id, message); - }); - }); - } - } + start() { if (!this.isRunning) { this.isRunning = true; + this._processFtp(); this._process(); } } @@ -127,6 +99,33 @@ export class UploadQueue { this._processSubmit(index, data); } + // Submit remote files as single batch request + _processFtp() { + const list = []; + for (const index of this.queue.keys()) { + const model = this.opts.get(index); + if (model.status === "queued" && model.fileMode === "ftp") { + list.push({ index, ...model }); + this.remove(index); + } + } + if (list.length > 0) { + const data = uploadPayload(list, this.opts.historyId); + sendPayload(data, { + success: (message) => { + list.forEach((model) => { + this.opts.success(model.index, message); + }); + }, + error: (message) => { + list.forEach((model) => { + this.opts.error(model.index, message); + }); + }, + }); + } + } + // Get next item to be processed _processIndex() { return this.queue.keys().next().value; diff --git a/client/src/utils/upload-submit.js b/client/src/utils/upload-submit.js index 9f3af3dab65..14625d20a62 100644 --- a/client/src/utils/upload-submit.js +++ b/client/src/utils/upload-submit.js @@ -12,7 +12,7 @@ function buildFingerprint(cnf) { }; } -function submitPayload(payload, cnf) { +export function sendPayload(payload, cnf) { axios .post(`${getAppRoot()}api/tools/fetch`, payload) .then((response) => { @@ -31,7 +31,7 @@ function tusUpload(uploadables, index, data, tusEndpoint, cnf) { if (!uploadable) { // We've uploaded all files or blobs; delete files from data and submit fetch payload delete data["files"]; - return submitPayload(data, cnf); + return sendPayload(data, cnf); } console.debug(`Starting chunked upload for ${uploadable.name} [chunkSize=${chunkSize}].`); const upload = new tus.Upload(uploadable, { @@ -115,7 +115,7 @@ export function uploadSubmit(config) { if (data.targets.length && data.targets[0].elements.length) { const pasted = data.targets[0].elements[0]; if (pasted.src == "url") { - return submitPayload(data, cnf); + return sendPayload(data, cnf); } else { const blob = new Blob([pasted.paste_content]); blob.name = data.targets[0].elements[0].name || "default"; diff --git a/test/integration_selenium/test_upload_file_sources.py b/test/integration_selenium/test_upload_file_sources.py index dc8ec613bac..a6fa321fa15 100644 --- a/test/integration_selenium/test_upload_file_sources.py +++ b/test/integration_selenium/test_upload_file_sources.py @@ -20,7 +20,7 @@ class TestPosixFileSourceSeleniumIntegration(PosixFileSourceSetup, SeleniumInteg def test_upload_from_posix(self): self.admin_login() self.components.upload.start.wait_for_and_click() - self.components.upload.ftp_add.wait_for_and_click() + self.components.upload.file_dialog.wait_for_and_click() self.components.upload.file_source_selector(path="gxfiles://posix_test").wait_for_and_click() self.components.upload.file_source_selector(path="gxfiles://posix_test/a").wait_for_and_click() self.components.upload.file_dialog_ok.wait_for_and_click() diff --git a/test/integration_selenium/test_upload_ftp.py b/test/integration_selenium/test_upload_ftp.py index ce4d0fa1707..c851783116a 100644 --- a/test/integration_selenium/test_upload_ftp.py +++ b/test/integration_selenium/test_upload_ftp.py @@ -21,15 +21,16 @@ class TestUploadFtpSeleniumIntegration(SeleniumIntegrationTestCase): def ftp_dir(cls): return cls.temp_config_dir("ftp") - def _upload_all(self): + def _upload_all(self, n): self.home() self.components.upload.start.wait_for_and_click() - self.components.upload.ftp_add.wait_for_and_click() - self.components.upload.ftp_popup.wait_for_visible() - for item in self.components.upload.ftp_items().all(): - item.click() - self.components.upload.ftp_close.wait_for_and_click() - self.components.upload.row(n=0).wait_for_visible() + self.components.upload.file_dialog.wait_for_and_click() + self.components.upload.file_source_selector(path="gxftp://").wait_for_and_click() + for i in range(n): + self.components.upload.file_source_selector(path="gxftp://%i.txt" % i).wait_for_and_click() + self.components.upload.file_dialog_ok.wait_for_and_click() + for i in range(n): + self.components.upload.row(n=i).wait_for_visible() self.upload_start() self.sleep_for(self.wait_types.UX_RENDER) self.wait_for_history() @@ -43,18 +44,18 @@ class TestUploadFtpSeleniumIntegration(SeleniumIntegrationTestCase): @selenium_test def test_upload_simplest(self): user_ftp_dir = self._create_ftp_dir() - file_path = os.path.join(user_ftp_dir, "1.txt") + file_path = os.path.join(user_ftp_dir, "0.txt") with open(file_path, "w") as f: f.write("Hello World!") - self._upload_all() + self._upload_all(1) @selenium_test def test_upload_multiple(self): user_ftp_dir = self._create_ftp_dir() - file_path = os.path.join(user_ftp_dir, "1.txt") + file_path = os.path.join(user_ftp_dir, "0.txt") with open(file_path, "w") as f: f.write("Hello World!") - file_path = os.path.join(user_ftp_dir, "2.txt") + file_path = os.path.join(user_ftp_dir, "1.txt") with open(file_path, "w") as f: f.write("Hello Galaxy!") - self._upload_all() + self._upload_all(2)