Merge branch 'release_21.01' into dev

This commit is contained in:
Nicola Soranzo
2021-01-19 17:51:43 +00:00
22 changed files with 260 additions and 46 deletions
@@ -1,5 +1,6 @@
<template>
<b-form-input v-model="localValue" :placeholder="placeholder" @click="selectFile"> </b-form-input>
<b-form-input class="directory-form-input" v-model="localValue" :placeholder="placeholder" @click="selectFile">
</b-form-input>
</template>
<script>
@@ -1,11 +1,12 @@
<template>
<span>
<b
><a :href="link">{{ link }}</a></b
><a class="generated-export-link" :href="link">{{ link }}</a></b
>
<font-awesome-icon
v-b-tooltip.hover
title="Copy export URL to your clipboard"
class="copy-export-link"
icon="link"
style="cursor: pointer;"
@click="copyUrl"
@@ -13,9 +14,9 @@
<i
title="Information about when the history export was generated is included in the job details. Additionally, if there are issues with export, the job details may help figure out the underlying problem or communicate issues to your Galaxy administrator."
>
(<a href="#" @click="showDetails">view job details</a>)
(<a class="show-job-link" href="#" @click="showDetails">view job details</a>)
</i>
<b-modal v-model="details" scrollable ok-only>
<b-modal v-model="details" modal-class="job-information-modal" scrollable ok-only hide-header>
<job-information :job_id="historyExport.job_id" :include-times="true" />
</b-modal>
</span>
@@ -1,5 +1,5 @@
<template>
<span>
<span class="history-export-component">
<h2>Export history archive</h2>
<span v-if="initializing">
<loading-span message="Loading server configuration." />
@@ -7,13 +7,15 @@
<span v-else-if="hasWritableFileSources">
<b-card no-body>
<b-tabs pills card vertical>
<b-tab title="to a link" active>
<b-tab title="to a link" title-link-class="tab-export-to-link" active>
<b-card-text>
<ToLink :history-id="historyId" />
</b-card-text>
</b-tab>
<b-tab title="to a remote file">
<ToRemoteFile :history-id="historyId" />
<b-tab title="to a remote file" title-link-class="tab-export-to-file">
<b-card-text>
<ToRemoteFile :history-id="historyId" />
</b-card-text>
</b-tab>
</b-tabs>
</b-card>
@@ -153,10 +153,7 @@ export default {
waitOnJob(jobId)
.then((jobResponse) => {
this.waitingOnJob = false;
this.loadingExports = true;
// Race condition, for some reasons the job API returns
// ok before the JEHA...
setTimeout(this.loadExports, 2000);
this.loadExports();
})
.catch(this.handleError);
},
@@ -1,5 +1,5 @@
<template>
<div>
<div class="export-to-remote-file">
<b-alert v-if="errorMessage" show variant="danger" dismissible @dismissed="errorMessage = null">
{{ errorMessage }}
<JobError
@@ -20,14 +20,14 @@
</div>
<div v-else>
<b-form-group
id="directory"
id="fieldset-directory"
label-for="directory"
description="Select a 'remote files' directory to export history archive to."
class="mt-3"
>
<files-input v-model="directory" mode="directory" :requireWritable="true" />
<files-input id="directory" v-model="directory" mode="directory" :requireWritable="true" />
</b-form-group>
<b-form-group id="name" label-for="name" description="Give the exported file a name." class="mt-3">
<b-form-group id="fieldset-name" label-for="name" description="Give the exported file a name." class="mt-3">
<b-form-input id="name" v-model="name" placeholder="Name" required></b-form-input>
</b-form-group>
<b-row align-h="end">
+1 -1
View File
@@ -1,5 +1,5 @@
<template>
<b-card title="Import a history from an archive">
<b-card body-class="history-import-component" title="Import a history from an archive">
<b-alert v-if="errorMessage" variant="danger" dismissible @dismissed="errorMessage = null" show>
{{ errorMessage }}
<JobError
@@ -11,7 +11,7 @@
</div>
<!-- TODO: modal for reporting error. -->
</b-card-text>
<b-modal v-model="showInfo" scrollable ok-only>
<b-modal v-model="showInfo" modal-class="job-information-modal" scrollable ok-only hide-header>
<job-information :job_id="job.id" :include-times="true" />
</b-modal>
</b-card>
+5 -2
View File
@@ -1731,7 +1731,11 @@ class JobWrapper(HasResourceParameters):
# Certain tools require tasks to be completed after job execution
# ( this used to be performed in the "exec_after_process" hook, but hooks are deprecated ).
param_dict = self.get_param_dict(job)
self.tool.exec_after_process(self.app, inp_data, out_data, param_dict, job=job, final_job_state=final_job_state)
try:
self.tool.exec_after_process(self.app, inp_data, out_data, param_dict, job=job, final_job_state=final_job_state)
except Exception:
log.exception(f"exec_after_process hook failed for job {self.job_id}")
final_job_state = job.states.ERROR
# Call 'exec_after_process' hook
self.tool.call_hook('exec_after_process', self.app, inp_data=inp_data,
out_data=out_data, param_dict=param_dict,
@@ -1834,7 +1838,6 @@ class JobWrapper(HasResourceParameters):
if e.errno != errno.ENOENT:
raise
self.external_output_metadata.cleanup_external_metadata(self.sa_session)
galaxy.tools.imp_exp.JobImportHistoryArchiveWrapper(self.app, self.job_id).cleanup_after_job()
if delete_files:
self.object_store.delete(self.get_job(), base_dir='job_work', entire_dir=True, dir_only=True, obj_dir=True)
except Exception:
+1 -1
View File
@@ -626,7 +626,7 @@ class ModelImportStore(metaclass=abc.ABCMeta):
if input_key in object_import_tracker.hdcas_by_key:
hdca = object_import_tracker.hdcas_by_key[input_key]
if input_key in object_import_tracker.hdca_copied_from_sinks:
hdca = object_import_tracker.hdca_copied_from_sinks[input_key]
hdca = object_import_tracker.hdcas_by_key[object_import_tracker.hdca_copied_from_sinks[input_key]]
return hdca
def _find_dce(input_key):
+36 -2
View File
@@ -517,14 +517,24 @@ class NavigatesGalaxy(HasDriver):
center_element = self.driver.find_element_by_css_selector("#center")
action_chains.move_to_element(center_element).click().perform()
def perform_upload(self, test_path, ext=None, genome=None, ext_all=None, genome_all=None):
def perform_upload(self, test_path, **kwd):
self._perform_upload(test_path=test_path, **kwd)
def perform_upload_of_pasted_content(self, paste_data, **kwd):
self._perform_upload(paste_data=paste_data, **kwd)
def _perform_upload(self, test_path=None, paste_data=None, ext=None, genome=None, ext_all=None, genome_all=None):
self.home()
self.upload_start_click()
self.upload_set_footer_extension(ext_all)
self.upload_set_footer_genome(genome_all)
self.upload_queue_local_file(test_path)
if test_path:
self.upload_queue_local_file(test_path)
else:
assert paste_data is not None
self.upload_paste_data(paste_data)
if ext is not None:
self.wait_for_selector_visible('.upload-extension')
@@ -635,6 +645,13 @@ class NavigatesGalaxy(HasDriver):
file_upload = self.wait_for_selector('div#%s input[type="file"]' % tab_id)
file_upload.send_keys(test_path)
def upload_paste_data(self, pasted_content, tab_id="regular"):
tab_locator = f"div#{tab_id}"
self.wait_for_and_click_selector(f"{tab_locator} button#btn-new")
textarea = self.wait_for_selector(f"{tab_locator} .upload-text-content")
textarea.send_keys(pasted_content)
def upload_rule_start(self):
self.upload_start_click()
self.upload_tab_click("rule-based")
@@ -865,6 +882,11 @@ class NavigatesGalaxy(HasDriver):
self.wait_for_and_click_selector("#workflow-save-button")
self.sleep_for(self.wait_types.DATABASE_OPERATION)
def navigate_to_histories_page(self):
self.home()
self.click_masthead_user()
self.components.masthead.histories.wait_for_and_click()
def navigate_to_user_preferences(self):
self.home()
self.click_masthead_user()
@@ -1169,6 +1191,18 @@ class NavigatesGalaxy(HasDriver):
search_selector = '#standard-search .advanced-search-toggle'
self.wait_for_and_click_selector(search_selector)
@retry_during_transitions
def histories_get_history_names(self):
self.sleep_for(self.wait_types.UX_RENDER)
names = []
grid = self.wait_for_selector('#grid-table-body')
for row in grid.find_elements_by_tag_name('tr'):
td = row.find_elements_by_tag_name('td')
name = td[1].text if td[0].text == '' else td[0].text
if name != "No items" and not name.startswith("No matching entries found"):
names.append(name)
return names
def history_panel_add_tags(self, tags):
tag_icon_selector = self.components.history_panel.tag_icon
tag_area_selector = self.components.history_panel.tag_area
+33
View File
@@ -188,6 +188,9 @@ history_panel:
options_show_history_structure:
type: xpath
selector: '//a[text()="Show Structure"]'
options_show_export_history_to_file:
type: xpath
selector: '//a[text()="Export History to File"]'
new_history_button: '#history-new-button'
multi_view_button: '#history-view-multi-button'
@@ -283,6 +286,9 @@ collection_builders:
reverse_datasets: ".reverse-column .column-datasets"
histories:
labels:
import_button: 'Import from file'
sharing:
selectors:
unshare_user_button: '.unshare_user'
@@ -292,6 +298,33 @@ histories:
labels:
unshare: 'Unshare'
files_dialog:
selectors:
ftp_row: 'span[title="gxftp://"]'
row: 'span[title="${uri}"]'
history_export:
selectors:
export_link: '.export-link'
running: '.history-export-component .loading-icon'
generated_export_link: '.generated-export-link'
copy_export_link: '.copy-export-link'
show_job_link: '.show-job-link'
job_table: '.info_data_table'
job_table_ok: '.job-information-modal .btn-primary'
tab_export_to_file: '.tab-export-to-file'
directory_input: '.directory-form-input'
name_input: '.export-to-remote-file #name'
export_button: '.export-button'
success_message: '.history-export-component .alert-success'
history_import:
selectors:
radio_button_remote_files: '.history-import-component .fa-folder-open'
import_button: '.import-button'
running: '.history-import-component .loading-icon'
success_message: '.history-import-component .alert-success'
pages:
selectors:
create: '.manage-table-actions .action-button'
+7
View File
@@ -55,6 +55,7 @@ from galaxy.tools.actions.data_manager import DataManagerToolAction
from galaxy.tools.actions.data_source import DataSourceToolAction
from galaxy.tools.actions.model_operations import ModelOperationToolAction
from galaxy.tools.cache import ToolDocumentCache
from galaxy.tools.imp_exp import JobImportHistoryArchiveWrapper
from galaxy.tools.parameters import (
check_param,
params_from_strings,
@@ -2649,6 +2650,12 @@ class ExportHistoryTool(Tool):
class ImportHistoryTool(Tool):
tool_type = 'import_history'
def exec_after_process(self, app, inp_data, out_data, param_dict, job, final_job_state=None, **kwds):
super().exec_after_process(app, inp_data, out_data, param_dict, job=job, **kwds)
if final_job_state != DETECTED_JOB_STATE.OK:
return
JobImportHistoryArchiveWrapper(self.app, job.id).cleanup_after_job()
class InteractiveTool(Tool):
tool_type = 'interactive'
+2 -1
View File
@@ -107,10 +107,11 @@ def execute(trans, tool, mapping_params, history, rerun_remap_job_id=None, colle
else:
# Make sure collections, implicit jobs etc are flushed even if there are no precreated output datasets
trans.sa_session.flush()
tool_id = tool.id
for job in execution_tracker.successful_jobs:
# Put the job in the queue if tracking in memory
tool.app.job_manager.enqueue(job, tool=tool, flush=False)
trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id)
trans.log_event("Added job to the job queue, id: %s" % str(job.id), tool_id=tool_id)
trans.sa_session.flush()
if has_remaining_jobs:
+1 -2
View File
@@ -1,4 +1,3 @@
import copy
import json
import logging
import math
@@ -70,7 +69,7 @@ def safe_dumps(*args, **kwargs):
try:
dumped = json.dumps(*args, allow_nan=False, **kwargs)
except ValueError:
obj = swap_inf_nan(copy.deepcopy(args[0]))
obj = swap_inf_nan(args[0])
dumped = json.dumps(obj, allow_nan=False, **kwargs)
if kwargs.get('escape_closing_tags', True):
return dumped.replace('</', '<\\/')
+1 -1
View File
@@ -425,7 +425,7 @@ class ConfiguresHandlers:
log.debug("(%s) No handler pool (uWSGI farm) for '%s' found", obj.log_str(), tag)
raise HandlerAssignmentSkip()
else:
if flush:
if flush or not obj.id:
_timed_flush_obj(obj)
message = message_callback()
self.app.application_stack.send_message(pool, message)
+34 -3
View File
@@ -129,12 +129,14 @@ def extract_steps(trans, history=None, job_ids=None, dataset_ids=None, dataset_c
assoc_name = assoc.name
if ToolOutputCollectionPart.is_named_collection_part_name(assoc_name):
continue
if assoc_name.startswith("__new_primary_file"):
continue
if job in summary.implicit_map_jobs:
hid = None
for implicit_pair in jobs[job]:
query_assoc_name, dataset_collection = implicit_pair
if query_assoc_name == assoc_name or assoc_name.startswith("__new_primary_file_%s|" % query_assoc_name):
hid = dataset_collection.hid
hid = summary.hid(dataset_collection)
if hid is None:
template = "Failed to find matching implicit job - job id is %s, implicit pairs are %s, assoc_name is %s."
message = template % (job.id, jobs[job], assoc_name)
@@ -142,9 +144,12 @@ def extract_steps(trans, history=None, job_ids=None, dataset_ids=None, dataset_c
raise Exception("Failed to extract job.")
else:
if hasattr(assoc, "dataset"):
hid = assoc.dataset.hid
has_hid = assoc.dataset
else:
hid = assoc.dataset_collection_instance.hid
has_hid = assoc.dataset_collection_instance
hid = summary.hid(has_hid)
if hid in hid_to_output_pair:
log.warning("duplicate hid found in extract_steps [%s]" % hid)
hid_to_output_pair[hid] = (step, assoc.name)
return steps
@@ -196,8 +201,29 @@ class WorkflowSummary:
self.implicit_map_jobs = []
self.collection_types = {}
self.hda_hid_in_history = {}
self.hdca_hid_in_history = {}
self.__summarize()
def hid(self, object):
if object.history_content_type == "dataset_collection":
if object.id in self.hdca_hid_in_history:
return self.hdca_hid_in_history[object.id]
elif object.history == self.history:
return object.hid
else:
log.warning("extraction issue, using hdca hid from outside current history and unmapped")
return object.hid
else:
if object.id in self.hda_hid_in_history:
return self.hda_hid_in_history[object.id]
elif object.history == self.history:
return object.hid
else:
log.warning("extraction issue, using hda hid from outside current history and unmapped")
return object.hid
def __summarize(self):
# Make a first pass handle all singleton jobs, input dataset and dataset collections
# just grab the implicitly mapped jobs and handle in second pass. Second pass is
@@ -215,7 +241,10 @@ class WorkflowSummary:
self.__summarize_dataset(content)
def __summarize_dataset_collection(self, dataset_collection):
hid_in_history = dataset_collection.hid
dataset_collection = self.__original_hdca(dataset_collection)
self.hdca_hid_in_history[dataset_collection.id] = hid_in_history
hid = dataset_collection.hid
self.collection_types[hid] = dataset_collection.collection.collection_type
cja = dataset_collection.creating_job_associations
@@ -264,7 +293,9 @@ class WorkflowSummary:
if not self.__check_state(dataset):
return
hid_in_history = dataset.hid
original_hda = self.__original_hda(dataset)
self.hda_hid_in_history[original_hda.id] = hid_in_history
if not original_hda.creating_job_associations:
self.jobs[FakeJob(dataset)] = [(None, dataset)]
@@ -1,6 +1,5 @@
from .framework import (
retry_assertion_during_transitions,
retry_during_transitions,
selenium_test,
SharedStateSeleniumTestCase,
)
@@ -251,17 +250,8 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase):
else:
self.assertEqual(intersection, set())
@retry_during_transitions
def get_histories(self):
self.sleep_for(self.wait_types.UX_RENDER)
names = []
grid = self.wait_for_selector('#grid-table-body')
for row in grid.find_elements_by_tag_name('tr'):
td = row.find_elements_by_tag_name('td')
name = td[1].text if td[0].text == '' else td[0].text
if name != "No items" and not name.startswith("No matching entries found"):
names.append(name)
return names
return self.histories_get_history_names()
def set_filter(self, selector, value):
filter_input = self.wait_for_selector_clickable(selector)
@@ -275,11 +265,6 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase):
self.wait_for_and_click_selector(close_button_selector)
self.sleep_for(self.wait_types.UX_RENDER)
def navigate_to_histories_page(self):
self.home()
self.click_masthead_user() # Open masthead menu
self.components.masthead.histories.wait_for_and_click()
def setup_shared_state(self):
SavedHistoriesTestCase.user_email = self._get_random_email()
SavedHistoriesTestCase.history1_name = self._get_random_name()
@@ -0,0 +1,44 @@
from .framework import (
selenium_test,
SeleniumTestCase
)
class HistoryExportTestCase(SeleniumTestCase):
ensure_registered = True
@selenium_test
def test_history_export(self):
gx_selenium_context = self
gx_selenium_context.perform_upload_of_pasted_content("my cool content")
gx_selenium_context.history_panel_wait_for_hid_ok(1)
gx_selenium_context.click_history_options()
gx_selenium_context.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
history_export = gx_selenium_context.components.history_export
history_export.export_link.wait_for_and_click()
history_export.running.wait_for_visible()
history_export.running.wait_for_absent(wait_type=gx_selenium_context.wait_types.JOB_COMPLETION)
history_export.generated_export_link.wait_for_visible()
history_export.copy_export_link.wait_for_visible()
history_export.job_table.assert_absent_or_hidden()
history_export.show_job_link.wait_for_and_click()
history_export.job_table.wait_for_present()
history_export.job_table_ok.wait_for_and_click()
history_export.job_table.wait_for_absent()
gx_selenium_context.click_history_options()
gx_selenium_context.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
# this time the exported link is still there
history_export.generated_export_link.wait_for_visible()
history_export.export_link.assert_absent()
gx_selenium_context.perform_upload_of_pasted_content("my cool content part 2")
gx_selenium_context.history_panel_wait_for_hid_ok(2)
gx_selenium_context.click_history_options()
gx_selenium_context.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
# now we have a generated link and a link to update to the newest export
history_export.generated_export_link.wait_for_visible()
history_export.export_link.wait_for_visible()
@@ -1,5 +1,7 @@
import json
import pytest
from galaxy.selenium.navigates_galaxy import retry_call_during_transitions
from galaxy_test.base import rules_test_data
from galaxy_test.base.populators import (
@@ -183,6 +185,8 @@ class LoggedInToolFormTestCase(SeleniumTestCase):
@selenium_test
@managed_history
@skip_if_github_down
@pytest.mark.gtn_screenshot
@pytest.mark.local
def test_run_apply_rules_tutorial(self):
self.home()
self.upload_rule_start()
+6
View File
@@ -213,6 +213,8 @@ PRJDA60709 SAMD00016382 DRX000480 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR0
self.screenshot("rules_example_3_14_paired_identifier_set")
@selenium_test
@pytest.mark.gtn_screenshot
@pytest.mark.local
def test_rules_example_4_accessions(self):
# http://www.uniprot.org/uniprot/?query=proteome:UP000052092+AND+proteomecomponent:%22Genome%22
self._setup_uniprot_example()
@@ -245,6 +247,8 @@ PRJDA60709 SAMD00016382 DRX000480 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR0
rule_builder.view_source.wait_for_visible()
@selenium_test
@pytest.mark.gtn_screenshot
@pytest.mark.local
def test_rules_example_5_matching_collections(self):
self._setup_uniprot_example()
self.screenshot("rules_example_5_1_inputs")
@@ -273,6 +277,8 @@ PRJDA60709 SAMD00016382 DRX000480 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR0
self.screenshot("rules_example_5_9_mapping")
@selenium_test
@pytest.mark.gtn_screenshot
@pytest.mark.local
def test_rules_example_6_nested_lists(self):
self.home()
self.perform_upload(self.get_filename("rules/PRJNA355367.tsv"))
@@ -0,0 +1,65 @@
import os
from .framework import (
selenium_test,
SeleniumIntegrationTestCase
)
class HistoryImportExportFtpSeleniumIntegrationTestCase(SeleniumIntegrationTestCase):
ensure_registered = True
@classmethod
def handle_galaxy_config_kwds(cls, config):
ftp_dir = cls.ftp_dir()
os.makedirs(ftp_dir)
config["ftp_upload_dir"] = ftp_dir
config["ftp_upload_site"] = "ftp://ftp.galaxyproject.com"
@classmethod
def ftp_dir(cls):
return cls.temp_config_dir("ftp")
@selenium_test
def test_history_import_export(self):
email = self.get_logged_in_user()["email"]
user_ftp_dir = os.path.join(self.ftp_dir(), email)
os.makedirs(user_ftp_dir)
gx_selenium_context = self
gx_selenium_context.perform_upload_of_pasted_content("my cool content")
gx_selenium_context.history_panel_wait_for_hid_ok(1)
gx_selenium_context.click_history_options()
gx_selenium_context.components.history_panel.options_show_export_history_to_file.wait_for_and_click()
history_export = gx_selenium_context.components.history_export
files_dialog = gx_selenium_context.components.files_dialog
# we land on link version, but go to export to file
history_export.export_link.wait_for_visible()
history_export.tab_export_to_file.wait_for_and_click()
history_export.export_link.wait_for_absent_or_hidden()
history_export.directory_input.wait_for_and_click()
files_dialog.ftp_row.wait_for_and_click()
history_export.name_input.wait_for_and_send_keys("my_export.tar.gz")
history_export.export_button.wait_for_and_click()
history_export.running.wait_for_visible()
history_export.running.wait_for_absent(wait_type=gx_selenium_context.wait_types.JOB_COMPLETION)
history_export.success_message.wait_for_visible()
gx_selenium_context.navigate_to_histories_page()
gx_selenium_context.components.histories.import_button.wait_for_and_click()
history_import = gx_selenium_context.components.history_import
history_import.radio_button_remote_files.wait_for_and_click()
files_dialog.ftp_row.wait_for_and_click()
files_dialog.row(uri="gxftp://my_export.tar.gz").wait_for_and_click()
history_import.import_button.wait_for_and_click()
history_import.running.wait_for_visible()
history_import.running.wait_for_absent(wait_type=gx_selenium_context.wait_types.JOB_COMPLETION)
history_import.success_message.wait_for_visible()
gx_selenium_context.navigate_to_histories_page()
newest_history_name = gx_selenium_context.histories_get_history_names()[0]
assert newest_history_name.startswith("imported from archive")
@@ -110,6 +110,7 @@ class MockTrans:
class MockHda:
def __init__(self, state='ok', output_name='out1', job=None):
self.hid = 1
self.id = 123
self.state = state
self.copied_from_history_dataset_association = None