mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
John Chilton
083bdcbdd8
Vue-based component for defining collections by applying rules to a list of files or more general spreadsheet style information (e.g. sample sheets or tabular data from data sources containing URL or FTP file paths for files along with metadata). The widget is fairly complex but very broadly is broken into two panes - one to preview how rules are applied to build up tabular data defining collections (each row corresponding to a file with columns for metadata and such) and one that displays defined rules and allows for editing of these rules and creation of new ones. The goal behind defining rules this way instead of allowing the user to interact with the spreadsheet display directly is to enable scaling up collection creation. If a user wishes to upload hundreds of datasets - interacting with a widget directly for each input doesn't scale well and would be error prone. If a user wishes to upload hundreds of thousands of datasets - even loading this information in the GUI may not scale (though I've been impressed with the performance so far of this approach) and so we can potentially just display a preview of some of the rows and process the final set of rules on the backend. Since we can handle an arbitrary number of columns this way, we can define multiple list identifiers per file and so we can easily construct nested lists. Hence this allows creation of not just potentially larger collections but arbitrarily complex lists as well. Paired identifiers via indicator columns are also implemented. In order to operate over lists of datasets directly - the multi-select history widget now has a new option "Build Collection from Rules" along side the other collection builders. This mode uses the well established dataset collection API to build collections from HDAs. In order to operate on lists of FTP files or URLs - the upload widget has a new tab "Rule-based" tab that allows users to paste in tabular data or select a history dataset and then send this tabular data to the new builder widget. This will be extended to include FTP directories for instance over time. This mode uses the new data fetch API to build collections and handle uploads of arbitrary collections of files. The preview of the tabular data generated via rules is done via [Handsontable](https://handsontable.com/) - a JavaScript spreadsheet widget with a VueJS [wrapper component](https://github.com/handsontable/vue-handsontable-official). This turns out to be a fairly nice application for reactive components - as rules are added or modified the spreadsheet just naturally updates. In my hands the widget scales very nicely - I've uploaded files with tens of thousands of rows and rules modifying the data and changing the spreadsheet do not seem to cause siignificant delays in the web browser.
329 lines
15 KiB
Python
329 lines
15 KiB
Python
import os
|
|
|
|
from selenium.webdriver.common.keys import Keys
|
|
|
|
from .framework import (
|
|
selenium_test,
|
|
SeleniumTestCase,
|
|
UsesHistoryItemAssertions,
|
|
)
|
|
|
|
|
|
class UploadsTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
|
|
|
|
@selenium_test
|
|
def test_upload_simplest(self):
|
|
self.perform_upload(self.get_filename("1.sam"))
|
|
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
history_contents = self.current_history_contents()
|
|
history_count = len(history_contents)
|
|
assert history_count == 1, "Incorrect number of items in history - expected 1, found %d" % history_count
|
|
|
|
hda = history_contents[0]
|
|
assert hda["name"] == '1.sam', hda
|
|
assert hda["extension"] == "sam", hda
|
|
|
|
self.history_panel_click_item_title(hid=1, wait=True)
|
|
self.assert_item_dbkey_displayed_as(1, "?")
|
|
|
|
@selenium_test
|
|
def test_upload_specify_ext(self):
|
|
self.perform_upload(self.get_filename("1.sam"), ext="txt")
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
history_contents = self.current_history_contents()
|
|
hda = history_contents[0]
|
|
assert hda["name"] == '1.sam'
|
|
assert hda["extension"] == "txt", hda
|
|
|
|
@selenium_test
|
|
def test_upload_specify_genome(self):
|
|
self.perform_upload(self.get_filename("1.sam"), genome="hg18")
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
|
|
self.history_panel_click_item_title(hid=1, wait=True)
|
|
self.assert_item_dbkey_displayed_as(1, "hg18")
|
|
|
|
@selenium_test
|
|
def test_upload_specify_ext_all(self):
|
|
self.perform_upload(self.get_filename("1.sam"), ext_all="txt")
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
history_contents = self.current_history_contents()
|
|
hda = history_contents[0]
|
|
assert hda["name"] == '1.sam'
|
|
assert hda["extension"] == "txt", hda
|
|
|
|
@selenium_test
|
|
def test_upload_specify_genome_all(self):
|
|
self.perform_upload(self.get_filename("1.sam"), genome_all="hg18")
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
|
|
self.history_panel_click_item_title(hid=1, wait=True)
|
|
self.assert_item_dbkey_displayed_as(1, "hg18")
|
|
|
|
@selenium_test
|
|
def test_upload_list(self):
|
|
self.upload_list([self.get_filename("1.tabular")], name="Test List")
|
|
self.history_panel_wait_for_hid_ok(2)
|
|
# Make sure modals disappeared - both List creator (TODO: upload).
|
|
self.wait_for_selector_absent_or_hidden(".collection-creator")
|
|
|
|
self.assert_item_name(2, "Test List")
|
|
|
|
# Make sure source item is hidden when the collection is created.
|
|
self.history_panel_wait_for_hid_hidden(1)
|
|
|
|
@selenium_test
|
|
def test_upload_pair(self):
|
|
self.upload_list([self.get_filename("1.tabular"), self.get_filename("2.tabular")], name="Test Pair")
|
|
self.history_panel_wait_for_hid_ok(3)
|
|
# Make sure modals disappeared - both collection creator (TODO: upload).
|
|
self.wait_for_selector_absent_or_hidden(".collection-creator")
|
|
|
|
self.assert_item_name(3, "Test Pair")
|
|
|
|
# Make sure source items are hidden when the collection is created.
|
|
self.history_panel_wait_for_hid_hidden(1)
|
|
self.history_panel_wait_for_hid_hidden(2)
|
|
|
|
@selenium_test
|
|
def test_upload_pair_specify_extension(self):
|
|
self.upload_list([self.get_filename("1.tabular"), self.get_filename("2.tabular")], name="Test Pair", ext="txt", hide_source_items=False)
|
|
self.history_panel_wait_for_hid_ok(3)
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
|
|
history_contents = self.current_history_contents()
|
|
hda = history_contents[0]
|
|
assert hda["name"] == '1.tabular'
|
|
assert hda["extension"] == "txt", hda
|
|
|
|
@selenium_test
|
|
def test_upload_paired_list(self):
|
|
self.upload_paired_list([self.get_filename("1.tabular"), self.get_filename("2.tabular")], name="Test Paired List")
|
|
self.history_panel_wait_for_hid_ok(3)
|
|
# Make sure modals disappeared - both collection creator (TODO: upload).
|
|
self.wait_for_selector_absent_or_hidden(".collection-creator")
|
|
self.assert_item_name(3, "Test Paired List")
|
|
|
|
# Make sure source items are hidden when the collection is created.
|
|
self.history_panel_wait_for_hid_hidden(1)
|
|
self.history_panel_wait_for_hid_hidden(2)
|
|
|
|
@selenium_test
|
|
def test_rules_example_1_datasets(self):
|
|
# Test case generated for:
|
|
# https://www.ebi.ac.uk/ena/data/view/PRJDA60709
|
|
self.home()
|
|
self.upload_rule_start()
|
|
self.screenshot("rules_example_1_1_rules_landing")
|
|
self.components.upload.rule_source_content.wait_for_and_send_keys("""study_accession sample_accession experiment_accession fastq_ftp
|
|
PRJDA60709 SAMD00016379 DRX000475 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000770/DRR000770.fastq.gz
|
|
PRJDA60709 SAMD00016383 DRX000476 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000771/DRR000771.fastq.gz
|
|
PRJDA60709 SAMD00016380 DRX000477 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000772/DRR000772.fastq.gz
|
|
PRJDA60709 SAMD00016378 DRX000478 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000773/DRR000773.fastq.gz
|
|
PRJDA60709 SAMD00016381 DRX000479 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000774/DRR000774.fastq.gz
|
|
PRJDA60709 SAMD00016382 DRX000480 ftp.sra.ebi.ac.uk/vol1/fastq/DRR000/DRR000775/DRR000775.fastq.gz""")
|
|
self.screenshot("rules_example_1_2_paste")
|
|
self.upload_rule_build()
|
|
rule_builder = self.components.rule_builder
|
|
rule_builder._.wait_for_and_click()
|
|
self.screenshot("rules_example_1_3_initial_rules")
|
|
rule_builder.menu_button_filter.wait_for_and_click()
|
|
self.screenshot("rule_builder_filters")
|
|
rule_builder.menu_item_rule_type(rule_type="add-filter-count").wait_for_and_click()
|
|
filter_editor = rule_builder.rule_editor(rule_type="add-filter-count")
|
|
filter_editor_element = filter_editor.wait_for_visible()
|
|
filter_input = filter_editor_element.find_element_by_css_selector("input[type='number']")
|
|
filter_input.clear()
|
|
filter_input.send_keys("1")
|
|
self.screenshot("rules_example_1_4_filter_header")
|
|
rule_builder.rule_editor_ok.wait_for_and_click()
|
|
self.rule_builder_set_mapping("url", "D")
|
|
self.rule_builder_set_mapping("name", "C", screenshot_name="rules_example_1_5_mapping_edit")
|
|
self.screenshot("rules_example_1_6_mapping_set")
|
|
self.select2_set_value(".rule-option-extension", "fastqsanger.gz")
|
|
self.screenshot("rules_example_1_7_extension_set")
|
|
# rule_builder.main_button_ok.wait_for_and_click()
|
|
# self.history_panel_wait_for_hid_ok(6)
|
|
# self.screenshot("rules_example_1_6_download_complete")
|
|
|
|
@selenium_test
|
|
def test_rules_example_2_list(self):
|
|
self.perform_upload(self.get_filename("rules/PRJDA60709.tsv"))
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
self.upload_rule_start()
|
|
self.upload_rule_set_data_type("Collection")
|
|
self.upload_rule_set_input_type("History Dataset")
|
|
self.upload_rule_set_dataset("1:")
|
|
self.screenshot("rules_example_2_1_inputs")
|
|
self.upload_rule_build()
|
|
rule_builder = self.components.rule_builder
|
|
rule_builder._.wait_for_and_click()
|
|
self.screenshot("rules_example_2_2_initial_rules")
|
|
# Filter header.
|
|
self.rule_builder_filter_count(1)
|
|
self.rule_builder_set_mapping("url", "D")
|
|
self.rule_builder_set_mapping("list-identifiers", "C")
|
|
self.rule_builder_set_extension("fastqsanger.gz")
|
|
self.screenshot("rules_example_2_3_rules")
|
|
self.rule_builder_set_collection_name("PRJDA60709")
|
|
self.screenshot("rules_example_2_4_name")
|
|
# rule_builder.main_buton_ok.wait_for_and_click()
|
|
# self.history_panel_wait_for_hid_ok(2)
|
|
# self.screenshot("rules_example_2_5_download_complete")
|
|
|
|
@selenium_test
|
|
def test_rules_example_3_list_pairs(self):
|
|
self.perform_upload(self.get_filename("rules/PRJDB3920.tsv"))
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
self.upload_rule_start()
|
|
self.upload_rule_set_data_type("Collection")
|
|
self.upload_rule_set_input_type("History Dataset")
|
|
self.upload_rule_set_dataset("1:")
|
|
self.screenshot("rules_example_3_1_inputs")
|
|
self.upload_rule_build()
|
|
rule_builder = self.components.rule_builder
|
|
rule_builder._.wait_for_and_click()
|
|
self.screenshot("rules_example_3_2_initial_rules")
|
|
# Filter header.
|
|
self.rule_builder_filter_count(1)
|
|
self.rule_builder_set_mapping("list-identifiers", "C")
|
|
self.select2_set_value(".rule-option-extension", "fastqsanger.gz")
|
|
self.screenshot("rules_example_3_3_old_rules")
|
|
self.rule_builder_add_regex_groups("D", 2, "(.*);(.*)", screenshot_name="rules_example_3_4_regex")
|
|
self.screenshot("rules_example_3_6_with_regexes")
|
|
# Remove A also?
|
|
self.rule_builder_remove_columns(["D"], screenshot_name="rules_example_3_7_removed_column")
|
|
self.rule_builder_split_columns(["D"], ["E"], screenshot_name="rules_example_3_8_split_columns")
|
|
self.screenshot("rules_example_3_9_columns_are_split")
|
|
self.rule_builder_add_regex_groups("D", 1, ".*_(\d).fastq.gz", screenshot_name="rules_example_3_10_regex_paired")
|
|
self.screenshot("rules_example_3_11_has_paired_id")
|
|
self.rule_builder_swap_columns("D", "E", screenshot_name="rules_example_3_12_swap_columns")
|
|
self.screenshot("rules_example_3_13_swapped_columns")
|
|
self.rule_builder_set_mapping("paired-identifier", "D")
|
|
self.rule_builder_set_mapping("url", "E")
|
|
self.rule_builder_set_collection_name("PRJDB3920")
|
|
self.screenshot("rules_example_3_14_paired_identifier_set")
|
|
|
|
@selenium_test
|
|
def test_rules_example_4_accessions(self):
|
|
# http://www.uniprot.org/uniprot/?query=proteome:UP000052092+AND+proteomecomponent:%22Genome%22
|
|
self._setup_uniprot_example()
|
|
self.screenshot("rules_example_4_1_inputs")
|
|
self.upload_rule_build()
|
|
rule_builder = self.components.rule_builder
|
|
rule_builder._.wait_for_and_click()
|
|
self.screenshot("rules_example_4_2_initial_rules")
|
|
self.rule_builder_filter_count(1)
|
|
self.rule_builder_remove_columns(["B", "C", "E", "F", "G"])
|
|
self.rule_builder_set_mapping("info", "B")
|
|
self.screenshot("rules_example_4_3_basic_rules")
|
|
self.rule_builder_sort("A", screenshot_name="rules_example_4_4_sort")
|
|
|
|
self.rule_builder_add_regex_replacement("A", ".*", "http://www.uniprot.org/uniprot/$&.fasta", screenshot_name="rules_example_4_5_url")
|
|
self.screenshot("rules_example_4_6_url_built")
|
|
self.rule_builder_set_mapping("list-identifiers", "A")
|
|
self.rule_builder_set_mapping("url", "C")
|
|
self.rule_builder_set_extension("fasta")
|
|
self.rule_builder_set_collection_name("UP000052092")
|
|
self.screenshot("rules_example_4_7_mapping_extension_and_name")
|
|
|
|
rule_builder.view_source.wait_for_and_click()
|
|
text_area_elem = rule_builder.source.wait_for_visible()
|
|
|
|
self.screenshot("rules_example_4_8_source")
|
|
self.write_screenshot_directory_file("rules_example_4_8_text", text_area_elem.get_attribute("value"))
|
|
|
|
rule_builder.main_button_ok.wait_for_and_click()
|
|
rule_builder.view_source.wait_for_visible()
|
|
|
|
@selenium_test
|
|
def test_rules_example_5_matching_collections(self):
|
|
self._setup_uniprot_example()
|
|
self.screenshot("rules_example_5_1_inputs")
|
|
self.upload_rule_build()
|
|
|
|
rule_builder = self.components.rule_builder
|
|
rule_builder.view_source.wait_for_and_click()
|
|
|
|
text_area_elem = rule_builder.source.wait_for_visible()
|
|
|
|
content = self._read_rules_test_data_file("uniprot.json")
|
|
text_area_elem.clear()
|
|
text_area_elem.send_keys(content)
|
|
self.screenshot("rules_example_5_2_source")
|
|
rule_builder.main_button_ok.wait_for_and_click()
|
|
rule_builder.view_source.wait_for_visible()
|
|
self.screenshot("rules_example_5_3_initial_rules")
|
|
self.rule_builder_add_value("fasta", screenshot_name="rules_example_5_4_fasta")
|
|
self.rule_builder_add_value("UP000052092 FASTA")
|
|
self.rule_builder_add_regex_replacement("A", ".*", "http://www.uniprot.org/uniprot/$&.gff", screenshot_name="rules_example_5_5_url")
|
|
self.rule_builder_add_value("gff3")
|
|
self.rule_builder_add_value("UP000052092 GFF3")
|
|
self._scroll_to_end_of_table()
|
|
self.screenshot("rules_example_5_6_presplit")
|
|
self.rule_builder_split_columns(["C", "D", "E"], ["F", "G", "H"], screenshot_name="rules_example_5_7_split_columns")
|
|
self.screenshot("rules_example_5_8_split")
|
|
self.rule_builder_set_mapping("file-type", "D")
|
|
self.rule_builder_set_mapping("collection-name", "E")
|
|
self.screenshot("rules_example_5_9_mapping")
|
|
|
|
@selenium_test
|
|
def test_rules_example_6_nested_lists(self):
|
|
self.home()
|
|
self.perform_upload(self.get_filename("rules/PRJNA355367.tsv"))
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
self.upload_rule_start()
|
|
self.upload_rule_set_data_type("Collection")
|
|
self.upload_rule_set_input_type("History Dataset")
|
|
self.upload_rule_set_dataset("1:")
|
|
self.screenshot("rules_example_6_1_paste")
|
|
self.upload_rule_build()
|
|
|
|
rule_builder = self.components.rule_builder
|
|
rule_builder._.wait_for_and_click()
|
|
|
|
self.screenshot("rules_example_6_2_rules_landing")
|
|
|
|
self.rule_builder_filter_count(1)
|
|
|
|
self.rule_builder_set_mapping("url", "J")
|
|
self.rule_builder_set_extension("sra")
|
|
self._scroll_to_end_of_table()
|
|
self.screenshot("rules_example_6_3_end_of_table")
|
|
self.rule_builder_add_regex_groups("L", 1, "([^\d]+)\d+", screenshot_name="rules_example_6_4_regex")
|
|
self.rule_builder_set_mapping("list-identifiers", ["M", "A"], screenshot_name="rules_example_6_5_multiple_identifiers_edit")
|
|
self._scroll_to_end_of_table()
|
|
self.screenshot("rules_example_6_6_multiple_identifiers")
|
|
self.rule_builder_set_collection_name("PRJNA355367")
|
|
self.screenshot("rules_example_6_7_named")
|
|
|
|
def _read_rules_test_data_file(self, name):
|
|
with open(self.test_data_resolver.get_filename(os.path.join("rules", name)), "r") as f:
|
|
return f.read()
|
|
|
|
def _scroll_to_end_of_table(self):
|
|
rule_builder = self.components.rule_builder
|
|
table_elem = rule_builder.table.wait_for_visible()
|
|
first_cell = table_elem.find_elements_by_css_selector("td")[0]
|
|
action_chains = self.action_chains()
|
|
action_chains.move_to_element(first_cell)
|
|
action_chains.click(first_cell)
|
|
for i in range(15):
|
|
action_chains.send_keys(Keys.ARROW_RIGHT)
|
|
action_chains.perform()
|
|
|
|
def _setup_uniprot_example(self):
|
|
self.perform_upload(self.get_filename("rules/uniprot.tsv"))
|
|
self.history_panel_wait_for_hid_ok(1)
|
|
self.upload_rule_start()
|
|
self.upload_rule_set_data_type("Collection")
|
|
self.upload_rule_set_input_type("History Dataset")
|
|
self.upload_rule_set_dataset("1:")
|
|
|
|
# @selenium_test
|
|
# def test_rules_example_5_matching_collections(self):
|
|
# self.rule_builder_add_value("Protein FASTA")
|
|
# self.rule_builder_add_value("gff")
|
|
# self.rule_builder_add_value("Protein GFF")
|