Merge remote-tracking branch 'jmchilton/dev' into tdt_tests

test/selenium_tests/framework.py

@@ -42,6 +42,7 @@ DEFAULT_ADMIN_PASSWORD = "testpass"
 
 TIMEOUT_MULTIPLIER = float(os.environ.get("GALAXY_TEST_TIMEOUT_MULTIPLIER", DEFAULT_TIMEOUT_MULTIPLIER))
 GALAXY_TEST_ERRORS_DIRECTORY = os.environ.get("GALAXY_TEST_ERRORS_DIRECTORY", DEFAULT_TEST_ERRORS_DIRECTORY)
+GALAXY_TEST_SCREENSHOTS_DIRECTORY = os.environ.get("GALAXY_TEST_SCREENSHOTS_DIRECTORY", None)
 # Test browser can be ["CHROME", "FIREFOX", "OPERA", "PHANTOMJS"]
 GALAXY_TEST_SELENIUM_BROWSER = os.environ.get("GALAXY_TEST_SELENIUM_BROWSER", DEFAULT_SELENIUM_BROWSER)
 GALAXY_TEST_SELENIUM_REMOTE = os.environ.get("GALAXY_TEST_SELENIUM_REMOTE", DEFAULT_SELENIUM_REMOTE)
@@ -57,6 +58,14 @@ GALAXY_TEST_SELENIUM_USER_PASSWORD = os.environ.get("GALAXY_TEST_SELENIUM_USER_P
 GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL = os.environ.get("GALAXY_TEST_SELENIUM_ADMIN_USER_EMAIL", DEFAULT_ADMIN_USER)
 GALAXY_TEST_SELENIUM_ADMIN_USER_PASSWORD = os.environ.get("GALAXY_TEST_SELENIUM_ADMIN_USER_PASSWORD", DEFAULT_ADMIN_PASSWORD)
 
+# JS code to execute in Galaxy JS console to setup localStorage of session for logging and
+# logging "flatten" messages because it seems Selenium (with Chrome at least) only grabs
+# the first argument to console.XXX when recovering the browser log.
+SETUP_LOGGING_JS = '''
+window.localStorage && window.localStorage.setItem("galaxy:debug", true);
+window.localStorage && window.localStorage.setItem("galaxy:debug:flatten", true);
+'''
+
 try:
     from nose.tools import nottest
 except ImportError:
@@ -107,8 +116,11 @@ def dump_test_information(self, name_prefix):
             snapshot.write_to_error_directory(write_file)
 
         for log_type in ["browser", "driver"]:
+            full_log = self.driver.get_log(log_type)
+            trimmed_log = [l for l in full_log if l["level"] not in ["DEBUG", "INFO"]]
             try:
-                write_file("%s.log.json" % log_type, json.dumps(self.driver.get_log(log_type)))
+                write_file("%s.log.json" % log_type, json.dumps(trimmed_log, indent=True))
+                write_file("%s.log.verbose.json" % log_type, json.dumps(full_log, indent=True))
             except Exception:
                 continue
         iframes = self.driver.find_elements_by_css_selector("iframe")
@@ -214,8 +226,33 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy, UsesApiTestCaseMixin
             raise exception
 
     def snapshot(self, description):
+        """Create a debug snapshot (DOM, screenshot, etc...) that is written out on tool failure.
+
+        This information will be automatically written to a per-test directory created for all
+        failed tests.
+        """
         self.snapshots.append(TestSnapshot(self.driver, len(self.snapshots), description))
 
+    def screenshot(self, label):
+        """If GALAXY_TEST_SCREENSHOTS_DIRECTORY is set create a screenshot there named <label>.png.
+
+        Unlike the above "snapshot" feature, this will be written out regardless and not in a per-test
+        directory. The above method is used for debugging failures within a specific test. This method
+        if more for creating a set of images to augment automated testing with manual human inspection
+        after a test or test suite has executed.
+        """
+        if GALAXY_TEST_SCREENSHOTS_DIRECTORY is None:
+            return
+        if not os.path.exists(GALAXY_TEST_SCREENSHOTS_DIRECTORY):
+            os.makedirs(GALAXY_TEST_SCREENSHOTS_DIRECTORY)
+        target = os.path.join(GALAXY_TEST_SCREENSHOTS_DIRECTORY, label + ".png")
+        copy = 1
+        while os.path.exists(target):
+            # Maybe previously a test re-run - keep the original.
+            target = os.path.join(GALAXY_TEST_SCREENSHOTS_DIRECTORY, "%s-%d.png" % (label, copy))
+            copy += 1
+        self.driver.save_screenshot(target)
+
     def reset_driver_and_session(self):
         self.tear_down_driver()
         self.setup_driver_and_session()
@@ -227,9 +264,15 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy, UsesApiTestCaseMixin
         # to increase this.
         self.driver.set_window_size(1280, 900)
 
+        self._setup_galaxy_logging()
+
         if self.ensure_registered:
             self.login()
 
+    def _setup_galaxy_logging(self):
+        self.home()
+        self.driver.execute_script(SETUP_LOGGING_JS)
+
     def login(self):
         if GALAXY_TEST_SELENIUM_USER_EMAIL:
             assert GALAXY_TEST_SELENIUM_USER_PASSWORD, "If GALAXY_TEST_SELENIUM_USER_EMAIL is set, a password must be set also with GALAXY_TEST_SELENIUM_USER_PASSWORD"

test/selenium_tests/test_collection_builders.py

@@ -17,7 +17,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list)
 
         self.collection_builder_set_name("my cool list")
-
+        self.screenshot("collection_builder_list")
         self.collection_builder_create()
         self.history_panel_wait_for_hid_ok(2)
 
@@ -48,7 +48,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.history_panel_muli_operation_select_hid(2)
         self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_pair)
         self.collection_builder_set_name("my awesome pair")
-
+        self.screenshot("collection_builder_pair")
         self.collection_builder_create()
         self.history_panel_wait_for_hid_ok(3)
 
@@ -67,7 +67,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_click_paired_item("forward", 0)
         self.collection_builder_click_paired_item("reverse", 1)
         self.collection_builder_set_name("my awesome paired list")
-
+        self.screenshot("collection_builder_paired_list")
         self.collection_builder_create()
         self.history_panel_wait_for_hid_ok(3)
 

test/selenium_tests/test_history_dataset_state.py

@@ -34,6 +34,8 @@ class HistoryDatasetStateTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.assert_item_info_includes(1, 'uploaded fasta file')
         self.assert_item_peek_includes(1, ">hg17")
 
+        self.screenshot("history_panel_dataset_expanded")
+
         self._assert_action_buttons(1)
 
     def _assert_title_buttons(self, hid, expected_buttons=['display', 'edit', 'delete']):

test/selenium_tests/test_history_sharing.py

@@ -106,7 +106,7 @@ class HistorySharingTestCase(SeleniumTestCase):
         with self.main_panel():
             self.components.histories.sharing.share_with_a_user_button.wait_for_and_click()
 
-    def share_history_with_user(self, user_id=None, user_email=None, assert_valid=False):
+    def share_history_with_user(self, user_id=None, user_email=None, assert_valid=False, screenshot=False):
         """Share the current history with a target user by ID or email.
 
         ``user_email`` will be used to enter in the share form unless ``user_id``
@@ -121,6 +121,8 @@ class HistorySharingTestCase(SeleniumTestCase):
             # line, in future dispatch on actual select2 div present or not.
             # self.select2_set_value(form_selector, email)
             self.fill(form, {"email": user_id or user_email})
+            if screenshot:
+                self.screenshot("history_sharing_user")
             self.click_submit(form)
         if assert_valid:
             with self.main_panel():

test/selenium_tests/test_jupyter.py

@@ -1,3 +1,5 @@
+from base.populators import flakey
+
 from .framework import (
     managed_history,
     selenium_test,
@@ -11,6 +13,7 @@ class JupyterTestCase(SeleniumTestCase):
 
     ensure_registered = True
 
+    @flakey
     @selenium_test
     @managed_history
     def test_jupyter_session(self):

test/selenium_tests/test_library_contents.py

@@ -37,6 +37,7 @@ class LibraryContentsTestCase(SeleniumTestCase):
         history_elements[0].find_element_by_css_selector("input").click()
         # Add
         self.sleep_for(self.wait_types.UX_RENDER)
+        self.screenshot("libraries_dataset_import")
         self.wait_for_and_click(self.navigation.libraries.folder.selectors.import_datasets_ok_button)
         # Let the progress bar disappear...
         self.wait_for_absent_or_hidden(self.navigation.libraries.folder.selectors.import_progress_bar)
@@ -87,6 +88,7 @@ class LibraryContentsTestCase(SeleniumTestCase):
         self.sleep_for(self.wait_types.UX_RENDER)
         self.wait_for_selector_clickable(".ui-modal #button-0").click()
         self.wait_for_overlays_cleared()
+        self.screenshot("libraries_show_details")
 
     @retry_assertion_during_transitions
     def _assert_num_displayed_items_is(self, n):

test/selenium_tests/test_library_landing.py

@@ -18,6 +18,7 @@ class LibraryLandingTestCase(SeleniumTestCase):
         num_displayed_libraries = self._num_displayed_libraries()
         self.libraries_index_click_create_new()
         self.wait_for_selector_visible(".ui-modal")
+        self.screenshot("libraries_new")
         close_button = self.wait_for_selector_clickable("#button-1")
         close_button.click()
         self.wait_for_overlays_cleared()
@@ -46,6 +47,7 @@ class LibraryLandingTestCase(SeleniumTestCase):
         name_box = self.wait_for_selector_clickable(".input_library_name")
         name_box.send_keys(new_name)
 
+        self.screenshot("libraries_rename")
         save_button = self.wait_for_selector_clickable(".save_library_btn")
         save_button.click()
 
@@ -71,9 +73,12 @@ class LibraryLandingTestCase(SeleniumTestCase):
         self.wait_for_overlays_cleared()
         self.libraries_index_create(namebase + " c")
 
+        self.screenshot("libraries_index")
+
         self.libraries_index_search_for(namebase)
 
         self._assert_num_displayed_libraries_is(3)
+        self.screenshot("libraries_index_search")
 
         self._assert_names_are([namebase + " a", namebase + " b", namebase + " c"])
         self.libraries_index_sort_click()

test/selenium_tests/test_published_histories_grid.py

@@ -15,7 +15,7 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase):
     @selenium_test
     def test_history_grid_search_standard(self):
         self.navigate_to_published_histories_page()
-
+        self.screenshot("histories_published_grid")
         self.published_grid_search_for(self.history1_name)
         self.assert_grid_histories_are([self.history1_name])
 
@@ -37,6 +37,7 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase):
 
         # Search by name
         self.set_filter(name_filter_selector, self.history1_name)
+        self.screenshot("histories_published_grid_advanced")
         self.assert_grid_histories_are([self.history1_name])
         self.unset_filter('name', self.history1_name)
 
@@ -88,9 +89,16 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase):
         if tags is None:
             raise AssertionError('Failed to find history with name [%s]' % self.history1_name)
 
-        tag_button_selector = '.tag-area > .tag-button:first-child > .tag-name'
-        tag_button = tags.find_element_by_css_selector(tag_button_selector)
-        self.assertEqual(tag_button.text, self.history1_tags[0])
+        tag_button_selector = '.tag-area > .tag-button > .tag-name'
+        tag_buttons = tags.find_elements_by_css_selector(tag_button_selector)
+        tag_button_text = None
+        target_tag_button_text = self.history1_tags[0]
+        for tag_button in tag_buttons:
+            tag_button_text = tag_button.text
+            if tag_button_text == target_tag_button_text:
+                break
+
+        self.assertEqual(tag_button_text, target_tag_button_text)
         tag_button.click()
 
         self.assert_grid_histories_are([self.history1_name, self.history3_name], False)

test/selenium_tests/test_saved_histories.py

@@ -18,6 +18,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase):
     def test_history_switch(self):
         self._login()
         self.navigate_to_saved_histories_page()
+        self.screenshot("histories_saved_grid")
         self.click_popup_option(self.history2_name, 'Switch')
         self.sleep_for(self.wait_types.UX_RENDER)
 

test/selenium_tests/test_stock_tours.py

@@ -1,7 +1,6 @@
 import os
 
 from galaxy.util import galaxy_root_path
-
 from .framework import (
     selenium_test,
     SeleniumTestCase

test/selenium_tests/test_tool_form.py

@@ -95,7 +95,10 @@ class ToolFormTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.home()
         self.tool_open("head")
         self.tool_set_value("input", "1.fasta", expected_type="data")
+
+        self.screenshot("tool_form_simple_data")
         self.tool_form_execute()
+
         self.history_panel_wait_for_hid_ok(3)
 
         latest_hda = self.latest_history_item()

test/selenium_tests/test_workflow_editor.py

@@ -32,6 +32,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.wait_for_selector_visible("#__workflow__inputs__ .toolTitle")
         input_links = self.driver.find_elements_by_css_selector("#__workflow__inputs__ .toolTitle a")
         input_links[0].click()
+        self.screenshot("workflow_editor_data_input")
         # TODO: verify box is highlighted and side panel is a form describing input.
         # More work needs to be done to develop testing abstractions for doing these things.
 
@@ -42,6 +43,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.workflow_index_open()
         self.workflow_index_click_option("Edit")
         self.sleep_for(self.wait_types.UX_RENDER)
+        self.screenshot("workflow_editor_edit_menu")
         self.workflow_editor_click_option("Save As")
 
     @selenium_test
@@ -51,6 +53,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.workflow_index_open()
         self.workflow_index_click_option("Edit")
         self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
+        self.screenshot("workflow_editor_tool_upgrade")
 
     @selenium_test
     def test_editor_invalid_tool_state(self):
@@ -60,6 +63,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.workflow_index_click_option("Edit")
         self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
         self.assert_modal_has_text("Using default: '1'")
+        self.screenshot("workflow_editor_invalid_state")
 
     @selenium_test
     def test_missing_tools(self):
@@ -77,6 +81,7 @@ steps:
         self.workflow_index_open()
         self.workflow_index_click_option("Edit")
         self.assert_modal_has_text("Tool is not installed")
+        self.screenshot("workflow_editor_missing_tool")
 
     def workflow_create_new(self, name=None, annotation=None):
         self.workflow_index_open()

test/selenium_tests/test_workflow_management.py

@@ -27,7 +27,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
         self.workflow_index_click_option("View")
         title = self.wait_for_selector(".page-body h3")
         assert "TestWorkflow1" in title.text
-
+        self.screenshot("workflow_manage_view")
         # TODO: Test display of steps...
 
     @selenium_test
@@ -64,6 +64,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
             self.assertEqual(self.workflow_index_tags(), ["cooltag"])
 
         check_tags()
+        self.screenshot("workflow_manage_tags")
 
     @selenium_test
     def test_index_search(self):
@@ -71,6 +72,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
         self._workflow_import_from_url()
         self.workflow_index_rename("searchforthis")
         self._assert_showing_n_workflows(1)
+        self.screenshot("workflow_manage_search")
 
         self.workflow_index_search_for("doesnotmatch")
         self._assert_showing_n_workflows(0)
@@ -100,6 +102,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
 
         self.workflow_index_open()
         assert_published_column_text_is("Yes")
+        self.screenshot("workflow_manage_published")
 
     @retry_assertion_during_transitions
     def _assert_showing_n_workflows(self, n):

test/selenium_tests/test_workflow_run.py

@@ -23,6 +23,7 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.workflow_index_open()
         self.workflow_index_click_option("Run")
 
+        self.screenshot("workflow_manage_run_simple")
         self.workflow_run_submit()
 
         self.history_panel_wait_for_hid_ok(2, allowed_force_refreshes=1)
@@ -39,3 +40,4 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.sleep_for(self.wait_types.UX_TRANSITION)
         # Check that this tool form contains a warning about different versions.
         self.assert_warning_message(contains="different versions")
+        self.screenshot("workflow_manage_run_tool_upgrade")