Merge pull request #4971 from jmchilton/selenium_screenshot_suite

Capture labelled screenshots of Galaxy during PR CI testing.

.ci/jenkins/selenium/run_tests.sh

@@ -2,10 +2,22 @@
 
 # Enable retries on tests to reduce chances of transient failures.
 : ${GALAXY_TEST_SELENIUM_RETRIES:=1}
-: ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors}
+
+# If in Jenkins environment, use it for artifacts.
+if [ -n "$BUILD_NUMBER" ];
+then
+    : ${GALAXY_TEST_ERRORS_DIRECTORY:=${BUILD_NUMBER}-test-errors}
+    : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=${BUILD_NUMBER}-test-screenshots}
+else
+    : ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors}
+    : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=database/test-screenshots}
+fi
+
+mkdir -p "$GALAXY_TEST_ERRORS_DIRECTORY"
+mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY"
 
 # Start Selenium server in the test Docker container.
-DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
+DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
 export DOCKER_RUN_EXTRA_ARGS
 
 ./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --selenium "$@"

test/selenium_tests/framework.py

@@ -42,6 +42,7 @@ DEFAULT_ADMIN_PASSWORD = "testpass"
 
 TIMEOUT_MULTIPLIER = float(os.environ.get("GALAXY_TEST_TIMEOUT_MULTIPLIER", DEFAULT_TIMEOUT_MULTIPLIER))
 GALAXY_TEST_ERRORS_DIRECTORY = os.environ.get("GALAXY_TEST_ERRORS_DIRECTORY", DEFAULT_TEST_ERRORS_DIRECTORY)
+GALAXY_TEST_SCREENSHOTS_DIRECTORY = os.environ.get("GALAXY_TEST_SCREENSHOTS_DIRECTORY", None)
 # Test browser can be ["CHROME", "FIREFOX", "OPERA", "PHANTOMJS"]
 GALAXY_TEST_SELENIUM_BROWSER = os.environ.get("GALAXY_TEST_SELENIUM_BROWSER", DEFAULT_SELENIUM_BROWSER)
 GALAXY_TEST_SELENIUM_REMOTE = os.environ.get("GALAXY_TEST_SELENIUM_REMOTE", DEFAULT_SELENIUM_REMOTE)
@@ -225,8 +226,33 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy, UsesApiTestCaseMixin
             raise exception
 
     def snapshot(self, description):
+        """Create a debug snapshot (DOM, screenshot, etc...) that is written out on tool failure.
+
+        This information will be automatically written to a per-test directory created for all
+        failed tests.
+        """
         self.snapshots.append(TestSnapshot(self.driver, len(self.snapshots), description))
 
+    def screenshot(self, label):
+        """If GALAXY_TEST_SCREENSHOTS_DIRECTORY is set create a screenshot there named <label>.png.
+
+        Unlike the above "snapshot" feature, this will be written out regardless and not in a per-test
+        directory. The above method is used for debugging failures within a specific test. This method
+        if more for creating a set of images to augment automated testing with manual human inspection
+        after a test or test suite has executed.
+        """
+        if GALAXY_TEST_SCREENSHOTS_DIRECTORY is None:
+            return
+        if not os.path.exists(GALAXY_TEST_SCREENSHOTS_DIRECTORY):
+            os.makedirs(GALAXY_TEST_SCREENSHOTS_DIRECTORY)
+        target = os.path.join(GALAXY_TEST_SCREENSHOTS_DIRECTORY, label + ".png")
+        copy = 1
+        while os.path.exists(target):
+            # Maybe previously a test re-run - keep the original.
+            target = os.path.join(GALAXY_TEST_SCREENSHOTS_DIRECTORY, "%s-%d.png" % (label, copy))
+            copy += 1
+        self.driver.save_screenshot(target)
+
     def reset_driver_and_session(self):
         self.tear_down_driver()
         self.setup_driver_and_session()

test/selenium_tests/test_collection_builders.py

@@ -17,7 +17,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_list)
 
         self.collection_builder_set_name("my cool list")
-
+        self.screenshot("collection_builder_list")
         self.collection_builder_create()
         self.history_panel_wait_for_hid_ok(2)
 
@@ -48,7 +48,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.history_panel_muli_operation_select_hid(2)
         self.history_panel_multi_operation_action_click(self.navigation.history_panel.multi_operations.labels.build_pair)
         self.collection_builder_set_name("my awesome pair")
-
+        self.screenshot("collection_builder_pair")
         self.collection_builder_create()
         self.history_panel_wait_for_hid_ok(3)
 
@@ -67,7 +67,7 @@ class CollectionBuildersTestCase(SeleniumTestCase):
         self.collection_builder_click_paired_item("forward", 0)
         self.collection_builder_click_paired_item("reverse", 1)
         self.collection_builder_set_name("my awesome paired list")
-
+        self.screenshot("collection_builder_paired_list")
         self.collection_builder_create()
         self.history_panel_wait_for_hid_ok(3)
 

test/selenium_tests/test_history_dataset_state.py

@@ -34,6 +34,8 @@ class HistoryDatasetStateTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.assert_item_info_includes(1, 'uploaded fasta file')
         self.assert_item_peek_includes(1, ">hg17")
 
+        self.screenshot("history_panel_dataset_expanded")
+
         self._assert_action_buttons(1)
 
     def _assert_title_buttons(self, hid, expected_buttons=['display', 'edit', 'delete']):

test/selenium_tests/test_history_sharing.py

@@ -106,7 +106,7 @@ class HistorySharingTestCase(SeleniumTestCase):
         with self.main_panel():
             self.components.histories.sharing.share_with_a_user_button.wait_for_and_click()
 
-    def share_history_with_user(self, user_id=None, user_email=None, assert_valid=False):
+    def share_history_with_user(self, user_id=None, user_email=None, assert_valid=False, screenshot=False):
         """Share the current history with a target user by ID or email.
 
         ``user_email`` will be used to enter in the share form unless ``user_id``
@@ -121,6 +121,8 @@ class HistorySharingTestCase(SeleniumTestCase):
             # line, in future dispatch on actual select2 div present or not.
             # self.select2_set_value(form_selector, email)
             self.fill(form, {"email": user_id or user_email})
+            if screenshot:
+                self.screenshot("history_sharing_user")
             self.click_submit(form)
         if assert_valid:
             with self.main_panel():

test/selenium_tests/test_library_contents.py

@@ -37,6 +37,7 @@ class LibraryContentsTestCase(SeleniumTestCase):
         history_elements[0].find_element_by_css_selector("input").click()
         # Add
         self.sleep_for(self.wait_types.UX_RENDER)
+        self.screenshot("libraries_dataset_import")
         self.wait_for_and_click(self.navigation.libraries.folder.selectors.import_datasets_ok_button)
         # Let the progress bar disappear...
         self.wait_for_absent_or_hidden(self.navigation.libraries.folder.selectors.import_progress_bar)
@@ -87,6 +88,7 @@ class LibraryContentsTestCase(SeleniumTestCase):
         self.sleep_for(self.wait_types.UX_RENDER)
         self.wait_for_selector_clickable(".ui-modal #button-0").click()
         self.wait_for_overlays_cleared()
+        self.screenshot("libraries_show_details")
 
     @retry_assertion_during_transitions
     def _assert_num_displayed_items_is(self, n):

test/selenium_tests/test_library_landing.py

@@ -18,6 +18,7 @@ class LibraryLandingTestCase(SeleniumTestCase):
         num_displayed_libraries = self._num_displayed_libraries()
         self.libraries_index_click_create_new()
         self.wait_for_selector_visible(".ui-modal")
+        self.screenshot("libraries_new")
         close_button = self.wait_for_selector_clickable("#button-1")
         close_button.click()
         self.wait_for_overlays_cleared()
@@ -46,6 +47,7 @@ class LibraryLandingTestCase(SeleniumTestCase):
         name_box = self.wait_for_selector_clickable(".input_library_name")
         name_box.send_keys(new_name)
 
+        self.screenshot("libraries_rename")
         save_button = self.wait_for_selector_clickable(".save_library_btn")
         save_button.click()
 
@@ -71,9 +73,12 @@ class LibraryLandingTestCase(SeleniumTestCase):
         self.wait_for_overlays_cleared()
         self.libraries_index_create(namebase + " c")
 
+        self.screenshot("libraries_index")
+
         self.libraries_index_search_for(namebase)
 
         self._assert_num_displayed_libraries_is(3)
+        self.screenshot("libraries_index_search")
 
         self._assert_names_are([namebase + " a", namebase + " b", namebase + " c"])
         self.libraries_index_sort_click()

test/selenium_tests/test_published_histories_grid.py

@@ -15,7 +15,7 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase):
     @selenium_test
     def test_history_grid_search_standard(self):
         self.navigate_to_published_histories_page()
-
+        self.screenshot("histories_published_grid")
         self.published_grid_search_for(self.history1_name)
         self.assert_grid_histories_are([self.history1_name])
 
@@ -37,6 +37,7 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase):
 
         # Search by name
         self.set_filter(name_filter_selector, self.history1_name)
+        self.screenshot("histories_published_grid_advanced")
         self.assert_grid_histories_are([self.history1_name])
         self.unset_filter('name', self.history1_name)
 

test/selenium_tests/test_saved_histories.py

@@ -18,6 +18,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase):
     def test_history_switch(self):
         self._login()
         self.navigate_to_saved_histories_page()
+        self.screenshot("histories_saved_grid")
         self.click_popup_option(self.history2_name, 'Switch')
         self.sleep_for(self.wait_types.UX_RENDER)
 

test/selenium_tests/test_tool_form.py

@@ -95,6 +95,7 @@ class ToolFormTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.home()
         self.tool_open("head")
         self.tool_set_value("input", "1.fasta", expected_type="data")
+        self.screenshot("tool_form_simple_data")
         self.tool_execute()
         self.history_panel_wait_for_hid_ok(3)
 

test/selenium_tests/test_workflow_editor.py

@@ -32,6 +32,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.wait_for_selector_visible("#__workflow__inputs__ .toolTitle")
         input_links = self.driver.find_elements_by_css_selector("#__workflow__inputs__ .toolTitle a")
         input_links[0].click()
+        self.screenshot("workflow_editor_data_input")
         # TODO: verify box is highlighted and side panel is a form describing input.
         # More work needs to be done to develop testing abstractions for doing these things.
 
@@ -42,6 +43,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.workflow_index_open()
         self.workflow_index_click_option("Edit")
         self.sleep_for(self.wait_types.UX_RENDER)
+        self.screenshot("workflow_editor_edit_menu")
         self.workflow_editor_click_option("Save As")
 
     @selenium_test
@@ -51,6 +53,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.workflow_index_open()
         self.workflow_index_click_option("Edit")
         self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
+        self.screenshot("workflow_editor_tool_upgrade")
 
     @selenium_test
     def test_editor_invalid_tool_state(self):
@@ -60,6 +63,7 @@ class WorkflowEditorTestCase(SeleniumTestCase):
         self.workflow_index_click_option("Edit")
         self.assert_modal_has_text("Using version '0.2' instead of version '0.0.1'")
         self.assert_modal_has_text("Using default: '1'")
+        self.screenshot("workflow_editor_invalid_state")
 
     @selenium_test
     def test_missing_tools(self):
@@ -77,6 +81,7 @@ steps:
         self.workflow_index_open()
         self.workflow_index_click_option("Edit")
         self.assert_modal_has_text("Tool is not installed")
+        self.screenshot("workflow_editor_missing_tool")
 
     def workflow_create_new(self, name=None, annotation=None):
         self.workflow_index_open()

test/selenium_tests/test_workflow_management.py

@@ -27,7 +27,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
         self.workflow_index_click_option("View")
         title = self.wait_for_selector(".page-body h3")
         assert "TestWorkflow1" in title.text
-
+        self.screenshot("workflow_manage_view")
         # TODO: Test display of steps...
 
     @selenium_test
@@ -64,6 +64,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
             self.assertEqual(self.workflow_index_tags(), ["cooltag"])
 
         check_tags()
+        self.screenshot("workflow_manage_tags")
 
     @selenium_test
     def test_index_search(self):
@@ -71,6 +72,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
         self._workflow_import_from_url()
         self.workflow_index_rename("searchforthis")
         self._assert_showing_n_workflows(1)
+        self.screenshot("workflow_manage_search")
 
         self.workflow_index_search_for("doesnotmatch")
         self._assert_showing_n_workflows(0)
@@ -100,6 +102,7 @@ class WorkflowManagementTestCase(SeleniumTestCase):
 
         self.workflow_index_open()
         assert_published_column_text_is("Yes")
+        self.screenshot("workflow_manage_published")
 
     @retry_assertion_during_transitions
     def _assert_showing_n_workflows(self, n):

test/selenium_tests/test_workflow_run.py

@@ -23,6 +23,7 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.workflow_index_open()
         self.workflow_index_click_option("Run")
 
+        self.screenshot("workflow_manage_run_simple")
         self.workflow_run_submit()
 
         self.history_panel_wait_for_hid_ok(2, allowed_force_refreshes=1)
@@ -39,3 +40,4 @@ class WorkflowRunTestCase(SeleniumTestCase, UsesHistoryItemAssertions):
         self.sleep_for(self.wait_types.UX_TRANSITION)
         # Check that this tool form contains a warning about different versions.
         self.assert_warning_message(contains="different versions")
+        self.screenshot("workflow_manage_run_tool_upgrade")