Merge pull request #4655 from jmchilton/selenium_fixes_29

Aggressively retry buggy submit_login() function in Selenium tests.

test/selenium_tests/framework.py

@@ -83,15 +83,19 @@ def selenium_test(f):
                     result_name = f.__name__ + datetime.datetime.now().strftime("%Y%m%d%H%M%s")
                     target_directory = os.path.join(GALAXY_TEST_ERRORS_DIRECTORY, result_name)
 
-                    def write_file(name, content):
+                    def write_file(name, content, raw=False):
                         with open(os.path.join(target_directory, name), "wb") as buf:
-                            buf.write(content.encode("utf-8"))
+                            buf.write(content.encode("utf-8") if not raw else content)
 
                     os.makedirs(target_directory)
                     self.driver.save_screenshot(os.path.join(target_directory, "last.png"))
                     write_file("page_source.txt", self.driver.page_source)
                     write_file("DOM.txt", self.driver.execute_script("return document.documentElement.outerHTML"))
                     write_file("stacktrace.txt", traceback.format_exc())
+
+                    for snapshot in getattr(self, "snapshots", []):
+                        snapshot.write_to_error_directory(write_file)
+
                     for log_type in ["browser", "driver"]:
                         try:
                             write_file("%s.log.json" % log_type, json.dumps(self.driver.get_log(log_type)))
@@ -116,6 +120,22 @@ def selenium_test(f):
 retry_assertion_during_transitions = partial(retry_during_transitions, exception_check=lambda e: isinstance(e, AssertionError))
 
 
+class TestSnapshot(object):
+
+    def __init__(self, driver, index, description):
+        self.screenshot_binary = driver.get_screenshot_as_png()
+        self.description = description
+        self.index = index
+        self.exc = traceback.format_exc()
+        self.stack = traceback.format_stack()
+
+    def write_to_error_directory(self, write_file_func):
+        prefix = "%d-%s" % (self.index, self.description)
+        write_file_func("%s-screenshot.png" % prefix, self.screenshot_binary, raw=True)
+        write_file_func("%s-traceback.txt" % prefix, self.exc)
+        write_file_func("%s-stack.txt" % prefix, str(self.stack))
+
+
 class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy):
 
     framework_tool_and_types = True
@@ -130,6 +150,7 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy):
         else:
             self.target_url_from_selenium = self.url
         self.setup_driver_and_session()
+        self.snapshots = []
 
     def tearDown(self):
         exception = None
@@ -146,6 +167,9 @@ class SeleniumTestCase(FunctionalTestCase, NavigatesGalaxy):
         if exception is not None:
             raise exception
 
+    def snapshot(self, description):
+        self.snapshots.append(TestSnapshot(self.driver, len(self.snapshots), description))
+
     def reset_driver_and_session(self):
         self.tear_down_driver()
         self.setup_driver_and_session()

test/selenium_tests/test_custom_builds.py

@@ -12,7 +12,7 @@ class CustomBuildsTestcase(SharedStateSeleniumTestCase):
     def setUp(self):
         super(CustomBuildsTestcase, self).setUp()
         self.home()  # ensure Galaxy is loaded
-        self.submit_login(self.user_email)
+        self.submit_login(self.user_email, retries=2)
 
     @selenium_test
     def test_build_add(self):

test/selenium_tests/test_history_sharing.py

@@ -1,27 +1,30 @@
 from .framework import SeleniumTestCase
 from .framework import selenium_test
 
+# Remove hack when submit_login works more consistently.
+VALID_LOGIN_RETRIES = 3
+
 
 class HistorySharingTestCase(SeleniumTestCase):
 
     @selenium_test
     def test_sharing_valid(self):
         user1_email, user2_email, history_id = self.setup_two_users_with_one_shared_history()
-        self.submit_login(user2_email)
+        self.submit_login(user2_email, retries=VALID_LOGIN_RETRIES)
         response = self.api_get("histories/%s" % history_id, raw=True)
         assert response.status_code == 200, response.json()
 
     @selenium_test
     def test_sharing_valid_by_id(self):
         user1_email, user2_email, history_id = self.setup_two_users_with_one_shared_history(share_by_id=True)
-        self.submit_login(user2_email)
+        self.submit_login(user2_email, retries=VALID_LOGIN_RETRIES)
         response = self.api_get("histories/%s" % history_id, raw=True)
         assert response.status_code == 200, response.json()
 
     @selenium_test
     def test_unsharing(self):
         user1_email, user2_email, history_id = self.setup_two_users_with_one_shared_history()
-        self.submit_login(user1_email)
+        self.submit_login(user1_email, retries=VALID_LOGIN_RETRIES)
         self.navigate_to_history_share_page()
 
         with self.main_panel():
@@ -36,7 +39,7 @@ class HistorySharingTestCase(SeleniumTestCase):
             self.assert_selector_absent("#user-0-popup")
 
         self.logout_if_needed()
-        self.submit_login(user2_email)
+        self.submit_login(user2_email, retries=VALID_LOGIN_RETRIES)
         response = self.api_get("histories/%s" % history_id, raw=True)
         assert response.status_code == 403
 
@@ -80,7 +83,7 @@ class HistorySharingTestCase(SeleniumTestCase):
         user2_id = self.api_get("users")[0]["id"]
         self.logout_if_needed()
 
-        self.submit_login(user1_email)
+        self.submit_login(user1_email, retries=VALID_LOGIN_RETRIES)
         # Can't share an empty history...
         self.perform_upload(self.get_filename("1.txt"))
         self.wait_for_history()

test/selenium_tests/test_saved_histories.py

@@ -12,7 +12,7 @@ class SavedHistoriesTestCase(SharedStateSeleniumTestCase):
     def setUp(self):
         super(SavedHistoriesTestCase, self).setUp()
         self.home()
-        self.submit_login(self.user_email)
+        self.submit_login(self.user_email, retries=3)
 
     @selenium_test
     def test_saved_histories_list(self):