Merge pull request #4972 from jmchilton/flakey

Add @flakey test annotation and "skip" them if the fail on Jenkins.

.ci/jenkins/selenium/run_tests.sh

@@ -20,4 +20,4 @@ mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY"
 DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
 export DOCKER_RUN_EXTRA_ARGS
 
-./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --selenium "$@"
+./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --skip_flakey_fails --selenium "$@"

run_tests.sh

@@ -157,6 +157,8 @@ Extra options:
  --external_master_key Master API key used to configure external tests.
  --external_user_key   User API used for external tests - not required if
                        external_master_key is specified.
+  --skip_flakey_fails  Skip flakey tests on error (sets
+                       GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1).
 
 Environment Variables:
 
@@ -210,6 +212,8 @@ GALAXY_TEST_FETCH_DATA          Fetch remote test data to
                                 command-line.
 GALAXY_TEST_DATA_REPO_CACHE     Where to cache remote test data to (default to
                                 test-data-cache).
+GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR
+                                Skip tests annotated with @flakey on test errors.
 HTTP_ACCEPT_LANGUAGE            Defaults to 'en'
 GALAXY_TEST_NO_CLEANUP          Do not cleanup main test directory after tests,
                                 the deprecated option TOOL_SHED_TEST_NO_CLEANUP
@@ -380,6 +384,11 @@ do
           find test -iname '*pyc' -exec rm -rf {} \;
           shift
           ;;
+      -skip_flakey_fails|--skip_flakey_fails)
+          GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1
+          export GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR
+          shift
+          ;;
       -with_framework_test_tools|--with_framework_test_tools)
           with_framework_test_tools_arg="-with_framework_test_tools"
           shift

test/base/populators.py

@@ -1,5 +1,6 @@
 import contextlib
 import json
+import os
 import time
 from functools import wraps
 from operator import itemgetter
@@ -23,6 +24,24 @@ workflow_random_x2_str = resource_string(__name__, "data/test_workflow_2.ga")
 
 DEFAULT_TIMEOUT = 60  # Secs to wait for state to turn ok
 
+SKIP_FLAKEY_TESTS_ON_ERROR = os.environ.get("GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR", None)
+
+
+def flakey(method):
+
+    @wraps(method)
+    def wrapped_method(test_case, *args, **kwargs):
+        try:
+            method(test_case, *args, **kwargs)
+        except Exception:
+            if SKIP_FLAKEY_TESTS_ON_ERROR:
+                from nose.plugins.skip import SkipTest
+                raise SkipTest()
+            else:
+                raise
+
+    return wrapped_method
+
 
 def skip_without_tool(tool_id):
     """Decorate an API test method as requiring a specific tool.

test/selenium_tests/test_jupyter.py

@@ -1,3 +1,5 @@
+from base.populators import flakey
+
 from .framework import (
     managed_history,
     selenium_test,
@@ -11,6 +13,7 @@ class JupyterTestCase(SeleniumTestCase):
 
     ensure_registered = True
 
+    @flakey
     @selenium_test
     @managed_history
     def test_jupyter_session(self):