From 875fb3641fbc70bc406be2744174e4945f61b99d Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 9 Nov 2017 16:24:55 -0500
Subject: [PATCH] Add @flakey test annotation and "skip" them if the fail on
 Jenkins.

Use it to skip the Juypter test that keeps randomly failing builds if there are problems.
---
 .ci/jenkins/selenium/run_tests.sh   |  2 +-
 run_tests.sh                        |  9 +++++++++
 test/base/populators.py             | 19 +++++++++++++++++++
 test/selenium_tests/test_jupyter.py |  3 +++
 4 files changed, 32 insertions(+), 1 deletion(-)

diff --git a/.ci/jenkins/selenium/run_tests.sh b/.ci/jenkins/selenium/run_tests.sh
index 24df53232cf..c00ecc33cd5 100644
--- a/.ci/jenkins/selenium/run_tests.sh
+++ b/.ci/jenkins/selenium/run_tests.sh
@@ -8,4 +8,4 @@
 DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}"
 export DOCKER_RUN_EXTRA_ARGS
 
-./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --selenium "$@"
+./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --skip_flakey_fails --selenium "$@"
diff --git a/run_tests.sh b/run_tests.sh
index 589c8e5d19b..a083c91a3ee 100755
--- a/run_tests.sh
+++ b/run_tests.sh
@@ -157,6 +157,8 @@ Extra options:
  --external_master_key Master API key used to configure external tests.
  --external_user_key   User API used for external tests - not required if
                        external_master_key is specified.
+  --skip_flakey_fails  Skip flakey tests on error (sets
+                       GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1).
 
 Environment Variables:
 
@@ -210,6 +212,8 @@ GALAXY_TEST_FETCH_DATA          Fetch remote test data to
                                 command-line.
 GALAXY_TEST_DATA_REPO_CACHE     Where to cache remote test data to (default to
                                 test-data-cache).
+GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR
+                                Skip tests annotated with @flakey on test errors.
 HTTP_ACCEPT_LANGUAGE            Defaults to 'en'
 GALAXY_TEST_NO_CLEANUP          Do not cleanup main test directory after tests,
                                 the deprecated option TOOL_SHED_TEST_NO_CLEANUP
@@ -380,6 +384,11 @@ do
           find test -iname '*pyc' -exec rm -rf {} \;
           shift
           ;;
+      -skip_flakey_fails|--skip_flakey_fails)
+          GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR=1
+          export GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR
+          shift
+          ;;
       -with_framework_test_tools|--with_framework_test_tools)
           with_framework_test_tools_arg="-with_framework_test_tools"
           shift
diff --git a/test/base/populators.py b/test/base/populators.py
index 46a9557ccab..0d063de88bf 100644
--- a/test/base/populators.py
+++ b/test/base/populators.py
@@ -1,5 +1,6 @@
 import contextlib
 import json
+import os
 import time
 from functools import wraps
 from operator import itemgetter
@@ -23,6 +24,24 @@ workflow_random_x2_str = resource_string(__name__, "data/test_workflow_2.ga")
 
 DEFAULT_TIMEOUT = 60  # Secs to wait for state to turn ok
 
+SKIP_FLAKEY_TESTS_ON_ERROR = os.environ.get("GALAXY_TEST_SKIP_FLAKEY_TESTS_ON_ERROR", None)
+
+
+def flakey(method):
+
+    @wraps(method)
+    def wrapped_method(test_case, *args, **kwargs):
+        try:
+            method(test_case, *args, **kwargs)
+        except Exception:
+            if SKIP_FLAKEY_TESTS_ON_ERROR:
+                from nose.plugins.skip import SkipTest
+                raise SkipTest()
+            else:
+                raise
+
+    return wrapped_method
+
 
 def skip_without_tool(tool_id):
     """Decorate an API test method as requiring a specific tool.
diff --git a/test/selenium_tests/test_jupyter.py b/test/selenium_tests/test_jupyter.py
index 4d46a3dbef8..14a74c07a34 100644
--- a/test/selenium_tests/test_jupyter.py
+++ b/test/selenium_tests/test_jupyter.py
@@ -1,3 +1,5 @@
+from base.populators import flakey
+
 from .framework import (
     managed_history,
     selenium_test,
@@ -11,6 +13,7 @@ class JupyterTestCase(SeleniumTestCase):
 
     ensure_registered = True
 
+    @flakey
     @selenium_test
     @managed_history
     def test_jupyter_session(self):