Prevent transient job state test failures from failing the build.

A precondition to the real meat of the test is failing to be met sometimes in production - this failure is indicating a problem with the test and not with Galaxy. I previously tried to address this with https://github.com/galaxyproject/galaxy/pull/3988 but that didn't work. Now if the trainsient ok occurs the test will be skipped. Most of the time it won't skip and the rest of the test will execute and ensure there aren't regressions in behaviors related to cleaning up datasets after job completion. I'm placing the skips a couple different places so hopefully we can get a stack trace at somepoint - knowing where the code is when the job state is changing from running to ok will help puzzle out what is handing for the two minutes the job is running in the test framework.

test/api/test_jobs.py

@@ -133,9 +133,17 @@ class JobsApiTestCase(api.ApiTestCase):
     def test_deleting_output_keep_running_until_all_deleted(self):
         history_id, job_state, outputs = self._setup_running_two_output_job(120)
 
+        self._hack_to_skip_test_if_state_ok(job_state)
+
         # Delete one of the two outputs and make sure the job is still running.
         self._raw_update_history_item(history_id, outputs[0]["id"], {"deleted": True})
+
+        self._hack_to_skip_test_if_state_ok(job_state)
+
         time.sleep(1)
+
+        self._hack_to_skip_test_if_state_ok(job_state)
+
         state = job_state().json()["state"]
         assert state == "running", state
 
@@ -159,12 +167,17 @@ class JobsApiTestCase(api.ApiTestCase):
             output_dataset_paths = []
             output_dataset_paths_exist = False
 
+        self._hack_to_skip_test_if_state_ok(job_state)
+
         current_state = job_state().json()["state"]
         assert current_state == "running", current_state
 
         # Purge one of the two outputs and make sure the job is still running.
         self._raw_update_history_item(history_id, outputs[0]["id"], {"purged": True})
         time.sleep(1)
+
+        self._hack_to_skip_test_if_state_ok(job_state)
+
         current_state = job_state().json()["state"]
         assert current_state == "running", current_state
 
@@ -210,6 +223,12 @@ class JobsApiTestCase(api.ApiTestCase):
             assert os.path.exists(output_dataset_paths[1])
             assert not os.path.exists(output_dataset_paths[0])
 
+    def _hack_to_skip_test_if_state_ok(self, job_state):
+        from nose.plugins.skip import SkipTest
+        if job_state().json()["state"] == "ok":
+            message = "Job state switch from running to ok too quickly - the rest of the test requires the job to be in a running state. Skipping test."
+            raise SkipTest(message)
+
     def _setup_running_two_output_job(self, sleep_time):
         history_id = self.dataset_populator.new_history()
         payload = self.dataset_populator.run_tool_payload(