diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample
index 8028f9917c3..d92b517788d 100644
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -1148,6 +1148,20 @@ use_interactive = True
# be configured through the job configuration file.
#job_config_file = config/job_conf.xml
+# When jobs fail due to job runner problems, Galaxy can be configured to retry
+# these or reroute the jobs to new destinations. Very fine control of this is
+# available with resubmit declarations in job_conf.xml. For simple deployments
+# of Galaxy though, the following attribute can define resubmission conditions
+# for all job destinations. If any job destination defines even one
+# resubmission condition explicitly in job_conf.xml - the condition described
+# by this option will not apply to that destination. For instance, the condition:
+# 'attempt < 3 and unknown_error and (time_running < 300 or time_since_queued < 300)'
+# would retry up to two times jobs that didn't fail due to detected memory or
+# walltime limits but did fail quickly (either while queueing or running). The
+# commented out default below results in no default job resubmission condition,
+# failing jobs are just failed outright.
+#default_job_resubmission_condition =
+
# In multiprocess configurations, notification between processes about new jobs
# must be done via the database. In single process configurations, this can be
# done in memory, which is a bit quicker.
diff --git a/config/job_conf.xml.sample_advanced b/config/job_conf.xml.sample_advanced
index fedd7deaec9..fe09304cf3d 100644
--- a/config/job_conf.xml.sample_advanced
+++ b/config/job_conf.xml.sample_advanced
@@ -574,22 +574,36 @@
-
--time=00:05:00 --nodes=1
@@ -603,6 +617,11 @@
--mem-per-cpu=512
+
+
+
+
+
+
+
+
+
+
+ integration.resubmission_rules
+
+
+
+
+
+
+
+
+
+ python
+ initial_destination
+
+
+
+
+
+
+
+
+
+
+
+
+
+ test_name,failure_state,initial_destination,run_for
+
+
+
+
+
+
+
diff --git a/test/integration/resubmission_runners.py b/test/integration/resubmission_runners.py
index 39cbb7c01f8..572fece61fe 100644
--- a/test/integration/resubmission_runners.py
+++ b/test/integration/resubmission_runners.py
@@ -63,4 +63,24 @@ class AssertionJobRunner(LocalJobRunner):
super(AssertionJobRunner, self).queue_job(job_wrapper)
+class FailOnlyFirstJobRunner(LocalJobRunner):
+ """Job runner that knows about test cases and checks final state assumptions."""
+
+ tests_seen = []
+
+ def queue_job(self, job_wrapper):
+ resource_parameters = job_wrapper.get_resource_parameters()
+ try:
+ test_name = resource_parameters["test_name"]
+ except KeyError:
+ job_wrapper.fail("Job resource parameter test_name not set as required for this job runner.")
+ return
+
+ if test_name in self.tests_seen:
+ super(FailOnlyFirstJobRunner, self).queue_job(job_wrapper)
+ else:
+ self.tests_seen.append(test_name)
+ self._fail_job_local(job_wrapper, "Failing first attempt")
+
+
__all__ = ('FailsJobRunner', 'AssertionJobRunner')
diff --git a/test/integration/test_job_resubmission.py b/test/integration/test_job_resubmission.py
index cefe368d36e..cffbdb5e5e8 100644
--- a/test/integration/test_job_resubmission.py
+++ b/test/integration/test_job_resubmission.py
@@ -6,10 +6,27 @@ from base import integration_util
SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
JOB_RESUBMISSION_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_job_conf.xml")
+JOB_RESUBMISSION_DEFAULT_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_default_job_conf.xml")
JOB_RESUBMISSION_JOB_RESOURCES_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_job_resource_parameters_conf.xml")
-class JobResubmissionIntegrationTestCase(integration_util.IntegrationTestCase):
+class _BaseResubmissionIntegerationTestCase(integration_util.IntegrationTestCase):
+ framework_tool_and_types = True
+
+ def _assert_job_passes(self, resource_parameters):
+ self._run_tool_test("simple_constructs", resource_parameters=resource_parameters)
+
+ def _assert_job_fails(self, resource_parameters):
+ exception_thrown = False
+ try:
+ self._run_tool_test("simple_constructs", resource_parameters=resource_parameters)
+ except Exception:
+ exception_thrown = True
+
+ assert exception_thrown
+
+
+class JobResubmissionIntegrationTestCase(_BaseResubmissionIntegerationTestCase):
framework_tool_and_types = True
@@ -85,14 +102,16 @@ class JobResubmissionIntegrationTestCase(integration_util.IntegrationTestCase):
"initial_destination": "resubmit_after_two_delays",
"failure_state": "unknown_error"})
- def _assert_job_passes(self, resource_parameters):
- self._run_tool_test("simple_constructs", resource_parameters=resource_parameters)
- def _assert_job_fails(self, resource_parameters):
- exception_thrown = False
- try:
- self._run_tool_test("simple_constructs", resource_parameters=resource_parameters)
- except Exception:
- exception_thrown = True
+class JobResubmissionDefaultIntegrationTestCase(_BaseResubmissionIntegerationTestCase):
- assert exception_thrown
+ framework_tool_and_types = True
+
+ @classmethod
+ def handle_galaxy_config_kwds(cls, config):
+ config["default_job_resubmission_condition"] = "attempt < 2"
+ config["job_config_file"] = JOB_RESUBMISSION_DEFAULT_JOB_CONFIG_FILE
+ config["job_resource_params_file"] = JOB_RESUBMISSION_JOB_RESOURCES_CONFIG_FILE
+
+ def test_default_resubmission(self):
+ self._assert_job_passes(resource_parameters={"test_name": "test_default_resubmission"})
diff --git a/test/unit/jobs/test_job_configuration.py b/test/unit/jobs/test_job_configuration.py
index 9045f768a35..ec19701b158 100644
--- a/test/unit/jobs/test_job_configuration.py
+++ b/test/unit/jobs/test_job_configuration.py
@@ -22,6 +22,7 @@ class JobConfXmlParserTestCase( unittest.TestCase ):
use_tasked_jobs=False,
job_resource_params_file="/tmp/fake_absent_path",
config_dict={},
+ default_job_resubmission_condition="",
)
self.__write_config_from( SIMPLE_JOB_CONF )
self.app = bunch.Bunch( config=self.config, job_metrics=MockJobMetrics() )