diff --git a/lib/galaxy/jobs/runners/state_handlers/resubmit.py b/lib/galaxy/jobs/runners/state_handlers/resubmit.py index 25ccd5bf8e5..f9cbd744f7c 100644 --- a/lib/galaxy/jobs/runners/state_handlers/resubmit.py +++ b/lib/galaxy/jobs/runners/state_handlers/resubmit.py @@ -12,7 +12,8 @@ log = logging.getLogger(__name__) MESSAGES = dict( walltime_reached='it reached the walltime', memory_limit_reached='it exceeded the amount of allocated memory', - unknown_error='it encountered an unknown error' + unknown_error='it encountered an unknown error', + tool_detected='it encountered a tool detected error condition', ) @@ -41,6 +42,7 @@ def eval_condition(condition, job_state): condition_locals = { "walltime_reached": runner_state == JobState.runner_states.WALLTIME_REACHED, "memory_limit_reached": runner_state == JobState.runner_states.MEMORY_LIMIT_REACHED, + "tool_detected_failure": runner_state == JobState.runner_states.TOOL_DETECT_ERROR, "unknown_error": JobState.runner_states.UNKNOWN_ERROR, "any_failure": True, "any_potential_job_failure": True, # Add a hook here - later on allow tools to describe things that are definitely input problems. @@ -66,6 +68,7 @@ def failure(app, job_runner, job_state): if (runner_state not in (JobState.runner_states.WALLTIME_REACHED, JobState.runner_states.MEMORY_LIMIT_REACHED, JobState.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER, + JobState.runner_states.TOOL_DETECT_ERROR, JobState.runner_states.UNKNOWN_ERROR)): # not set or not a handleable runner state return @@ -184,6 +187,7 @@ class _ExpressionContext(object): "walltime_reached": runner_state == JobState.runner_states.WALLTIME_REACHED, "memory_limit_reached": runner_state == JobState.runner_states.MEMORY_LIMIT_REACHED, "unknown_error": JobState.runner_states.UNKNOWN_ERROR, + "tool_detected_failure": runner_state == JobState.runner_states.TOOL_DETECT_ERROR, "any_failure": True, "any_potential_job_failure": True, # Add a hook here - later on allow tools to describe things that are definitely input problems. "attempt": attempt, diff --git a/test/functional/tools/exit_code_from_env.xml b/test/functional/tools/exit_code_from_env.xml new file mode 100644 index 00000000000..970a7272e48 --- /dev/null +++ b/test/functional/tools/exit_code_from_env.xml @@ -0,0 +1,26 @@ + + + '$out_file1'; + : \${GX_TARGET_EXIT_CODE:-0}; + exit \${GX_TARGET_EXIT_CODE}; + ]]> + + + + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index b3a586e2977..112103d7e85 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -62,6 +62,7 @@ + diff --git a/test/integration/resubmission_tool_detected_always_error_job_conf.xml b/test/integration/resubmission_tool_detected_always_error_job_conf.xml new file mode 100644 index 00000000000..7b498cf6e5b --- /dev/null +++ b/test/integration/resubmission_tool_detected_always_error_job_conf.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + 4 + + + + diff --git a/test/integration/resubmission_tool_detected_resubmit_job_conf.xml b/test/integration/resubmission_tool_detected_resubmit_job_conf.xml new file mode 100644 index 00000000000..d7cecd554cf --- /dev/null +++ b/test/integration/resubmission_tool_detected_resubmit_job_conf.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + 4 + + + + + 0 + + + + diff --git a/test/integration/test_job_resubmission.py b/test/integration/test_job_resubmission.py index f4b586c3db2..1785749eec2 100644 --- a/test/integration/test_job_resubmission.py +++ b/test/integration/test_job_resubmission.py @@ -10,19 +10,21 @@ JOB_RESUBMISSION_DEFAULT_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resub JOB_RESUBMISSION_DYNAMIC_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_dynamic_job_conf.xml") JOB_RESUBMISSION_SMALL_MEMORY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_small_memory_job_conf.xml") JOB_RESUBMISSION_SMALL_MEMORY_RESUBMISSION_TO_LARGE_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_small_memory_resubmission_to_large_job_conf.xml") +JOB_RESUBMISSION_TOOL_DETECTED_ALWAYS_ERROR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_tool_detected_always_error_job_conf.xml") +JOB_RESUBMISSION_TOOL_DETECTED_RESUBMIT_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_tool_detected_resubmit_job_conf.xml") JOB_RESUBMISSION_JOB_RESOURCES_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_job_resource_parameters_conf.xml") class _BaseResubmissionIntegerationTestCase(integration_util.IntegrationTestCase): framework_tool_and_types = True - def _assert_job_passes(self, resource_parameters={}): - self._run_tool_test("exit_code_oom", resource_parameters=resource_parameters) + def _assert_job_passes(self, tool_id="exit_code_oom", resource_parameters={}): + self._run_tool_test(tool_id, resource_parameters=resource_parameters) - def _assert_job_fails(self, resource_parameters={}): + def _assert_job_fails(self, tool_id="exit_code_oom", resource_parameters={}): exception_thrown = False try: - self._run_tool_test("exit_code_oom", resource_parameters=resource_parameters) + self._run_tool_test(tool_id, resource_parameters=resource_parameters) except Exception: exception_thrown = True @@ -143,7 +145,8 @@ class JobResubmissionSmallMemoryIntegrationTestCase(_BaseResubmissionIntegeratio self._assert_job_fails() -# Verify the test tool fails if only a small amount of memory is allocated. +# Verify the test tool will resubmit on failure tested above and will then pass with +# proper resubmission condition. class JobResubmissionSmallMemoryResubmitsToLargeIntegrationTestCase(_BaseResubmissionIntegerationTestCase): @classmethod @@ -152,3 +155,26 @@ class JobResubmissionSmallMemoryResubmitsToLargeIntegrationTestCase(_BaseResubmi def test_dynamic_resubmission(self): self._assert_job_passes() + + +# Verify the test tool fails with an exit code issue. +class JobResubmissionToolDetectedErrorIntegrationTestCase(_BaseResubmissionIntegerationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["job_config_file"] = JOB_RESUBMISSION_TOOL_DETECTED_ALWAYS_ERROR_JOB_CONFIG_FILE + + def test_dynamic_resubmission(self): + self._assert_job_fails(tool_id="exit_code_from_env") + + +# Verify the test tool will resubmit on failure tested above and will then pass in +# an environment without a tool indicated error. +class JobResubmissionToolDetectedErrorResubmitsIntegrationTestCase(_BaseResubmissionIntegerationTestCase): + + @classmethod + def handle_galaxy_config_kwds(cls, config): + config["job_config_file"] = JOB_RESUBMISSION_TOOL_DETECTED_RESUBMIT_JOB_CONFIG_FILE + + def test_dynamic_resubmission(self): + self._assert_job_passes(tool_id="exit_code_from_env")