Fix resubmission on tool error detection.

Parts of this were wired up before but pieces were missing and so it was broken.

lib/galaxy/jobs/runners/state_handlers/resubmit.py

@@ -12,7 +12,8 @@ log = logging.getLogger(__name__)
 MESSAGES = dict(
     walltime_reached='it reached the walltime',
     memory_limit_reached='it exceeded the amount of allocated memory',
-    unknown_error='it encountered an unknown error'
+    unknown_error='it encountered an unknown error',
+    tool_detected='it encountered a tool detected error condition',
 )
 
 
@@ -41,6 +42,7 @@ def eval_condition(condition, job_state):
     condition_locals = {
         "walltime_reached": runner_state == JobState.runner_states.WALLTIME_REACHED,
         "memory_limit_reached": runner_state == JobState.runner_states.MEMORY_LIMIT_REACHED,
+        "tool_detected_failure": runner_state == JobState.runner_states.TOOL_DETECT_ERROR,
         "unknown_error": JobState.runner_states.UNKNOWN_ERROR,
         "any_failure": True,
         "any_potential_job_failure": True,  # Add a hook here - later on allow tools to describe things that are definitely input problems.
@@ -66,6 +68,7 @@ def failure(app, job_runner, job_state):
     if (runner_state not in (JobState.runner_states.WALLTIME_REACHED,
                              JobState.runner_states.MEMORY_LIMIT_REACHED,
                              JobState.runner_states.JOB_OUTPUT_NOT_RETURNED_FROM_CLUSTER,
+                             JobState.runner_states.TOOL_DETECT_ERROR,
                              JobState.runner_states.UNKNOWN_ERROR)):
         # not set or not a handleable runner state
         return
@@ -184,6 +187,7 @@ class _ExpressionContext(object):
                 "walltime_reached": runner_state == JobState.runner_states.WALLTIME_REACHED,
                 "memory_limit_reached": runner_state == JobState.runner_states.MEMORY_LIMIT_REACHED,
                 "unknown_error": JobState.runner_states.UNKNOWN_ERROR,
+                "tool_detected_failure": runner_state == JobState.runner_states.TOOL_DETECT_ERROR,
                 "any_failure": True,
                 "any_potential_job_failure": True,  # Add a hook here - later on allow tools to describe things that are definitely input problems.
                 "attempt": attempt,

test/functional/tools/exit_code_from_env.xml

@@ -0,0 +1,26 @@
+<tool id="exit_code_from_env" name="exit_code_from_env">
+    <!-- tool errors out with identified OOM error if less than 10MB are allocated. -->
+    <command detect_errors="exit_code" oom_exit_code="42"><![CDATA[
+        echo 'Hello' > '$out_file1';
+        : \${GX_TARGET_EXIT_CODE:-0};
+        exit \${GX_TARGET_EXIT_CODE};
+    ]]></command>
+    <inputs>
+        <param name="input" type="integer" label="Dummy" value="6" />
+    </inputs>
+    <outputs>
+        <data name="out_file1" />
+    </outputs>
+    <help>
+    </help>
+    <tests>
+        <test>
+            <param name="input" value="5" />
+            <output name="out_file1">
+                <assert_contents>
+                    <has_line line="Hello" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -62,6 +62,7 @@
   <tool file="version_command_tool_dir.xml" />
   <tool file="exit_code_oom.xml" />
   <tool file="exit_code_from_file.xml" />
+  <tool file="exit_code_from_env.xml" />
   <tool file="gzipped_inputs.xml" />
   <tool file="output_order.xml" />
   <tool file="output_format.xml" />

test/integration/resubmission_tool_detected_always_error_job_conf.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0"?>
+<job_conf>
+    <plugins>
+        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="2"/>
+    </plugins>
+
+    <handlers>
+        <handler id="main"/>
+    </handlers>
+
+    <destinations default="local_bad">
+        <destination id="local_bad" runner="local">
+            <env id="GX_TARGET_EXIT_CODE">4</env>
+        </destination>
+    </destinations>
+
+</job_conf>

test/integration/resubmission_tool_detected_resubmit_job_conf.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0"?>
+<job_conf>
+    <plugins>
+        <plugin id="local" type="runner" load="galaxy.jobs.runners.local:LocalJobRunner" workers="2"/>
+    </plugins>
+
+    <handlers>
+        <handler id="main"/>
+    </handlers>
+
+    <destinations default="local_resubmit">
+        <destination id="local_resubmit" runner="local">
+            <env id="GX_TARGET_EXIT_CODE">4</env>
+            <resubmit condition="tool_detected_failure" destination="local_good" />
+        </destination>
+
+        <destination id="local_good" runner="local">
+            <env id="GX_TARGET_EXIT_CODE">0</env>
+        </destination>
+    </destinations>
+
+</job_conf>

test/integration/test_job_resubmission.py

@@ -10,19 +10,21 @@ JOB_RESUBMISSION_DEFAULT_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resub
 JOB_RESUBMISSION_DYNAMIC_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_dynamic_job_conf.xml")
 JOB_RESUBMISSION_SMALL_MEMORY_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_small_memory_job_conf.xml")
 JOB_RESUBMISSION_SMALL_MEMORY_RESUBMISSION_TO_LARGE_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_small_memory_resubmission_to_large_job_conf.xml")
+JOB_RESUBMISSION_TOOL_DETECTED_ALWAYS_ERROR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_tool_detected_always_error_job_conf.xml")
+JOB_RESUBMISSION_TOOL_DETECTED_RESUBMIT_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_tool_detected_resubmit_job_conf.xml")
 JOB_RESUBMISSION_JOB_RESOURCES_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "resubmission_job_resource_parameters_conf.xml")
 
 
 class _BaseResubmissionIntegerationTestCase(integration_util.IntegrationTestCase):
     framework_tool_and_types = True
 
-    def _assert_job_passes(self, resource_parameters={}):
-        self._run_tool_test("exit_code_oom", resource_parameters=resource_parameters)
+    def _assert_job_passes(self, tool_id="exit_code_oom", resource_parameters={}):
+        self._run_tool_test(tool_id, resource_parameters=resource_parameters)
 
-    def _assert_job_fails(self, resource_parameters={}):
+    def _assert_job_fails(self, tool_id="exit_code_oom", resource_parameters={}):
         exception_thrown = False
         try:
-            self._run_tool_test("exit_code_oom", resource_parameters=resource_parameters)
+            self._run_tool_test(tool_id, resource_parameters=resource_parameters)
         except Exception:
             exception_thrown = True
 
@@ -143,7 +145,8 @@ class JobResubmissionSmallMemoryIntegrationTestCase(_BaseResubmissionIntegeratio
         self._assert_job_fails()
 
 
-# Verify the test tool fails if only a small amount of memory is allocated.
+# Verify the test tool will resubmit on failure tested above and will then pass with
+# proper resubmission condition.
 class JobResubmissionSmallMemoryResubmitsToLargeIntegrationTestCase(_BaseResubmissionIntegerationTestCase):
 
     @classmethod
@@ -152,3 +155,26 @@ class JobResubmissionSmallMemoryResubmitsToLargeIntegrationTestCase(_BaseResubmi
 
     def test_dynamic_resubmission(self):
         self._assert_job_passes()
+
+
+# Verify the test tool fails with an exit code issue.
+class JobResubmissionToolDetectedErrorIntegrationTestCase(_BaseResubmissionIntegerationTestCase):
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["job_config_file"] = JOB_RESUBMISSION_TOOL_DETECTED_ALWAYS_ERROR_JOB_CONFIG_FILE
+
+    def test_dynamic_resubmission(self):
+        self._assert_job_fails(tool_id="exit_code_from_env")
+
+
+# Verify the test tool will resubmit on failure tested above and will then pass in
+# an environment without a tool indicated error.
+class JobResubmissionToolDetectedErrorResubmitsIntegrationTestCase(_BaseResubmissionIntegerationTestCase):
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        config["job_config_file"] = JOB_RESUBMISSION_TOOL_DETECTED_RESUBMIT_JOB_CONFIG_FILE
+
+    def test_dynamic_resubmission(self):
+        self._assert_job_passes(tool_id="exit_code_from_env")