Merge pull request #11673 from mvdbeek/fix_fail_job_cycle

[21.01] Prevent endless fail_job cycles

lib/galaxy/jobs/__init__.py

@@ -1334,7 +1334,10 @@ class JobWrapper(HasResourceParameters):
             ActionBox.execute(self.app, self.sa_session, pja, job)
         # If the job was deleted, call tool specific fail actions (used for e.g. external metadata) and clean up
         if self.tool:
-            self.tool.job_failed(self, message, exception)
+            try:
+                self.tool.job_failed(self, message, exception)
+            except Exception:
+                log.exception(f"Error occured while calling tool specific fail actions for job {job.id}")
         cleanup_job = self.cleanup_job
         delete_files = cleanup_job == 'always' or (cleanup_job == 'onsuccess' and job.state == job.states.DELETED)
         self.cleanup(delete_files=delete_files)

lib/galaxy/jobs/runners/__init__.py

@@ -141,7 +141,9 @@ class BaseJobRunner:
                     job_state = JobState(job_wrapper=arg, job_destination={})
                 else:
                     job_state = arg
-                self.work_queue.put((self.fail_job, job_state))
+                if method != self.fail_job:
+                    # Prevent fail_job cycle in the work_queue
+                    self.work_queue.put((self.fail_job, job_state))
 
     # Causes a runner's `queue_job` method to be called from a worker thread
     def put(self, job_wrapper):