From ae12cc92eb12cada527cb44dc5c66ae7b355545e Mon Sep 17 00:00:00 2001
From: Nate Coraor <nate@bx.psu.edu>
Date: Thu, 25 Oct 2018 15:44:31 -0400
Subject: [PATCH] Catch all exceptions around runner shutdown so it doesn't
 prevent other runners from attempting shutdown.

---
 lib/galaxy/jobs/handler.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/lib/galaxy/jobs/handler.py b/lib/galaxy/jobs/handler.py
index bfc916719fe..a481bba6e48 100644
--- a/lib/galaxy/jobs/handler.py
+++ b/lib/galaxy/jobs/handler.py
@@ -965,5 +965,12 @@ class DefaultJobDispatcher(object):
             job_wrapper.fail(DEFAULT_JOB_PUT_FAILURE_MESSAGE)
 
     def shutdown(self):
-        for runner in self.job_runners.values():
-            runner.shutdown()
+        for name, runner in self.job_runners.items():
+            failures = []
+            try:
+                runner.shutdown()
+            except Exception:
+                failures.append(name)
+                log.exception("Failed to shutdown runner %s", name)
+            if failures:
+                raise Exception("Failed to shutdown runners: %s" % ', '.join(failures))