Merge pull request #11434 from mvdbeek/recover_workflow_handler_from_database_hiccup

[20.09] Wrap core of workflow monitor thread in try/except
This commit is contained in:
Marius van den Beek
2021-02-23 08:57:41 +01:00
committed by GitHub
+14 -11
View File
@@ -283,19 +283,22 @@ class WorkflowRequestMonitor(Monitors):
def __monitor(self):
to_monitor = self.workflow_scheduling_manager.active_workflow_schedulers
while self.monitor_running:
if self.invocation_grabber:
self.invocation_grabber.grab_unhandled_items()
try:
if self.invocation_grabber:
self.invocation_grabber.grab_unhandled_items()
monitor_step_timer = self.app.execution_timer_factory.get_timer(
'internal.galaxy.workflows.scheduling_manager.monitor_step',
'Workflow scheduling manager monitor step complete.'
)
for workflow_scheduler_id, workflow_scheduler in to_monitor.items():
if not self.monitor_running:
return
monitor_step_timer = self.app.execution_timer_factory.get_timer(
'internal.galaxy.workflows.scheduling_manager.monitor_step',
'Workflow scheduling manager monitor step complete.'
)
for workflow_scheduler_id, workflow_scheduler in to_monitor.items():
if not self.monitor_running:
return
self.__schedule(workflow_scheduler_id, workflow_scheduler)
log.trace(monitor_step_timer.to_str())
self.__schedule(workflow_scheduler_id, workflow_scheduler)
log.trace(monitor_step_timer.to_str())
except Exception:
log.exception('An exception occured scheduling while scheduling workflows')
self._monitor_sleep(1)
def __schedule(self, workflow_scheduler_id, workflow_scheduler):