From 65a42e9bc6d368fe40c5eb1da1314de8668ebaf0 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 30 Mar 2015 12:32:47 -0400 Subject: [PATCH] Fix database deadlocks when running large workflows. This fix was first tested and initially verfied by Marius van den Beek with subsequent parallel verification from Philip Mabon and Aaron Petkau. Likely these deadlocks were introduced in 14.10 when I added an update time to workflow invocations so we can re-evaluate them over time - an explanation for why the deadlocks are likely occuring can be found in the code comments. --- lib/galaxy/jobs/__init__.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index df96bf209d7..d1fc3c7e706 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1251,6 +1251,16 @@ class JobWrapper( object ): if job.user: job.user.total_disk_usage += bytes + # Emperically, we need to update job.user and + # job.workflow_invocation_step.workflow_invocation in separate + # transactions. Best guess as to why is that the workflow_invocation + # may or may not exist when the job is first loaded by the handler - + # and depending on whether it is or not sqlalchemy orders the updates + # differently and deadlocks can occur (one thread updates user and + # waits on invocation and the other updates invocation and waits on + # user). + self.sa_session.flush() + # fix permissions for path in [ dp.real_path for dp in self.get_mutable_output_fnames() ]: util.umask_fix_perms( path, self.app.config.umask, 0666, self.app.config.gid )