From 44b988ca2f55448f4eb31a167c5869bcbb63f157 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 15 Apr 2021 19:44:24 +0200 Subject: [PATCH] Restore setting hdca.update_time and fix the subcontainer query to filter the update time on the hdca, not on the DatasetCollection. This is necessary for HDCA state changes in the new history panel. Using the `with_for_update` should work around the observed deadlocks, and `skip_locked` should prevent any slowdown when many processes finish jobs belonging to the same implicit collection. --- lib/galaxy/managers/history_contents.py | 5 +---- lib/galaxy/model/__init__.py | 24 ++++++++++++++++++++++-- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/history_contents.py b/lib/galaxy/managers/history_contents.py index d8d4db39973..0f58b4f05bb 100644 --- a/lib/galaxy/managers/history_contents.py +++ b/lib/galaxy/managers/history_contents.py @@ -364,12 +364,9 @@ class HistoryContentsManager(containers.ContainerManagerMixin): # TODO: should be purgable? fix purged=literal(False), extension=literal(None), - # these are attached instead to the inner collection joined below - create_time=model.DatasetCollection.create_time, - update_time=model.DatasetCollection.update_time ) subquery = self._session().query(*columns) - # for the HDCA's we need to join the DatasetCollection since it has update/create times + # for the HDCA's we need to join the DatasetCollection since it has the populated_state subquery = subquery.join(model.DatasetCollection, model.DatasetCollection.id == component_class.collection_id) if history_id: diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index c102fe6ab23..a02783f3357 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1177,6 +1177,21 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): return rval + def update_hdca_update_time_for_job(self, update_time, sa_session): + subq = sa_session.query(HistoryDatasetCollectionAssociation.id) \ + .join(ImplicitCollectionJobs) \ + .join(ImplicitCollectionJobsJobAssociation) \ + .filter(ImplicitCollectionJobsJobAssociation.job_id == self.id) \ + .with_for_update(skip_locked=True).subquery() + implicit_statement = HistoryDatasetCollectionAssociation.table.update() \ + .where(HistoryDatasetCollectionAssociation.table.c.id.in_(subq)) \ + .values(update_time=update_time) + explicit_statement = HistoryDatasetCollectionAssociation.table.update() \ + .where(HistoryDatasetCollectionAssociation.table.c.job_id == self.id) \ + .values(update_time=update_time) + sa_session.execute(implicit_statement) + sa_session.execute(explicit_statement) + def set_final_state(self, final_state): self.set_state(final_state) # TODO: migrate to where-in subqueries? @@ -1186,9 +1201,12 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): WHERE job_id = :job_id; ''' sa_session = object_session(self) + update_time = galaxy.model.orm.now.now() + log.debug(f'Final state update time: {update_time}') + self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session) params = { 'job_id': self.id, - 'update_time': galaxy.model.orm.now.now() + 'update_time': update_time } sa_session.execute(statement, params) @@ -1256,11 +1274,13 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById): ); '''] sa_session = object_session(self) + update_time = galaxy.model.orm.now.now() + self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session) params = { 'job_id': self.id, 'state': self.state, 'info': self.info, - 'update_time': galaxy.model.orm.now.now() + 'update_time': update_time } for statement in statements: sa_session.execute(statement, params)