Restore setting hdca.update_time

and fix the subcontainer query to filter the update time on the hdca,
not on the DatasetCollection. This is necessary for HDCA state changes
in the new history panel.
Using the `with_for_update` should work around the observed deadlocks,
and `skip_locked` should prevent any slowdown when many processes
finish jobs belonging to the same implicit collection.

lib/galaxy/managers/history_contents.py

@@ -364,12 +364,9 @@ class HistoryContentsManager(containers.ContainerManagerMixin):
             # TODO: should be purgable? fix
             purged=literal(False),
             extension=literal(None),
-            # these are attached instead to the inner collection joined below
-            create_time=model.DatasetCollection.create_time,
-            update_time=model.DatasetCollection.update_time
         )
         subquery = self._session().query(*columns)
-        # for the HDCA's we need to join the DatasetCollection since it has update/create times
+        # for the HDCA's we need to join the DatasetCollection since it has the populated_state
         subquery = subquery.join(model.DatasetCollection,
             model.DatasetCollection.id == component_class.collection_id)
         if history_id:

lib/galaxy/model/__init__.py

@@ -1177,6 +1177,21 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
 
         return rval
 
+    def update_hdca_update_time_for_job(self, update_time, sa_session):
+        subq = sa_session.query(HistoryDatasetCollectionAssociation.id) \
+            .join(ImplicitCollectionJobs) \
+            .join(ImplicitCollectionJobsJobAssociation) \
+            .filter(ImplicitCollectionJobsJobAssociation.job_id == self.id) \
+            .with_for_update(skip_locked=True).subquery()
+        implicit_statement = HistoryDatasetCollectionAssociation.table.update() \
+            .where(HistoryDatasetCollectionAssociation.table.c.id.in_(subq)) \
+            .values(update_time=update_time)
+        explicit_statement = HistoryDatasetCollectionAssociation.table.update() \
+            .where(HistoryDatasetCollectionAssociation.table.c.job_id == self.id) \
+            .values(update_time=update_time)
+        sa_session.execute(implicit_statement)
+        sa_session.execute(explicit_statement)
+
     def set_final_state(self, final_state):
         self.set_state(final_state)
         # TODO: migrate to where-in subqueries?
@@ -1186,9 +1201,12 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
             WHERE job_id = :job_id;
         '''
         sa_session = object_session(self)
+        update_time = galaxy.model.orm.now.now()
+        log.debug(f'Final state update time: {update_time}')
+        self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session)
         params = {
             'job_id': self.id,
-            'update_time': galaxy.model.orm.now.now()
+            'update_time': update_time
         }
         sa_session.execute(statement, params)
 
@@ -1256,11 +1274,13 @@ class Job(JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
             );
         ''']
         sa_session = object_session(self)
+        update_time = galaxy.model.orm.now.now()
+        self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session)
         params = {
             'job_id': self.id,
             'state': self.state,
             'info': self.info,
-            'update_time': galaxy.model.orm.now.now()
+            'update_time': update_time
         }
         for statement in statements:
             sa_session.execute(statement, params)