Merge pull request #13157 from mvdbeek/fix_hiding_on_rerun

[21.09] Fix re-run with rerun_remap_job_id hiding collection

lib/galaxy/model/__init__.py

@@ -295,15 +295,20 @@ class HasName:
 
 class UsesCreateAndUpdateTime:
 
+    update_time: DateTime
+
     @property
     def seconds_since_updated(self):
-        update_time = self.update_time or galaxy.model.orm.now.now()  # In case not yet flushed
-        return (galaxy.model.orm.now.now() - update_time).total_seconds()
+        update_time = self.update_time or now()  # In case not yet flushed
+        return (now() - update_time).total_seconds()
 
     @property
     def seconds_since_created(self):
-        create_time = self.create_time or galaxy.model.orm.now.now()  # In case not yet flushed
-        return (galaxy.model.orm.now.now() - create_time).total_seconds()
+        create_time = self.create_time or now()  # In case not yet flushed
+        return (now() - create_time).total_seconds()
+
+    def update(self):
+        self.update_time = now()
 
 
 class WorkerProcess(Base, UsesCreateAndUpdateTime, _HasTable):
@@ -776,7 +781,7 @@ class PasswordResetToken(Base, _HasTable):
         else:
             self.token = unique_id()
         self.user = user
-        self.expiration_time = galaxy.model.orm.now.now() + timedelta(hours=24)
+        self.expiration_time = now() + timedelta(hours=24)
 
 
 class DynamicTool(Base, Dictifiable, RepresentById):
@@ -1452,7 +1457,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
             WHERE job_id = :job_id;
         '''
         sa_session = object_session(self)
-        update_time = galaxy.model.orm.now.now()
+        update_time = now()
         self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked)
         params = {
             'job_id': self.id,
@@ -1524,7 +1529,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, RepresentById):
             );
         ''']
         sa_session = object_session(self)
-        update_time = galaxy.model.orm.now.now()
+        update_time = now()
         self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked)
         params = {
             'job_id': self.id,
@@ -3466,7 +3471,7 @@ def datatype_for_extension(extension, datatypes_registry=None):
     return ret
 
 
-class DatasetInstance:
+class DatasetInstance(UsesCreateAndUpdateTime):
     """A base class for all 'dataset instances', HDAs, LDAs, etc"""
     states = Dataset.states
     conversion_messages = Dataset.conversion_messages
@@ -3519,9 +3524,6 @@ class DatasetInstance:
     def peek(self, peek):
         self._peek = unicodify(peek, strip_null=True)
 
-    def update(self):
-        self.update_time = galaxy.model.orm.now.now()
-
     @property
     def ext(self):
         return self.extension
@@ -5430,7 +5432,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, RepresentById):
         return rval
 
 
-class DatasetCollectionInstance(HasName):
+class DatasetCollectionInstance(HasName, UsesCreateAndUpdateTime):
 
     @property
     def state(self):
@@ -5692,6 +5694,8 @@ class HistoryDatasetCollectionAssociation(
                 deleted=self.deleted,
                 job_source_id=self.job_source_id,
                 job_source_type=self.job_source_type,
+                create_time=self.create_time.isoformat(),
+                update_time=self.update_time.isoformat(),
                 **self._base_to_dict(view=view)
             )
 

lib/galaxy/tools/actions/__init__.py

@@ -623,6 +623,8 @@ class DefaultToolAction:
                 assert old_job.session_id == galaxy_session.id, f'({old_job.id}/{current_job.id}): Old session id ({old_job.session_id}) does not match rerun session id ({galaxy_session.id})'
             else:
                 raise Exception(f'({old_job.id}/{current_job.id}): Remapping via the API is not (yet) supported')
+            # Start by hiding current job outputs before taking over the old job's (implicit) outputs.
+            current_job.hide_outputs(flush=False)
             # Duplicate PJAs before remap.
             for pjaa in old_job.post_job_actions:
                 current_job.add_post_job_action(pjaa.post_job_action)
@@ -658,9 +660,9 @@ class DefaultToolAction:
                     for job in hdca.implicit_collection_jobs.jobs:
                         if job.job_id == old_job.id:
                             job.job_id = current_job.id
+                hdca.update()
             for jtoidca in old_job.output_dataset_collections:
                 jtoidca.dataset_collection.replace_failed_elements(remapped_hdas)
-            current_job.hide_outputs(flush=False)
         except Exception:
             log.exception('Cannot remap rerun dependencies.')
 

lib/galaxy_test/api/test_jobs.py

@@ -5,6 +5,7 @@ import time
 from operator import itemgetter
 
 import requests
+from dateutil.parser import isoparse
 
 from galaxy_test.api.test_tools import TestsTools
 from galaxy_test.base.api_asserts import assert_status_code_is_ok
@@ -269,18 +270,21 @@ steps:
                                                                          assert_ok=False)
             assert dataset['visible']
 
+    def _run_map_over_error(self, history_id):
+        hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-1", "1 2 3")]).json()
+        inputs = {
+            'error_bool': 'true',
+            'dataset': {
+                'batch': True,
+                'values': [{'src': 'hdca', 'id': hdca1['id']}],
+            }
+        }
+        return self._run_detect_errors(history_id=history_id, inputs=inputs)
+
     @skip_without_tool("detect_errors_aggressive")
     def test_no_unhide_on_error_if_mapped_over(self):
         with self.dataset_populator.test_history() as history_id:
-            hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-1", "1 2 3")]).json()
-            inputs = {
-                'error_bool': 'true',
-                'dataset': {
-                    'batch': True,
-                    'values': [{'src': 'hdca', 'id': hdca1['id']}],
-                }
-            }
-            run_response = self._run_detect_errors(history_id=history_id, inputs=inputs)
+            run_response = self._run_map_over_error(history_id)
             job_id = run_response['jobs'][0]["id"]
             self.dataset_populator.wait_for_job(job_id)
             job = self.dataset_populator.get_job_details(job_id).json()
@@ -290,6 +294,33 @@ steps:
                                                                          assert_ok=False)
             assert not dataset['visible']
 
+    def test_no_hide_on_rerun(self):
+        with self.dataset_populator.test_history() as history_id:
+            run_response = self._run_map_over_error(history_id)
+            job_id = run_response['jobs'][0]["id"]
+            self.dataset_populator.wait_for_job(job_id)
+            failed_hdca = self.dataset_populator.get_history_collection_details(
+                history_id=history_id,
+                content_id=run_response['implicit_collections'][0]['id'],
+                assert_ok=False,
+            )
+            first_update_time = failed_hdca['update_time']
+            assert failed_hdca['visible']
+            rerun_params = self._get(f"jobs/{job_id}/build_for_rerun").json()
+            inputs = rerun_params['state_inputs']
+            inputs['rerun_remap_job_id'] = job_id
+            rerun_response = self._run_detect_errors(history_id=history_id, inputs=inputs)
+            rerun_job_id = rerun_response['jobs'][0]["id"]
+            self.dataset_populator.wait_for_job(rerun_job_id)
+            # Verify source hdca is still visible
+            hdca = self.dataset_populator.get_history_collection_details(
+                history_id=history_id,
+                content_id=run_response['implicit_collections'][0]['id'],
+                assert_ok=False,
+            )
+            assert hdca['visible']
+            assert isoparse(hdca['update_time']) > (isoparse(first_update_time))
+
     @skip_without_tool('empty_output')
     def test_common_problems(self):
         with self.dataset_populator.test_history() as history_id: