From 8a16c56cac74a83ddb964f0bc6d95577c07aef88 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 12 Jan 2022 20:00:37 +0100 Subject: [PATCH 1/3] Fix re-run with rerun_remap_job_id hiding collection Fixes https://github.com/galaxyproject/galaxy/issues/13052 --- lib/galaxy/tools/actions/__init__.py | 3 ++- lib/galaxy_test/api/test_jobs.py | 38 +++++++++++++++++++++------- 2 files changed, 31 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 55b71fb233a..1624f72bda0 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -639,6 +639,8 @@ class DefaultToolAction(ToolAction): assert old_job.session_id == galaxy_session.id, f'({old_job.id}/{current_job.id}): Old session id ({old_job.session_id}) does not match rerun session id ({galaxy_session.id})' else: raise Exception(f'({old_job.id}/{current_job.id}): Remapping via the API is not (yet) supported') + # Start by hiding current job outputs before taking over the old job's (implicit) outputs. + current_job.hide_outputs(flush=False) # Duplicate PJAs before remap. for pjaa in old_job.post_job_actions: current_job.add_post_job_action(pjaa.post_job_action) @@ -676,7 +678,6 @@ class DefaultToolAction(ToolAction): job.job_id = current_job.id for jtoidca in old_job.output_dataset_collections: jtoidca.dataset_collection.replace_failed_elements(remapped_hdas) - current_job.hide_outputs(flush=False) except Exception: log.exception('Cannot remap rerun dependencies.') diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index a9edf51a8e3..e48eaf43180 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -268,18 +268,21 @@ steps: assert_ok=False) assert dataset['visible'] + def _run_map_over_error(self, history_id): + hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-1", "1 2 3")]).json() + inputs = { + 'error_bool': 'true', + 'dataset': { + 'batch': True, + 'values': [{'src': 'hdca', 'id': hdca1['id']}], + } + } + return self._run_detect_errors(history_id=history_id, inputs=inputs) + @skip_without_tool("detect_errors_aggressive") def test_no_unhide_on_error_if_mapped_over(self): with self.dataset_populator.test_history() as history_id: - hdca1 = self.dataset_collection_populator.create_list_in_history(history_id, contents=[("sample1-1", "1 2 3")]).json() - inputs = { - 'error_bool': 'true', - 'dataset': { - 'batch': True, - 'values': [{'src': 'hdca', 'id': hdca1['id']}], - } - } - run_response = self._run_detect_errors(history_id=history_id, inputs=inputs) + run_response = self._run_map_over_error(history_id) job_id = run_response['jobs'][0]["id"] self.dataset_populator.wait_for_job(job_id) job = self.dataset_populator.get_job_details(job_id).json() @@ -289,6 +292,23 @@ steps: assert_ok=False) assert not dataset['visible'] + def test_no_hide_on_rerun(self): + with self.dataset_populator.test_history() as history_id: + run_response = self._run_map_over_error(history_id) + assert run_response['implicit_collections'][0]['visible'] + job_id = run_response['jobs'][0]["id"] + rerun_params = self._get(f"jobs/{job_id}/build_for_rerun").json() + inputs = rerun_params['state_inputs'] + inputs['rerun_remap_job_id'] = job_id + self._run_detect_errors(history_id=history_id, inputs=inputs) + # Verify source hdca is still visible + hdca = self.dataset_populator.get_history_collection_details( + history_id=history_id, + content_id=run_response['implicit_collections'][0]['id'], + assert_ok=False, + ) + assert hdca['visible'] + @skip_without_tool('empty_output') def test_common_problems(self): with self.dataset_populator.test_history() as history_id: From 7c4568fe90415f5475766f4a3671f22a707cd3cc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 14 Jan 2022 11:49:04 +0100 Subject: [PATCH 2/3] Set collection update_time when replacing failed elements That allows the new history to update the collection. --- lib/galaxy/model/__init__.py | 28 ++++++++++++++++------------ lib/galaxy/tools/actions/__init__.py | 1 + lib/galaxy_test/api/test_jobs.py | 15 +++++++++++++-- 3 files changed, 30 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index f679b0493bd..500e2d80fd3 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -325,15 +325,20 @@ class HasName: class UsesCreateAndUpdateTime: + update_time: DateTime + @property def seconds_since_updated(self): - update_time = self.update_time or galaxy.model.orm.now.now() # In case not yet flushed - return (galaxy.model.orm.now.now() - update_time).total_seconds() + update_time = self.update_time or now() # In case not yet flushed + return (now() - update_time).total_seconds() @property def seconds_since_created(self): - create_time = self.create_time or galaxy.model.orm.now.now() # In case not yet flushed - return (galaxy.model.orm.now.now() - create_time).total_seconds() + create_time = self.create_time or now() # In case not yet flushed + return (now() - create_time).total_seconds() + + def update(self): + self.update_time = now() class WorkerProcess(Base, UsesCreateAndUpdateTime, _HasTable): @@ -805,7 +810,7 @@ class PasswordResetToken(Base, _HasTable): else: self.token = unique_id() self.user = user - self.expiration_time = galaxy.model.orm.now.now() + timedelta(hours=24) + self.expiration_time = now() + timedelta(hours=24) class DynamicTool(Base, Dictifiable, RepresentById): @@ -1474,7 +1479,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): WHERE job_id = :job_id; ''' sa_session = object_session(self) - update_time = galaxy.model.orm.now.now() + update_time = now() self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked) params = { 'job_id': self.id, @@ -1546,7 +1551,7 @@ class Job(Base, JobLike, UsesCreateAndUpdateTime, Dictifiable, Serializable): ); '''] sa_session = object_session(self) - update_time = galaxy.model.orm.now.now() + update_time = now() self.update_hdca_update_time_for_job(update_time=update_time, sa_session=sa_session, supports_skip_locked=supports_skip_locked) params = { 'job_id': self.id, @@ -3499,7 +3504,7 @@ def datatype_for_extension(extension, datatypes_registry=None) -> "Data": return ret -class DatasetInstance(_HasTable): +class DatasetInstance(UsesCreateAndUpdateTime, _HasTable): """A base class for all 'dataset instances', HDAs, LDAs, etc""" states = Dataset.states conversion_messages = Dataset.conversion_messages @@ -3552,9 +3557,6 @@ class DatasetInstance(_HasTable): def peek(self, peek): self._peek = unicodify(peek, strip_null=True) - def update(self): - self.update_time = galaxy.model.orm.now.now() - @property def ext(self): return self.extension @@ -5480,7 +5482,7 @@ class DatasetCollection(Base, Dictifiable, UsesAnnotations, Serializable): return rval -class DatasetCollectionInstance(HasName): +class DatasetCollectionInstance(HasName, UsesCreateAndUpdateTime): @property def state(self): @@ -5723,6 +5725,8 @@ class HistoryDatasetCollectionAssociation( deleted=self.deleted, job_source_id=self.job_source_id, job_source_type=self.job_source_type, + create_time=self.create_time.isoformat(), + update_time=self.update_time.isoformat(), **self._base_to_dict(view=view) ) diff --git a/lib/galaxy/tools/actions/__init__.py b/lib/galaxy/tools/actions/__init__.py index 1624f72bda0..6ee80c94bca 100644 --- a/lib/galaxy/tools/actions/__init__.py +++ b/lib/galaxy/tools/actions/__init__.py @@ -676,6 +676,7 @@ class DefaultToolAction(ToolAction): for job in hdca.implicit_collection_jobs.jobs: if job.job_id == old_job.id: job.job_id = current_job.id + hdca.update() for jtoidca in old_job.output_dataset_collections: jtoidca.dataset_collection.replace_failed_elements(remapped_hdas) except Exception: diff --git a/lib/galaxy_test/api/test_jobs.py b/lib/galaxy_test/api/test_jobs.py index e48eaf43180..9af31548f73 100644 --- a/lib/galaxy_test/api/test_jobs.py +++ b/lib/galaxy_test/api/test_jobs.py @@ -6,6 +6,7 @@ import urllib.parse from operator import itemgetter import requests +from dateutil.parser import isoparse from galaxy_test.api.test_tools import TestsTools from galaxy_test.base.api_asserts import assert_status_code_is_ok @@ -295,12 +296,21 @@ steps: def test_no_hide_on_rerun(self): with self.dataset_populator.test_history() as history_id: run_response = self._run_map_over_error(history_id) - assert run_response['implicit_collections'][0]['visible'] job_id = run_response['jobs'][0]["id"] + self.dataset_populator.wait_for_job(job_id) + failed_hdca = self.dataset_populator.get_history_collection_details( + history_id=history_id, + content_id=run_response['implicit_collections'][0]['id'], + assert_ok=False, + ) + first_update_time = failed_hdca['update_time'] + assert failed_hdca['visible'] rerun_params = self._get(f"jobs/{job_id}/build_for_rerun").json() inputs = rerun_params['state_inputs'] inputs['rerun_remap_job_id'] = job_id - self._run_detect_errors(history_id=history_id, inputs=inputs) + rerun_response = self._run_detect_errors(history_id=history_id, inputs=inputs) + rerun_job_id = rerun_response['jobs'][0]["id"] + self.dataset_populator.wait_for_job(rerun_job_id) # Verify source hdca is still visible hdca = self.dataset_populator.get_history_collection_details( history_id=history_id, @@ -308,6 +318,7 @@ steps: assert_ok=False, ) assert hdca['visible'] + assert isoparse(hdca['update_time']) > (isoparse(first_update_time)) @skip_without_tool('empty_output') def test_common_problems(self): From 9dcea35ae59bcfd22c57407d76a439e6155eff9b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Jan 2022 10:14:28 +0100 Subject: [PATCH 3/3] Show yaml files in preview The `application/yaml` mediatype might be the logical mediatype for yaml (there's no standard yet), but we want to see a textual preview (in the absence of a viewer). Fixes https://github.com/galaxyproject/galaxy/issues/13168 --- lib/galaxy/datatypes/text.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 2f20a120597..4fbe8926462 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -12,7 +12,7 @@ import tempfile import yaml -from galaxy.datatypes.data import get_file_peek, Text +from galaxy.datatypes.data import get_file_peek, Headers, Text from galaxy.datatypes.metadata import MetadataElement, MetadataParameter from galaxy.datatypes.sniff import ( build_sniff_from_prefix, @@ -884,6 +884,13 @@ class Yaml(Text): """Returns the mime type of the datatype""" return 'application/yaml' + def _yield_user_file_content(self, trans, from_dataset, filename, headers: Headers): + # Override non-standard application/yaml mediatype with + # non-standard text/x-yaml, so preview is shown in preview iframe, + # instead of downloading the file. + headers['content-type'] = 'text/x-yaml' + return super()._yield_user_file_content(trans, from_dataset, filename, headers) + def _looks_like_yaml(self, file_prefix: FilePrefix): # Pattern used by SequenceSplitLocations if file_prefix.file_size < 50000 and not file_prefix.truncated: