From 8e32e93d6d8e048cbc77cb94f7c97b5ae0cfc6d6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 30 Jan 2018 14:47:39 -0500 Subject: [PATCH] Implement more collection workflow post job actions. Implement hide, delete, add tag, and remove tag for output collections and mapped over collections. Fixes https://github.com/galaxyproject/galaxy/issues/1680. --- lib/galaxy/jobs/actions/post.py | 51 +++++-- lib/galaxy/workflow/modules.py | 2 +- test/api/test_workflows.py | 167 +++++++++++++++++++++- test/base/workflows_format_2/converter.py | 22 +++ 4 files changed, 231 insertions(+), 11 deletions(-) diff --git a/lib/galaxy/jobs/actions/post.py b/lib/galaxy/jobs/actions/post.py index 822fb83634c..6003d68963d 100644 --- a/lib/galaxy/jobs/actions/post.py +++ b/lib/galaxy/jobs/actions/post.py @@ -86,7 +86,7 @@ class RenameDatasetAction(DefaultJobAction): verbose_name = "Rename Dataset" @classmethod - def execute_on_mapped_over(cls, app, sa_session, action, step_inputs, step_outputs, replacement_dict): + def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict): # Prevent renaming a dataset to the empty string. input_names = {} # Lookp through inputs find one with "to_be_replaced" input @@ -226,6 +226,16 @@ class HideDatasetAction(DefaultJobAction): if dataset_assoc.dataset.state != dataset_assoc.dataset.states.ERROR and (action.output_name == '' or dataset_assoc.name == action.output_name): dataset_assoc.dataset.visible = False + for dataset_collection_assoc in job.output_dataset_collection_instances: + if action.output_name == '' or dataset_collection_assoc.name == action.output_name: + dataset_collection_assoc.dataset_collection_instance.visible = False + + @classmethod + def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict): + for name, step_output in step_outputs.items(): + if action.output_name == '' or name == action.output_name: + step_output.visible = False + @classmethod def get_short_str(cls, pja): return "Hide output '%s'." % escape(pja.output_name) @@ -242,6 +252,16 @@ class DeleteDatasetAction(DefaultJobAction): if action.output_name == '' or dataset_assoc.name == action.output_name: dataset_assoc.dataset.deleted = True + for dataset_collection_assoc in job.output_dataset_collection_instances: + if action.output_name == '' or dataset_collection_assoc.name == action.output_name: + dataset_collection_assoc.dataset_collection_instance.deleted = True + + @classmethod + def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict): + for name, step_output in step_outputs.items(): + if action.output_name == '' or name == action.output_name: + step_output.deleted = True + @classmethod def get_short_str(cls, pja): return "Delete this dataset after creation." @@ -357,6 +377,15 @@ class TagDatasetAction(DefaultJobAction): action = "Add" direction = "to" + @classmethod + def execute_on_mapped_over(cls, trans, sa_session, action, step_inputs, step_outputs, replacement_dict): + if action.action_arguments: + tags = [t.replace('#', 'name:') if t.startswith('#') else t for t in [t.strip() for t in action.action_arguments.get('tags', '').split(',') if t.strip()]] + if tags: + for name, step_output in step_outputs.items(): + if action.output_name == '' or name == action.output_name: + cls._execute(trans.app, trans.user, step_output, tags) + @classmethod def execute(cls, app, sa_session, action, job, replacement_dict): if action.action_arguments: @@ -365,11 +394,16 @@ class TagDatasetAction(DefaultJobAction): for dataset_assoc in job.output_datasets: if action.output_name == '' or dataset_assoc.name == action.output_name: cls._execute(app, job.user, dataset_assoc.dataset, tags) + + for dataset_collection_assoc in job.output_dataset_collection_instances: + if action.output_name == '' or dataset_collection_assoc.name == action.output_name: + cls._execute(app, job.user, dataset_collection_assoc.dataset_collection_instance, tags) + sa_session.flush() @classmethod - def _execute(cls, app, user, dataset, tags): - app.tag_handler.add_tags_from_list(user, dataset, tags) + def _execute(cls, app, user, output, tags): + app.tag_handler.add_tags_from_list(user, output, tags) @classmethod def get_short_str(cls, pja): @@ -389,8 +423,8 @@ class RemoveTagDatasetAction(TagDatasetAction): direction = "from" @classmethod - def _execute(cls, app, user, dataset, tags): - app.tag_handler.remove_tags_from_list(user, dataset, tags) + def _execute(cls, app, user, output, tags): + app.tag_handler.remove_tags_from_list(user, output, tags) class ActionBox(object): @@ -413,7 +447,8 @@ class ActionBox(object): 'TagDatasetAction', 'RemoveTagDatasetAction'] # Actions that will be applied to implicit mapped over collection outputs and not # just individual outputs when steps include mapped over tools and implicit collection outputs. - mapped_over_output_actions = ['RenameDatasetAction'] + mapped_over_output_actions = ['RenameDatasetAction', 'HideDatasetAction', + 'TagDatasetAction', 'RemoveTagDatasetAction'] @classmethod def get_short_str(cls, action): @@ -445,9 +480,9 @@ class ActionBox(object): return npd @classmethod - def execute_on_mapped_over(cls, app, sa_session, pja, step_inputs, step_outputs, replacement_dict=None): + def execute_on_mapped_over(cls, trans, sa_session, pja, step_inputs, step_outputs, replacement_dict=None): if pja.action_type in ActionBox.actions: - ActionBox.actions[pja.action_type].execute_on_mapped_over(app, sa_session, pja, step_inputs, step_outputs, replacement_dict) + ActionBox.actions[pja.action_type].execute_on_mapped_over(trans, sa_session, pja, step_inputs, step_outputs, replacement_dict) @classmethod def execute(cls, app, sa_session, pja, job, replacement_dict=None): diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 0daea4f2d0b..a7b4303badc 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -986,7 +986,7 @@ class ToolModule(WorkflowModule): effective_post_job_actions = self._effective_post_job_actions(step) for pja in effective_post_job_actions: if pja.action_type in ActionBox.mapped_over_output_actions: - ActionBox.execute_on_mapped_over(self.trans.app, self.trans.sa_session, pja, step_inputs, step_outputs, replacement_dict) + ActionBox.execute_on_mapped_over(self.trans, self.trans.sa_session, pja, step_inputs, step_outputs, replacement_dict) def _handle_post_job_actions(self, step, job, replacement_dict): # Create new PJA associations with the created job, to be run on completion. diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 18b051647ee..bfdb59997e3 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -2000,7 +2000,7 @@ test_data: {} assert okay_dataset["state"] == "ok" @skip_without_tool("cat") - def test_run_rename_on_mapped_over_dataset(self): + def test_run_rename_on_mapped_over_collection(self): history_id = self.dataset_populator.new_history() self._run_jobs(""" class: GalaxyWorkflow @@ -2036,7 +2036,7 @@ test_data: assert name == "my new name", name @skip_without_tool("cat") - def test_run_rename_based_on_inputs_on_mapped_over_dataset(self): + def test_run_rename_based_on_inputs_on_mapped_over_collection(self): history_id = self.dataset_populator.new_history() self._run_jobs(""" class: GalaxyWorkflow @@ -2283,6 +2283,169 @@ test_data: name = content["name"] assert name == "the_dataset_pair suffix", name + @skip_without_tool("collection_creates_pair") + def test_run_hide_on_collection_output(self): + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" +class: GalaxyWorkflow +inputs: + - id: input1 +steps: + - tool_id: collection_creates_pair + state: + input1: + $link: input1 + outputs: + paired_output: + hide: true +test_data: + input1: + value: 1.fasta + type: File + name: fasta1 +""", history_id=history_id) + details1 = self.dataset_populator.get_history_collection_details(history_id, hid=4, wait=True, assert_ok=True) + + assert details1["history_content_type"] == "dataset_collection" + assert not details1["visible"], details1 + + @skip_without_tool("cat") + def test_run_hide_on_mapped_over_collection(self): + history_id = self.dataset_populator.new_history() + self._run_jobs(""" +class: GalaxyWorkflow +inputs: + - id: input1 + type: data_collection_input + collection_type: list +steps: + - tool_id: cat + label: first_cat + state: + input1: + $link: input1 + outputs: + out_file1: + hide: true +test_data: + input1: + type: list + name: the_dataset_list + elements: + - identifier: el1 + value: 1.fastq + type: File +""", history_id=history_id) + + content = self.dataset_populator.get_history_dataset_details(history_id, hid=4, wait=True, assert_ok=True) + assert content["history_content_type"] == "dataset" + assert content["visible"] is False + + content = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) + assert content["history_content_type"] == "dataset_collection", content + assert content["visible"] is False + + @skip_without_tool("collection_creates_pair") + def test_run_add_tag_on_collection_output(self): + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" +class: GalaxyWorkflow +inputs: + - id: input1 +steps: + - tool_id: collection_creates_pair + state: + input1: + $link: input1 + outputs: + paired_output: + add_tags: + - "name:foo" +test_data: + input1: + value: 1.fasta + type: File + name: fasta1 +""", history_id=history_id) + details1 = self.dataset_populator.get_history_collection_details(history_id, hid=4, wait=True, assert_ok=True) + + assert details1["history_content_type"] == "dataset_collection" + assert details1["tags"][0] == "name:foo", details1 + + @skip_without_tool("collection_creates_pair") + def test_run_add_tag_on_mapped_over_collection(self): + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" +class: GalaxyWorkflow +inputs: + - id: input1 + type: data_collection_input + collection_type: list +steps: + - tool_id: cat + label: first_cat + state: + input1: + $link: input1 + outputs: + out_file1: + add_tags: + - "name:foo" +test_data: + input1: + type: list + name: the_dataset_list + elements: + - identifier: el1 + value: 1.fastq + type: File +""", history_id=history_id) + details1 = self.dataset_populator.get_history_collection_details(history_id, hid=3, wait=True, assert_ok=True) + + assert details1["history_content_type"] == "dataset_collection" + assert details1["tags"][0] == "name:foo", details1 + + @skip_without_tool("collection_creates_pair") + @skip_without_tool("cat") + def test_run_remove_tag_on_collection_output(self): + with self.dataset_populator.test_history() as history_id: + self._run_jobs(""" +class: GalaxyWorkflow +inputs: + - id: input1 +steps: + - tool_id: cat + label: first_cat + state: + input1: + $link: input1 + outputs: + out_file1: + add_tags: + - "name:foo" + - tool_id: collection_creates_pair + state: + input1: + $link: first_cat#out_file1 + outputs: + paired_output: + remove_tags: + - "name:foo" +test_data: + input1: + value: 1.fasta + type: File + name: fasta1 +""", history_id=history_id) + details_dataset_with_tag = self.dataset_populator.get_history_dataset_details(history_id, hid=2, wait=True, assert_ok=True) + + assert details_dataset_with_tag["history_content_type"] == "dataset", details_dataset_with_tag + assert details_dataset_with_tag["tags"][0] == "name:foo", details_dataset_with_tag + + details_collection_without_tag = self.dataset_populator.get_history_collection_details(history_id, hid=5, wait=True, assert_ok=True) + assert details_collection_without_tag["history_content_type"] == "dataset_collection", details_collection_without_tag + assert len(details_collection_without_tag["tags"]) == 0, details_collection_without_tag + @skip_without_tool("cat1") def test_run_with_runtime_pja(self): workflow = self.workflow_populator.load_workflow(name="test_for_pja_runtime") diff --git a/test/base/workflows_format_2/converter.py b/test/base/workflows_format_2/converter.py index c35ec2797c0..6820e517fe1 100644 --- a/test/base/workflows_format_2/converter.py +++ b/test/base/workflows_format_2/converter.py @@ -390,6 +390,28 @@ def transform_tool(context, step): ) post_job_actions[action_name] = action + add_tags = output.get("add_tags", []) + if add_tags: + action_name = "TagDatasetAction%s" % name + arguments = dict(tags=",".join(add_tags)) + action = _action( + "TagDatasetAction", + name, + arguments + ) + post_job_actions[action_name] = action + + remove_tags = output.get("remove_tags", []) + if remove_tags: + action_name = "RemoveTagDatasetAction%s" % name + arguments = dict(tags=",".join(remove_tags)) + action = _action( + "RemoveTagDatasetAction", + name, + arguments + ) + post_job_actions[action_name] = action + del step["outputs"]