From 090b6af592182bed9df4eafb9b3a857afefceed8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 1 Aug 2019 11:52:11 +0200 Subject: [PATCH 1/8] Only set workflow tags if they are include in workflow update payload We accept different kinds of payloads here. The summary payload as sent by the workflow overview page includes a tags key, while the workflow payload sent by the workflow editor only includes the steps (tag updates are sent through a different endpoint), so anytime a workflow is saved in the workflow editor we would fallback to an empty list as the new tags, effectively removing all tags. Fixes https://github.com/galaxyproject/galaxy/issues/6888. --- lib/galaxy/webapps/galaxy/api/workflows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 51d21275bcc..7a03f34b8be 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -552,7 +552,8 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn if (trans.security.decode_id(id) in entries): trans.get_user().stored_workflow_menu_entries.remove(entries[trans.security.decode_id(id)]) # set tags - trans.app.tag_handler.set_tags_from_list(user=trans.user, item=stored_workflow, new_tags_list=workflow_dict.get('tags', [])) + if 'tags' in workflow_dict: + trans.app.tag_handler.set_tags_from_list(user=trans.user, item=stored_workflow, new_tags_list=workflow_dict['tags']) if 'steps' in workflow_dict: try: From 98321919fdf7dfc1e2d2d81f67e53ec915f3b105 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Tue, 30 Jul 2019 13:28:41 +0200 Subject: [PATCH 2/8] Fix ignored assert_command tests These were swalled becasue the `raw_test_dict` already lists them under `command_line`, not `command`. (https://github.com/galaxyproject/galaxy/blob/418f1220c8f6b21bcf3b4ddcf9d19f29bdb2c373/lib/galaxy/tools/test.py#L49) --- lib/galaxy/tools/verify/interactor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tools/verify/interactor.py b/lib/galaxy/tools/verify/interactor.py index 036d2a78760..97737259db2 100644 --- a/lib/galaxy/tools/verify/interactor.py +++ b/lib/galaxy/tools/verify/interactor.py @@ -1064,7 +1064,7 @@ class ToolTestDescription(object): self.exception = processed_test_dict.get("exception", None) self.output_collections = map(TestCollectionOutputDef.from_dict, processed_test_dict.get("output_collections", [])) - self.command_line = processed_test_dict.get("command", None) + self.command_line = processed_test_dict.get("command_line", None) self.stdout = processed_test_dict.get("stdout", None) self.stderr = processed_test_dict.get("stderr", None) self.expect_exit_code = processed_test_dict.get("expect_exit_code", None) From e164290c5ca872adc4d2cd9d6ee8e1b60a2f2210 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 1 Aug 2019 12:25:30 +0200 Subject: [PATCH 3/8] Always return serialized workflow when updating This is more predictable, just returning the payload if there are no changes isn't that useful. --- lib/galaxy/webapps/galaxy/api/workflows.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/workflows.py b/lib/galaxy/webapps/galaxy/api/workflows.py index 7a03f34b8be..9da77b9eec0 100644 --- a/lib/galaxy/webapps/galaxy/api/workflows.py +++ b/lib/galaxy/webapps/galaxy/api/workflows.py @@ -566,9 +566,6 @@ class WorkflowsAPIController(BaseAPIController, UsesStoredWorkflowMixin, UsesAnn ) except workflows.MissingToolsException: raise exceptions.MessageException("This workflow contains missing tools. It cannot be saved until they have been removed from the workflow or installed.") - else: - # We only adjusted tags and menu entry - return payload else: message = "Updating workflow requires dictionary containing 'workflow' attribute with new JSON description." raise exceptions.RequestParameterInvalidException(message) From ae84a553f64b7d2738be2f960096e55a38b782ae Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 1 Aug 2019 12:27:02 +0200 Subject: [PATCH 4/8] Add tests for correct tag saving/updating via workflow update API --- test/api/test_workflows.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/test/api/test_workflows.py b/test/api/test_workflows.py index 4f357742b4f..c25b33df595 100644 --- a/test/api/test_workflows.py +++ b/test/api/test_workflows.py @@ -343,6 +343,20 @@ class WorkflowsApiTestCase(BaseWorkflowsApiTestCase): # Make sure the positions have been updated. map(tweak_step, updated_workflow_content['steps'].items()) + def test_update_tags(self): + workflow_object = self.workflow_populator.load_workflow(name="test_import") + upload_response = self.__test_upload(workflow=workflow_object) + workflow = upload_response.json() + workflow['tags'] = ['a_tag', 'b_tag'] + update_response = self._update_workflow(workflow['id'], workflow).json() + assert update_response['tags'] == ['a_tag', 'b_tag'] + del workflow['tags'] + update_response = self._update_workflow(workflow['id'], workflow).json() + assert update_response['tags'] == ['a_tag', 'b_tag'] + workflow['tags'] = [] + update_response = self._update_workflow(workflow['id'], workflow).json() + assert update_response['tags'] == [] + def test_update_no_tool_id(self): workflow_object = self.workflow_populator.load_workflow(name="test_import") upload_response = self.__test_upload(workflow=workflow_object) From 6bdb1fde0bbc5f04e4ad00bb29a97cd6db78e499 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 15 Jul 2019 13:57:00 -0400 Subject: [PATCH 5/8] remove libpython requirement for uwsgi building --- .circleci/config.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 01b3105991a..26cd1f6ffed 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -140,9 +140,6 @@ jobs: steps: - *restore_repo_cache - *install_tox - - run: sudo apt-get update - # For uwsgi - - run: sudo apt-get install -y libpython3.5-dev - run: tox -e py35-first_startup validate_test_tools: docker: From 3de99087d9bfa5f5dab0f7d071345186e0890a40 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 6 Aug 2019 16:33:13 -0400 Subject: [PATCH 6/8] Don't parse HTML for tool help search index. Just use the simpler raw rST (or Markdown in the future). This should prevent all rst_to_html loading from happening at Galaxy startup - the existing caching should allow it to just be loaded and cached on first use. --- lib/galaxy/tools/__init__.py | 7 +++++++ lib/galaxy/tools/search/__init__.py | 18 +++++++++--------- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index e9490bb99d0..288258ba1eb 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1246,6 +1246,13 @@ class Tool(Dictifiable): self.__ensure_help() return self.__help_by_page + @property + def raw_help(self): + # may return rst (or Markdown in the future) + tool_source = self.__help_source + help_text = tool_source.parse_help() + return help_text + def __ensure_help(self): with HELP_UNINITIALIZED: if self.__help is HELP_UNINITIALIZED: diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index 7f68703bc06..a7d38c3076f 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -104,15 +104,15 @@ class ToolBoxSearch(object): add_doc_kwds['stub'] = to_unicode(id) if tool.labels: add_doc_kwds['labels'] = to_unicode(" ".join(tool.labels)) - if index_help and tool.help: - try: - raw_html = tool.help.render(host_url="", static_path="") - cleantext = clean(raw_html, tags=[''], strip=True).replace('\n', ' ') - add_doc_kwds['help'] = to_unicode(cleantext) - except Exception: - # Don't fail to build index just because a help message - # won't render. - pass + if index_help: + raw_help = tool.raw_help + if raw_help: + try: + add_doc_kwds['help'] = to_unicode(raw_help) + except Exception: + # Don't fail to build index just because a help message + # won't render. + pass return add_doc_kwds def search(self, q, tool_name_boost, tool_section_boost, tool_description_boost, tool_label_boost, tool_stub_boost, tool_help_boost, tool_search_limit, tool_enable_ngram_search, tool_ngram_minsize, tool_ngram_maxsize): From 42d1a1fb1f28bce9ec8500d789c078f3aa111ae3 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Tue, 6 Aug 2019 16:55:17 -0400 Subject: [PATCH 7/8] drop unused import --- lib/galaxy/tools/search/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/tools/search/__init__.py b/lib/galaxy/tools/search/__init__.py index a7d38c3076f..2c894bd56cb 100644 --- a/lib/galaxy/tools/search/__init__.py +++ b/lib/galaxy/tools/search/__init__.py @@ -6,7 +6,6 @@ import logging import re import tempfile -from bleach import clean from whoosh import analysis from whoosh.analysis import StandardAnalyzer from whoosh.fields import ( From f4388a3d1150bb9c6f9caf052cb0e3abd9822f92 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 6 Aug 2019 11:22:05 -0400 Subject: [PATCH 8/8] [19.05] Try to fix too many workflow step output associations. Certain entries on main have hundreds of thousands of such records because they're being created at each iteration of workflow scheduling. This leads to serious memory and runtime issues over time. Cleanup extra rows with: ```sql DELETE FROM workflow_invocation_step_output_dataset_association WHERE id NOT IN ( SELECT max(w.id) FROM workflow_invocation_step_output_dataset_association as w GROUP BY workflow_invocation_step_id, dataset_id, output_name ) ``` --- lib/galaxy/workflow/modules.py | 13 +------------ lib/galaxy/workflow/run.py | 4 ++-- 2 files changed, 3 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index e605d4c72d4..58b407b5d27 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -594,7 +594,7 @@ class InputModule(WorkflowModule): progress.set_outputs_for_input(invocation_step, step_outputs) def recover_mapping(self, invocation_step, progress): - progress.set_outputs_for_input(invocation_step) + progress.set_outputs_for_input(invocation_step, already_persisted=True) class InputDataModule(InputModule): @@ -1305,17 +1305,6 @@ class ToolModule(WorkflowModule): return complete - def recover_mapping(self, invocation_step, progress): - outputs = {} - - for output_dataset_assoc in invocation_step.output_datasets: - outputs[output_dataset_assoc.output_name] = output_dataset_assoc.dataset - - for output_dataset_collection_assoc in invocation_step.output_dataset_collections: - outputs[output_dataset_collection_assoc.output_name] = output_dataset_collection_assoc.dataset_collection - - progress.set_step_outputs(invocation_step, outputs) - def _effective_post_job_actions(self, step): effective_post_job_actions = step.post_job_actions[:] for key, value in self.runtime_post_job_actions.items(): diff --git a/lib/galaxy/workflow/run.py b/lib/galaxy/workflow/run.py index 3336139eab3..ae20dcb943f 100644 --- a/lib/galaxy/workflow/run.py +++ b/lib/galaxy/workflow/run.py @@ -405,7 +405,7 @@ class WorkflowProgress(object): else: return step_outputs[output_name] - def set_outputs_for_input(self, invocation_step, outputs=None): + def set_outputs_for_input(self, invocation_step, outputs=None, already_persisted=False): step = invocation_step.workflow_step if outputs is None: @@ -420,7 +420,7 @@ class WorkflowProgress(object): elif step_id in self.inputs_by_step_id: outputs['output'] = self.inputs_by_step_id[step_id] - self.set_step_outputs(invocation_step, outputs) + self.set_step_outputs(invocation_step, outputs, already_persisted=already_persisted) def set_step_outputs(self, invocation_step, outputs, already_persisted=False): step = invocation_step.workflow_step