From 46fed3d3441ece2a60175d2af3903e52ef24c272 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 21 Jul 2021 15:21:26 -0400 Subject: [PATCH 1/3] Fix Pulsar's ability to read tool provided metadata when path is overridden. --- lib/galaxy/jobs/runners/pulsar.py | 11 ++++++++--- test/integration/test_pulsar_embedded.py | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index bf7c9d9a2a3..15a6aa3c977 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -6,6 +6,7 @@ More information on Pulsar can be found at https://pulsar.readthedocs.io/ . import errno import logging import os +import re import subprocess from time import sleep @@ -758,6 +759,10 @@ class PulsarJobRunner(AsynchronousJobRunner): output_files = self.get_output_files(job_wrapper) metadata_directory = os.path.join(job_wrapper.working_directory, "metadata") metadata_strategy = job_wrapper.get_destination_configuration('metadata_strategy', None) + tool = job_wrapper.tool + tool_provided_metadata_file_path = tool.provided_metadata_file + tool_provided_metadata_style = tool.provided_metadata_style + dynamic_outputs = None # use default if metadata_strategy == "extended" and PulsarJobRunner.__remote_metadata(client): # if Pulsar is doing remote metadata and the remote metadata is extended, @@ -766,10 +771,10 @@ class PulsarJobRunner(AsynchronousJobRunner): else: # otherwise collect everything we might need dynamic_outputs = DEFAULT_DYNAMIC_COLLECTION_PATTERN[:] + # grab discovered outputs... dynamic_outputs.extend(job_wrapper.tool.output_discover_patterns) - tool = job_wrapper.tool - tool_provided_metadata_file_path = tool.provided_metadata_file - tool_provided_metadata_style = tool.provided_metadata_style + # grab too provided metadata (galaxy.json) also... + dynamic_outputs.append(re.escape(tool_provided_metadata_file_path)) dynamic_file_sources = [ {"path": tool_provided_metadata_file_path, "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy"} ] diff --git a/test/integration/test_pulsar_embedded.py b/test/integration/test_pulsar_embedded.py index db9bd8baedf..75d1df29276 100644 --- a/test/integration/test_pulsar_embedded.py +++ b/test/integration/test_pulsar_embedded.py @@ -29,4 +29,6 @@ test_tools = integration_util.integration_tool_runner([ "vcf_bgzip_test", "environment_variables", "multi_output_assign_primary_ext_dbkey", + "strict_shell", + "tool_provided_metadata_9", ]) From 2ffa18d836b40bebab911e59ab7b33e00c22e0d2 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 10 Aug 2021 14:13:55 -0400 Subject: [PATCH 2/3] Update lib/galaxy/jobs/runners/pulsar.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Björn Grüning --- lib/galaxy/jobs/runners/pulsar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 15a6aa3c977..0dac01bdea7 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -773,7 +773,7 @@ class PulsarJobRunner(AsynchronousJobRunner): dynamic_outputs = DEFAULT_DYNAMIC_COLLECTION_PATTERN[:] # grab discovered outputs... dynamic_outputs.extend(job_wrapper.tool.output_discover_patterns) - # grab too provided metadata (galaxy.json) also... + # grab tool provided metadata (galaxy.json) also... dynamic_outputs.append(re.escape(tool_provided_metadata_file_path)) dynamic_file_sources = [ {"path": tool_provided_metadata_file_path, "type": "galaxy" if tool_provided_metadata_style == "default" else "legacy_galaxy"} From 718b70b17bfe30ddfc8078d6ab72456770cf14ec Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 6 Sep 2021 11:28:47 +0100 Subject: [PATCH 3/3] Downgrade Cheetah3 to 3.2.6.post1 Version 3.2.6.post2 was yanked: https://pypi.org/project/Cheetah3/3.2.6.post2/ Fix the following warning on startup: ``` 2021-09-03T22:30:41.3786134Z WARNING: The candidate selected for download or install is a yanked version: 'cheetah3' candidate (version 3.2.6.post2 at https://files.pythonhosted.org/packages/a4/d4/0325ef3dfd3edf545b5ff0aec15f248d8bf2e5cdda9e56a7b68187181703/Cheetah3-3.2.6.post2-cp36-cp36m-manylinux1_x86_64.whl#sha256=b37e99194c71eea8156c4cd66381257a3a4c45b13c5f7e9fdbcd2772ab0752e8 (from https://pypi.org/simple/cheetah3/) (requires-python:>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*)) 2021-09-03T22:30:41.3789180Z Reason for being yanked: Failed ``` --- lib/galaxy/dependencies/dev-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index d1346637796..df7658ae5ca 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -32,7 +32,7 @@ celery==5.0.5; python_version >= "3.6" certifi==2020.12.5; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" and python_version >= "3.6" cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" -cheetah3==3.2.6.post2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") +cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") circus==0.17.1 click-didyoumean==0.0.3; python_version >= "3.6" click-plugins==1.1.1; python_version >= "3.6" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index b3db1a5e313..a0a0d68a7ff 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -30,7 +30,7 @@ celery==5.0.5; python_version >= "3.6" certifi==2020.12.5; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" cffi==1.14.5; implementation_name == "pypy" and python_version >= "3.6" and (python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.4.0") chardet==4.0.0; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" -cheetah3==3.2.6.post2; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") +cheetah3==3.2.6.post1; (python_version >= "2.7" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") circus==0.17.1 click-didyoumean==0.0.3; python_version >= "3.6" click-plugins==1.1.1; python_version >= "3.6"