From d9ca2ed983bdd7b9368dbc1fea4a25e3ad9bec88 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 9 Apr 2021 13:42:06 -0400 Subject: [PATCH 1/2] Transfer fewer files when using extended + remote metadata. --- lib/galaxy/dependencies/dev-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- lib/galaxy/jobs/runners/pulsar.py | 8 ++++++++ pyproject.toml | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 73020b1aee7..48a48bf69be 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -143,7 +143,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.2 +pulsar-galaxy-lib==0.14.4 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 000d8452733..db476244a9b 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -126,7 +126,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.2 +pulsar-galaxy-lib==0.14.4 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 522770b0dbf..4813d5bbb77 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -19,6 +19,7 @@ from pulsar.client import ( ClientInputs, ClientJobDescription, ClientOutputs, + EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN, finish_job as pulsar_finish_job, PathMapper, PulsarClientTransportError, @@ -754,12 +755,19 @@ class PulsarJobRunner(AsynchronousJobRunner): work_dir_outputs = self.get_work_dir_outputs(job_wrapper) output_files = self.get_output_files(job_wrapper) metadata_directory = os.path.join(job_wrapper.working_directory, "metadata") + metadata_strategy = job_wrapper.get_destination_configuration('metadata_strategy', None) + dynamic_outputs = None # use default + if metadata_strategy == "extended" and PulsarJobRunner.__remote_metadata(client): + # if Pulsar is doing remote metdata and the remote metadata is extended, + # we only need to recover the final model store. + dynamic_outputs = EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN client_outputs = ClientOutputs( working_directory=job_wrapper.tool_working_directory, metadata_directory=metadata_directory, work_dir_outputs=work_dir_outputs, output_files=output_files, version_file=job_wrapper.get_version_string_path(), + dynamic_outputs=dynamic_outputs, ) return client_outputs diff --git a/pyproject.toml b/pyproject.toml index b6aa16e2369..d84715b43d0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ paramiko = "*" Parsley = "*" Paste = "*" psutil = "*" -pulsar-galaxy-lib = "==0.14.2" +pulsar-galaxy-lib = ">=0.14.4" pycryptodome = "*" pykwalify = "*" pyparsing = "*" From cc41f30af6bf76093d93861aae2b95c04b6a4c26 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 15 Apr 2021 15:01:28 -0400 Subject: [PATCH 2/2] Better extended metadata configuration for Pulsar (don't copy outputs) --- lib/galaxy/dependencies/dev-requirements.txt | 2 +- .../dependencies/pinned-requirements.txt | 2 +- pyproject.toml | 2 +- ...dded_pulsar_metadata_extended_job_conf.yml | 24 +++++++++++++++++++ .../test_pulsar_embedded_extended_metadata.py | 2 +- 5 files changed, 28 insertions(+), 4 deletions(-) create mode 100644 test/integration/embedded_pulsar_metadata_extended_job_conf.yml diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 48a48bf69be..3ee0028c051 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -143,7 +143,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.4 +pulsar-galaxy-lib==0.14.5 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index db476244a9b..fc3a20a6b45 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -126,7 +126,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.4 +pulsar-galaxy-lib==0.14.5 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" diff --git a/pyproject.toml b/pyproject.toml index d84715b43d0..73c84e30b9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ paramiko = "*" Parsley = "*" Paste = "*" psutil = "*" -pulsar-galaxy-lib = ">=0.14.4" +pulsar-galaxy-lib = ">=0.14.5" pycryptodome = "*" pykwalify = "*" pyparsing = "*" diff --git a/test/integration/embedded_pulsar_metadata_extended_job_conf.yml b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml new file mode 100644 index 00000000000..76c7b8b5d22 --- /dev/null +++ b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml @@ -0,0 +1,24 @@ +runners: + local: + load: galaxy.jobs.runners.local:LocalJobRunner + pulsar_embed: + load: galaxy.jobs.runners.pulsar:PulsarEmbeddedJobRunner + +execution: + default: pulsar_embed + environments: + local: + runner: local + pulsar_embed: + runner: pulsar_embed + remote_metadata: true + default_file_action: copy + # don't copy outputs, not needed. + file_actions: + paths: + - path_types: output + action: none + +tools: +- id: upload1 + environment: local diff --git a/test/integration/test_pulsar_embedded_extended_metadata.py b/test/integration/test_pulsar_embedded_extended_metadata.py index 7200f1aab7d..17cd3c02e5b 100644 --- a/test/integration/test_pulsar_embedded_extended_metadata.py +++ b/test/integration/test_pulsar_embedded_extended_metadata.py @@ -5,7 +5,7 @@ import os from galaxy_test.driver import integration_util SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) -EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_job_conf.yml") +EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_extended_job_conf.yml") class EmbeddedAndExtendedMetadataPulsarIntegrationInstance(integration_util.IntegrationInstance):