diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index 73020b1aee7..3ee0028c051 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -143,7 +143,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.2 +pulsar-galaxy-lib==0.14.5 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 000d8452733..fc3a20a6b45 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -126,7 +126,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6" protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6" prov==1.5.1; python_version >= "3.6" and python_version < "4" psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0") -pulsar-galaxy-lib==0.14.2 +pulsar-galaxy-lib==0.14.5 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0" pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" pyasn1==0.4.8; python_version >= "3.5" and python_version < "4" diff --git a/lib/galaxy/jobs/runners/pulsar.py b/lib/galaxy/jobs/runners/pulsar.py index 522770b0dbf..4813d5bbb77 100644 --- a/lib/galaxy/jobs/runners/pulsar.py +++ b/lib/galaxy/jobs/runners/pulsar.py @@ -19,6 +19,7 @@ from pulsar.client import ( ClientInputs, ClientJobDescription, ClientOutputs, + EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN, finish_job as pulsar_finish_job, PathMapper, PulsarClientTransportError, @@ -754,12 +755,19 @@ class PulsarJobRunner(AsynchronousJobRunner): work_dir_outputs = self.get_work_dir_outputs(job_wrapper) output_files = self.get_output_files(job_wrapper) metadata_directory = os.path.join(job_wrapper.working_directory, "metadata") + metadata_strategy = job_wrapper.get_destination_configuration('metadata_strategy', None) + dynamic_outputs = None # use default + if metadata_strategy == "extended" and PulsarJobRunner.__remote_metadata(client): + # if Pulsar is doing remote metdata and the remote metadata is extended, + # we only need to recover the final model store. + dynamic_outputs = EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN client_outputs = ClientOutputs( working_directory=job_wrapper.tool_working_directory, metadata_directory=metadata_directory, work_dir_outputs=work_dir_outputs, output_files=output_files, version_file=job_wrapper.get_version_string_path(), + dynamic_outputs=dynamic_outputs, ) return client_outputs diff --git a/pyproject.toml b/pyproject.toml index b6aa16e2369..73c84e30b9f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,7 +56,7 @@ paramiko = "*" Parsley = "*" Paste = "*" psutil = "*" -pulsar-galaxy-lib = "==0.14.2" +pulsar-galaxy-lib = ">=0.14.5" pycryptodome = "*" pykwalify = "*" pyparsing = "*" diff --git a/test/integration/embedded_pulsar_metadata_extended_job_conf.yml b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml new file mode 100644 index 00000000000..76c7b8b5d22 --- /dev/null +++ b/test/integration/embedded_pulsar_metadata_extended_job_conf.yml @@ -0,0 +1,24 @@ +runners: + local: + load: galaxy.jobs.runners.local:LocalJobRunner + pulsar_embed: + load: galaxy.jobs.runners.pulsar:PulsarEmbeddedJobRunner + +execution: + default: pulsar_embed + environments: + local: + runner: local + pulsar_embed: + runner: pulsar_embed + remote_metadata: true + default_file_action: copy + # don't copy outputs, not needed. + file_actions: + paths: + - path_types: output + action: none + +tools: +- id: upload1 + environment: local diff --git a/test/integration/test_pulsar_embedded_extended_metadata.py b/test/integration/test_pulsar_embedded_extended_metadata.py index 7200f1aab7d..17cd3c02e5b 100644 --- a/test/integration/test_pulsar_embedded_extended_metadata.py +++ b/test/integration/test_pulsar_embedded_extended_metadata.py @@ -5,7 +5,7 @@ import os from galaxy_test.driver import integration_util SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__)) -EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_job_conf.yml") +EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_extended_job_conf.yml") class EmbeddedAndExtendedMetadataPulsarIntegrationInstance(integration_util.IntegrationInstance):