Merge pull request #11852 from jmchilton/less_transfer

Test a better Pulsar configuration for extended metadata.

lib/galaxy/dependencies/dev-requirements.txt

@@ -143,7 +143,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.2
+pulsar-galaxy-lib==0.14.5
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or python_full_version >= "3.4.0" and python_version >= "3.6" and implementation_name == "pypy"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"

lib/galaxy/dependencies/pinned-requirements.txt

@@ -126,7 +126,7 @@ prompt-toolkit==3.0.3; python_version >= "3.6"
 protobuf==3.15.7; python_version >= "3.6" and python_full_version < "3.0.0" or python_full_version >= "3.6.0" and python_version >= "3.6"
 prov==1.5.1; python_version >= "3.6" and python_version < "4"
 psutil==5.8.0; (python_version >= "2.6" and python_full_version < "3.0.0") or (python_full_version >= "3.4.0")
-pulsar-galaxy-lib==0.14.2
+pulsar-galaxy-lib==0.14.5
 py==1.10.0; python_version >= "3.6" and python_full_version < "3.0.0" and implementation_name == "pypy" or implementation_name == "pypy" and python_version >= "3.6" and python_full_version >= "3.4.0"
 pyasn1-modules==0.2.8; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.6.0"
 pyasn1==0.4.8; python_version >= "3.5" and python_version < "4"

lib/galaxy/jobs/runners/pulsar.py

@@ -19,6 +19,7 @@ from pulsar.client import (
     ClientInputs,
     ClientJobDescription,
     ClientOutputs,
+    EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN,
     finish_job as pulsar_finish_job,
     PathMapper,
     PulsarClientTransportError,
@@ -754,12 +755,19 @@ class PulsarJobRunner(AsynchronousJobRunner):
         work_dir_outputs = self.get_work_dir_outputs(job_wrapper)
         output_files = self.get_output_files(job_wrapper)
         metadata_directory = os.path.join(job_wrapper.working_directory, "metadata")
+        metadata_strategy = job_wrapper.get_destination_configuration('metadata_strategy', None)
+        dynamic_outputs = None  # use default
+        if metadata_strategy == "extended" and PulsarJobRunner.__remote_metadata(client):
+            # if Pulsar is doing remote metdata and the remote metadata is extended,
+            # we only need to recover the final model store.
+            dynamic_outputs = EXTENDED_METADATA_DYNAMIC_COLLECTION_PATTERN
         client_outputs = ClientOutputs(
             working_directory=job_wrapper.tool_working_directory,
             metadata_directory=metadata_directory,
             work_dir_outputs=work_dir_outputs,
             output_files=output_files,
             version_file=job_wrapper.get_version_string_path(),
+            dynamic_outputs=dynamic_outputs,
         )
         return client_outputs
 

pyproject.toml

@@ -56,7 +56,7 @@ paramiko = "*"
 Parsley = "*"
 Paste = "*"
 psutil = "*"
-pulsar-galaxy-lib = "==0.14.2"
+pulsar-galaxy-lib = ">=0.14.5"
 pycryptodome = "*"
 pykwalify = "*"
 pyparsing = "*"

test/integration/embedded_pulsar_metadata_extended_job_conf.yml

@@ -0,0 +1,24 @@
+runners:
+  local:
+    load: galaxy.jobs.runners.local:LocalJobRunner
+  pulsar_embed:
+    load: galaxy.jobs.runners.pulsar:PulsarEmbeddedJobRunner
+
+execution:
+  default: pulsar_embed
+  environments:
+    local:
+      runner: local
+    pulsar_embed:
+      runner: pulsar_embed
+      remote_metadata: true
+      default_file_action: copy
+      # don't copy outputs, not needed.
+      file_actions:
+        paths:
+          - path_types: output
+            action: none
+
+tools:
+- id: upload1
+  environment: local

test/integration/test_pulsar_embedded_extended_metadata.py

@@ -5,7 +5,7 @@ import os
 from galaxy_test.driver import integration_util
 
 SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
-EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_job_conf.yml")
+EMBEDDED_PULSAR_JOB_CONFIG_FILE = os.path.join(SCRIPT_DIRECTORY, "embedded_pulsar_metadata_extended_job_conf.yml")
 
 
 class EmbeddedAndExtendedMetadataPulsarIntegrationInstance(integration_util.IntegrationInstance):