Merge pull request #9956 from mvdbeek/pulsar_uuid_extended_test

[20.05] Allow setting metadata_strategy per destination

doc/source/admin/galaxy_options.rst

@@ -3563,6 +3563,22 @@
 :Type: bool
 
 
+~~~~~~~~~~~~~~~~~~~~~
+``metadata_strategy``
+~~~~~~~~~~~~~~~~~~~~~
+
+:Description:
+    Determines how metadata will be set. Valid values are `directory`,
+    `extended` and `legacy`. In extended mode jobs will decide if a
+    tool run failed, the object stores configuration is serialized and
+    made available to the job and is used for writing output datasets
+    to the object store as part of the job and dynamic output
+    discovery (e.g. discovered datasets <discover_datasets>,
+    unpopulated collections, etc) happens as part of the job.
+:Default: ``directory``
+:Type: str
+
+
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 ``retry_metadata_internally``
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

lib/galaxy/config/sample/galaxy.yml.sample

@@ -1741,6 +1741,15 @@ galaxy:
   # database.
   #enable_job_recovery: true
 
+  # Determines how metadata will be set. Valid values are `directory`,
+  # `extended` and `legacy`. In extended mode jobs will decide if a tool
+  # run failed, the object stores configuration is serialized and made
+  # available to the job and is used for writing output datasets to the
+  # object store as part of the job and dynamic output discovery (e.g.
+  # discovered datasets <discover_datasets>, unpopulated collections,
+  # etc) happens as part of the job.
+  #metadata_strategy: directory
+
   # Although it is fairly reliable, setting metadata can occasionally
   # fail.  In these instances, you can choose to retry setting it
   # internally or leave it in a failed state (since retrying internally

lib/galaxy/config/sample/job_conf.xml.sample_advanced

@@ -401,6 +401,17 @@
                try it early - it will slightly speed up local jobs by
                embedding metadata calculation in job script itself.
           -->
+          <param id="metadata_strategy">directory</param>
+          <!-- If set, overwrites metadata_strategy (configurable in galaxy.yml).
+               Valid values are `directory` (default), `extended` and `legacy`.
+               Setting metadata_strategy to `extended` requires that all object stores
+               that can be written to store datasets by uuid. In `extended` mode jobs
+               will decide if a tool run failed, the object stores configuration
+               is serialized and made available to the job and is used for writing output
+               datasets to the object store as part of the job and dynamic output discovery
+               (e.g. discovered datasets <discover_datasets>, unpopulated collections, etc)
+               happens as part of the job.
+          -->
           <job_metrics />
           <!-- Above element demonstrates embedded job metrics definition - see
                job_metrics_conf.xml.sample for full documentation on possible nested

lib/galaxy/jobs/__init__.py

@@ -906,17 +906,20 @@ class JobWrapper(HasResourceParameters):
         self.output_paths = None
         self.output_hdas_and_paths = None
         self.tool_provided_job_metadata = None
-        # Wrapper holding the info required to restore and clean up from files used for setting metadata externally
-        metadata_strategy_override = None
-        if job.tasks:
-            metadata_strategy_override = "directory"
-        self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override)
         self.job_runner_mapper = JobRunnerMapper(self, queue.dispatcher.url_to_destination, self.app.job_config)
         self.params = None
         if job.params:
             self.params = loads(job.params)
         if use_persisted_destination:
             self.job_runner_mapper.cached_job_destination = JobDestination(from_job=job)
+        # Wrapper holding the info required to restore and clean up from files used for setting metadata externally
+        try:
+            metadata_strategy_override = self.get_destination_configuration('metadata_strategy', None)
+        except JobMappingException:
+            metadata_strategy_override = None
+        if job.tasks:
+            metadata_strategy_override = "directory"
+        self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override)
 
         self.__commands_in_new_shell = True
         self.__user_system_pwent = None

lib/galaxy/webapps/galaxy/config_schema.yml

@@ -2638,6 +2638,18 @@ mapping:
           it can "recover" them when it starts).  This is not safe to use if you are
           running more than one Galaxy server using the same database.
 
+      metadata_strategy:
+        type: str
+        required: false
+        default: directory
+        desc: |
+          Determines how metadata will be set. Valid values are `directory`, `extended` and `legacy`.
+          In extended mode jobs will decide if a tool run failed, the object stores
+          configuration is serialized and made available to the job and is used for
+          writing output datasets to the object store as part of the job and dynamic
+          output discovery (e.g. discovered datasets <discover_datasets>, unpopulated collections,
+          etc) happens as part of the job.
+
       retry_metadata_internally:
         type: bool
         default: true

test/integration/embedded_pulsar_metadata_job_conf.yml

@@ -13,6 +13,7 @@ execution:
       runner: pulsar_embed
       remote_metadata: true
       default_file_action: copy
+      metadata_strategy: directory
 
 tools:
 - id: upload1

test/integration/test_pulsar_embedded_metadata.py

@@ -16,6 +16,10 @@ class EmbeddedMetadataPulsarIntegrationInstance(integration_util.IntegrationInst
     @classmethod
     def handle_galaxy_config_kwds(cls, config):
         config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE
+        config['object_store_store_by'] = 'uuid'
+        # We set the global metadata_strategy to `extended, but`metadata_strategy is
+        # being overridden in embedded_pulsar_metadata_job_conf.yml, since extended_metadata does not yet work on pulsar
+        config['metadata_strategy'] = 'extended'
 
 
 instance = integration_util.integration_module_instance(EmbeddedMetadataPulsarIntegrationInstance)