From 0c78aca542ab544e82897aa3e0249dfb8146c11e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 4 Jul 2020 19:16:35 +0200 Subject: [PATCH 1/4] Allow setting metadata_strategy per destination This allows instances that mix classic job runners with pulsar to use metadata_strategy: extended on destinations that are not served by pulsar. --- lib/galaxy/jobs/__init__.py | 10 +++++----- test/integration/embedded_pulsar_metadata_job_conf.yml | 1 + test/integration/test_pulsar_embedded_metadata.py | 4 ++++ 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 4c882990f58..ad4cf4fde23 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -906,17 +906,17 @@ class JobWrapper(HasResourceParameters): self.output_paths = None self.output_hdas_and_paths = None self.tool_provided_job_metadata = None - # Wrapper holding the info required to restore and clean up from files used for setting metadata externally - metadata_strategy_override = None - if job.tasks: - metadata_strategy_override = "directory" - self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override) self.job_runner_mapper = JobRunnerMapper(self, queue.dispatcher.url_to_destination, self.app.job_config) self.params = None if job.params: self.params = loads(job.params) if use_persisted_destination: self.job_runner_mapper.cached_job_destination = JobDestination(from_job=job) + # Wrapper holding the info required to restore and clean up from files used for setting metadata externally + metadata_strategy_override = self.get_destination_configuration('metadata_strategy', None) + if job.tasks: + metadata_strategy_override = "directory" + self.external_output_metadata = get_metadata_compute_strategy(self.app.config, job.id, metadata_strategy_override=metadata_strategy_override) self.__commands_in_new_shell = True self.__user_system_pwent = None diff --git a/test/integration/embedded_pulsar_metadata_job_conf.yml b/test/integration/embedded_pulsar_metadata_job_conf.yml index 76520bf4ed5..451162866a0 100644 --- a/test/integration/embedded_pulsar_metadata_job_conf.yml +++ b/test/integration/embedded_pulsar_metadata_job_conf.yml @@ -13,6 +13,7 @@ execution: runner: pulsar_embed remote_metadata: true default_file_action: copy + metadata_strategy: directory tools: - id: upload1 diff --git a/test/integration/test_pulsar_embedded_metadata.py b/test/integration/test_pulsar_embedded_metadata.py index 4ebae4a5037..14dc616281d 100644 --- a/test/integration/test_pulsar_embedded_metadata.py +++ b/test/integration/test_pulsar_embedded_metadata.py @@ -16,6 +16,10 @@ class EmbeddedMetadataPulsarIntegrationInstance(integration_util.IntegrationInst @classmethod def handle_galaxy_config_kwds(cls, config): config["job_config_file"] = EMBEDDED_PULSAR_JOB_CONFIG_FILE + config['object_store_store_by'] = 'uuid' + # metadata_strategy is being set to directory in embedded_pulsar_metadata_job_conf.yml, + # since extended_metadata doesn't work yet on pulsar + config['metadata_strategy'] = 'extended' instance = integration_util.integration_module_instance(EmbeddedMetadataPulsarIntegrationInstance) From ea3f48a5ec1220a1d1d0b614cbb66cbdeebaa79d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 4 Jul 2020 19:41:11 +0200 Subject: [PATCH 2/4] Document metadata_strategy --- doc/source/admin/galaxy_options.rst | 16 ++++++++++++++++ lib/galaxy/config/sample/galaxy.yml.sample | 9 +++++++++ .../config/sample/job_conf.xml.sample_advanced | 11 +++++++++++ lib/galaxy/webapps/galaxy/config_schema.yml | 12 ++++++++++++ 4 files changed, 48 insertions(+) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 40defa60e8a..1682b35b062 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -3563,6 +3563,22 @@ :Type: bool +~~~~~~~~~~~~~~~~~~~~~ +``metadata_strategy`` +~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Determines how metadata will be set. Valid values are `directory`, + `extended` and `legacy`. In extended mode jobs will decide if a + tool run failed, the object stores configuration is serialized + and made available to the job and is used for writing output + datasets to the object store as part of the job and dynamic + output discovery (e.g. discovered datasets , + unpopulated collections, etc) happens as part of the job. +:Default: ``directory`` +:Type: str + + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ ``retry_metadata_internally`` ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 1fe69acef2a..c28ab6fab5b 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -1741,6 +1741,15 @@ galaxy: # database. #enable_job_recovery: true + # Determines how metadata will be set. Valid values are `directory`, + # `extended` and `legacy`. In extended mode jobs will decide if a tool + # run failed, the object stores configuration is serialized and made + # available to the job and is used for writing output datasets to the + # object store as part of the job and dynamic output discovery (e.g. + # discovered datasets , unpopulated collections, + # etc) happens as part of the job. + #metadata_strategy: directory + # Although it is fairly reliable, setting metadata can occasionally # fail. In these instances, you can choose to retry setting it # internally or leave it in a failed state (since retrying internally diff --git a/lib/galaxy/config/sample/job_conf.xml.sample_advanced b/lib/galaxy/config/sample/job_conf.xml.sample_advanced index 53254618eba..f44a940a95e 100644 --- a/lib/galaxy/config/sample/job_conf.xml.sample_advanced +++ b/lib/galaxy/config/sample/job_conf.xml.sample_advanced @@ -401,6 +401,17 @@ try it early - it will slightly speed up local jobs by embedding metadata calculation in job script itself. --> + directory + directory -