From f50a02078ab48c55471577c12064aa1bfa9ce97f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 8 Jul 2020 16:41:32 +0200 Subject: [PATCH 1/3] Import MetadataFile in extended_metadata mode We used to skip MetadaFile, despite it being generated and pushed to the object store. And we need to allow MetadataFile as a marshalled value when there is no associated ID yet. Should fix ``` Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: Traceback (most recent call last): Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: File "/opt/galaxy/server/lib/galaxy/jobs/runners/pulsar.py", line 390, in queue_job Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: job_id = pulsar_submit_job(client, client_job_description, remote_job_config) Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: File "/opt/galaxy/venv/lib/python3.6/site-packages/pulsar/client/staging/up.py", line 27, in submit_job Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: file_stager = FileStager(client, client_job_description, job_config) Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: File "/opt/galaxy/venv/lib/python3.6/site-packages/pulsar/client/staging/up.py", line 123, in __init__ Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: self.__upload_input_files() Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: File "/opt/galaxy/venv/lib/python3.6/site-packages/pulsar/client/staging/up.py", line 236, in __upload_input_files Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: self.__upload_input_metadata_file(client_input.action_source) Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: File "/opt/galaxy/venv/lib/python3.6/site-packages/pulsar/client/staging/up.py", line 255, in __upload_input_metadata_file Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: self.transfer_tracker.handle_transfer_source(input_action_source, path_type.INPUT, name=remote_name) Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: File "/opt/galaxy/venv/lib/python3.6/site-packages/pulsar/client/staging/up.py", line 462, in handle_transfer_source Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: action = self.__action_for_transfer(source, type, contents) Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: File "/opt/galaxy/venv/lib/python3.6/site-packages/pulsar/client/staging/up.py", line 518, in __action_for_transfer Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: raise Exception(message) Jul 08 11:25:23 test.internal.usegalaxy.eu.novalocal python[873067]: Exception: __action_for_transfer called on non-existent file - [None] ``` reported in https://github.com/galaxyproject/galaxy/issues/9968 --- lib/galaxy/metadata/set_metadata.py | 2 +- lib/galaxy/model/metadata.py | 8 +++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/metadata/set_metadata.py b/lib/galaxy/metadata/set_metadata.py index 5724c5543d2..c8077132cc4 100644 --- a/lib/galaxy/metadata/set_metadata.py +++ b/lib/galaxy/metadata/set_metadata.py @@ -174,7 +174,7 @@ def set_metadata_portable(): # Load outputs. import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store) - export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True) + export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False) for output_name, output_dict in outputs.items(): if extended_metadata_collection: diff --git a/lib/galaxy/model/metadata.py b/lib/galaxy/model/metadata.py index c257e2960e4..0bbc87eeb4c 100644 --- a/lib/galaxy/model/metadata.py +++ b/lib/galaxy/model/metadata.py @@ -524,8 +524,7 @@ class FileParameter(MetadataParameter): return None if isinstance(value, galaxy.model.MetadataFile) or isinstance(value, MetadataTempFile): return value - mf = session.query(galaxy.model.MetadataFile).get(value) - return mf + return session.query(galaxy.model.MetadataFile).get(value) def make_copy(self, value, target_context, source_context): value = self.wrap(value, object_session(target_context.parent)) @@ -540,7 +539,10 @@ class FileParameter(MetadataParameter): @classmethod def marshal(cls, value): if isinstance(value, galaxy.model.MetadataFile): - value = value.id + # We want to push value.id to the database, but need to skip this when no session is available, + # as in extended_metadata mode, so there we just accept MetadataFile. + # We will only serialize MetadataFile in this mode and not push to the database, so this is OK. + value = value.id or value return value def from_external_value(self, value, parent, path_rewriter=None): From a6aa1c932e7780228405c6d3e87d104e1c08dd25 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 9 Jul 2020 12:09:11 +0200 Subject: [PATCH 2/3] Extend pulsar metadata testing to cover MetadataFile inputs --- test/functional/tools/metadata_bam.xml | 5 +++-- test/integration/test_pulsar_embedded_metadata.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/test/functional/tools/metadata_bam.xml b/test/functional/tools/metadata_bam.xml index 2f20be5b531..6ee6bfa417c 100644 --- a/test/functional/tools/metadata_bam.xml +++ b/test/functional/tools/metadata_bam.xml @@ -1,5 +1,6 @@ - - echo "${ref_names}" > "${output_of_input_metadata}" + + "${output_of_input_metadata}"]]> diff --git a/test/integration/test_pulsar_embedded_metadata.py b/test/integration/test_pulsar_embedded_metadata.py index cd59298a45d..4ba5a600c7d 100644 --- a/test/integration/test_pulsar_embedded_metadata.py +++ b/test/integration/test_pulsar_embedded_metadata.py @@ -20,8 +20,9 @@ class EmbeddedMetadataPulsarIntegrationInstance(integration_util.IntegrationInst # We set the global metadata_strategy to `extended, but`metadata_strategy is # being overridden in embedded_pulsar_metadata_job_conf.yml, since extended_metadata does not yet work on pulsar config['metadata_strategy'] = 'extended' + config['retry_metadata_internally'] = False instance = integration_util.integration_module_instance(EmbeddedMetadataPulsarIntegrationInstance) -test_tools = integration_util.integration_tool_runner(["simple_constructs"]) +test_tools = integration_util.integration_tool_runner(["simple_constructs", "metadata_bam"]) From 6648890328521dbc8896c9000f8c0616f42587b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bj=C3=B6rn=20Gr=C3=BCning?= Date: Thu, 9 Jul 2020 15:21:43 +0200 Subject: [PATCH 3/3] Update test/functional/tools/metadata_bam.xml Co-authored-by: Nicola Soranzo --- test/functional/tools/metadata_bam.xml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/functional/tools/metadata_bam.xml b/test/functional/tools/metadata_bam.xml index 6ee6bfa417c..6cf73b2e9c3 100644 --- a/test/functional/tools/metadata_bam.xml +++ b/test/functional/tools/metadata_bam.xml @@ -1,6 +1,8 @@ - "${output_of_input_metadata}"]]> + '${output_of_input_metadata}' + ]]>