Merge pull request #9974 from mvdbeek/extended_metadata_fixes

[20.05] Fix MetadataFile handing with metadata_strategy: extended

lib/galaxy/metadata/set_metadata.py

@@ -174,7 +174,7 @@ def set_metadata_portable():
 
         # Load outputs.
         import_model_store = store.imported_store_for_metadata('metadata/outputs_new', object_store=object_store)
-        export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True)
+        export_store = store.DirectoryModelExportStore('metadata/outputs_populated', serialize_dataset_objects=True, for_edit=True, strip_metadata_files=False)
 
     for output_name, output_dict in outputs.items():
         if extended_metadata_collection:

lib/galaxy/model/metadata.py

@@ -524,8 +524,7 @@ class FileParameter(MetadataParameter):
             return None
         if isinstance(value, galaxy.model.MetadataFile) or isinstance(value, MetadataTempFile):
             return value
-        mf = session.query(galaxy.model.MetadataFile).get(value)
-        return mf
+        return session.query(galaxy.model.MetadataFile).get(value)
 
     def make_copy(self, value, target_context, source_context):
         value = self.wrap(value, object_session(target_context.parent))
@@ -540,7 +539,10 @@ class FileParameter(MetadataParameter):
     @classmethod
     def marshal(cls, value):
         if isinstance(value, galaxy.model.MetadataFile):
-            value = value.id
+            # We want to push value.id to the database, but need to skip this when no session is available,
+            # as in extended_metadata mode, so there we just accept MetadataFile.
+            # We will only serialize MetadataFile in this mode and not push to the database, so this is OK.
+            value = value.id or value
         return value
 
     def from_external_value(self, value, parent, path_rewriter=None):

test/functional/tools/metadata_bam.xml

@@ -1,5 +1,8 @@
-<tool id="metadata_bam" name="metadata BAM" version="1.0.0">
-  <command>echo "${ref_names}" &gt; "${output_of_input_metadata}"</command>
+<tool id="metadata_bam" name="metadata BAM" version="1.0.0" profile="16.04">
+  <command><![CDATA[
+#assert $input_bam.metadata.bam_index
+echo '${ref_names}' > '${output_of_input_metadata}'
+  ]]></command>
   <inputs>
     <param name="input_bam" type="data" format="bam" label="BAM File"/>
     <param name="ref_names" type="select" optional="False" label="Select references you would like to restrict bam to" multiple="True">

test/integration/test_pulsar_embedded_metadata.py

@@ -20,8 +20,9 @@ class EmbeddedMetadataPulsarIntegrationInstance(integration_util.IntegrationInst
         # We set the global metadata_strategy to `extended, but`metadata_strategy is
         # being overridden in embedded_pulsar_metadata_job_conf.yml, since extended_metadata does not yet work on pulsar
         config['metadata_strategy'] = 'extended'
+        config['retry_metadata_internally'] = False
 
 
 instance = integration_util.integration_module_instance(EmbeddedMetadataPulsarIntegrationInstance)
 
-test_tools = integration_util.integration_tool_runner(["simple_constructs"])
+test_tools = integration_util.integration_tool_runner(["simple_constructs", "metadata_bam"])