Merge pull request #7186 from jmchilton/import_metadata_files

Fix regeneration of metadata files for types besides bam during import.

lib/galaxy/managers/datasets.py

@@ -470,14 +470,13 @@ class _UnflattenedMetadataDatasetAssociationSerializer(base.ModelSerializer,
         Cycle through meta files and return them as a list of dictionaries.
         """
         meta_files = []
-        for meta_type in dataset_assoc.metadata.spec.keys():
-            if isinstance(dataset_assoc.metadata.spec[meta_type].param, galaxy.datatypes.metadata.FileParameter):
-                meta_files.append(
-                    dict(file_type=meta_type,
-                         download_url=self.url_for('history_contents_metadata_file',
-                                                   history_id=self.app.security.encode_id(dataset_assoc.history_id),
-                                                   history_content_id=self.app.security.encode_id(dataset_assoc.id),
-                                                   metadata_file=meta_type)))
+        for meta_type in dataset_assoc.metadata_file_types:
+            meta_files.append(
+                dict(file_type=meta_type,
+                     download_url=self.url_for('history_contents_metadata_file',
+                                               history_id=self.app.security.encode_id(dataset_assoc.history_id),
+                                               history_content_id=self.app.security.encode_id(dataset_assoc.id),
+                                               metadata_file=meta_type)))
         return meta_files
 
     def serialize_metadata(self, dataset_assoc, key, excluded=None, **context):

lib/galaxy/model/__init__.py

@@ -2236,6 +2236,15 @@ class DatasetInstance(object):
         # Needs to accept a MetadataCollection, a bunch, or a dict
         self._metadata = self.metadata.make_dict_copy(bunch)
     metadata = property(get_metadata, set_metadata)
+
+    @property
+    def metadata_file_types(self):
+        meta_types = []
+        for meta_type in self.metadata.spec.keys():
+            if isinstance(self.metadata.spec[meta_type].param, galaxy.model.metadata.FileParameter):
+                meta_types.append(meta_type)
+        return meta_types
+
     # This provide backwards compatibility with using the old dbkey
     # field in the database.  That field now maps to "old_dbkey" (see mapping.py).
 

lib/galaxy/tools/__init__.py

@@ -2250,6 +2250,13 @@ class SetMetadataTool(Tool):
     tool_type = 'set_metadata'
     requires_setting_metadata = False
 
+    def regenerate_imported_metadata_if_needed(self, hda, history, job):
+        if len(hda.metadata_file_types) > 0:
+            self.tool_action.execute_via_app(
+                self, self.app, job.session_id,
+                history.id, job.user, incoming={'input1': hda}, overwrite=False
+            )
+
     def exec_after_process(self, app, inp_data, out_data, param_dict, job=None):
         for name, dataset in inp_data.items():
             external_metadata = JobExternalOutputMetadataWrapper(job)

lib/galaxy/tools/imp_exp/__init__.py

@@ -182,12 +182,9 @@ class JobImportHistoryArchiveWrapper(UsesAnnotations):
                             self.sa_session.flush()
                         """
 
-                    # Although metadata is set above, need to set metadata to recover BAI for BAMs.
-                    if hda.extension == 'bam':
-                        self.app.datatypes_registry.set_external_metadata_tool.tool_action.execute_via_app(
-                            self.app.datatypes_registry.set_external_metadata_tool, self.app, jiha.job.session_id,
-                            new_history.id, jiha.job.user, incoming={'input1': hda}, overwrite=False
-                        )
+                    self.app.datatypes_registry.set_external_metadata_tool.regenerate_imported_metadata_if_needed(
+                        hda, new_history, jiha.job
+                    )
 
                 #
                 # Create jobs.

test/api/test_histories.py

@@ -182,17 +182,30 @@ class HistoriesApiTestCase(api.ApiTestCase):
         history_id = self.dataset_populator.new_history(name=history_name)
         self.dataset_populator.new_dataset(history_id, content="1 2 3")
         imported_history_id = self._reimport_history(history_id, history_name)
-
-        contents_response = self._get("histories/%s/contents" % imported_history_id)
-        self._assert_status_code_is(contents_response, 200)
-        contents = contents_response.json()
-        assert len(contents) == 1
+        self._assert_history_length(imported_history_id, 1)
         imported_content = self.dataset_populator.get_history_dataset_content(
             history_id=imported_history_id,
-            dataset_id=contents[0]["id"]
+            hid=1,
         )
         assert imported_content == "1 2 3\n"
 
+    def test_import_metadata_regeneration(self):
+        history_name = "for_import_metadata_regeneration"
+        history_id = self.dataset_populator.new_history(name=history_name)
+        self.dataset_populator.new_dataset(history_id, content=open(self.test_data_resolver.get_filename("1.bam")), file_type='bam')
+        imported_history_id = self._reimport_history(history_id, history_name)
+        self._assert_history_length(imported_history_id, 1)
+        import_bam_metadata = self.dataset_populator.get_history_dataset_details(
+            history_id=imported_history_id,
+            hid=1,
+        )
+        bai_metadata = import_bam_metadata["meta_files"][0]
+        assert bai_metadata["file_type"] == "bam_index"
+        api_url = bai_metadata["download_url"].split("api/", 1)[1]
+        bai_response = self._get(api_url)
+        assert bai_response.status_code == 200
+        assert len(bai_response.content) > 4
+
     def test_import_export_collection(self):
         from nose.plugins.skip import SkipTest
         raise SkipTest("Collection import/export not yet implemented")
@@ -203,10 +216,7 @@ class HistoriesApiTestCase(api.ApiTestCase):
 
         imported_history_id = self._reimport_history(history_id, history_name)
 
-        contents_response = self._get("histories/%s/contents" % imported_history_id)
-        self._assert_status_code_is(contents_response, 200)
-        contents = contents_response.json()
-        assert len(contents) == 3
+        self._assert_history_length(imported_history_id, 3)
 
     def _reimport_history(self, history_id, history_name):
         # Ensure the history is ready to go...
@@ -242,6 +252,12 @@ class HistoriesApiTestCase(api.ApiTestCase):
 
         return imported_history_id
 
+    def _assert_history_length(self, history_id, n):
+        contents_response = self._get("histories/%s/contents" % history_id)
+        self._assert_status_code_is(contents_response, 200)
+        contents = contents_response.json()
+        assert len(contents) == n, contents
+
     def test_create_tag(self):
         post_data = dict(name="TestHistoryForTag")
         history_id = self._post("histories", data=post_data).json()["id"]