Implement export of FileParameter / MetadataFile files

lib/galaxy/model/__init__.py

@@ -2895,6 +2895,17 @@ class DatasetInstance:
                 meta_types.append(meta_type)
         return meta_types
 
+    def get_metadata_file_paths_and_extensions(self):
+        metadata = self.metadata
+        metadata_files = []
+        for metadata_name in self.metadata_file_types:
+            file_ext = metadata.spec[metadata_name].file_ext
+            metadata_file = metadata[metadata_name]
+            if metadata_file:
+                path = metadata_file.file_name
+                metadata_files.append((file_ext, path))
+        return metadata_files
+
     # This provide backwards compatibility with using the old dbkey
     # field in the database.  That field now maps to "old_dbkey" (see mapping.py).
 
@@ -4291,7 +4302,22 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById):
             hda_attributes=('extension',),
             return_entities=(Dataset,)
         )
-        return [(row[:-2], row[-2], row[-1].file_name) for row in q]
+        return [(row[:-2], row.extension, row.Dataset.file_name) for row in q]
+
+    @property
+    def element_identifiers_extensions_paths_and_metadata_files(self):
+        q = self._get_nested_collection_attributes(
+            element_attributes=('element_identifier',),
+            hda_attributes=('extension',),
+            return_entities=(HistoryDatasetAssociation, Dataset)
+        )
+        results = []
+        for row in q:
+            result = [row[:-3], row.extension, row.Dataset.file_name]
+            hda = row.HistoryDatasetAssociation
+            result.append(hda.get_metadata_file_paths_and_extensions())
+            results.append(result)
+        return results
 
     @property
     def waiting_for_elements(self):
@@ -5924,6 +5950,8 @@ class MetadataFile(StorableObject, RepresentById):
             self.history_dataset = dataset
         elif isinstance(dataset, LibraryDatasetDatasetAssociation):
             self.library_dataset = dataset
+        self.hda_id = None
+        self.lda_id = None
         self.name = name
 
     @property

lib/galaxy/tools/wrappers.py

@@ -322,6 +322,10 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper):
         safe_element_identifier = filesystem_safe_string(self.element_identifier, max_len=max_len)
         return f"{safe_element_identifier}.{self.file_ext}"
 
+    @property
+    def all_metadata_files(self):
+        return self.unsanitized.get_metadata_file_paths_and_extensions()
+
     @property
     def is_collection(self):
         return False
@@ -447,11 +451,12 @@ class DatasetListWrapper(list, ToolParameterValueWrapper, HasDatasets):
 
 class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
 
-    def __init__(self, job_working_directory, has_collection, **kwargs):
+    def __init__(self, job_working_directory, has_collection, datatypes_registry=None, **kwargs):
         super().__init__()
         self.job_working_directory = job_working_directory
         self._dataset_elements_cache = {}
-        self._element_identifiers_extensions_and_paths = None
+        self._element_identifiers_extensions_paths_and_metadata_files = None
+        self.datatypes_registry = datatypes_registry
         self.kwargs = kwargs
 
         if has_collection is None:
@@ -517,18 +522,22 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets):
 
     @property
     def all_paths(self):
-        return [path for _, _, path in self.all_element_identifiers_extensions_and_paths]
+        return [path for _, _, path, _ in self.element_identifiers_extensions_paths_and_metadata_files]
 
     @property
-    def all_element_identifiers_extensions_and_paths(self):
-        if self._element_identifiers_extensions_and_paths is None:
-            self._element_identifiers_extensions_and_paths = self.collection.element_identifiers_extensions_and_paths
-        return self._element_identifiers_extensions_and_paths
+    def all_metadata_files(self):
+        return [metadata_files for _, _, _, metadata_files in self.element_identifiers_extensions_paths_and_metadata_files]
+
+    @property
+    def element_identifiers_extensions_paths_and_metadata_files(self):
+        if self._element_identifiers_extensions_paths_and_metadata_files is None:
+            self._element_identifiers_extensions_paths_and_metadata_files = self.collection.element_identifiers_extensions_paths_and_metadata_files
+        return self._element_identifiers_extensions_paths_and_metadata_files
 
     @property
     def all_element_identifiers_and_extensions_filesystem_safe(self):
         safe_element_identifiers = []
-        for element_identifiers, extension, _ in self.all_element_identifiers_extensions_and_paths:
+        for element_identifiers, extension, *_ in self.element_identifiers_extensions_paths_and_metadata_files:
             datatype = self.datatypes_registry.get_datatype_by_extension(extension)
             if datatype:
                 extension = getattr(datatype, 'file_ext_export_alias', extension)

test/integration/test_remote_files.py

@@ -155,7 +155,7 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase):
             }
             response = dataset_populator.run_tool("export_remote", inputs, history_id)
             dataset_populator.wait_for_job(response["jobs"][0]["id"], assert_ok=True)
-            with open(os.path.join(ftp_dir, 'my_cool_utf8_name_😻.txt')) as f:
+            with open(os.path.join(ftp_dir, 'my_cool', 'utf8_name_😻.txt')) as f:
                 assert 'example content\n' == f.read()
 
     def test_export_remote_tool_default_duplicate_name_fails(self):
@@ -204,6 +204,7 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase):
             dataset_populator.wait_for_job(response["jobs"][0]["id"], assert_ok=True)
             with open(os.path.join(ftp_dir, 'foo_1.vcf.gz'), 'rb') as export, open(VCF_GZ_PATH, 'rb') as vcf_gz:
                 assert export.read() == vcf_gz.read()
+            assert os.path.exists(os.path.join(ftp_dir, 'foo_1.vcf.gz.tbi'))
 
     def test_export_remote_tool_collection_structure(self):
         dataset_populator = self.dataset_populator

test/unit/data/test_galaxy_mapping.py

@@ -2,6 +2,7 @@ import collections
 import os
 import unittest
 import uuid
+from tempfile import NamedTemporaryFile
 
 import pytest
 from sqlalchemy import inspect
@@ -10,6 +11,7 @@ import galaxy.datatypes.registry
 import galaxy.model
 import galaxy.model.mapping as mapping
 from galaxy.model.database_utils import create_database
+from galaxy.model.metadata import MetadataTempFile
 from galaxy.model.security import GalaxyRBACAgent
 
 datatypes_registry = galaxy.datatypes.registry.Registry()
@@ -324,11 +326,20 @@ class MappingTests(BaseModelTestCase):
         u = model.User(email="mary2@example.com", password="password")
         h1 = model.History(name="History 1", user=u)
         d1 = model.HistoryDatasetAssociation(extension="bam", history=h1, create_dataset=True, sa_session=model.session)
+        index = NamedTemporaryFile("w")
+        index.write("cool bam index")
+        index2 = NamedTemporaryFile("w")
+        index2.write("cool bam index 2")
+        metadata_dict = {"bam_index": MetadataTempFile.from_JSON({"kwds": {}, "filename": index.name}), "bam_csi_index": MetadataTempFile.from_JSON({"kwds": {}, "filename": index2.name})}
+        d1.metadata.from_JSON_dict(json_dict=metadata_dict)
+        assert d1.metadata.bam_index
+        assert d1.metadata.bam_csi_index
+        assert isinstance(d1.metadata.bam_index, model.MetadataFile)
+        assert isinstance(d1.metadata.bam_csi_index, model.MetadataFile)
         d2 = model.HistoryDatasetAssociation(extension="txt", history=h1, create_dataset=True, sa_session=model.session)
         c1 = model.DatasetCollection(collection_type='paired')
         dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0)
         dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1)
-
         c2 = model.DatasetCollection(collection_type="list:paired")
         dce3 = model.DatasetCollectionElement(collection=c2, element=c1, element_identifier="inner_list", element_index=0)
         c3 = model.DatasetCollection(collection_type="list:list")
@@ -349,6 +360,7 @@ class MappingTests(BaseModelTestCase):
         assert c2.dataset_action_tuples == []
         assert c2.populated_optimized
         assert c2.dataset_states_and_extensions_summary == ({'new'}, {'txt', 'bam'})
+        assert c2.element_identifiers_extensions_paths_and_metadata_files == [[('inner_list', 'forward'), 'bam', 'mock_dataset_14.dat', [('bai', 'mock_dataset_14.dat'), ('bam.csi', 'mock_dataset_14.dat')]], [('inner_list', 'reverse'), 'txt', 'mock_dataset_14.dat', []]]
         assert c3.dataset_instances == []
         assert c3.dataset_elements == []
         assert c3.dataset_states_and_extensions_summary == (set(), set())
@@ -889,6 +901,9 @@ class MockObjectStore:
     def get_store_by(self, *args, **kwds):
         return 'id'
 
+    def update_from_file(self, *arg, **kwds):
+        pass
+
 
 def get_suite():
     suite = unittest.TestSuite()

tools/data_export/export_remote.py

@@ -26,6 +26,15 @@ def check_for_duplicate_name(files_to_export):
         sys.exit(f"Duplicate export filenames given: {', '.join(duplicates)}, failing export")
 
 
+def write_if_not_exists(file_sources, target_uri, real_data_path):
+    file_source_path = file_sources.get_file_source_path(target_uri)
+    if os.path.exists(file_source_path.path):
+        print(f'Error: File "{file_source_path.path}" already exists. Skipping.')
+        return 1
+    file_source = file_source_path.file_source
+    file_source.write_from(file_source_path.path, real_data_path)
+
+
 def main(argv=None):
     if argv is None:
         argv = sys.argv[1:]
@@ -33,6 +42,7 @@ def main(argv=None):
     exit_code = 0
     file_sources = get_file_sources(args.file_sources)
     directory_uri = args.directory_uri
+    export_metadata_files = args.export_metadata_files
     with open(args.files_to_export) as f:
         files_to_export = json.load(f)
     counter = 0
@@ -44,13 +54,14 @@ def main(argv=None):
             target_uri = directory_uri + name
         else:
             target_uri = directory_uri + "/" + name
-        file_source_path = file_sources.get_file_source_path(target_uri)
-        if os.path.exists(file_source_path.path):
-            print(f'Error: File "{file_source_path.path}" already exists. Skipping.')
+        if write_if_not_exists(file_sources, target_uri, real_data_path):
             exit_code = 1
-            continue
-        file_source = file_source_path.file_source
-        file_source.write_from(file_source_path.path, real_data_path)
+        if export_metadata_files:
+            metadata_files = entry.get('metadata_files', [])
+            for extension, path in metadata_files:
+                metadata_file_uri = f"{target_uri}.{extension}"
+                if write_if_not_exists(file_sources, metadata_file_uri, path):
+                    exit_code = 1
         counter += 1
     print(f"{counter} out of {len(files_to_export)} files have been exported.\n")
     sys.exit(exit_code)
@@ -61,6 +72,7 @@ def _parser():
     parser.add_argument("--directory-uri", type=str, help="directory target URI")
     parser.add_argument("--file-sources", type=str, help="file sources json")
     parser.add_argument("--files-to-export", type=str, help="files to export")
+    parser.add_argument("--export-metadata-files", type=bool, help="export metadata files", default=True)
     return parser
 
 

tools/data_export/export_remote.xml

@@ -10,6 +10,7 @@ python '$__tool_directory__/export_remote.py'
     --file-sources '$file_sources'
     --directory-uri '$d_uri'
     --files-to-export '$files_to_export'
+    --export-metadata-files $include_metadata_files
     > '$out'
     ]]></command>
     <configfiles>
@@ -17,11 +18,11 @@ python '$__tool_directory__/export_remote.py'
         <configfile name="files_to_export">#import json
 #from galaxy.util import filesystem_safe_string
 #if $export_type.export_type_selector == "datasets_auto":
-#set $fileconfig = json.dumps([{"real_data_path": str(infile), "name": infile.name_and_ext_filesystem_safe} for infile in $infiles])
+#set $fileconfig = json.dumps([{"real_data_path": str(infile), "name": infile.name_and_ext_filesystem_safe, "metadata_files": infile.all_metadata_files} for infile in $infiles])
 #else if $export_type.export_type_selector == "collection_auto":
-#set $fileconfig = json.dumps([{"real_data_path": data_path, "name": identifier_and_extension} for data_path, identifier_and_extension in zip($export_type.incollection.all_paths, $export_type.incollection.all_element_identifiers_and_extensions_filesystem_safe)])
+#set $fileconfig = json.dumps([{"real_data_path": data_path, "name": identifier_and_extension, "metadata_files": metadata_files} for data_path, identifier_and_extension, metadata_files in zip($export_type.incollection.all_paths, $export_type.incollection.all_element_identifiers_and_extensions_filesystem_safe, $export_type.incollection.all_metadata_files)])
 #else
-#set $fileconfig = json.dumps([{"real_data_path": str(dataset["infile"]), "name": filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=())} for dataset in $export_type.datasets])
+#set $fileconfig = json.dumps([{"real_data_path": str(dataset["infile"]), "name": filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=()), "metadata_files": dataset['infile'].all_metadata_files} for dataset in $export_type.datasets])
 #end if
 $fileconfig
         </configfile>
@@ -52,6 +53,7 @@ $fileconfig
             </when>
         </conditional>
         <param type="directory_uri" name="d_uri" label="Directory URI" />
+        <param name="include_metadata_files" type="boolean" checked="true" label="Include metadata files in export?" help="Examples of metadata files are bam, cram and vcf indexes that can be regenerated from raw data" />
     </inputs>
     <outputs>
         <data name="out" format="txt" label="Export logs"/>