diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 53854926d17..a60dbce0b5a 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2895,6 +2895,17 @@ class DatasetInstance: meta_types.append(meta_type) return meta_types + def get_metadata_file_paths_and_extensions(self): + metadata = self.metadata + metadata_files = [] + for metadata_name in self.metadata_file_types: + file_ext = metadata.spec[metadata_name].file_ext + metadata_file = metadata[metadata_name] + if metadata_file: + path = metadata_file.file_name + metadata_files.append((file_ext, path)) + return metadata_files + # This provide backwards compatibility with using the old dbkey # field in the database. That field now maps to "old_dbkey" (see mapping.py). @@ -4291,7 +4302,22 @@ class DatasetCollection(Dictifiable, UsesAnnotations, RepresentById): hda_attributes=('extension',), return_entities=(Dataset,) ) - return [(row[:-2], row[-2], row[-1].file_name) for row in q] + return [(row[:-2], row.extension, row.Dataset.file_name) for row in q] + + @property + def element_identifiers_extensions_paths_and_metadata_files(self): + q = self._get_nested_collection_attributes( + element_attributes=('element_identifier',), + hda_attributes=('extension',), + return_entities=(HistoryDatasetAssociation, Dataset) + ) + results = [] + for row in q: + result = [row[:-3], row.extension, row.Dataset.file_name] + hda = row.HistoryDatasetAssociation + result.append(hda.get_metadata_file_paths_and_extensions()) + results.append(result) + return results @property def waiting_for_elements(self): @@ -5924,6 +5950,8 @@ class MetadataFile(StorableObject, RepresentById): self.history_dataset = dataset elif isinstance(dataset, LibraryDatasetDatasetAssociation): self.library_dataset = dataset + self.hda_id = None + self.lda_id = None self.name = name @property diff --git a/lib/galaxy/tools/wrappers.py b/lib/galaxy/tools/wrappers.py index ba6a1c15f76..296fa801bfd 100644 --- a/lib/galaxy/tools/wrappers.py +++ b/lib/galaxy/tools/wrappers.py @@ -322,6 +322,10 @@ class DatasetFilenameWrapper(ToolParameterValueWrapper): safe_element_identifier = filesystem_safe_string(self.element_identifier, max_len=max_len) return f"{safe_element_identifier}.{self.file_ext}" + @property + def all_metadata_files(self): + return self.unsanitized.get_metadata_file_paths_and_extensions() + @property def is_collection(self): return False @@ -447,11 +451,12 @@ class DatasetListWrapper(list, ToolParameterValueWrapper, HasDatasets): class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets): - def __init__(self, job_working_directory, has_collection, **kwargs): + def __init__(self, job_working_directory, has_collection, datatypes_registry=None, **kwargs): super().__init__() self.job_working_directory = job_working_directory self._dataset_elements_cache = {} - self._element_identifiers_extensions_and_paths = None + self._element_identifiers_extensions_paths_and_metadata_files = None + self.datatypes_registry = datatypes_registry self.kwargs = kwargs if has_collection is None: @@ -517,18 +522,22 @@ class DatasetCollectionWrapper(ToolParameterValueWrapper, HasDatasets): @property def all_paths(self): - return [path for _, _, path in self.all_element_identifiers_extensions_and_paths] + return [path for _, _, path, _ in self.element_identifiers_extensions_paths_and_metadata_files] @property - def all_element_identifiers_extensions_and_paths(self): - if self._element_identifiers_extensions_and_paths is None: - self._element_identifiers_extensions_and_paths = self.collection.element_identifiers_extensions_and_paths - return self._element_identifiers_extensions_and_paths + def all_metadata_files(self): + return [metadata_files for _, _, _, metadata_files in self.element_identifiers_extensions_paths_and_metadata_files] + + @property + def element_identifiers_extensions_paths_and_metadata_files(self): + if self._element_identifiers_extensions_paths_and_metadata_files is None: + self._element_identifiers_extensions_paths_and_metadata_files = self.collection.element_identifiers_extensions_paths_and_metadata_files + return self._element_identifiers_extensions_paths_and_metadata_files @property def all_element_identifiers_and_extensions_filesystem_safe(self): safe_element_identifiers = [] - for element_identifiers, extension, _ in self.all_element_identifiers_extensions_and_paths: + for element_identifiers, extension, *_ in self.element_identifiers_extensions_paths_and_metadata_files: datatype = self.datatypes_registry.get_datatype_by_extension(extension) if datatype: extension = getattr(datatype, 'file_ext_export_alias', extension) diff --git a/test/integration/test_remote_files.py b/test/integration/test_remote_files.py index 9927b53e8f5..622e772fa6f 100644 --- a/test/integration/test_remote_files.py +++ b/test/integration/test_remote_files.py @@ -155,7 +155,7 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase): } response = dataset_populator.run_tool("export_remote", inputs, history_id) dataset_populator.wait_for_job(response["jobs"][0]["id"], assert_ok=True) - with open(os.path.join(ftp_dir, 'my_cool_utf8_name_😻.txt')) as f: + with open(os.path.join(ftp_dir, 'my_cool', 'utf8_name_😻.txt')) as f: assert 'example content\n' == f.read() def test_export_remote_tool_default_duplicate_name_fails(self): @@ -204,6 +204,7 @@ class RemoteFilesIntegrationTestCase(ConfiguresRemoteFilesIntegrationTestCase): dataset_populator.wait_for_job(response["jobs"][0]["id"], assert_ok=True) with open(os.path.join(ftp_dir, 'foo_1.vcf.gz'), 'rb') as export, open(VCF_GZ_PATH, 'rb') as vcf_gz: assert export.read() == vcf_gz.read() + assert os.path.exists(os.path.join(ftp_dir, 'foo_1.vcf.gz.tbi')) def test_export_remote_tool_collection_structure(self): dataset_populator = self.dataset_populator diff --git a/test/unit/data/test_galaxy_mapping.py b/test/unit/data/test_galaxy_mapping.py index 44a58f6c288..5cbe0a58a0b 100644 --- a/test/unit/data/test_galaxy_mapping.py +++ b/test/unit/data/test_galaxy_mapping.py @@ -2,6 +2,7 @@ import collections import os import unittest import uuid +from tempfile import NamedTemporaryFile import pytest from sqlalchemy import inspect @@ -10,6 +11,7 @@ import galaxy.datatypes.registry import galaxy.model import galaxy.model.mapping as mapping from galaxy.model.database_utils import create_database +from galaxy.model.metadata import MetadataTempFile from galaxy.model.security import GalaxyRBACAgent datatypes_registry = galaxy.datatypes.registry.Registry() @@ -324,11 +326,20 @@ class MappingTests(BaseModelTestCase): u = model.User(email="mary2@example.com", password="password") h1 = model.History(name="History 1", user=u) d1 = model.HistoryDatasetAssociation(extension="bam", history=h1, create_dataset=True, sa_session=model.session) + index = NamedTemporaryFile("w") + index.write("cool bam index") + index2 = NamedTemporaryFile("w") + index2.write("cool bam index 2") + metadata_dict = {"bam_index": MetadataTempFile.from_JSON({"kwds": {}, "filename": index.name}), "bam_csi_index": MetadataTempFile.from_JSON({"kwds": {}, "filename": index2.name})} + d1.metadata.from_JSON_dict(json_dict=metadata_dict) + assert d1.metadata.bam_index + assert d1.metadata.bam_csi_index + assert isinstance(d1.metadata.bam_index, model.MetadataFile) + assert isinstance(d1.metadata.bam_csi_index, model.MetadataFile) d2 = model.HistoryDatasetAssociation(extension="txt", history=h1, create_dataset=True, sa_session=model.session) c1 = model.DatasetCollection(collection_type='paired') dce1 = model.DatasetCollectionElement(collection=c1, element=d1, element_identifier="forward", element_index=0) dce2 = model.DatasetCollectionElement(collection=c1, element=d2, element_identifier="reverse", element_index=1) - c2 = model.DatasetCollection(collection_type="list:paired") dce3 = model.DatasetCollectionElement(collection=c2, element=c1, element_identifier="inner_list", element_index=0) c3 = model.DatasetCollection(collection_type="list:list") @@ -349,6 +360,7 @@ class MappingTests(BaseModelTestCase): assert c2.dataset_action_tuples == [] assert c2.populated_optimized assert c2.dataset_states_and_extensions_summary == ({'new'}, {'txt', 'bam'}) + assert c2.element_identifiers_extensions_paths_and_metadata_files == [[('inner_list', 'forward'), 'bam', 'mock_dataset_14.dat', [('bai', 'mock_dataset_14.dat'), ('bam.csi', 'mock_dataset_14.dat')]], [('inner_list', 'reverse'), 'txt', 'mock_dataset_14.dat', []]] assert c3.dataset_instances == [] assert c3.dataset_elements == [] assert c3.dataset_states_and_extensions_summary == (set(), set()) @@ -889,6 +901,9 @@ class MockObjectStore: def get_store_by(self, *args, **kwds): return 'id' + def update_from_file(self, *arg, **kwds): + pass + def get_suite(): suite = unittest.TestSuite() diff --git a/tools/data_export/export_remote.py b/tools/data_export/export_remote.py index 5c19bb699ad..31182042104 100644 --- a/tools/data_export/export_remote.py +++ b/tools/data_export/export_remote.py @@ -26,6 +26,15 @@ def check_for_duplicate_name(files_to_export): sys.exit(f"Duplicate export filenames given: {', '.join(duplicates)}, failing export") +def write_if_not_exists(file_sources, target_uri, real_data_path): + file_source_path = file_sources.get_file_source_path(target_uri) + if os.path.exists(file_source_path.path): + print(f'Error: File "{file_source_path.path}" already exists. Skipping.') + return 1 + file_source = file_source_path.file_source + file_source.write_from(file_source_path.path, real_data_path) + + def main(argv=None): if argv is None: argv = sys.argv[1:] @@ -33,6 +42,7 @@ def main(argv=None): exit_code = 0 file_sources = get_file_sources(args.file_sources) directory_uri = args.directory_uri + export_metadata_files = args.export_metadata_files with open(args.files_to_export) as f: files_to_export = json.load(f) counter = 0 @@ -44,13 +54,14 @@ def main(argv=None): target_uri = directory_uri + name else: target_uri = directory_uri + "/" + name - file_source_path = file_sources.get_file_source_path(target_uri) - if os.path.exists(file_source_path.path): - print(f'Error: File "{file_source_path.path}" already exists. Skipping.') + if write_if_not_exists(file_sources, target_uri, real_data_path): exit_code = 1 - continue - file_source = file_source_path.file_source - file_source.write_from(file_source_path.path, real_data_path) + if export_metadata_files: + metadata_files = entry.get('metadata_files', []) + for extension, path in metadata_files: + metadata_file_uri = f"{target_uri}.{extension}" + if write_if_not_exists(file_sources, metadata_file_uri, path): + exit_code = 1 counter += 1 print(f"{counter} out of {len(files_to_export)} files have been exported.\n") sys.exit(exit_code) @@ -61,6 +72,7 @@ def _parser(): parser.add_argument("--directory-uri", type=str, help="directory target URI") parser.add_argument("--file-sources", type=str, help="file sources json") parser.add_argument("--files-to-export", type=str, help="files to export") + parser.add_argument("--export-metadata-files", type=bool, help="export metadata files", default=True) return parser diff --git a/tools/data_export/export_remote.xml b/tools/data_export/export_remote.xml index 75f927d73b8..39b8ac256fa 100644 --- a/tools/data_export/export_remote.xml +++ b/tools/data_export/export_remote.xml @@ -10,6 +10,7 @@ python '$__tool_directory__/export_remote.py' --file-sources '$file_sources' --directory-uri '$d_uri' --files-to-export '$files_to_export' + --export-metadata-files $include_metadata_files > '$out' ]]> @@ -17,11 +18,11 @@ python '$__tool_directory__/export_remote.py' #import json #from galaxy.util import filesystem_safe_string #if $export_type.export_type_selector == "datasets_auto": -#set $fileconfig = json.dumps([{"real_data_path": str(infile), "name": infile.name_and_ext_filesystem_safe} for infile in $infiles]) +#set $fileconfig = json.dumps([{"real_data_path": str(infile), "name": infile.name_and_ext_filesystem_safe, "metadata_files": infile.all_metadata_files} for infile in $infiles]) #else if $export_type.export_type_selector == "collection_auto": -#set $fileconfig = json.dumps([{"real_data_path": data_path, "name": identifier_and_extension} for data_path, identifier_and_extension in zip($export_type.incollection.all_paths, $export_type.incollection.all_element_identifiers_and_extensions_filesystem_safe)]) +#set $fileconfig = json.dumps([{"real_data_path": data_path, "name": identifier_and_extension, "metadata_files": metadata_files} for data_path, identifier_and_extension, metadata_files in zip($export_type.incollection.all_paths, $export_type.incollection.all_element_identifiers_and_extensions_filesystem_safe, $export_type.incollection.all_metadata_files)]) #else -#set $fileconfig = json.dumps([{"real_data_path": str(dataset["infile"]), "name": filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=())} for dataset in $export_type.datasets]) +#set $fileconfig = json.dumps([{"real_data_path": str(dataset["infile"]), "name": filesystem_safe_string(str(dataset["name"]), 255, invalid_chars=()), "metadata_files": dataset['infile'].all_metadata_files} for dataset in $export_type.datasets]) #end if $fileconfig @@ -52,6 +53,7 @@ $fileconfig +