Merge branch 'release_23.0' into release_23.1

lib/galaxy/job_execution/output_collect.py

@@ -732,18 +732,19 @@ def collect_extra_files(object_store, dataset, job_working_directory):
         # Fall back to working dir, remove in 23.2
         output_location = "working"
         temp_file_path = os.path.join(job_working_directory, output_location, file_name)
-    extra_dir = None
+    if not os.path.exists(temp_file_path):
+        # no outputs to working directory, but may still need to push form cache to backend
+        temp_file_path = dataset.extra_files_path
     try:
         # This skips creation of directories - object store
         # automatically creates them.  However, empty directories will
         # not be created in the object store at all, which might be a
         # problem.
         for root, _dirs, files in os.walk(temp_file_path):
-            extra_dir = root.replace(os.path.join(job_working_directory, output_location), "", 1).lstrip(os.path.sep)
             for f in files:
                 object_store.update_from_file(
                     dataset.dataset,
-                    extra_dir=extra_dir,
+                    extra_dir=os.path.normpath(os.path.join(file_name, os.path.relpath(root, temp_file_path))),
                     alt_name=f,
                     file_name=os.path.join(root, f),
                     create=True,

lib/galaxy/metadata/set_metadata.py

@@ -527,7 +527,7 @@ def write_job_metadata(tool_job_working_directory, job_metadata, set_meta, tool_
         new_dataset = Dataset(id=-i, external_filename=new_dataset_filename)
         extra_files = file_dict.get("extra_files", None)
         if extra_files is not None:
-            new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "working", extra_files)
+            new_dataset._extra_files_path = os.path.join(tool_job_working_directory, "outputs", extra_files)
         new_dataset.state = new_dataset.states.OK
         new_dataset_instance = HistoryDatasetAssociation(
             id=-i, dataset=new_dataset, extension=file_dict.get("ext", "data")

lib/galaxy/objectstore/s3.py

@@ -49,6 +49,21 @@ log = logging.getLogger(__name__)
 logging.getLogger("boto").setLevel(logging.INFO)  # Otherwise boto is quite noisy
 
 
+def download_directory(bucket, remote_folder, local_path):
+    # List objects in the specified S3 folder
+    objects = bucket.list(prefix=remote_folder)
+
+    for obj in objects:
+        remote_file_path = obj.key
+        local_file_path = os.path.join(local_path, os.path.relpath(remote_file_path, remote_folder))
+
+        # Create directories if they don't exist
+        os.makedirs(os.path.dirname(local_file_path), exist_ok=True)
+
+        # Download the file
+        obj.get_contents_to_filename(local_file_path)
+
+
 def parse_config_xml(config_xml):
     try:
         a_xml = config_xml.findall("auth")[0]
@@ -652,7 +667,8 @@ class S3ObjectStore(ConcreteObjectStore, CloudConfigMixin):
             return cache_path
         # Check if the file exists in persistent storage and, if it does, pull it into cache
         elif self._exists(obj, **kwargs):
-            if dir_only:  # Directories do not get pulled into cache
+            if dir_only:
+                download_directory(self._bucket, rel_path, cache_path)
                 return cache_path
             else:
                 if self._pull_into_cache(rel_path):

lib/galaxy/tool_util/data/__init__.py

@@ -1174,8 +1174,10 @@ class ToolDataTableManager(Dictifiable):
         out_data: Dict[str, OutputDataset],
         bundle_description: DataTableBundleProcessorDescription,
         repo_info: Optional[RepoInfo],
-    ) -> None:
+    ) -> Dict[str, OutputDataset]:
+        """Writes bundle and returns bundle path."""
         data_manager_dict = _data_manager_dict(out_data, ensure_single_output=True)
+        bundle_datasets: Dict[str, OutputDataset] = {}
         for output_name, dataset in out_data.items():
             if dataset.ext != "data_manager_json":
                 continue
@@ -1190,6 +1192,8 @@ class ToolDataTableManager(Dictifiable):
             bundle_path = os.path.join(extra_files_path, BUNDLE_INDEX_FILE_NAME)
             with open(bundle_path, "w") as fw:
                 json.dump(bundle.dict(), fw)
+            bundle_datasets[bundle_path] = dataset
+        return bundle_datasets
 
 
 SUPPORTED_DATA_TABLE_TYPES = TabularToolDataTable

lib/galaxy/tools/__init__.py

@@ -3200,7 +3200,17 @@ class DataManagerTool(OutputParameterJSONTool):
         elif data_manager_mode == "dry_run":
             pass
         elif data_manager_mode == "bundle":
-            data_manager.write_bundle(out_data)
+            for bundle_path, dataset in data_manager.write_bundle(out_data).items():
+                dataset = cast(model.HistoryDatasetAssociation, dataset)
+                dataset.dataset.object_store.update_from_file(
+                    dataset.dataset,
+                    extra_dir=dataset.dataset.extra_files_path_name,
+                    file_name=bundle_path,
+                    alt_name=os.path.basename(bundle_path),
+                    create=True,
+                    preserve_symlinks=True,
+                )
+
         else:
             raise Exception("Unknown data manager mode encountered type...")
 

lib/galaxy/tools/data_manager/manager.py

@@ -236,9 +236,9 @@ class DataManager:
     def write_bundle(
         self,
         out_data: Dict[str, OutputDataset],
-    ) -> None:
+    ):
         tool_data_tables = self.data_managers.app.tool_data_tables
-        tool_data_tables.write_bundle(
+        return tool_data_tables.write_bundle(
             out_data,
             self.processor_description,
             self.repo_info,

test/integration/test_tool_data_bundles.py

@@ -1,10 +1,12 @@
 import os
+import shutil
 
 from galaxy.util.compression_utils import decompress_bytes_to_directory
+from .objectstore._base import BaseSwiftObjectStoreIntegrationTestCase
 from .test_tool_data_delete import DataManagerIntegrationTestCase
 
 
-class TestDataBundlesIntegration(DataManagerIntegrationTestCase):
+class TestDataBundlesIntegration(BaseSwiftObjectStoreIntegrationTestCase, DataManagerIntegrationTestCase):
     def test_admin_build_data_bundle_by_uri(self):
         original_count = self._testbeta_field_count()
 
@@ -24,10 +26,14 @@ class TestDataBundlesIntegration(DataManagerIntegrationTestCase):
         post_job_count = self._testbeta_field_count()
         assert original_count == post_job_count
 
+        shutil.rmtree(self.object_store_cache_path)
+        os.makedirs(self.object_store_cache_path)
+
         content = self.dataset_populator.get_history_dataset_content(
             history_id, to_ext="data_manager_json", type="bytes"
         )
         temp_directory = decompress_bytes_to_directory(content)
+        assert os.path.exists(os.path.join(temp_directory, "newvalue.txt"))
         uri = f"file://{os.path.normpath(temp_directory)}"
         data = {
             "source": {