Merge pull request #18855 from mvdbeek/fix_init_from_object_store_id

[24.0] Fix discovered outputs with directory metadata and distributed object

lib/galaxy/model/store/discover.py

@@ -132,11 +132,6 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
                 )
                 self.persist_object(primary_data)
 
-                if init_from:
-                    self.permission_provider.copy_dataset_permissions(init_from, primary_data)
-                    primary_data.state = init_from.state
-                else:
-                    self.permission_provider.set_default_hda_permissions(primary_data)
             else:
                 ld = galaxy.model.LibraryDataset(folder=library_folder, name=name)
                 ldda = galaxy.model.LibraryDatasetDatasetAssociation(
@@ -208,6 +203,7 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
                     filename=filename,
                     link_data=link_data,
                     output_name=output_name,
+                    init_from=init_from,
                 )
             else:
                 storage_callbacks.append(
@@ -218,11 +214,14 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
                         filename=filename,
                         link_data=link_data,
                         output_name=output_name,
+                        init_from=init_from,
                     )
                 )
         return primary_data
 
-    def finalize_storage(self, primary_data, dataset_attributes, extra_files, filename, link_data, output_name):
+    def finalize_storage(
+        self, primary_data, dataset_attributes, extra_files, filename, link_data, output_name, init_from
+    ):
         if primary_data.dataset.purged:
             # metadata won't be set, maybe we should do that, then purge ?
             primary_data.dataset.file_size = 0
@@ -243,6 +242,13 @@ class ModelPersistenceContext(metaclass=abc.ABCMeta):
         else:
             # We are sure there are no extra files, so optimize things that follow by settting total size also.
             primary_data.set_size(no_extra_files=True)
+
+        if init_from:
+            self.permission_provider.copy_dataset_permissions(init_from, primary_data)
+            primary_data.state = init_from.state
+        else:
+            self.permission_provider.set_default_hda_permissions(primary_data)
+
         # TODO: this might run set_meta after copying the file to the object store, which could be inefficient if job working directory is closer to the node.
         self.set_datasets_metadata(datasets=[primary_data], datasets_attributes=[dataset_attributes])
 

test/integration/objectstore/test_jobs.py

@@ -3,12 +3,17 @@
 import os
 import string
 
+from galaxy_test.driver.integration_util import (
+    integration_module_instance,
+    integration_tool_runner,
+)
 from ._base import (
     BaseObjectStoreIntegrationTestCase,
     files_count,
 )
+from .test_selection_with_resource_parameters import DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE
 
-DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(
+HIERARCHICAL_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(
     """<?xml version="1.0"?>
 <object_store type="hierarchical">
     <backends>
@@ -39,7 +44,20 @@ DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE = string.Template(
 TEST_INPUT_FILES_CONTENT = "1 2 3"
 
 
-class TestObjectStoreJobsIntegration(BaseObjectStoreIntegrationTestCase):
+class TestDistributedObjectStore(BaseObjectStoreIntegrationTestCase):
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        super().handle_galaxy_config_kwds(config)
+        config["metadata_strategy"] = "directory"
+        config["object_store_store_by"] = "uuid"
+        cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config)
+
+
+instance = integration_module_instance(TestDistributedObjectStore)
+test_tools = integration_tool_runner(["all_output_types"])
+
+
+class TestObjectStoreJobsIntegration(TestDistributedObjectStore):
     # setup by _configure_object_store
     files1_path: str
     files2_path: str
@@ -48,7 +66,7 @@ class TestObjectStoreJobsIntegration(BaseObjectStoreIntegrationTestCase):
     @classmethod
     def handle_galaxy_config_kwds(cls, config):
         super().handle_galaxy_config_kwds(config)
-        cls._configure_object_store(DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE, config)
+        cls._configure_object_store(HIERARCHICAL_OBJECT_STORE_CONFIG_TEMPLATE, config)
 
     def setUp(self):
         super().setUp()
@@ -68,7 +86,7 @@ class TestObjectStoreJobsIntegration(BaseObjectStoreIntegrationTestCase):
     def test_files_count_and_content_in_each_objectstore_backend(self):
         """
         According to the ObjectStore configuration given in the
-        `DISTRIBUTED_OBJECT_STORE_CONFIG_TEMPLATE` variable, datasets
+        `HIERARCHICAL_OBJECT_STORE_CONFIG_TEMPLATE` variable, datasets
         can be stored on three backends, named:
             -   primary/files1;
             -   primary/files2;

test/integration/test_extended_metadata.py

@@ -43,6 +43,7 @@ TEST_TOOL_IDS = [
     "collection_creates_dynamic_nested_from_json_elements",
     "implicit_conversion",
     "environment_variables",
+    "all_output_types",
 ]