Merge pull request #17890 from mvdbeek/fix_empty_nested_collection_element_count

[24.0] Always serialize element_count and populated when listing contents

lib/galaxy/managers/collections_util.py

@@ -1,5 +1,9 @@
 import logging
 import math
+from typing import (
+    Any,
+    Dict,
+)
 
 from galaxy import (
     exceptions,
@@ -153,7 +157,9 @@ def dictify_dataset_collection_instance(
     return dict_value
 
 
-def dictify_element_reference(element, rank_fuzzy_counts=None, recursive=True, security=None):
+def dictify_element_reference(
+    element: model.DatasetCollectionElement, rank_fuzzy_counts=None, recursive=True, security=None
+):
     """Load minimal details of elements required to show outline of contents in history panel.
 
     History panel can use this reference to expand to full details if individual dataset elements
@@ -161,27 +167,28 @@ def dictify_element_reference(element, rank_fuzzy_counts=None, recursive=True, s
     """
     dictified = element.to_dict(view="element")
     if (element_object := element.element_object) is not None:
-        object_details = dict(
+        object_details: Dict[str, Any] = dict(
             id=element_object.id,
             model_class=element_object.__class__.__name__,
         )
-        if element.child_collection:
+        if isinstance(element_object, model.DatasetCollection):
             object_details["collection_type"] = element_object.collection_type
+            object_details["element_count"] = element_object.element_count
+            object_details["populated"] = element_object.populated_optimized
 
             # Recursively yield elements for each nested collection...
             if recursive:
-                child_collection = element.child_collection
-                elements, rest_fuzzy_counts = get_fuzzy_count_elements(child_collection, rank_fuzzy_counts)
+                elements, rest_fuzzy_counts = get_fuzzy_count_elements(element_object, rank_fuzzy_counts)
                 object_details["elements"] = [
                     dictify_element_reference(_, rank_fuzzy_counts=rest_fuzzy_counts, recursive=recursive)
                     for _ in elements
                 ]
-                object_details["element_count"] = child_collection.element_count
         else:
             object_details["state"] = element_object.state
             object_details["hda_ldda"] = "hda"
-            object_details["history_id"] = element_object.history_id
-            object_details["tags"] = element_object.make_tag_string_list()
+            if isinstance(element_object, model.HistoryDatasetAssociation):
+                object_details["history_id"] = element_object.history_id
+                object_details["tags"] = element_object.make_tag_string_list()
 
         dictified["object"] = object_details
     else:

lib/galaxy/model/__init__.py

@@ -4996,6 +4996,8 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
     Resource class that creates a relation between a dataset and a user history.
     """
 
+    history_id: Optional[int]
+
     def __init__(
         self,
         hid=None,
@@ -7153,7 +7155,9 @@ class DatasetCollectionElement(Base, Dictifiable, Serializable):
         return self.element_type == "dataset_collection"
 
     @property
-    def element_object(self):
+    def element_object(
+        self,
+    ) -> Optional[Union[HistoryDatasetAssociation, LibraryDatasetDatasetAssociation, DatasetCollection]]:
         if self.hda:
             return self.hda
         elif self.ldda:

lib/galaxy/tools/__init__.py

@@ -3461,7 +3461,9 @@ class FilterDatasetsTool(DatabaseOperationTool):
 
     @staticmethod
     def element_is_valid(element: model.DatasetCollectionElement):
-        return element.element_object.is_ok
+        element_object = element.element_object
+        assert isinstance(element_object, model.DatasetInstance)
+        return element_object.is_ok
 
     def produce_outputs(self, trans, out_data, output_collections, incoming, history, **kwds):
         collection = incoming["input"]
@@ -3503,7 +3505,9 @@ class FilterFailedDatasetsTool(FilterDatasetsTool):
 
     @staticmethod
     def element_is_valid(element: model.DatasetCollectionElement):
-        return element.element_object.is_ok
+        element_object = element.element_object
+        assert isinstance(element_object, model.DatasetInstance)
+        return element_object.is_ok
 
 
 class KeepSuccessDatasetsTool(FilterDatasetsTool):
@@ -3514,12 +3518,14 @@ class KeepSuccessDatasetsTool(FilterDatasetsTool):
 
     @staticmethod
     def element_is_valid(element: model.DatasetCollectionElement):
+        element_object = element.element_object
+        assert isinstance(element_object, model.DatasetInstance)
         if (
-            element.element_object.state != model.Dataset.states.PAUSED
-            and element.element_object.state in model.Dataset.non_ready_states
+            element_object.state != model.Dataset.states.PAUSED
+            and element_object.state in model.Dataset.non_ready_states
         ):
             raise ToolInputsNotReadyException("An input dataset is pending.")
-        return element.element_object.is_ok
+        return element_object.is_ok
 
 
 class FilterEmptyDatasetsTool(FilterDatasetsTool):
@@ -3528,10 +3534,11 @@ class FilterEmptyDatasetsTool(FilterDatasetsTool):
 
     @staticmethod
     def element_is_valid(element: model.DatasetCollectionElement):
-        dataset_instance: model.DatasetInstance = element.element_object
-        if dataset_instance.has_data():
+        element_object = element.element_object
+        assert isinstance(element_object, model.DatasetInstance)
+        if element_object.has_data():
             # We have data, but it might just be a compressed archive of nothing
-            file_name = dataset_instance.get_file_name()
+            file_name = element_object.get_file_name()
             _, fh = get_fileobj_raw(file_name, mode="rb")
             if len(fh.read(1)):
                 return True

lib/galaxy_test/api/test_dataset_collections.py

@@ -455,7 +455,7 @@ class TestDatasetCollectionsApi(ApiTestCase):
         # Get contents_url from history contents, use it to show the first level
         # of collection contents in the created HDCA, then use it again to drill
         # down into the nested collection contents
-        hdca = self.dataset_collection_populator.create_list_of_list_in_history(history_id).json()
+        hdca = self.dataset_collection_populator.create_list_of_list_in_history(history_id, wait=True).json()
         root_contents_url = self._get_contents_url_for_hdca(history_id, hdca)
 
         # check root contents for this collection
@@ -466,6 +466,8 @@ class TestDatasetCollectionsApi(ApiTestCase):
         # drill down, retrieve nested collection contents
         assert "object" in root_contents[0]
         assert "contents_url" in root_contents[0]["object"]
+        assert root_contents[0]["object"]["element_count"] == 3
+        assert root_contents[0]["object"]["populated"]
         drill_contents_url = root_contents[0]["object"]["contents_url"]
         drill_contents = self._get(drill_contents_url).json()
         assert len(drill_contents) == len(hdca["elements"][0]["object"]["elements"])