Merge pull request #4771 from jmchilton/copy_children

Remove unused concept of dataset children.

lib/galaxy/jobs/__init__.py

@@ -1388,7 +1388,6 @@ class JobWrapper(object, HasResourceParameters):
             # Maybe this is a legacy job, use the job working directory instead
             tool_working_directory = self.working_directory
         collected_datasets = {
-            'children': self.tool.collect_child_datasets(out_data, tool_working_directory),
             'primary': self.tool.collect_primary_datasets(out_data, self.get_tool_provided_job_metadata(), tool_working_directory, input_ext, input_dbkey)
         }
         self.tool.collect_dynamic_collections(

lib/galaxy/model/__init__.py

@@ -1351,7 +1351,7 @@ class History(HasTags, Dictifiable, UsesAnnotations, HasName):
             hdas = self.active_datasets
         for hda in hdas:
             # Copy HDA.
-            new_hda = hda.copy(copy_children=True)
+            new_hda = hda.copy()
             new_history.add_dataset(new_hda, set_hid=False, quota=applies_to_quota)
             db_session.add(new_hda)
             db_session.flush()
@@ -1856,7 +1856,7 @@ class Dataset(StorableObject):
         """Detects whether there is any data"""
         return self.get_size() > 0
 
-    def mark_deleted(self, include_children=True):
+    def mark_deleted(self):
         self.deleted = True
 
     def is_multi_byte(self):
@@ -2189,12 +2189,6 @@ class DatasetInstance(object):
     def clear_associated_files(self, metadata_safe=False, purge=False):
         raise Exception("Unimplemented")
 
-    def get_child_by_designation(self, designation):
-        for child in self.children:
-            if child.designation == designation:
-                return child
-        return None
-
     def get_converter_types(self):
         return self.datatype.get_converter_types(self, _get_datatypes_registry())
 
@@ -2211,23 +2205,14 @@ class DatasetInstance(object):
     def extend_validation_errors(self, validation_errors):
         self.validation_errors.extend(validation_errors)
 
-    def mark_deleted(self, include_children=True):
+    def mark_deleted(self):
         self.deleted = True
-        if include_children:
-            for child in self.children:
-                child.mark_deleted()
 
-    def mark_undeleted(self, include_children=True):
+    def mark_undeleted(self):
         self.deleted = False
-        if include_children:
-            for child in self.children:
-                child.mark_undeleted()
 
-    def mark_unhidden(self, include_children=True):
+    def mark_unhidden(self):
         self.visible = True
-        if include_children:
-            for child in self.children:
-                child.mark_unhidden()
 
     def undeletable(self):
         if self.purged:
@@ -2390,7 +2375,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
         self.copied_from_history_dataset_association = copied_from_history_dataset_association
         self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association
 
-    def copy(self, copy_children=False, parent_id=None):
+    def copy(self, parent_id=None):
         """
         Create a copy of this HDA.
         """
@@ -2415,9 +2400,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
         hda.set_size()
         # Need to set after flushed, as MetadataFiles require dataset.id
         hda.metadata = self.metadata
-        if copy_children:
-            for child in self.children:
-                child.copy(copy_children=copy_children, parent_id=hda.id)
         if not self.datatype.copy_safe_peek:
             # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
             hda.set_peek()
@@ -2481,12 +2463,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
         library_dataset.library_dataset_dataset_association_id = ldda.id
         object_session(self).add(library_dataset)
         object_session(self).flush()
-        for child in self.children:
-            child.to_library_dataset_dataset_association(trans,
-                                                         target_folder=target_folder,
-                                                         replace_dataset=replace_dataset,
-                                                         parent_id=ldda.id,
-                                                         user=ldda.user)
         if not self.datatype.copy_safe_peek:
             # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
             ldda.set_peek()
@@ -2520,7 +2496,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
         # Anon users are handled just by their single history size.
         if not user:
             return rval
-        # Gets an HDA and its children's disk usage, if the user does not already
+        # Gets an HDA disk usage, if the user does not already
         #   have an association of the same dataset
         if not self.dataset.library_associations and not self.purged and not self.dataset.purged:
             for hda in self.dataset.history_associations:
@@ -2530,8 +2506,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot
                     break
             else:
                 rval += self.get_total_size()
-        for child in self.children:
-            rval += child.get_disk_usage(user)
         return rval
 
     def to_dict(self, view='collection', expose_dataset_path=False):
@@ -2946,14 +2920,12 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName):
         hda.metadata = self.metadata  # need to set after flushed, as MetadataFiles require dataset.id
         if add_to_history and target_history:
             target_history.add_dataset(hda)
-        for child in self.children:
-            child.to_history_dataset_association(target_history=target_history, parent_id=hda.id, add_to_history=False)
         if not self.datatype.copy_safe_peek:
             hda.set_peek()  # in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
         sa_session.flush()
         return hda
 
-    def copy(self, copy_children=False, parent_id=None, target_folder=None):
+    def copy(self, parent_id=None, target_folder=None):
         sa_session = object_session(self)
         ldda = LibraryDatasetDatasetAssociation(name=self.name,
                                                 info=self.info,
@@ -2977,9 +2949,6 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName):
         sa_session.flush()
         # Need to set after flushed, as MetadataFiles require dataset.id
         ldda.metadata = self.metadata
-        if copy_children:
-            for child in self.children:
-                child.copy(copy_children=copy_children, parent_id=ldda.id)
         if not self.datatype.copy_safe_peek:
             # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
             ldda.set_peek()
@@ -3591,7 +3560,7 @@ class DatasetCollectionElement(object, Dictifiable):
                     element_destination=element_destination
                 )
             else:
-                new_element_object = element_object.copy(copy_children=True)
+                new_element_object = element_object.copy()
                 if destination is not None and element_object.hidden_beneath_collection_instance:
                     new_element_object.hidden_beneath_collection_instance = destination
                 # Ideally we would not need to give the following

lib/galaxy/model/mapping.py

@@ -1605,18 +1605,6 @@ simple_mapping(model.HistoryDatasetAssociation,
     implicitly_converted_parent_datasets=relation(model.ImplicitlyConvertedDatasetAssociation,
         primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.hda_id ==
                      model.HistoryDatasetAssociation.table.c.id)),
-    children=relation(model.HistoryDatasetAssociation,
-        primaryjoin=(model.HistoryDatasetAssociation.table.c.parent_id ==
-                      model.HistoryDatasetAssociation.table.c.id),
-        backref=backref("parent",
-            primaryjoin=(model.HistoryDatasetAssociation.table.c.parent_id ==
-                         model.HistoryDatasetAssociation.table.c.id),
-            remote_side=[model.HistoryDatasetAssociation.table.c.id], uselist=False)),
-    visible_children=relation(model.HistoryDatasetAssociation,
-        primaryjoin=(
-            (model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id) &
-            (model.HistoryDatasetAssociation.table.c.visible == true())),
-        remote_side=[model.HistoryDatasetAssociation.table.c.id]),
     tags=relation(model.HistoryDatasetAssociationTagAssociation,
         order_by=model.HistoryDatasetAssociationTagAssociation.table.c.id,
         backref='history_tag_associations'),
@@ -1987,19 +1975,6 @@ mapper(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssoci
     implicitly_converted_datasets=relation(model.ImplicitlyConvertedDatasetAssociation,
         primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.ldda_parent_id ==
                      model.LibraryDatasetDatasetAssociation.table.c.id)),
-    children=relation(model.LibraryDatasetDatasetAssociation,
-        primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.parent_id ==
-                      model.LibraryDatasetDatasetAssociation.table.c.id),
-        backref=backref("parent",
-            primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.parent_id ==
-                         model.LibraryDatasetDatasetAssociation.table.c.id),
-            remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id])),
-    visible_children=relation(model.LibraryDatasetDatasetAssociation,
-        primaryjoin=(
-            (model.LibraryDatasetDatasetAssociation.table.c.parent_id == model.LibraryDatasetDatasetAssociation.table.c.id) &
-            (model.LibraryDatasetDatasetAssociation.table.c.visible == true())
-        ),
-        remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id]),
     tags=relation(model.LibraryDatasetDatasetAssociationTagAssociation,
                   order_by=model.LibraryDatasetDatasetAssociationTagAssociation.table.c.id,
                   backref='history_tag_associations'),

lib/galaxy/tools/__init__.py

@@ -1,7 +1,6 @@
 """
 Classes encapsulating galaxy tools and tool configuration.
 """
-import glob
 import json
 import logging
 import os
@@ -1560,73 +1559,6 @@ class Tool(object, Dictifiable):
         """
         pass
 
-    def collect_child_datasets(self, output, job_working_directory):
-        """
-        Look for child dataset files, create HDA and attach to parent.
-        """
-        children = {}
-        # Loop through output file names, looking for generated children in
-        # form of 'child_parentId_designation_visibility_extension'
-        for name, outdata in output.items():
-            filenames = []
-            if 'new_file_path' in self.app.config.collect_outputs_from:
-                filenames.extend(glob.glob(os.path.join(self.app.config.new_file_path, "child_%i_*" % outdata.id)))
-            if 'job_working_directory' in self.app.config.collect_outputs_from:
-                filenames.extend(glob.glob(os.path.join(job_working_directory, "child_%i_*" % outdata.id)))
-            for filename in filenames:
-                if name not in children:
-                    children[name] = {}
-                fields = os.path.basename(filename).split("_")
-                designation = fields[2]
-                visible = fields[3].lower()
-                if visible == "visible":
-                    visible = True
-                else:
-                    visible = False
-                ext = fields[4].lower()
-                child_dataset = self.app.model.HistoryDatasetAssociation(extension=ext,
-                                                                         parent_id=outdata.id,
-                                                                         designation=designation,
-                                                                         visible=visible,
-                                                                         dbkey=outdata.dbkey,
-                                                                         create_dataset=True,
-                                                                         sa_session=self.sa_session)
-                self.app.security_agent.copy_dataset_permissions(outdata.dataset, child_dataset.dataset)
-                # Move data from temp location to dataset location
-                self.app.object_store.update_from_file(child_dataset.dataset, file_name=filename, create=True)
-                self.sa_session.add(child_dataset)
-                self.sa_session.flush()
-                child_dataset.set_size()
-                child_dataset.name = "Secondary Dataset (%s)" % (designation)
-                child_dataset.init_meta()
-                child_dataset.set_meta()
-                child_dataset.set_peek()
-                # Associate new dataset with job
-                job = None
-                for assoc in outdata.creating_job_associations:
-                    job = assoc.job
-                    break
-                if job:
-                    assoc = self.app.model.JobToOutputDatasetAssociation('__new_child_file_%s|%s__' % (name, designation), child_dataset)
-                    assoc.job = job
-                    self.sa_session.add(assoc)
-                    self.sa_session.flush()
-                child_dataset.state = outdata.state
-                self.sa_session.add(child_dataset)
-                self.sa_session.flush()
-                # Add child to return dict
-                children[name][designation] = child_dataset
-                # Need to update all associated output hdas, i.e. history was
-                # shared with job running
-                for dataset in outdata.dataset.history_associations:
-                    if outdata == dataset:
-                        continue
-                    # Create new child dataset
-                    child_data = child_dataset.copy(parent_id=dataset.id)
-                    self.sa_session.add(child_data)
-                    self.sa_session.flush()
-        return children
-
     def collect_primary_datasets(self, output, tool_provided_metadata, job_working_directory, input_ext, input_dbkey="?"):
         """
         Find any additional datasets generated by a tool and attach (for

lib/galaxy/tools/evaluation.py

@@ -91,7 +91,6 @@ class ToolEvaluator(object):
                     self.dataset = dataset
                     self.file_name = dataset.file_name
                     self.metadata = dict()
-                    self.children = []
 
             special = get_special()
             if special:
@@ -303,9 +302,6 @@ class ToolEvaluator(object):
                         dataset_path = input_dataset_paths[real_path]
                         wrapper_kwds['dataset_path'] = dataset_path
                 param_dict[name] = DatasetFilenameWrapper(data, **wrapper_kwds)
-            if data:
-                for child in data.children:
-                    param_dict["_CHILD___%s___%s" % (name, child.designation)] = DatasetFilenameWrapper(child)
 
     def __populate_output_collection_wrappers(self, param_dict, output_collections, output_paths, job_working_directory):
         output_dataset_paths = dataset_path_rewrites(output_paths)
@@ -355,8 +351,6 @@ class ToolEvaluator(object):
             # Provide access to a path to store additional files
             # TODO: path munging for cluster/dataset server relocatability
             param_dict[name].files_path = os.path.abspath(os.path.join(job_working_directory, "dataset_%s_files" % (hda.dataset.id)))
-            for child in hda.children:
-                param_dict["_CHILD___%s___%s" % (name, child.designation)] = DatasetFilenameWrapper(child)
         for out_name, output in self.tool.outputs.items():
             if out_name not in param_dict and output.filters:
                 # Assume the reason we lack this output is because a filter

lib/galaxy/tools/imp_exp/__init__.py

@@ -6,7 +6,7 @@ import shutil
 import tempfile
 from json import dumps, loads
 
-from sqlalchemy.orm import eagerload, eagerload_all
+from sqlalchemy.orm import eagerload_all
 from sqlalchemy.sql import expression
 
 from galaxy import model
@@ -322,7 +322,6 @@ class JobExportHistoryArchiveWrapper(object, UsesAnnotations):
         """
         query = (trans.sa_session.query(trans.model.HistoryDatasetAssociation)
                  .filter(trans.model.HistoryDatasetAssociation.history == history)
-                 .options(eagerload("children"))
                  .join("dataset")
                  .options(eagerload_all("dataset.actions"))
                  .order_by(trans.model.HistoryDatasetAssociation.hid)

lib/galaxy/webapps/galaxy/controllers/dataset.py

@@ -1238,7 +1238,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
                     else:
                         for hist in target_histories:
                             if content.history_content_type == "dataset":
-                                hist.add_dataset(content.copy(copy_children=True))
+                                hist.add_dataset(content.copy())
                             else:
                                 copy_collected_datasets = True
                                 copy_kwds = {}
@@ -1296,7 +1296,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE
                     invalid_datasets += 1
                 else:
                     for hist in target_histories:
-                        dataset_copy = data.copy(copy_children=True)
+                        dataset_copy = data.copy()
                         if imported:
                             dataset_copy.name = "imported: " + dataset_copy.name
                         hist.add_dataset(dataset_copy)

lib/galaxy/webapps/galaxy/controllers/history.py

@@ -495,7 +495,6 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt
         trans.sa_session.expunge(trans.history)
         history = trans.sa_session.query(model.History).options(
             eagerload_all('active_datasets.creating_job_associations.job.workflow_invocation_step.workflow_invocation.workflow'),
-            eagerload_all('active_datasets.children')
         ).get(id)
         assert history
         # TODO: formalize to trans.show_error

lib/galaxy/webapps/galaxy/controllers/root.py

@@ -221,25 +221,6 @@ class RootController(controller.JSAppLauncher, UsesAnnotations):
         else:
             return "No dataset with id '%s'" % str(id)
 
-    @web.expose
-    def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"):
-        """Returns child data directly into the browser, based upon parent_id and designation.
-        """
-        # TODO: unencoded id
-        try:
-            data = trans.sa_session.query(self.app.model.HistoryDatasetAssociation).get(parent_id)
-            if data:
-                child = data.get_child_by_designation(designation)
-                if child:
-                    current_user_roles = trans.get_current_user_roles()
-                    if trans.app.security_agent.can_access_dataset(current_user_roles, child):
-                        return self.display(trans, id=child.id, tofile=tofile, toext=toext)
-                    else:
-                        return "You are not privileged to access this dataset."
-        except Exception:
-            pass
-        return "A child named %s could not be found for data %s" % (designation, parent_id)
-
     @web.expose
     def display_as(self, trans, id=None, display_app=None, **kwd):
         """Returns a file in a format that can successfully be displayed in display_app.

lib/galaxy/workflow/modules.py

@@ -362,7 +362,7 @@ class InputModule(WorkflowModule):
             for input_dataset_hda in list(step_outputs.values()):
                 content_type = input_dataset_hda.history_content_type
                 if content_type == "dataset":
-                    new_hda = input_dataset_hda.copy(copy_children=True)
+                    new_hda = input_dataset_hda.copy()
                     invocation.history.add_dataset(new_hda)
                     step_outputs['input_ds_copy'] = new_hda
                 elif content_type == "dataset_collection":