From d5b71415f62d97487ef898934875da65a2b29c47 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 9 Oct 2017 11:47:38 -0400 Subject: [PATCH] Remove unused concept of dataset children. We didn't really have any corresponding to active tools, we know if they are present they break multiple different parts of Galaxy. Lots of database loads and complexity we are not using and that is broken. --- lib/galaxy/jobs/__init__.py | 1 - lib/galaxy/model/__init__.py | 49 +++---------- lib/galaxy/model/mapping.py | 25 ------- lib/galaxy/tools/__init__.py | 68 ------------------- lib/galaxy/tools/evaluation.py | 6 -- lib/galaxy/tools/imp_exp/__init__.py | 3 +- .../webapps/galaxy/controllers/dataset.py | 4 +- .../webapps/galaxy/controllers/history.py | 1 - lib/galaxy/webapps/galaxy/controllers/root.py | 19 ------ lib/galaxy/workflow/modules.py | 2 +- 10 files changed, 13 insertions(+), 165 deletions(-) diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 08457df9aa2..13378fb6605 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1388,7 +1388,6 @@ class JobWrapper(object, HasResourceParameters): # Maybe this is a legacy job, use the job working directory instead tool_working_directory = self.working_directory collected_datasets = { - 'children': self.tool.collect_child_datasets(out_data, tool_working_directory), 'primary': self.tool.collect_primary_datasets(out_data, self.get_tool_provided_job_metadata(), tool_working_directory, input_ext, input_dbkey) } self.tool.collect_dynamic_collections( diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1c287f0dc2a..ecfde77474e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1351,7 +1351,7 @@ class History(HasTags, Dictifiable, UsesAnnotations, HasName): hdas = self.active_datasets for hda in hdas: # Copy HDA. - new_hda = hda.copy(copy_children=True) + new_hda = hda.copy() new_history.add_dataset(new_hda, set_hid=False, quota=applies_to_quota) db_session.add(new_hda) db_session.flush() @@ -1856,7 +1856,7 @@ class Dataset(StorableObject): """Detects whether there is any data""" return self.get_size() > 0 - def mark_deleted(self, include_children=True): + def mark_deleted(self): self.deleted = True def is_multi_byte(self): @@ -2189,12 +2189,6 @@ class DatasetInstance(object): def clear_associated_files(self, metadata_safe=False, purge=False): raise Exception("Unimplemented") - def get_child_by_designation(self, designation): - for child in self.children: - if child.designation == designation: - return child - return None - def get_converter_types(self): return self.datatype.get_converter_types(self, _get_datatypes_registry()) @@ -2211,23 +2205,14 @@ class DatasetInstance(object): def extend_validation_errors(self, validation_errors): self.validation_errors.extend(validation_errors) - def mark_deleted(self, include_children=True): + def mark_deleted(self): self.deleted = True - if include_children: - for child in self.children: - child.mark_deleted() - def mark_undeleted(self, include_children=True): + def mark_undeleted(self): self.deleted = False - if include_children: - for child in self.children: - child.mark_undeleted() - def mark_unhidden(self, include_children=True): + def mark_unhidden(self): self.visible = True - if include_children: - for child in self.children: - child.mark_unhidden() def undeletable(self): if self.purged: @@ -2390,7 +2375,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot self.copied_from_history_dataset_association = copied_from_history_dataset_association self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association - def copy(self, copy_children=False, parent_id=None): + def copy(self, parent_id=None): """ Create a copy of this HDA. """ @@ -2415,9 +2400,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot hda.set_size() # Need to set after flushed, as MetadataFiles require dataset.id hda.metadata = self.metadata - if copy_children: - for child in self.children: - child.copy(copy_children=copy_children, parent_id=hda.id) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs hda.set_peek() @@ -2481,12 +2463,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot library_dataset.library_dataset_dataset_association_id = ldda.id object_session(self).add(library_dataset) object_session(self).flush() - for child in self.children: - child.to_library_dataset_dataset_association(trans, - target_folder=target_folder, - replace_dataset=replace_dataset, - parent_id=ldda.id, - user=ldda.user) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs ldda.set_peek() @@ -2520,7 +2496,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot # Anon users are handled just by their single history size. if not user: return rval - # Gets an HDA and its children's disk usage, if the user does not already + # Gets an HDA disk usage, if the user does not already # have an association of the same dataset if not self.dataset.library_associations and not self.purged and not self.dataset.purged: for hda in self.dataset.history_associations: @@ -2530,8 +2506,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot break else: rval += self.get_total_size() - for child in self.children: - rval += child.get_disk_usage(user) return rval def to_dict(self, view='collection', expose_dataset_path=False): @@ -2946,14 +2920,12 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName): hda.metadata = self.metadata # need to set after flushed, as MetadataFiles require dataset.id if add_to_history and target_history: target_history.add_dataset(hda) - for child in self.children: - child.to_history_dataset_association(target_history=target_history, parent_id=hda.id, add_to_history=False) if not self.datatype.copy_safe_peek: hda.set_peek() # in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs sa_session.flush() return hda - def copy(self, copy_children=False, parent_id=None, target_folder=None): + def copy(self, parent_id=None, target_folder=None): sa_session = object_session(self) ldda = LibraryDatasetDatasetAssociation(name=self.name, info=self.info, @@ -2977,9 +2949,6 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName): sa_session.flush() # Need to set after flushed, as MetadataFiles require dataset.id ldda.metadata = self.metadata - if copy_children: - for child in self.children: - child.copy(copy_children=copy_children, parent_id=ldda.id) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs ldda.set_peek() @@ -3591,7 +3560,7 @@ class DatasetCollectionElement(object, Dictifiable): element_destination=element_destination ) else: - new_element_object = element_object.copy(copy_children=True) + new_element_object = element_object.copy() if destination is not None and element_object.hidden_beneath_collection_instance: new_element_object.hidden_beneath_collection_instance = destination # Ideally we would not need to give the following diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 95b5ebdf329..b9b8c184265 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1605,18 +1605,6 @@ simple_mapping(model.HistoryDatasetAssociation, implicitly_converted_parent_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.hda_id == model.HistoryDatasetAssociation.table.c.id)), - children=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.HistoryDatasetAssociation.table.c.parent_id == - model.HistoryDatasetAssociation.table.c.id), - backref=backref("parent", - primaryjoin=(model.HistoryDatasetAssociation.table.c.parent_id == - model.HistoryDatasetAssociation.table.c.id), - remote_side=[model.HistoryDatasetAssociation.table.c.id], uselist=False)), - visible_children=relation(model.HistoryDatasetAssociation, - primaryjoin=( - (model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id) & - (model.HistoryDatasetAssociation.table.c.visible == true())), - remote_side=[model.HistoryDatasetAssociation.table.c.id]), tags=relation(model.HistoryDatasetAssociationTagAssociation, order_by=model.HistoryDatasetAssociationTagAssociation.table.c.id, backref='history_tag_associations'), @@ -1987,19 +1975,6 @@ mapper(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssoci implicitly_converted_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.ldda_parent_id == model.LibraryDatasetDatasetAssociation.table.c.id)), - children=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.parent_id == - model.LibraryDatasetDatasetAssociation.table.c.id), - backref=backref("parent", - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.parent_id == - model.LibraryDatasetDatasetAssociation.table.c.id), - remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id])), - visible_children=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=( - (model.LibraryDatasetDatasetAssociation.table.c.parent_id == model.LibraryDatasetDatasetAssociation.table.c.id) & - (model.LibraryDatasetDatasetAssociation.table.c.visible == true()) - ), - remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id]), tags=relation(model.LibraryDatasetDatasetAssociationTagAssociation, order_by=model.LibraryDatasetDatasetAssociationTagAssociation.table.c.id, backref='history_tag_associations'), diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 722a9dc8043..efe64712c45 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1,7 +1,6 @@ """ Classes encapsulating galaxy tools and tool configuration. """ -import glob import json import logging import os @@ -1560,73 +1559,6 @@ class Tool(object, Dictifiable): """ pass - def collect_child_datasets(self, output, job_working_directory): - """ - Look for child dataset files, create HDA and attach to parent. - """ - children = {} - # Loop through output file names, looking for generated children in - # form of 'child_parentId_designation_visibility_extension' - for name, outdata in output.items(): - filenames = [] - if 'new_file_path' in self.app.config.collect_outputs_from: - filenames.extend(glob.glob(os.path.join(self.app.config.new_file_path, "child_%i_*" % outdata.id))) - if 'job_working_directory' in self.app.config.collect_outputs_from: - filenames.extend(glob.glob(os.path.join(job_working_directory, "child_%i_*" % outdata.id))) - for filename in filenames: - if name not in children: - children[name] = {} - fields = os.path.basename(filename).split("_") - designation = fields[2] - visible = fields[3].lower() - if visible == "visible": - visible = True - else: - visible = False - ext = fields[4].lower() - child_dataset = self.app.model.HistoryDatasetAssociation(extension=ext, - parent_id=outdata.id, - designation=designation, - visible=visible, - dbkey=outdata.dbkey, - create_dataset=True, - sa_session=self.sa_session) - self.app.security_agent.copy_dataset_permissions(outdata.dataset, child_dataset.dataset) - # Move data from temp location to dataset location - self.app.object_store.update_from_file(child_dataset.dataset, file_name=filename, create=True) - self.sa_session.add(child_dataset) - self.sa_session.flush() - child_dataset.set_size() - child_dataset.name = "Secondary Dataset (%s)" % (designation) - child_dataset.init_meta() - child_dataset.set_meta() - child_dataset.set_peek() - # Associate new dataset with job - job = None - for assoc in outdata.creating_job_associations: - job = assoc.job - break - if job: - assoc = self.app.model.JobToOutputDatasetAssociation('__new_child_file_%s|%s__' % (name, designation), child_dataset) - assoc.job = job - self.sa_session.add(assoc) - self.sa_session.flush() - child_dataset.state = outdata.state - self.sa_session.add(child_dataset) - self.sa_session.flush() - # Add child to return dict - children[name][designation] = child_dataset - # Need to update all associated output hdas, i.e. history was - # shared with job running - for dataset in outdata.dataset.history_associations: - if outdata == dataset: - continue - # Create new child dataset - child_data = child_dataset.copy(parent_id=dataset.id) - self.sa_session.add(child_data) - self.sa_session.flush() - return children - def collect_primary_datasets(self, output, tool_provided_metadata, job_working_directory, input_ext, input_dbkey="?"): """ Find any additional datasets generated by a tool and attach (for diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 582f20b019a..5cd92aa913f 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -91,7 +91,6 @@ class ToolEvaluator(object): self.dataset = dataset self.file_name = dataset.file_name self.metadata = dict() - self.children = [] special = get_special() if special: @@ -303,9 +302,6 @@ class ToolEvaluator(object): dataset_path = input_dataset_paths[real_path] wrapper_kwds['dataset_path'] = dataset_path param_dict[name] = DatasetFilenameWrapper(data, **wrapper_kwds) - if data: - for child in data.children: - param_dict["_CHILD___%s___%s" % (name, child.designation)] = DatasetFilenameWrapper(child) def __populate_output_collection_wrappers(self, param_dict, output_collections, output_paths, job_working_directory): output_dataset_paths = dataset_path_rewrites(output_paths) @@ -355,8 +351,6 @@ class ToolEvaluator(object): # Provide access to a path to store additional files # TODO: path munging for cluster/dataset server relocatability param_dict[name].files_path = os.path.abspath(os.path.join(job_working_directory, "dataset_%s_files" % (hda.dataset.id))) - for child in hda.children: - param_dict["_CHILD___%s___%s" % (name, child.designation)] = DatasetFilenameWrapper(child) for out_name, output in self.tool.outputs.items(): if out_name not in param_dict and output.filters: # Assume the reason we lack this output is because a filter diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 94f7c5d275d..962aa423e6b 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -6,7 +6,7 @@ import shutil import tempfile from json import dumps, loads -from sqlalchemy.orm import eagerload, eagerload_all +from sqlalchemy.orm import eagerload_all from sqlalchemy.sql import expression from galaxy import model @@ -322,7 +322,6 @@ class JobExportHistoryArchiveWrapper(object, UsesAnnotations): """ query = (trans.sa_session.query(trans.model.HistoryDatasetAssociation) .filter(trans.model.HistoryDatasetAssociation.history == history) - .options(eagerload("children")) .join("dataset") .options(eagerload_all("dataset.actions")) .order_by(trans.model.HistoryDatasetAssociation.hid) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 74e32c5254d..1627d6684f3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -1238,7 +1238,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE else: for hist in target_histories: if content.history_content_type == "dataset": - hist.add_dataset(content.copy(copy_children=True)) + hist.add_dataset(content.copy()) else: copy_collected_datasets = True copy_kwds = {} @@ -1296,7 +1296,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE invalid_datasets += 1 else: for hist in target_histories: - dataset_copy = data.copy(copy_children=True) + dataset_copy = data.copy() if imported: dataset_copy.name = "imported: " + dataset_copy.name hist.add_dataset(dataset_copy) diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index cf7ff9fc786..b943e26459f 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -495,7 +495,6 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt trans.sa_session.expunge(trans.history) history = trans.sa_session.query(model.History).options( eagerload_all('active_datasets.creating_job_associations.job.workflow_invocation_step.workflow_invocation.workflow'), - eagerload_all('active_datasets.children') ).get(id) assert history # TODO: formalize to trans.show_error diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 2d95c7d4dea..7cf6ae4689e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -221,25 +221,6 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): else: return "No dataset with id '%s'" % str(id) - @web.expose - def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"): - """Returns child data directly into the browser, based upon parent_id and designation. - """ - # TODO: unencoded id - try: - data = trans.sa_session.query(self.app.model.HistoryDatasetAssociation).get(parent_id) - if data: - child = data.get_child_by_designation(designation) - if child: - current_user_roles = trans.get_current_user_roles() - if trans.app.security_agent.can_access_dataset(current_user_roles, child): - return self.display(trans, id=child.id, tofile=tofile, toext=toext) - else: - return "You are not privileged to access this dataset." - except Exception: - pass - return "A child named %s could not be found for data %s" % (designation, parent_id) - @web.expose def display_as(self, trans, id=None, display_app=None, **kwd): """Returns a file in a format that can successfully be displayed in display_app. diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 300b2d6a61c..d5fcbf641cf 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -362,7 +362,7 @@ class InputModule(WorkflowModule): for input_dataset_hda in list(step_outputs.values()): content_type = input_dataset_hda.history_content_type if content_type == "dataset": - new_hda = input_dataset_hda.copy(copy_children=True) + new_hda = input_dataset_hda.copy() invocation.history.add_dataset(new_hda) step_outputs['input_ds_copy'] = new_hda elif content_type == "dataset_collection":