diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 08457df9aa2..13378fb6605 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1388,7 +1388,6 @@ class JobWrapper(object, HasResourceParameters): # Maybe this is a legacy job, use the job working directory instead tool_working_directory = self.working_directory collected_datasets = { - 'children': self.tool.collect_child_datasets(out_data, tool_working_directory), 'primary': self.tool.collect_primary_datasets(out_data, self.get_tool_provided_job_metadata(), tool_working_directory, input_ext, input_dbkey) } self.tool.collect_dynamic_collections( diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 1c287f0dc2a..ecfde77474e 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1351,7 +1351,7 @@ class History(HasTags, Dictifiable, UsesAnnotations, HasName): hdas = self.active_datasets for hda in hdas: # Copy HDA. - new_hda = hda.copy(copy_children=True) + new_hda = hda.copy() new_history.add_dataset(new_hda, set_hid=False, quota=applies_to_quota) db_session.add(new_hda) db_session.flush() @@ -1856,7 +1856,7 @@ class Dataset(StorableObject): """Detects whether there is any data""" return self.get_size() > 0 - def mark_deleted(self, include_children=True): + def mark_deleted(self): self.deleted = True def is_multi_byte(self): @@ -2189,12 +2189,6 @@ class DatasetInstance(object): def clear_associated_files(self, metadata_safe=False, purge=False): raise Exception("Unimplemented") - def get_child_by_designation(self, designation): - for child in self.children: - if child.designation == designation: - return child - return None - def get_converter_types(self): return self.datatype.get_converter_types(self, _get_datatypes_registry()) @@ -2211,23 +2205,14 @@ class DatasetInstance(object): def extend_validation_errors(self, validation_errors): self.validation_errors.extend(validation_errors) - def mark_deleted(self, include_children=True): + def mark_deleted(self): self.deleted = True - if include_children: - for child in self.children: - child.mark_deleted() - def mark_undeleted(self, include_children=True): + def mark_undeleted(self): self.deleted = False - if include_children: - for child in self.children: - child.mark_undeleted() - def mark_unhidden(self, include_children=True): + def mark_unhidden(self): self.visible = True - if include_children: - for child in self.children: - child.mark_unhidden() def undeletable(self): if self.purged: @@ -2390,7 +2375,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot self.copied_from_history_dataset_association = copied_from_history_dataset_association self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association - def copy(self, copy_children=False, parent_id=None): + def copy(self, parent_id=None): """ Create a copy of this HDA. """ @@ -2415,9 +2400,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot hda.set_size() # Need to set after flushed, as MetadataFiles require dataset.id hda.metadata = self.metadata - if copy_children: - for child in self.children: - child.copy(copy_children=copy_children, parent_id=hda.id) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs hda.set_peek() @@ -2481,12 +2463,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot library_dataset.library_dataset_dataset_association_id = ldda.id object_session(self).add(library_dataset) object_session(self).flush() - for child in self.children: - child.to_library_dataset_dataset_association(trans, - target_folder=target_folder, - replace_dataset=replace_dataset, - parent_id=ldda.id, - user=ldda.user) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs ldda.set_peek() @@ -2520,7 +2496,7 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot # Anon users are handled just by their single history size. if not user: return rval - # Gets an HDA and its children's disk usage, if the user does not already + # Gets an HDA disk usage, if the user does not already # have an association of the same dataset if not self.dataset.library_associations and not self.purged and not self.dataset.purged: for hda in self.dataset.history_associations: @@ -2530,8 +2506,6 @@ class HistoryDatasetAssociation(DatasetInstance, HasTags, Dictifiable, UsesAnnot break else: rval += self.get_total_size() - for child in self.children: - rval += child.get_disk_usage(user) return rval def to_dict(self, view='collection', expose_dataset_path=False): @@ -2946,14 +2920,12 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName): hda.metadata = self.metadata # need to set after flushed, as MetadataFiles require dataset.id if add_to_history and target_history: target_history.add_dataset(hda) - for child in self.children: - child.to_history_dataset_association(target_history=target_history, parent_id=hda.id, add_to_history=False) if not self.datatype.copy_safe_peek: hda.set_peek() # in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs sa_session.flush() return hda - def copy(self, copy_children=False, parent_id=None, target_folder=None): + def copy(self, parent_id=None, target_folder=None): sa_session = object_session(self) ldda = LibraryDatasetDatasetAssociation(name=self.name, info=self.info, @@ -2977,9 +2949,6 @@ class LibraryDatasetDatasetAssociation(DatasetInstance, HasName): sa_session.flush() # Need to set after flushed, as MetadataFiles require dataset.id ldda.metadata = self.metadata - if copy_children: - for child in self.children: - child.copy(copy_children=copy_children, parent_id=ldda.id) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs ldda.set_peek() @@ -3591,7 +3560,7 @@ class DatasetCollectionElement(object, Dictifiable): element_destination=element_destination ) else: - new_element_object = element_object.copy(copy_children=True) + new_element_object = element_object.copy() if destination is not None and element_object.hidden_beneath_collection_instance: new_element_object.hidden_beneath_collection_instance = destination # Ideally we would not need to give the following diff --git a/lib/galaxy/model/mapping.py b/lib/galaxy/model/mapping.py index 95b5ebdf329..b9b8c184265 100644 --- a/lib/galaxy/model/mapping.py +++ b/lib/galaxy/model/mapping.py @@ -1605,18 +1605,6 @@ simple_mapping(model.HistoryDatasetAssociation, implicitly_converted_parent_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.hda_id == model.HistoryDatasetAssociation.table.c.id)), - children=relation(model.HistoryDatasetAssociation, - primaryjoin=(model.HistoryDatasetAssociation.table.c.parent_id == - model.HistoryDatasetAssociation.table.c.id), - backref=backref("parent", - primaryjoin=(model.HistoryDatasetAssociation.table.c.parent_id == - model.HistoryDatasetAssociation.table.c.id), - remote_side=[model.HistoryDatasetAssociation.table.c.id], uselist=False)), - visible_children=relation(model.HistoryDatasetAssociation, - primaryjoin=( - (model.HistoryDatasetAssociation.table.c.parent_id == model.HistoryDatasetAssociation.table.c.id) & - (model.HistoryDatasetAssociation.table.c.visible == true())), - remote_side=[model.HistoryDatasetAssociation.table.c.id]), tags=relation(model.HistoryDatasetAssociationTagAssociation, order_by=model.HistoryDatasetAssociationTagAssociation.table.c.id, backref='history_tag_associations'), @@ -1987,19 +1975,6 @@ mapper(model.LibraryDatasetDatasetAssociation, model.LibraryDatasetDatasetAssoci implicitly_converted_datasets=relation(model.ImplicitlyConvertedDatasetAssociation, primaryjoin=(model.ImplicitlyConvertedDatasetAssociation.table.c.ldda_parent_id == model.LibraryDatasetDatasetAssociation.table.c.id)), - children=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.parent_id == - model.LibraryDatasetDatasetAssociation.table.c.id), - backref=backref("parent", - primaryjoin=(model.LibraryDatasetDatasetAssociation.table.c.parent_id == - model.LibraryDatasetDatasetAssociation.table.c.id), - remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id])), - visible_children=relation(model.LibraryDatasetDatasetAssociation, - primaryjoin=( - (model.LibraryDatasetDatasetAssociation.table.c.parent_id == model.LibraryDatasetDatasetAssociation.table.c.id) & - (model.LibraryDatasetDatasetAssociation.table.c.visible == true()) - ), - remote_side=[model.LibraryDatasetDatasetAssociation.table.c.id]), tags=relation(model.LibraryDatasetDatasetAssociationTagAssociation, order_by=model.LibraryDatasetDatasetAssociationTagAssociation.table.c.id, backref='history_tag_associations'), diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index 722a9dc8043..efe64712c45 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -1,7 +1,6 @@ """ Classes encapsulating galaxy tools and tool configuration. """ -import glob import json import logging import os @@ -1560,73 +1559,6 @@ class Tool(object, Dictifiable): """ pass - def collect_child_datasets(self, output, job_working_directory): - """ - Look for child dataset files, create HDA and attach to parent. - """ - children = {} - # Loop through output file names, looking for generated children in - # form of 'child_parentId_designation_visibility_extension' - for name, outdata in output.items(): - filenames = [] - if 'new_file_path' in self.app.config.collect_outputs_from: - filenames.extend(glob.glob(os.path.join(self.app.config.new_file_path, "child_%i_*" % outdata.id))) - if 'job_working_directory' in self.app.config.collect_outputs_from: - filenames.extend(glob.glob(os.path.join(job_working_directory, "child_%i_*" % outdata.id))) - for filename in filenames: - if name not in children: - children[name] = {} - fields = os.path.basename(filename).split("_") - designation = fields[2] - visible = fields[3].lower() - if visible == "visible": - visible = True - else: - visible = False - ext = fields[4].lower() - child_dataset = self.app.model.HistoryDatasetAssociation(extension=ext, - parent_id=outdata.id, - designation=designation, - visible=visible, - dbkey=outdata.dbkey, - create_dataset=True, - sa_session=self.sa_session) - self.app.security_agent.copy_dataset_permissions(outdata.dataset, child_dataset.dataset) - # Move data from temp location to dataset location - self.app.object_store.update_from_file(child_dataset.dataset, file_name=filename, create=True) - self.sa_session.add(child_dataset) - self.sa_session.flush() - child_dataset.set_size() - child_dataset.name = "Secondary Dataset (%s)" % (designation) - child_dataset.init_meta() - child_dataset.set_meta() - child_dataset.set_peek() - # Associate new dataset with job - job = None - for assoc in outdata.creating_job_associations: - job = assoc.job - break - if job: - assoc = self.app.model.JobToOutputDatasetAssociation('__new_child_file_%s|%s__' % (name, designation), child_dataset) - assoc.job = job - self.sa_session.add(assoc) - self.sa_session.flush() - child_dataset.state = outdata.state - self.sa_session.add(child_dataset) - self.sa_session.flush() - # Add child to return dict - children[name][designation] = child_dataset - # Need to update all associated output hdas, i.e. history was - # shared with job running - for dataset in outdata.dataset.history_associations: - if outdata == dataset: - continue - # Create new child dataset - child_data = child_dataset.copy(parent_id=dataset.id) - self.sa_session.add(child_data) - self.sa_session.flush() - return children - def collect_primary_datasets(self, output, tool_provided_metadata, job_working_directory, input_ext, input_dbkey="?"): """ Find any additional datasets generated by a tool and attach (for diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index 582f20b019a..5cd92aa913f 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -91,7 +91,6 @@ class ToolEvaluator(object): self.dataset = dataset self.file_name = dataset.file_name self.metadata = dict() - self.children = [] special = get_special() if special: @@ -303,9 +302,6 @@ class ToolEvaluator(object): dataset_path = input_dataset_paths[real_path] wrapper_kwds['dataset_path'] = dataset_path param_dict[name] = DatasetFilenameWrapper(data, **wrapper_kwds) - if data: - for child in data.children: - param_dict["_CHILD___%s___%s" % (name, child.designation)] = DatasetFilenameWrapper(child) def __populate_output_collection_wrappers(self, param_dict, output_collections, output_paths, job_working_directory): output_dataset_paths = dataset_path_rewrites(output_paths) @@ -355,8 +351,6 @@ class ToolEvaluator(object): # Provide access to a path to store additional files # TODO: path munging for cluster/dataset server relocatability param_dict[name].files_path = os.path.abspath(os.path.join(job_working_directory, "dataset_%s_files" % (hda.dataset.id))) - for child in hda.children: - param_dict["_CHILD___%s___%s" % (name, child.designation)] = DatasetFilenameWrapper(child) for out_name, output in self.tool.outputs.items(): if out_name not in param_dict and output.filters: # Assume the reason we lack this output is because a filter diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 94f7c5d275d..962aa423e6b 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -6,7 +6,7 @@ import shutil import tempfile from json import dumps, loads -from sqlalchemy.orm import eagerload, eagerload_all +from sqlalchemy.orm import eagerload_all from sqlalchemy.sql import expression from galaxy import model @@ -322,7 +322,6 @@ class JobExportHistoryArchiveWrapper(object, UsesAnnotations): """ query = (trans.sa_session.query(trans.model.HistoryDatasetAssociation) .filter(trans.model.HistoryDatasetAssociation.history == history) - .options(eagerload("children")) .join("dataset") .options(eagerload_all("dataset.actions")) .order_by(trans.model.HistoryDatasetAssociation.hid) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index 74e32c5254d..1627d6684f3 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -1238,7 +1238,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE else: for hist in target_histories: if content.history_content_type == "dataset": - hist.add_dataset(content.copy(copy_children=True)) + hist.add_dataset(content.copy()) else: copy_collected_datasets = True copy_kwds = {} @@ -1296,7 +1296,7 @@ class DatasetInterface(BaseUIController, UsesAnnotations, UsesItemRatings, UsesE invalid_datasets += 1 else: for hist in target_histories: - dataset_copy = data.copy(copy_children=True) + dataset_copy = data.copy() if imported: dataset_copy.name = "imported: " + dataset_copy.name hist.add_dataset(dataset_copy) diff --git a/lib/galaxy/webapps/galaxy/controllers/history.py b/lib/galaxy/webapps/galaxy/controllers/history.py index cf7ff9fc786..b943e26459f 100644 --- a/lib/galaxy/webapps/galaxy/controllers/history.py +++ b/lib/galaxy/webapps/galaxy/controllers/history.py @@ -495,7 +495,6 @@ class HistoryController(BaseUIController, SharableMixin, UsesAnnotations, UsesIt trans.sa_session.expunge(trans.history) history = trans.sa_session.query(model.History).options( eagerload_all('active_datasets.creating_job_associations.job.workflow_invocation_step.workflow_invocation.workflow'), - eagerload_all('active_datasets.children') ).get(id) assert history # TODO: formalize to trans.show_error diff --git a/lib/galaxy/webapps/galaxy/controllers/root.py b/lib/galaxy/webapps/galaxy/controllers/root.py index 2d95c7d4dea..7cf6ae4689e 100644 --- a/lib/galaxy/webapps/galaxy/controllers/root.py +++ b/lib/galaxy/webapps/galaxy/controllers/root.py @@ -221,25 +221,6 @@ class RootController(controller.JSAppLauncher, UsesAnnotations): else: return "No dataset with id '%s'" % str(id) - @web.expose - def display_child(self, trans, parent_id=None, designation=None, tofile=None, toext=".txt"): - """Returns child data directly into the browser, based upon parent_id and designation. - """ - # TODO: unencoded id - try: - data = trans.sa_session.query(self.app.model.HistoryDatasetAssociation).get(parent_id) - if data: - child = data.get_child_by_designation(designation) - if child: - current_user_roles = trans.get_current_user_roles() - if trans.app.security_agent.can_access_dataset(current_user_roles, child): - return self.display(trans, id=child.id, tofile=tofile, toext=toext) - else: - return "You are not privileged to access this dataset." - except Exception: - pass - return "A child named %s could not be found for data %s" % (designation, parent_id) - @web.expose def display_as(self, trans, id=None, display_app=None, **kwd): """Returns a file in a format that can successfully be displayed in display_app. diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index 300b2d6a61c..d5fcbf641cf 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -362,7 +362,7 @@ class InputModule(WorkflowModule): for input_dataset_hda in list(step_outputs.values()): content_type = input_dataset_hda.history_content_type if content_type == "dataset": - new_hda = input_dataset_hda.copy(copy_children=True) + new_hda = input_dataset_hda.copy() invocation.history.add_dataset(new_hda) step_outputs['input_ds_copy'] = new_hda elif content_type == "dataset_collection":