diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index 8a4f7819644..d196e73d138 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -14,7 +14,7 @@ from galaxy import model from galaxy.exceptions import MalformedContents from galaxy.exceptions import ObjectNotFound from galaxy.model.item_attrs import add_item_annotation, get_item_annotation_str -from galaxy.util import unicodify +from galaxy.util import in_directory, unicodify from galaxy.version import VERSION_MAJOR log = logging.getLogger(__name__) @@ -38,11 +38,6 @@ class JobImportHistoryArchiveWrapper: def cleanup_after_job(self): """ Set history, datasets, and jobs' attributes and clean up archive directory. """ - def file_in_dir(file_path, a_dir): - """ Returns true if file is in directory. """ - abs_file_path = os.path.abspath(file_path) - return os.path.split(abs_file_path)[0] == a_dir - # # Import history. # @@ -119,23 +114,33 @@ class JobImportHistoryArchiveWrapper: sa_session=self.sa_session) if 'uuid' in dataset_attrs: hda.dataset.uuid = dataset_attrs["uuid"] - if dataset_attrs.get('exported', True) is False: - hda.state = hda.states.DISCARDED - hda.deleted = True - hda.purged = True - else: - hda.state = hda.states.OK self.sa_session.add(hda) self.sa_session.flush() new_history.add_dataset(hda, genome_build=None) hda.hid = dataset_attrs['hid'] # Overwrite default hid set when HDA added to history. self.sa_session.flush() - if dataset_attrs.get('exported', True) is True: + + file_name = dataset_attrs.get('file_name') + if file_name: # Do security check and move/copy dataset data. + archive_path = os.path.abspath(os.path.join(archive_dir, file_name)) + if os.path.islink(archive_path): + raise MalformedContents("Invalid dataset path: %s" % archive_path) + temp_dataset_file_name = \ - os.path.realpath(os.path.abspath(os.path.join(archive_dir, dataset_attrs['file_name']))) - if not file_in_dir(temp_dataset_file_name, os.path.join(archive_dir, "datasets")): + os.path.realpath(archive_path) + + if not in_directory(temp_dataset_file_name, archive_dir): raise MalformedContents("Invalid dataset path: %s" % temp_dataset_file_name) + + if not file_name or not os.path.exists(temp_dataset_file_name): + hda.state = hda.states.DISCARDED + hda.deleted = True + hda.purged = True + hda.dataset.deleted = True + hda.dataset.purged = True + else: + hda.state = hda.states.OK self.app.object_store.update_from_file(hda.dataset, file_name=temp_dataset_file_name, create=True) # Import additional files if present. Histories exported previously might not have this attribute set. @@ -154,6 +159,9 @@ class JobImportHistoryArchiveWrapper: create=True) hda.dataset.set_total_size() # update the filesize record in the database + if hda.deleted: + hda.dataset.deleted = True + if user: add_item_annotation(self.sa_session, user, hda, dataset_attrs['annotation']) # TODO: Set tags. @@ -298,6 +306,8 @@ class JobExportHistoryArchiveWrapper: class HistoryDatasetAssociationEncoder(json.JSONEncoder): """ Custom JSONEncoder for a HistoryDatasetAssociation. """ + include_files = True + def default(self, obj): """ Encode an HDA, default encoding for everything else. """ if isinstance(obj, model.HistoryDatasetAssociation): @@ -320,26 +330,23 @@ class JobExportHistoryArchiveWrapper: "annotation": unicodify(getattr(obj, 'annotation', '')), "tags": obj.make_tag_string_list() } + if self.include_files: + try: + rval['file_name'] = obj.file_name + except ObjectNotFound: + rval['file_name'] = None - try: - rval['file_name'] = obj.file_name - except ObjectNotFound: - rval['file_name'] = None + if obj.extra_files_path_exists(): + rval['extra_files_path'] = obj.extra_files_path + else: + rval['extra_files_path'] = None - if obj.extra_files_path_exists(): - rval['extra_files_path'] = obj.extra_files_path - else: - rval['extra_files_path'] = None - - if not obj.visible and not include_hidden: - rval['exported'] = False - elif obj.deleted and not include_deleted: - rval['exported'] = False - else: - rval['exported'] = True return rval return json.JSONEncoder.default(self, obj) + class NoFilesHistoryDatasetAssociationEncoder(HistoryDatasetAssociationEncoder): + include_files = False + # # Create attributes/metadata files for export. # @@ -383,7 +390,7 @@ class JobExportHistoryArchiveWrapper: dump(datasets_attrs, datasets_attrs_out, cls=HistoryDatasetAssociationEncoder) with open(datasets_attrs_filename + ".provenance", 'w') as provenance_attrs_out: - dump(provenance_attrs, provenance_attrs_out, cls=HistoryDatasetAssociationEncoder) + dump(provenance_attrs, provenance_attrs_out, cls=NoFilesHistoryDatasetAssociationEncoder) # # Write jobs attributes file. diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index ca7f9dff4e6..324524d5a04 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -49,32 +49,31 @@ def create_archive(history_attrs_file, datasets_attrs_file, jobs_attrs_file, out # Add datasets to archive and update dataset attributes. # TODO: security check to ensure that files added are in Galaxy dataset directory? for dataset_attrs in datasets_attrs: - if dataset_attrs['exported']: - dataset_file_name = dataset_attrs['file_name'] # Full file name. - dataset_hid = dataset_attrs['hid'] - dataset_archive_name = os.path.join('datasets', - get_dataset_filename(dataset_attrs['name'], dataset_attrs['extension'], dataset_hid)) - history_archive.add(dataset_file_name, arcname=dataset_archive_name) + dataset_file_name = dataset_attrs['file_name'] # Full file name. + dataset_hid = dataset_attrs['hid'] + dataset_archive_name = os.path.join('datasets', + get_dataset_filename(dataset_attrs['name'], dataset_attrs['extension'], dataset_hid)) + history_archive.add(dataset_file_name, arcname=dataset_archive_name) - # Include additional files for example, files/images included in HTML output. - extra_files_path = dataset_attrs['extra_files_path'] - if extra_files_path: - try: - file_list = os.listdir(extra_files_path) - except OSError: - file_list = [] + # Include additional files for example, files/images included in HTML output. + extra_files_path = dataset_attrs['extra_files_path'] + if extra_files_path: + try: + file_list = os.listdir(extra_files_path) + except OSError: + file_list = [] - if len(file_list): - dataset_extra_files_path = 'datasets/extra_files_path_%s' % dataset_hid - for fname in file_list: - history_archive.add(os.path.join(extra_files_path, fname), - arcname=(os.path.join(dataset_extra_files_path, fname))) - dataset_attrs['extra_files_path'] = dataset_extra_files_path - else: - dataset_attrs['extra_files_path'] = '' + if len(file_list): + dataset_extra_files_path = 'datasets/extra_files_path_%s' % dataset_hid + for fname in file_list: + history_archive.add(os.path.join(extra_files_path, fname), + arcname=(os.path.join(dataset_extra_files_path, fname))) + dataset_attrs['extra_files_path'] = dataset_extra_files_path + else: + dataset_attrs['extra_files_path'] = '' - # Update dataset filename to be archive name. - dataset_attrs['file_name'] = dataset_archive_name + # Update dataset filename to be archive name. + dataset_attrs['file_name'] = dataset_archive_name # Rewrite dataset attributes file. with open(datasets_attrs_file, 'w') as datasets_attrs_out: diff --git a/test/api/test_histories.py b/test/api/test_histories.py index fdc8eed3c6d..3985d273727 100644 --- a/test/api/test_histories.py +++ b/test/api/test_histories.py @@ -184,12 +184,15 @@ class HistoriesApiTestCase(api.ApiTestCase): self.dataset_populator.new_dataset(history_id, content="1 2 3") deleted_hda = self.dataset_populator.new_dataset(history_id, content="1 2 3") self.dataset_populator.delete_dataset(history_id, deleted_hda["id"]) + deleted_details = self.dataset_populator.get_history_dataset_details(history_id, id=deleted_hda["id"]) + assert deleted_details["deleted"] imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2) def upload_job_check(job): assert job["tool_id"] == "upload1" def check_discarded(hda): + assert hda["deleted"] assert hda["state"] == "discarded", hda assert hda["purged"] is True