opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Simplify import/export history.

Browse Source
This commit is contained in:
John Chilton
2019-03-24 13:53:20 -04:00
parent 5bc8235aa2
commit b32beb1653
3 changed files with 63 additions and 54 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tools/imp_exp/__init__.py
+38 -31
View File
@@ -14,7 +14,7 @@ from galaxy import model
from galaxy.exceptions import MalformedContents
from galaxy.exceptions import ObjectNotFound
from galaxy.model.item_attrs import add_item_annotation, get_item_annotation_str
from galaxy.util import unicodify
from galaxy.util import in_directory, unicodify
from galaxy.version import VERSION_MAJOR
log = logging.getLogger(__name__)
@@ -38,11 +38,6 @@ class JobImportHistoryArchiveWrapper:
def cleanup_after_job(self):
""" Set history, datasets, and jobs' attributes and clean up archive directory. """
def file_in_dir(file_path, a_dir):
""" Returns true if file is in directory. """
abs_file_path = os.path.abspath(file_path)
return os.path.split(abs_file_path)[0] == a_dir
#
# Import history.
#
@@ -119,23 +114,33 @@ class JobImportHistoryArchiveWrapper:
sa_session=self.sa_session)
if 'uuid' in dataset_attrs:
hda.dataset.uuid = dataset_attrs["uuid"]
if dataset_attrs.get('exported', True) is False:
hda.state = hda.states.DISCARDED
hda.deleted = True
hda.purged = True
else:
hda.state = hda.states.OK
self.sa_session.add(hda)
self.sa_session.flush()
new_history.add_dataset(hda, genome_build=None)
hda.hid = dataset_attrs['hid'] # Overwrite default hid set when HDA added to history.
self.sa_session.flush()
if dataset_attrs.get('exported', True) is True:
file_name = dataset_attrs.get('file_name')
if file_name:
# Do security check and move/copy dataset data.
archive_path = os.path.abspath(os.path.join(archive_dir, file_name))
if os.path.islink(archive_path):
raise MalformedContents("Invalid dataset path: %s" % archive_path)
temp_dataset_file_name = \
os.path.realpath(os.path.abspath(os.path.join(archive_dir, dataset_attrs['file_name'])))
if not file_in_dir(temp_dataset_file_name, os.path.join(archive_dir, "datasets")):
os.path.realpath(archive_path)
if not in_directory(temp_dataset_file_name, archive_dir):
raise MalformedContents("Invalid dataset path: %s" % temp_dataset_file_name)
if not file_name or not os.path.exists(temp_dataset_file_name):
hda.state = hda.states.DISCARDED
hda.deleted = True
hda.purged = True
hda.dataset.deleted = True
hda.dataset.purged = True
else:
hda.state = hda.states.OK
self.app.object_store.update_from_file(hda.dataset, file_name=temp_dataset_file_name, create=True)
# Import additional files if present. Histories exported previously might not have this attribute set.
@@ -154,6 +159,9 @@ class JobImportHistoryArchiveWrapper:
create=True)
hda.dataset.set_total_size() # update the filesize record in the database
if hda.deleted:
hda.dataset.deleted = True
if user:
add_item_annotation(self.sa_session, user, hda, dataset_attrs['annotation'])
# TODO: Set tags.
@@ -298,6 +306,8 @@ class JobExportHistoryArchiveWrapper:
class HistoryDatasetAssociationEncoder(json.JSONEncoder):
""" Custom JSONEncoder for a HistoryDatasetAssociation. """
include_files = True
def default(self, obj):
""" Encode an HDA, default encoding for everything else. """
if isinstance(obj, model.HistoryDatasetAssociation):
@@ -320,26 +330,23 @@ class JobExportHistoryArchiveWrapper:
"annotation": unicodify(getattr(obj, 'annotation', '')),
"tags": obj.make_tag_string_list()
}
if self.include_files:
try:
rval['file_name'] = obj.file_name
except ObjectNotFound:
rval['file_name'] = None
try:
rval['file_name'] = obj.file_name
except ObjectNotFound:
rval['file_name'] = None
if obj.extra_files_path_exists():
rval['extra_files_path'] = obj.extra_files_path
else:
rval['extra_files_path'] = None
if obj.extra_files_path_exists():
rval['extra_files_path'] = obj.extra_files_path
else:
rval['extra_files_path'] = None
if not obj.visible and not include_hidden:
rval['exported'] = False
elif obj.deleted and not include_deleted:
rval['exported'] = False
else:
rval['exported'] = True
return rval
return json.JSONEncoder.default(self, obj)
class NoFilesHistoryDatasetAssociationEncoder(HistoryDatasetAssociationEncoder):
include_files = False
#
# Create attributes/metadata files for export.
#
@@ -383,7 +390,7 @@ class JobExportHistoryArchiveWrapper:
dump(datasets_attrs, datasets_attrs_out, cls=HistoryDatasetAssociationEncoder)
with open(datasets_attrs_filename + ".provenance", 'w') as provenance_attrs_out:
dump(provenance_attrs, provenance_attrs_out, cls=HistoryDatasetAssociationEncoder)
dump(provenance_attrs, provenance_attrs_out, cls=NoFilesHistoryDatasetAssociationEncoder)
#
# Write jobs attributes file.
lib/galaxy/tools/imp_exp/export_history.py
+22 -23
View File
@@ -49,32 +49,31 @@ def create_archive(history_attrs_file, datasets_attrs_file, jobs_attrs_file, out
# Add datasets to archive and update dataset attributes.
# TODO: security check to ensure that files added are in Galaxy dataset directory?
for dataset_attrs in datasets_attrs:
if dataset_attrs['exported']:
dataset_file_name = dataset_attrs['file_name'] # Full file name.
dataset_hid = dataset_attrs['hid']
dataset_archive_name = os.path.join('datasets',
get_dataset_filename(dataset_attrs['name'], dataset_attrs['extension'], dataset_hid))
history_archive.add(dataset_file_name, arcname=dataset_archive_name)
dataset_file_name = dataset_attrs['file_name'] # Full file name.
dataset_hid = dataset_attrs['hid']
dataset_archive_name = os.path.join('datasets',
get_dataset_filename(dataset_attrs['name'], dataset_attrs['extension'], dataset_hid))
history_archive.add(dataset_file_name, arcname=dataset_archive_name)
# Include additional files for example, files/images included in HTML output.
extra_files_path = dataset_attrs['extra_files_path']
if extra_files_path:
try:
file_list = os.listdir(extra_files_path)
except OSError:
file_list = []
# Include additional files for example, files/images included in HTML output.
extra_files_path = dataset_attrs['extra_files_path']
if extra_files_path:
try:
file_list = os.listdir(extra_files_path)
except OSError:
file_list = []
if len(file_list):
dataset_extra_files_path = 'datasets/extra_files_path_%s' % dataset_hid
for fname in file_list:
history_archive.add(os.path.join(extra_files_path, fname),
arcname=(os.path.join(dataset_extra_files_path, fname)))
dataset_attrs['extra_files_path'] = dataset_extra_files_path
else:
dataset_attrs['extra_files_path'] = ''
if len(file_list):
dataset_extra_files_path = 'datasets/extra_files_path_%s' % dataset_hid
for fname in file_list:
history_archive.add(os.path.join(extra_files_path, fname),
arcname=(os.path.join(dataset_extra_files_path, fname)))
dataset_attrs['extra_files_path'] = dataset_extra_files_path
else:
dataset_attrs['extra_files_path'] = ''
# Update dataset filename to be archive name.
dataset_attrs['file_name'] = dataset_archive_name
# Update dataset filename to be archive name.
dataset_attrs['file_name'] = dataset_archive_name
# Rewrite dataset attributes file.
with open(datasets_attrs_file, 'w') as datasets_attrs_out:
test/api/test_histories.py
+3
View File
@@ -184,12 +184,15 @@ class HistoriesApiTestCase(api.ApiTestCase):
self.dataset_populator.new_dataset(history_id, content="1 2 3")
deleted_hda = self.dataset_populator.new_dataset(history_id, content="1 2 3")
self.dataset_populator.delete_dataset(history_id, deleted_hda["id"])
deleted_details = self.dataset_populator.get_history_dataset_details(history_id, id=deleted_hda["id"])
assert deleted_details["deleted"]
imported_history_id = self._reimport_history(history_id, history_name, wait_on_history_length=2)
def upload_job_check(job):
assert job["tool_id"] == "upload1"
def check_discarded(hda):
assert hda["deleted"]
assert hda["state"] == "discarded", hda
assert hda["purged"] is True