diff --git a/lib/galaxy/tools/imp_exp/__init__.py b/lib/galaxy/tools/imp_exp/__init__.py index bfa7d6fd675..8a4f7819644 100644 --- a/lib/galaxy/tools/imp_exp/__init__.py +++ b/lib/galaxy/tools/imp_exp/__init__.py @@ -429,15 +429,15 @@ class JobExportHistoryArchiveWrapper: job_attrs['update_time'] = job.update_time.isoformat() # Get the job's parameters + params_dict = {} try: params_objects = job.get_param_values(self.app) + for name, value in params_objects.items(): + params_dict[name] = value except Exception: # Could not get job params. - continue + pass - params_dict = {} - for name, value in params_objects.items(): - params_dict[name] = value job_attrs['params'] = params_dict # -- Get input, output datasets. -- diff --git a/lib/galaxy/tools/imp_exp/export_history.py b/lib/galaxy/tools/imp_exp/export_history.py index cb35a888abe..ca7f9dff4e6 100644 --- a/lib/galaxy/tools/imp_exp/export_history.py +++ b/lib/galaxy/tools/imp_exp/export_history.py @@ -90,16 +90,18 @@ def create_archive(history_attrs_file, datasets_attrs_file, jobs_attrs_file, out # Status. print('Created history archive.') + return 0 except Exception as e: print('Error creating history archive: %s' % str(e), file=sys.stderr) + return 1 -def main(): +def main(argv=None): # Parse command line. parser = optparse.OptionParser() parser.add_option('-G', '--gzip', dest='gzip', action="store_true", help='Compress archive using gzip.') parser.add_option('--galaxy-version', dest='galaxy_version', help='Galaxy version that initiated the command.', default=None) - (options, args) = parser.parse_args() + (options, args) = parser.parse_args(argv) galaxy_version = options.galaxy_version if galaxy_version is None: galaxy_version = "19.01" if len(args) == 4 else "19.05" @@ -118,7 +120,7 @@ def main(): job_attrs = os.path.join(temp_directory, 'jobs_attrs.txt') # Create archive. - create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip) + return create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip) if __name__ == "__main__": diff --git a/test/unit/test_objectstore.py b/test/unit/test_objectstore.py index d8050ad17fa..afd8998e7bc 100644 --- a/test/unit/test_objectstore.py +++ b/test/unit/test_objectstore.py @@ -740,7 +740,7 @@ def test_config_parse_azure(): class TestConfig(object): - def __init__(self, config_str, clazz=None): + def __init__(self, config_str=DISK_TEST_CONFIG, clazz=None): self.temp_directory = mkdtemp() if config_str.startswith("<"): config_file = "store.xml" diff --git a/test/unit/tools/test_history_imp_exp.py b/test/unit/tools/test_history_imp_exp.py index fb8a4b13674..75a186f24ba 100644 --- a/test/unit/tools/test_history_imp_exp.py +++ b/test/unit/tools/test_history_imp_exp.py @@ -7,8 +7,9 @@ from tempfile import mkdtemp from galaxy import model from galaxy.exceptions import MalformedContents -from galaxy.tools.imp_exp import JobImportHistoryArchiveWrapper, unpack_tar_gz_archive +from galaxy.tools.imp_exp import JobExportHistoryArchiveWrapper, JobImportHistoryArchiveWrapper, unpack_tar_gz_archive from galaxy.tools.imp_exp.export_history import create_archive +from ..test_objectstore import TestConfig from ..unittest_utils.galaxy_mock import MockApp @@ -25,19 +26,26 @@ class MockSetExternalTool(object): pass -def _run_jihaw_cleanup(archive_dir): - app = MockApp() - app.datatypes_registry.set_external_metadata_tool = MockSetExternalTool() - +def _run_jihaw_cleanup(archive_dir, app=None): + app = app or _mock_app() job = model.Job() job.tool_stderr = '' jiha = model.JobImportHistoryArchive(job=job, archive_dir=archive_dir) app.model.context.current.add_all([job, jiha]) app.model.context.flush() - jihaw = JobImportHistoryArchiveWrapper(app, 1) # yeehaw! + jihaw = JobImportHistoryArchiveWrapper(app, job.id) # yeehaw! return app, jihaw.cleanup_after_job() +def _mock_app(): + app = MockApp() + test_object_store_config = TestConfig() + app.object_store = test_object_store_config.object_store + app.model.Dataset.object_store = app.object_store + app.datatypes_registry.set_external_metadata_tool = MockSetExternalTool() + return app + + def _run_jihaw_cleanup_check_secure(history_archive, msg): malformed = False try: @@ -101,6 +109,131 @@ def test_history_import_abspath_in_metadata(): _run_jihaw_cleanup_check_secure(history_archive, 'Absolute path in datasets_attrs.txt allowed') +def test_export_dataset(): + app, sa_session, h = _setup_history_for_export("Datasets History") + + d1, d2 = _create_datasets(sa_session, h, 2) + + j = model.Job() + j.user = h.user + j.tool_id = "cat1" + + j.add_input_dataset("input1", d1) + j.add_output_dataset("out_file1", d2) + + sa_session.add(d1) + sa_session.add(d2) + sa_session.add(h) + sa_session.add(j) + sa_session.flush() + + app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True) + app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True) + + imported_history = _import_export(app, h) + + datasets = list(imported_history.contents_iter(types=["dataset"])) + assert len(datasets) == 2 + imported_job = datasets[1].creating_job + assert imported_job + assert imported_job.output_datasets + assert imported_job.output_datasets[0].dataset == datasets[1] + + assert imported_job.input_datasets + assert imported_job.input_datasets[0].dataset == datasets[0] + + assert datasets[0].state == 'ok' + assert datasets[1].state == 'ok' + + with open(datasets[0].file_name, "r") as f: + assert f.read().startswith("chr1 4225 19670") + with open(datasets[1].file_name, "r") as f: + assert f.read().startswith("chr1\t147962192\t147962580\tNM_005997_cds_0_0_chr1_147962193_r\t0\t-") + + +def test_export_dataset_with_deleted_and_purged(): + app, sa_session, h = _setup_history_for_export("Datasets History with deleted") + + d1, d2 = _create_datasets(sa_session, h, 2) + + # Maybe use abstractions for deleting? + d1.deleted = True + d1.dataset.deleted = True + d1.dataset.purged = False + + d2.deleted = True + d2.dataset.deleted = True + d2.dataset.purged = True + + j1 = model.Job() + j1.user = h.user + j1.tool_id = "cat1" + j1.add_output_dataset("out_file1", d1) + + j2 = model.Job() + j2.user = h.user + j2.tool_id = "cat1" + j2.add_output_dataset("out_file1", d2) + + sa_session.add(d1) + sa_session.add(d2) + sa_session.add(j1) + sa_session.add(j2) + sa_session.add(h) + sa_session.flush() + + assert d1.deleted + + app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True) + app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True) + + imported_history = _import_export(app, h) + + datasets = list(imported_history.contents_iter(types=["dataset"])) + assert len(datasets) == 1 + + assert datasets[0].state == 'discarded' + assert datasets[0].deleted + assert datasets[0].dataset.deleted + assert datasets[0].creating_job + + +def _create_datasets(sa_session, history, n): + return [model.HistoryDatasetAssociation(extension="txt", history=history, create_dataset=True, sa_session=sa_session, hid=i + 1) for i in range(n)] + + +def _setup_history_for_export(history_name): + app = _mock_app() + sa_session = app.model.context + + email = history_name.replace(" ", "-") + "-user@example.org" + u = model.User(email=email, password="password") + h = model.History(name=history_name, user=u) + + return app, sa_session, h + + +def _import_export(app, h, dest_export=None): + if dest_export is None: + dest_parent = mkdtemp() + dest_export = os.path.join(dest_parent, "moo.tgz") + + jeha = model.JobExportHistoryArchive(job=None, history=h, + dataset=None, + compressed=True) + wrapper = JobExportHistoryArchiveWrapper(app, 1) + wrapper.setup_job(jeha) + + from galaxy.tools.imp_exp import export_history + ret = export_history.main(["--gzip", jeha.temp_directory, dest_export]) + assert ret == 0 + + _, imported_history = import_archive(dest_export, app=app) + print(_) + assert imported_history + return imported_history + + def test_import_1901_default(): app, new_history = import_archive('test-data/exports/1901_two_datasets.tgz') assert new_history @@ -133,7 +266,7 @@ def test_import_1901_default(): assert json.loads(param_dict['queries'])[0]['input2'] == dataset0.id -def import_archive(archive_path): +def import_archive(archive_path, app=None): dest_parent = mkdtemp() dest_dir = os.path.join(dest_parent, 'dest') @@ -144,7 +277,7 @@ def import_archive(archive_path): args = (archive_path, dest_dir) unpack_tar_gz_archive.main(options, args) - app, new_history = _run_jihaw_cleanup(dest_dir) + app, new_history = _run_jihaw_cleanup(dest_dir, app=app) return app, new_history @@ -170,6 +303,7 @@ def test_history_import_relpath_in_archive(): """ dest_parent = mkdtemp() with HistoryArchive(arcname_prefix='../insecure') as history_archive: + history_archive.write_metafiles() history_archive.write_file('datasets/Pasted_Entry_1.txt', 'foo') history_archive.finalize() @@ -182,6 +316,7 @@ def test_history_import_abspath_in_archive(): """ dest_parent = mkdtemp() arcname_prefix = os.path.abspath(os.path.join(dest_parent, 'insecure')) + with HistoryArchive(arcname_prefix=arcname_prefix) as history_archive: history_archive.write_metafiles() history_archive.write_file('datasets/Pasted_Entry_1.txt', 'foo')