More import/export unit-style testing.

lib/galaxy/tools/imp_exp/__init__.py

@@ -429,15 +429,15 @@ class JobExportHistoryArchiveWrapper:
             job_attrs['update_time'] = job.update_time.isoformat()
 
             # Get the job's parameters
+            params_dict = {}
             try:
                 params_objects = job.get_param_values(self.app)
+                for name, value in params_objects.items():
+                    params_dict[name] = value
             except Exception:
                 # Could not get job params.
-                continue
+                pass
 
-            params_dict = {}
-            for name, value in params_objects.items():
-                params_dict[name] = value
             job_attrs['params'] = params_dict
 
             # -- Get input, output datasets. --

lib/galaxy/tools/imp_exp/export_history.py

@@ -90,16 +90,18 @@ def create_archive(history_attrs_file, datasets_attrs_file, jobs_attrs_file, out
 
         # Status.
         print('Created history archive.')
+        return 0
     except Exception as e:
         print('Error creating history archive: %s' % str(e), file=sys.stderr)
+        return 1
 
 
-def main():
+def main(argv=None):
     # Parse command line.
     parser = optparse.OptionParser()
     parser.add_option('-G', '--gzip', dest='gzip', action="store_true", help='Compress archive using gzip.')
     parser.add_option('--galaxy-version', dest='galaxy_version', help='Galaxy version that initiated the command.', default=None)
-    (options, args) = parser.parse_args()
+    (options, args) = parser.parse_args(argv)
     galaxy_version = options.galaxy_version
     if galaxy_version is None:
         galaxy_version = "19.01" if len(args) == 4 else "19.05"
@@ -118,7 +120,7 @@ def main():
         job_attrs = os.path.join(temp_directory, 'jobs_attrs.txt')
 
     # Create archive.
-    create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip)
+    return create_archive(history_attrs, dataset_attrs, job_attrs, out_file, gzip)
 
 
 if __name__ == "__main__":

test/unit/test_objectstore.py

@@ -740,7 +740,7 @@ def test_config_parse_azure():
 
 
 class TestConfig(object):
-    def __init__(self, config_str, clazz=None):
+    def __init__(self, config_str=DISK_TEST_CONFIG, clazz=None):
         self.temp_directory = mkdtemp()
         if config_str.startswith("<"):
             config_file = "store.xml"

test/unit/tools/test_history_imp_exp.py

@@ -7,8 +7,9 @@ from tempfile import mkdtemp
 
 from galaxy import model
 from galaxy.exceptions import MalformedContents
-from galaxy.tools.imp_exp import JobImportHistoryArchiveWrapper, unpack_tar_gz_archive
+from galaxy.tools.imp_exp import JobExportHistoryArchiveWrapper, JobImportHistoryArchiveWrapper, unpack_tar_gz_archive
 from galaxy.tools.imp_exp.export_history import create_archive
+from ..test_objectstore import TestConfig
 from ..unittest_utils.galaxy_mock import MockApp
 
 
@@ -25,19 +26,26 @@ class MockSetExternalTool(object):
         pass
 
 
-def _run_jihaw_cleanup(archive_dir):
-    app = MockApp()
-    app.datatypes_registry.set_external_metadata_tool = MockSetExternalTool()
-
+def _run_jihaw_cleanup(archive_dir, app=None):
+    app = app or _mock_app()
     job = model.Job()
     job.tool_stderr = ''
     jiha = model.JobImportHistoryArchive(job=job, archive_dir=archive_dir)
     app.model.context.current.add_all([job, jiha])
     app.model.context.flush()
-    jihaw = JobImportHistoryArchiveWrapper(app, 1)  # yeehaw!
+    jihaw = JobImportHistoryArchiveWrapper(app, job.id)  # yeehaw!
     return app, jihaw.cleanup_after_job()
 
 
+def _mock_app():
+    app = MockApp()
+    test_object_store_config = TestConfig()
+    app.object_store = test_object_store_config.object_store
+    app.model.Dataset.object_store = app.object_store
+    app.datatypes_registry.set_external_metadata_tool = MockSetExternalTool()
+    return app
+
+
 def _run_jihaw_cleanup_check_secure(history_archive, msg):
     malformed = False
     try:
@@ -101,6 +109,131 @@ def test_history_import_abspath_in_metadata():
         _run_jihaw_cleanup_check_secure(history_archive, 'Absolute path in datasets_attrs.txt allowed')
 
 
+def test_export_dataset():
+    app, sa_session, h = _setup_history_for_export("Datasets History")
+
+    d1, d2 = _create_datasets(sa_session, h, 2)
+
+    j = model.Job()
+    j.user = h.user
+    j.tool_id = "cat1"
+
+    j.add_input_dataset("input1", d1)
+    j.add_output_dataset("out_file1", d2)
+
+    sa_session.add(d1)
+    sa_session.add(d2)
+    sa_session.add(h)
+    sa_session.add(j)
+    sa_session.flush()
+
+    app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True)
+    app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True)
+
+    imported_history = _import_export(app, h)
+
+    datasets = list(imported_history.contents_iter(types=["dataset"]))
+    assert len(datasets) == 2
+    imported_job = datasets[1].creating_job
+    assert imported_job
+    assert imported_job.output_datasets
+    assert imported_job.output_datasets[0].dataset == datasets[1]
+
+    assert imported_job.input_datasets
+    assert imported_job.input_datasets[0].dataset == datasets[0]
+
+    assert datasets[0].state == 'ok'
+    assert datasets[1].state == 'ok'
+
+    with open(datasets[0].file_name, "r") as f:
+        assert f.read().startswith("chr1    4225    19670")
+    with open(datasets[1].file_name, "r") as f:
+        assert f.read().startswith("chr1\t147962192\t147962580\tNM_005997_cds_0_0_chr1_147962193_r\t0\t-")
+
+
+def test_export_dataset_with_deleted_and_purged():
+    app, sa_session, h = _setup_history_for_export("Datasets History with deleted")
+
+    d1, d2 = _create_datasets(sa_session, h, 2)
+
+    # Maybe use abstractions for deleting?
+    d1.deleted = True
+    d1.dataset.deleted = True
+    d1.dataset.purged = False
+
+    d2.deleted = True
+    d2.dataset.deleted = True
+    d2.dataset.purged = True
+
+    j1 = model.Job()
+    j1.user = h.user
+    j1.tool_id = "cat1"
+    j1.add_output_dataset("out_file1", d1)
+
+    j2 = model.Job()
+    j2.user = h.user
+    j2.tool_id = "cat1"
+    j2.add_output_dataset("out_file1", d2)
+
+    sa_session.add(d1)
+    sa_session.add(d2)
+    sa_session.add(j1)
+    sa_session.add(j2)
+    sa_session.add(h)
+    sa_session.flush()
+
+    assert d1.deleted
+
+    app.object_store.update_from_file(d1, file_name="test-data/1.txt", create=True)
+    app.object_store.update_from_file(d2, file_name="test-data/2.bed", create=True)
+
+    imported_history = _import_export(app, h)
+
+    datasets = list(imported_history.contents_iter(types=["dataset"]))
+    assert len(datasets) == 1
+
+    assert datasets[0].state == 'discarded'
+    assert datasets[0].deleted
+    assert datasets[0].dataset.deleted
+    assert datasets[0].creating_job
+
+
+def _create_datasets(sa_session, history, n):
+    return [model.HistoryDatasetAssociation(extension="txt", history=history, create_dataset=True, sa_session=sa_session, hid=i + 1) for i in range(n)]
+
+
+def _setup_history_for_export(history_name):
+    app = _mock_app()
+    sa_session = app.model.context
+
+    email = history_name.replace(" ", "-") + "-user@example.org"
+    u = model.User(email=email, password="password")
+    h = model.History(name=history_name, user=u)
+
+    return app, sa_session, h
+
+
+def _import_export(app, h, dest_export=None):
+    if dest_export is None:
+        dest_parent = mkdtemp()
+        dest_export = os.path.join(dest_parent, "moo.tgz")
+
+    jeha = model.JobExportHistoryArchive(job=None, history=h,
+                                         dataset=None,
+                                         compressed=True)
+    wrapper = JobExportHistoryArchiveWrapper(app, 1)
+    wrapper.setup_job(jeha)
+
+    from galaxy.tools.imp_exp import export_history
+    ret = export_history.main(["--gzip", jeha.temp_directory, dest_export])
+    assert ret == 0
+
+    _, imported_history = import_archive(dest_export, app=app)
+    print(_)
+    assert imported_history
+    return imported_history
+
+
 def test_import_1901_default():
     app, new_history = import_archive('test-data/exports/1901_two_datasets.tgz')
     assert new_history
@@ -133,7 +266,7 @@ def test_import_1901_default():
     assert json.loads(param_dict['queries'])[0]['input2'] == dataset0.id
 
 
-def import_archive(archive_path):
+def import_archive(archive_path, app=None):
     dest_parent = mkdtemp()
     dest_dir = os.path.join(dest_parent, 'dest')
 
@@ -144,7 +277,7 @@ def import_archive(archive_path):
 
     args = (archive_path, dest_dir)
     unpack_tar_gz_archive.main(options, args)
-    app, new_history = _run_jihaw_cleanup(dest_dir)
+    app, new_history = _run_jihaw_cleanup(dest_dir, app=app)
     return app, new_history
 
 
@@ -170,6 +303,7 @@ def test_history_import_relpath_in_archive():
     """
     dest_parent = mkdtemp()
     with HistoryArchive(arcname_prefix='../insecure') as history_archive:
+
         history_archive.write_metafiles()
         history_archive.write_file('datasets/Pasted_Entry_1.txt', 'foo')
         history_archive.finalize()
@@ -182,6 +316,7 @@ def test_history_import_abspath_in_archive():
     """
     dest_parent = mkdtemp()
     arcname_prefix = os.path.abspath(os.path.join(dest_parent, 'insecure'))
+
     with HistoryArchive(arcname_prefix=arcname_prefix) as history_archive:
         history_archive.write_metafiles()
         history_archive.write_file('datasets/Pasted_Entry_1.txt', 'foo')