Establish unittest utils for galaxy-data packages.

Progress toward https://github.com/galaxyproject/galaxy/issues/8364. Cleans up dependency into a proper DAG so we can move for instance test_model_discovery.py into galaxy-data's unittests.

lib/galaxy/model/unittest_utils/__init__.py

@@ -0,0 +1,8 @@
+"""Interface provided by galaxy-data modules for unittest utilities for reuse by other modules."""
+from .data_app import GalaxyDataTestApp, GalaxyDataTestConfig
+
+
+__all__ = [
+    'GalaxyDataTestApp',
+    'GalaxyDataTestConfig',
+]

lib/galaxy/model/unittest_utils/data_app.py

@@ -0,0 +1,90 @@
+"""Mock GalaxyApp exposing config + functionality required for galaxy-data package.
+
+There is a more expansive MockApp in test/unit/unittest_utils - but it isn't packaged
+and it has dependencies from across the app. This is mock application and config is
+more appropriate for testing galaxy-data functionality and will be included with
+galaxy-data.
+"""
+import os
+import shutil
+import tempfile
+
+from galaxy import model, objectstore
+from galaxy.datatypes import registry
+from galaxy.model.mapping import GalaxyModelMapping, init
+from galaxy.model.security import GalaxyRBACAgent
+from galaxy.security.idencoding import IdEncodingHelper
+from galaxy.util.bunch import Bunch
+
+
+GALAXY_TEST_UNITTEST_SECRET = '6e46ed6483a833c100e68cc3f1d0dd76'
+GALAXY_TEST_IN_MEMORY_DB_CONNECTION = "sqlite:///:memory:"
+
+
+class GalaxyDataTestConfig(Bunch):
+    """Minimal Galaxy mock config object that exposes and uses only what is needed for the galaxy-data package."""
+    security: IdEncodingHelper
+    database_connection: str
+    root: str
+    data_dir: str
+    _remove_root: bool
+
+    def __init__(self, root=None, **kwd):
+        Bunch.__init__(self, **kwd)
+        if not root:
+            root = tempfile.mkdtemp()
+            self._remove_root = True
+        else:
+            self._remove_root = False
+        self.root = root
+        self.data_dir = os.path.join(root, 'database')
+
+        self.security = IdEncodingHelper(id_secret=GALAXY_TEST_UNITTEST_SECRET)
+        self.database_connection = kwd.get('database_connection', GALAXY_TEST_IN_MEMORY_DB_CONNECTION)
+
+        # objectstore config values...
+        self.object_store_config_file = ''
+        self.object_store = 'disk'
+        self.object_store_check_old_style = False
+        self.object_store_cache_path = '/tmp/cache'
+        self.object_store_store_by = "uuid"
+
+        self.umask = os.umask(0o77)
+        self.gid = os.getgid()
+        # objectstore config directories...
+        self.jobs_directory = os.path.join(self.data_dir, 'jobs_directory')
+        self.new_file_path = os.path.join(self.data_dir, 'tmp')
+        self.file_path = os.path.join(self.data_dir, 'files')
+
+    def __del__(self):
+        if self._remove_root:
+            shutil.rmtree(self.root)
+
+
+class GalaxyDataTestApp():
+    """Minimal Galaxy mock app object that exposes and uses only what is needed for the galaxy-data package."""
+    security: IdEncodingHelper
+    model: GalaxyModelMapping
+    security_agent: GalaxyRBACAgent
+
+    def __init__(self, config: GalaxyDataTestConfig = None, **kwd):
+        config = config or GalaxyDataTestConfig(**kwd)
+        self.config = config
+        self.security = config.security
+        self.object_store = objectstore.build_object_store_from_config(self.config)
+        self.model = init("/tmp", self.config.database_connection, create_tables=True, object_store=self.object_store)
+        self.security_agent = self.model.security_agent
+        self.init_datatypes()
+
+    def init_datatypes(self):
+        datatypes_registry = registry.Registry()
+        datatypes_registry.load_datatypes()
+        model.set_datatypes_registry(datatypes_registry)
+        datatypes_registry.set_external_metadata_tool = MockSetExternalTool()
+        self.datatypes_registry = datatypes_registry
+
+
+class MockSetExternalTool:
+
+    def regenerate_imported_metadata_if_needed(self, *args, **kwds):
+        pass

packages/data/setup.py

@@ -47,6 +47,7 @@ PACKAGES = [
     'galaxy.model.tool_shed_install',
     'galaxy.model.tool_shed_install.migrate',
     'galaxy.model.tool_shed_install.migrate.versions',
+    'galaxy.model.unittest_utils',
     'galaxy.model.view',
     'galaxy.quota',
     'galaxy.security',

test/unit/model/test_model_discovery.py

@@ -4,7 +4,7 @@ from tempfile import mkdtemp
 from galaxy import model
 from galaxy.model import store
 from galaxy.model.store.discover import persist_target_to_export_store
-from .tools.test_history_imp_exp import _mock_app
+from galaxy.model.unittest_utils import GalaxyDataTestApp
 
 
 def test_model_create_context_persist_hdas():
@@ -23,7 +23,7 @@ def test_model_create_context_persist_hdas():
             "md5": "e5d21b1ea57fc9a31f8ea0110531bf3d",
         }],
     }
-    app = _mock_app(store_by="uuid")
+    app = _mock_app()
     temp_directory = mkdtemp()
     with store.DirectoryModelExportStore(temp_directory, serialize_dataset_objects=True) as export_store:
         persist_target_to_export_store(target, export_store, app.object_store, work_directory)
@@ -54,7 +54,7 @@ def test_model_create_context_persist_error_hda():
             "error_message": "Failed to download some URL I guess",
         }],
     }
-    app = _mock_app(store_by="uuid")
+    app = _mock_app()
     temp_directory = mkdtemp()
     with store.DirectoryModelExportStore(temp_directory, serialize_dataset_objects=True) as export_store:
         persist_target_to_export_store(target, export_store, app.object_store, work_directory)
@@ -171,7 +171,7 @@ def test_persist_target_hdca():
         }]
     }
 
-    app = _mock_app(store_by="uuid")
+    app = _mock_app()
     temp_directory = mkdtemp()
     with store.DirectoryModelExportStore(temp_directory, serialize_dataset_objects=True) as export_store:
         persist_target_to_export_store(target, export_store, app.object_store, work_directory)
@@ -200,7 +200,7 @@ def _assert_one_library_created(sa_session):
 
 
 def _import_library_target(target, work_directory):
-    app = _mock_app(store_by="uuid")
+    app = _mock_app()
     temp_directory = mkdtemp()
     with store.DirectoryModelExportStore(temp_directory, app=app, serialize_dataset_objects=True) as export_store:
         persist_target_to_export_store(target, export_store, app.object_store, work_directory)
@@ -233,3 +233,7 @@ def _import_directory_to_history(app, target, work_directory):
         import_model_store.perform_import(import_history)
 
     return import_history
+
+
+def _mock_app():
+    return GalaxyDataTestApp()

test/unit/tools/test_history_imp_exp.py

@@ -20,12 +20,6 @@ HISTORY_ATTRS = '''{"hid_counter": 2, "update_time": "2016-02-08 18:38:38.705058
 JOBS_ATTRS = '''[{"info": null, "tool_id": "upload1", "update_time": "2016-02-08T18:39:23.356482", "stdout": "", "input_mapping": {}, "tool_version": "1.1.4", "traceback": null, "command_line": "python /galaxy/tools/data_source/upload.py /galaxy /scratch/tmppwU9rD /scratch/tmpP4_45Y 1:/scratch/jobs/000/dataset_1_files:/data/000/dataset_1.dat", "exit_code": 0, "output_datasets": [1], "state": "ok", "create_time": "2016-02-08T18:38:39.153873", "params": {"files": [{"to_posix_lines": "Yes", "NAME": "None", "file_data": null, "space_to_tab": null, "url_paste": "/scratch/strio_url_paste_o6nrv8", "__index__": 0, "ftp_files": "", "uuid": "None"}], "paramfile": "/scratch/tmpP4_45Y", "file_type": "auto", "files_metadata": {"file_type": "auto", "__current_case__": 41}, "async_datasets": "None", "dbkey": "?"}, "stderr": ""}]'''
 
 
-class MockSetExternalTool:
-
-    def regenerate_imported_metadata_if_needed(self, *args, **kwds):
-        pass
-
-
 def _run_jihaw_cleanup(archive_dir, app=None):
     app = app or _mock_app()
     job = model.Job()
@@ -43,7 +37,6 @@ def _mock_app(store_by="id"):
     test_object_store_config = TestConfig(store_by=store_by)
     app.object_store = test_object_store_config.object_store
     app.model.Dataset.object_store = app.object_store
-    app.datatypes_registry.set_external_metadata_tool = MockSetExternalTool()
     return app
 
 

test/unit/unittest_utils/galaxy_mock.py

@@ -9,17 +9,15 @@ from sqlalchemy.orm.scoping import scoped_session
 
 from galaxy import (
     di,
-    model,
-    objectstore,
     quota,
 )
 from galaxy.auth import AuthManager
-from galaxy.datatypes import registry
 from galaxy.jobs.manager import NoopManager
 from galaxy.managers.users import UserManager
-from galaxy.model import mapping, tags
+from galaxy.model import tags
 from galaxy.model.base import SharedModelMapping
 from galaxy.model.mapping import GalaxyModelMapping
+from galaxy.model.unittest_utils import GalaxyDataTestApp, GalaxyDataTestConfig
 from galaxy.security import idencoding
 from galaxy.structured_app import BasicApp, MinimalManagerApp, StructuredApp
 from galaxy.tool_util.deps.containers import NullContainerFinder
@@ -63,28 +61,24 @@ def buildMockEnviron(**kwargs):
     return environ
 
 
-class MockApp(di.Container):
+class MockApp(di.Container, GalaxyDataTestApp):
 
     def __init__(self, config=None, **kwargs):
         super().__init__()
+        config = config or MockAppConfig(**kwargs)
+        GalaxyDataTestApp.__init__(self, config=config, **kwargs)
         self[BasicApp] = self
         self[MinimalManagerApp] = self
         self[StructuredApp] = self
-        self.config = config or MockAppConfig(**kwargs)
-        self.security = self.config.security
         self[idencoding.IdEncodingHelper] = self.security
         self.name = kwargs.get('name', 'galaxy')
-        self.object_store = objectstore.build_object_store_from_config(self.config)
-        self.model = mapping.init("/tmp", self.config.database_connection, create_tables=True, object_store=self.object_store)
         self[SharedModelMapping] = self.model
         self[GalaxyModelMapping] = self.model
         self[scoped_session] = self.model.context
-        self.security_agent = self.model.security_agent
         self.visualizations_registry = MockVisualizationsRegistry()
         self.tag_handler = tags.GalaxyTagHandler(self.model.context)
         self[tags.GalaxyTagHandler] = self.tag_handler
         self.quota_agent = quota.DatabaseQuotaAgent(self.model)
-        self.init_datatypes()
         self.job_config = Bunch(
             dynamic_params=None,
             destinations={},
@@ -109,12 +103,6 @@ class MockApp(di.Container):
             return "/mock/url"
         self.url_for = url_for
 
-    def init_datatypes(self):
-        datatypes_registry = registry.Registry()
-        datatypes_registry.load_datatypes()
-        model.set_datatypes_registry(datatypes_registry)
-        self.datatypes_registry = datatypes_registry
-
     def wait_for_toolbox_reload(self, toolbox):
         # TODO: If the tpm test case passes, does the operation really
         # need to wait.
@@ -129,38 +117,20 @@ class MockLock:
         pass
 
 
-class MockAppConfig(Bunch):
+class MockAppConfig(GalaxyDataTestConfig):
 
     class MockSchema(Bunch):
         pass
 
-    def __init__(self, root=None, **kwargs):
-        Bunch.__init__(self, **kwargs)
-        if not root:
-            root = tempfile.mkdtemp()
-            self._remove_root = True
-        else:
-            self._remove_root = False
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
         self.schema = self.MockSchema()
-        self.security = idencoding.IdEncodingHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76')
-        self.database_connection = kwargs.get('database_connection', "sqlite:///:memory:")
         self.use_remote_user = kwargs.get('use_remote_user', False)
         self.enable_celery_tasks = False
-        self.data_dir = os.path.join(root, 'database')
-        self.file_path = os.path.join(self.data_dir, 'files')
-        self.jobs_directory = os.path.join(self.data_dir, 'jobs_directory')
-        self.new_file_path = os.path.join(self.data_dir, 'tmp')
-        self.tool_data_path = os.path.join(root, 'tool-data')
+        self.tool_data_path = os.path.join(self.root, 'tool-data')
         self.tool_dependency_dir = None
         self.metadata_strategy = 'legacy'
 
-        self.object_store_config_file = ''
-        self.object_store = 'disk'
-        self.object_store_check_old_style = False
-        self.object_store_cache_path = '/tmp/cache'
-        self.umask = os.umask(0o77)
-        self.gid = os.getgid()
-
         self.user_activation_on = False
         self.new_user_dataset_access_role_default_private = False
 
@@ -191,9 +161,8 @@ class MockAppConfig(Bunch):
         self.version_major = "19.09"
 
         # set by MockDir
-        self.root = root
         self.enable_tool_document_cache = False
-        self.tool_cache_data_dir = os.path.join(root, 'tool_cache')
+        self.tool_cache_data_dir = os.path.join(self.root, 'tool_cache')
         self.delay_tool_initialization = True
         self.external_chown_script = None
 
@@ -217,16 +186,11 @@ class MockAppConfig(Bunch):
             return False
         raise AttributeError(name)
 
-    def __del__(self):
-        if self._remove_root:
-            shutil.rmtree(self.root)
-
 
 class MockWebapp:
 
     def __init__(self, **kwargs):
         self.name = kwargs.get('name', 'galaxy')
-        self.security = idencoding.IdEncodingHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76')
 
 
 class MockTrans:
@@ -235,6 +199,7 @@ class MockTrans:
         self.app = app or MockApp(**kwargs)
         self.model = self.app.model
         self.webapp = MockWebapp(**kwargs)
+        self.webapp.security = self.app.security
         self.sa_session = self.app.model.session
         self.workflow_building_mode = False
         self.error_message = None