diff --git a/lib/galaxy/model/unittest_utils/__init__.py b/lib/galaxy/model/unittest_utils/__init__.py new file mode 100644 index 00000000000..788bdc9ef5f --- /dev/null +++ b/lib/galaxy/model/unittest_utils/__init__.py @@ -0,0 +1,8 @@ +"""Interface provided by galaxy-data modules for unittest utilities for reuse by other modules.""" +from .data_app import GalaxyDataTestApp, GalaxyDataTestConfig + + +__all__ = [ + 'GalaxyDataTestApp', + 'GalaxyDataTestConfig', +] diff --git a/lib/galaxy/model/unittest_utils/data_app.py b/lib/galaxy/model/unittest_utils/data_app.py new file mode 100644 index 00000000000..54b038fb499 --- /dev/null +++ b/lib/galaxy/model/unittest_utils/data_app.py @@ -0,0 +1,90 @@ +"""Mock GalaxyApp exposing config + functionality required for galaxy-data package. + +There is a more expansive MockApp in test/unit/unittest_utils - but it isn't packaged +and it has dependencies from across the app. This is mock application and config is +more appropriate for testing galaxy-data functionality and will be included with +galaxy-data. +""" +import os +import shutil +import tempfile + +from galaxy import model, objectstore +from galaxy.datatypes import registry +from galaxy.model.mapping import GalaxyModelMapping, init +from galaxy.model.security import GalaxyRBACAgent +from galaxy.security.idencoding import IdEncodingHelper +from galaxy.util.bunch import Bunch + + +GALAXY_TEST_UNITTEST_SECRET = '6e46ed6483a833c100e68cc3f1d0dd76' +GALAXY_TEST_IN_MEMORY_DB_CONNECTION = "sqlite:///:memory:" + + +class GalaxyDataTestConfig(Bunch): + """Minimal Galaxy mock config object that exposes and uses only what is needed for the galaxy-data package.""" + security: IdEncodingHelper + database_connection: str + root: str + data_dir: str + _remove_root: bool + + def __init__(self, root=None, **kwd): + Bunch.__init__(self, **kwd) + if not root: + root = tempfile.mkdtemp() + self._remove_root = True + else: + self._remove_root = False + self.root = root + self.data_dir = os.path.join(root, 'database') + + self.security = IdEncodingHelper(id_secret=GALAXY_TEST_UNITTEST_SECRET) + self.database_connection = kwd.get('database_connection', GALAXY_TEST_IN_MEMORY_DB_CONNECTION) + + # objectstore config values... + self.object_store_config_file = '' + self.object_store = 'disk' + self.object_store_check_old_style = False + self.object_store_cache_path = '/tmp/cache' + self.object_store_store_by = "uuid" + + self.umask = os.umask(0o77) + self.gid = os.getgid() + # objectstore config directories... + self.jobs_directory = os.path.join(self.data_dir, 'jobs_directory') + self.new_file_path = os.path.join(self.data_dir, 'tmp') + self.file_path = os.path.join(self.data_dir, 'files') + + def __del__(self): + if self._remove_root: + shutil.rmtree(self.root) + + +class GalaxyDataTestApp(): + """Minimal Galaxy mock app object that exposes and uses only what is needed for the galaxy-data package.""" + security: IdEncodingHelper + model: GalaxyModelMapping + security_agent: GalaxyRBACAgent + + def __init__(self, config: GalaxyDataTestConfig = None, **kwd): + config = config or GalaxyDataTestConfig(**kwd) + self.config = config + self.security = config.security + self.object_store = objectstore.build_object_store_from_config(self.config) + self.model = init("/tmp", self.config.database_connection, create_tables=True, object_store=self.object_store) + self.security_agent = self.model.security_agent + self.init_datatypes() + + def init_datatypes(self): + datatypes_registry = registry.Registry() + datatypes_registry.load_datatypes() + model.set_datatypes_registry(datatypes_registry) + datatypes_registry.set_external_metadata_tool = MockSetExternalTool() + self.datatypes_registry = datatypes_registry + + +class MockSetExternalTool: + + def regenerate_imported_metadata_if_needed(self, *args, **kwds): + pass diff --git a/packages/data/setup.py b/packages/data/setup.py index b0037cff0c1..263571e25d5 100644 --- a/packages/data/setup.py +++ b/packages/data/setup.py @@ -47,6 +47,7 @@ PACKAGES = [ 'galaxy.model.tool_shed_install', 'galaxy.model.tool_shed_install.migrate', 'galaxy.model.tool_shed_install.migrate.versions', + 'galaxy.model.unittest_utils', 'galaxy.model.view', 'galaxy.quota', 'galaxy.security', diff --git a/test/unit/test_model_discovery.py b/test/unit/model/test_model_discovery.py similarity index 97% rename from test/unit/test_model_discovery.py rename to test/unit/model/test_model_discovery.py index 3e595b29788..f3fe2091fca 100644 --- a/test/unit/test_model_discovery.py +++ b/test/unit/model/test_model_discovery.py @@ -4,7 +4,7 @@ from tempfile import mkdtemp from galaxy import model from galaxy.model import store from galaxy.model.store.discover import persist_target_to_export_store -from .tools.test_history_imp_exp import _mock_app +from galaxy.model.unittest_utils import GalaxyDataTestApp def test_model_create_context_persist_hdas(): @@ -23,7 +23,7 @@ def test_model_create_context_persist_hdas(): "md5": "e5d21b1ea57fc9a31f8ea0110531bf3d", }], } - app = _mock_app(store_by="uuid") + app = _mock_app() temp_directory = mkdtemp() with store.DirectoryModelExportStore(temp_directory, serialize_dataset_objects=True) as export_store: persist_target_to_export_store(target, export_store, app.object_store, work_directory) @@ -54,7 +54,7 @@ def test_model_create_context_persist_error_hda(): "error_message": "Failed to download some URL I guess", }], } - app = _mock_app(store_by="uuid") + app = _mock_app() temp_directory = mkdtemp() with store.DirectoryModelExportStore(temp_directory, serialize_dataset_objects=True) as export_store: persist_target_to_export_store(target, export_store, app.object_store, work_directory) @@ -171,7 +171,7 @@ def test_persist_target_hdca(): }] } - app = _mock_app(store_by="uuid") + app = _mock_app() temp_directory = mkdtemp() with store.DirectoryModelExportStore(temp_directory, serialize_dataset_objects=True) as export_store: persist_target_to_export_store(target, export_store, app.object_store, work_directory) @@ -200,7 +200,7 @@ def _assert_one_library_created(sa_session): def _import_library_target(target, work_directory): - app = _mock_app(store_by="uuid") + app = _mock_app() temp_directory = mkdtemp() with store.DirectoryModelExportStore(temp_directory, app=app, serialize_dataset_objects=True) as export_store: persist_target_to_export_store(target, export_store, app.object_store, work_directory) @@ -233,3 +233,7 @@ def _import_directory_to_history(app, target, work_directory): import_model_store.perform_import(import_history) return import_history + + +def _mock_app(): + return GalaxyDataTestApp() diff --git a/test/unit/tools/test_history_imp_exp.py b/test/unit/tools/test_history_imp_exp.py index 14254d080e5..2322d094e28 100644 --- a/test/unit/tools/test_history_imp_exp.py +++ b/test/unit/tools/test_history_imp_exp.py @@ -20,12 +20,6 @@ HISTORY_ATTRS = '''{"hid_counter": 2, "update_time": "2016-02-08 18:38:38.705058 JOBS_ATTRS = '''[{"info": null, "tool_id": "upload1", "update_time": "2016-02-08T18:39:23.356482", "stdout": "", "input_mapping": {}, "tool_version": "1.1.4", "traceback": null, "command_line": "python /galaxy/tools/data_source/upload.py /galaxy /scratch/tmppwU9rD /scratch/tmpP4_45Y 1:/scratch/jobs/000/dataset_1_files:/data/000/dataset_1.dat", "exit_code": 0, "output_datasets": [1], "state": "ok", "create_time": "2016-02-08T18:38:39.153873", "params": {"files": [{"to_posix_lines": "Yes", "NAME": "None", "file_data": null, "space_to_tab": null, "url_paste": "/scratch/strio_url_paste_o6nrv8", "__index__": 0, "ftp_files": "", "uuid": "None"}], "paramfile": "/scratch/tmpP4_45Y", "file_type": "auto", "files_metadata": {"file_type": "auto", "__current_case__": 41}, "async_datasets": "None", "dbkey": "?"}, "stderr": ""}]''' -class MockSetExternalTool: - - def regenerate_imported_metadata_if_needed(self, *args, **kwds): - pass - - def _run_jihaw_cleanup(archive_dir, app=None): app = app or _mock_app() job = model.Job() @@ -43,7 +37,6 @@ def _mock_app(store_by="id"): test_object_store_config = TestConfig(store_by=store_by) app.object_store = test_object_store_config.object_store app.model.Dataset.object_store = app.object_store - app.datatypes_registry.set_external_metadata_tool = MockSetExternalTool() return app diff --git a/test/unit/unittest_utils/galaxy_mock.py b/test/unit/unittest_utils/galaxy_mock.py index 2e50ed279f5..7c03038a162 100644 --- a/test/unit/unittest_utils/galaxy_mock.py +++ b/test/unit/unittest_utils/galaxy_mock.py @@ -9,17 +9,15 @@ from sqlalchemy.orm.scoping import scoped_session from galaxy import ( di, - model, - objectstore, quota, ) from galaxy.auth import AuthManager -from galaxy.datatypes import registry from galaxy.jobs.manager import NoopManager from galaxy.managers.users import UserManager -from galaxy.model import mapping, tags +from galaxy.model import tags from galaxy.model.base import SharedModelMapping from galaxy.model.mapping import GalaxyModelMapping +from galaxy.model.unittest_utils import GalaxyDataTestApp, GalaxyDataTestConfig from galaxy.security import idencoding from galaxy.structured_app import BasicApp, MinimalManagerApp, StructuredApp from galaxy.tool_util.deps.containers import NullContainerFinder @@ -63,28 +61,24 @@ def buildMockEnviron(**kwargs): return environ -class MockApp(di.Container): +class MockApp(di.Container, GalaxyDataTestApp): def __init__(self, config=None, **kwargs): super().__init__() + config = config or MockAppConfig(**kwargs) + GalaxyDataTestApp.__init__(self, config=config, **kwargs) self[BasicApp] = self self[MinimalManagerApp] = self self[StructuredApp] = self - self.config = config or MockAppConfig(**kwargs) - self.security = self.config.security self[idencoding.IdEncodingHelper] = self.security self.name = kwargs.get('name', 'galaxy') - self.object_store = objectstore.build_object_store_from_config(self.config) - self.model = mapping.init("/tmp", self.config.database_connection, create_tables=True, object_store=self.object_store) self[SharedModelMapping] = self.model self[GalaxyModelMapping] = self.model self[scoped_session] = self.model.context - self.security_agent = self.model.security_agent self.visualizations_registry = MockVisualizationsRegistry() self.tag_handler = tags.GalaxyTagHandler(self.model.context) self[tags.GalaxyTagHandler] = self.tag_handler self.quota_agent = quota.DatabaseQuotaAgent(self.model) - self.init_datatypes() self.job_config = Bunch( dynamic_params=None, destinations={}, @@ -109,12 +103,6 @@ class MockApp(di.Container): return "/mock/url" self.url_for = url_for - def init_datatypes(self): - datatypes_registry = registry.Registry() - datatypes_registry.load_datatypes() - model.set_datatypes_registry(datatypes_registry) - self.datatypes_registry = datatypes_registry - def wait_for_toolbox_reload(self, toolbox): # TODO: If the tpm test case passes, does the operation really # need to wait. @@ -129,38 +117,20 @@ class MockLock: pass -class MockAppConfig(Bunch): +class MockAppConfig(GalaxyDataTestConfig): class MockSchema(Bunch): pass - def __init__(self, root=None, **kwargs): - Bunch.__init__(self, **kwargs) - if not root: - root = tempfile.mkdtemp() - self._remove_root = True - else: - self._remove_root = False + def __init__(self, **kwargs): + super().__init__(**kwargs) self.schema = self.MockSchema() - self.security = idencoding.IdEncodingHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76') - self.database_connection = kwargs.get('database_connection', "sqlite:///:memory:") self.use_remote_user = kwargs.get('use_remote_user', False) self.enable_celery_tasks = False - self.data_dir = os.path.join(root, 'database') - self.file_path = os.path.join(self.data_dir, 'files') - self.jobs_directory = os.path.join(self.data_dir, 'jobs_directory') - self.new_file_path = os.path.join(self.data_dir, 'tmp') - self.tool_data_path = os.path.join(root, 'tool-data') + self.tool_data_path = os.path.join(self.root, 'tool-data') self.tool_dependency_dir = None self.metadata_strategy = 'legacy' - self.object_store_config_file = '' - self.object_store = 'disk' - self.object_store_check_old_style = False - self.object_store_cache_path = '/tmp/cache' - self.umask = os.umask(0o77) - self.gid = os.getgid() - self.user_activation_on = False self.new_user_dataset_access_role_default_private = False @@ -191,9 +161,8 @@ class MockAppConfig(Bunch): self.version_major = "19.09" # set by MockDir - self.root = root self.enable_tool_document_cache = False - self.tool_cache_data_dir = os.path.join(root, 'tool_cache') + self.tool_cache_data_dir = os.path.join(self.root, 'tool_cache') self.delay_tool_initialization = True self.external_chown_script = None @@ -217,16 +186,11 @@ class MockAppConfig(Bunch): return False raise AttributeError(name) - def __del__(self): - if self._remove_root: - shutil.rmtree(self.root) - class MockWebapp: def __init__(self, **kwargs): self.name = kwargs.get('name', 'galaxy') - self.security = idencoding.IdEncodingHelper(id_secret='6e46ed6483a833c100e68cc3f1d0dd76') class MockTrans: @@ -235,6 +199,7 @@ class MockTrans: self.app = app or MockApp(**kwargs) self.model = self.app.model self.webapp = MockWebapp(**kwargs) + self.webapp.security = self.app.security self.sa_session = self.app.model.session self.workflow_building_mode = False self.error_message = None