From fe63ba9dcffcecb9c4b596d27edb8dff7779472b Mon Sep 17 00:00:00 2001 From: claudiofr Date: Tue, 21 Feb 2023 15:43:50 -0500 Subject: [PATCH] Clear the beaker caches after creating them on app startup Clear the caches in the event the cache type is database. This will delete any rows in the cache table that could be left over from a prior run of the app preventing an accumulation of stale data. --- lib/galaxy/app.py | 2 + lib/galaxy/managers/citations.py | 14 ++++--- .../unittest_utils/beaker_testing_utils.py | 17 ++++++++ lib/galaxy/tools/biotools.py | 14 ++++--- .../app/managers/test_CitationsManager_db.py | 38 +++++++++++------- .../tool_util/test_resolution_cache_db.py | 40 +++++++++++++------ 6 files changed, 85 insertions(+), 40 deletions(-) create mode 100644 lib/galaxy/model/unittest_utils/beaker_testing_utils.py diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 767cc27bbd0..7fbc2ac43fa 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -330,6 +330,8 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi mulled_resolution_cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache( "mulled_resolution" ) + # If using database cache clear cache table contents + mulled_resolution_cache.clear() self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) self._set_enabled_container_types() index_help = getattr(self.config, "index_tool_help", True) diff --git a/lib/galaxy/managers/citations.py b/lib/galaxy/managers/citations.py index cf88e566060..c5c5c36a853 100644 --- a/lib/galaxy/managers/citations.py +++ b/lib/galaxy/managers/citations.py @@ -38,14 +38,16 @@ class CitationsManager: class DoiCache: def __init__(self, config): cache_opts = { - "cache.type": getattr(config, "citation_cache_type", "ext:database"), - "cache.data_dir": getattr(config, "citation_cache_data_dir", None), - "cache.lock_dir": getattr(config, "citation_cache_lock_dir", None), - "cache.url": getattr(config, "citation_cache_url", None), - "cache.table_name": getattr(config, "citation_cache_table_name", None), - "cache.schema_name": getattr(config, "citation_cache_schema_name", None), + "cache.type": config.citation_cache_type, + "cache.data_dir": config.citation_cache_data_dir, + "cache.lock_dir": config.citation_cache_lock_dir, + "cache.url": config.citation_cache_url, + "cache.table_name": config.citation_cache_table_name, + "cache.schema_name": config.citation_cache_schema_name, } self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi") + # If using database cache clear cache table contents + self._cache.clear() def _raw_get_bibtex(self, doi): doi_url = f"https://doi.org/{doi}" diff --git a/lib/galaxy/model/unittest_utils/beaker_testing_utils.py b/lib/galaxy/model/unittest_utils/beaker_testing_utils.py new file mode 100644 index 00000000000..1bfd2bf7f7c --- /dev/null +++ b/lib/galaxy/model/unittest_utils/beaker_testing_utils.py @@ -0,0 +1,17 @@ +from sqlalchemy import ( + MetaData, + select, + Table, +) + +from galaxy.model.unittest_utils.model_testing_utils import disposing_engine + + +def is_cache_empty(url: str, namespace: str, beaker_table: str = "beaker_cache") -> bool: + """Check if there are any entries for a given namespace in a beaker cache db table""" + with disposing_engine(url) as eng: + metadata_obj = MetaData() + table = Table(beaker_table, metadata_obj, autoload_with=eng) + with eng.connect() as conn: + result = conn.execute(select(table).where(table.c.namespace == namespace)) + return result.fetchone() is None diff --git a/lib/galaxy/tools/biotools.py b/lib/galaxy/tools/biotools.py index 3f8c1050cc8..d2c0a82738f 100644 --- a/lib/galaxy/tools/biotools.py +++ b/lib/galaxy/tools/biotools.py @@ -16,13 +16,15 @@ def get_galaxy_biotools_metadata_source(config) -> BiotoolsMetadataSource: biotools_metadata_source_config.content_directory = config.biotools_content_directory biotools_metadata_source_config.use_api = config.biotools_use_api cache_opts = { - "cache.type": getattr(config, "biotools_service_cache_type", "ext:database"), - "cache.data_dir": getattr(config, "biotools_service_cache_data_dir", None), - "cache.lock_dir": getattr(config, "biotools_service_cache_lock_dir", None), - "cache.url": getattr(config, "biotools_service_cache_url", config.database_connection), - "cache.table_name": getattr(config, "biotools_service_cache_table_name", None), - "cache.schema_name": getattr(config, "biotools_service_cache_schema_name", None), + "cache.type": config.biotools_service_cache_type, + "cache.data_dir": config.biotools_service_cache_data_dir, + "cache.lock_dir": config.biotools_service_cache_lock_dir, + "cache.url": config.biotools_service_cache_url, + "cache.table_name": config.biotools_service_cache_table_name, + "cache.schema_name": config.biotools_service_cache_schema_name, } cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi") + # If using database cache clear cache table contents + cache.clear() biotools_metadata_source_config.cache = cache return get_biotools_metadata_source(biotools_metadata_source_config) diff --git a/test/unit/app/managers/test_CitationsManager_db.py b/test/unit/app/managers/test_CitationsManager_db.py index 7812d996216..8c7b8f8eb29 100644 --- a/test/unit/app/managers/test_CitationsManager_db.py +++ b/test/unit/app/managers/test_CitationsManager_db.py @@ -1,28 +1,36 @@ -import pytest from beaker.cache import CacheManager from beaker.util import parse_cache_config_options import galaxy.config from galaxy.managers.citations import DoiCache +from galaxy.model.unittest_utils.beaker_testing_utils import is_cache_empty +from galaxy.model.unittest_utils.migration_scripts_testing_utils import tmp_directory # noqa: F401 +from galaxy.model.unittest_utils.model_testing_utils import ( # noqa: F401 (url_factory is a fixture we have to import explicitly) + create_and_drop_database, + url_factory, +) class MockDoiCache(DoiCache): - def __init__(self, config): + def __init__(self, config, db_url): cache_opts = { - "cache.type": getattr(config, "citation_cache_type", "ext:database"), - "cache.data_dir": getattr(config, "citation_cache_data_dir", None), - "cache.url": "sqlite://", - "cache.table_name": getattr(config, "citation_cache_table_name", None), - "cache.schema_name": getattr(config, "citation_cache_schema_name", None), + "cache.type": "ext:database", + "cache.data_dir": config.citation_cache_data_dir, + "cache.url": db_url, + "cache.table_name": config.citation_cache_table_name, + "cache.schema_name": config.citation_cache_schema_name, } self._cache = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("doi") + self._cache.clear() -@pytest.fixture() -def doi_cache(): - return MockDoiCache(galaxy.config.GalaxyAppConfiguration(override_tempdir=False)) - - -def test_DoiCache(doi_cache): - assert "Jörg" in doi_cache.get_bibtex("10.1093/bioinformatics/bts252") - assert "Özkurt" in doi_cache.get_bibtex("10.1101/2021.12.24.474111") +def test_DoiCache(url_factory): # noqa: F811 + db_url = url_factory() + with create_and_drop_database(db_url): + doi_cache = MockDoiCache(galaxy.config.GalaxyAppConfiguration(override_tempdir=False), db_url) + assert is_cache_empty(db_url, "doi") + assert "Jörg" in doi_cache.get_bibtex("10.1093/bioinformatics/bts252") + assert "Özkurt" in doi_cache.get_bibtex("10.1101/2021.12.24.474111") + assert not is_cache_empty(db_url, "doi") + doi_cache._cache.clear() + assert is_cache_empty(db_url, "doi") diff --git a/test/unit/tool_util/test_resolution_cache_db.py b/test/unit/tool_util/test_resolution_cache_db.py index 4c448e329ac..156fbbc7c8a 100644 --- a/test/unit/tool_util/test_resolution_cache_db.py +++ b/test/unit/tool_util/test_resolution_cache_db.py @@ -5,6 +5,12 @@ from beaker.cache import CacheManager from beaker.util import parse_cache_config_options import galaxy.config +from galaxy.model.unittest_utils.beaker_testing_utils import is_cache_empty +from galaxy.model.unittest_utils.migration_scripts_testing_utils import tmp_directory # noqa: F401 +from galaxy.model.unittest_utils.model_testing_utils import ( # noqa: F401 (url_factory is a fixture we have to import explicitly) + create_and_drop_database, + url_factory, +) from galaxy.tool_util.deps.container_resolvers import ResolutionCache from galaxy.tool_util.deps.mulled.util import ( _namespace_has_repo_name, @@ -13,25 +19,33 @@ from galaxy.tool_util.deps.mulled.util import ( TAG_CACHE_KEY, ) +cache_namespace = "mulled_resolution" -@pytest.fixture() + +@pytest.fixture(scope="module") def appconfig(): return galaxy.config.GalaxyAppConfiguration(override_tempdir=False) @pytest.fixture() -def resolution_cache(tmpdir, appconfig): - resolution_cache = ResolutionCache() - cache_opts = { - "cache.type": "ext:database", - "cache.expire": "1", - "cache.url": "sqlite://", - "cache.schema_name": appconfig.mulled_resolution_cache_schema_name, - "cache.table_name": appconfig.mulled_resolution_cache_table_name, - } - cm = CacheManager(**parse_cache_config_options(cache_opts)).get_cache("mulled_resolution") - resolution_cache.mulled_resolution_cache = cm - return resolution_cache +def resolution_cache(url_factory, appconfig): # noqa: F811 + db_url = url_factory() + with create_and_drop_database(db_url): + resolution_cache = ResolutionCache() + cache_opts = { + "cache.type": "ext:database", + "cache.expire": "1", + "cache.url": db_url, + "cache.schema_name": appconfig.mulled_resolution_cache_schema_name, + "cache.table_name": appconfig.mulled_resolution_cache_table_name, + } + cm = CacheManager(**parse_cache_config_options(cache_opts)).get_cache(cache_namespace) + cm.clear() + resolution_cache.mulled_resolution_cache = cm + yield resolution_cache + assert not is_cache_empty(db_url, cache_namespace) + cm.clear() + assert is_cache_empty(db_url, cache_namespace) def test_resolution_cache_namepace_has_repo_name(resolution_cache):