From a327e34c89cbc5e95c232cc0bf6bb3dc93e8aecb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 12 Jan 2022 12:07:15 +0100 Subject: [PATCH 1/3] Limit lifetime of repository cache --- lib/galaxy/app.py | 6 ++--- lib/galaxy/queue_worker.py | 4 --- lib/galaxy/structured_app.py | 2 +- .../installed_repository_metadata_manager.py | 1 - lib/galaxy/tool_shed/util/repository_util.py | 25 +++++++++---------- .../tool_shed/util/tool_dependency_util.py | 1 - lib/galaxy/tool_util/toolbox/base.py | 4 ++- lib/galaxy/tools/cache.py | 12 +++------ .../galaxy/controllers/admin_toolshed.py | 3 +-- .../tools/test_tool_shed_repository_cache.py | 10 ++++---- 10 files changed, 28 insertions(+), 40 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index e8a89669ba9..db4b507609e 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -232,7 +232,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): ("job manager", self._shutdown_job_manager), ("application heartbeat", self._shutdown_heartbeat), ("repository manager", self._shutdown_repo_manager), - ("database connection repository cache", self._shutdown_repo_cache), ("database connection", self._shutdown_model), ("application stack", self._shutdown_application_stack), ] @@ -384,6 +383,8 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): self.url_for = url_for self.server_starttime = int(time.time()) # used for cachebusting + # Limit lifetime of tool shed repository cache to app startup + self.tool_shed_repository_cache = None log.info(f"Galaxy app startup finished {startup_timer}") def _shutdown_queue_worker(self): @@ -408,9 +409,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): def _shutdown_repo_manager(self): self.update_repository_manager.shutdown() - def _shutdown_repo_cache(self): - self.tool_shed_repository_cache.shutdown() - def _shutdown_application_stack(self): self.application_stack.shutdown() diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 16e7f553df7..9dc382e6e7b 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -181,8 +181,6 @@ def _get_new_toolbox(app, save_integrated_tool_panel=True): """ from galaxy import tools from galaxy.tools.special_tools import load_lib_tools - if hasattr(app, 'tool_shed_repository_cache'): - app.tool_shed_repository_cache.rebuild() tool_configs = app.config.tool_configs new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app, save_integrated_tool_panel=save_integrated_tool_panel) @@ -199,8 +197,6 @@ def reload_data_managers(app, **kwargs): reload_timer = util.ExecutionTimer() from galaxy.tools.data_manager.manager import DataManagers log.debug("Executing data managers reload on '%s'", app.config.server_name) - if hasattr(app, 'tool_shed_repository_cache'): - app.tool_shed_repository_cache.rebuild() app._configure_tool_data_tables(from_shed_config=False) reload_tool_data_tables(app) reload_count = app.data_managers._reload_count diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index 715f42ceab7..642c1ea48e0 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -112,7 +112,7 @@ class StructuredApp(MinimalManagerApp): error_reports: Any # 'galaxy.tools.error_reports.ErrorReports' job_config: Any # 'galaxy.jobs.JobConfiguration' tool_cache: Any # 'galaxy.tools.cache.ToolCache' - tool_shed_repository_cache: Any # 'galaxy.tools.cache.ToolShedRepositoryCache' + tool_shed_repository_cache: Optional[Any] # 'galaxy.tools.cache.ToolShedRepositoryCache' watchers: Any # 'galaxy.config_watchers.ConfigWatchers' installed_repository_manager: Any # 'galaxy.tool_shed.galaxy_install.installed_repository_manager.InstalledRepositoryManager' workflow_scheduling_manager: Any # 'galaxy.workflow.scheduling_manager.WorkflowSchedulingManager' diff --git a/lib/galaxy/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py b/lib/galaxy/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py index 6619ef98773..72ae00b8c7f 100644 --- a/lib/galaxy/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py @@ -94,7 +94,6 @@ class InstalledRepositoryMetadataManager(MetadataGenerator): self.update_in_shed_tool_config() self.app.install_model.context.add(self.repository) self.app.install_model.context.flush() - self.app.tool_shed_repository_cache.rebuild() log.debug(f'Metadata has been reset on repository {self.repository.name}.') else: log.debug(f'Metadata did not need to be reset on repository {self.repository.name}.') diff --git a/lib/galaxy/tool_shed/util/repository_util.py b/lib/galaxy/tool_shed/util/repository_util.py index dcad79e1ce0..ccf64076234 100644 --- a/lib/galaxy/tool_shed/util/repository_util.py +++ b/lib/galaxy/tool_shed/util/repository_util.py @@ -106,7 +106,7 @@ def create_or_update_tool_shed_repository(app, name, description, installed_chan deleted = False uninstalled = False tool_shed_repository = \ - get_installed_repository(app, tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=installed_changeset_revision, refresh=True) + get_installed_repository(app, tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=installed_changeset_revision) if tool_shed_repository: log.debug("Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'.", name, status) tool_shed_repository.description = description @@ -208,15 +208,17 @@ def get_installed_repository(app, tool_shed=None, name=None, owner=None, changes """ # We store the port, if one exists, in the database. tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed) - if from_cache and hasattr(app, 'tool_shed_repository_cache'): - if refresh: - app.tool_shed_repository_cache.rebuild() - return app.tool_shed_repository_cache.get_installed_repository(tool_shed=tool_shed, - name=name, - owner=owner, - installed_changeset_revision=installed_changeset_revision, - changeset_revision=changeset_revision, - repository_id=repository_id) + if from_cache: + tsr_cache = getattr(app, 'tool_shed_repository_cache', None) + if tsr_cache: + return app.tool_shed_repository_cache.get_installed_repository( + tool_shed=tool_shed, + name=name, + owner=owner, + installed_changeset_revision=installed_changeset_revision, + changeset_revision=changeset_revision, + repository_id=repository_id + ) query = app.install_model.context.query(app.install_model.ToolShedRepository) if repository_id: clause_list = [app.install_model.ToolShedRepository.table.c.id == repository_id] @@ -239,8 +241,6 @@ def get_installed_tool_shed_repository(app, id): else: id = [id] return_list = False - if hasattr(app, 'tool_shed_repository_cache'): - app.tool_shed_repository_cache.rebuild() repository_ids = [app.security.decode_id(i) for i in id] rval = [get_installed_repository(app=app, repository_id=repo_id, from_cache=False) for repo_id in repository_ids] if return_list: @@ -377,7 +377,6 @@ def get_repository_for_dependency_relationship(app, tool_shed, name, owner, chan message += "required parameters is None: tool_shed: %s, name: %s, owner: %s, changeset_revision: %s " % \ (str(tool_shed), str(name), str(owner), str(changeset_revision)) raise Exception(message) - app.tool_shed_repository_cache.rebuild() repository = get_installed_repository(app=app, tool_shed=tool_shed, name=name, diff --git a/lib/galaxy/tool_shed/util/tool_dependency_util.py b/lib/galaxy/tool_shed/util/tool_dependency_util.py index 7105bb92f16..318cec119c7 100644 --- a/lib/galaxy/tool_shed/util/tool_dependency_util.py +++ b/lib/galaxy/tool_shed/util/tool_dependency_util.py @@ -388,5 +388,4 @@ def set_tool_dependency_attributes(app, tool_dependency, status, error_message=N tool_dependency.status = status sa_session.add(tool_dependency) sa_session.flush() - app.tool_shed_repository_cache.rebuild() return tool_dependency diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index ed58ec431e3..7fc3f810225 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -807,7 +807,9 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): repository_path, tool_path ) - self.app.tool_shed_repository_cache.add_local_repository(repository) + tsr_cache = self.app.tool_shed_repository_cache + if tsr_cache: + tsr_cache.add_local_repository(repository) return repository def _get_tool_shed_repository(self, tool_shed, name, owner, installed_changeset_revision): diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index ca23c0768c8..22f82fe488c 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -13,7 +13,6 @@ from sqlalchemy.orm import ( defer, joinedload, ) -from sqlalchemy.orm.scoping import scoped_session from sqlalchemy.orm.session import sessionmaker from sqlitedict import SqliteDict @@ -288,20 +287,20 @@ class ToolShedRepositoryCache: repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]] def __init__(self, session: sessionmaker): - engine = session().bind - self.session = scoped_session(sessionmaker(engine)) + self.session = session() # Contains ToolConfRepository objects created from shed_tool_conf.xml entries self.local_repositories = [] # Repositories loaded from database self.repositories = [] self.repos_by_tuple = defaultdict(list) - self.rebuild() + self._rebuild() + self.session.close() def add_local_repository(self, repository): self.local_repositories.append(repository) self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) - def rebuild(self): + def _rebuild(self): self.repositories = self.session.query(ToolShedRepository).options( defer(ToolShedRepository.metadata), joinedload('tool_dependencies') ).all() @@ -325,6 +324,3 @@ class ToolShedRepositoryCache: continue return repo return None - - def shutdown(self) -> None: - self.session.close() diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 7e92798353c..8ef04b5b1f0 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1241,8 +1241,7 @@ class AdminToolshed(AdminGalaxy): tool_shed=tool_shed_url, name=name, owner=owner, - changeset_revision=changeset_revision, - refresh=True) + changeset_revision=changeset_revision) if changeset_revision and latest_changeset_revision and latest_ctx_rev: if changeset_revision == latest_changeset_revision: message = f"The installed repository named '{name}' is current, there are no updates available. " diff --git a/test/unit/tools/test_tool_shed_repository_cache.py b/test/unit/tools/test_tool_shed_repository_cache.py index 26966703b07..9b4501f4f4d 100644 --- a/test/unit/tools/test_tool_shed_repository_cache.py +++ b/test/unit/tools/test_tool_shed_repository_cache.py @@ -9,21 +9,21 @@ def test_empty_repo_cache(tool_shed_repository_cache): def test_add_repository_to_repository_cache(tool_shed_repository_cache, repos): - tool_shed_repository_cache.rebuild() + tool_shed_repository_cache._rebuild() assert len(tool_shed_repository_cache.repositories) == 10 assert len(tool_shed_repository_cache.local_repositories) == 0 def test_add_repository_and_tool_conf_repository_to_repository_cache(tool_shed_repository_cache, repos, tool_conf_repos): - tool_shed_repository_cache.rebuild() + tool_shed_repository_cache._rebuild() assert len(tool_shed_repository_cache.repositories) == 10 assert len(tool_shed_repository_cache.local_repositories) == 10 - tool_shed_repository_cache.rebuild() + tool_shed_repository_cache._rebuild() assert len(tool_shed_repository_cache.repositories) == 10 assert len(tool_shed_repository_cache.local_repositories) == 10 create_repo(tool_shed_repository_cache.session, '21', '20') tool_shed_repository_cache.session.flush() - tool_shed_repository_cache.rebuild() + tool_shed_repository_cache._rebuild() assert len(tool_shed_repository_cache.repositories) == 11 assert len(tool_shed_repository_cache.local_repositories) == 10 @@ -41,7 +41,7 @@ def test_add_repository_and_tool_conf_repository_to_repository_cache(tool_shed_r ('github.com', 'example', 'galaxyproject', '19', '18', None, True), ]) def test_get_installed_repository(tool_shed_repository_cache, repos, tool_conf_repos, tool_shed, name, owner, changeset_revision, installed_changeset_revision, repository_id, repo_exists): - tool_shed_repository_cache.rebuild() + tool_shed_repository_cache._rebuild() repo = tool_shed_repository_cache.get_installed_repository( tool_shed=tool_shed, name=name, From afd94218f03e4fd507ba42a77bb33f649b382874 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Wed, 12 Jan 2022 19:48:16 +0100 Subject: [PATCH 2/3] Don't need to use a joinedload anymore The cache is only used on startup, not during dependency resolution. Also rename _rebuild, since we only use it once now. --- lib/galaxy/tools/cache.py | 11 ++++------- test/unit/tools/test_tool_shed_repository_cache.py | 10 +++++----- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 22f82fe488c..12e571e018d 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -9,10 +9,7 @@ from collections import defaultdict from threading import Lock from typing import Dict, List, Tuple -from sqlalchemy.orm import ( - defer, - joinedload, -) +from sqlalchemy.orm import defer from sqlalchemy.orm.session import sessionmaker from sqlitedict import SqliteDict @@ -293,16 +290,16 @@ class ToolShedRepositoryCache: # Repositories loaded from database self.repositories = [] self.repos_by_tuple = defaultdict(list) - self._rebuild() + self._build() self.session.close() def add_local_repository(self, repository): self.local_repositories.append(repository) self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) - def _rebuild(self): + def _build(self): self.repositories = self.session.query(ToolShedRepository).options( - defer(ToolShedRepository.metadata), joinedload('tool_dependencies') + defer(ToolShedRepository.metadata) ).all() repos_by_tuple = defaultdict(list) for repository in self.repositories + self.local_repositories: diff --git a/test/unit/tools/test_tool_shed_repository_cache.py b/test/unit/tools/test_tool_shed_repository_cache.py index 9b4501f4f4d..436147303ca 100644 --- a/test/unit/tools/test_tool_shed_repository_cache.py +++ b/test/unit/tools/test_tool_shed_repository_cache.py @@ -9,21 +9,21 @@ def test_empty_repo_cache(tool_shed_repository_cache): def test_add_repository_to_repository_cache(tool_shed_repository_cache, repos): - tool_shed_repository_cache._rebuild() + tool_shed_repository_cache._build() assert len(tool_shed_repository_cache.repositories) == 10 assert len(tool_shed_repository_cache.local_repositories) == 0 def test_add_repository_and_tool_conf_repository_to_repository_cache(tool_shed_repository_cache, repos, tool_conf_repos): - tool_shed_repository_cache._rebuild() + tool_shed_repository_cache._build() assert len(tool_shed_repository_cache.repositories) == 10 assert len(tool_shed_repository_cache.local_repositories) == 10 - tool_shed_repository_cache._rebuild() + tool_shed_repository_cache._build() assert len(tool_shed_repository_cache.repositories) == 10 assert len(tool_shed_repository_cache.local_repositories) == 10 create_repo(tool_shed_repository_cache.session, '21', '20') tool_shed_repository_cache.session.flush() - tool_shed_repository_cache._rebuild() + tool_shed_repository_cache._build() assert len(tool_shed_repository_cache.repositories) == 11 assert len(tool_shed_repository_cache.local_repositories) == 10 @@ -41,7 +41,7 @@ def test_add_repository_and_tool_conf_repository_to_repository_cache(tool_shed_r ('github.com', 'example', 'galaxyproject', '19', '18', None, True), ]) def test_get_installed_repository(tool_shed_repository_cache, repos, tool_conf_repos, tool_shed, name, owner, changeset_revision, installed_changeset_revision, repository_id, repo_exists): - tool_shed_repository_cache._rebuild() + tool_shed_repository_cache._build() repo = tool_shed_repository_cache.get_installed_repository( tool_shed=tool_shed, name=name, From bf712670f2d62673da16bd88916c48d43abe7cb3 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Jan 2022 13:35:36 +0100 Subject: [PATCH 3/3] Drop refresh argument of get_installed_repository --- lib/galaxy/tool_shed/util/repository_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_shed/util/repository_util.py b/lib/galaxy/tool_shed/util/repository_util.py index ccf64076234..e4ac37fc334 100644 --- a/lib/galaxy/tool_shed/util/repository_util.py +++ b/lib/galaxy/tool_shed/util/repository_util.py @@ -201,7 +201,7 @@ def get_ids_of_tool_shed_repositories_being_installed(app, as_string=False): return installing_repository_ids -def get_installed_repository(app, tool_shed=None, name=None, owner=None, changeset_revision=None, installed_changeset_revision=None, repository_id=None, refresh=False, from_cache=False): +def get_installed_repository(app, tool_shed=None, name=None, owner=None, changeset_revision=None, installed_changeset_revision=None, repository_id=None, from_cache=False): """ Return a tool shed repository database record defined by the combination of a toolshed, repository name, repository owner and either current or originally installed changeset_revision.