opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #13154 from mvdbeek/limit_repository_cache_lifetime

Browse Source 
[21.09] Limit lifetime of repository cache
This commit is contained in:
Marius van den Beek
2022-01-13 16:10:40 +01:00
committed by GitHub
parent a2c4f12a78 bf712670f2
commit 2a76f41841
10 changed files with 31 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/app.py
+2 -4
View File
@@ -232,7 +232,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
("job manager", self._shutdown_job_manager),
("application heartbeat", self._shutdown_heartbeat),
("repository manager", self._shutdown_repo_manager),
("database connection repository cache", self._shutdown_repo_cache),
("database connection", self._shutdown_model),
("application stack", self._shutdown_application_stack),
]
@@ -384,6 +383,8 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
self.url_for = url_for
self.server_starttime = int(time.time()) # used for cachebusting
# Limit lifetime of tool shed repository cache to app startup
self.tool_shed_repository_cache = None
log.info(f"Galaxy app startup finished {startup_timer}")
def _shutdown_queue_worker(self):
@@ -408,9 +409,6 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
def _shutdown_repo_manager(self):
self.update_repository_manager.shutdown()
def _shutdown_repo_cache(self):
self.tool_shed_repository_cache.shutdown()
def _shutdown_application_stack(self):
self.application_stack.shutdown()
lib/galaxy/queue_worker.py
-4
View File
@@ -181,8 +181,6 @@ def _get_new_toolbox(app, save_integrated_tool_panel=True):
"""
from galaxy import tools
from galaxy.tools.special_tools import load_lib_tools
if hasattr(app, 'tool_shed_repository_cache'):
app.tool_shed_repository_cache.rebuild()
tool_configs = app.config.tool_configs
new_toolbox = tools.ToolBox(tool_configs, app.config.tool_path, app, save_integrated_tool_panel=save_integrated_tool_panel)
@@ -199,8 +197,6 @@ def reload_data_managers(app, **kwargs):
reload_timer = util.ExecutionTimer()
from galaxy.tools.data_manager.manager import DataManagers
log.debug("Executing data managers reload on '%s'", app.config.server_name)
if hasattr(app, 'tool_shed_repository_cache'):
app.tool_shed_repository_cache.rebuild()
app._configure_tool_data_tables(from_shed_config=False)
reload_tool_data_tables(app)
reload_count = app.data_managers._reload_count
lib/galaxy/structured_app.py
+1 -1
View File
@@ -112,7 +112,7 @@ class StructuredApp(MinimalManagerApp):
error_reports: Any # 'galaxy.tools.error_reports.ErrorReports'
job_config: Any # 'galaxy.jobs.JobConfiguration'
tool_cache: Any # 'galaxy.tools.cache.ToolCache'
tool_shed_repository_cache: Any # 'galaxy.tools.cache.ToolShedRepositoryCache'
tool_shed_repository_cache: Optional[Any] # 'galaxy.tools.cache.ToolShedRepositoryCache'
watchers: Any # 'galaxy.config_watchers.ConfigWatchers'
installed_repository_manager: Any # 'galaxy.tool_shed.galaxy_install.installed_repository_manager.InstalledRepositoryManager'
workflow_scheduling_manager: Any # 'galaxy.workflow.scheduling_manager.WorkflowSchedulingManager'
lib/galaxy/tool_shed/galaxy_install/metadata/installed_repository_metadata_manager.py
-1
View File
@@ -94,7 +94,6 @@ class InstalledRepositoryMetadataManager(MetadataGenerator):
self.update_in_shed_tool_config()
self.app.install_model.context.add(self.repository)
self.app.install_model.context.flush()
self.app.tool_shed_repository_cache.rebuild()
log.debug(f'Metadata has been reset on repository {self.repository.name}.')
else:
log.debug(f'Metadata did not need to be reset on repository {self.repository.name}.')
lib/galaxy/tool_shed/util/repository_util.py
+13 -14
View File
@@ -106,7 +106,7 @@ def create_or_update_tool_shed_repository(app, name, description, installed_chan
deleted = False
uninstalled = False
tool_shed_repository = \
get_installed_repository(app, tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=installed_changeset_revision, refresh=True)
get_installed_repository(app, tool_shed=tool_shed, name=name, owner=owner, installed_changeset_revision=installed_changeset_revision)
if tool_shed_repository:
log.debug("Updating an existing row for repository '%s' in the tool_shed_repository table, status set to '%s'.", name, status)
tool_shed_repository.description = description
@@ -201,22 +201,24 @@ def get_ids_of_tool_shed_repositories_being_installed(app, as_string=False):
return installing_repository_ids
def get_installed_repository(app, tool_shed=None, name=None, owner=None, changeset_revision=None, installed_changeset_revision=None, repository_id=None, refresh=False, from_cache=False):
def get_installed_repository(app, tool_shed=None, name=None, owner=None, changeset_revision=None, installed_changeset_revision=None, repository_id=None, from_cache=False):
"""
Return a tool shed repository database record defined by the combination of a toolshed, repository name,
repository owner and either current or originally installed changeset_revision.
"""
# We store the port, if one exists, in the database.
tool_shed = common_util.remove_protocol_from_tool_shed_url(tool_shed)
if from_cache and hasattr(app, 'tool_shed_repository_cache'):
if refresh:
app.tool_shed_repository_cache.rebuild()
return app.tool_shed_repository_cache.get_installed_repository(tool_shed=tool_shed,
name=name,
owner=owner,
installed_changeset_revision=installed_changeset_revision,
changeset_revision=changeset_revision,
repository_id=repository_id)
if from_cache:
tsr_cache = getattr(app, 'tool_shed_repository_cache', None)
if tsr_cache:
return app.tool_shed_repository_cache.get_installed_repository(
tool_shed=tool_shed,
name=name,
owner=owner,
installed_changeset_revision=installed_changeset_revision,
changeset_revision=changeset_revision,
repository_id=repository_id
)
query = app.install_model.context.query(app.install_model.ToolShedRepository)
if repository_id:
clause_list = [app.install_model.ToolShedRepository.table.c.id == repository_id]
@@ -239,8 +241,6 @@ def get_installed_tool_shed_repository(app, id):
else:
id = [id]
return_list = False
if hasattr(app, 'tool_shed_repository_cache'):
app.tool_shed_repository_cache.rebuild()
repository_ids = [app.security.decode_id(i) for i in id]
rval = [get_installed_repository(app=app, repository_id=repo_id, from_cache=False) for repo_id in repository_ids]
if return_list:
@@ -377,7 +377,6 @@ def get_repository_for_dependency_relationship(app, tool_shed, name, owner, chan
message += "required parameters is None: tool_shed: %s, name: %s, owner: %s, changeset_revision: %s " % \
(str(tool_shed), str(name), str(owner), str(changeset_revision))
raise Exception(message)
app.tool_shed_repository_cache.rebuild()
repository = get_installed_repository(app=app,
tool_shed=tool_shed,
name=name,
lib/galaxy/tool_shed/util/tool_dependency_util.py
-1
View File
@@ -388,5 +388,4 @@ def set_tool_dependency_attributes(app, tool_dependency, status, error_message=N
tool_dependency.status = status
sa_session.add(tool_dependency)
sa_session.flush()
app.tool_shed_repository_cache.rebuild()
return tool_dependency
lib/galaxy/tool_util/toolbox/base.py
+3 -1
View File
@@ -807,7 +807,9 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
repository_path,
tool_path
)
self.app.tool_shed_repository_cache.add_local_repository(repository)
tsr_cache = self.app.tool_shed_repository_cache
if tsr_cache:
tsr_cache.add_local_repository(repository)
return repository
def _get_tool_shed_repository(self, tool_shed, name, owner, installed_changeset_revision):
lib/galaxy/tools/cache.py
+6 -13
View File
@@ -9,11 +9,7 @@ from collections import defaultdict
from threading import Lock
from typing import Dict, List, Tuple
from sqlalchemy.orm import (
defer,
joinedload,
)
from sqlalchemy.orm.scoping import scoped_session
from sqlalchemy.orm import defer
from sqlalchemy.orm.session import sessionmaker
from sqlitedict import SqliteDict
@@ -288,22 +284,22 @@ class ToolShedRepositoryCache:
repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]]
def __init__(self, session: sessionmaker):
engine = session().bind
self.session = scoped_session(sessionmaker(engine))
self.session = session()
# Contains ToolConfRepository objects created from shed_tool_conf.xml entries
self.local_repositories = []
# Repositories loaded from database
self.repositories = []
self.repos_by_tuple = defaultdict(list)
self.rebuild()
self._build()
self.session.close()
def add_local_repository(self, repository):
self.local_repositories.append(repository)
self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
def rebuild(self):
def _build(self):
self.repositories = self.session.query(ToolShedRepository).options(
defer(ToolShedRepository.metadata), joinedload('tool_dependencies')
defer(ToolShedRepository.metadata)
).all()
repos_by_tuple = defaultdict(list)
for repository in self.repositories + self.local_repositories:
@@ -325,6 +321,3 @@ class ToolShedRepositoryCache:
continue
return repo
return None
def shutdown(self) -> None:
self.session.close()
lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py
+1 -2
View File
@@ -1241,8 +1241,7 @@ class AdminToolshed(AdminGalaxy):
tool_shed=tool_shed_url,
name=name,
owner=owner,
changeset_revision=changeset_revision,
refresh=True)
changeset_revision=changeset_revision)
if changeset_revision and latest_changeset_revision and latest_ctx_rev:
if changeset_revision == latest_changeset_revision:
message = f"The installed repository named '{name}' is current, there are no updates available. "
test/unit/tools/test_tool_shed_repository_cache.py
+5 -5
View File
@@ -9,21 +9,21 @@ def test_empty_repo_cache(tool_shed_repository_cache):
def test_add_repository_to_repository_cache(tool_shed_repository_cache, repos):
tool_shed_repository_cache.rebuild()
tool_shed_repository_cache._build()
assert len(tool_shed_repository_cache.repositories) == 10
assert len(tool_shed_repository_cache.local_repositories) == 0
def test_add_repository_and_tool_conf_repository_to_repository_cache(tool_shed_repository_cache, repos, tool_conf_repos):
tool_shed_repository_cache.rebuild()
tool_shed_repository_cache._build()
assert len(tool_shed_repository_cache.repositories) == 10
assert len(tool_shed_repository_cache.local_repositories) == 10
tool_shed_repository_cache.rebuild()
tool_shed_repository_cache._build()
assert len(tool_shed_repository_cache.repositories) == 10
assert len(tool_shed_repository_cache.local_repositories) == 10
create_repo(tool_shed_repository_cache.session, '21', '20')
tool_shed_repository_cache.session.flush()
tool_shed_repository_cache.rebuild()
tool_shed_repository_cache._build()
assert len(tool_shed_repository_cache.repositories) == 11
assert len(tool_shed_repository_cache.local_repositories) == 10
@@ -41,7 +41,7 @@ def test_add_repository_and_tool_conf_repository_to_repository_cache(tool_shed_r
('github.com', 'example', 'galaxyproject', '19', '18', None, True),
])
def test_get_installed_repository(tool_shed_repository_cache, repos, tool_conf_repos, tool_shed, name, owner, changeset_revision, installed_changeset_revision, repository_id, repo_exists):
tool_shed_repository_cache.rebuild()
tool_shed_repository_cache._build()
repo = tool_shed_repository_cache.get_installed_repository(
tool_shed=tool_shed,
name=name,