From 7bf6a581249ec409d33ccc060816b9f505185165 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 8 Mar 2017 20:48:21 +0100 Subject: [PATCH 1/3] Remove outdated items from tool cache --- lib/galaxy/queue_worker.py | 1 + lib/galaxy/tools/toolbox/cache.py | 16 ++++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/lib/galaxy/queue_worker.py b/lib/galaxy/queue_worker.py index 0ca7b928d77..c63db688621 100644 --- a/lib/galaxy/queue_worker.py +++ b/lib/galaxy/queue_worker.py @@ -87,6 +87,7 @@ def reload_tool(app, **kwargs): def reload_toolbox(app, **kwargs): log.debug("Executing toolbox reload on '%s'", app.config.server_name) reload_count = app.toolbox._reload_count + app.tool_cache.cleanup() app.toolbox = _get_new_toolbox(app) app.toolbox._reload_count = reload_count + 1 diff --git a/lib/galaxy/tools/toolbox/cache.py b/lib/galaxy/tools/toolbox/cache.py index 2c2968ad12e..fb800c24f7f 100644 --- a/lib/galaxy/tools/toolbox/cache.py +++ b/lib/galaxy/tools/toolbox/cache.py @@ -1,3 +1,6 @@ +import os + +from galaxy.util.hash_util import md5_hash_file class ToolCache(object): @@ -7,9 +10,20 @@ class ToolCache(object): """ def __init__(self): + self._hash_by_tool_paths = {} self._tools_by_path = {} self._tool_paths_by_id = {} + def cleanup(self): + """Remove uninstalled tools from tool cache if they are not on disk anymore or if their content has changed.""" + clean_paths = {path: tool.all_ids for path, tool in self._tools_by_path.items() if not os.path.exists(path) or md5_hash_file(path) != self._hash_by_tool_paths[path]} + for path, tool_ids in clean_paths.items(): + del self._hash_by_tool_paths[path] + del self._tools_by_path[path] + for tool_id in tool_ids: + if tool_id in self._tool_paths_by_id: + del self._tool_paths_by_id[tool_id] + def get_tool(self, config_filename): """ Get the tool from the cache if the tool is up to date. """ @@ -22,6 +36,8 @@ class ToolCache(object): del self._tools_by_path[config_filename] def cache_tool(self, config_filename, tool): + tool_hash = md5_hash_file(config_filename) tool_id = str( tool.id ) + self._hash_by_tool_paths[config_filename] = tool_hash self._tool_paths_by_id[tool_id] = config_filename self._tools_by_path[config_filename] = tool From 0d80b8807a70ff94a6f8dbee6279207c53869755 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 9 Mar 2017 13:32:42 +0100 Subject: [PATCH 2/3] Remove hash when expiring tool from tool cache and rename `clean_paths` to `paths_to_cleanup` (thx @nsoranzo). --- lib/galaxy/tools/toolbox/cache.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tools/toolbox/cache.py b/lib/galaxy/tools/toolbox/cache.py index fb800c24f7f..57de2a276ca 100644 --- a/lib/galaxy/tools/toolbox/cache.py +++ b/lib/galaxy/tools/toolbox/cache.py @@ -16,8 +16,8 @@ class ToolCache(object): def cleanup(self): """Remove uninstalled tools from tool cache if they are not on disk anymore or if their content has changed.""" - clean_paths = {path: tool.all_ids for path, tool in self._tools_by_path.items() if not os.path.exists(path) or md5_hash_file(path) != self._hash_by_tool_paths[path]} - for path, tool_ids in clean_paths.items(): + paths_to_cleanup = {path: tool.all_ids for path, tool in self._tools_by_path.items() if not os.path.exists(path) or md5_hash_file(path) != self._hash_by_tool_paths[path]} + for path, tool_ids in paths_to_cleanup.items(): del self._hash_by_tool_paths[path] del self._tools_by_path[path] for tool_id in tool_ids: @@ -32,6 +32,7 @@ class ToolCache(object): def expire_tool(self, tool_id): if tool_id in self._tool_paths_by_id: config_filename = self._tool_paths_by_id[tool_id] + del self._hash_by_tool_paths[path] del self._tool_paths_by_id[tool_id] del self._tools_by_path[config_filename] From 7b375664634075c1ad261eff42a2355637cfc363 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 9 Mar 2017 14:31:53 +0100 Subject: [PATCH 3/3] Use config_filename throughout ToolCache --- lib/galaxy/tools/toolbox/cache.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/toolbox/cache.py b/lib/galaxy/tools/toolbox/cache.py index 57de2a276ca..ee576d1d865 100644 --- a/lib/galaxy/tools/toolbox/cache.py +++ b/lib/galaxy/tools/toolbox/cache.py @@ -17,9 +17,9 @@ class ToolCache(object): def cleanup(self): """Remove uninstalled tools from tool cache if they are not on disk anymore or if their content has changed.""" paths_to_cleanup = {path: tool.all_ids for path, tool in self._tools_by_path.items() if not os.path.exists(path) or md5_hash_file(path) != self._hash_by_tool_paths[path]} - for path, tool_ids in paths_to_cleanup.items(): - del self._hash_by_tool_paths[path] - del self._tools_by_path[path] + for config_filename, tool_ids in paths_to_cleanup.items(): + del self._hash_by_tool_paths[config_filename] + del self._tools_by_path[config_filename] for tool_id in tool_ids: if tool_id in self._tool_paths_by_id: del self._tool_paths_by_id[tool_id] @@ -32,7 +32,7 @@ class ToolCache(object): def expire_tool(self, tool_id): if tool_id in self._tool_paths_by_id: config_filename = self._tool_paths_by_id[tool_id] - del self._hash_by_tool_paths[path] + del self._hash_by_tool_paths[config_filename] del self._tool_paths_by_id[tool_id] del self._tools_by_path[config_filename]