From ad68b154d4cbecdbc2e96e58d9d3c4153e014eeb Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 24 Mar 2017 16:30:15 +0100 Subject: [PATCH] Make ToolConfWatcher watch ToolCache If any item in the tool cache has changed (local tool update, repository update, tool deleted) the toolbox will be reloaded and any tool changes are applied. Should fix https://github.com/galaxyproject/galaxy/issues/3813. --- lib/galaxy/tools/toolbox/base.py | 2 +- lib/galaxy/tools/toolbox/cache.py | 25 +++++++++++++++++++++++-- lib/galaxy/tools/toolbox/watcher.py | 13 +++++++++---- test/unit/tools/test_toolbox.py | 1 + 4 files changed, 34 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 402e490bb47..b432f3df69c 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -79,7 +79,7 @@ class AbstractToolBox( Dictifiable, ManagesIntegratedToolPanelMixin, object ): if tool_conf_watcher: self._tool_conf_watcher = tool_conf_watcher # Avoids (re-)starting threads in uwsgi else: - self._tool_conf_watcher = get_tool_conf_watcher(lambda: self.handle_reload_toolbox()) + self._tool_conf_watcher = get_tool_conf_watcher(reload_callback=lambda: self.handle_reload_toolbox(), tool_cache=self.app.tool_cache) self._filter_factory = FilterFactory( self ) self._tool_tag_manager = tool_tag_manager( app ) self._init_tools_from_configs( config_filenames ) diff --git a/lib/galaxy/tools/toolbox/cache.py b/lib/galaxy/tools/toolbox/cache.py index ee576d1d865..e1036f1bec7 100644 --- a/lib/galaxy/tools/toolbox/cache.py +++ b/lib/galaxy/tools/toolbox/cache.py @@ -1,4 +1,5 @@ import os +import time from galaxy.util.hash_util import md5_hash_file @@ -13,16 +14,34 @@ class ToolCache(object): self._hash_by_tool_paths = {} self._tools_by_path = {} self._tool_paths_by_id = {} + self._mod_time_by_path = {} def cleanup(self): - """Remove uninstalled tools from tool cache if they are not on disk anymore or if their content has changed.""" - paths_to_cleanup = {path: tool.all_ids for path, tool in self._tools_by_path.items() if not os.path.exists(path) or md5_hash_file(path) != self._hash_by_tool_paths[path]} + """ + Remove uninstalled tools from tool cache if they are not on disk anymore or if their content has changed. + + Returns list of tool_ids that have been removed. + """ + paths_to_cleanup = {path: tool.all_ids for path, tool in self._tools_by_path.items() if self._should_cleanup(path)} + removed_tool_ids = [] for config_filename, tool_ids in paths_to_cleanup.items(): + removed_tool_ids.extend(tool_ids) del self._hash_by_tool_paths[config_filename] del self._tools_by_path[config_filename] for tool_id in tool_ids: if tool_id in self._tool_paths_by_id: del self._tool_paths_by_id[tool_id] + return removed_tool_ids + + def _should_cleanup(self, config_filename): + """Return True of `config_filename` does not exist or if modtime and hash have changes, else return False.""" + if not os.path.exists(config_filename): + return True + new_mtime = time.ctime(os.path.getmtime(config_filename)) + if self._mod_time_by_path[config_filename] != new_mtime: + if md5_hash_file(config_filename) != self._hash_by_tool_paths[config_filename]: + return True + return False def get_tool(self, config_filename): """ Get the tool from the cache if the tool is up to date. @@ -35,10 +54,12 @@ class ToolCache(object): del self._hash_by_tool_paths[config_filename] del self._tool_paths_by_id[tool_id] del self._tools_by_path[config_filename] + del self._mod_time_by_path[config_filename] def cache_tool(self, config_filename, tool): tool_hash = md5_hash_file(config_filename) tool_id = str( tool.id ) self._hash_by_tool_paths[config_filename] = tool_hash + self._mod_time_by_path[config_filename] = time.ctime(os.path.getmtime(config_filename)) self._tool_paths_by_id[tool_id] = config_filename self._tools_by_path[config_filename] = tool diff --git a/lib/galaxy/tools/toolbox/watcher.py b/lib/galaxy/tools/toolbox/watcher.py index 862b30017ee..cbd9fa75a4f 100644 --- a/lib/galaxy/tools/toolbox/watcher.py +++ b/lib/galaxy/tools/toolbox/watcher.py @@ -48,8 +48,8 @@ def get_observer_class(config_value, default, monitor_what_str): return observer_class -def get_tool_conf_watcher(reload_callback): - return ToolConfWatcher(reload_callback) +def get_tool_conf_watcher(reload_callback, tool_cache=None): + return ToolConfWatcher(reload_callback=reload_callback, tool_cache=tool_cache) def get_tool_data_dir_watcher(tool_data_tables, config): @@ -73,8 +73,9 @@ def get_tool_watcher(toolbox, config): class ToolConfWatcher(object): - def __init__(self, reload_callback): + def __init__(self, reload_callback, tool_cache=None): self.paths = {} + self.cache = tool_cache self._active = False self._lock = threading.Lock() self.thread = threading.Thread(target=self.check, name="ToolConfWatcher.thread") @@ -92,6 +93,7 @@ class ToolConfWatcher(object): self.thread.join() def check(self): + """Check for changes in self.paths or self.cache and call the event handler.""" hashes = { key: None for key in self.paths.keys() } while self._active: do_reload = False @@ -113,7 +115,10 @@ class ToolConfWatcher(object): hashes[path] = new_hash log.debug("The file '%s' has changes.", path) do_reload = True - + if not do_reload and self.cache: + removed_ids = self.cache.cleanup() + if removed_ids: + do_reload = True if do_reload: with self._lock: t = threading.Thread(target=self.event_handler.on_any_event) diff --git a/test/unit/tools/test_toolbox.py b/test/unit/tools/test_toolbox.py index 6e9a2729fea..a3856640aa1 100644 --- a/test/unit/tools/test_toolbox.py +++ b/test/unit/tools/test_toolbox.py @@ -57,6 +57,7 @@ class BaseToolBoxTestCase( unittest.TestCase, tools_support.UsesApp, tools_supp self.reindexed = False self.setup_app( mock_model=False ) install_model = mapping.init( "sqlite:///:memory:", create_tables=True ) + self.app.tool_cache = None self.app.install_model = install_model self.app.reindex_tool_search = self.__reindex itp_config = os.path.join(self.test_directory, "integrated_tool_panel.xml")