diff --git a/lib/galaxy/queue_worker/__init__.py b/lib/galaxy/queue_worker/__init__.py index a998c012f00..74ef479a7da 100644 --- a/lib/galaxy/queue_worker/__init__.py +++ b/lib/galaxy/queue_worker/__init__.py @@ -296,6 +296,7 @@ def _get_new_toolbox(app: "UniverseApplication", save_integrated_tool_panel: boo tool_configs = app.config.tool_configs with app._toolbox_lock: + old_toolbox = app._toolbox new_toolbox: ToolBox if getattr(app.config, "use_lazy_toolbox", False) and getattr(app, "tool_source_store", None) is not None: new_toolbox = LazyToolBox( @@ -320,6 +321,15 @@ def _get_new_toolbox(app: "UniverseApplication", save_integrated_tool_panel: boo for tool in new_toolbox.data_manager_tools.values(): new_toolbox.register_tool(tool) app._toolbox = new_toolbox + # Retire the superseded box's background store watcher. In lazy mode + # each toolbox spawns a ToolSourceStoreWatcher daemon bound to itself; + # without this every reload leaks a polling thread that keeps grabbing + # app._toolbox_lock and disposing the shared store's engines on each + # republish. Only the watcher is stopped — in-flight reads on other + # threads still work, and the shared app.tool_source_store stays open. + # Eager ToolBoxes have no watcher, so guard on the lazy type. + if isinstance(old_toolbox, LazyToolBox) and old_toolbox is not new_toolbox: + old_toolbox.stop_watcher() def reload_data_managers(app, **kwargs): diff --git a/lib/galaxy/tools/lazy_toolbox.py b/lib/galaxy/tools/lazy_toolbox.py index 437669c37d1..9e5b073c8d8 100644 --- a/lib/galaxy/tools/lazy_toolbox.py +++ b/lib/galaxy/tools/lazy_toolbox.py @@ -1699,6 +1699,18 @@ class LazyToolBox(ToolBox): if entry and entry.hidden: tool.hidden = True + def stop_watcher(self) -> None: + """Stop the background store-freshness watcher, if one is running. + + The reload path uses this to retire a superseded toolbox's watcher + thread without the class-level ``ToolLineage.reset()`` and index/store + teardown of ``close()`` — the replacement box is already live and the + shared ``tool_source_store`` must stay open. Idempotent. + """ + if self._store_watcher is not None: + self._store_watcher.shutdown() + self._store_watcher = None + def close(self) -> None: """Drop in-memory state at app shutdown. @@ -1708,9 +1720,7 @@ class LazyToolBox(ToolBox): ``tool_source_store`` before the next boot wires up a fresh toolbox. Idempotent; safe to call more than once. """ - if self._store_watcher is not None: - self._store_watcher.shutdown() - self._store_watcher = None + self.stop_watcher() with self._cache_lock: self._tool_object_cache.clear() self._tool_index = None diff --git a/test/unit/app/queue_worker/test_reload_toolbox_watcher.py b/test/unit/app/queue_worker/test_reload_toolbox_watcher.py new file mode 100644 index 00000000000..0c87558f704 --- /dev/null +++ b/test/unit/app/queue_worker/test_reload_toolbox_watcher.py @@ -0,0 +1,68 @@ +import threading +from types import SimpleNamespace +from unittest.mock import MagicMock + +from galaxy import queue_worker +from galaxy.queue_worker import _get_new_toolbox +from galaxy.tools.lazy_toolbox import LazyToolBox + + +class FakeWatcher: + def __init__(self): + self.stopped = False + + def shutdown(self): + self.stopped = True + + +def _fake_new_toolbox(*args, **kwargs): + box = SimpleNamespace(data_manager_tools={}) + box.register_tool = lambda tool: None + return box + + +def _fake_app(old_toolbox): + app = SimpleNamespace() + app._toolbox_lock = threading.RLock() + app._toolbox = old_toolbox + app.toolbox = old_toolbox + app.datatypes_registry = MagicMock() + app.tool_source_store = None + app.config = SimpleNamespace( + use_lazy_toolbox=False, + tool_configs=[], + tool_path="/tmp/tools", + lazy_toolbox_cache_size=500, + ) + return app + + +def _patch_builders(monkeypatch): + monkeypatch.setattr(queue_worker, "ToolBox", _fake_new_toolbox) + monkeypatch.setattr(queue_worker, "load_lib_tools", lambda toolbox: None) + + +def test_replacement_stops_superseded_lazy_watcher(monkeypatch): + _patch_builders(monkeypatch) + old = LazyToolBox.__new__(LazyToolBox) + old.data_manager_tools = {} + watcher = FakeWatcher() + old._store_watcher = watcher # type: ignore[assignment] + app = _fake_app(old) + + _get_new_toolbox(app) + + assert watcher.stopped is True + assert old._store_watcher is None + assert app._toolbox is not old + + +def test_eager_old_toolbox_is_left_untouched(monkeypatch): + _patch_builders(monkeypatch) + old = SimpleNamespace(data_manager_tools={}) + app = _fake_app(old) + + _get_new_toolbox(app) + + assert app._toolbox is not old + assert not hasattr(old, "_store_watcher")