From f16e7fcec4d87ec21ee2b8a5f02387ca1020a6af Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 8 Jul 2026 15:24:20 +0200 Subject: [PATCH] watch tool source stores: poll freshness, reload on CVMFS publish MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit New opt-in watch_tool_source_stores (+ tool_source_store_watch_interval, default 60s): a per-process thread polls the freshness probe of every read-only named store — one extended-attribute read per store per tick for CVMFS — and reacts to token transitions. inotify does not fire on CVMFS, so polling is the only reliable signal, and catalog TTL bounds propagation anyway. On a transition the store's engine is disposed before the index reload: CVMFS keeps serving the pre-publish file to descriptors that were open before the catalog update, so pooled sqlite connections pin the old snapshot until dropped. The same disposal now also runs for read-only members in invalidate_index_cache, making the existing admin reload API and reload_tool_source_cache broadcasts correct on CVMFS as the manual trigger. The reload itself reuses invalidate_index_cache (stub registration, removal reconcile) plus a per-changed-store whoosh rebuild guarded by the corpus-signature skip. Watchable stores are read-only members with a probe: writable stores change through this process group's own populate paths, which already broadcast their own reloads. The watcher compares against the last *seen* token, not the persisted one, so a store whose publisher hasn't repopulated yet logs once instead of re-firing every tick. Claude-Session: https://claude.ai/code/session_018L7ZmCv2ubKA3JNeSL8Pkr --- doc/source/admin/galaxy_options.rst | 29 ++++++++ .../_galaxy_config_schema_attributes.py | 2 + lib/galaxy/config/sample/galaxy.yml.sample | 15 ++++ lib/galaxy/config/schemas/config_schema.yml | 23 +++++++ lib/galaxy/tools/lazy_toolbox.py | 69 +++++++++++++++++++ lib/galaxy/tools/source_store/watcher.py | 3 +- 6 files changed, 140 insertions(+), 1 deletion(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 1bd2ecb3867..7ba4dfcf33f 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -499,6 +499,35 @@ :Type: int +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``watch_tool_source_stores`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Poll the freshness probes of read-only tool source stores and + reload the lazy toolbox index when a store's token changes (for + example, a new CVMFS repository revision was published). + Filesystem event watchers do not fire on CVMFS, so polling is the + only reliable signal; each poll costs one extended-attribute read + per watched store. Only meaningful with ``use_lazy_toolbox`` and a + ``tool_source_stores`` entry declaring a freshness probe. The + admin toolbox-reload API remains available as a manual trigger. +:Default: ``false`` +:Type: bool + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``tool_source_store_watch_interval`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Seconds between freshness polls when ``watch_tool_source_stores`` + is enabled. CVMFS clients only refresh their catalogs every few + minutes, so sub-minute polling buys nothing. +:Default: ``60.0`` +:Type: float + + ~~~~~~~~~~~~~~~~~~~~~~~ ``tool_dependency_dir`` ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/lib/galaxy/config/_galaxy_config_schema_attributes.py b/lib/galaxy/config/_galaxy_config_schema_attributes.py index 110c3c9e662..5ca504b0fc6 100644 --- a/lib/galaxy/config/_galaxy_config_schema_attributes.py +++ b/lib/galaxy/config/_galaxy_config_schema_attributes.py @@ -41,6 +41,8 @@ class GalaxyAppConfigurationAttributes: tool_source_stores: Any use_lazy_toolbox: bool lazy_toolbox_cache_size: int + watch_tool_source_stores: bool + tool_source_store_watch_interval: float tool_dependency_dir: str | None dependency_resolvers_config_file: str conda_prefix: str | None diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index a7ea56ea6f7..33c5a97018e 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -638,6 +638,21 @@ galaxy: # parsing cost for popular tools at the expense of memory. #lazy_toolbox_cache_size: 500 + # Poll the freshness probes of read-only tool source stores and reload + # the lazy toolbox index when a store's token changes (for example, a + # new CVMFS repository revision was published). Filesystem event + # watchers do not fire on CVMFS, so polling is the only reliable + # signal; each poll costs one extended-attribute read per watched + # store. Only meaningful with ``use_lazy_toolbox`` and a + # ``tool_source_stores`` entry declaring a freshness probe. The admin + # toolbox-reload API remains available as a manual trigger. + #watch_tool_source_stores: false + + # Seconds between freshness polls when ``watch_tool_source_stores`` is + # enabled. CVMFS clients only refresh their catalogs every few + # minutes, so sub-minute polling buys nothing. + #tool_source_store_watch_interval: 60.0 + # Various dependency resolver configuration parameters will have # defaults set relative to this path, such as the default conda # prefix, default Galaxy packages path, legacy tool shed dependencies diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 91fa452922a..d311534d526 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -390,6 +390,29 @@ mapping: in its in-memory LRU cache. Larger values reduce repeat parsing cost for popular tools at the expense of memory. + watch_tool_source_stores: + type: bool + default: false + required: false + desc: | + Poll the freshness probes of read-only tool source stores and + reload the lazy toolbox index when a store's token changes (for + example, a new CVMFS repository revision was published). + Filesystem event watchers do not fire on CVMFS, so polling is the + only reliable signal; each poll costs one extended-attribute read + per watched store. Only meaningful with ``use_lazy_toolbox`` and + a ``tool_source_stores`` entry declaring a freshness probe. The + admin toolbox-reload API remains available as a manual trigger. + + tool_source_store_watch_interval: + type: float + default: 60.0 + required: false + desc: | + Seconds between freshness polls when ``watch_tool_source_stores`` + is enabled. CVMFS clients only refresh their catalogs every few + minutes, so sub-minute polling buys nothing. + tool_dependency_dir: type: str default: dependencies diff --git a/lib/galaxy/tools/lazy_toolbox.py b/lib/galaxy/tools/lazy_toolbox.py index 03f908c6b84..f931d5a98f5 100644 --- a/lib/galaxy/tools/lazy_toolbox.py +++ b/lib/galaxy/tools/lazy_toolbox.py @@ -43,11 +43,13 @@ from galaxy.tools.source_store.index import ( ToolIndexEntry, ) from galaxy.tools.source_store.populator import ( + build_whoosh_for_store, conf_to_store_map, DEFAULT_STORE_NAME, populate_for_paths, populate_store_inline, ) +from galaxy.tools.source_store.watcher import ToolSourceStoreWatcher from galaxy.util.tool_version import remove_version_from_guid from . import ( create_tool_from_source, @@ -494,6 +496,7 @@ class LazyToolBox(ToolBox): # only the guid landed in the panel. Filled after the eager walk # via ``_rebuild_shed_short_id_map``. self._shed_short_id_to_guids: dict[str, set[str]] = {} + self._store_watcher: ToolSourceStoreWatcher | None = None # Eager init — its ``_init_tools_from_configs`` is overridden so the # walk goes through our ``create_tool`` seam and hands back LazyTool @@ -519,6 +522,59 @@ class LazyToolBox(ToolBox): self._tools_loaded_from_store, ) + self._start_store_watcher() + + def _start_store_watcher(self) -> None: + """Poll externally-published stores for freshness-token changes. + + Only read-only members with a probe are watched: writable stores + change through this process's own populate paths, which broadcast + their own reloads, and CVMFS (the read-only publishing model) + delivers no filesystem events to react to — polling one + extended-attribute read per store per tick is the whole cost. + """ + if not self.app.config.watch_tool_source_stores: + return + if not isinstance(self._store, CompositeToolSourceStore): + log.info("watch_tool_source_stores is enabled but no named tool source stores are configured") + return + members = [(n, m) for n, m in self._store.members if m.read_only and m.has_freshness_probe] + if not members: + log.info("watch_tool_source_stores is enabled but no read-only store declares a freshness probe") + return + self._store_watcher = ToolSourceStoreWatcher( + members=members, + interval=self.app.config.tool_source_store_watch_interval, + on_change=self._on_store_freshness_change, + ) + self._store_watcher.start() + log.info( + "Watching tool source store(s) %s for freshness changes every %gs", + sorted(n for n, _ in members), + self.app.config.tool_source_store_watch_interval, + ) + + def _on_store_freshness_change(self, changed_names: list[str]) -> None: + """A watched store was republished: reload index state and search. + + ``invalidate_index_cache`` handles the reload dance (it also + disposes read-only members' engines — see there). The whoosh + rebuild runs here rather than in the reload path because only a + republished store can grow the corpus without a local populate; + its corpus-signature check makes re-runs no-ops, and concurrent + rebuilds from peer processes degrade to one winner (whoosh lock, + errors swallowed and logged by ``build_whoosh_for_store``). + """ + self.invalidate_index_cache() + if not isinstance(self._store, CompositeToolSourceStore): + return + for name, member in self._store.members: + if name not in changed_names: + continue + index = member.load_index() + if index is not None: + build_whoosh_for_store(self.app.config, name, index) + def _init_tools_from_configs(self, config_filenames: list[str]) -> None: """Load the persistent ``ToolIndex`` before delegating to the eager walk. @@ -1142,6 +1198,16 @@ class LazyToolBox(ToolBox): # tool right after the install's own broadcast). with self.app._toolbox_lock: try: + # Read-only members are published externally (CVMFS); a + # descriptor opened before the publish keeps serving the old + # snapshot forever, so dropping pooled connections — not just + # the cached index — is what makes the re-read below actually + # see the new file. Writable stores were updated through this + # process group's own connections and only need the cache drop. + if isinstance(self._store, CompositeToolSourceStore): + for _name, member in self._store.members: + if member.read_only: + member.dispose() self._store.invalidate_index_cache() except Exception as e: log.debug(f"Store invalidate_index_cache raised: {e}") @@ -1310,6 +1376,9 @@ class LazyToolBox(ToolBox): ``tool_source_store`` before the next boot wires up a fresh toolbox. Idempotent; safe to call more than once. """ + if self._store_watcher is not None: + self._store_watcher.shutdown() + self._store_watcher = None with self._cache_lock: self._tool_object_cache.clear() self._tool_index = None diff --git a/lib/galaxy/tools/source_store/watcher.py b/lib/galaxy/tools/source_store/watcher.py index dc1405c1b52..b92373f9070 100644 --- a/lib/galaxy/tools/source_store/watcher.py +++ b/lib/galaxy/tools/source_store/watcher.py @@ -4,7 +4,8 @@ A store published outside this Galaxy process — the CVMFS model, where a publisher repopulates the sqlite bundle in the same transaction that ships new tools — changes without any local filesystem event (inotify does not fire on CVMFS). Polling each store's freshness probe is the only reliable -signal, and it is cheap: one probe per store per tick. +signal, and it is cheap: one probe per store per tick, a single +extended-attribute read for CVMFS stores. The watcher itself is deliberately dumb: it detects token transitions and hands the changed store names to ``on_change``. Reload mechanics —