Address cached toolbox review feedback

.github/workflows/integration.yaml

@@ -16,11 +16,6 @@ on:
     # Run at midnight UTC every Tuesday
     - cron: '0 0 * * 2'
   workflow_dispatch:
-    inputs:
-      use_cached_toolbox:
-        description: "Run with use_cached_toolbox=true (CachedToolBox)"
-        type: boolean
-        default: true
 env:
   GALAXY_TEST_RAISE_EXCEPTION_ON_HISTORYLESS_HDA: '1'
   GALAXY_CONFIG_SQLALCHEMY_WARN_20: '1'
@@ -94,9 +89,6 @@ jobs:
           echo "GALAXY_CONFIG_OVERRIDE_METADATA_STRATEGY=extended" >> $GITHUB_ENV
           # Skip outputs_to_working_directory: true in integration tests, doesn't work with pulsar
           # echo "GALAXY_CONFIG_OVERRIDE_OUTPUTS_TO_WORKING_DIRECTORY=true" >> $GITHUB_ENV
-      - if: github.event_name == 'workflow_dispatch' && inputs.use_cached_toolbox
-        run: |
-          echo "GALAXY_CONFIG_OVERRIDE_USE_CACHED_TOOLBOX=true" >> $GITHUB_ENV
       - name: Prune unused docker image, volumes and containers
         run: docker system prune -a -f
       - name: Clean dotnet folder for space

doc/source/dev/tool_source_storage.rst

@@ -224,8 +224,8 @@ model. Tools are parsed in a
 ``ThreadPoolExecutor`` (``--parallel``, default 4 workers); each tool is
 matched to its source path and carried forward when its raw file hash is unchanged
 (``--incremental``, the default). Once the JSON index is committed the
-populator rebuilds the Whoosh search index (``search.py``) so ranked tool
-search stays in sync with the stored sources.
+rendered cached toolbox builds one Whoosh corpus per panel view from the
+merged ``ToolIndex`` and that view's in-memory membership.
 
 Watch mode (``--watch``) uses ``watchdog`` to monitor every directory yielded
 by ``discover_tools``. File events are debounced (default 2 s), the changed
@@ -255,12 +255,12 @@ materialising tools:
   ``ToolIndex`` entries when the cached toolbox is active, and iterate the toolbox
   otherwise.
 
-``CachedToolboxSearch`` (``tools/search/__init__.py``) queries the whoosh index
-of *every* configured store — the default plus each named per-conf store —
-via ``ToolWhooshIndex.search_scored``, then merges the per-store hit lists by
-BM25 score and post-filters them to the requested panel view. A tool served
-from a named store is therefore reachable through ``/api/tools?q=`` even
-though its source lives outside the default store.
+``CachedToolboxSearch`` (``tools/search/__init__.py``) builds and queries one
+Whoosh index for each rendered panel view. Membership is read directly from
+the toolbox and is not persisted in ``ToolIndex``. Eager and cached search use
+the same document normalization, fields, boosts, scoring, and 20,000-character
+help-text bound, so the same panel corpus produces the same ordered results.
+Named-store tools participate through the merged runtime index.
 
 App Wiring
 ----------
@@ -285,11 +285,6 @@ expansions is expensive and shouldn't block worker startup. Keeping the
 populator separate also lets it run on a single host while many web workers
 share the resulting store.
 
-**Why subclass ToolBox instead of building a parallel hierarchy?**
-``trans.app.toolbox`` is referenced from hundreds of call sites that expect
-the full ToolBox interface. Subclassing keeps the Liskov-substitution
-property and lets unmodified callers benefit from lazy loading transparently.
-
 **Why hash-keyed storage?** Content-addressed storage gives us cheap
 deduplication across versions and shed installations, and idempotent
 incremental updates: re-running the populator over an unchanged tree is
@@ -298,14 +293,6 @@ effectively a no-op.
 Testing
 -------
 
-- Store unit tests: ``test/unit/app/tools/source_store/`` exercises each backend
-  through the ``ToolSourceStore`` interface (``test_stores.py``,
-  ``test_sqlite_store.py``, ``test_composite_store.py``,
-  ``test_index_versions.py``, ``test_multi_store_search.py``).
-- Populator/discovery unit tests: ``test/unit/scripts/tool_source/``
-  (``test_populate_store.py``, ``test_discover.py``,
-  ``test_build_index_entry.py``, ``test_whoosh_dir.py``). These use fakes
-  (not mocks) of ``ToolSourceStore`` so behavior is verified against the real
-  interface.
-- Integration tests: ``test/integration/test_tool_source_storage.py`` spins
-  up Galaxy against the store and verifies end-to-end behavior.
+Tests cover backend contracts, discovery and population, per-view search
+parity, lazy API behavior, store recovery, and selected integration workflows
+under both eager and cached toolboxes.

lib/galaxy/app/__init__.py

@@ -412,10 +412,8 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
         self._set_enabled_container_types()
         index_help = getattr(self.config, "index_tool_help", True)
         if self._use_cached_toolbox():
-            # Populator owns the whoosh index in cached-toolbox mode; the toolbox search
-            # singleton is a thin reader. The toolbox is threaded in so
-            # ``search`` can scope hits to the requested panel view. See
-            # ``CachedToolboxSearch``.
+            # Build search corpora from rendered cached panel views without
+            # materializing their tool stubs.
             search_singleton: ToolBoxSearch = CachedToolboxSearch(self.config, self.toolbox)
         else:
             search_singleton = ToolBoxSearch(
@@ -442,11 +440,8 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
         stats = self.tool_source_store.get_stats()
         tool_count = stats.get("count", 0)
         log.info(f"Initialized tool source store (backend: {stats.get('backend', 'unknown')}, tools: {tool_count})")
-        # Hand the store + (eventual) cached toolbox to ``Galaxy.shutdown()``
-        # so embedded restarts (IntegrationTestCase.restart) drop their
-        # in-memory caches before the next boot wires its own. Without this,
-        # the prior boot's cached ToolIndex sticks around long enough to
-        # race with the new boot's _load_index_from_store.
+        # Close cached state before a replacement application boot wires its
+        # own store and toolbox, so no prior ToolIndex survives the handoff.
         self.haltables.insert(1, ("tool source store", self._shutdown_tool_source_store))
         self.haltables.insert(2, ("cached toolbox", self._shutdown_cached_toolbox))
 
@@ -532,7 +527,11 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
 
     def reindex_tool_search(self) -> None:
         # Call this when tools are added or removed.
-        self.toolbox_search.build_index(tool_cache=self.tool_cache, toolbox=self.toolbox)
+        self.toolbox_search.build_index(
+            tool_cache=self.tool_cache,
+            toolbox=self.toolbox,
+            index_help=self.config.index_tool_help,
+        )
         self.tool_cache.reset_status()
 
     def _set_enabled_container_types(self):

lib/galaxy/queue_worker/__init__.py

@@ -410,6 +410,7 @@ def reload_tool_source_cache(app, **kwargs):
     toolbox = app.toolbox
     if isinstance(toolbox, CachedToolBox):
         toolbox.invalidate_index_cache()
+        app.reindex_tool_search()
         log.info("Tool source index cache invalidated")
 
     # Invalidate the tool source store cache if it exists

lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py

@@ -25,6 +25,19 @@ from galaxy.util.tool_shed.xml_util import parse_xml
 log = logging.getLogger(__name__)
 
 
+def toolbox_with_new_tool_path(config_filename: str, tool_path: str) -> Element:
+    """Return a toolbox root that preserves attributes and overrides its path."""
+    root_attributes: dict[str, str] = {}
+    if os.path.exists(config_filename):
+        existing_tree, _error_message = parse_xml(config_filename)
+        if existing_tree is not None:
+            for key, value in existing_tree.getroot().attrib.items():
+                if isinstance(key, str) and isinstance(value, str):
+                    root_attributes[key] = value
+    root_attributes["tool_path"] = tool_path
+    return Element("toolbox", root_attributes)
+
+
 def _collect_new_tool_paths(elem_list, tool_path: str, shed_tool_conf: str) -> dict[str, str | None]:
     """Walk ``elem_list`` and map each new ``<tool>``'s absolute path to its guid.
 
@@ -150,7 +163,7 @@ class ToolPanelManager:
             use_cached_toolbox = self.app.config.use_cached_toolbox
             if use_cached_toolbox:
                 # The populator writes ``StoredToolSource`` + ``ToolIndexEntry``
-                # + whoosh for every new tool file, then broadcasts
+                # for every new tool file, then broadcasts
                 # ``reload_tool_source_cache`` so peer Galaxy processes
                 # refresh. ``create_tool`` raises on index miss, so this MUST
                 # run before ``load_item`` reaches the seam — and the
@@ -177,7 +190,6 @@ class ToolPanelManager:
                     populate_for_paths(
                         self.app.config,
                         paths=list(new_path_guids),
-                        rebuild_whoosh=True,
                         path_guids=new_path_guids,
                         app=self.app,
                     )
@@ -195,6 +207,7 @@ class ToolPanelManager:
                         load_panel_dict=True,
                         guid=config_elem.get("guid"),
                     )
+                self.app.reindex_tool_search()
             else:
                 # Eager path: append + load each elem, then persist the
                 # updated shed_tool_conf.
@@ -220,21 +233,7 @@ class ToolPanelManager:
         value of config_filename.
         """
         try:
-            # Managed shed confs may carry publisher/operator attributes such
-            # as ``store`` or ``monitor``.  Rebuilding the root from only
-            # ``tool_path`` used to silently discard them on every install,
-            # update, metadata refresh, or uninstall.
-            root_attributes: dict[str, str] = {}
-            if os.path.exists(config_filename):
-                existing_tree, _error_message = parse_xml(config_filename)
-                if existing_tree is not None:
-                    for key, value in existing_tree.getroot().attrib.items():
-                        if isinstance(key, str) and isinstance(value, str):
-                            root_attributes[key] = value
-            # The resolved tool path passed by the manager remains
-            # authoritative even if the old root contained another value.
-            root_attributes["tool_path"] = tool_path
-            root = Element("toolbox", root_attributes)
+            root = toolbox_with_new_tool_path(config_filename, tool_path)
             for elem in config_elems:
                 root.append(elem)
             with RenamedTemporaryFile(config_filename, mode="w") as fh:

lib/galaxy/tool_util/toolbox/lineages/interface.py

@@ -75,10 +75,8 @@ class ToolLineage:
     def reset(cls) -> None:
         """Clear the global ``lineages_by_id`` cache.
 
-        ``lineages_by_id`` is a class attribute, so a ``ToolLineage`` built
-        in a prior process / embedded ``IntegrationTestCase.restart()`` boot
-        carries its ``tool_versions`` SortedSet into the next boot. Wired
-        into ``CachedToolBox.close()`` so a restart sees a clean cache.
+        ``lineages_by_id`` is a class attribute, so toolbox shutdown clears
+        it before another boot reconstructs lineages from current metadata.
         """
         with cls.lock:
             cls.lineages_by_id.clear()

lib/galaxy/tools/cached_toolbox.py

@@ -746,7 +746,7 @@ class CachedToolBox(ToolBox):
         return False
 
     def _run_inline_populator(self) -> None:
-        """Cold-start hook: write the index + whoosh in this process.
+        """Cold-start hook: write source rows and metadata in this process.
 
         Wraps ``populator.populate_store_inline`` so boot can use the same
         single-writer machinery as the CLI script and the shed-install
@@ -756,10 +756,7 @@ class CachedToolBox(ToolBox):
         load instead of degrading silently.
         """
         log.info("CachedToolBox: running populator inline to backfill the index")
-        populate_store_inline(
-            self.app.config,
-            rebuild_whoosh=True,
-        )
+        populate_store_inline(self.app.config)
 
     def _index_versions_for(self, tool_id: str) -> list[str]:
         """Return every version present in the index for ``tool_id``.
@@ -1545,23 +1542,17 @@ class CachedToolBox(ToolBox):
     def close(self) -> None:
         """Drop in-memory state at app shutdown.
 
-        Wired into ``GalaxyUniverseApplication.haltables`` so an embedded
-        restart (``IntegrationTestCase.restart``) releases the LRU cache,
-        the ``ToolIndex`` reference, and the link back to the
-        ``tool_source_store`` before the next boot wires up a fresh
-        toolbox. Idempotent; safe to call more than once.
+        Release the LRU cache, ``ToolIndex`` reference, and store link before
+        a replacement boot wires a fresh toolbox. Idempotent; safe to call
+        more than once.
         """
         with self._cache_lock:
             self._tool_object_cache.clear()
         self._cached_tools.clear()
         self._tool_index = None
         self._store = None
-        # ``ToolLineage.lineages_by_id`` is a *class*-level dict, so a
-        # ``ToolLineage`` from a prior process / embedded restart would
-        # otherwise carry its ``tool_versions`` SortedSet across boots and
-        # shadow the new boot's index versions. Reset on shutdown so the
-        # next ``CachedLineageMap.get`` rebuilds from the freshly-loaded
-        # index.
+        # ``ToolLineage.lineages_by_id`` is class-level, so reset it on
+        # shutdown before the next toolbox rebuilds from its index.
         ToolLineage.reset()
         # ``_tools_by_id`` and friends still get GC'd when the surrounding
         # app object drops. We don't clear them here because the eager
@@ -1615,6 +1606,7 @@ class CachedToolBox(ToolBox):
             current = getattr(self.app, "toolbox", None)
             if current is not self and isinstance(current, CachedToolBox):
                 current._remove_tool_in_memory(tool_id, remove_from_panel=remove_from_panel)
+        self.app.reindex_tool_search()
         return result
 
     def _remove_tool_in_memory(self, tool_id: str, remove_from_panel: bool = True):

lib/galaxy/tools/search/__init__.py

@@ -27,7 +27,6 @@ Filters - various filters are available for processing content as the index is
 
 import logging
 import os
-import re
 import shutil
 from typing import (
     Any,
@@ -35,36 +34,19 @@ from typing import (
     TYPE_CHECKING,
 )
 
-from whoosh import (
-    analysis,
-    index,
-)
+from whoosh import index
 from whoosh.fields import Schema
-from whoosh.qparser import (
-    MultifieldParser,
-    OrGroup,
-)
-from whoosh.scoring import (
-    BM25F,
-    Frequency,
-    MultiWeighting,
-)
 from whoosh.writing import AsyncWriter
 
 from galaxy.config import GalaxyAppConfiguration
-from galaxy.tools.source_store.populator import (
-    DEFAULT_STORE_NAME,
-    whoosh_dir_for_store,
-)
 from galaxy.tools.source_store.search import (
+    build_search_document,
     build_search_schema,
+    search_whoosh_index,
     ToolSearchTuning,
     ToolWhooshIndex,
 )
-from galaxy.util import (
-    ExecutionTimer,
-    unicodify,
-)
+from galaxy.util import ExecutionTimer
 
 if TYPE_CHECKING:
     from galaxy.tools import (
@@ -132,75 +114,47 @@ class ToolBoxSearch:
 
 
 class CachedToolboxSearch(ToolBoxSearch):
-    """Drop-in for :class:`ToolBoxSearch` in cached-toolbox mode.
-
-    The populator (``galaxy.tools.source_store.populator``) builds and owns
-    one whoosh index per store; this class is a thin reader that opens them
-    on each query and merges hits by score.
-
-    Per-view scoping mirrors the eager :class:`ToolBoxSearch`: the merged hits
-    are filtered down to the tools the requested panel view holds, and an
-    unknown ``panel_view`` raises ``KeyError``. Membership is read off the
-    toolbox's rendered panel by id (:meth:`AbstractToolBox.panel_view_tool_ids`),
-    so no tool is materialised just to filter.
-
-    ``build_index`` is a no-op (the populator's job). ``index_count`` is
-    still incremented so :func:`galaxy.queue_worker.rebuild_toolbox_search_index`
-    keeps its watermark check happy — it stops at "in sync with the
-    toolbox reload count" without re-building anything.
-    """
+    """Build one metadata-only Whoosh corpus per rendered panel view."""
 
     def __init__(self, config: GalaxyAppConfiguration, toolbox: Optional["ToolBox"] = None) -> None:
-        # Skip ToolBoxSearch.__init__ — it walks ``toolbox.panel_views()`` and
-        # builds a ToolPanelViewSearch per view. Under the cached toolbox the
-        # populator owns the whoosh indexes; view scoping is a post-filter on
-        # the merged hits instead of a per-view index.
         self.config = config
         self._toolbox = toolbox
-        self.panel_searches: dict[str, ToolPanelViewSearch] = {}
+        self.panel_searches: dict[str, ToolWhooshIndex] = {}
+        self._panel_view_ids: set[str] = set()
         self.index_count = -1
+        if toolbox is not None:
+            self._sync_panel_searches(toolbox)
 
     def build_index(self, tool_cache: Any, toolbox: Any, index_help: bool = True) -> None:
-        # Populator side owns whoosh writes; bump the watermark so the
-        # rebuild_toolbox_search_index control task observes "in sync".
+        self._toolbox = toolbox
+        self._sync_panel_searches(toolbox)
+        tool_index = toolbox.tool_index
+        if tool_index is not None:
+            for panel_view_id, searcher in self.panel_searches.items():
+                searcher.build(tool_index, toolbox.panel_view_tool_ids(panel_view_id))
         self.index_count += 1
 
     def search(self, q: str, panel_view: str, config: GalaxyAppConfiguration) -> list[str]:
-        # Resolve view membership first so an unknown view raises ``KeyError``
-        # even when whoosh search is disabled — parity with eager
-        # ``ToolBoxSearch.search``. ``None`` (no toolbox wired, e.g. in unit
-        # tests) skips scoping and returns the raw merged hits.
-        member_ids: set[str] | None = None
-        if self._toolbox is not None:
-            member_ids = self._toolbox.panel_view_tool_ids(panel_view)
+        if panel_view not in self._panel_view_ids:
+            raise KeyError(f"Unknown panel_view specified {panel_view}")
         if not config.tool_search_index_dir:
-            # No index dir means whoosh search is off entirely.
             return []
-        # The populator writes one whoosh index per store — searching only
-        # the default's would make every named-store tool invisible to
-        # ``/api/tools?q=``. Search each configured store's index and merge
-        # by score. Over-searching a catalog store no conf references is
-        # harmless: the panel-view filter (and ``resolve_search_hit``
-        # downstream) drops ids not placed in this toolbox.
-        store_names = [DEFAULT_STORE_NAME, *sorted(config.tool_source_stores or {})]
-        tuning = ToolSearchTuning.from_config(config)
-        scored: dict[str, float] = {}
-        for store_name in store_names:
-            index_dir = whoosh_dir_for_store(config.tool_search_index_dir, store_name)
-            assert index_dir  # tool_search_index_dir checked above
-            searcher = ToolWhooshIndex(index_dir=index_dir, tuning=tuning)
-            # limit=None matches ToolPanelViewSearch below, which searches
-            # unlimited — capping here would truncate uniform-score matches.
-            for tool_id, score in searcher.search_scored(q, limit=None):
-                if tool_id not in scored or score > scored[tool_id]:
-                    scored[tool_id] = score
-        # BM25 scores from different indexes aren't strictly comparable
-        # (per-corpus statistics), but interleaving by score beats
-        # concatenation; ties keep first-seen order (default store first).
-        ordered = [tool_id for tool_id, _score in sorted(scored.items(), key=lambda kv: -kv[1])]
-        if member_ids is None:
-            return ordered
-        return [tool_id for tool_id in ordered if tool_id in member_ids]
+        return self.panel_searches[panel_view].search(q, limit=None)
+
+    def _sync_panel_searches(self, toolbox: "ToolBox") -> None:
+        panel_view_ids = {panel_view.id for panel_view in toolbox.panel_views()}
+        self._panel_view_ids = panel_view_ids
+        if not self.config.tool_search_index_dir:
+            self.panel_searches = {}
+            return
+        tuning = ToolSearchTuning.from_config(self.config)
+        self.panel_searches = {
+            panel_view_id: ToolWhooshIndex(
+                index_dir=os.path.join(self.config.tool_search_index_dir, panel_view_id),
+                tuning=tuning,
+            )
+            for panel_view_id in panel_view_ids
+        }
 
 
 class ToolPanelViewSearch:
@@ -217,14 +171,12 @@ class ToolPanelViewSearch:
         index_help: bool = True,
     ) -> None:
         """Build the schema and validate against the index."""
-        # Shared with the store's ``ToolWhooshIndex`` so eager and cached
-        # search rank identically; ``help_boost`` adds the eager-only help
-        # field the populator can't index.
+        tuning = ToolSearchTuning.from_config(config)
         self.schema = build_search_schema(
-            ToolSearchTuning.from_config(config),
-            help_boost=config.tool_help_boost,
+            tuning,
+            help_boost=tuning.help_boost if index_help else None,
         )
-        self.rex = analysis.RegexTokenizer()
+        self.tuning = tuning
         self.index_dir = index_dir
         self.panel_view_id = panel_view_id
         self.index = self._index_setup()
@@ -312,51 +264,22 @@ class ToolPanelViewSearch:
         tool: "Tool",
         index_help: bool = True,
     ) -> dict[str, str | list[str]]:
-        def clean(s: str) -> str:
-            """Remove hyphens as they are Whoosh wildcards."""
-            if "-" in s:
-                return " ".join(token.text for token in self.rex(s))
-            else:
-                return s
-
-        if tool.tool_type == "manage_data":
-            #  Do not add data managers to the public index
-            return {}
-        add_doc_kwds: dict[str, str | list[str]] = {
-            "id": unicodify(tool.id),
-            "id_exact": unicodify(tool.id),
-            "name": clean(tool.name),
-            "description": unicodify(tool.description),
-            "section": tool.get_panel_section()[1] or "",
-            "edam_operations": [clean(_) for _ in tool.edam_operations or []],
-            "edam_topics": [clean(_) for _ in tool.edam_topics or []],
-            "repository": unicodify(tool.repository_name),
-            "owner": unicodify(tool.repository_owner),
-            "help": unicodify(""),
-        }
-        if tool.guid:
-            # Create a stub consisting of owner, repo, and tool from guid
-            slash_indexes = [m.start() for m in re.finditer("/", tool.guid)]
-            id_stub = tool.guid[(slash_indexes[1] + 1) : slash_indexes[4]]
-            add_doc_kwds["stub"] = clean(id_stub)
-        else:
-            add_doc_kwds["stub"] = unicodify(tool.id)
-        if tool.labels:
-            add_doc_kwds["labels"] = unicodify(" ".join(tool.labels))
-        if tool.tool_tags:
-            add_doc_kwds["tool_tags"] = unicodify(",".join(tool.tool_tags))
-        if index_help:
-            raw_help = tool.raw_help
-            if raw_help:
-                try:
-                    add_doc_kwds["help"] = unicodify(raw_help)
-                except Exception:
-                    # Don't fail to build index when help fails to parse
-                    pass
-
-        add_doc_kwds["name_exact"] = add_doc_kwds["name"]
-
-        return add_doc_kwds
+        document = build_search_document(
+            tool_id=tool.id,
+            guid=tool.guid,
+            name=tool.name,
+            description=tool.description,
+            section=tool.get_panel_section()[1],
+            edam_operations=tool.edam_operations,
+            edam_topics=tool.edam_topics,
+            repository=tool.repository_name,
+            owner=tool.repository_owner,
+            labels=tool.labels,
+            tool_tags=tool.tool_tags,
+            help_text=tool.raw_help if index_help else None,
+            tool_type=tool.tool_type,
+        )
+        return document or {}
 
     def search(
         self,
@@ -364,40 +287,5 @@ class ToolPanelViewSearch:
         config: GalaxyAppConfiguration,
     ) -> list[str]:
         """Perform search on the in-memory index."""
-        # Change field boosts for searcher
-        self.searcher = self.index.searcher(
-            weighting=MultiWeighting(
-                Frequency(),
-                help=BM25F(K1=config.tool_help_bm25f_k1),
-            )
-        )
-        fields = [
-            "id",
-            "id_exact",
-            "name",
-            "name_exact",
-            "description",
-            "section",
-            "edam_operations",
-            "edam_topics",
-            "repository",
-            "owner",
-            "help",
-            "labels",
-            "tool_tags",
-            "stub",
-        ]
-        self.parser = MultifieldParser(
-            fields,
-            schema=self.schema,
-            group=OrGroup,
-        )
-        parsed_query = self.parser.parse(q)
-        hits = self.searcher.search(
-            parsed_query,
-            limit=None,
-            sortedby="",
-            terms=True,
-        )
-
-        return [hit["id"] for hit in hits]
+        tuning = ToolSearchTuning.from_config(config)
+        return [tool_id for tool_id, _score in search_whoosh_index(self.index, q, tuning, limit=None)]

lib/galaxy/tools/source_store/interface.py

@@ -237,11 +237,9 @@ class ToolSourceStore(ABC):
     def close(self) -> None:  # noqa: B027 — intentional empty default
         """Release any state the store is holding.
 
-        Wired into ``GalaxyUniverseApplication.haltables`` so a Python-side
-        ``app.shutdown()`` (e.g. the embedded ``IntegrationTestCase.restart()``
-        path) clears references that would otherwise survive into the next
-        boot. Default is a no-op; backends holding an engine or cache
-        override, and the composite store propagates.
+        Application shutdown clears references before a replacement boot.
+        Default is a no-op; backends holding an engine or cache override, and
+        the composite store propagates.
         """
 
 

lib/galaxy/tools/source_store/populator.py

@@ -3,8 +3,8 @@ Tool source store populator.
 
 Walks Galaxy's tool configuration files, parses each tool source, and writes
 the canonical ``StoredToolSource`` + ``ToolIndex`` rows into every writable
-tool source store. The populator is the single writer of the index and the
-whoosh search index; store consumers are read-only.
+tool source store. The rendered toolbox owns its per-view search indexes;
+the populator only persists source and metadata rows.
 
 Two execution modes:
 
@@ -21,7 +21,6 @@ loop that updates the store as tool files change on disk and broadcasts a
 import argparse
 import hashlib
 import logging
-import os
 import signal
 import sys
 import threading
@@ -96,10 +95,7 @@ from galaxy.tools.source_store.manifest import (
     build_manifest,
     write_manifest,
 )
-from galaxy.tools.source_store.search import (
-    ToolSearchTuning,
-    ToolWhooshIndex,
-)
+from galaxy.tools.source_store.search import MAX_TOOL_SEARCH_HELP_CHARS
 from galaxy.util import listify
 from galaxy.util.hash_util import md5_hash_file
 from galaxy.util.properties import load_app_properties
@@ -116,12 +112,6 @@ from galaxy.util.watcher import (
 
 log = logging.getLogger(__name__)
 
-# Upper bound on the help text carried onto each ``ToolIndexEntry``. Tool help
-# is occasionally enormous (embedded tables, long tutorials); cap it so the
-# persisted index and whoosh corpus stay bounded while still covering the help
-# body for search.
-MAX_HELP_TEXT_CHARS = 20000
-
 
 class _ReloadNotificationConfig(Protocol):
     amqp_internal_connection: str | None
@@ -308,7 +298,7 @@ class ToolFileWatcher:
         If the content is new, delegates to :func:`populate_for_paths`,
         which runs the full parse-and-persist path: ``StoredToolSource``
         write, ``ToolIndexEntry`` build (with section + labels from the
-        conf walk), whoosh rebuild, and the ``reload_tool_source_cache``
+        conf walk), metadata rebuild, and the ``reload_tool_source_cache``
         broadcast — same machinery a shed install hits.
         """
         try:
@@ -341,7 +331,6 @@ class ToolFileWatcher:
         self._populate(
             self.config,
             paths=[path],
-            rebuild_whoosh=True,
             write_manifests=self._write_manifests,
         )
         log.info("Updated tool: %s", path)
@@ -364,7 +353,6 @@ class ToolFileWatcher:
         self._populate(
             self.config,
             paths=siblings,
-            rebuild_whoosh=True,
             write_manifests=self._write_manifests,
         )
         log.info("Macro change in %s — re-expanded %d sibling file(s)", macro_path, len(siblings))
@@ -386,43 +374,6 @@ class ToolFileWatcher:
 
 DEFAULT_STORE_NAME = "__default__"
 
-# Sub-directory under ``config.tool_search_index_dir`` where the default
-# store's whoosh index lives.
-_WHOOSH_DEFAULT_SUBDIR = "_store_default"
-
-
-def whoosh_dir_for_store(tool_search_index_dir: str | None, store_name: str) -> str | None:
-    """Resolve the on-disk whoosh dir for ``store_name``.
-
-    Returns ``None`` if the config doesn't define ``tool_search_index_dir``
-    (whoosh search is then disabled). The default store maps to a fixed
-    sub-dir; named stores get their own sub-dir.
-    """
-    if not tool_search_index_dir:
-        return None
-    sub = _WHOOSH_DEFAULT_SUBDIR if store_name == DEFAULT_STORE_NAME else store_name
-    return os.path.join(tool_search_index_dir, sub)
-
-
-def build_whoosh_for_store(config: GalaxyAppConfiguration, store_name: str, tool_index: ToolIndex) -> None:
-    """Rebuild the whoosh index for ``store_name`` from ``tool_index``.
-
-    No-ops if ``tool_search_index_dir`` is unset. Logs and swallows whoosh
-    failures: the toolbox surfaces them to users at query time (the
-    populator's job is to write what it can; an unbuildable index is a
-    deploy issue, not a populator-CLI fatal).
-    """
-    index_dir = whoosh_dir_for_store(config.tool_search_index_dir, store_name)
-    if index_dir is None:
-        return
-    try:
-        tuning = ToolSearchTuning.from_config(config)
-        searcher = ToolWhooshIndex(index_dir=index_dir, tuning=tuning)
-        count = searcher.build(tool_index)
-        log.info("Built whoosh index for store %s at %s (%d docs)", store_name, index_dir, count)
-    except Exception as e:
-        log.error("Whoosh build for store %s failed: %s", store_name, e)
-
 
 def _merge_panel_order(previous: list[ToolPanelItem], rebuilt: list[ToolPanelItem]) -> list[ToolPanelItem]:
     """Reorder a full rebuild's placements to the previous index's order.
@@ -526,14 +477,14 @@ def build_index_entry_from_source(
         requirements, containers, _, _, _ = tool_source.parse_requirements()
         tests = tool_source.parse_tests_to_dict().get("tests", [])
 
-        # Capture help text for the whoosh corpus. Parse failures drop help
+        # Capture bounded help text for the toolbox-owned search corpus. Parse failures drop help
         # for this entry rather than failing the populate — a malformed help
         # block must not lose the whole tool.
         help_text = ""
         try:
             parsed_help = tool_source.parse_help()
             if parsed_help and parsed_help.content:
-                help_text = parsed_help.content[:MAX_HELP_TEXT_CHARS]
+                help_text = parsed_help.content[:MAX_TOOL_SEARCH_HELP_CHARS]
         except Exception:
             help_text = ""
 
@@ -665,7 +616,6 @@ def populate_store_inline(
     dry_run: bool = False,
     incremental: bool = True,
     verbose: bool = False,
-    rebuild_whoosh: bool = True,
     broadcast: bool = False,
     target: str | None = None,
     prune: bool = False,
@@ -896,8 +846,8 @@ def populate_store_inline(
     # One ordered stream matters: same-id twins are resolved by index
     # add-order, so consuming as_completed (thread timing) or adding carried
     # entries before parsed ones flips which twin wins ``entries[id]`` run to
-    # run — changing its panel section in the whoosh corpus and defeating the
-    # corpus-signature rebuild skip.
+    # run — changing its panel section in the search corpus even though no
+    # input changed.
     index_inputs: dict[str, list[ToolIndexEntry | tuple[DiscoveredTool, StoredToolSource, Any]]] = {
         name: [] for name in writable_names
     }
@@ -1026,12 +976,6 @@ def populate_store_inline(
                         log.info("Skipping manifest for non-file SQLite store %s", store_name)
                     else:
                         log.info("Wrote tool source store manifest for %s to %s", store_name, manifest_path)
-            # Rebuild the whoosh search index from the persisted ToolIndex.
-            # Single-writer principle: the toolbox stops re-building this in
-            # the search hot path.
-            if rebuild_whoosh:
-                build_whoosh_for_store(config, store_name, index)
-
         if broadcast:
             # Tell peer Galaxy processes to drop their cached index so the
             # next request reloads what we just wrote.
@@ -1044,7 +988,6 @@ def populate_for_paths(
     config: GalaxyAppConfiguration,
     paths: list[str],
     *,
-    rebuild_whoosh: bool = True,
     path_guids: dict[str, str | None] | None = None,
     app=None,
 ) -> dict[str, int]:
@@ -1062,7 +1005,6 @@ def populate_for_paths(
     return populate_store_inline(
         config,
         paths=paths,
-        rebuild_whoosh=rebuild_whoosh,
         broadcast=True,
         path_guids=path_guids,
         app=app,
@@ -1072,7 +1014,6 @@ def populate_for_paths(
 def reconcile_index(
     config: GalaxyAppConfiguration,
     *,
-    rebuild_whoosh: bool = True,
     app=None,
 ) -> dict[str, int]:
     """Full prune-enabled scan; used by ``reset_shed_tools``.
@@ -1087,7 +1028,6 @@ def reconcile_index(
         config,
         paths=None,
         prune=True,
-        rebuild_whoosh=rebuild_whoosh,
         broadcast=True,
         app=app,
     )

lib/galaxy/tools/source_store/search.py

@@ -5,6 +5,7 @@ import logging
 import os
 import re
 import shutil
+from collections.abc import Collection
 from dataclasses import dataclass
 
 from whoosh import (
@@ -23,7 +24,11 @@ from whoosh.qparser import (
     MultifieldParser,
     OrGroup,
 )
-from whoosh.scoring import BM25F
+from whoosh.scoring import (
+    BM25F,
+    Frequency,
+    MultiWeighting,
+)
 
 from galaxy.config import GalaxyAppConfiguration
 from galaxy.tool_util.ontologies.ontology_data import curated_tool_tags
@@ -45,6 +50,11 @@ log = logging.getLogger(__name__)
 # Matching sidecar signatures let ``build`` skip an unchanged corpus.
 _CORPUS_SIGNATURE_FILE = "corpus.md5"
 
+# Tool help can contain embedded tables and tutorials large enough to dominate
+# both the persisted metadata index and Whoosh. Eager and cached search must
+# apply the same bound so their corpora and ranking remain equivalent.
+MAX_TOOL_SEARCH_HELP_CHARS = 20_000
+
 
 @dataclass(frozen=True)
 class ToolSearchTuning:
@@ -64,6 +74,7 @@ class ToolSearchTuning:
     # Defaults preserve existing explicit tuning literals.
     help_boost: float = 1.0
     index_tool_help: bool = True
+    help_bm25f_k1: float = 1.2
 
     @classmethod
     def from_config(cls, config: GalaxyAppConfiguration) -> "ToolSearchTuning":
@@ -81,6 +92,7 @@ class ToolSearchTuning:
             ngram_factor=float(config.tool_ngram_factor),
             help_boost=float(config.tool_help_boost),
             index_tool_help=bool(config.index_tool_help),
+            help_bm25f_k1=float(config.tool_help_bm25f_k1),
         )
 
 
@@ -139,53 +151,129 @@ def _clean(s: str | None) -> str:
     return text
 
 
-def _entry_to_doc(entry: ToolIndexEntry, *, include_help: bool = False) -> dict | None:
-    """Turn a ``ToolIndexEntry`` into the document shape ``Schema`` expects.
-
-    ``include_help`` projects the entry's captured help text into a ``help``
-    field, mirroring the eager toolbox's ``index_tool_help`` behaviour. It's
-    off by default so callers that build a help-less schema never emit a field
-    the schema lacks.
-    """
-    if entry.tool_type == DataManagerTool.tool_type:
+def build_search_document(
+    *,
+    tool_id: str,
+    name: str,
+    description: str | None = None,
+    section: str | None = None,
+    edam_operations: Collection[str] | None = None,
+    edam_topics: Collection[str] | None = None,
+    repository: str | None = None,
+    owner: str | None = None,
+    labels: Collection[str] | None = None,
+    tool_tags: Collection[str] | None = None,
+    guid: str | None = None,
+    help_text: object | None = None,
+    tool_type: str = "default",
+) -> dict | None:
+    """Build the common eager/cached Whoosh document for one tool."""
+    if tool_type == DataManagerTool.tool_type:
         return None
-    name_clean = _clean(entry.name)
+    name_clean = _clean(name)
     doc: dict = {
-        "id": unicodify(entry.id),
-        "id_exact": unicodify(entry.id),
+        "id": unicodify(tool_id),
+        "id_exact": unicodify(tool_id),
         "name": name_clean,
         "name_exact": name_clean,
-        "description": unicodify(entry.description or ""),
-        "section": unicodify(entry.panel_section_name or ""),
-        "edam_operations": [_clean(op) for op in entry.edam_operations or []],
-        "edam_topics": [_clean(topic) for topic in entry.edam_topics or []],
-        "repository": unicodify(entry.repository_name or ""),
-        "owner": unicodify(entry.repository_owner or ""),
+        "description": unicodify(description or ""),
+        "section": unicodify(section or ""),
+        "edam_operations": [_clean(op) for op in edam_operations or []],
+        "edam_topics": [_clean(topic) for topic in edam_topics or []],
+        "repository": unicodify(repository or ""),
+        "owner": unicodify(owner or ""),
     }
-    # GUID has shape ``shed/repos/owner/repo/tool/version``. The eager path
-    # carves out ``owner/repo/tool`` as the search stub; fall back to the
-    # plain id for local tools.
-    if "/" in entry.id:
-        slash_indexes = [m.start() for m in re.finditer("/", entry.id)]
+    # A Tool Shed GUID has shape ``shed/repos/owner/repo/tool/version``.
+    # Carve out ``owner/repo/tool`` as the search stub.
+    stub_source = guid or tool_id
+    if guid:
+        slash_indexes = [m.start() for m in re.finditer("/", stub_source)]
         if len(slash_indexes) >= 5:
-            doc["stub"] = _clean(entry.id[slash_indexes[1] + 1 : slash_indexes[4]])
+            doc["stub"] = _clean(stub_source[slash_indexes[1] + 1 : slash_indexes[4]])
         else:
-            doc["stub"] = unicodify(entry.id)
+            doc["stub"] = unicodify(tool_id)
     else:
-        doc["stub"] = unicodify(entry.id)
-    if entry.labels:
-        doc["labels"] = unicodify(" ".join(entry.labels))
-    tool_id = (entry.id or "").lower()
-    all_ids = [tool_id]
-    if is_shed_guid(tool_id):
-        all_ids = [tool_id, remove_version_from_guid(tool_id) or tool_id, short_tool_id(tool_id)]
-    if tags := curated_tool_tags(all_ids):
-        doc["tool_tags"] = unicodify(",".join(tags))
-    if include_help and entry.help_text:
-        doc["help"] = unicodify(entry.help_text)
+        doc["stub"] = unicodify(tool_id)
+    if labels:
+        doc["labels"] = unicodify(" ".join(labels))
+    if tool_tags is None:
+        normalized_id = tool_id.lower()
+        all_ids = [normalized_id]
+        if is_shed_guid(normalized_id):
+            all_ids = [
+                normalized_id,
+                remove_version_from_guid(normalized_id) or normalized_id,
+                short_tool_id(normalized_id),
+            ]
+        tool_tags = curated_tool_tags(all_ids)
+    if tool_tags:
+        doc["tool_tags"] = unicodify(",".join(tool_tags))
+    if help_text is not None:
+        help_content = getattr(help_text, "content", help_text)
+        doc["help"] = unicodify(help_content or "")[:MAX_TOOL_SEARCH_HELP_CHARS]
     return doc
 
 
+def entry_to_search_document(entry: ToolIndexEntry, *, include_help: bool = False) -> dict | None:
+    """Project one cached metadata entry into the common document shape."""
+    entry_id = entry.id
+    return build_search_document(
+        tool_id=entry_id,
+        guid=entry_id if is_shed_guid(entry_id) else None,
+        name=entry.name,
+        description=entry.description,
+        section=entry.panel_section_name,
+        edam_operations=entry.edam_operations,
+        edam_topics=entry.edam_topics,
+        repository=entry.repository_name,
+        owner=entry.repository_owner,
+        labels=entry.labels,
+        help_text=entry.help_text if include_help else None,
+        tool_type=entry.tool_type,
+    )
+
+
+def search_whoosh_index(
+    ix: index.FileIndex,
+    query: str,
+    tuning: ToolSearchTuning,
+    limit: int | None = None,
+) -> list[tuple[str, float]]:
+    """Search an eager or cached index with one parser and scoring policy."""
+    if not query or not query.strip():
+        return []
+    search_fields = [
+        "id",
+        "id_exact",
+        "name",
+        "name_exact",
+        "stub",
+        "description",
+        "section",
+        "edam_operations",
+        "edam_topics",
+        "repository",
+        "owner",
+        "labels",
+        "tool_tags",
+    ]
+    if "help" in ix.schema.names():
+        search_fields.append("help")
+    parser = MultifieldParser(search_fields, schema=ix.schema, group=OrGroup)
+    parsed = parser.parse(query)
+    weighting = MultiWeighting(
+        Frequency(),
+        help=BM25F(K1=tuning.help_bm25f_k1),
+    )
+    with ix.searcher(weighting=weighting) as searcher:
+        hits = searcher.search(parsed, limit=None, sortedby="", terms=True)
+        scored = [(hit["id"], hit.score) for hit in hits]
+    # Whoosh does not define a stable tie order. Sharing an explicit secondary
+    # key makes eager and cached result ordering deterministic.
+    scored.sort(key=lambda hit: (-hit[1], hit[0]))
+    return scored if limit is None else scored[:limit]
+
+
 class ToolWhooshIndex:
     """Build and query a Whoosh index over ``ToolIndexEntry`` rows."""
 
@@ -221,13 +309,18 @@ class ToolWhooshIndex:
             + json.dumps(sorted(docs, key=lambda d: d["id"]), sort_keys=True)
         )
 
-    def build(self, tool_index: ToolIndex) -> int:
-        """Rebuild from entries, returning zero when the corpus is unchanged."""
+    def build(self, tool_index: ToolIndex, tool_ids: Collection[str] | None = None) -> int:
+        """Rebuild one panel-view corpus, or all entries when unscoped."""
         docs = []
-        for entry in tool_index.entries.values():
+        entries = (
+            tool_index.entries.values()
+            if tool_ids is None
+            else (tool_index.entries[tool_id] for tool_id in sorted(tool_ids) if tool_id in tool_index.entries)
+        )
+        for entry in entries:
             if entry.hidden:
                 continue
-            doc = _entry_to_doc(entry, include_help=self.index_help)
+            doc = entry_to_search_document(entry, include_help=self.index_help)
             if doc is None:
                 continue
             docs.append(doc)
@@ -265,36 +358,10 @@ class ToolWhooshIndex:
         return [tool_id for tool_id, _score in self.search_scored(query, limit=limit)]
 
     def search_scored(self, query: str, limit: int | None = None) -> list[tuple[str, float]]:
-        """Like :meth:`search`, but pair each tool id with its BM25 score.
-
-        Callers merging hits across several store indexes need the scores
-        to interleave results instead of concatenating whole result lists.
-        """
+        """Like :meth:`search`, but pair each tool id with its score."""
         if not query or not query.strip():
             return []
         if not (os.path.isdir(self.index_dir) and index.exists_in(self.index_dir)):
             return []
         ix = index.open_dir(self.index_dir)
-        search_fields = [
-            "id_exact",
-            "name",
-            "name_exact",
-            "stub",
-            "description",
-            "section",
-            "edam_operations",
-            "edam_topics",
-            "repository",
-            "owner",
-            "labels",
-            "tool_tags",
-        ]
-        # ``help`` is only present when the corpus was built with help indexing
-        # on; querying a field the on-disk schema lacks raises in the parser.
-        if "help" in ix.schema.names():
-            search_fields.append("help")
-        parser = MultifieldParser(search_fields, schema=ix.schema, group=OrGroup)
-        parsed = parser.parse(query)
-        with ix.searcher(weighting=BM25F()) as searcher:
-            hits = searcher.search(parsed, limit=limit)
-            return [(hit["id"], hit.score) for hit in hits]
+        return search_whoosh_index(ix, query, self.tuning, limit=limit)

lib/galaxy/tools/source_store/watcher.py

@@ -9,8 +9,8 @@ signal, and it is cheap: one probe per store per tick.
 The watcher itself is deliberately dumb: it detects token transitions and
 hands the changed store names to ``on_change``. Reload mechanics —
 disposing engines so new connections see the new CVMFS snapshot, index
-reload, panel refresh, whoosh rebuild — belong to the callback owner
-(``CachedToolBox``).
+reload, panel refresh, and search rebuild — belong to the toolbox callback
+owner.
 """
 
 import logging

lib/galaxy/webapps/galaxy/services/tools.py

@@ -691,22 +691,17 @@ class ToolsService(ServiceBase):
     ) -> list[str]:
         """Search tools via the ``app.toolbox_search`` singleton.
 
-        In cached-toolbox mode that singleton is :class:`CachedToolboxSearch`, which reads
-        the populator-owned whoosh index; in eager mode it's
-        :class:`ToolBoxSearch` walking ``tool_cache``. Both expose the same
-        ``search(q, panel_view, config)`` interface, so this method doesn't
-        branch on which toolbox flavour is active.
+        Cached and eager toolbox search both own a Whoosh corpus for each
+        rendered panel view and expose the same
+        ``search(q, panel_view, config)`` interface.
 
         Every hit is resolved with a per-tool access check
         (``allow_user_access``, e.g. ``require_login`` tools for anonymous
         users) so denied tools are filtered out.
 
-        In cached-toolbox mode ``get_tool`` would *materialise* every hit (it loads the
-        tool from the store on demand) and the populator-owned whoosh index
-        can carry ids that aren't loaded in this toolbox, so hits are instead
-        resolved against the registered stubs via
-        :meth:`CachedToolBox.resolve_search_hit` — no parse, and hits foreign to
-        this toolbox are skipped just as eager search skips them.
+        In cached-toolbox mode ``get_tool`` would materialise every hit, so
+        hits are instead resolved against registered stubs via
+        :meth:`CachedToolBox.resolve_search_hit`.
         """
         cached_toolbox = self._get_cached_toolbox(trans)
         results: list[str] = []

lib/galaxy_test/driver/integration_util.py

@@ -53,6 +53,15 @@ SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
 VAULT_CONF = os.path.join(SCRIPT_DIRECTORY, "vault_conf.yml")
 
 
+class CachedToolBoxIntegrationMixin:
+    """Run an existing integration-test configuration with lazy tools."""
+
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        super().handle_galaxy_config_kwds(config)  # type: ignore[misc]
+        config["use_cached_toolbox"] = True
+
+
 def docker_run(image, name, *args, detach=True, remove=True, ports=None, env_vars: dict[str, str] | None = None):
     cmd = ["docker", "run"]
 

lib/galaxy_test/driver/uses_shed.py

@@ -91,7 +91,7 @@ class UsesShed(UsesShedApi):
         # would otherwise survive the ``reload_toolbox`` below: ``CachedToolBox``
         # only re-runs the populator when discovery turns up a *new* path.
         try:
-            reconcile_index(self._app.config, rebuild_whoosh=True, app=self._app)
+            reconcile_index(self._app.config, app=self._app)
         except Exception as e:
             log.warning("reset_shed_tools: reconcile_index raised (continuing): %s", e)
         # deleting the containing folder doesn't trigger a toolbox reload, so signal it now and wait until it's done

test/integration/test_data_manager.py

@@ -201,3 +201,10 @@ class TestDataManagerIntegration(integration_util.IntegrationTestCase, UsesShed)
     @classmethod
     def get_secure_ascii_digits(cls, n=12):
         return "".join(random.SystemRandom().choice(string.ascii_lowercase + string.digits) for _ in range(12))
+
+
+class TestCachedDataManagerIntegration(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestDataManagerIntegration,
+):
+    pass

test/integration/test_data_manager_workflow_bundle.py

@@ -5,6 +5,7 @@ from galaxy_test.base.populators import (
     WorkflowPopulator,
 )
 from galaxy_test.base.uses_shed_api import UsesShedApi
+from galaxy_test.driver import integration_util
 from .test_containerized_jobs import ContainerizedIntegrationTestCase
 
 
@@ -107,3 +108,10 @@ test_data:
                 # the 60s default invocation timeout is sized for lightweight tools.
                 timeout=180,
             )
+
+
+class TestCachedDataManagerWorkflowInvocation(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestDataManagerWorkflowInvocation,
+):
+    pass

test/integration/test_edam_toolbox.py

@@ -61,3 +61,17 @@ class TestEdamToolboxDefaultIntegration(integration_util.IntegrationTestCase):
         assert isinstance(views, dict)
         edam_panel_view = views["ontology:edam_topics"]
         assert edam_panel_view["view_type"] == "ontology"
+
+
+class TestCachedEdamToolboxIntegration(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestEdamToolboxIntegration,
+):
+    pass
+
+
+class TestCachedEdamToolboxDefaultIntegration(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestEdamToolboxDefaultIntegration,
+):
+    pass

test/integration/test_panel_views.py

@@ -251,3 +251,17 @@ def get_sections(index_panel):
 
 def model_classes(elements):
     return [x["model_class"] for _, x in elements.items()]
+
+
+class TestCachedPanelViewsFromDirectoryIntegration(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestPanelViewsFromDirectoryIntegration,
+):
+    pass
+
+
+class TestCachedPanelViewsWithShedTools(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestPanelViewsWithShedTools,
+):
+    pass

test/integration/test_repository_operations.py

@@ -176,3 +176,10 @@ class TestRepositoryInstallIntegrationTestCase(integration_util.IntegrationTestC
         if assert_ok:
             self._assert_status_code_is_ok(response)
         return response.json()
+
+
+class TestCachedRepositoryInstallIntegrationTestCase(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestRepositoryInstallIntegrationTestCase,
+):
+    pass

test/integration/test_tool_source_storage.py

@@ -10,8 +10,6 @@ plumbing.
 import os
 import tempfile
 
-import pytest
-
 from galaxy.queue_worker import reload_toolbox
 from galaxy.tools.cached_toolbox import CachedToolBox
 from galaxy.tools.source_store import ToolIndex
@@ -50,11 +48,12 @@ class BaseToolSourceStorageIntegrationTestCase(integration_util.IntegrationTestC
 class TestEagerBootSkipsStore(BaseToolSourceStorageIntegrationTestCase):
     """Default deployments never initialize a tool source store."""
 
+    @classmethod
+    def handle_galaxy_config_kwds(cls, config):
+        super().handle_galaxy_config_kwds(config)
+        config["use_cached_toolbox"] = False
+
     def test_no_store_initialized(self):
-        if self._app.config.use_cached_toolbox:
-            # GALAXY_CONFIG_OVERRIDE_USE_CACHED_TOOLBOX (the cached-toolbox CI dispatch)
-            # trumps per-class config kwds, so an eager boot is impossible here.
-            pytest.skip("use_cached_toolbox forced on by environment override")
         assert self._app.tool_source_store is None
         self._test_api_tools_list()
         self._test_api_tools_show()
@@ -397,10 +396,8 @@ class TestCachedToolBoxApi(BaseToolSourceStorageIntegrationTestCase):
         _probe("tests_summary", self._get("tools/tests_summary"))
         # all_requirements is admin-only; it aggregates from the index in cached-toolbox mode.
         _probe("all_requirements", self._get("tools/all_requirements", admin=True))
-        # Search resolves whoosh hits against the registered stubs, not through
-        # the materialising get_tool — the populator-owned index carries ~150
-        # tool_conf.xml.sample ids that aren't loaded here, and resolving them
-        # via get_tool would parse every one.
+        # Search resolves Whoosh hits against registered stubs, not through the
+        # materialising get_tool.
         _probe("search", self._get("tools", data={"q": "Concatenate multiple datasets"}))
 
         assert all(v == 0 for v in deltas.values()), f"batch endpoints materialised tools (expected 0 each): {deltas}"
@@ -427,6 +424,7 @@ class TestCachedToolBoxApi(BaseToolSourceStorageIntegrationTestCase):
         tool = tools_by_id["cat1"]
         assert tool is not None
         assert getattr(tool, "tool_requirements", None) is not None
+        assert tool.dependencies == []
         # ``.copy()`` is what ``ContainerFinder.find_best_container_description``
         # invokes via ``copy.copy`` on the registry; placeholder ``None``
         # values would crash callers iterating the copy.

test/integration/test_user_defined_tool_job_conf.py

@@ -104,3 +104,10 @@ class TestUserDefinedToolRecommendedJobSetupTPV(TestUserDefinedToolRecommendedJo
             cores = response["outputs"][0]
             cores_content = self.dataset_populator.get_history_dataset_content(history_id, content_id=cores["id"])
             assert cores_content == "2.0\n"
+
+
+class TestCachedUserDefinedToolRecommendedJobSetupTPV(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestUserDefinedToolRecommendedJobSetupTPV,
+):
+    pass

test/integration/test_workflow_invocation.py

@@ -294,3 +294,10 @@ steps:
             # Wait for workflow to complete — should delay then succeed
             invocation = self.workflow_populator.wait_for_invocation_and_completion(invocation_id)
             assert invocation["state"] == "completed", invocation
+
+
+class TestCachedWorkflowInvocation(
+    integration_util.CachedToolBoxIntegrationMixin,
+    TestWorkflowInvocation,
+):
+    pass

test/integration/test_workflow_refactoring.py

@@ -1046,6 +1046,31 @@ def _step_with_label(native_dict, label):
     raise AssertionError(f"Failed to find step with label {label}")
 
 
+class TestCachedWorkflowUpgradeAllSteps(
+    integration_util.CachedToolBoxIntegrationMixin,
+    integration_util.IntegrationTestCase,
+    UsesShedApi,
+):
+    """Cached regression for the workflow-upgrade failure seen in dispatch runs."""
+
+    framework_tool_and_types = True
+
+    def setUp(self):
+        super().setUp()
+        self.workflow_populator = WorkflowPopulator(self.galaxy_interactor)
+
+    _export_for_update = TestWorkflowRefactoringIntegration._export_for_update
+    _refactor = TestWorkflowRefactoringIntegration._refactor
+    _manager = TestWorkflowRefactoringIntegration._manager
+    _most_recent_stored_workflow = TestWorkflowRefactoringIntegration._most_recent_stored_workflow
+    _recent_stored_workflow = TestWorkflowRefactoringIntegration._recent_stored_workflow
+    _latest_workflow = TestWorkflowRefactoringIntegration._latest_workflow
+    _increment_nested_workflow_version = TestWorkflowRefactoringIntegration._increment_nested_workflow_version
+
+    def test_upgrade_all_steps(self):
+        TestWorkflowRefactoringIntegration.test_upgrade_all_steps(self)
+
+
 class MockTrans(ProvidesAppContext):
     def __init__(self, app, user):
         self._app = app

test/unit/app/tools/source_store/test_multi_store_search.py

@@ -9,14 +9,7 @@ from galaxy.tools.source_store.index import (
     ToolIndex,
     ToolIndexEntry,
 )
-from galaxy.tools.source_store.populator import (
-    DEFAULT_STORE_NAME,
-    whoosh_dir_for_store,
-)
-from galaxy.tools.source_store.search import (
-    ToolSearchTuning,
-    ToolWhooshIndex,
-)
+from galaxy.tools.source_store.search import ToolSearchTuning
 
 _TUNING = ToolSearchTuning(
     id_boost=9.0,
@@ -33,89 +26,89 @@ _TUNING = ToolSearchTuning(
 )
 
 
-def _build_store_index(index_root, store_name, entries):
-    tool_index = ToolIndex()
-    for entry in entries:
-        tool_index.add_entry(entry)
-    index_dir = whoosh_dir_for_store(index_root, store_name)
-    assert index_dir
-    whoosh = ToolWhooshIndex(index_dir=index_dir, tuning=_TUNING)
-    assert whoosh.build(tool_index) == len(entries)
-
-
-def test_search_merges_hits_across_store_indexes(tmp_path, monkeypatch):
-    monkeypatch.setattr(ToolSearchTuning, "from_config", classmethod(lambda cls, config: _TUNING))
-    index_root = str(tmp_path)
-    _build_store_index(
-        index_root,
-        DEFAULT_STORE_NAME,
-        [ToolIndexEntry(id="local_mapper", name="Sequence mapper", version="1.0")],
-    )
-    _build_store_index(
-        index_root,
-        "cvmfs_main",
-        [ToolIndexEntry(id="cvmfs_mapper", name="Sequence mapper deluxe", version="2.0")],
-    )
-    config = cast(
-        GalaxyAppConfiguration,
-        SimpleNamespace(
-            tool_search_index_dir=index_root,
-            tool_source_stores={"cvmfs_main": {"type": "sqlalchemy"}},
-        ),
-    )
-    hits = CachedToolboxSearch(config).search("mapper", panel_view="default", config=config)
-    assert set(hits) == {"local_mapper", "cvmfs_mapper"}
-
-
-def test_search_without_index_dir_returns_empty():
-    config = cast(GalaxyAppConfiguration, SimpleNamespace(tool_search_index_dir=None, tool_source_stores={}))
-    assert CachedToolboxSearch(config).search("mapper", panel_view="default", config=config) == []
-
-
 class _FakeToolbox:
-    def __init__(self, views):
+    def __init__(self, entries, views):
+        self.tool_index = ToolIndex()
+        for entry in entries:
+            self.tool_index.add_entry(entry)
         self._views = views
 
+    def panel_views(self):
+        return [SimpleNamespace(id=view_id) for view_id in self._views]
+
     def panel_view_tool_ids(self, panel_view_id):
         return self._views[panel_view_id]
 
 
-def _single_store_config(index_root):
+def _config(index_root):
     return cast(
         GalaxyAppConfiguration,
-        SimpleNamespace(tool_search_index_dir=index_root, tool_source_stores={}),
+        SimpleNamespace(tool_search_index_dir=index_root),
     )
 
 
+def _search(index_root, entries, views, monkeypatch):
+    monkeypatch.setattr(ToolSearchTuning, "from_config", classmethod(lambda cls, config: _TUNING))
+    config = _config(index_root)
+    toolbox = _FakeToolbox(entries, views)
+    search = CachedToolboxSearch(config, toolbox)  # type: ignore[arg-type]
+    search.build_index(tool_cache=None, toolbox=toolbox)
+    return search, config
+
+
+def test_search_indexes_merged_runtime_tool_index(tmp_path, monkeypatch):
+    search, config = _search(
+        str(tmp_path),
+        [
+            ToolIndexEntry(id="local_mapper", name="Sequence mapper", version="1.0"),
+            ToolIndexEntry(id="cvmfs_mapper", name="Sequence mapper deluxe", version="2.0"),
+        ],
+        {"default": {"local_mapper", "cvmfs_mapper"}},
+        monkeypatch,
+    )
+
+    assert set(search.search("mapper", panel_view="default", config=config)) == {
+        "local_mapper",
+        "cvmfs_mapper",
+    }
+
+
+def test_search_without_index_dir_returns_empty(monkeypatch):
+    search, config = _search(
+        None,
+        [ToolIndexEntry(id="local_mapper", name="Sequence mapper", version="1.0")],
+        {"default": {"local_mapper"}},
+        monkeypatch,
+    )
+
+    assert search.search("mapper", panel_view="default", config=config) == []
+
+
 def test_search_unknown_panel_view_raises_key_error(tmp_path, monkeypatch):
-    monkeypatch.setattr(ToolSearchTuning, "from_config", classmethod(lambda cls, config: _TUNING))
-    index_root = str(tmp_path)
-    _build_store_index(
-        index_root, DEFAULT_STORE_NAME, [ToolIndexEntry(id="local_mapper", name="Sequence mapper", version="1.0")]
+    search, config = _search(
+        str(tmp_path),
+        [ToolIndexEntry(id="local_mapper", name="Sequence mapper", version="1.0")],
+        {"default": {"local_mapper"}},
+        monkeypatch,
     )
-    config = _single_store_config(index_root)
-    search = CachedToolboxSearch(config, _FakeToolbox({"default": {"local_mapper"}}))  # type: ignore[arg-type]
+
     with pytest.raises(KeyError):
         search.search("mapper", panel_view="does_not_exist", config=config)
 
 
-def test_search_scopes_hits_to_requested_panel_view(tmp_path, monkeypatch):
-    monkeypatch.setattr(ToolSearchTuning, "from_config", classmethod(lambda cls, config: _TUNING))
-    index_root = str(tmp_path)
-    _build_store_index(
-        index_root,
-        DEFAULT_STORE_NAME,
+def test_search_builds_distinct_panel_view_corpora(tmp_path, monkeypatch):
+    search, config = _search(
+        str(tmp_path),
         [
             ToolIndexEntry(id="local_mapper", name="Sequence mapper", version="1.0"),
             ToolIndexEntry(id="other_mapper", name="Sequence mapper deluxe", version="1.0"),
         ],
+        {"default": {"local_mapper", "other_mapper"}, "restricted": {"local_mapper"}},
+        monkeypatch,
     )
-    config = _single_store_config(index_root)
-    search = CachedToolboxSearch(
-        config,
-        _FakeToolbox({"default": {"local_mapper", "other_mapper"}, "restricted": {"local_mapper"}}),  # type: ignore[arg-type]
-    )
-    assert set(search.search("mapper", panel_view="default", config=config)) == {"local_mapper", "other_mapper"}
-    # The restricted view holds only one of the two matching tools; the
-    # out-of-view hit must be dropped.
+
+    assert set(search.search("mapper", panel_view="default", config=config)) == {
+        "local_mapper",
+        "other_mapper",
+    }
     assert search.search("mapper", panel_view="restricted", config=config) == ["local_mapper"]

test/unit/app/tools/source_store/test_populator_entry.py

@@ -13,8 +13,8 @@ from galaxy.tools.source_store.discover import (
 from galaxy.tools.source_store.interface import StoredToolSource
 from galaxy.tools.source_store.populator import (
     build_index_entry_from_source,
-    MAX_HELP_TEXT_CHARS,
 )
+from galaxy.tools.source_store.search import MAX_TOOL_SEARCH_HELP_CHARS
 
 _TOOL_XML = """<tool id="help_tool" name="Help Tool" version="1.0">
   <command>echo</command>
@@ -76,8 +76,8 @@ def test_entry_help_text_empty_without_help_block(tmp_path):
 
 
 def test_entry_help_text_capped(tmp_path):
-    huge = "quaxifier " * (MAX_HELP_TEXT_CHARS // 2)
+    huge = "quaxifier " * (MAX_TOOL_SEARCH_HELP_CHARS // 2)
     xml = _TOOL_XML.replace("This wraps the quaxifier subroutine.", huge)
     entry = _build(tmp_path, xml)
     assert entry is not None
-    assert len(entry.help_text) <= MAX_HELP_TEXT_CHARS
+    assert len(entry.help_text) <= MAX_TOOL_SEARCH_HELP_CHARS

test/unit/app/tools/source_store/test_search_parity.py

@@ -0,0 +1,123 @@
+from types import SimpleNamespace
+from typing import cast
+
+from galaxy.config import GalaxyAppConfiguration
+from galaxy.tools.search import ToolPanelViewSearch
+from galaxy.tools.source_store.index import (
+    ToolIndex,
+    ToolIndexEntry,
+)
+from galaxy.tools.source_store.search import (
+    entry_to_search_document,
+    MAX_TOOL_SEARCH_HELP_CHARS,
+    ToolSearchTuning,
+    ToolWhooshIndex,
+)
+
+_TUNING = ToolSearchTuning(
+    id_boost=20.0,
+    name_boost=10.0,
+    name_exact_multiplier=2.0,
+    stub_boost=5.0,
+    section_boost=4.0,
+    description_boost=3.0,
+    label_boost=3.0,
+    ngram_minsize=3,
+    ngram_maxsize=4,
+    enable_ngram_search=True,
+    ngram_factor=0.5,
+    help_boost=1.0,
+    help_bm25f_k1=1.2,
+)
+
+
+def _tool(entry: ToolIndexEntry):
+    return SimpleNamespace(
+        id=entry.id,
+        guid=None,
+        name=entry.name,
+        description=entry.description,
+        get_panel_section=lambda: (entry.panel_section_id, entry.panel_section_name),
+        edam_operations=entry.edam_operations,
+        edam_topics=entry.edam_topics,
+        repository_name=entry.repository_name,
+        repository_owner=entry.repository_owner,
+        labels=entry.labels,
+        tool_tags=[],
+        raw_help=entry.help_text,
+        tool_type=entry.tool_type,
+    )
+
+
+def test_eager_and_cached_documents_share_help_bound(tmp_path, monkeypatch):
+    monkeypatch.setattr(ToolSearchTuning, "from_config", classmethod(lambda cls, config: _TUNING))
+    config = cast(GalaxyAppConfiguration, SimpleNamespace())
+    eager = ToolPanelViewSearch("default", str(tmp_path / "eager"), config)
+    entry = ToolIndexEntry(
+        id="mapper",
+        version="1.0",
+        name="Sequence mapper",
+        description="Maps sequences",
+        panel_section_id="mapping",
+        panel_section_name="Mapping",
+        labels=["featured"],
+        edam_operations=["Mapping"],
+        edam_topics=["Genomics"],
+        help_text=("h" * MAX_TOOL_SEARCH_HELP_CHARS) + "not-indexed",
+    )
+
+    eager_document = eager._create_doc(_tool(entry))
+    cached_document = entry_to_search_document(entry, include_help=True)
+
+    assert eager_document == cached_document
+    assert len(eager_document["help"]) == MAX_TOOL_SEARCH_HELP_CHARS
+
+
+def test_eager_help_content_object_is_normalized(tmp_path, monkeypatch):
+    monkeypatch.setattr(ToolSearchTuning, "from_config", classmethod(lambda cls, config: _TUNING))
+    config = cast(GalaxyAppConfiguration, SimpleNamespace())
+    eager = ToolPanelViewSearch("default", str(tmp_path / "eager"), config)
+    entry = ToolIndexEntry(id="mapper", version="1.0", name="Sequence mapper")
+    tool = _tool(entry)
+    tool.raw_help = SimpleNamespace(content="quaxifier help")
+
+    assert eager._create_doc(tool)["help"] == "quaxifier help"
+
+
+def test_eager_and_cached_indexes_return_same_ordered_hits(tmp_path, monkeypatch):
+    monkeypatch.setattr(ToolSearchTuning, "from_config", classmethod(lambda cls, config: _TUNING))
+    config = cast(GalaxyAppConfiguration, SimpleNamespace())
+    entries = [
+        ToolIndexEntry(
+            id="mapper_a",
+            version="1.0",
+            name="Sequence mapper",
+            description="Maps genomic reads",
+            panel_section_name="Mapping",
+            labels=["featured"],
+            help_text="aligns reads with quaxifier",
+        ),
+        ToolIndexEntry(
+            id="mapper_b",
+            version="1.0",
+            name="Mapper deluxe",
+            description="Maps sequences",
+            panel_section_name="Mapping",
+            labels=["standard"],
+            help_text="alignment helper",
+        ),
+    ]
+    tool_index = ToolIndex()
+    for entry in entries:
+        tool_index.add_entry(entry)
+
+    eager = ToolPanelViewSearch("default", str(tmp_path / "eager"), config)
+    with eager.index.writer() as writer:
+        for entry in entries:
+            writer.add_document(**eager._create_doc(_tool(entry)))
+
+    cached = ToolWhooshIndex(str(tmp_path / "cached"), _TUNING)
+    cached.build(tool_index, set(tool_index.entries))
+
+    for query in ("mapper", "mapper_a", "genomic", "Mapping", "featured", "quaxifier"):
+        assert eager.search(query, config) == cached.search(query)

test/unit/app/tools/source_store/test_whoosh_build.py

@@ -6,6 +6,7 @@ from galaxy.tools.source_store.index import (
     ToolIndexEntry,
 )
 from galaxy.tools.source_store.search import (
+    MAX_TOOL_SEARCH_HELP_CHARS,
     ToolSearchTuning,
     ToolWhooshIndex,
 )
@@ -54,6 +55,14 @@ def test_changed_corpus_rebuilds_and_drops_stale_docs(tmp_path):
     assert searcher.search("caller") == []
 
 
+def test_build_scopes_corpus_to_panel_membership(tmp_path):
+    searcher = ToolWhooshIndex(index_dir=str(tmp_path / "ix"), tuning=_TUNING)
+
+    assert searcher.build(_index("mapper", "caller"), {"mapper"}) == 1
+    assert searcher.search("mapper") == ["mapper"]
+    assert searcher.search("caller") == []
+
+
 def test_data_managers_are_excluded_from_search(tmp_path):
     tool_index = ToolIndex()
     tool_index.add_entry(
@@ -101,6 +110,21 @@ def test_help_only_phrase_matches_when_help_indexed(tmp_path):
     assert ToolWhooshIndex(index_dir=index_dir, tuning=_TUNING).search("quaxifier") == ["mytool"]
 
 
+def test_help_after_shared_limit_is_not_indexed(tmp_path):
+    index_dir = str(tmp_path / "ix")
+    bounded_prefix = "a" * MAX_TOOL_SEARCH_HELP_CHARS
+    ToolWhooshIndex(index_dir=index_dir, tuning=_TUNING).build(_help_index("mytool", f"{bounded_prefix} quaxifier"))
+
+    assert ToolWhooshIndex(index_dir=index_dir, tuning=_TUNING).search("quaxifier") == []
+
+
+def test_plain_id_field_is_searchable(tmp_path):
+    index_dir = str(tmp_path / "ix")
+    ToolWhooshIndex(index_dir=index_dir, tuning=_TUNING).build(_index("exact_tool_id"))
+
+    assert ToolWhooshIndex(index_dir=index_dir, tuning=_TUNING).search("exact_tool_id") == ["exact_tool_id"]
+
+
 def test_help_omitted_when_index_tool_help_disabled(tmp_path):
     index_dir = str(tmp_path / "ix")
     help_off = ToolSearchTuning(

test/unit/scripts/tool_source/test_populate_store.py

@@ -380,7 +380,6 @@ class TestIncrementalFastPath:
             paths=[str(tool_path)],
             path_guids={str(tool_path): guid},
             incremental=True,
-            rebuild_whoosh=False,
         )
         store, index = _load_default_index(cfg)
         entry = index.entries[guid]
@@ -402,7 +401,6 @@ class TestIncrementalFastPath:
             cfg,
             pattern="fastp.xml",
             incremental=True,
-            rebuild_whoosh=False,
         )
         assert result["unchanged"] == 1
         store.invalidate_index_cache()
@@ -443,8 +441,8 @@ class TestTwinDeterminism:
     Regression: the winner among same-id/same-version twins is decided by
     index add-order. Consuming pool results as_completed — or adding
     carried-forward entries before re-parsed ones — flipped the winner (and
-    its panel section in the whoosh corpus) between runs, defeating the
-    whoosh corpus-signature rebuild skip.
+    its panel section in the search corpus) between runs even though no input
+    changed.
     """
 
     def test_twin_winner_stable_across_runs(self, tmp_path):

test/unit/scripts/tool_source/test_whoosh_dir.py

@@ -1,25 +0,0 @@
-import sys
-from pathlib import Path
-
-galaxy_root = Path(__file__).parent.parent.parent.parent.parent
-sys.path.insert(0, str(galaxy_root / "lib"))
-
-from galaxy.tools.source_store.populator import (
-    DEFAULT_STORE_NAME,
-    whoosh_dir_for_store,
-)
-
-
-def test_default_store_maps_to_store_default_subdir():
-    assert (
-        whoosh_dir_for_store("/var/galaxy/tool_search", DEFAULT_STORE_NAME) == "/var/galaxy/tool_search/_store_default"
-    )
-
-
-def test_named_store_gets_named_subdir():
-    assert whoosh_dir_for_store("/var/galaxy/tool_search", "cvmfs_mirror") == "/var/galaxy/tool_search/cvmfs_mirror"
-
-
-def test_none_or_empty_base_yields_none():
-    assert whoosh_dir_for_store(None, DEFAULT_STORE_NAME) is None
-    assert whoosh_dir_for_store("", DEFAULT_STORE_NAME) is None