Add LazyToolBox and wire it into the Galaxy app

LazyToolBox extends ToolBox and loads each Tool on demand from the
configured tool source store, backed by an LRU cache sized via
``lazy_toolbox_cache_size``. The pre-computed index lets batch
endpoints (panels, tests summary, requirements) avoid materializing
every tool object.

- tools/lazy_toolbox.py: the on-demand toolbox plus its index cache
  invalidation hook.
- tool_util/toolbox/base.py and parser.py: small extension points
  needed for lazy loading.
- tools/__init__.py: helpers shared with LazyToolBox.
- queue_worker/__init__.py: route ``_get_new_toolbox`` through
  LazyToolBox when opted in, and add the
  ``reload_tool_source_cache`` control message used by
  populate_store --watch to invalidate caches across processes.
- app/__init__.py and galaxy_mock.py: build the tool source store at
  app startup and use LazyToolBox when ``use_lazy_toolbox`` is set.

lib/galaxy/app/__init__.py

@@ -387,9 +387,78 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
         # right TOOL_TAG_MAPPING.
         configure_tool_tag_mapping(self.config.tool_tag_mappings_file)
 
+        # Initialize tool source store if configured
+        self._init_tool_source_store()
+
         self.dynamic_tool_manager = DynamicToolManager(self)
         self._toolbox_lock = threading.RLock()
-        self._toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self)
+
+        # Use LazyToolBox if tool source store is available and populated
+        if self._use_lazy_toolbox():
+            self._toolbox = self._create_lazy_toolbox()
+        else:
+            self._toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self)
+
+        # Initialize container finder and toolbox search (requires toolbox)
+        self._init_container_finder()
+        self._set_enabled_container_types()
+        index_help = getattr(self.config, "index_tool_help", True)
+        self.toolbox_search = self._register_singleton(
+            ToolBoxSearch,
+            ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help),
+        )
+
+    def _init_tool_source_store(self) -> None:
+        """Initialize the tool source store for efficient tool loading.
+
+        Misconfiguration (bad backend name, missing required setting) raises
+        ``ConfigurationError`` from ``build_tool_source_store`` — we let it
+        propagate so the operator sees the failure at startup.
+        """
+        # Lazy import: avoids pulling in optional backend deps at module load.
+        from galaxy.tool_source_store import (
+            build_tool_source_store,
+            ToolSourceStore,
+        )
+
+        self.tool_source_store: Optional[ToolSourceStore] = self._register_singleton(
+            ToolSourceStore,  # type: ignore[type-abstract,unused-ignore]
+            build_tool_source_store(self.config, self.model.context),  # type: ignore[arg-type,unused-ignore]
+        )
+        stats = self.tool_source_store.get_stats()
+        tool_count = stats.get("count", 0)
+        log.info(f"Initialized tool source store (backend: {stats.get('backend', 'unknown')}, tools: {tool_count})")
+
+    def _use_lazy_toolbox(self) -> bool:
+        """Determine whether to use LazyToolBox instead of regular ToolBox.
+
+        Opt-in is explicit: only ``use_lazy_toolbox: true`` activates the
+        lazy toolbox. A populated store on its own (e.g. brought in by a
+        per-conf ``store="..."`` attribute) does *not* flip a default
+        deployment to lazy mode — that has to be a deliberate choice.
+        """
+        if self.tool_source_store is None:
+            return False
+        return bool(self.config.use_lazy_toolbox)
+
+    def _create_lazy_toolbox(self) -> "tools.ToolBox":
+        """Create a LazyToolBox instance."""
+        # Lazy import: avoids circular import between galaxy.app and galaxy.tools.
+        from galaxy.tools.lazy_toolbox import LazyToolBox
+
+        cache_size = self.config.lazy_toolbox_cache_size
+        log.info(f"Using LazyToolBox with cache_size={cache_size}")
+
+        return LazyToolBox(
+            config_filenames=self.config.tool_configs,
+            tool_root_dir=self.config.tool_path,
+            app=self,  # type: ignore[arg-type]
+            tool_source_store=self.tool_source_store,
+            cache_size=cache_size,
+        )
+
+    def _init_container_finder(self):
+        """Initialize the container finder for dependency resolution."""
         galaxy_root_dir = os.path.abspath(self.config.root)
         file_path = os.path.abspath(self.config.file_path)
         app_info = AppInfo(
@@ -423,16 +492,6 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi
                 "mulled_resolution"
             )
         self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache)
-        self._set_enabled_container_types()
-        index_help = getattr(self.config, "index_tool_help", True)
-        self.toolbox_search = self._register_singleton(
-            ToolBoxSearch,
-            ToolBoxSearch(
-                self.toolbox,
-                index_dir=self.config.tool_search_index_dir,
-                index_help=index_help,
-            ),
-        )
 
     @property
     def toolbox(self) -> tools.ToolBox:

lib/galaxy/app_unittest_utils/galaxy_mock.py

@@ -291,8 +291,9 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin):
         self.track_jobs_in_database = False
         self.amqp_internal_connection = None
         self.tool_configs = []
-        self.tool_source_database_connection = f"sqlite:///{os.path.join(self.data_dir, 'tool_sources.sqlite')}"
+        self.tool_source_store = "database"
         self.tool_source_stores = None
+        self.use_lazy_toolbox = False
         self.manage_dependency_relationships = False
         self.enable_tool_shed_check = False
         self.monitor_thread_join_timeout = 1

lib/galaxy/queue_worker/__init__.py

@@ -288,12 +288,26 @@ def _get_new_toolbox(app: "UniverseApplication", save_integrated_tool_panel: boo
     """
     tool_configs = app.config.tool_configs
 
-    new_toolbox = ToolBox(
-        tool_configs,
-        app.config.tool_path,
-        app,
-        save_integrated_tool_panel=save_integrated_tool_panel,
-    )
+    new_toolbox: ToolBox
+    if getattr(app.config, "use_lazy_toolbox", False) and getattr(app, "tool_source_store", None) is not None:
+        # Lazy import: avoids circular import between galaxy.queue_worker and galaxy.tools.
+        from galaxy.tools.lazy_toolbox import LazyToolBox
+
+        new_toolbox = LazyToolBox(
+            config_filenames=tool_configs,
+            tool_root_dir=app.config.tool_path,
+            app=app,
+            tool_source_store=app.tool_source_store,
+            cache_size=getattr(app.config, "lazy_toolbox_cache_size", 500),
+            save_integrated_tool_panel=save_integrated_tool_panel,
+        )
+    else:
+        new_toolbox = ToolBox(
+            tool_configs,
+            app.config.tool_path,
+            app,
+            save_integrated_tool_panel=save_integrated_tool_panel,
+        )
     new_toolbox.data_manager_tools = app.toolbox.data_manager_tools
     app.datatypes_registry.load_datatype_converters(new_toolbox, use_cached=True)
     app.datatypes_registry.load_external_metadata_tool(new_toolbox)
@@ -378,6 +392,29 @@ def reload_tour(app, **kwargs):
     log.debug("Tour reloaded")
 
 
+def reload_tool_source_cache(app, **kwargs):
+    """
+    Reload the tool source cache/index.
+
+    This is typically triggered by an external process (like populate_store.py --watch)
+    when tool files change on disk.
+    """
+    from galaxy.tools.lazy_toolbox import LazyToolBox
+
+    log.debug("Executing tool source cache reload on '%s'", app.config.server_name)
+
+    # Invalidate the lazy toolbox cache if the active toolbox is a LazyToolBox.
+    toolbox = app.toolbox
+    if isinstance(toolbox, LazyToolBox):
+        toolbox.invalidate_index_cache()
+        log.info("Tool source index cache invalidated")
+
+    # Invalidate the tool source store cache if it exists
+    if app.tool_source_store is not None:
+        app.tool_source_store.invalidate_index_cache()
+        log.info("Tool source store cache invalidated")
+
+
 def __job_rule_module_names(app: "MinimalManagerApp"):
     rules_module_names = {"galaxy.jobs.rules"}
     if app.job_config.dynamic_params is not None:
@@ -554,6 +591,7 @@ control_message_to_task = {
     "entry_point_update": entry_point_update,
     "subscribe_history_viewer": subscribe_history_viewer,
     "unsubscribe_history_viewer": unsubscribe_history_viewer,
+    "reload_tool_source_cache": reload_tool_source_cache,
 }
 
 

lib/galaxy/tool_util/toolbox/base.py

@@ -5,7 +5,6 @@ import os
 import string
 import time
 from collections import namedtuple
-from collections.abc import Iterator
 from errno import ENOENT
 from typing import (
     Any,
@@ -30,7 +29,6 @@ from galaxy.exceptions import (
     RequestParameterInvalidException,
 )
 from galaxy.util import (
-    Element,
     etree,
     ExecutionTimer,
     listify,
@@ -39,7 +37,6 @@ from galaxy.util import (
     unicodify,
 )
 from galaxy.util.bunch import Bunch
-from galaxy.util.path import StrPath
 from .filters import FilterFactory
 from .integrated_panel import ManagesIntegratedToolPanelMixin
 from .lineages import LineageMap
@@ -88,6 +85,8 @@ if TYPE_CHECKING:
 
     # both web transactions and FastAPI/agents request contexts render tool panels
     PanelViewTrans: TypeAlias = "GalaxyWebTransaction | SessionRequestContext"
+    from galaxy.util import Element
+    from galaxy.util.path import StrPath
 
 log = logging.getLogger(__name__)
 
@@ -172,47 +171,6 @@ class ToolLoadConfigurationConflict(Exception):
     pass
 
 
-def walk_tool_directories(directory: StrPath, recursive: bool) -> Iterator[tuple[str, list[str]]]:
-    """Yield ``(directory, files)`` for ``directory`` and, when ``recursive``,
-    each subdirectory - skipping hidden/private (``.``/``_`` prefixed) entries.
-
-    ``files`` are candidate tool file paths in sorted order; filtering them
-    (e.g. via ``looks_like_a_tool``) is the caller's responsibility.
-    """
-    files = []
-    subdirs = []
-    for name in sorted(os.listdir(directory)):
-        if name.startswith((".", "_")):
-            # Very unlikely that we want to load tools from a hidden or private folder
-            continue
-        child = os.path.join(str(directory), name)
-        if os.path.isdir(child):
-            subdirs.append(child)
-        else:
-            files.append(child)
-    yield str(directory), files
-    if recursive:
-        for subdir in subdirs:
-            yield from walk_tool_directories(subdir, recursive)
-
-
-def resolve_tool_path(tool_path: str | None, config_filename: str, default_tool_path: "StrPath | None" = None) -> str:
-    """Resolve a tool conf's ``tool_path`` attribute to the directory its tool
-    files are relative to.
-
-    Expands the ``${tool_conf_dir}`` template; falls back to
-    ``default_tool_path`` (the toolbox's ``tool_root_dir``, i.e.
-    ``config.tool_path``) when the conf doesn't set one.
-    """
-    if not tool_path:
-        # Default to backward compatible config setting.
-        return str(default_tool_path) if default_tool_path else ""
-    # Allow use of ${tool_conf_dir} in toolbox config files.
-    tool_conf_dir = os.path.dirname(config_filename)
-    tool_path_vars = {"tool_conf_dir": tool_conf_dir}
-    return string.Template(tool_path).safe_substitute(tool_path_vars)
-
-
 class AbstractToolBox(ManagesIntegratedToolPanelMixin):
     """
     Abstract container for managing a ToolPanel - containing tools and
@@ -333,7 +291,11 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin):
             config_value = getattr(config, "default_panel_view", None)
         return config_value or self.__default_panel_view
 
-    def create_tool(self, config_file: StrPath, **kwds: Any) -> "Tool":
+    def default_panel_view(self, trans):
+        """Public accessor for the default tool panel view name."""
+        return self._default_panel_view(trans)
+
+    def create_tool(self, config_file: "StrPath", **kwds) -> "Tool":
         raise NotImplementedError()
 
     def create_dynamic_tool(self, dynamic_tool: "DynamicTool") -> "Tool":
@@ -427,7 +389,7 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin):
             config_elems = []
         tool_conf_type = "shed tool" if parsing_shed_tool_conf else "tool"
         log.debug("Tool path for %s configuration %s is %s", tool_conf_type, config_filename, tool_path)
-        tool_path = resolve_tool_path(tool_path, config_filename, self._tool_root_dir)
+        tool_path = self.__resolve_tool_path(tool_path, config_filename)
         # Only load the panel_dict under certain conditions.
         load_panel_dict = not self._integrated_tool_panel_config_has_contents
         for item in tool_conf_source.parse_items():
@@ -602,6 +564,17 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin):
         tool_id = tool.id
         return self._tool_panel.get_section_for_tool_id(tool_id)
 
+    def __resolve_tool_path(self, tool_path, config_filename):
+        if not tool_path:
+            # Default to backward compatible config setting.
+            tool_path = self._tool_root_dir
+        else:
+            # Allow use of __tool_conf_dir__ in toolbox config files.
+            tool_conf_dir = os.path.dirname(config_filename)
+            tool_path_vars = {"tool_conf_dir": tool_conf_dir}
+            tool_path = string.Template(tool_path).safe_substitute(tool_path_vars)
+        return tool_path
+
     def add_tool_to_tool_panel_view(self, tool, view_panel_component):
         self.__add_tool_to_tool_panel(tool, view_panel_component)
 
@@ -1036,7 +1009,7 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin):
             log.exception("Error reading tool from path: %s", path)
 
     def get_tool_repository_from_xml_item(
-        self, elem: Element, path: str
+        self, elem: "Element", path: str
     ) -> Union[ToolConfRepository, "ToolShedRepository"]:
         tool_shed_el = elem.find("tool_shed")
         assert tool_shed_el is not None
@@ -1195,14 +1168,14 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin):
 
     def __watch_directory(
         self,
-        directory: StrPath,
-        elems: ToolPanelElements,
-        integrated_elems: ToolPanelElements,
+        directory: "StrPath",
+        elems,
+        integrated_elems,
         load_panel_dict: bool,
         recursive: bool,
         force_watch: bool = False,
     ) -> None:
-        def quick_load(tool_file: StrPath, async_load: bool = True) -> str | None:
+        def quick_load(tool_file: "StrPath", async_load: bool = True) -> str | None:
             if not self._looks_like_a_tool(str(tool_file)):
                 return None
             try:
@@ -1227,24 +1200,30 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin):
                 log.exception("Failed to load potential tool %s.", tool_file)
             return None
 
+        tool_loaded = False
         if not os.path.isdir(directory):
             log.error("Failed to read tool directory %s.", directory)
             return
-        for dirpath, files in walk_tool_directories(directory, recursive):
-            tool_loaded = False
-            for child_path in files:
-                if self._looks_like_a_tool(child_path):
-                    tool_loaded = bool(quick_load(child_path, async_load=False)) or tool_loaded
-            if (tool_loaded or (force_watch and dirpath == str(directory))) and self._tool_watcher:
-                self._tool_watcher.watch_directory(dirpath, quick_load)
+        for name in os.listdir(directory):
+            if name.startswith((".", "_")):
+                # Very unlikely that we want to load tools from a hidden or private folder
+                continue
+            child_path = os.path.join(directory, name)
+            if os.path.isdir(child_path) and recursive:
+                self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive)
+            elif self._looks_like_a_tool(child_path):
+                tool_id = quick_load(child_path, async_load=False)
+                tool_loaded = bool(tool_id)
+        if (tool_loaded or force_watch) and self._tool_watcher:
+            self._tool_watcher.watch_directory(directory, quick_load)
 
     def load_tool(
         self,
-        config_file: StrPath,
-        guid: str | None = None,
-        tool_shed_repository: "ToolConfRepository | ToolShedRepository | None" = None,
+        config_file: "StrPath",
+        guid=None,
+        tool_shed_repository=None,
         use_cached: bool = False,
-        **kwds: Any,
+        **kwds,
     ) -> "Tool":
         """Load a single tool from the file named by `config_file` and return an instance of `Tool`."""
         # Parse XML configuration file and get the root element
@@ -1279,12 +1258,12 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin):
             if self._tool_config_watcher:
                 [self._tool_config_watcher.watch_file(macro_path) for macro_path in tool._macro_paths]
 
-    def add_tool_to_cache(self, tool: "Tool", config_file: StrPath) -> None:
+    def add_tool_to_cache(self, tool: "Tool", config_file: "StrPath") -> None:
         tool_cache: ToolCache | None = getattr(self.app, "tool_cache", None)
         if tool_cache:
             tool_cache.cache_tool(config_file, tool)
 
-    def load_tool_from_cache(self, config_file: StrPath, recover_tool: bool = False) -> Union["Tool", None]:
+    def load_tool_from_cache(self, config_file: "StrPath", recover_tool: bool = False) -> Union["Tool", None]:
         tool_cache: ToolCache | None = getattr(self.app, "tool_cache", None)
         tool = None
         if tool_cache:

lib/galaxy/tool_util/toolbox/parser.py

@@ -8,6 +8,7 @@ from abc import (
     ABCMeta,
     abstractmethod,
 )
+from typing import Optional
 
 import yaml
 
@@ -39,7 +40,7 @@ class ToolConfSource(metaclass=ABCMeta):
         """Monitor the toolbox configuration source for changes and reload."""
         return DEFAULT_MONITOR
 
-    def parse_store_name(self) -> str | None:
+    def parse_store_name(self) -> Optional[str]:
         """Return the named tool source store this conf routes to, or None.
 
         Lets a single tool_conf opt into a non-default store (e.g. a
@@ -68,7 +69,7 @@ class XmlToolConfSource(ToolConfSource):
     def parse_monitor(self):
         return string_as_bool(self.root.get("monitor", DEFAULT_MONITOR))
 
-    def parse_store_name(self) -> str | None:
+    def parse_store_name(self) -> Optional[str]:
         return self.root.get("store") or None
 
 
@@ -90,7 +91,7 @@ class YamlToolConfSource(ToolConfSource):
     def is_shed_tool_conf(self):
         return False
 
-    def parse_store_name(self) -> str | None:
+    def parse_store_name(self) -> Optional[str]:
         store = self.as_dict.get("store")
         return store or None
 

lib/galaxy/tools/__init__.py

@@ -76,6 +76,7 @@ from galaxy.tool_util.deps import (
 )
 from galaxy.tool_util.deps.requirements import CredentialsRequirement
 from galaxy.tool_util.fetcher import ToolLocationFetcher
+from galaxy.tool_util.id_util import extract_tool_id_from_file
 from galaxy.tool_util.identifiers import uri_safe_tool_id
 from galaxy.tool_util.loader import (
     imported_macro_paths,
@@ -524,6 +525,8 @@ class ToolBox(AbstractToolBox):
         self, config_filenames: list[str], tool_root_dir, app, save_integrated_tool_panel: bool = True
     ) -> None:
         self._reload_count = 0
+        self._tools_loaded_from_store = 0
+        self._tools_parsed_from_file = 0
         self.tool_location_fetcher = ToolLocationFetcher()
         # This is here to deal with the old default value, which doesn't make
         # sense in an "installed Galaxy" world.
@@ -552,6 +555,35 @@ class ToolBox(AbstractToolBox):
         else:
             self._init_dependency_manager()
 
+        # Log tool loading summary
+        self._log_tool_loading_summary()
+
+    def _log_tool_loading_summary(self):
+        """Log a summary of how tools were loaded (from store vs parsed from file)."""
+        total_tools = len(self._tools_by_id)
+        store_count = self._tools_loaded_from_store
+        file_count = self._tools_parsed_from_file
+
+        if store_count > 0 or file_count > 0:
+            store = getattr(self.app, "tool_source_store", None)
+            backend = "unknown"
+            if store:
+                try:
+                    stats = store.get_stats()
+                    backend = stats.get("backend", "unknown")
+                except Exception:
+                    pass
+
+            if store_count > 0 and file_count == 0:
+                log.info(f"Loaded {total_tools} tools from tool source store ({backend}), 0 parsed from files")
+            elif store_count == 0:
+                log.info(f"Loaded {total_tools} tools by parsing from files (no store configured or empty)")
+            else:
+                log.info(
+                    f"Loaded {total_tools} tools: {store_count} from store ({backend}), "
+                    f"{file_count} parsed from files"
+                )
+
     def tool_tag_manager(self):
         if hasattr(self.app.config, "get_bool") and self.app.config.get_bool("enable_tool_tags", False):
             return PersistentToolTagManager(self.app)
@@ -620,21 +652,91 @@ class ToolBox(AbstractToolBox):
         return self._tools_by_id
 
     def create_tool(self, config_file: StrPath, **kwds) -> "Tool":
-        tool_source = self.get_expanded_tool_source(config_file)
+        # Pass guid to enable direct store lookup for shed tools
+        guid = kwds.get("guid")
+        tool_source = self.get_expanded_tool_source(config_file, tool_id=guid)
         return self._create_tool_from_source(tool_source, config_file=config_file, **kwds)
 
-    def get_expanded_tool_source(self, config_file: StrPath) -> ToolSource:
+    def get_expanded_tool_source(self, config_file: StrPath, tool_id: Optional[str] = None) -> ToolSource:
+        # Try to load from tool source store first (pre-parsed, macro-expanded)
+        tool_source = self._get_tool_source_from_store(config_file, tool_id=tool_id)
+        if tool_source is not None:
+            self._tools_loaded_from_store += 1
+            return tool_source
+
+        # Fall back to parsing from file
         try:
-            return get_tool_source(
+            tool_source = get_tool_source(
                 config_file,
                 enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False),
                 tool_location_fetcher=self.tool_location_fetcher,
             )
+            self._tools_parsed_from_file += 1
+            return tool_source
         except Exception as e:
             # capture and log parsing errors
             global_tool_errors.add_error(config_file, "Tool XML parsing", e)
             raise e
 
+    def _get_tool_source_from_store(self, config_file: StrPath, tool_id: Optional[str] = None) -> Optional[ToolSource]:
+        """
+        Try to load tool source from the pre-parsed store.
+
+        Args:
+            config_file: Path to the tool XML file.
+            tool_id: Optional tool ID (guid for shed tools) to look up directly.
+
+        Returns:
+            ToolSource if found in store, None otherwise.
+        """
+        store = getattr(self.app, "tool_source_store", None)
+        if store is None:
+            return None
+
+        stored = None
+
+        # If we have a tool_id, try direct lookup first (fastest)
+        if tool_id:
+            sources = store.get_by_tool_id(tool_id)
+            if sources:
+                # Get most recent version
+                stored = sources[0]
+
+        # If no tool_id or not found, try to match by tool_dir
+        if stored is None:
+            # Quick extraction of tool_id from raw XML without full macro expansion
+            try:
+                extracted_id = extract_tool_id_from_file(str(config_file), max_read=2000)
+                if extracted_id:
+                    sources = store.get_by_tool_id(extracted_id)
+                    if sources:
+                        # Check if any source matches this file's directory
+                        config_dir = str(Path(config_file).parent)
+                        for source in sources:
+                            if source.tool_dir == config_dir:
+                                stored = source
+                                break
+                        if stored is None:
+                            # Just use the first one if dir doesn't match
+                            stored = sources[0]
+            except Exception:
+                pass
+
+        if stored is None:
+            return None
+
+        # Create tool source from stored content
+        try:
+            tool_source = get_tool_source(
+                raw_tool_source=stored.raw_source,
+                tool_source_class=stored.tool_source_class,
+            )
+            log.debug(f"Loaded tool source from store: {stored.tool_id} ({config_file})")
+            return tool_source
+        except Exception as e:
+            log.warning(f"Error loading tool source from store for {config_file}: {e}")
+            return None
+
     def _create_tool_from_source(self, tool_source: ToolSource, **kwds):
         return create_tool_from_source(self.app, tool_source, **kwds)