diff --git a/lib/galaxy/app/__init__.py b/lib/galaxy/app/__init__.py index e9d19862fe7..8442d546141 100644 --- a/lib/galaxy/app/__init__.py +++ b/lib/galaxy/app/__init__.py @@ -387,9 +387,78 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi # right TOOL_TAG_MAPPING. configure_tool_tag_mapping(self.config.tool_tag_mappings_file) + # Initialize tool source store if configured + self._init_tool_source_store() + self.dynamic_tool_manager = DynamicToolManager(self) self._toolbox_lock = threading.RLock() - self._toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) + + # Use LazyToolBox if tool source store is available and populated + if self._use_lazy_toolbox(): + self._toolbox = self._create_lazy_toolbox() + else: + self._toolbox = tools.ToolBox(self.config.tool_configs, self.config.tool_path, self) + + # Initialize container finder and toolbox search (requires toolbox) + self._init_container_finder() + self._set_enabled_container_types() + index_help = getattr(self.config, "index_tool_help", True) + self.toolbox_search = self._register_singleton( + ToolBoxSearch, + ToolBoxSearch(self.toolbox, index_dir=self.config.tool_search_index_dir, index_help=index_help), + ) + + def _init_tool_source_store(self) -> None: + """Initialize the tool source store for efficient tool loading. + + Misconfiguration (bad backend name, missing required setting) raises + ``ConfigurationError`` from ``build_tool_source_store`` — we let it + propagate so the operator sees the failure at startup. + """ + # Lazy import: avoids pulling in optional backend deps at module load. + from galaxy.tool_source_store import ( + build_tool_source_store, + ToolSourceStore, + ) + + self.tool_source_store: Optional[ToolSourceStore] = self._register_singleton( + ToolSourceStore, # type: ignore[type-abstract,unused-ignore] + build_tool_source_store(self.config, self.model.context), # type: ignore[arg-type,unused-ignore] + ) + stats = self.tool_source_store.get_stats() + tool_count = stats.get("count", 0) + log.info(f"Initialized tool source store (backend: {stats.get('backend', 'unknown')}, tools: {tool_count})") + + def _use_lazy_toolbox(self) -> bool: + """Determine whether to use LazyToolBox instead of regular ToolBox. + + Opt-in is explicit: only ``use_lazy_toolbox: true`` activates the + lazy toolbox. A populated store on its own (e.g. brought in by a + per-conf ``store="..."`` attribute) does *not* flip a default + deployment to lazy mode — that has to be a deliberate choice. + """ + if self.tool_source_store is None: + return False + return bool(self.config.use_lazy_toolbox) + + def _create_lazy_toolbox(self) -> "tools.ToolBox": + """Create a LazyToolBox instance.""" + # Lazy import: avoids circular import between galaxy.app and galaxy.tools. + from galaxy.tools.lazy_toolbox import LazyToolBox + + cache_size = self.config.lazy_toolbox_cache_size + log.info(f"Using LazyToolBox with cache_size={cache_size}") + + return LazyToolBox( + config_filenames=self.config.tool_configs, + tool_root_dir=self.config.tool_path, + app=self, # type: ignore[arg-type] + tool_source_store=self.tool_source_store, + cache_size=cache_size, + ) + + def _init_container_finder(self): + """Initialize the container finder for dependency resolution.""" galaxy_root_dir = os.path.abspath(self.config.root) file_path = os.path.abspath(self.config.file_path) app_info = AppInfo( @@ -423,16 +492,6 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi "mulled_resolution" ) self.container_finder = containers.ContainerFinder(app_info, mulled_resolution_cache=mulled_resolution_cache) - self._set_enabled_container_types() - index_help = getattr(self.config, "index_tool_help", True) - self.toolbox_search = self._register_singleton( - ToolBoxSearch, - ToolBoxSearch( - self.toolbox, - index_dir=self.config.tool_search_index_dir, - index_help=index_help, - ), - ) @property def toolbox(self) -> tools.ToolBox: diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index a8c41dd4bd8..6a91ff822b5 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -291,8 +291,9 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.track_jobs_in_database = False self.amqp_internal_connection = None self.tool_configs = [] - self.tool_source_database_connection = f"sqlite:///{os.path.join(self.data_dir, 'tool_sources.sqlite')}" + self.tool_source_store = "database" self.tool_source_stores = None + self.use_lazy_toolbox = False self.manage_dependency_relationships = False self.enable_tool_shed_check = False self.monitor_thread_join_timeout = 1 diff --git a/lib/galaxy/queue_worker/__init__.py b/lib/galaxy/queue_worker/__init__.py index 7358a37c6c4..9224371fdf4 100644 --- a/lib/galaxy/queue_worker/__init__.py +++ b/lib/galaxy/queue_worker/__init__.py @@ -288,12 +288,26 @@ def _get_new_toolbox(app: "UniverseApplication", save_integrated_tool_panel: boo """ tool_configs = app.config.tool_configs - new_toolbox = ToolBox( - tool_configs, - app.config.tool_path, - app, - save_integrated_tool_panel=save_integrated_tool_panel, - ) + new_toolbox: ToolBox + if getattr(app.config, "use_lazy_toolbox", False) and getattr(app, "tool_source_store", None) is not None: + # Lazy import: avoids circular import between galaxy.queue_worker and galaxy.tools. + from galaxy.tools.lazy_toolbox import LazyToolBox + + new_toolbox = LazyToolBox( + config_filenames=tool_configs, + tool_root_dir=app.config.tool_path, + app=app, + tool_source_store=app.tool_source_store, + cache_size=getattr(app.config, "lazy_toolbox_cache_size", 500), + save_integrated_tool_panel=save_integrated_tool_panel, + ) + else: + new_toolbox = ToolBox( + tool_configs, + app.config.tool_path, + app, + save_integrated_tool_panel=save_integrated_tool_panel, + ) new_toolbox.data_manager_tools = app.toolbox.data_manager_tools app.datatypes_registry.load_datatype_converters(new_toolbox, use_cached=True) app.datatypes_registry.load_external_metadata_tool(new_toolbox) @@ -378,6 +392,29 @@ def reload_tour(app, **kwargs): log.debug("Tour reloaded") +def reload_tool_source_cache(app, **kwargs): + """ + Reload the tool source cache/index. + + This is typically triggered by an external process (like populate_store.py --watch) + when tool files change on disk. + """ + from galaxy.tools.lazy_toolbox import LazyToolBox + + log.debug("Executing tool source cache reload on '%s'", app.config.server_name) + + # Invalidate the lazy toolbox cache if the active toolbox is a LazyToolBox. + toolbox = app.toolbox + if isinstance(toolbox, LazyToolBox): + toolbox.invalidate_index_cache() + log.info("Tool source index cache invalidated") + + # Invalidate the tool source store cache if it exists + if app.tool_source_store is not None: + app.tool_source_store.invalidate_index_cache() + log.info("Tool source store cache invalidated") + + def __job_rule_module_names(app: "MinimalManagerApp"): rules_module_names = {"galaxy.jobs.rules"} if app.job_config.dynamic_params is not None: @@ -554,6 +591,7 @@ control_message_to_task = { "entry_point_update": entry_point_update, "subscribe_history_viewer": subscribe_history_viewer, "unsubscribe_history_viewer": unsubscribe_history_viewer, + "reload_tool_source_cache": reload_tool_source_cache, } diff --git a/lib/galaxy/tool_util/toolbox/base.py b/lib/galaxy/tool_util/toolbox/base.py index c21271cfb74..4f95ccd1717 100644 --- a/lib/galaxy/tool_util/toolbox/base.py +++ b/lib/galaxy/tool_util/toolbox/base.py @@ -5,7 +5,6 @@ import os import string import time from collections import namedtuple -from collections.abc import Iterator from errno import ENOENT from typing import ( Any, @@ -30,7 +29,6 @@ from galaxy.exceptions import ( RequestParameterInvalidException, ) from galaxy.util import ( - Element, etree, ExecutionTimer, listify, @@ -39,7 +37,6 @@ from galaxy.util import ( unicodify, ) from galaxy.util.bunch import Bunch -from galaxy.util.path import StrPath from .filters import FilterFactory from .integrated_panel import ManagesIntegratedToolPanelMixin from .lineages import LineageMap @@ -88,6 +85,8 @@ if TYPE_CHECKING: # both web transactions and FastAPI/agents request contexts render tool panels PanelViewTrans: TypeAlias = "GalaxyWebTransaction | SessionRequestContext" + from galaxy.util import Element + from galaxy.util.path import StrPath log = logging.getLogger(__name__) @@ -172,47 +171,6 @@ class ToolLoadConfigurationConflict(Exception): pass -def walk_tool_directories(directory: StrPath, recursive: bool) -> Iterator[tuple[str, list[str]]]: - """Yield ``(directory, files)`` for ``directory`` and, when ``recursive``, - each subdirectory - skipping hidden/private (``.``/``_`` prefixed) entries. - - ``files`` are candidate tool file paths in sorted order; filtering them - (e.g. via ``looks_like_a_tool``) is the caller's responsibility. - """ - files = [] - subdirs = [] - for name in sorted(os.listdir(directory)): - if name.startswith((".", "_")): - # Very unlikely that we want to load tools from a hidden or private folder - continue - child = os.path.join(str(directory), name) - if os.path.isdir(child): - subdirs.append(child) - else: - files.append(child) - yield str(directory), files - if recursive: - for subdir in subdirs: - yield from walk_tool_directories(subdir, recursive) - - -def resolve_tool_path(tool_path: str | None, config_filename: str, default_tool_path: "StrPath | None" = None) -> str: - """Resolve a tool conf's ``tool_path`` attribute to the directory its tool - files are relative to. - - Expands the ``${tool_conf_dir}`` template; falls back to - ``default_tool_path`` (the toolbox's ``tool_root_dir``, i.e. - ``config.tool_path``) when the conf doesn't set one. - """ - if not tool_path: - # Default to backward compatible config setting. - return str(default_tool_path) if default_tool_path else "" - # Allow use of ${tool_conf_dir} in toolbox config files. - tool_conf_dir = os.path.dirname(config_filename) - tool_path_vars = {"tool_conf_dir": tool_conf_dir} - return string.Template(tool_path).safe_substitute(tool_path_vars) - - class AbstractToolBox(ManagesIntegratedToolPanelMixin): """ Abstract container for managing a ToolPanel - containing tools and @@ -333,7 +291,11 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): config_value = getattr(config, "default_panel_view", None) return config_value or self.__default_panel_view - def create_tool(self, config_file: StrPath, **kwds: Any) -> "Tool": + def default_panel_view(self, trans): + """Public accessor for the default tool panel view name.""" + return self._default_panel_view(trans) + + def create_tool(self, config_file: "StrPath", **kwds) -> "Tool": raise NotImplementedError() def create_dynamic_tool(self, dynamic_tool: "DynamicTool") -> "Tool": @@ -427,7 +389,7 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): config_elems = [] tool_conf_type = "shed tool" if parsing_shed_tool_conf else "tool" log.debug("Tool path for %s configuration %s is %s", tool_conf_type, config_filename, tool_path) - tool_path = resolve_tool_path(tool_path, config_filename, self._tool_root_dir) + tool_path = self.__resolve_tool_path(tool_path, config_filename) # Only load the panel_dict under certain conditions. load_panel_dict = not self._integrated_tool_panel_config_has_contents for item in tool_conf_source.parse_items(): @@ -602,6 +564,17 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): tool_id = tool.id return self._tool_panel.get_section_for_tool_id(tool_id) + def __resolve_tool_path(self, tool_path, config_filename): + if not tool_path: + # Default to backward compatible config setting. + tool_path = self._tool_root_dir + else: + # Allow use of __tool_conf_dir__ in toolbox config files. + tool_conf_dir = os.path.dirname(config_filename) + tool_path_vars = {"tool_conf_dir": tool_conf_dir} + tool_path = string.Template(tool_path).safe_substitute(tool_path_vars) + return tool_path + def add_tool_to_tool_panel_view(self, tool, view_panel_component): self.__add_tool_to_tool_panel(tool, view_panel_component) @@ -1036,7 +1009,7 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): log.exception("Error reading tool from path: %s", path) def get_tool_repository_from_xml_item( - self, elem: Element, path: str + self, elem: "Element", path: str ) -> Union[ToolConfRepository, "ToolShedRepository"]: tool_shed_el = elem.find("tool_shed") assert tool_shed_el is not None @@ -1195,14 +1168,14 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): def __watch_directory( self, - directory: StrPath, - elems: ToolPanelElements, - integrated_elems: ToolPanelElements, + directory: "StrPath", + elems, + integrated_elems, load_panel_dict: bool, recursive: bool, force_watch: bool = False, ) -> None: - def quick_load(tool_file: StrPath, async_load: bool = True) -> str | None: + def quick_load(tool_file: "StrPath", async_load: bool = True) -> str | None: if not self._looks_like_a_tool(str(tool_file)): return None try: @@ -1227,24 +1200,30 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): log.exception("Failed to load potential tool %s.", tool_file) return None + tool_loaded = False if not os.path.isdir(directory): log.error("Failed to read tool directory %s.", directory) return - for dirpath, files in walk_tool_directories(directory, recursive): - tool_loaded = False - for child_path in files: - if self._looks_like_a_tool(child_path): - tool_loaded = bool(quick_load(child_path, async_load=False)) or tool_loaded - if (tool_loaded or (force_watch and dirpath == str(directory))) and self._tool_watcher: - self._tool_watcher.watch_directory(dirpath, quick_load) + for name in os.listdir(directory): + if name.startswith((".", "_")): + # Very unlikely that we want to load tools from a hidden or private folder + continue + child_path = os.path.join(directory, name) + if os.path.isdir(child_path) and recursive: + self.__watch_directory(child_path, elems, integrated_elems, load_panel_dict, recursive) + elif self._looks_like_a_tool(child_path): + tool_id = quick_load(child_path, async_load=False) + tool_loaded = bool(tool_id) + if (tool_loaded or force_watch) and self._tool_watcher: + self._tool_watcher.watch_directory(directory, quick_load) def load_tool( self, - config_file: StrPath, - guid: str | None = None, - tool_shed_repository: "ToolConfRepository | ToolShedRepository | None" = None, + config_file: "StrPath", + guid=None, + tool_shed_repository=None, use_cached: bool = False, - **kwds: Any, + **kwds, ) -> "Tool": """Load a single tool from the file named by `config_file` and return an instance of `Tool`.""" # Parse XML configuration file and get the root element @@ -1279,12 +1258,12 @@ class AbstractToolBox(ManagesIntegratedToolPanelMixin): if self._tool_config_watcher: [self._tool_config_watcher.watch_file(macro_path) for macro_path in tool._macro_paths] - def add_tool_to_cache(self, tool: "Tool", config_file: StrPath) -> None: + def add_tool_to_cache(self, tool: "Tool", config_file: "StrPath") -> None: tool_cache: ToolCache | None = getattr(self.app, "tool_cache", None) if tool_cache: tool_cache.cache_tool(config_file, tool) - def load_tool_from_cache(self, config_file: StrPath, recover_tool: bool = False) -> Union["Tool", None]: + def load_tool_from_cache(self, config_file: "StrPath", recover_tool: bool = False) -> Union["Tool", None]: tool_cache: ToolCache | None = getattr(self.app, "tool_cache", None) tool = None if tool_cache: diff --git a/lib/galaxy/tool_util/toolbox/parser.py b/lib/galaxy/tool_util/toolbox/parser.py index 0d1c284daf5..191630a2931 100644 --- a/lib/galaxy/tool_util/toolbox/parser.py +++ b/lib/galaxy/tool_util/toolbox/parser.py @@ -8,6 +8,7 @@ from abc import ( ABCMeta, abstractmethod, ) +from typing import Optional import yaml @@ -39,7 +40,7 @@ class ToolConfSource(metaclass=ABCMeta): """Monitor the toolbox configuration source for changes and reload.""" return DEFAULT_MONITOR - def parse_store_name(self) -> str | None: + def parse_store_name(self) -> Optional[str]: """Return the named tool source store this conf routes to, or None. Lets a single tool_conf opt into a non-default store (e.g. a @@ -68,7 +69,7 @@ class XmlToolConfSource(ToolConfSource): def parse_monitor(self): return string_as_bool(self.root.get("monitor", DEFAULT_MONITOR)) - def parse_store_name(self) -> str | None: + def parse_store_name(self) -> Optional[str]: return self.root.get("store") or None @@ -90,7 +91,7 @@ class YamlToolConfSource(ToolConfSource): def is_shed_tool_conf(self): return False - def parse_store_name(self) -> str | None: + def parse_store_name(self) -> Optional[str]: store = self.as_dict.get("store") return store or None diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index a30892eb4bb..c5baf21d4cc 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -76,6 +76,7 @@ from galaxy.tool_util.deps import ( ) from galaxy.tool_util.deps.requirements import CredentialsRequirement from galaxy.tool_util.fetcher import ToolLocationFetcher +from galaxy.tool_util.id_util import extract_tool_id_from_file from galaxy.tool_util.identifiers import uri_safe_tool_id from galaxy.tool_util.loader import ( imported_macro_paths, @@ -524,6 +525,8 @@ class ToolBox(AbstractToolBox): self, config_filenames: list[str], tool_root_dir, app, save_integrated_tool_panel: bool = True ) -> None: self._reload_count = 0 + self._tools_loaded_from_store = 0 + self._tools_parsed_from_file = 0 self.tool_location_fetcher = ToolLocationFetcher() # This is here to deal with the old default value, which doesn't make # sense in an "installed Galaxy" world. @@ -552,6 +555,35 @@ class ToolBox(AbstractToolBox): else: self._init_dependency_manager() + # Log tool loading summary + self._log_tool_loading_summary() + + def _log_tool_loading_summary(self): + """Log a summary of how tools were loaded (from store vs parsed from file).""" + total_tools = len(self._tools_by_id) + store_count = self._tools_loaded_from_store + file_count = self._tools_parsed_from_file + + if store_count > 0 or file_count > 0: + store = getattr(self.app, "tool_source_store", None) + backend = "unknown" + if store: + try: + stats = store.get_stats() + backend = stats.get("backend", "unknown") + except Exception: + pass + + if store_count > 0 and file_count == 0: + log.info(f"Loaded {total_tools} tools from tool source store ({backend}), 0 parsed from files") + elif store_count == 0: + log.info(f"Loaded {total_tools} tools by parsing from files (no store configured or empty)") + else: + log.info( + f"Loaded {total_tools} tools: {store_count} from store ({backend}), " + f"{file_count} parsed from files" + ) + def tool_tag_manager(self): if hasattr(self.app.config, "get_bool") and self.app.config.get_bool("enable_tool_tags", False): return PersistentToolTagManager(self.app) @@ -620,21 +652,91 @@ class ToolBox(AbstractToolBox): return self._tools_by_id def create_tool(self, config_file: StrPath, **kwds) -> "Tool": - tool_source = self.get_expanded_tool_source(config_file) + # Pass guid to enable direct store lookup for shed tools + guid = kwds.get("guid") + tool_source = self.get_expanded_tool_source(config_file, tool_id=guid) return self._create_tool_from_source(tool_source, config_file=config_file, **kwds) - def get_expanded_tool_source(self, config_file: StrPath) -> ToolSource: + def get_expanded_tool_source(self, config_file: StrPath, tool_id: Optional[str] = None) -> ToolSource: + # Try to load from tool source store first (pre-parsed, macro-expanded) + tool_source = self._get_tool_source_from_store(config_file, tool_id=tool_id) + if tool_source is not None: + self._tools_loaded_from_store += 1 + return tool_source + + # Fall back to parsing from file try: - return get_tool_source( + tool_source = get_tool_source( config_file, enable_beta_formats=getattr(self.app.config, "enable_beta_tool_formats", False), tool_location_fetcher=self.tool_location_fetcher, ) + self._tools_parsed_from_file += 1 + return tool_source except Exception as e: # capture and log parsing errors global_tool_errors.add_error(config_file, "Tool XML parsing", e) raise e + def _get_tool_source_from_store(self, config_file: StrPath, tool_id: Optional[str] = None) -> Optional[ToolSource]: + """ + Try to load tool source from the pre-parsed store. + + Args: + config_file: Path to the tool XML file. + tool_id: Optional tool ID (guid for shed tools) to look up directly. + + Returns: + ToolSource if found in store, None otherwise. + """ + store = getattr(self.app, "tool_source_store", None) + if store is None: + return None + + stored = None + + # If we have a tool_id, try direct lookup first (fastest) + if tool_id: + sources = store.get_by_tool_id(tool_id) + if sources: + # Get most recent version + stored = sources[0] + + # If no tool_id or not found, try to match by tool_dir + if stored is None: + # Quick extraction of tool_id from raw XML without full macro expansion + try: + extracted_id = extract_tool_id_from_file(str(config_file), max_read=2000) + if extracted_id: + sources = store.get_by_tool_id(extracted_id) + if sources: + # Check if any source matches this file's directory + config_dir = str(Path(config_file).parent) + for source in sources: + if source.tool_dir == config_dir: + stored = source + break + if stored is None: + # Just use the first one if dir doesn't match + stored = sources[0] + except Exception: + pass + + if stored is None: + return None + + # Create tool source from stored content + try: + tool_source = get_tool_source( + raw_tool_source=stored.raw_source, + tool_source_class=stored.tool_source_class, + ) + log.debug(f"Loaded tool source from store: {stored.tool_id} ({config_file})") + return tool_source + except Exception as e: + log.warning(f"Error loading tool source from store for {config_file}: {e}") + return None + def _create_tool_from_source(self, tool_source: ToolSource, **kwds): return create_tool_from_source(self.app, tool_source, **kwds) diff --git a/lib/galaxy/tools/lazy_toolbox.py b/lib/galaxy/tools/lazy_toolbox.py new file mode 100644 index 00000000000..97aef9a7bbb --- /dev/null +++ b/lib/galaxy/tools/lazy_toolbox.py @@ -0,0 +1,1111 @@ +""" +Lazy ToolBox - On-demand tool loading with LRU caching. + +This module provides a LazyToolBox that extends ToolBox but keeps only a +lightweight index in memory and loads full Tool objects on-demand with +LRU eviction. +""" + +import hashlib +import logging +import os +import string +import threading +from datetime import datetime +from typing import ( + Any, + Literal, + Optional, + overload, + TYPE_CHECKING, + Union, +) +from uuid import UUID + +from cachetools import LRUCache + +from galaxy.tool_source_store import ( + StoredToolSource, + ToolSourceStore, +) +from galaxy.tool_source_store.index import ( + ToolIndex, + ToolIndexEntry, +) +from galaxy.tool_util.id_util import ( + extract_short_id_from_guid, + extract_tool_id_from_file, +) +from galaxy.tool_util.parser import get_tool_source +from galaxy.tool_util.toolbox.base import DynamicToolConfDict +from galaxy.tool_util.toolbox.filters import FilterFactory +from galaxy.tool_util.toolbox.lineages import LineageMap +from galaxy.tool_util.toolbox.panel import ( + ToolPanelElements, + ToolSection, +) +from galaxy.tool_util.toolbox.views.edam import ( + EdamPanelMode, + EdamToolPanelView, +) +from galaxy.tool_util.toolbox.views.interface import ( + ToolPanelView, + ToolPanelViewModel, + ToolPanelViewModelType, +) +from galaxy.tool_util.toolbox.views.sources import StaticToolBoxViewSources +from galaxy.util import listify +from . import ( + create_tool_from_source, + ToolBox, +) + +if TYPE_CHECKING: + from galaxy.app import UniverseApplication + from galaxy.model import User + from galaxy.tools import Tool + +log = logging.getLogger(__name__) + + +class DefaultToolPanelView(ToolPanelView): + """Default tool panel view for LazyToolBox.""" + + def __init__(self, toolbox: "LazyToolBox"): + self.toolbox = toolbox + + def apply_view(self, base_tool_panel, toolbox_registry): + return self.toolbox._tool_panel + + def to_model(self) -> ToolPanelViewModel: + return ToolPanelViewModel( + id="default", + name="Full Tool Panel", + description="Galaxy's fully configured toolbox panel.", + model_class="DefaultToolPanelView", + view_type=ToolPanelViewModelType.default_type, + searchable=True, + ) + + +class LazyToolBox(ToolBox): + """ + ToolBox that loads tools on-demand from the tool source store. + + Extends ToolBox but overrides initialization to avoid loading all tools + at startup. Keeps a lightweight index in memory for API responses, + but only loads full Tool objects when needed for execution or form building. + """ + + def __init__( + self, + config_filenames: list[str], + tool_root_dir: str, + app: "UniverseApplication", + tool_source_store: Optional[ToolSourceStore], + cache_size: int = 500, + save_integrated_tool_panel: bool = True, + ) -> None: + """ + Initialize the lazy toolbox. + + Args: + config_filenames: Tool configuration files (used for panel structure). + tool_root_dir: Root directory for tools. + app: Galaxy application instance. + tool_source_store: The tool source store to load from. + cache_size: Maximum number of Tool objects to cache in memory. + save_integrated_tool_panel: Whether to save integrated tool panel. + """ + # Store references before any initialization + self._store = tool_source_store + self._tool_object_cache: LRUCache = LRUCache(maxsize=cache_size) + self._cache_lock = threading.RLock() + self._reload_count = 0 + + # Initialize core attributes that AbstractToolBox.__init__ would set + # We do this manually to avoid loading all tools + self._init_lazy_toolbox( + config_filenames=config_filenames, + tool_root_dir=tool_root_dir, + app=app, + save_integrated_tool_panel=save_integrated_tool_panel, + ) + + # Load tool index from store + self._tool_index: Optional[ToolIndex] = None + self._load_index_from_store() + + # Populate _tools_by_id with stub entries from index + # This allows has_tool() and similar checks to work without loading + self._populate_tool_registry_from_index() + + log.info(f"LazyToolBox initialized with {len(self._tools_by_id)} tools (cache_size={cache_size})") + + def _init_lazy_toolbox( + self, + config_filenames: list[str], + tool_root_dir: str, + app: "UniverseApplication", + save_integrated_tool_panel: bool, + ) -> None: + """ + Initialize toolbox attributes without loading tools. + + This replicates the essential parts of AbstractToolBox.__init__ + without calling _init_tools_from_configs which loads all tools. + """ + # From ToolBox.__init__ — imported lazily to avoid circular import via + # galaxy.tools -> galaxy.tool_util.fetcher -> galaxy.tools. + from galaxy.tool_util.fetcher import ToolLocationFetcher + + self.tool_location_fetcher = ToolLocationFetcher() + self._tools_loaded_from_store = 0 + self._tools_parsed_from_file = 0 + + # From AbstractToolBox.__init__ + self._dynamic_tool_confs: list[DynamicToolConfDict] = [] + self._tools_by_id: dict[str, Tool] = {} + self._tools_by_uuid: dict[UUID, Tool] = {} + self._tool_versions_by_id: dict[str, dict[Union[str, None], Tool]] = {} + self._tools_by_old_id: dict[str, list[Tool]] = {} + self._workflows_by_id: dict[str, Any] = {} + self._tool_to_dict_cache: dict[str, dict[str, Any]] = {} + self._tool_to_dict_cache_admin: dict[str, dict[str, Any]] = {} + self._tool_panel = ToolPanelElements() + self._index = 0 + self.data_manager_tools: dict[str, Tool] = {} + self._lineage_map = LineageMap(app) + + # Tool root dir handling from ToolBox + if tool_root_dir == "./tools": + tool_root_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), "bundled")) + self._tool_root_dir = tool_root_dir + self.app = app + + # Initialize integrated tool panel (from ManagesIntegratedToolPanelMixin) + self._init_integrated_tool_panel(app.config) + + # Watchers and filters + self._tool_watcher = self.app.watchers.tool_watcher + self._tool_config_watcher = self.app.watchers.tool_config_watcher + self._filter_factory = FilterFactory(self) + self._tool_tag_manager = self.tool_tag_manager() + + # Initialize panel views + view_sources = StaticToolBoxViewSources( + view_directories=app.config.panel_views_dir, + view_dicts=app.config.panel_views, + ) + # Store default panel view - we override _default_panel_view() method to use this + self._default_panel_view_name = app.config.default_panel_view + self._setup_panel_views(view_sources) + + # Initialize dependency manager + self._init_dependency_manager() + + # Load panel structure from config files (sections, labels) + # but don't load the actual tools + self._init_panel_structure_from_configs(config_filenames) + + # Load tool panel views (required for panel_has_tool checks) + if self.app.name == "galaxy": + self._load_tool_panel_views() + + if save_integrated_tool_panel: + self._save_integrated_tool_panel() + + def _default_panel_view(self, trans): + """ + Override AbstractToolBox._default_panel_view to avoid name-mangled attribute access. + + Returns the default panel view for the given transaction, respecting + per-host configuration if available. + """ + config = self.app.config + if hasattr(config, "config_value_for_host"): + config_value = config.config_value_for_host("default_panel_view", trans.host) + else: + config_value = getattr(config, "default_panel_view", None) + return config_value or self._default_panel_view_name + + def _setup_panel_views(self, view_sources) -> None: + """Set up tool panel views.""" + tool_panel_views_list: list[ToolPanelView] = [DefaultToolPanelView(self)] + + for edam_view in listify(self.app.config.edam_panel_views): + mode = EdamPanelMode[edam_view] + tool_panel_views_list.append(EdamToolPanelView(self.app.datatypes_registry.edam, mode=mode)) + + if view_sources is not None: + # Lazy import: only needed when there are static panel views to register. + from galaxy.tool_util.toolbox.views.static import StaticToolPanelView + + for definition in view_sources.get_definitions(): + tool_panel_views_list.append(StaticToolPanelView(definition)) + + self._tool_panel_views = {} + for tool_panel_view in tool_panel_views_list: + self._tool_panel_views[tool_panel_view.to_model().id] = tool_panel_view + + self._tool_panel_view_rendered: dict[str, ToolPanelElements] = {} + + def _init_panel_structure_from_configs(self, config_filenames: list[str]) -> None: + """ + Load panel structure (sections, labels) from config files. + + This parses the tool configs to get the panel layout and builds + a mapping of tool_id -> section info for use with the index. + """ + # Lazy import: only the panel-loading code path needs this parser. + from galaxy.tool_util.toolbox.parser import get_toolbox_parser + + # Map tool_id -> (section_id, section_name) + self._tool_section_map: dict[str, tuple] = {} + + config_filenames = listify(config_filenames) + + for config_filename in config_filenames: + if not self.can_load_config_file(config_filename): + continue + try: + tool_conf_source = get_toolbox_parser(config_filename) + tool_path = tool_conf_source.parse_tool_path() + if not tool_path: + tool_path = self._tool_root_dir + else: + tool_conf_dir = os.path.dirname(config_filename) + tool_path_vars = {"tool_conf_dir": tool_conf_dir} + tool_path = string.Template(tool_path).safe_substitute(tool_path_vars) + + parsing_shed_tool_conf = tool_conf_source.is_shed_tool_conf() + + for item in tool_conf_source.parse_items(): + try: + item_type = getattr(item, "type", None) + if item_type == "section": + section_id = item.get("id") + section_name = item.get("name", section_id) + section_dict = { + "id": section_id, + "name": section_name, + "version": item.get("version", ""), + } + if section_id and section_id not in self._tool_panel: + section = ToolSection(section_dict) + self._tool_panel.append_section(section_id, section) + + # Extract tools in this section + if section_id: + self._extract_tools_from_section(item, section_id, section_name, tool_path) + + elif item_type == "label": + label_id = item.get("id") + label_text = item.get("text", "") + if label_id and label_id not in self._tool_panel: + # Lazy import: only needed when a label element is encountered. + from galaxy.tool_util.toolbox.panel import ToolSectionLabel + + label = ToolSectionLabel({"id": label_id, "text": label_text}) + self._tool_panel[f"label_{label_id}"] = label + + elif item_type == "tool": + # Tool at root level (no section). sample_tool_conf.xml + # ships some tools both inside a ``
`` and at + # root level; we don't want the root-level pass to + # clobber the section assignment we recorded a few + # iterations earlier. Only register if we haven't + # already seen this tool with a non-empty section. + tool_id = self._extract_tool_id_from_item(item, tool_path) + if tool_id: + existing = self._tool_section_map.get(tool_id) + if existing is None or existing[0] is None: + self._tool_section_map[tool_id] = (None, None) + except Exception as e: + log.debug(f"Error processing item in {config_filename}: {e}") + + if parsing_shed_tool_conf: + if os.access(config_filename, os.W_OK): + shed_tool_conf_dict = dict( + config_filename=config_filename, + tool_path=tool_path, + config_elems=[], + ) + self._dynamic_tool_confs.append(shed_tool_conf_dict) + + except FileNotFoundError: + log.debug(f"Tool config file not found: {config_filename}") + except Exception as e: + log.warning(f"Error parsing tool config {config_filename}: {e}") + + log.info(f"Built tool section map with {len(self._tool_section_map)} entries") + # Log some sample entries for debugging + sample_entries = list(self._tool_section_map.items())[:5] + for tool_id, (section_id, _section_name) in sample_entries: + log.debug(f" Section map sample: {tool_id} -> {section_id}") + + def _extract_tools_from_section(self, section_item, section_id: str, section_name: str, tool_path: str) -> None: + """Extract tool IDs from a section and add to section map.""" + if not hasattr(section_item, "items"): + return + + for sub_item in section_item.items: + try: + item_type = getattr(sub_item, "type", None) + if item_type == "tool": + tool_id = self._extract_tool_id_from_item(sub_item, tool_path) + if tool_id: + self._tool_section_map[tool_id] = (section_id, section_name) + except Exception as e: + log.debug(f"Error extracting tool from section {section_id}: {e}") + + def _extract_tool_id_from_item(self, item, tool_path: str) -> Optional[str]: + """Extract tool ID from a tool item - either from guid or by parsing the file.""" + # For shed tools, use the guid directly + guid = item.get("guid") + if guid: + return guid + + # For regular tools, we need to get the ID from the file attribute + # and optionally parse the tool XML to get the actual ID + tool_file = item.get("file") + if not tool_file: + return None + + # Try to extract tool ID from file + tool_path_full = os.path.join(tool_path, tool_file) + tool_id = extract_tool_id_from_file(tool_path_full, max_read=2000) + if tool_id: + return tool_id + + # Fall back to using filename without extension as ID hint + return os.path.splitext(os.path.basename(tool_file))[0] + + def _load_index_from_store(self) -> None: + """Load the tool index from store.""" + log.debug("Loading tool index from store...") + if self._store is None: + log.info("No tool source store configured") + self._tool_index = ToolIndex() + return + self._tool_index = self._store.load_index() + + if self._tool_index is None or len(self._tool_index.entries) == 0: + # Check if store has tools but index is missing/empty + stored_hashes = list(self._store.list_all()) + if stored_hashes: + log.info(f"Index empty but store has {len(stored_hashes)} tools - rebuilding index...") + self._rebuild_index_from_store(stored_hashes) + else: + # Empty store + empty index — first lazy boot. Walk every + # configured tool conf and populate the default store so the + # toolbox is non-empty without requiring an explicit + # populate_store.py run. + log.info("Empty tool source store; bootstrapping from tool configs...") + self._bootstrap_store_from_configs() + else: + log.info(f"Loaded tool index with {len(self._tool_index.entries)} entries") + + def _bootstrap_store_from_configs(self) -> None: + """One-shot populate: parse every tool conf, write canonical sources, build index. + + Runs only when the active store is empty. Idempotent — if a hash + already exists in the store the per-source `store()` call is a + no-op. Composite layering means writes land on the writable + default member; read-only members (CVMFS bundles) are skipped by + the composite itself. + """ + # Lazy imports — only the bootstrap path needs these. + import sys as _sys + from pathlib import Path as _Path + + from galaxy.tool_source_store import StoredToolSource as _StoredToolSource + from galaxy.util import xml_to_string + + if self._store is None: + self._tool_index = ToolIndex() + return + + # scripts/tool_source/_discover.py is a script-local helper; expose it on + # sys.path the same way scripts/tool_source/populate_store.py does. + scripts_dir = _Path(__file__).resolve().parents[3] / "scripts" / "tool_source" + if str(scripts_dir) not in _sys.path: + _sys.path.insert(0, str(scripts_dir)) + try: + from _discover import discover_tools + except ImportError as e: + log.warning(f"Could not import discover_tools for bootstrap: {e}") + self._tool_index = ToolIndex() + return + + config = self.app.config + entries: dict[str, ToolIndexEntry] = {} + stored_count = 0 + + for discovered in discover_tools(config, include_bundled=True): + tool_path = discovered.path + try: + tool_source = get_tool_source(config_file=tool_path) + root = tool_source.xml_tree.getroot() # type: ignore[attr-defined] + expanded_content = xml_to_string(root, pretty=True) + except Exception as e: + log.debug(f"Bootstrap skipping {tool_path}: {e}") + continue + content_hash = hashlib.sha256(expanded_content.encode("utf-8")).hexdigest() + tool_id = tool_source.parse_id() or discovered.guid + if not tool_id: + continue + stored = _StoredToolSource( + hash=content_hash, + tool_source_class=type(tool_source).__name__, + raw_source=expanded_content, + tool_id=tool_id, + tool_version=tool_source.parse_version(), + tool_dir=str(_Path(tool_path).parent), + stored_at=datetime.utcnow(), + ) + try: + self._store.store(stored) + stored_count += 1 + except Exception as e: + log.debug(f"Bootstrap could not store {tool_path}: {e}") + continue + entry = self._build_index_entry_from_stored(stored) + if entry and entry.id: + entries[entry.id] = entry + + self._tool_index = ToolIndex( + entries=entries, + by_section={}, + version=hashlib.md5(str(sorted(entries.keys())).encode()).hexdigest()[:8], + built_at=datetime.utcnow(), + ) + try: + self._store.store_index(self._tool_index) + except Exception as e: + log.warning(f"Bootstrap could not persist index: {e}") + log.info(f"Bootstrap complete: stored {stored_count} sources, index has {len(entries)} entries") + + def _rebuild_index_from_store(self, stored_hashes: list[str]) -> None: + """Rebuild the index from stored tool sources.""" + assert self._store is not None + entries: dict[str, ToolIndexEntry] = {} + + for source_hash in stored_hashes: + stored = self._store.get(source_hash) + if stored: + try: + entry = self._build_index_entry_from_stored(stored) + if entry and entry.id: + entries[entry.id] = entry + except Exception as e: + log.warning(f"Error building index entry for {source_hash}: {e}") + + self._tool_index = ToolIndex( + entries=entries, + by_section={}, + version=hashlib.md5(str(sorted(entries.keys())).encode()).hexdigest()[:8], + built_at=datetime.utcnow(), + ) + + # Save the rebuilt index + try: + self._store.store_index(self._tool_index) + log.info(f"Rebuilt and saved tool index with {len(entries)} entries") + except Exception as e: + log.warning(f"Could not save rebuilt index: {e}") + + def _build_index_entry_from_stored(self, stored: StoredToolSource) -> Optional[ToolIndexEntry]: + """Build an index entry from a stored tool source.""" + try: + tool_source = get_tool_source( + raw_tool_source=stored.raw_source, + tool_source_class=stored.tool_source_class, + ) + + tool_id = tool_source.parse_id() or stored.tool_id + if not tool_id: + return None + + # Safely get optional attributes + uuid_val = None + if hasattr(tool_source, "parse_uuid"): + try: + parsed_uuid = tool_source.parse_uuid() + uuid_val = str(parsed_uuid) if parsed_uuid else None + except Exception: + pass + + hidden = False + if hasattr(tool_source, "parse_hidden"): + try: + hidden = tool_source.parse_hidden() + except Exception: + pass + + return ToolIndexEntry( + id=tool_id, + uuid=uuid_val, + version=tool_source.parse_version(), + name=tool_source.parse_name() or "", + description=tool_source.parse_description() or "", + source_hash=stored.hash, + source_class=stored.tool_source_class, + hidden=hidden, + indexed_at=datetime.utcnow(), + ) + except Exception as e: + log.debug(f"Error parsing tool source for index: {e}") + return None + + def load_item( + self, + item, + tool_path, + panel_dict=None, + integrated_panel_dict=None, + load_panel_dict: bool = True, + guid=None, + index: Optional[int] = None, + ) -> None: + """Persist newly installed tools to the store + index without pinning a Tool object. + + Called at runtime by the shed-install path + (``tool_panel_manager.add_to_tool_panel`` → + ``self.app.toolbox.load_item(...)``). The eager ``ToolBox.load_item`` + materializes a fully parsed ``Tool`` and registers it in + ``_tools_by_id`` for the lifetime of the process — defeating the + whole point of the lazy path. We replace it for ``tool`` items + with the lazy-equivalent work; section/label/workflow/tool_dir + items still go through super since they only mutate panel + structure. + """ + from galaxy.tool_util.toolbox.parser import ensure_tool_conf_item + + item = ensure_tool_conf_item(item) + if getattr(item, "type", None) != "tool": + super().load_item( + item, + tool_path=tool_path, + panel_dict=panel_dict, + integrated_panel_dict=integrated_panel_dict, + load_panel_dict=load_panel_dict, + guid=guid, + index=index, + ) + return + + if self._store is None or self._tool_index is None: + # No lazy infra wired — fall back to eager load so the install still works. + super().load_item( + item, + tool_path=tool_path, + panel_dict=panel_dict, + integrated_panel_dict=integrated_panel_dict, + load_panel_dict=load_panel_dict, + guid=guid, + index=index, + ) + return + + with self.app._toolbox_lock: + self._lazy_register_tool_item(item, tool_path, guid=guid) + + def _lazy_register_tool_item(self, item, tool_path: str, guid: Optional[str] = None) -> None: + """Persist a single tool item's source to the store and add an index entry. + + Reused by the shed-install ``load_item`` override. Does *not* + instantiate a ``Tool`` object — the next ``get_tool`` call + lazy-loads it. + """ + from galaxy.tool_source_store import StoredToolSource as _StoredToolSource + from galaxy.util import xml_to_string + + assert self._store is not None + assert self._tool_index is not None + + tool_file = item.get("file") + if not tool_file: + log.debug("Lazy load_item skipped: tool item has no 'file' attribute") + return + tool_full_path = os.path.join(tool_path, tool_file) + try: + tool_source = get_tool_source(config_file=tool_full_path) + root = tool_source.xml_tree.getroot() # type: ignore[attr-defined] + expanded_content = xml_to_string(root, pretty=True) + except Exception as e: + log.warning(f"Lazy load_item could not parse {tool_full_path}: {e}") + return + + content_hash = hashlib.sha256(expanded_content.encode("utf-8")).hexdigest() + tool_id = tool_source.parse_id() or guid or item.get("guid") + if not tool_id: + log.debug(f"Lazy load_item: no tool_id resolvable for {tool_full_path}") + return + + stored = _StoredToolSource( + hash=content_hash, + tool_source_class=type(tool_source).__name__, + raw_source=expanded_content, + tool_id=tool_id, + tool_version=tool_source.parse_version(), + tool_dir=os.path.dirname(tool_full_path), + stored_at=datetime.utcnow(), + ) + try: + self._store.store(stored) + except Exception as e: + log.warning(f"Lazy load_item could not store {tool_full_path}: {e}") + return + + entry = self._build_index_entry_from_stored(stored) + if entry and entry.id: + self._tool_index.entries[entry.id] = entry + self._tool_index.invalidate_caches() + try: + self._store.store_index(self._tool_index) + except Exception as e: + log.warning(f"Lazy load_item could not persist index: {e}") + + # Fan out to peer Galaxy processes so they reload the index. + try: + from galaxy.queue_worker import send_control_task + + send_control_task(self.app, "reload_tool_source_cache") + except Exception as e: + log.debug(f"Lazy load_item could not broadcast reload_tool_source_cache: {e}") + + def _populate_tool_registry_from_index(self) -> None: + """ + Populate _tools_by_id with None placeholders from index. + + This allows has_tool() checks to work without loading Tool objects. + The actual Tool objects are loaded on-demand in get_tool(). + """ + if self._tool_index is None: + return + + # Update index entries with section info from tool_conf.xml + # Build reverse map: short_id -> section_info for faster lookup + if hasattr(self, "_tool_section_map"): + short_id_to_section: dict[str, tuple] = {} + for map_tool_id, mapped_section in self._tool_section_map.items(): + # Store exact ID + short_id_to_section[map_tool_id] = mapped_section + # For guids, also store the short tool ID + short_id = extract_short_id_from_guid(map_tool_id) + if short_id and short_id != map_tool_id and short_id not in short_id_to_section: + short_id_to_section[short_id] = mapped_section + else: + short_id_to_section = {} + + section_updates = 0 + section_info: Optional[tuple] + for tool_id, entry in self._tool_index.entries.items(): + section_info = None + + # Try exact match first + if tool_id in short_id_to_section: + section_info = short_id_to_section[tool_id] + + if section_info: + section_id, section_name = section_info + if section_id and not entry.panel_section_id: + entry.panel_section_id = section_id + entry.panel_section_name = section_name + section_updates += 1 + + # Store None as placeholder - actual Tool loaded on demand + self._tools_by_id[tool_id] = None # type: ignore[assignment] + + # Initialize version tracking + if tool_id not in self._tool_versions_by_id: + self._tool_versions_by_id[tool_id] = {} + if entry.version: + self._tool_versions_by_id[tool_id][entry.version] = None # type: ignore[assignment] + + # Add to panel if section info available + if entry.panel_section_id and entry.panel_section_id in self._tool_panel: + section = self._tool_panel[entry.panel_section_id] + if isinstance(section, ToolSection): + self._tool_panel.record_section_for_tool_id(tool_id, entry.panel_section_id, section.name or "") + + # Debug: check for mismatches + index_ids = set(self._tool_index.entries.keys()) if self._tool_index else set() + map_ids = set(self._tool_section_map.keys()) if hasattr(self, "_tool_section_map") else set() + matched = index_ids & map_ids + log.info( + f"Section map has {len(map_ids)} entries, index has {len(index_ids)} entries, {len(matched)} matched, {section_updates} updated" + ) + if map_ids and index_ids: + # Show sample IDs from each for comparison + log.info(f" Sample index IDs: {list(index_ids)[:3]}") + log.info(f" Sample map IDs: {list(map_ids)[:3]}") + + # === Override get_tool for lazy loading === + + @overload + def get_tool( + self, + tool_id: Optional[str] = None, + tool_version: Optional[str] = None, + tool_uuid: Optional[Union[UUID, str]] = None, + get_all_versions: Literal[False] = False, + exact: Optional[bool] = False, + user: Optional["User"] = None, + ) -> Optional["Tool"]: ... + + @overload + def get_tool( + self, + tool_id: Optional[str] = None, + tool_version: Optional[str] = None, + tool_uuid: Optional[Union[UUID, str]] = None, + get_all_versions: Literal[True] = True, + exact: Optional[bool] = False, + user: Optional["User"] = None, + ) -> list["Tool"]: ... + + def get_tool( + self, + tool_id: Optional[str] = None, + tool_version: Optional[str] = None, + tool_uuid: Optional[Union[UUID, str]] = None, + get_all_versions: Optional[bool] = False, + exact: Optional[bool] = False, + user: Optional["User"] = None, + ) -> Union[Optional["Tool"], list["Tool"]]: + """ + Get a tool, loading from store on-demand if needed. + + Overrides ToolBox.get_tool to implement lazy loading. + """ + # Lazy import: galaxy.exceptions pulls in webapp framework deps. + from galaxy.exceptions import ( + ObjectNotFound, + RequestParameterInvalidException, + ) + + if tool_id is None and tool_uuid is None: + raise RequestParameterInvalidException("get_tool cannot be called with both tool_id and tool_uuid as None") + + # Handle UUID lookup + if tool_uuid: + if user: + unprivileged_tool = self.get_unprivileged_tool_or_none(user, tool_uuid=tool_uuid) + if unprivileged_tool: + return unprivileged_tool + tool_uuid = tool_uuid if isinstance(tool_uuid, UUID) else UUID(tool_uuid) + tool_from_uuid = self._get_tool_by_uuid(tool_uuid) + if tool_from_uuid is None: + raise ObjectNotFound(f"Failed to find a tool with uuid [{tool_uuid}]") + tool_id = tool_from_uuid.id + + assert tool_id + + if tool_version: + tool_version = str(tool_version) + + if get_all_versions and exact: + raise RequestParameterInvalidException( + "get_tool cannot be called with both get_all_versions and exact as True" + ) + + # Check if we have this tool in our index + if self._tool_index and tool_id in self._tool_index.entries: + tool = self._load_tool_on_demand(tool_id, tool_version) + if tool: + if get_all_versions: + return [tool] # TODO: support multiple versions + return tool + + # Fall back to parent implementation for tools not in our index + # (dynamic tools, data manager tools, etc.) + if get_all_versions: + return super().get_tool( + tool_id=tool_id, + tool_version=tool_version, + tool_uuid=tool_uuid, + get_all_versions=True, + exact=exact, + user=user, + ) + return super().get_tool( + tool_id=tool_id, + tool_version=tool_version, + tool_uuid=tool_uuid, + get_all_versions=False, + exact=exact, + user=user, + ) + + def _load_tool_on_demand(self, tool_id: str, tool_version: Optional[str] = None) -> Optional["Tool"]: + """ + Load a tool from the store on-demand. + + Uses LRU cache to avoid reloading frequently used tools. + """ + cache_key = f"{tool_id}:{tool_version or 'latest'}" + + # Check cache first + with self._cache_lock: + if cache_key in self._tool_object_cache: + return self._tool_object_cache[cache_key] + + # Check if already loaded in _tools_by_id + existing = self._tools_by_id.get(tool_id) + if existing is not None: + with self._cache_lock: + self._tool_object_cache[cache_key] = existing + return existing + + # Get entry from index + if self._tool_index is None or self._store is None: + return None + + entry = self._tool_index.get(tool_id) + if not entry: + return None + + # Load source from store + stored = self._store.get(entry.source_hash) + if not stored: + log.warning(f"Tool source not found for {tool_id} (hash: {entry.source_hash})") + return None + + # Create Tool object + try: + tool = self._create_tool_from_stored_source(stored) + log.debug(f"Lazy-loaded tool: {tool_id}") + except Exception as e: + log.error(f"Error creating tool {tool_id}: {e}") + return None + + # Register the tool + self._register_loaded_tool(tool) + + # Add to cache + with self._cache_lock: + self._tool_object_cache[cache_key] = tool + + return tool + + def _create_tool_from_stored_source(self, stored: StoredToolSource) -> "Tool": + """Create a Tool object from stored source.""" + tool_source = get_tool_source( + raw_tool_source=stored.raw_source, + tool_source_class=stored.tool_source_class, + ) + return create_tool_from_source( + self.app, + tool_source, + tool_dir=stored.tool_dir, + ) + + def invalidate_index_cache(self) -> None: + """Drop cached tool index so the next read picks up out-of-band updates. + + Wired to the ``reload_tool_source_cache`` queue-worker control + message: a populator on another host (or another Galaxy process) + writes new sources/index to the shared store, then publishes + the message — every process calls this method to re-read the + index. The per-process LRU of materialized ``Tool`` objects + stays warm because content-addressed sources can't go stale + for a given hash. + """ + if self._store is not None: + try: + self._store.invalidate_index_cache() + except Exception as e: + log.debug(f"Store invalidate_index_cache raised: {e}") + self._tool_index = None + self._load_index_from_store() + + def _register_loaded_tool(self, tool: "Tool") -> None: + """Register a lazily-loaded tool in the toolbox registries.""" + tool_id = tool.id + if not tool_id: + return + + self._tools_by_id[tool_id] = tool + + version = tool.version + if tool_id not in self._tool_versions_by_id: + self._tool_versions_by_id[tool_id] = {} + self._tool_versions_by_id[tool_id][version] = tool + + # Tool uses 'guid' not 'uuid' + if hasattr(tool, "uuid") and tool.uuid: + self._tools_by_uuid[tool.uuid] = tool + + # Update lineage + self._lineage_map.register(tool) + + # === Override has_tool to check index === + + def has_tool( + self, + tool_id: Optional[str], + tool_version: Optional[str] = None, + tool_uuid: Optional[Union[UUID, str]] = None, + exact: bool = False, + user: Optional["User"] = None, + ) -> bool: + """Check if tool exists, using index for fast lookup.""" + if tool_id and self._tool_index and tool_id in self._tool_index.entries: + return True + # Fall back to parent for UUID lookups and edge cases + return super().has_tool( + tool_id=tool_id, + tool_version=tool_version, + tool_uuid=tool_uuid, + exact=exact, + user=user, + ) + + # === Override tools() to iterate loaded tools === + + def tools(self): + """ + Return loaded tools. + + Note: This only returns tools that have been loaded on-demand. + For a full list, use the index. + """ + return {k: v for k, v in self._tools_by_id.items() if v is not None}.items() + + # === Index access methods === + + @property + def tool_index(self) -> Optional[ToolIndex]: + """Get the tool index.""" + return self._tool_index + + def get_tool_ids(self) -> list[str]: + """Get all tool IDs from index.""" + if self._tool_index: + return list(self._tool_index.entries.keys()) + return [] + + def get_index_entry(self, tool_id: str) -> Optional[ToolIndexEntry]: + """Get index entry for a tool without loading it.""" + if self._tool_index: + return self._tool_index.get(tool_id) + return None + + # === Required property overrides === + + @property + def all_requirements(self): + """Get all tool requirements from index (no tool loading needed).""" + if self._tool_index: + requirements = set() + for entry in self._tool_index.entries.values(): + for req in entry.requirements: + # Convert dict to hashable tuple + req_tuple = (req.get("name"), req.get("version"), req.get("type")) + requirements.add(req_tuple) + return [ + {"name": r[0], "version": r[1], "type": r[2]} for r in requirements if r[0] # Filter out empty names + ] + return [] + + # === Override to_dict for API responses === + + def to_dict( + self, trans, in_panel: bool = True, tool_help: bool = False, view: Optional[str] = None, **kwds + ) -> list[dict[str, Any]]: + """ + Create a dictionary representation of the toolbox. + + For the *flat* listing (``in_panel=False``) we serve straight from + the index — no Tool loading needed. The panel listing + (``in_panel=True``) defers to the parent so the section-aware + response shape the UI expects is preserved. + + Note: tool_help is ignored since we don't load the full tool. + """ + if self._tool_index is None: + return [] + + if in_panel: + return super().to_dict(trans, in_panel=True, tool_help=tool_help, view=view, **kwds) + + rval = [] + + # Return data directly from index - no tool loading needed! + for _tool_id, entry in self._tool_index.entries.items(): + # Skip hidden tools unless requested + if entry.hidden and not kwds.get("include_hidden", False): + continue + + # Convert index entry to API dict format + tool_dict = self._index_entry_to_api_dict(entry) + rval.append(tool_dict) + + log.debug(f"LazyToolBox.to_dict: returning {len(rval)} tools from index (no loading)") + return rval + + def _index_entry_to_api_dict(self, entry: ToolIndexEntry) -> dict[str, Any]: + """Convert an index entry to the format expected by /api/tools.""" + return { + "id": entry.id, + "name": entry.name, + "version": entry.version, + "description": entry.description, + "labels": entry.labels if entry.labels else [], + "edam_operations": entry.edam_operations if entry.edam_operations else [], + "edam_topics": entry.edam_topics if entry.edam_topics else [], + "hidden": entry.hidden, + "model_class": "Tool", + "panel_section_id": entry.panel_section_id, + "panel_section_name": entry.panel_section_name, + # Minimal fields that indicate this is from index + "link": f"/api/tools/{entry.id}", + "min_width": -1, + "target": "galaxy_main", + } + + def to_panel_view(self, trans, view="default_panel_view", **kwds) -> dict[str, dict]: + """ + Create a panel view representation of the toolbox. + + For LazyToolBox, returns tools from index organized by section. + """ + if self._tool_index is None: + return {} + + view_contents: dict[str, dict] = {} + + # Group tools by section from index + sections: dict[str, dict[str, Any]] = {} + uncategorized_tools: list[dict[str, Any]] = [] + + for _tool_id, entry in self._tool_index.entries.items(): + if entry.hidden and not kwds.get("include_hidden", False): + continue + + tool_dict = self._index_entry_to_api_dict(entry) + + section_id = entry.panel_section_id + if section_id: + if section_id not in sections: + sections[section_id] = { + "id": section_id, + "name": entry.panel_section_name or section_id, + "model_class": "ToolSection", + "elems": [], + } + sections[section_id]["elems"].append(tool_dict) + else: + uncategorized_tools.append(tool_dict) + + # Add sections to view_contents + for section_id, section_dict in sections.items(): + view_contents[section_id] = section_dict + + # Add uncategorized tools directly + for tool_dict in uncategorized_tools: + view_contents[tool_dict["id"]] = tool_dict + + return view_contents