From f9f078b2df2730fd95f1aad96a65aebb7ab4eb09 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 6 Jul 2026 21:00:06 +0200 Subject: [PATCH] Align store layer with #23067 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adopt #23067's store infrastructure wholesale where it is canonical, keeping only genuine lazy-toolbox additions as deltas on top: - Store package: adopt #23067's factory.py/interface.py split and facade __init__.py, and its URL-only SqlAlchemyToolSourceStore. Re-apply lazy-only deltas — ToolIndexEntry panel-contract fields (icon/xrefs/model_class/ form_style/is_workflow_compatible/source_path) and data_manager_id; composite per-version index merge; scored multi-store whoosh search (search_scored + tool_tags field); populator panel-contract derivation via expand_ontology_data + biotools and data_manager_id stamping; data-manager / converter discovery in discover.py; benchmarks.py. - Config: replace tool_source_store + tool_source_disk_path with the single SQLAlchemy URI tool_source_database_connection (defaulted in config/__init__.py, validated via try_parsing, schema attr added), adopt #23067's tool_source_stores wording, and keep the branch-only use_lazy_toolbox / lazy_toolbox_cache_size options. Regenerated galaxy.yml.sample, galaxy_options.rst, and the schema-type stub. galaxy_mock uses tool_source_database_connection. - Docs: adopt #23067's tool_source_storage.rst (admin + dev) as the base and re-add the lazy sections (LazyToolBox, batch-endpoint integration, materialisation-count guard, LazyToolboxSearch multi-store search, benchmarks). - Tests: adopt #23067's store + scripts unit tests; re-add the ours-only composite entries_by_version merge test, data-manager discovery / build-index tests, and multi-store search test, all on the URI config. Claude-Session: https://claude.ai/code/session_018L7ZmCv2ubKA3JNeSL8Pkr --- doc/source/admin/galaxy_options.rst | 26 ++ doc/source/admin/tool_source_storage.rst | 66 ++-- doc/source/dev/tool_source_storage.rst | 127 ++++--- lib/galaxy/app_unittest_utils/galaxy_mock.py | 3 +- .../_galaxy_config_schema_attributes.py | 2 + lib/galaxy/config/sample/galaxy.yml.sample | 12 + lib/galaxy/tools/source_store/__init__.py | 354 +----------------- lib/galaxy/tools/source_store/composite.py | 23 +- lib/galaxy/tools/source_store/factory.py | 6 +- test/integration/test_tool_source_storage.py | 9 +- test/unit/app/tools/source_store/conftest.py | 2 +- .../source_store/test_composite_store.py | 40 +- .../tools/source_store/test_sqlite_store.py | 24 +- 13 files changed, 215 insertions(+), 479 deletions(-) diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 60b120a5d50..b48a4514921 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -459,6 +459,32 @@ :Type: map +~~~~~~~~~~~~~~~~~~~~ +``use_lazy_toolbox`` +~~~~~~~~~~~~~~~~~~~~ + +:Description: + When true, use the LazyToolBox which loads tools on demand from + the tool source store. Otherwise (the default), the traditional + eager ToolBox is used and any per-conf ``store="..."`` attributes + on tool_conf files are ignored. Opt-in is explicit: a populated + tool source store does not flip a default deployment to lazy mode. +:Default: ``None`` +:Type: bool + + +~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``lazy_toolbox_cache_size`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Maximum number of fully constructed Tool objects the LazyToolBox + keeps in its in-memory LRU cache. Larger values reduce repeat + parsing cost for popular tools at the expense of memory. +:Default: ``500`` +:Type: int + + ~~~~~~~~~~~~~~~~~~~~~~~ ``tool_dependency_dir`` ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/admin/tool_source_storage.rst b/doc/source/admin/tool_source_storage.rst index 58421576715..5bec7774ad9 100644 --- a/doc/source/admin/tool_source_storage.rst +++ b/doc/source/admin/tool_source_storage.rst @@ -1,8 +1,9 @@ Tool Source Storage =================== -Galaxy can cache parsed tool sources to improve startup time and reduce memory usage -when serving batch API endpoints. This is especially useful for large Galaxy installations +Galaxy can pre-parse and store tool sources, plus a lightweight index, in a +configurable backend, and load full ``Tool`` objects on demand through the +``LazyToolBox``. This is especially useful for large Galaxy installations with thousands of tools. Overview @@ -18,38 +19,37 @@ The tool source storage system addresses these issues by: 1. Pre-parsing and storing tool sources in a configurable backend 2. Maintaining a lightweight index in memory for fast API responses -3. Loading full Tool objects on-demand with LRU caching +3. Loading full ``Tool`` objects on demand with LRU caching Configuration ------------- Tool source storage is configured in ``galaxy.yml``. The following options are available: -Backend Selection -^^^^^^^^^^^^^^^^^ +Default Store +^^^^^^^^^^^^^ .. code-block:: yaml galaxy: - # Backend for storing tool sources ('sqlite', alias 'sqlalchemy') - tool_source_store: sqlite + # SQLAlchemy URI for storing tool sources. + tool_source_database_connection: sqlite:////srv/galaxy/tool_sources.sqlite -The store lives in a standalone database — a SQLite file under -``tool_source_disk_path`` (default: ``/tool_sources``) — never in -Galaxy's own database. It is a rebuildable cache: deleting it costs one -populator run. Nothing is initialized unless ``use_lazy_toolbox`` is -enabled. +The store lives in a standalone database - a SQLite file under +``/tool_sources.sqlite`` by default - separate from Galaxy's main +database. It is a rebuildable cache: deleting it costs one populator run. +This URI is used by tool source storage code paths, including the population +script and lazy toolbox consumers. Runtime use also requires a populated store +and a toolbox consumer configured to read from tool source storage. .. code-block:: yaml galaxy: - tool_source_store: sqlite - tool_source_disk_path: /path/to/tool_sources.sqlite + tool_source_database_connection: postgresql://galaxy@db.example.org/tool_sources Multi-host deployments must point every Galaxy process (web workers *and* -job handlers) at the same store: either a ``tool_source_disk_path`` on a -shared filesystem, or a named read-only store with a full SQLAlchemy -``url`` layered via ``tool_source_stores`` (see below). +job handlers) at the same store, such as a SQLite file on a shared filesystem +or a shared database URI. Toolbox Selection ^^^^^^^^^^^^^^^^^ @@ -65,7 +65,7 @@ The LazyToolBox is opt-in: leave ``use_lazy_toolbox`` unset (or false) and Galaxy uses the traditional eager ToolBox even when the store is populated or when a tool_conf carries a ``store="..."`` attribute. Set ``use_lazy_toolbox: true`` to activate lazy loading and per-conf store -routing. +routing. The store is only initialized when the LazyToolBox is enabled. Per-conf Store Routing (CVMFS Recipe) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -76,23 +76,20 @@ read-only SQLite bundle on CVMFS alongside a tool_conf, so worker processes can resolve every tool in that conf with local-cached lookups instead of one network round-trip per JSON file. -Declare the named stores under the new top-level ``tool_source_stores`` -key in ``galaxy.yml``. The ``sqlalchemy`` backend takes either a SQLAlchemy -``url`` or a ``path`` shortcut that builds a SQLite URL. SQLite is the -typical choice for CVMFS bundles (single self-contained file), but any -SQLAlchemy-supported database works: +Declare the named stores under the top-level ``tool_source_stores`` key in +``galaxy.yml``. Each entry takes a SQLAlchemy ``url`` and optional +``read_only`` flag. SQLite is the typical choice for CVMFS bundles (single +self-contained file), but any SQLAlchemy-supported database works: .. code-block:: yaml galaxy: - tool_source_store: sqlite # the writable default + tool_source_database_connection: sqlite:////srv/galaxy/tool_sources.sqlite tool_source_stores: cvmfs_main: - backend: sqlalchemy - path: /cvmfs/example.org/tools/sources.sqlite + url: sqlite:///file:/cvmfs/example.org/tools/sources.sqlite?mode=ro&uri=true read_only: true site_shared: - backend: sqlalchemy url: postgresql://galaxy_ro@db.example.org/tool_sources read_only: true @@ -128,7 +125,9 @@ it, ``populate_store.py`` populates **every writable store** referenced from a tool_conf in the same run. Once the bundle is in place on CVMFS (or any read-only mount), restart -Galaxy. +Galaxy. The ``read_only: true`` flag prevents Galaxy from writing through that +store. For SQLite connection-level read-only, use ``mode=ro&uri=true`` in the +SQLite URI as shown above. Cache Configuration ^^^^^^^^^^^^^^^^^^^ @@ -282,10 +281,10 @@ High memory usage 1. Reduce ``lazy_toolbox_cache_size`` to cache fewer Tool objects 2. Ensure ``use_lazy_toolbox: true`` is set in ``galaxy.yml`` -Migration from Traditional Toolbox ----------------------------------- +Populating an existing installation +----------------------------------- -To migrate an existing Galaxy installation to use tool source storage: +To set up tool source storage on an existing Galaxy installation: 1. Add the configuration to ``galaxy.yml``: @@ -302,5 +301,6 @@ To migrate an existing Galaxy installation to use tool source storage: 3. Restart Galaxy -The traditional toolbox will continue to work as a fallback if the tool source -store is not populated or if a specific tool is not found in the store. +If the store is not populated, or a specific tool is not found in it, the +LazyToolBox self-heals by populating the missing entries in-process, so the +traditional toolbox behavior is preserved as a fallback. diff --git a/doc/source/dev/tool_source_storage.rst b/doc/source/dev/tool_source_storage.rst index d81a79a8875..313c8e4b1e1 100644 --- a/doc/source/dev/tool_source_storage.rst +++ b/doc/source/dev/tool_source_storage.rst @@ -28,24 +28,25 @@ Module Layout :: lib/galaxy/tools/source_store/ - __init__.py ToolSourceStore ABC, StoredToolSource, build_tool_source_store() - sqlalchemy.py SqlAlchemyToolSourceStore (any SA URL; sqlite shortcut) + __init__.py Facade re-exporting the interface + factory + interface.py ToolSourceStore ABC, StoredToolSource, exceptions + factory.py build_tool_source_store() / build_named_store() + sqlalchemy.py SqlAlchemyToolSourceStore (any SQLAlchemy URL) composite.py CompositeToolSourceStore (per-conf routing, merged index) index.py ToolIndex, ToolIndexEntry (the lightweight metadata) + search.py ToolWhooshIndex (Whoosh index built from a ToolIndex) + discover.py discover_tools() — conf walk without booting a ToolBox + populator.py Population + watch logic (parse, store, index, broadcast) models.py Pydantic models for stored payloads - discover.py Tool-file discovery (conf walk without a ToolBox) - populator.py Store/index population (standalone + in-process) - search.py Whoosh index writer + LazyToolboxSearch benchmarks.py Store/index micro-benchmarks lib/galaxy/tools/lazy_toolbox.py LazyToolBox (subclass of ToolBox), LazyTool - lib/galaxy/tool_util/toolbox/ - base.py (small hook to support lazy mode) + lib/galaxy/tools/search/__init__.py LazyToolboxSearch (queries every store's index) lib/galaxy/tool_util/id_util.py Cheap tool-ID extraction (regex, no XML parser) - lib/galaxy/webapps/galaxy/services/tools.py Batch endpoints (lazy-aware) + lib/galaxy/webapps/galaxy/services/tools.py Batch endpoints (lazy-aware) - scripts/tool_source/populate_store.py CLI entry point for the populator + scripts/tool_source/populate_store.py CLI entry point for the populator Data Model ---------- @@ -61,9 +62,10 @@ rebuildable cache and does not participate in Galaxy's migrations or session lifecycle. **ToolIndex** — a single dataclass containing one ``ToolIndexEntry`` per tool, -holding everything the batch APIs need (id, name, description, panel section, -labels, EDAM, requirements, container info, test counts, hidden/disabled, -shed metadata). The index is serialized and gzip-compressed as a blob. +holding everything the batch APIs and the lazy panel render need (id, name, +description, panel section, labels, EDAM, xrefs, icon, requirements, container +info, test counts, hidden/disabled, shed metadata, ``data_manager_id``). The +index is serialized and gzip-compressed as a blob. The schema is auto-created on first open; ``tool_index`` holds a single row per index version. @@ -71,7 +73,7 @@ row per index version. Backend Abstraction ------------------- -``ToolSourceStore`` (in ``tools/source_store/__init__.py``) is an ABC defining: +``ToolSourceStore`` (in ``tools/source_store/interface.py``) is an ABC defining: - ``store/get/exists/delete/list_all/get_by_tool_id/count`` — per-tool source operations, all keyed by content hash. @@ -79,13 +81,12 @@ Backend Abstraction - ``get_stats()`` — backend-specific stats (count, size, backend name). ``build_tool_source_store(config)`` is the only entry point used -by Galaxy. It inspects ``config.tool_source_store`` to pick the backend -(currently ``sqlalchemy``, alias ``sqlite``). The store is only built -when ``use_lazy_toolbox`` is enabled — default deployments never -initialize it. -``ConfigurationError`` is raised for unknown backends or missing required -settings; it is allowed to propagate up so misconfiguration fails fast at -startup. +by Galaxy. It builds the default store from +``config.tool_source_database_connection`` and uses the same SQLAlchemy-backed +store implementation for all configured URIs. The store is only built when +``use_lazy_toolbox`` is enabled — default deployments never initialize it. +``ConfigurationError`` is raised for missing required settings and is allowed +to propagate up so misconfiguration fails fast at startup. The ABC defines a ``read_only: bool`` class attribute (default ``False``). ``ReadOnlyStoreError`` is raised by mutating methods of stores that opted @@ -124,11 +125,12 @@ case. The ``sqlalchemy`` backend (``sqlalchemy.py``) was added to make this useful for CVMFS: a single self-contained ``.sqlite`` file, opened with its own SQLAlchemy ``MetaData`` (independent of ``galaxy.model``) so the -file is portable, and openable with ``mode=ro&uri=true`` for read-only -mounts. Despite the name, the backend is not sqlite-specific — pass any -SQLAlchemy ``url`` (Postgres, MySQL, …) instead of ``path``. Auto schema -creation runs on first open; on remote backends operators may prefer to -manage migrations explicitly. +file is portable, and openable with a SQLite URI such as +``sqlite:///file:/cvmfs/example.org/tools/sources.sqlite?mode=ro&uri=true`` +for read-only mounts. Despite the name, the backend is not sqlite-specific - +pass any SQLAlchemy URL (Postgres, MySQL, ...). Auto schema creation runs on +first open; on remote backends operators may prefer to manage migrations +explicitly. Per-conf populator routing ^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -178,28 +180,37 @@ panel-structure discovery, where we just need the ID to map a file entry back to an index entry. Discovery -^^^^^^^^^ +--------- ``galaxy.tools.source_store.discover.discover_tools`` walks tool config files -and yields ``DiscoveredTool`` records. It is used by: +and yields ``DiscoveredTool`` records without booting a full ``ToolBox``. It is +used by: -- ``populate_store.py`` to find tools to parse and store. -- ``populate_store.py --watch`` to know which directories to monitor. -- (Indirectly) the LazyToolBox panel-structure code path. +- the populator to find tools to parse and store. +- watch mode to know which directories to monitor. +- callers that compare on-disk confs against the indexed tool set. +- (indirectly) the LazyToolBox panel-structure code path. -Pulling discovery out of ``ToolBox`` was deliberate: the population script -must run *without* a full app (or even a running Galaxy), and the watch -mode must run in a long-lived loop with no Galaxy process at all. +It also walks ``data_manager_conf``/``shed_data_manager_conf`` and the +datatype converters so data-manager and converter tools — loaded post-boot +outside any tool_conf — still land in the index. + +Pulling discovery out of ``ToolBox`` was deliberate: the populator must run +*without* a full app (or even a running Galaxy), and the watch mode must run in +a long-lived loop with no Galaxy process at all. Population Script ----------------- -``scripts/tool_source/populate_store.py`` runs out of process. It builds a -minimal app context (datatypes registry + SQLAlchemy model + config) and -calls ``build_tool_source_store`` with that context. Tools are parsed in a +``scripts/tool_source/populate_store.py`` is a thin CLI wrapper over +``galaxy.tools.source_store.populator.main``. It loads only the Galaxy +config and calls ``build_tool_source_store(config)`` — the standalone store +needs no datatypes registry or Galaxy model. Tools are parsed in a ``ThreadPoolExecutor`` (``--parallel``, default 4 workers); each tool is hashed and skipped if an entry with the same hash already exists -(``--incremental``, the default). +(``--incremental``, the default). Once the JSON index is committed the +populator rebuilds the Whoosh search index (``search.py``) so ranked tool +search stays in sync with the stored sources. Watch mode (``--watch``) uses ``watchdog`` to monitor every directory yielded by ``discover_tools``. File events are debounced (default 2 s), the changed @@ -231,9 +242,8 @@ materialising tools: and ``get_tool_to_dict`` serves ``LazyTool`` stubs from their index entries. - ``search_tools`` queries the ``app.toolbox_search`` singleton - (``LazyToolboxSearch`` over the populator-owned whoosh index in lazy - mode); hits are resolved against registered stubs via - ``LazyToolBox.resolve_search_hit`` with a per-hit access check. + (``LazyToolboxSearch`` in lazy mode); hits are resolved against registered + stubs via ``LazyToolBox.resolve_search_hit`` with a per-hit access check. - ``get_tests_summary`` and ``get_all_requirements`` answer from ``ToolIndex`` entries when the toolbox is lazy, and iterate the toolbox otherwise. @@ -241,24 +251,29 @@ materialising tools: The integration suite pins this: ``_lazy_materialize_count`` (bumped in the single materialise chokepoint) must not move across any of these endpoints. +``LazyToolboxSearch`` (``tools/search/__init__.py``) queries the whoosh index +of *every* configured store — the default plus each named per-conf store — +via ``ToolWhooshIndex.search_scored``, then merges the per-store hit lists by +BM25 score. A tool served from a named store is therefore reachable through +``/api/tools?q=`` even though its source lives outside the default store. + App Wiring ---------- ``galaxy.app.UniverseApplication.__init__`` calls ``_init_tool_source_store`` early and registers the result as a singleton under ``ToolSourceStore``. The toolbox is then chosen based on -``_use_lazy_toolbox()`` (explicit config override, otherwise auto-detect). -The store is exposed as ``app.tool_source_store`` and is ``Optional`` only -to satisfy type checkers — in practice the build either succeeds or raises -``ConfigurationError``. +``use_lazy_toolbox``. The store is exposed as ``app.tool_source_store`` and is +``Optional`` only to satisfy type checkers — in practice the build either +succeeds or raises ``ConfigurationError``. Design Notes ------------ -**Why a separate index instead of always-querying-the-store?** Batch -endpoints need O(N) access to N entries; doing that against the database on -every request is a latency hit. Keeping the index in-process and only paying -for invalidation on reload is the better tradeoff. +**Why a separate index instead of always querying the store?** A consumer +needs O(N) access to N entries; doing that against the backing store on every +request is a latency hit. Keeping the index in-process and only paying for +invalidation on reload is the better tradeoff. **Why an out-of-process populator?** Parsing tools and computing macro expansions is expensive and shouldn't block worker startup. Keeping the @@ -278,11 +293,15 @@ effectively a no-op. Testing ------- -- Unit tests: ``test/unit/app/tools/source_store/test_stores.py`` exercises each - backend through the ``ToolSourceStore`` interface. +- Store unit tests: ``test/unit/app/tools/source_store/`` exercises each backend + through the ``ToolSourceStore`` interface (``test_stores.py``, + ``test_sqlite_store.py``, ``test_composite_store.py``, + ``test_index_versions.py``, ``test_multi_store_search.py``). +- Populator/discovery unit tests: ``test/unit/scripts/tool_source/`` + (``test_populate_store.py``, ``test_discover.py``, + ``test_build_index_entry.py``, ``test_whoosh_dir.py``). These use fakes + (not mocks) of ``ToolSourceStore`` so behavior is verified against the real + interface. - Integration tests: ``test/integration/test_tool_source_storage.py`` spins - up Galaxy with each backend and verifies end-to-end behavior. -- Populator tests: ``test/unit/scripts/tool_source/test_populate_store.py`` - uses fakes (not mocks) of ``ToolSourceStore`` so behavior is verified - against the real interface. + up Galaxy against the store and verifies end-to-end behavior. - Benchmarks: ``python -m galaxy.tools.source_store.benchmarks --iterations 100``. diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index 4322caa77c7..06f6f7f3373 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -291,8 +291,7 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.track_jobs_in_database = False self.amqp_internal_connection = None self.tool_configs = [] - self.tool_source_store = "sqlite" - self.tool_source_disk_path = os.path.join(self.data_dir, "tool_sources") + self.tool_source_database_connection = f"sqlite:///{os.path.join(self.data_dir, 'tool_sources.sqlite')}" self.tool_source_stores = None self.use_lazy_toolbox = False self.manage_dependency_relationships = False diff --git a/lib/galaxy/config/_galaxy_config_schema_attributes.py b/lib/galaxy/config/_galaxy_config_schema_attributes.py index 0aa51434645..110c3c9e662 100644 --- a/lib/galaxy/config/_galaxy_config_schema_attributes.py +++ b/lib/galaxy/config/_galaxy_config_schema_attributes.py @@ -39,6 +39,8 @@ class GalaxyAppConfigurationAttributes: tool_path: str tool_source_database_connection: str | None tool_source_stores: Any + use_lazy_toolbox: bool + lazy_toolbox_cache_size: int tool_dependency_dir: str | None dependency_resolvers_config_file: str conda_prefix: str | None diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 1dc1e79e9bf..87fd65a4459 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -613,6 +613,18 @@ galaxy: # https://docs.galaxyproject.org/en/master/admin/tool_source_storage.html #tool_source_stores: null + # When true, use the LazyToolBox which loads tools on demand from the + # tool source store. Otherwise (the default), the traditional eager + # ToolBox is used and any per-conf ``store="..."`` attributes on + # tool_conf files are ignored. Opt-in is explicit: a populated tool + # source store does not flip a default deployment to lazy mode. + #use_lazy_toolbox: false + + # Maximum number of fully constructed Tool objects the LazyToolBox + # keeps in its in-memory LRU cache. Larger values reduce repeat + # parsing cost for popular tools at the expense of memory. + #lazy_toolbox_cache_size: 500 + # Various dependency resolver configuration parameters will have # defaults set relative to this path, such as the default conda # prefix, default Galaxy packages path, legacy tool shed dependencies diff --git a/lib/galaxy/tools/source_store/__init__.py b/lib/galaxy/tools/source_store/__init__.py index b022280da1b..a83de8905a0 100644 --- a/lib/galaxy/tools/source_store/__init__.py +++ b/lib/galaxy/tools/source_store/__init__.py @@ -1,347 +1,29 @@ """ -Tool Source Store - Pluggable storage backends for Galaxy tool sources. +Tool Source Store - standalone storage for Galaxy tool sources. -This module provides a configurable, pluggable tool source storage system -that enables storing and retrieving tool sources from multiple backends -(currently ``database`` and ``sqlalchemy``). +Tool sources and their derived ``ToolIndex`` live in a standalone SQLAlchemy +database chosen by connection URL (``tool_source_database_connection``; +defaults to a ``sqlite:///`` file, but any SQLAlchemy URL such as +``postgresql://`` works just as well). There is a single store +implementation, ``SqlAlchemyToolSourceStore``; a tool_conf may point at a +named store declared in ``tool_source_stores``, and those are layered over +the default in a ``CompositeToolSourceStore``. """ -import logging -from abc import ( - ABC, - abstractmethod, +from .factory import ( + build_named_store, + build_tool_source_store, ) -from collections.abc import Iterator -from dataclasses import ( - dataclass, - field, -) -from datetime import datetime -from typing import ( - Optional, - TYPE_CHECKING, -) - -if TYPE_CHECKING: - from galaxy.config import GalaxyAppConfiguration - -log = logging.getLogger(__name__) - - -@dataclass -class StoredToolSource: - """Representation of a stored tool source.""" - - hash: str # Content hash (SHA256) - tool_source_class: str # XmlToolSource, YamlToolSource, etc. - raw_source: str # Serialized tool source string - tool_id: str | None = None # Tool ID if known - tool_version: str | None = None # Tool version if known - tool_dir: str | None = None # Original tool directory - source_path: str | None = None # Original file path (used as a lookup key) - stored_at: datetime | None = None - metadata: dict | None = field(default_factory=dict) - - -class ToolSourceStore(ABC): - """Abstract base class for tool source storage backends.""" - - # Backends that wrap a read-only target (e.g. CVMFS-resident sqlite) - # set this to ``True`` so the populator and reload paths can skip them - # cleanly instead of crashing on a write attempt. - read_only: bool = False - - @abstractmethod - def store(self, tool_source: StoredToolSource) -> str: - """ - Store a tool source. - - Args: - tool_source: The tool source to store. - - Returns: - The storage key (hash). - """ - - @abstractmethod - def get(self, hash: str) -> StoredToolSource | None: - """ - Retrieve a tool source by hash. - - Args: - hash: The content hash of the tool source. - - Returns: - The stored tool source, or None if not found. - """ - - @abstractmethod - def exists(self, hash: str) -> bool: - """ - Check if a tool source exists. - - Args: - hash: The content hash to check. - - Returns: - True if the tool source exists. - """ - - @abstractmethod - def delete(self, hash: str) -> bool: - """ - Delete a tool source by hash. - - Args: - hash: The content hash of the tool source to delete. - - Returns: - True if deleted, False if not found. - """ - - @abstractmethod - def list_all(self) -> Iterator[str]: - """ - List all stored tool source hashes. - - Yields: - Content hashes of all stored tool sources. - """ - - @abstractmethod - def get_by_tool_id(self, tool_id: str, version: str | None = None) -> list[StoredToolSource]: - """ - Get tool sources by tool ID and optional version. - - Args: - tool_id: The tool ID to search for. - version: Optional version filter. - - Returns: - List of matching tool sources. - """ - - @abstractmethod - def get_by_source_path(self, source_path: str) -> StoredToolSource | None: - """ - Get the stored tool source for a given on-disk file path. - - The populator records ``source_path`` for every stored entry so the - eager / lazy load paths can resolve a config file to the - already-parsed source without guessing through ``tool_id`` (which can - collide across directories or be macro-expanded after the regex shortcut). - - Args: - source_path: Absolute path of the original tool config file. - - Returns: - Matching stored source, or None if nothing was populated from that file. - """ - - @abstractmethod - def count(self) -> int: - """Return the total number of stored tool sources.""" - - def get_stats(self) -> dict: - """Return storage statistics.""" - return {"count": self.count()} - - # Index operations - - @abstractmethod - def store_index(self, index: "ToolIndex") -> None: - """ - Store the complete tool index. - - Args: - index: The tool index to store. - """ - - @abstractmethod - def load_index(self) -> Optional["ToolIndex"]: - """ - Load the tool index. - - Returns: - The tool index, or None if not found. - """ - - @abstractmethod - def update_index_entry(self, entry: "ToolIndexEntry") -> None: - """ - Update a single index entry. - - Args: - entry: The index entry to update. - """ - - def remove_index_entry(self, tool_id: str) -> None: - """Remove a tool's entry from the persisted index. - - Counterpart of :meth:`update_index_entry` for uninstalls: the lazy - toolbox pops the entry from its in-memory index, but the persisted - singleton would hand it right back on the next cache invalidation - unless the removal is written through. - """ - index = self.load_index() - if index is None: - return - removed = index.entries.pop(tool_id, None) - removed_versions = index.entries_by_version.pop(tool_id, None) - if removed is None and removed_versions is None: - return - for section_tool_ids in index.by_section.values(): - if tool_id in section_tool_ids: - section_tool_ids.remove(tool_id) - index.invalidate_caches() - self.store_index(index) - - def invalidate_index_cache(self) -> None: # noqa: B027 — intentional empty default - """Drop any in-memory cached index so the next load_index() reads fresh. - - Backends override this when they cache; the default is a no-op. - """ - - def close(self) -> None: # noqa: B027 — intentional empty default - """Release any state the store is holding. - - Wired into ``GalaxyUniverseApplication.haltables`` so a Python-side - ``app.shutdown()`` (e.g. the embedded ``IntegrationTestCase.restart()`` - path) clears references that would otherwise survive into the next - boot. Default is a no-op; backends holding an engine or cache - override, and the composite store propagates. - """ - - -class ConfigurationError(Exception): - """Raised when there's a configuration error.""" - - -class ReadOnlyStoreError(Exception): - """Raised when a write is attempted against a read-only tool source store.""" - - -def _build_default_store( - config: "GalaxyAppConfiguration", -) -> ToolSourceStore: - """Build the default store from top-level ``tool_source_*`` config.""" - backend = config.tool_source_store - - if backend in ("sqlalchemy", "sqlite"): - from .sqlalchemy import SqlAlchemyToolSourceStore - - path = config.tool_source_disk_path - if path: - return SqlAlchemyToolSourceStore(path=path, read_only=False) - raise ConfigurationError(f"{backend!r} backend requires tool_source_disk_path") - - raise ConfigurationError(f"Unknown tool source store backend: {backend}") - - -def build_named_store( - name: str, - spec: dict, -) -> ToolSourceStore: - """Build a single named store from a ``tool_source_stores`` entry. - - ``spec`` is the dict from galaxy.yml — a ``backend`` plus its options - plus an optional ``read_only`` flag. - """ - if not isinstance(spec, dict): - raise ConfigurationError(f"tool_source_stores[{name!r}] must be a mapping") - backend = spec.get("backend") - read_only = bool(spec.get("read_only", False)) - - if backend in ("sqlalchemy", "sqlite"): - from .sqlalchemy import SqlAlchemyToolSourceStore - - url = spec.get("url") - path = spec.get("path") - if not url and not path: - raise ConfigurationError(f"tool_source_stores[{name!r}] requires a 'url' or 'path'") - return SqlAlchemyToolSourceStore(url=url, path=path, read_only=read_only) - - raise ConfigurationError(f"tool_source_stores[{name!r}] has unknown backend {backend!r}") - - -def _collect_per_conf_store_names(config: "GalaxyAppConfiguration") -> set[str]: - """Walk configured tool_confs and collect referenced store names.""" - if not config.tool_configs: - return set() - # Lazy import: avoids pulling parser code into deploys that don't need it. - from galaxy.tool_util.toolbox.parser import get_toolbox_parser - - names: set[str] = set() - for path in config.tool_configs: - try: - parser = get_toolbox_parser(path) - except Exception as e: - log.debug(f"skipping tool conf {path}: {e}") - continue - store = parser.parse_store_name() - if store: - names.add(store) - return names - - -def build_tool_source_store( - config: "GalaxyAppConfiguration", -) -> ToolSourceStore: - """Build the active tool source store, composing per-conf overrides. - - Returns the default store directly when no tool_conf opts into a named - override (zero overhead for the common case). Otherwise wraps the - default plus each referenced named store in a - :class:`CompositeToolSourceStore`, with the default consulted last and - receiving all writes. - - Args: - config: The Galaxy application configuration. - """ - default_store = _build_default_store(config) - - # Per-conf store="..." attributes are only meaningful when the LazyToolBox - # is the active toolbox. Opting in is explicit: anything other than - # ``use_lazy_toolbox: true`` keeps the traditional ToolBox, in which case - # nothing would query the named store. Treat such attributes as no-ops - # rather than failing on a catalog mismatch or doing wasted I/O. - if not config.use_lazy_toolbox: - referenced = _collect_per_conf_store_names(config) - if referenced: - log.info( - "use_lazy_toolbox is not enabled; ignoring store=... attributes " - f"from tool_confs (referenced: {sorted(referenced)})" - ) - return default_store - - referenced = _collect_per_conf_store_names(config) - if not referenced: - return default_store - - catalog = config.tool_source_stores or {} - members: list[tuple[str, ToolSourceStore]] = [] - for name in referenced: - if name not in catalog: - raise ConfigurationError( - f"tool_conf references store {name!r} but no such entry exists in tool_source_stores" - ) - members.append((name, build_named_store(name, catalog[name]))) - - # Default is consulted last so per-conf overrides shadow it on hash collisions. - members.append(("__default__", default_store)) - - # Lazy import to avoid composite always being pulled in. - from .composite import CompositeToolSourceStore - - return CompositeToolSourceStore(members=members, default="__default__") - - -# Re-export key classes — placed after the abstract base above to avoid circular -# imports between this module and ``index.py``/``database.py``. -from .index import ( # noqa: E402 +from .index import ( ToolIndex, ToolIndexEntry, ) +from .interface import ( + ConfigurationError, + ReadOnlyStoreError, + StoredToolSource, + ToolSourceStore, +) __all__ = [ "StoredToolSource", diff --git a/lib/galaxy/tools/source_store/composite.py b/lib/galaxy/tools/source_store/composite.py index deb55f6813c..cb35a766505 100644 --- a/lib/galaxy/tools/source_store/composite.py +++ b/lib/galaxy/tools/source_store/composite.py @@ -7,7 +7,7 @@ a designated *default* store. Used to layer e.g. a CVMFS-resident read-only sqlite bundle on top of the local writable store. The composite is invisible to the rest of Galaxy: it implements the same -:class:`ToolSourceStore` interface, and ``LazyToolBox`` / the populator +:class:`ToolSourceStore` interface, and consumers / the populator keep working unchanged. """ @@ -17,14 +17,14 @@ from typing import ( Any, ) -from . import ( - StoredToolSource, - ToolSourceStore, -) from .index import ( ToolIndex, ToolIndexEntry, ) +from .interface import ( + StoredToolSource, + ToolSourceStore, +) log = logging.getLogger(__name__) @@ -158,7 +158,7 @@ class CompositeToolSourceStore(ToolSourceStore): try: idx = member.load_index() except Exception as e: - log.warning(f"Failed to load index from store {name!r}: {e}") + log.error(f"Failed to load index from store {name!r}: {e}") continue if idx is None: continue @@ -196,7 +196,6 @@ class CompositeToolSourceStore(ToolSourceStore): merged.built_at = idx.built_at if not any_loaded: return None - merged.version = merged.compute_version() return merged def invalidate_index_cache(self) -> None: @@ -231,18 +230,10 @@ class CompositeToolSourceStore(ToolSourceStore): verdict = None return verdict - def commit(self) -> None: - """Propagate commit() to every writable member store.""" - for _name, member in self._members: - try: - member.commit() - except Exception as e: - log.warning(f"Composite store commit failed for member '{_name}': {e}") - def close(self) -> None: """Propagate close() to every member store.""" for _name, member in self._members: try: member.close() except Exception as e: - log.warning(f"Composite store close failed for member '{_name}': {e}") + log.error(f"Composite store close failed for member '{_name}': {e}") diff --git a/lib/galaxy/tools/source_store/factory.py b/lib/galaxy/tools/source_store/factory.py index f2043577039..3786ee4454d 100644 --- a/lib/galaxy/tools/source_store/factory.py +++ b/lib/galaxy/tools/source_store/factory.py @@ -81,12 +81,14 @@ def build_named_store( def _collect_per_conf_store_names(config: "GalaxyAppConfiguration") -> set[str]: """Walk configured tool_confs and collect referenced store names.""" + if not config.tool_configs: + return set() names: set[str] = set() - for path in config.all_tool_config_files(): + for path in config.tool_configs: try: parser = get_toolbox_parser(path) except Exception as e: - log.error(f"skipping tool conf {path}: {e}") + log.debug(f"skipping tool conf {path}: {e}") continue store = parser.parse_store_name() if store: diff --git a/test/integration/test_tool_source_storage.py b/test/integration/test_tool_source_storage.py index e62dac1fe12..808a9b640ee 100644 --- a/test/integration/test_tool_source_storage.py +++ b/test/integration/test_tool_source_storage.py @@ -89,7 +89,7 @@ class TestCompositeToolSourceStorage(BaseToolSourceStorageIntegrationTestCase): from galaxy.tools.source_store.sqlalchemy import SqlAlchemyToolSourceStore - SqlAlchemyToolSourceStore(path=cls._sqlite_path).count() + SqlAlchemyToolSourceStore(url=f"sqlite:///{cls._sqlite_path}").count() cls._conf_path = os.path.join(cls._tmpdir, "extra_tool_conf.xml") with open(cls._conf_path, "w") as f: @@ -103,8 +103,7 @@ class TestCompositeToolSourceStorage(BaseToolSourceStorageIntegrationTestCase): config["tool_config_file"] = list(existing_confs) + [cls._conf_path] config["tool_source_stores"] = { "cvmfs_main": { - "backend": "sqlalchemy", - "path": cls._sqlite_path, + "url": f"sqlite:///file:{cls._sqlite_path}?mode=ro&uri=true", "read_only": True, } } @@ -180,7 +179,9 @@ class TestLazyToolBoxReload(BaseToolSourceStorageIntegrationTestCase): store = self._app.tool_source_store assert store is not None - foreign_store = SqlAlchemyToolSourceStore(path=self._app.config.tool_source_disk_path) + connection = self._app.config.tool_source_database_connection + assert connection is not None + foreign_store = SqlAlchemyToolSourceStore(url=connection) foreign_store.store_index(ToolIndex()) store.invalidate_index_cache() # Drop one store row so the reload takes the inline-repopulate path. diff --git a/test/unit/app/tools/source_store/conftest.py b/test/unit/app/tools/source_store/conftest.py index 7ddf91bc9db..c592f5395fb 100644 --- a/test/unit/app/tools/source_store/conftest.py +++ b/test/unit/app/tools/source_store/conftest.py @@ -1,4 +1,4 @@ -"""Shared fixtures for tool_source_store unit tests.""" +"""Shared fixtures for tool source store unit tests.""" import pytest diff --git a/test/unit/app/tools/source_store/test_composite_store.py b/test/unit/app/tools/source_store/test_composite_store.py index 663411473d4..5c6aeb9ef98 100644 --- a/test/unit/app/tools/source_store/test_composite_store.py +++ b/test/unit/app/tools/source_store/test_composite_store.py @@ -30,10 +30,14 @@ def _src(hash, tool_id="t", version="1"): ) +def _sqlite_url(path): + return f"sqlite:///{path}" + + def test_priority_order_first_hit_wins(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) # Same hash, different tool_id payloads, to prove which member answered. a.store(_src("dup", tool_id="from_a")) b.store(_src("dup", tool_id="from_b")) @@ -45,8 +49,8 @@ def test_priority_order_first_hit_wins(two_paths): def test_writes_go_to_default(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) composite = CompositeToolSourceStore(members=[("a", a), ("b", b)], default="b") composite.store(_src("h1")) assert b.exists("h1") @@ -55,17 +59,17 @@ def test_writes_go_to_default(two_paths): def test_default_must_not_be_read_only(two_paths): pa, pb = two_paths - rw = SqliteToolSourceStore(path=pa) + rw = SqliteToolSourceStore(url=_sqlite_url(pa)) rw.store(_src("seed")) # so the file exists - ro = SqliteToolSourceStore(path=pa, read_only=True) + ro = SqliteToolSourceStore(url=_sqlite_url(pa), read_only=True) with pytest.raises(ValueError): CompositeToolSourceStore(members=[("ro", ro), ("rw", rw)], default="ro") def test_list_all_dedupes_across_members(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) a.store(_src("h1")) a.store(_src("dup")) b.store(_src("dup")) @@ -77,8 +81,8 @@ def test_list_all_dedupes_across_members(two_paths): def test_load_index_merges_and_dedupes(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) a.store_index( ToolIndex( entries={ @@ -105,8 +109,8 @@ def test_load_index_merges_and_dedupes(two_paths): def test_load_index_returns_none_when_no_member_has_one(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) composite = CompositeToolSourceStore(members=[("a", a), ("b", b)], default="b") assert composite.load_index() is None @@ -119,8 +123,8 @@ def test_invalidate_fans_out(two_paths): # cache hides the new entry; with composite.invalidate_index_cache() # the next load surfaces it. pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) a.store_index(ToolIndex(entries={"x": ToolIndexEntry(id="x")})) b.store_index(ToolIndex(entries={"y": ToolIndexEntry(id="y")})) a.load_index() @@ -128,10 +132,10 @@ def test_invalidate_fans_out(two_paths): # Out-of-band update via a fresh handle so the existing instance's # cache stays primed with the old value. - SqliteToolSourceStore(path=pa).store_index( + SqliteToolSourceStore(url=_sqlite_url(pa)).store_index( ToolIndex(entries={"x": ToolIndexEntry(id="x"), "x2": ToolIndexEntry(id="x2")}) ) - SqliteToolSourceStore(path=pb).store_index( + SqliteToolSourceStore(url=_sqlite_url(pb)).store_index( ToolIndex(entries={"y": ToolIndexEntry(id="y"), "y2": ToolIndexEntry(id="y2")}) ) @@ -156,8 +160,8 @@ def test_invalidate_fans_out(two_paths): def test_load_index_merges_entries_by_version(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) idx_a = ToolIndex() idx_a.add_entry(ToolIndexEntry(id="multi", name="v1", version="1.0")) idx_a.add_entry(ToolIndexEntry(id="multi", name="v2", version="2.0")) diff --git a/test/unit/app/tools/source_store/test_sqlite_store.py b/test/unit/app/tools/source_store/test_sqlite_store.py index 8cbd26aafeb..fafbce03710 100644 --- a/test/unit/app/tools/source_store/test_sqlite_store.py +++ b/test/unit/app/tools/source_store/test_sqlite_store.py @@ -33,8 +33,12 @@ def _source(hash="h1", tool_id="t1", version="1.0"): ) +def _sqlite_url(path): + return f"sqlite:///{path}" + + def test_store_and_retrieve_round_trip(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) store.store(_source()) got = store.get("h1") assert got is not None @@ -45,7 +49,7 @@ def test_store_and_retrieve_round_trip(sqlite_path): def test_get_by_tool_id_filters_by_version(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) store.store(_source(hash="h1", tool_id="t1", version="1.0")) store.store(_source(hash="h2", tool_id="t1", version="2.0")) assert {s.tool_version for s in store.get_by_tool_id("t1")} == {"1.0", "2.0"} @@ -53,7 +57,7 @@ def test_get_by_tool_id_filters_by_version(sqlite_path): def test_delete_returns_false_for_missing(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) assert store.delete("nope") is False store.store(_source()) assert store.delete("h1") is True @@ -61,7 +65,7 @@ def test_delete_returns_false_for_missing(sqlite_path): def test_index_round_trip(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) idx = ToolIndex(entries={"t1": ToolIndexEntry(id="t1", name="T1")}) store.store_index(idx) store.invalidate_index_cache() @@ -72,11 +76,11 @@ def test_index_round_trip(sqlite_path): def test_read_only_refuses_writes(sqlite_path): - rw = SqliteToolSourceStore(path=sqlite_path) + rw = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) rw.store(_source()) rw.store_index(ToolIndex(entries={"t1": ToolIndexEntry(id="t1", name="T1")})) - ro = SqliteToolSourceStore(path=sqlite_path, read_only=True) + ro = SqliteToolSourceStore(url=_sqlite_url(sqlite_path), read_only=True) assert ro.read_only is True fetched = ro.get("h1") assert fetched is not None @@ -89,14 +93,8 @@ def test_read_only_refuses_writes(sqlite_path): ro.store_index(ToolIndex()) -def test_read_only_missing_file_raises(tmp_path): - missing = tmp_path / "nope.sqlite" - with pytest.raises(FileNotFoundError): - SqliteToolSourceStore(path=str(missing), read_only=True) - - def test_get_stats_reports_backend_and_url(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) stats = store.get_stats() assert stats["backend"] == "sqlalchemy" assert stats["url"].startswith("sqlite:///")