diff --git a/doc/source/admin/tool_source_storage.rst b/doc/source/admin/tool_source_storage.rst index cdda15b55c8..58421576715 100644 --- a/doc/source/admin/tool_source_storage.rst +++ b/doc/source/admin/tool_source_storage.rst @@ -31,28 +31,25 @@ Backend Selection .. code-block:: yaml galaxy: - # Backend for storing tool sources: 'database' or 'sqlalchemy' - tool_source_store: database + # Backend for storing tool sources ('sqlite', alias 'sqlalchemy') + tool_source_store: sqlite -**Database Backend** (default) - -Stores tool sources in the Galaxy database. Best for: - -- Single-server deployments -- Installations where tools don't change frequently -- Simplest setup (no additional infrastructure) - -**SQLAlchemy Backend** - -Stores tool sources in a separate SQLAlchemy-managed database (typically a -SQLite file). Useful for shipping read-only tool source bundles via per-conf -``tool_source_stores`` entries (see CVMFS recipe below). +The store lives in a standalone database — a SQLite file under +``tool_source_disk_path`` (default: ``/tool_sources``) — never in +Galaxy's own database. It is a rebuildable cache: deleting it costs one +populator run. Nothing is initialized unless ``use_lazy_toolbox`` is +enabled. .. code-block:: yaml galaxy: - tool_source_store: sqlalchemy - tool_source_disk_path: /path/to/tool_sources.sqlite # SQLite shortcut + tool_source_store: sqlite + tool_source_disk_path: /path/to/tool_sources.sqlite + +Multi-host deployments must point every Galaxy process (web workers *and* +job handlers) at the same store: either a ``tool_source_disk_path`` on a +shared filesystem, or a named read-only store with a full SQLAlchemy +``url`` layered via ``tool_source_stores`` (see below). Toolbox Selection ^^^^^^^^^^^^^^^^^ @@ -88,7 +85,7 @@ SQLAlchemy-supported database works: .. code-block:: yaml galaxy: - tool_source_store: database # the writable default + tool_source_store: sqlite # the writable default tool_source_stores: cvmfs_main: backend: sqlalchemy @@ -295,7 +292,6 @@ To migrate an existing Galaxy installation to use tool source storage: .. code-block:: yaml galaxy: - tool_source_store: database use_lazy_toolbox: true 2. Run the population script: diff --git a/doc/source/dev/tool_source_storage.rst b/doc/source/dev/tool_source_storage.rst index a1f6e682d25..d81a79a8875 100644 --- a/doc/source/dev/tool_source_storage.rst +++ b/doc/source/dev/tool_source_storage.rst @@ -29,7 +29,6 @@ Module Layout lib/galaxy/tools/source_store/ __init__.py ToolSourceStore ABC, StoredToolSource, build_tool_source_store() - database.py DatabaseToolSourceStore (tool_source_record + tool_index tables) sqlalchemy.py SqlAlchemyToolSourceStore (any SA URL; sqlite shortcut) composite.py CompositeToolSourceStore (per-conf routing, merged index) index.py ToolIndex, ToolIndexEntry (the lightweight metadata) @@ -55,19 +54,19 @@ Two persistence concepts: **StoredToolSource** — the canonical macro-expanded XML/YAML for a tool, keyed by SHA-256 of the expanded content. Multiple versions of the same -``tool_id`` coexist as separate hashes. The ``database`` backend persists -these in the store-owned ``tool_source_record`` table (the ``tool_source`` -table belongs to the job-request path and has a different payload contract); -the ``sqlalchemy`` backend uses its own schema in a standalone database. +``tool_id`` coexist as separate hashes. The store keeps its own schema in a +standalone database (a SQLite file by default, any SQLAlchemy URL for shared +deployments) — deliberately outside Galaxy's database: the store is a +rebuildable cache and does not participate in Galaxy's migrations or session +lifecycle. **ToolIndex** — a single dataclass containing one ``ToolIndexEntry`` per tool, holding everything the batch APIs need (id, name, description, panel section, labels, EDAM, requirements, container info, test counts, hidden/disabled, shed metadata). The index is serialized and gzip-compressed as a blob. -The ``database`` backend gets the ``tool_index`` and ``tool_source_record`` -tables (migration ``f5a73c8b9d12``); ``tool_index`` holds a single row per -index version. +The schema is auto-created on first open; ``tool_index`` holds a single +row per index version. Backend Abstraction ------------------- @@ -79,9 +78,11 @@ Backend Abstraction - ``store_index/load_index/update_index_entry`` — index operations. - ``get_stats()`` — backend-specific stats (count, size, backend name). -``build_tool_source_store(config, sa_session)`` is the only entry point used +``build_tool_source_store(config)`` is the only entry point used by Galaxy. It inspects ``config.tool_source_store`` to pick the backend -(currently ``database`` and ``sqlalchemy``/``sqlite``). +(currently ``sqlalchemy``, alias ``sqlite``). The store is only built +when ``use_lazy_toolbox`` is enabled — default deployments never +initialize it. ``ConfigurationError`` is raised for unknown backends or missing required settings; it is allowed to propagate up so misconfiguration fails fast at startup. diff --git a/lib/galaxy/app/__init__.py b/lib/galaxy/app/__init__.py index cfecb51b8e0..69076214a51 100644 --- a/lib/galaxy/app/__init__.py +++ b/lib/galaxy/app/__init__.py @@ -419,19 +419,24 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi def _init_tool_source_store(self) -> None: """Initialize the tool source store for efficient tool loading. - Misconfiguration (bad backend name, missing required setting) raises - ``ConfigurationError`` from ``build_tool_source_store`` — we let it - propagate so the operator sees the failure at startup. + Default deployments never touch the store: it is only built when the + operator opted into ``use_lazy_toolbox``. Misconfiguration (bad + backend name, missing required setting) raises ``ConfigurationError`` + from ``build_tool_source_store`` — we let it propagate so the + operator sees the failure at startup. """ - # Lazy import: avoids pulling in optional backend deps at module load. + # Local import: avoids a circular import between galaxy.app and galaxy.tools. from galaxy.tools.source_store import ( build_tool_source_store, ToolSourceStore, ) - self.tool_source_store: ToolSourceStore | None = self._register_singleton( + self.tool_source_store: ToolSourceStore | None = None + if not self.config.use_lazy_toolbox: + return + self.tool_source_store = self._register_singleton( ToolSourceStore, # type: ignore[type-abstract,unused-ignore] - build_tool_source_store(self.config, self.model.context), # type: ignore[arg-type,unused-ignore] + build_tool_source_store(self.config), ) stats = self.tool_source_store.get_stats() tool_count = stats.get("count", 0) @@ -441,12 +446,6 @@ class MinimalGalaxyApplication(BasicSharedApp, HaltableContainer, SentryClientMi # in-memory caches before the next boot wires its own. Without this, # the prior boot's cached ToolIndex sticks around long enough to # race with the new boot's _load_index_from_store. - # ``_shutdown_tool_source_store`` commits any flushed-but-unwritten - # source/index rows on its way down (so the next embedded Galaxy - # boot doesn't have to re-bootstrap from configs). It must run - # before ``_shutdown_model`` disposes the engine — after that the - # session can't commit. Insert at index 1 (after object store, - # before database connection). self.haltables.insert(1, ("tool source store", self._shutdown_tool_source_store)) self.haltables.insert(2, ("lazy toolbox", self._shutdown_lazy_toolbox)) diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index 6a91ff822b5..4322caa77c7 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -291,7 +291,8 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.track_jobs_in_database = False self.amqp_internal_connection = None self.tool_configs = [] - self.tool_source_store = "database" + self.tool_source_store = "sqlite" + self.tool_source_disk_path = os.path.join(self.data_dir, "tool_sources") self.tool_source_stores = None self.use_lazy_toolbox = False self.manage_dependency_relationships = False diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index fdbf1b611b5..399ca32d27d 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -81,7 +81,6 @@ from sqlalchemy import ( Integer, join, JSON, - LargeBinary, literal, MetaData, not_, @@ -103,7 +102,6 @@ from sqlalchemy import ( update, VARCHAR, ) -from sqlalchemy.dialects import mysql from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.engine import CursorResult from sqlalchemy.exc import ( @@ -1428,66 +1426,6 @@ class ToolSource(Base, Dictifiable, RepresentById): dynamic_tool: Mapped[Optional["DynamicTool"]] = relationship() -class ToolSourceRecord(Base, Dictifiable, RepresentById): - """ - A tool source persisted by the tool source store. - - Deliberately separate from :class:`ToolSource`, whose rows are created - per executed tool by the job-request path and whose ``source`` column - carries the raw source string contract that path deserializes. The - populator keeps one row per ``source_path``; ``hash`` fingerprints the - expanded content and is deliberately non-unique — distinct files can - expand to identical content (``tools/data_source/upload.xml`` and its - ``test/functional/tools`` copy) yet each path must stay resolvable. - The populator may prune rows freely without affecting job records. - """ - - __tablename__ = "tool_source_record" - - dict_collection_visible_keys = ("id", "hash", "tool_id", "tool_version", "create_time", "update_time") - dict_element_visible_keys = ("id", "hash", "tool_id", "tool_version", "create_time", "update_time") - - id: Mapped[int] = mapped_column(primary_key=True) - hash: Mapped[str] = mapped_column(String(255), index=True, nullable=False) - source: Mapped[str] = mapped_column(Text().with_variant(mysql.LONGTEXT(), "mysql"), nullable=False) - source_class: Mapped[str] = mapped_column(TrimmedString(255)) - tool_id: Mapped[str | None] = mapped_column(String(255), index=True) - tool_version: Mapped[str | None] = mapped_column(String(255)) - tool_dir: Mapped[str | None] = mapped_column(Text, nullable=True) - source_path: Mapped[str | None] = mapped_column(Text, nullable=True) - stored_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) - source_metadata: Mapped[dict | None] = mapped_column(JSONType, nullable=True) - create_time: Mapped[datetime] = mapped_column(DateTime, default=now, nullable=False) - update_time: Mapped[datetime] = mapped_column(DateTime, default=now, onupdate=now, nullable=False) - - -class ToolIndexCache(Base, Dictifiable, RepresentById): - """ - Stores pre-computed tool index for fast API responses. - - The tool index contains lightweight metadata about all tools for - efficient batch endpoint access without loading full tool sources. - - The ``data`` column holds a gzipped JSON serialization of the index; - on MySQL the variant promotion to LONGBLOB is required because a real - tool index easily exceeds the 64KB BLOB default. Concurrent writers - must serialize their updates externally — the unique constraint on - ``version`` is intentionally singleton-like (delete-then-insert). - """ - - __tablename__ = "tool_index" - - dict_collection_visible_keys = ("id", "version", "built_at", "create_time", "update_time") - dict_element_visible_keys = ("id", "version", "built_at", "create_time", "update_time") - - id: Mapped[int] = mapped_column(primary_key=True) - version: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) - data: Mapped[bytes] = mapped_column(LargeBinary().with_variant(mysql.LONGBLOB(), "mysql"), nullable=False) - built_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) - create_time: Mapped[datetime] = mapped_column(DateTime, default=now, nullable=False) - update_time: Mapped[datetime] = mapped_column(DateTime, default=now, onupdate=now, nullable=False) - - class ToolRequest(Base, Dictifiable, RepresentById): """A captured request_internal payload for a tool execution. diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py b/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py deleted file mode 100644 index 594351a7b3f..00000000000 --- a/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py +++ /dev/null @@ -1,77 +0,0 @@ -"""Add tool source store tables (tool_index, tool_source_record) - -Revision ID: f5a73c8b9d12 -Revises: 28885b317f78 -Create Date: 2026-01-25 10:00:00.000000 - -""" - -import sqlalchemy as sa -from sqlalchemy.dialects import mysql - -from galaxy.model.custom_types import JSONType -from galaxy.model.migrations.util import ( - create_table, - drop_table, -) - -# revision identifiers, used by Alembic. -revision = "f5a73c8b9d12" -down_revision = "28885b317f78" -branch_labels = None -depends_on = None - -INDEX_TABLE_NAME = "tool_index" -SOURCE_TABLE_NAME = "tool_source_record" - - -def upgrade(): - """Create the tool source store's tables. - - ``tool_index`` stores a serialized ToolIndex object that provides fast - access to tool metadata for API responses without loading full tool - sources. ``tool_source_record`` stores the content-addressed tool - sources themselves — separate from ``tool_source``, whose rows belong - to the job-request path and carry a different payload contract. - """ - create_table( - INDEX_TABLE_NAME, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column("version", sa.String(64), nullable=False, unique=True), - sa.Column("data", sa.LargeBinary().with_variant(mysql.LONGBLOB(), "mysql"), nullable=False), - sa.Column("built_at", sa.DateTime, nullable=True), - sa.Column("create_time", sa.DateTime, nullable=False, server_default=sa.func.now()), - sa.Column( - "update_time", - sa.DateTime, - nullable=False, - server_default=sa.func.now(), - onupdate=sa.func.now(), - ), - ) - create_table( - SOURCE_TABLE_NAME, - sa.Column("id", sa.Integer, primary_key=True), - sa.Column("hash", sa.String(255), nullable=False, index=True), - sa.Column("source", sa.Text().with_variant(mysql.LONGTEXT(), "mysql"), nullable=False), - sa.Column("source_class", sa.String(255)), - sa.Column("tool_id", sa.String(255), index=True), - sa.Column("tool_version", sa.String(255)), - sa.Column("tool_dir", sa.Text, nullable=True), - sa.Column("source_path", sa.Text, nullable=True), - sa.Column("stored_at", sa.DateTime, nullable=True), - sa.Column("source_metadata", JSONType, nullable=True), - sa.Column("create_time", sa.DateTime, nullable=False, server_default=sa.func.now()), - sa.Column( - "update_time", - sa.DateTime, - nullable=False, - server_default=sa.func.now(), - onupdate=sa.func.now(), - ), - ) - - -def downgrade(): - drop_table(SOURCE_TABLE_NAME) - drop_table(INDEX_TABLE_NAME) diff --git a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py index 676f4419257..5883852ccb7 100644 --- a/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py +++ b/lib/galaxy/tool_shed/galaxy_install/tools/tool_panel_manager.py @@ -182,9 +182,6 @@ class ToolPanelManager: populate_for_paths( self.app.config, - # Lazy mode implies a full Galaxy app, which carries - # ``model`` beyond the InstallationTarget protocol. - self.app.model.context, # type: ignore[attr-defined] paths=list(new_path_guids), rebuild_whoosh=True, path_guids=new_path_guids, diff --git a/lib/galaxy/tools/lazy_toolbox.py b/lib/galaxy/tools/lazy_toolbox.py index 2e06debce98..ff4e32c2dcf 100644 --- a/lib/galaxy/tools/lazy_toolbox.py +++ b/lib/galaxy/tools/lazy_toolbox.py @@ -508,21 +508,6 @@ class LazyToolBox(ToolBox): # them at discovery time), so no post-walk sync is needed. self._rebuild_shed_short_id_map() - # Commit anything ``create_tool`` persisted during the eager walk. - # No-op when the store didn't see any new content. When the - # constructor runs in the queue-worker thread (``reload_toolbox`` - # control task), nothing later closes that transaction — on SQLite - # the open writer lock blocks the test driver's subsequent - # ``DELETE FROM repository_repository_dependency_association`` in - # ``reset_shed_tools`` (``test_repository_*`` teardown), on - # Postgres it leaves an idle-in-transaction row blocking those - # same ``DELETE``s for the rest of the shard run. - if self._store is not None: - try: - self._store.commit() - except Exception as e: - log.warning(f"LazyToolBox: post-init commit raised: {e}") - log.info( "LazyToolBox initialized with %d tools (cache_size=%d, parsed=%d, from_store=%d)", len(self._tools_by_id), @@ -598,7 +583,6 @@ class LazyToolBox(ToolBox): log.info("LazyToolBox: running populator inline to backfill the index") populate_store_inline( self.app.config, - self.app.model.context, rebuild_whoosh=True, ) @@ -886,7 +870,6 @@ class LazyToolBox(ToolBox): if self._store is not None: try: self._store.update_index_entry(entry) - self._store.commit() except Exception as e: log.warning("Persisting data_manager_id for %s raised: %s", entry.id, e) # LazyTool is duck-typed against Tool — the eager pipeline only @@ -924,7 +907,6 @@ class LazyToolBox(ToolBox): try: populate_for_paths( self.app.config, - self.app.model.context, [path], path_guids={path: guid}, ) @@ -1335,7 +1317,6 @@ class LazyToolBox(ToolBox): # resurrecting an uninstalled tool. try: self._store.remove_index_entry(tool_id) - self._store.commit() except Exception as e: log.warning("Persisting index removal of %s raised: %s", tool_id, e) # Installs converge across processes because every populate diff --git a/lib/galaxy/tools/source_store/__init__.py b/lib/galaxy/tools/source_store/__init__.py index 57c98ed5264..b022280da1b 100644 --- a/lib/galaxy/tools/source_store/__init__.py +++ b/lib/galaxy/tools/source_store/__init__.py @@ -24,7 +24,6 @@ from typing import ( if TYPE_CHECKING: from galaxy.config import GalaxyAppConfiguration - from galaxy.model.scoped_session import galaxy_scoped_session log = logging.getLogger(__name__) @@ -203,26 +202,14 @@ class ToolSourceStore(ABC): Backends override this when they cache; the default is a no-op. """ - def commit(self) -> None: # noqa: B027 — intentional empty default - """Commit any pending writes to durable storage. - - Backends that use a request-scoped session (``DatabaseToolSourceStore``) - only ``flush()`` inside ``store()`` / ``store_index()`` — the surrounding - session decides when to ``commit()``. The lazy toolbox bootstrap runs - *outside* a request, so it must drive the commit itself or every - bootstrap insert rolls back when the engine is disposed. File-backed - stores (``SqlAlchemyToolSourceStore``) already commit per write and - override this as a no-op; ``CompositeToolSourceStore`` propagates. - """ - def close(self) -> None: # noqa: B027 — intentional empty default """Release any state the store is holding. Wired into ``GalaxyUniverseApplication.haltables`` so a Python-side ``app.shutdown()`` (e.g. the embedded ``IntegrationTestCase.restart()`` path) clears references that would otherwise survive into the next - boot. Default is a no-op; ``DatabaseToolSourceStore`` and the - composite store override. + boot. Default is a no-op; backends holding an engine or cache + override, and the composite store propagates. """ @@ -236,18 +223,10 @@ class ReadOnlyStoreError(Exception): def _build_default_store( config: "GalaxyAppConfiguration", - sa_session: Optional["galaxy_scoped_session"], ) -> ToolSourceStore: """Build the default store from top-level ``tool_source_*`` config.""" backend = config.tool_source_store - if backend == "database": - from .database import DatabaseToolSourceStore - - if sa_session is None: - raise ConfigurationError("'database' backend requires a SQLAlchemy session") - return DatabaseToolSourceStore(sa_session) - if backend in ("sqlalchemy", "sqlite"): from .sqlalchemy import SqlAlchemyToolSourceStore @@ -260,15 +239,13 @@ def _build_default_store( def build_named_store( - sa_session: Optional["galaxy_scoped_session"], name: str, spec: dict, ) -> ToolSourceStore: """Build a single named store from a ``tool_source_stores`` entry. ``spec`` is the dict from galaxy.yml — a ``backend`` plus its options - plus an optional ``read_only`` flag. ``sa_session`` is only used for - the (unusual) ``database`` backend. + plus an optional ``read_only`` flag. """ if not isinstance(spec, dict): raise ConfigurationError(f"tool_source_stores[{name!r}] must be a mapping") @@ -284,17 +261,6 @@ def build_named_store( raise ConfigurationError(f"tool_source_stores[{name!r}] requires a 'url' or 'path'") return SqlAlchemyToolSourceStore(url=url, path=path, read_only=read_only) - if backend == "database": - from .database import DatabaseToolSourceStore - - if sa_session is None: - raise ConfigurationError( - f"tool_source_stores[{name!r}] uses the 'database' backend which requires a SQLAlchemy session" - ) - store = DatabaseToolSourceStore(sa_session) - store.read_only = read_only - return store - raise ConfigurationError(f"tool_source_stores[{name!r}] has unknown backend {backend!r}") @@ -320,7 +286,6 @@ def _collect_per_conf_store_names(config: "GalaxyAppConfiguration") -> set[str]: def build_tool_source_store( config: "GalaxyAppConfiguration", - sa_session: Optional["galaxy_scoped_session"], ) -> ToolSourceStore: """Build the active tool source store, composing per-conf overrides. @@ -332,10 +297,8 @@ def build_tool_source_store( Args: config: The Galaxy application configuration. - sa_session: Galaxy's scoped SQLAlchemy session, used by the - ``database`` backend. Other backends ignore it. """ - default_store = _build_default_store(config, sa_session) + default_store = _build_default_store(config) # Per-conf store="..." attributes are only meaningful when the LazyToolBox # is the active toolbox. Opting in is explicit: anything other than @@ -362,7 +325,7 @@ def build_tool_source_store( raise ConfigurationError( f"tool_conf references store {name!r} but no such entry exists in tool_source_stores" ) - members.append((name, build_named_store(sa_session, name, catalog[name]))) + members.append((name, build_named_store(name, catalog[name]))) # Default is consulted last so per-conf overrides shadow it on hash collisions. members.append(("__default__", default_store)) diff --git a/lib/galaxy/tools/source_store/database.py b/lib/galaxy/tools/source_store/database.py deleted file mode 100644 index ebc9816440a..00000000000 --- a/lib/galaxy/tools/source_store/database.py +++ /dev/null @@ -1,374 +0,0 @@ -""" -Database backend for Tool Source Store. - -This module provides a database-backed implementation of the ToolSourceStore -backed by the store-owned ``tool_source_record`` table (content-addressed -sources) and the ``tool_index`` table (serialized ToolIndex). The unrelated -``tool_source`` table belongs to the job-request path and is never touched. -""" - -import gzip -import json -import logging -from collections.abc import Iterator -from contextlib import contextmanager -from typing import ( - cast, -) - -from sqlalchemy import ( - func, - select, -) -from sqlalchemy.orm import Session - -from galaxy.model import ( - ToolIndexCache, - ToolSourceRecord, -) -from galaxy.model.scoped_session import galaxy_scoped_session -from . import ( - StoredToolSource, - ToolSourceStore, -) -from .index import ( - ToolIndex, - ToolIndexEntry, -) - -log = logging.getLogger(__name__) - - -class DatabaseToolSourceStore(ToolSourceStore): - """ - Database-backed tool source store. - - Uses the ``tool_source_record`` table for full tool sources and the - ``tool_index`` table for the serialized index. - """ - - def __init__(self, sa_session: galaxy_scoped_session): - """ - Initialize the database tool source store. - - Args: - sa_session: Galaxy's scoped SQLAlchemy session (``app.model.context``). - """ - self._sa_session = sa_session - self._cached_index: ToolIndex | None = None - - def _get_session(self) -> Session: - """Get the shared scoped session — used for writes that must commit - in the caller's context (request, queue worker, populator).""" - return cast(Session, self._sa_session) - - @contextmanager - def _read_session(self) -> Iterator[Session]: - """Yield a private Session for reads. - - Reads must not run on the shared scoped session: this code can - be called mid-request (``LazyTool`` materialise during workflow - ``inject_all``), and the ``rollback()`` we issue afterwards to - release the implicit read transaction would expire the caller's - request-scoped objects (e.g. ``workflow.steps``). A private - ``Session`` bound to the same engine sees the same committed - data but its lifecycle is isolated from the caller's. - """ - bind = self._sa_session.get_bind() - session = Session(bind=bind, autoflush=False, expire_on_commit=False) - try: - yield session - finally: - try: - session.close() - except Exception as e: - log.debug("read session close raised: %s", e) - - def store(self, tool_source: StoredToolSource) -> str: - """Store a tool source in the database. - - One row per ``source_path``: distinct files can expand to identical - content (same ``hash``), and each path must stay resolvable through - :meth:`get_by_source_path` — deduplicating on hash alone would leave - the second file's path pointing at nothing. A row whose path already - exists is updated in place when its content changed. Path-less - sources deduplicate on content hash. - """ - session = self._get_session() - - if tool_source.source_path: - existing = ( - session.execute( - select(ToolSourceRecord).where(ToolSourceRecord.source_path == tool_source.source_path).limit(1) - ) - .scalars() - .first() - ) - if existing is not None: - if existing.hash != tool_source.hash: - existing.hash = tool_source.hash - existing.source = tool_source.raw_source - existing.source_class = tool_source.tool_source_class - existing.tool_id = tool_source.tool_id - existing.tool_version = tool_source.tool_version - existing.tool_dir = tool_source.tool_dir - existing.stored_at = tool_source.stored_at - existing.source_metadata = tool_source.metadata or None - session.flush() - return tool_source.hash - else: - existing_id = ( - session.execute(select(ToolSourceRecord.id).where(ToolSourceRecord.hash == tool_source.hash).limit(1)) - .scalars() - .first() - ) - if existing_id: - return tool_source.hash - - model = ToolSourceRecord( - hash=tool_source.hash, - source=tool_source.raw_source, - source_class=tool_source.tool_source_class, - tool_id=tool_source.tool_id, - tool_version=tool_source.tool_version, - tool_dir=tool_source.tool_dir, - source_path=tool_source.source_path, - stored_at=tool_source.stored_at, - source_metadata=tool_source.metadata or None, - ) - session.add(model) - session.flush() - - return tool_source.hash - - def get(self, hash: str) -> StoredToolSource | None: - """Retrieve a tool source by hash. - - Several rows can share a hash (one per source path with identical - expanded content); any of them carries the same source. - """ - with self._read_session() as session: - model = ( - session.execute(select(ToolSourceRecord).where(ToolSourceRecord.hash == hash).limit(1)) - .scalars() - .first() - ) - if not model: - return None - return self._model_to_stored(model) - - def _model_to_stored(self, model: ToolSourceRecord) -> StoredToolSource: - """Convert database model to StoredToolSource.""" - return StoredToolSource( - hash=model.hash, - tool_source_class=model.source_class or "XmlToolSource", - raw_source=model.source or "", - tool_id=model.tool_id, - tool_version=model.tool_version, - tool_dir=model.tool_dir, - source_path=model.source_path, - stored_at=model.stored_at, - metadata=model.source_metadata or {}, - ) - - def exists(self, hash: str) -> bool: - """Check if a tool source exists.""" - with self._read_session() as session: - result = ( - session.execute(select(ToolSourceRecord.id).where(ToolSourceRecord.hash == hash).limit(1)) - .scalars() - .first() - ) - return result is not None - - def delete(self, hash: str) -> bool: - """Delete all rows carrying this hash (one per source path).""" - session = self._get_session() - - models = session.execute(select(ToolSourceRecord).where(ToolSourceRecord.hash == hash)).scalars().all() - - if not models: - return False - - for model in models: - session.delete(model) - session.flush() - return True - - def list_all(self) -> Iterator[str]: - """List all stored tool source hashes.""" - # Materialise eagerly so the private session can close before we - # yield — otherwise an outer caller could keep the session open - # indefinitely while iterating. - with self._read_session() as session: - result = session.execute(select(ToolSourceRecord.hash)).all() - for (hash_value,) in result: - if hash_value: - yield hash_value - - def get_by_tool_id(self, tool_id: str, version: str | None = None) -> list[StoredToolSource]: - """Get tool sources by tool ID and optional version.""" - with self._read_session() as session: - stmt = select(ToolSourceRecord).where(ToolSourceRecord.tool_id == tool_id) - if version is not None: - stmt = stmt.where(ToolSourceRecord.tool_version == version) - return [self._model_to_stored(model) for model in session.scalars(stmt)] - - def get_by_source_path(self, source_path: str) -> StoredToolSource | None: - """Get the stored source for a given on-disk file path. - - The populator writes one entry per file, so there is at most one match. - """ - with self._read_session() as session: - model = session.execute( - select(ToolSourceRecord).where(ToolSourceRecord.source_path == source_path).limit(1) - ).scalar_one_or_none() - if not model: - return None - return self._model_to_stored(model) - - def count(self) -> int: - """Return the total number of stored tool sources.""" - with self._read_session() as session: - result = session.execute(select(func.count(ToolSourceRecord.id))) - return result.scalar() or 0 - - def get_stats(self) -> dict: - """Return storage statistics.""" - return { - "count": self.count(), - "backend": "database", - } - - # Index operations - - def store_index(self, index: ToolIndex) -> None: - """ - Store the complete tool index. - - Stores the index as a gzip-compressed JSON blob in the tool_index table. - Uses versioning for cache invalidation. - """ - session = self._get_session() - - # Serialize and compress index - index_data = index.to_dict() - json_bytes = json.dumps(index_data).encode("utf-8") - compressed = gzip.compress(json_bytes) - - version = index.compute_version() - - # Singleton-style upsert: keep at most one row, identified by version. We - # update an existing row in place rather than DELETE+INSERT to avoid a - # window where the unique-version constraint can be violated by a - # concurrent writer. - model = session.execute(select(ToolIndexCache).order_by(ToolIndexCache.id)).scalar_one_or_none() - if model is None: - model = ToolIndexCache( - version=version, - data=compressed, - built_at=index.built_at, - ) - session.add(model) - else: - model.version = version - model.data = compressed - model.built_at = index.built_at - - session.flush() - self._cached_index = index - - def load_index(self) -> ToolIndex | None: - """Load the tool index from the tool_index table. - - Uses a private session so the implicit read transaction is - scoped to this call — the shared scoped session (which may be - request-bound or driving the cold-start populator) keeps its - own transaction state. - """ - if self._cached_index is not None: - return self._cached_index - - with self._read_session() as session: - # Try to load from new tool_index table first - model = session.execute(select(ToolIndexCache).order_by(ToolIndexCache.id.desc())).scalar_one_or_none() - - if model and model.data: - try: - json_bytes = gzip.decompress(model.data) - index_data = json.loads(json_bytes.decode("utf-8")) - self._cached_index = ToolIndex.from_dict(index_data) - return self._cached_index - except Exception as e: - log.warning(f"Failed to load index from tool_index table: {e}") - - return None - - def update_index_entry(self, entry: ToolIndexEntry) -> None: - """Update a single index entry.""" - index = self.load_index() - if index is None: - index = ToolIndex() - - # add_entry keeps ``entries_by_version`` in step with ``entries`` — - # versioned lookups (``get(tool_id, tool_version)``, the job-time - # materialise path) read the per-version map, and the two maps are - # serialized independently. - index.add_entry(entry) - index.invalidate_caches() - - # Update section mapping - if entry.panel_section_id: - if entry.panel_section_id not in index.by_section: - index.by_section[entry.panel_section_id] = [] - if entry.id not in index.by_section[entry.panel_section_id]: - index.by_section[entry.panel_section_id].append(entry.id) - - self.store_index(index) - - def invalidate_index_cache(self) -> None: - """Invalidate the cached index.""" - self._cached_index = None - - def commit(self) -> None: - """Commit pending writes on the shared scoped session.""" - if self._sa_session is not None: - try: - self._sa_session.commit() - except Exception as e: - log.warning(f"DatabaseToolSourceStore.commit raised: {e}") - - def close(self) -> None: - """Commit pending writes and drop in-memory state at app shutdown. - - ``store`` and ``store_index`` only ``flush()`` — they don't - ``commit()`` (so Galaxy's request-scoped session stays in - control of when its work lands on disk). On - ``IntegrationTestCase.restart()`` the prior Galaxy then disposes - its engine via ``_shutdown_model``, which forcibly closes - in-flight transactions; psycopg's abort path rolls them back. - Result: every shed-installed tool / bootstrapped index entry - the prior Galaxy wrote is gone when the next Galaxy starts — - the next boot sees an empty store and re-bootstraps from - configs (484 tool sources × XML parse + DB insert). - - On CI the second bootstrap consistently stalls a few seconds in - and never completes, hanging the test (``test_recovery``'s - post-restart Galaxy is the most obvious victim — its first - Galaxy bootstrapped fine, the second got stuck part-way through - ``discover_tools``). - - Commit on close so the next embedded Galaxy sees the - already-bootstrapped index and skips the second bootstrap - entirely. Outside the test driver this is a no-op for the - common case (production Galaxy doesn't restart in-process). - """ - if self._sa_session is not None: - try: - self._sa_session.commit() - except Exception as e: - log.debug(f"DatabaseToolSourceStore.close commit raised: {e}") - self._cached_index = None - # Don't null out the session — it's a scoped session shared with - # the rest of Galaxy. Just stop holding a strong reference to the - # cached index. diff --git a/lib/galaxy_test/driver/uses_shed.py b/lib/galaxy_test/driver/uses_shed.py index 8777945b5b4..cc8cd1c9012 100644 --- a/lib/galaxy_test/driver/uses_shed.py +++ b/lib/galaxy_test/driver/uses_shed.py @@ -92,7 +92,7 @@ class UsesShed(UsesShedApi): try: from galaxy.tools.source_store.populator import reconcile_index - reconcile_index(self._app.config, self._app.model.context, rebuild_whoosh=True) + reconcile_index(self._app.config, rebuild_whoosh=True) except Exception as e: log.warning("reset_shed_tools: reconcile_index raised (continuing): %s", e) # deleting the containing folder doesn't trigger a toolbox reload, so signal it now and wait until it's done diff --git a/test/integration/test_tool_source_storage.py b/test/integration/test_tool_source_storage.py index 28b85871bd1..e62dac1fe12 100644 --- a/test/integration/test_tool_source_storage.py +++ b/test/integration/test_tool_source_storage.py @@ -18,12 +18,6 @@ class BaseToolSourceStorageIntegrationTestCase(integration_util.IntegrationTestC """Base class for tool source storage integration tests.""" framework_tool_and_types = True - STORE_KIND: str = "database" - - @classmethod - def handle_galaxy_config_kwds(cls, config): - super().handle_galaxy_config_kwds(config) - config["tool_source_store"] = cls.STORE_KIND def _test_api_tools_list(self): # ``in_panel=False`` is what the Galaxy client uses @@ -46,8 +40,22 @@ class BaseToolSourceStorageIntegrationTestCase(integration_util.IntegrationTestC assert tool_info["id"] == tool_id -class TestDatabaseToolSourceStorage(BaseToolSourceStorageIntegrationTestCase): - """Integration tests with database tool source storage backend.""" +class TestEagerBootSkipsStore(BaseToolSourceStorageIntegrationTestCase): + """Default deployments never initialize a tool source store.""" + + def test_no_store_initialized(self): + assert self._app.tool_source_store is None + self._test_api_tools_list() + self._test_api_tools_show() + + +class TestSqliteToolSourceStorage(BaseToolSourceStorageIntegrationTestCase): + """Integration tests with the default sqlite tool source store.""" + + @classmethod + def handle_galaxy_config_kwds(cls, config): + super().handle_galaxy_config_kwds(config) + config["use_lazy_toolbox"] = True def test_api_tools_list(self): self._test_api_tools_list() @@ -55,10 +63,10 @@ class TestDatabaseToolSourceStorage(BaseToolSourceStorageIntegrationTestCase): def test_api_tools_show(self): self._test_api_tools_show() - def test_default_store_is_database_backend(self): - from galaxy.tools.source_store.database import DatabaseToolSourceStore + def test_default_store_is_sqlalchemy_backend(self): + from galaxy.tools.source_store.sqlalchemy import SqlAlchemyToolSourceStore - assert isinstance(self._app.tool_source_store, DatabaseToolSourceStore) + assert isinstance(self._app.tool_source_store, SqlAlchemyToolSourceStore) class TestCompositeToolSourceStorage(BaseToolSourceStorageIntegrationTestCase): @@ -87,7 +95,6 @@ class TestCompositeToolSourceStorage(BaseToolSourceStorageIntegrationTestCase): with open(cls._conf_path, "w") as f: f.write('\n\n') - config["tool_source_store"] = "database" config["use_lazy_toolbox"] = True existing_confs = config.get("tool_config_file") or "config/tool_conf.xml.sample" if isinstance(existing_confs, str): @@ -168,23 +175,18 @@ class TestLazyToolBoxReload(BaseToolSourceStorageIntegrationTestCase): # with its own, much smaller view, and the peer broadcast invalidates # our store cache. A subsequent reload must serve THIS instance's # tools from its inline repopulate, not the foreign index. - from typing import cast - - from galaxy.model.scoped_session import galaxy_scoped_session from galaxy.tools.source_store import ToolIndex - from galaxy.tools.source_store.database import DatabaseToolSourceStore + from galaxy.tools.source_store.sqlalchemy import SqlAlchemyToolSourceStore store = self._app.tool_source_store assert store is not None - foreign_store = DatabaseToolSourceStore(cast(galaxy_scoped_session, self._app.model.context)) + foreign_store = SqlAlchemyToolSourceStore(path=self._app.config.tool_source_disk_path) foreign_store.store_index(ToolIndex()) - foreign_store.commit() store.invalidate_index_cache() # Drop one store row so the reload takes the inline-repopulate path. upload_stored = store.get_by_tool_id("upload1") assert upload_stored store.delete(upload_stored[0].hash) - store.commit() from galaxy.queue_worker import reload_toolbox @@ -201,7 +203,6 @@ class TestLazyToolBoxReload(BaseToolSourceStorageIntegrationTestCase): assert store is not None for source_hash in list(store.list_all()): store.delete(source_hash) - store.commit() store.invalidate_index_cache() from galaxy.queue_worker import reload_toolbox diff --git a/test/unit/app/tools/test_lazy_tool.py b/test/unit/app/tools/test_lazy_tool.py index 64c45e3aa6c..f38329c8983 100644 --- a/test/unit/app/tools/test_lazy_tool.py +++ b/test/unit/app/tools/test_lazy_tool.py @@ -350,7 +350,7 @@ def test_create_tool_populates_adhoc_for_existing_file(tmp_path, monkeypatch): calls = {} - def fake_populate(config, session, paths, path_guids=None, **kwargs): + def fake_populate(config, paths, path_guids=None, **kwargs): calls["paths"] = paths calls["path_guids"] = path_guids