diff --git a/doc/source/dev/tool_source_storage.rst b/doc/source/dev/tool_source_storage.rst index 575998e98c5..ba321d6938a 100644 --- a/doc/source/dev/tool_source_storage.rst +++ b/doc/source/dev/tool_source_storage.rst @@ -53,15 +53,18 @@ Two persistence concepts: **StoredToolSource** — the canonical macro-expanded XML/YAML for a tool, keyed by SHA-256 of the expanded content. Multiple versions of the same ``tool_id`` coexist as separate hashes. The DB backend persists these in the -``tool_source`` table; the Redis and disk backends use their own layout. +store-owned ``tool_source_record`` table (the ``tool_source`` table belongs +to the job-request path and has a different payload contract); the Redis and +disk backends use their own layout. **ToolIndex** — a single dataclass containing one ``ToolIndexEntry`` per tool, holding everything the batch APIs need (id, name, description, panel section, labels, EDAM, requirements, container info, test counts, hidden/disabled, shed metadata). The index is serialized and gzip-compressed as a blob. -The DB backend gets a new ``tool_index`` table (migration -``f5a73c8b9d12_add_tool_index_table``) with a single row per index version. +The DB backend gets the ``tool_index`` and ``tool_source_record`` tables +(migration ``f5a73c8b9d12``); ``tool_index`` holds a single row per index +version. Redis stores the blob under a known key; disk stores it as a file. Backend Abstraction diff --git a/lib/galaxy/managers/tool_source.py b/lib/galaxy/managers/tool_source.py index a2872f5dcca..6e9aec1f641 100644 --- a/lib/galaxy/managers/tool_source.py +++ b/lib/galaxy/managers/tool_source.py @@ -67,19 +67,11 @@ def get_or_create_tool_source(session: Session, tool) -> ToolSource: def tool_source_identity_hash(tool: Any) -> str: dynamic_tool = getattr(tool, "dynamic_tool", None) + identity: tuple[str, ...] if dynamic_tool is not None and dynamic_tool.id is not None: identity = ("dynamic", str(dynamic_tool.id)) - return hashlib.sha256("\0".join(identity).encode("utf-8")).hexdigest() - return static_tool_source_identity_hash(tool.id, tool.version) - - -def static_tool_source_identity_hash(tool_id: str | None, tool_version: str | None) -> str: - """Identity hash for a persisted source with no dynamic-tool linkage. - - Shared with ``galaxy.tool_source_store`` writers, which persist - config-file tool sources and therefore always carry a static identity. - """ - identity = ("static", tool_id or "", tool_version or "") + else: + identity = ("static", tool.id or "", tool.version or "") return hashlib.sha256("\0".join(identity).encode("utf-8")).hexdigest() diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 906d99bfe08..01b0ab92f2f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -1428,6 +1428,36 @@ class ToolSource(Base, Dictifiable, RepresentById): dynamic_tool: Mapped[Optional["DynamicTool"]] = relationship() +class ToolSourceRecord(Base, Dictifiable, RepresentById): + """ + A tool source persisted by the tool source store. + + Deliberately separate from :class:`ToolSource`, whose rows are created + per executed tool by the job-request path and whose ``source`` column + carries the raw source string contract that path deserializes. Store + rows are content-addressed by ``hash`` and carry populator metadata; + the populator may prune them freely without affecting job records. + """ + + __tablename__ = "tool_source_record" + + dict_collection_visible_keys = ("id", "hash", "tool_id", "tool_version", "create_time", "update_time") + dict_element_visible_keys = ("id", "hash", "tool_id", "tool_version", "create_time", "update_time") + + id: Mapped[int] = mapped_column(primary_key=True) + hash: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False) + source: Mapped[str] = mapped_column(Text().with_variant(mysql.LONGTEXT(), "mysql"), nullable=False) + source_class: Mapped[str] = mapped_column(TrimmedString(255)) + tool_id: Mapped[str | None] = mapped_column(String(255), index=True) + tool_version: Mapped[str | None] = mapped_column(String(255)) + tool_dir: Mapped[str | None] = mapped_column(Text, nullable=True) + source_path: Mapped[str | None] = mapped_column(Text, nullable=True) + stored_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + source_metadata: Mapped[dict | None] = mapped_column(JSONType, nullable=True) + create_time: Mapped[datetime] = mapped_column(DateTime, default=now, nullable=False) + update_time: Mapped[datetime] = mapped_column(DateTime, default=now, onupdate=now, nullable=False) + + class ToolIndexCache(Base, Dictifiable, RepresentById): """ Stores pre-computed tool index for fast API responses. diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py b/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py index 10ce6aa5dbf..b35c2b91247 100644 --- a/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py @@ -1,4 +1,4 @@ -"""Add tool_index table for storing pre-computed tool index +"""Add tool source store tables (tool_index, tool_source_record) Revision ID: f5a73c8b9d12 Revises: 28885b317f78 @@ -9,6 +9,7 @@ Create Date: 2026-01-25 10:00:00.000000 import sqlalchemy as sa from sqlalchemy.dialects import mysql +from galaxy.model.custom_types import JSONType from galaxy.model.migrations.util import ( create_table, drop_table, @@ -20,18 +21,21 @@ down_revision = "28885b317f78" branch_labels = None depends_on = None -TABLE_NAME = "tool_index" +INDEX_TABLE_NAME = "tool_index" +SOURCE_TABLE_NAME = "tool_source_record" def upgrade(): - """Create tool_index table for storing pre-computed tool index. + """Create the tool source store's tables. - This table stores a serialized ToolIndex object that provides - fast access to tool metadata for API responses without loading - full tool sources. + ``tool_index`` stores a serialized ToolIndex object that provides fast + access to tool metadata for API responses without loading full tool + sources. ``tool_source_record`` stores the content-addressed tool + sources themselves — separate from ``tool_source``, whose rows belong + to the job-request path and carry a different payload contract. """ create_table( - TABLE_NAME, + INDEX_TABLE_NAME, sa.Column("id", sa.Integer, primary_key=True), sa.Column("version", sa.String(64), nullable=False, unique=True), sa.Column("data", sa.LargeBinary().with_variant(mysql.LONGBLOB(), "mysql"), nullable=False), @@ -45,7 +49,29 @@ def upgrade(): onupdate=sa.func.now(), ), ) + create_table( + SOURCE_TABLE_NAME, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("hash", sa.String(255), nullable=False, unique=True, index=True), + sa.Column("source", sa.Text().with_variant(mysql.LONGTEXT(), "mysql"), nullable=False), + sa.Column("source_class", sa.String(255)), + sa.Column("tool_id", sa.String(255), index=True), + sa.Column("tool_version", sa.String(255)), + sa.Column("tool_dir", sa.Text, nullable=True), + sa.Column("source_path", sa.Text, nullable=True), + sa.Column("stored_at", sa.DateTime, nullable=True), + sa.Column("source_metadata", JSONType, nullable=True), + sa.Column("create_time", sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.Column( + "update_time", + sa.DateTime, + nullable=False, + server_default=sa.func.now(), + onupdate=sa.func.now(), + ), + ) def downgrade(): - drop_table(TABLE_NAME) + drop_table(SOURCE_TABLE_NAME) + drop_table(INDEX_TABLE_NAME) diff --git a/lib/galaxy/tool_source_store/database.py b/lib/galaxy/tool_source_store/database.py index 9f441eb21b0..7bf14ea5073 100644 --- a/lib/galaxy/tool_source_store/database.py +++ b/lib/galaxy/tool_source_store/database.py @@ -2,8 +2,9 @@ Database backend for Tool Source Store. This module provides a database-backed implementation of the ToolSourceStore -that uses the existing tool_source table and adds a tool_index table for -lightweight metadata. +backed by the store-owned ``tool_source_record`` table (content-addressed +sources) and the ``tool_index`` table (serialized ToolIndex). The unrelated +``tool_source`` table belongs to the job-request path and is never touched. """ import gzip @@ -11,7 +12,6 @@ import json import logging from collections.abc import Iterator from contextlib import contextmanager -from datetime import datetime from typing import ( cast, ) @@ -22,10 +22,9 @@ from sqlalchemy import ( ) from sqlalchemy.orm import Session -from galaxy.managers.tool_source import static_tool_source_identity_hash from galaxy.model import ( ToolIndexCache, - ToolSource as ToolSourceModel, + ToolSourceRecord, ) from galaxy.model.scoped_session import galaxy_scoped_session from . import ( @@ -44,8 +43,8 @@ class DatabaseToolSourceStore(ToolSourceStore): """ Database-backed tool source store. - Uses the existing tool_source table for storing full tool sources - and a separate tool_index table for lightweight metadata. + Uses the ``tool_source_record`` table for full tool sources and the + ``tool_index`` table for the serialized index. """ def __init__(self, sa_session: galaxy_scoped_session): @@ -89,35 +88,22 @@ class DatabaseToolSourceStore(ToolSourceStore): """Store a tool source in the database.""" session = self._get_session() - # Check if already exists existing = session.execute( - select(ToolSourceModel).where(ToolSourceModel.hash == tool_source.hash) + select(ToolSourceRecord.id).where(ToolSourceRecord.hash == tool_source.hash) ).scalar_one_or_none() - if existing: return tool_source.hash - # Create new record - source_data = { - "raw": tool_source.raw_source, - "tool_source_class": tool_source.tool_source_class, - "tool_id": tool_source.tool_id, - "tool_version": tool_source.tool_version, - "tool_dir": tool_source.tool_dir, - "source_path": tool_source.source_path, - "stored_at": (tool_source.stored_at.isoformat() if tool_source.stored_at else None), - "metadata": tool_source.metadata, - } - - model = ToolSourceModel( + model = ToolSourceRecord( hash=tool_source.hash, - source=source_data, + source=tool_source.raw_source, source_class=tool_source.tool_source_class, tool_id=tool_source.tool_id, tool_version=tool_source.tool_version, - # Same "static" identity as galaxy.managers.tool_source.tool_source_identity_hash — - # store-backed sources never carry a dynamic tool. - identity_hash=static_tool_source_identity_hash(tool_source.tool_id, tool_source.tool_version), + tool_dir=tool_source.tool_dir, + source_path=tool_source.source_path, + stored_at=tool_source.stored_at, + source_metadata=tool_source.metadata or None, ) session.add(model) session.flush() @@ -128,38 +114,31 @@ class DatabaseToolSourceStore(ToolSourceStore): """Retrieve a tool source by hash.""" with self._read_session() as session: model = session.execute( - select(ToolSourceModel).where(ToolSourceModel.hash == hash) + select(ToolSourceRecord).where(ToolSourceRecord.hash == hash) ).scalar_one_or_none() if not model: return None return self._model_to_stored(model) - def _model_to_stored(self, model: ToolSourceModel) -> StoredToolSource: + def _model_to_stored(self, model: ToolSourceRecord) -> StoredToolSource: """Convert database model to StoredToolSource.""" - source_data = model.source or {} - - stored_at = source_data.get("stored_at") - if stored_at and isinstance(stored_at, str): - stored_at = datetime.fromisoformat(stored_at) - - assert model.hash is not None return StoredToolSource( hash=model.hash, - tool_source_class=source_data.get("tool_source_class", "XmlToolSource"), - raw_source=source_data.get("raw", ""), - tool_id=source_data.get("tool_id"), - tool_version=source_data.get("tool_version"), - tool_dir=source_data.get("tool_dir"), - source_path=source_data.get("source_path"), - stored_at=stored_at, - metadata=source_data.get("metadata", {}), + tool_source_class=model.source_class or "XmlToolSource", + raw_source=model.source or "", + tool_id=model.tool_id, + tool_version=model.tool_version, + tool_dir=model.tool_dir, + source_path=model.source_path, + stored_at=model.stored_at, + metadata=model.source_metadata or {}, ) def exists(self, hash: str) -> bool: """Check if a tool source exists.""" with self._read_session() as session: result = session.execute( - select(ToolSourceModel.id).where(ToolSourceModel.hash == hash) + select(ToolSourceRecord.id).where(ToolSourceRecord.hash == hash) ).scalar_one_or_none() return result is not None @@ -167,7 +146,7 @@ class DatabaseToolSourceStore(ToolSourceStore): """Delete a tool source by hash.""" session = self._get_session() - model = session.execute(select(ToolSourceModel).where(ToolSourceModel.hash == hash)).scalar_one_or_none() + model = session.execute(select(ToolSourceRecord).where(ToolSourceRecord.hash == hash)).scalar_one_or_none() if not model: return False @@ -182,7 +161,7 @@ class DatabaseToolSourceStore(ToolSourceStore): # yield — otherwise an outer caller could keep the session open # indefinitely while iterating. with self._read_session() as session: - result = session.execute(select(ToolSourceModel.hash)).all() + result = session.execute(select(ToolSourceRecord.hash)).all() for (hash_value,) in result: if hash_value: yield hash_value @@ -190,35 +169,28 @@ class DatabaseToolSourceStore(ToolSourceStore): def get_by_tool_id(self, tool_id: str, version: str | None = None) -> list[StoredToolSource]: """Get tool sources by tool ID and optional version.""" with self._read_session() as session: - # Query all and filter in Python since tool_id is in JSON - result = session.execute(select(ToolSourceModel)) - sources = [] - for (model,) in result: - source_data = model.source or {} - if source_data.get("tool_id") == tool_id: - if version is None or source_data.get("tool_version") == version: - sources.append(self._model_to_stored(model)) - return sources + stmt = select(ToolSourceRecord).where(ToolSourceRecord.tool_id == tool_id) + if version is not None: + stmt = stmt.where(ToolSourceRecord.tool_version == version) + return [self._model_to_stored(model) for model in session.scalars(stmt)] def get_by_source_path(self, source_path: str) -> StoredToolSource | None: """Get the stored source for a given on-disk file path. - ``source_path`` lives inside the JSON ``source`` blob so this scans the - table and filters in Python — same shape as ``get_by_tool_id``. The - populator writes one entry per file, so there is at most one match. + The populator writes one entry per file, so there is at most one match. """ with self._read_session() as session: - result = session.execute(select(ToolSourceModel)) - for (model,) in result: - source_data = model.source or {} - if source_data.get("source_path") == source_path: - return self._model_to_stored(model) - return None + model = session.execute( + select(ToolSourceRecord).where(ToolSourceRecord.source_path == source_path).limit(1) + ).scalar_one_or_none() + if not model: + return None + return self._model_to_stored(model) def count(self) -> int: """Return the total number of stored tool sources.""" with self._read_session() as session: - result = session.execute(select(func.count(ToolSourceModel.id))) + result = session.execute(select(func.count(ToolSourceRecord.id))) return result.scalar() or 0 def get_stats(self) -> dict: @@ -290,18 +262,6 @@ class DatabaseToolSourceStore(ToolSourceStore): except Exception as e: log.warning(f"Failed to load index from tool_index table: {e}") - # Fall back to legacy storage in tool_source table - legacy = session.execute( - select(ToolSourceModel).where(ToolSourceModel.hash == "__tool_index__") - ).scalar_one_or_none() - - if legacy: - source_data = legacy.source or {} - index_data = source_data.get("index") - if index_data: - self._cached_index = ToolIndex.from_dict(index_data) - return self._cached_index - return None def update_index_entry(self, entry: ToolIndexEntry) -> None: