From 8162853207965762a9dafb1d2da995cc812b4e8f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 3 Jul 2026 15:51:06 +0200 Subject: [PATCH 01/56] Add pluggable tool source store infrastructure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Introduces galaxy.tool_source_store: a queryable store of parsed, macro-expanded tool sources plus a pre-computed ToolIndex of the metadata batch consumers need (panel section, labels, EDAM, requirements, test counts, shed provenance). - Backends: database (store-owned tool_source_record + tool_index tables, migration f5a73c8b9d12), sqlalchemy (read-only SQLite bundles, e.g. CVMFS-shipped), and a composite that layers named stores over the default with read fall-through. - scripts/tool_source/populate_store.py walks tool_conf files and writes sources + index + whoosh search index; --watch updates the store on file changes and broadcasts reload_tool_source_cache. The populator is the single writer; consumers are read-only. - Per-conf opt-in via store="name" on a toolbox conf root, resolved against the tool_source_stores catalog in galaxy.yml. - The tool_source_record table is deliberately separate from tool_source, whose rows belong to the job-request path and carry a raw-string payload contract. Nothing reads from the store yet — an on-demand-loading toolbox consuming it is follow-up work (#22633). Extracted from that PR. --- doc/source/admin/index.rst | 1 + doc/source/admin/tool_source_storage.rst | 274 +++++ doc/source/dev/index.rst | 1 + doc/source/dev/tool_source_storage.rst | 206 ++++ lib/galaxy/app_unittest_utils/galaxy_mock.py | 2 + lib/galaxy/config/schemas/config_schema.yml | 54 + lib/galaxy/model/__init__.py | 59 ++ .../f5a73c8b9d12_add_tool_index_table.py | 77 ++ lib/galaxy/tool_source_store/__init__.py | 33 + lib/galaxy/tool_source_store/composite.py | 179 ++++ lib/galaxy/tool_source_store/database.py | 327 ++++++ lib/galaxy/tool_source_store/discover.py | 403 ++++++++ lib/galaxy/tool_source_store/factory.py | 134 +++ lib/galaxy/tool_source_store/index.py | 545 ++++++++++ lib/galaxy/tool_source_store/interface.py | 206 ++++ lib/galaxy/tool_source_store/models.py | 124 +++ lib/galaxy/tool_source_store/populator.py | 956 ++++++++++++++++++ lib/galaxy/tool_source_store/search.py | 255 +++++ lib/galaxy/tool_source_store/sqlalchemy.py | 306 ++++++ lib/galaxy/tool_util/toolbox/parser.py | 16 + lib/galaxy/tools/special_tools.py | 24 +- packages/app/src/galaxy/tool_source_store | 1 + scripts/tool_source/__init__.py | 1 + scripts/tool_source/populate_store.py | 22 + test/unit/scripts/__init__.py | 0 test/unit/scripts/tool_source/__init__.py | 0 .../tool_source/test_build_index_entry.py | 179 ++++ .../unit/scripts/tool_source/test_discover.py | 133 +++ .../tool_source/test_populate_store.py | 473 +++++++++ .../scripts/tool_source/test_whoosh_dir.py | 26 + test/unit/tool_source_store/__init__.py | 0 test/unit/tool_source_store/conftest.py | 27 + .../tool_source_store/test_composite_store.py | 154 +++ .../tool_source_store/test_index_versions.py | 155 +++ .../tool_source_store/test_sqlite_store.py | 121 +++ test/unit/tool_source_store/test_stores.py | 346 +++++++ 36 files changed, 5819 insertions(+), 1 deletion(-) create mode 100644 doc/source/admin/tool_source_storage.rst create mode 100644 doc/source/dev/tool_source_storage.rst create mode 100644 lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py create mode 100644 lib/galaxy/tool_source_store/__init__.py create mode 100644 lib/galaxy/tool_source_store/composite.py create mode 100644 lib/galaxy/tool_source_store/database.py create mode 100644 lib/galaxy/tool_source_store/discover.py create mode 100644 lib/galaxy/tool_source_store/factory.py create mode 100644 lib/galaxy/tool_source_store/index.py create mode 100644 lib/galaxy/tool_source_store/interface.py create mode 100644 lib/galaxy/tool_source_store/models.py create mode 100644 lib/galaxy/tool_source_store/populator.py create mode 100644 lib/galaxy/tool_source_store/search.py create mode 100644 lib/galaxy/tool_source_store/sqlalchemy.py create mode 120000 packages/app/src/galaxy/tool_source_store create mode 100644 scripts/tool_source/__init__.py create mode 100644 scripts/tool_source/populate_store.py create mode 100644 test/unit/scripts/__init__.py create mode 100644 test/unit/scripts/tool_source/__init__.py create mode 100644 test/unit/scripts/tool_source/test_build_index_entry.py create mode 100644 test/unit/scripts/tool_source/test_discover.py create mode 100644 test/unit/scripts/tool_source/test_populate_store.py create mode 100644 test/unit/scripts/tool_source/test_whoosh_dir.py create mode 100644 test/unit/tool_source_store/__init__.py create mode 100644 test/unit/tool_source_store/conftest.py create mode 100644 test/unit/tool_source_store/test_composite_store.py create mode 100644 test/unit/tool_source_store/test_index_versions.py create mode 100644 test/unit/tool_source_store/test_sqlite_store.py create mode 100644 test/unit/tool_source_store/test_stores.py diff --git a/doc/source/admin/index.rst b/doc/source/admin/index.rst index 7b065f21d0f..13cace25439 100644 --- a/doc/source/admin/index.rst +++ b/doc/source/admin/index.rst @@ -22,6 +22,7 @@ Galaxy Deployment & Administration ai_agents enable_headers_in_fetch_requests tool_panel + tool_source_storage data_tables mq dependency_resolvers diff --git a/doc/source/admin/tool_source_storage.rst b/doc/source/admin/tool_source_storage.rst new file mode 100644 index 00000000000..69c78498582 --- /dev/null +++ b/doc/source/admin/tool_source_storage.rst @@ -0,0 +1,274 @@ +Tool Source Storage +=================== + +Galaxy can pre-parse and store tool sources, plus a lightweight index, in a +configurable backend. This is especially useful for large Galaxy installations +with thousands of tools. + +A toolbox that consumes this store to load tools on demand is planned as +follow-up work; this document covers the store, the populator, and the index +that it will build on. + +Overview +-------- + +By default, Galaxy loads all tools into memory at startup. For installations with many tools, +this can: + +- Slow down Galaxy startup significantly +- Consume large amounts of memory + +The tool source storage system provides the groundwork to address these issues by: + +1. Pre-parsing and storing tool sources in a configurable backend +2. Maintaining a lightweight index for fast API responses + +Configuration +------------- + +Tool source storage is configured in ``galaxy.yml``. The following options are available: + +Backend Selection +^^^^^^^^^^^^^^^^^ + +.. code-block:: yaml + + galaxy: + # Backend for storing tool sources: 'database' or 'sqlalchemy' + tool_source_store: database + +**Database Backend** (default) + +Stores tool sources in the Galaxy database. Best for: + +- Single-server deployments +- Installations where tools don't change frequently +- Simplest setup (no additional infrastructure) + +**SQLAlchemy Backend** + +Stores tool sources in a separate SQLAlchemy-managed database (typically a +SQLite file). Useful for shipping read-only tool source bundles via per-conf +``tool_source_stores`` entries (see CVMFS recipe below). + +.. code-block:: yaml + + galaxy: + tool_source_store: sqlalchemy + tool_source_disk_path: /path/to/tool_sources.sqlite # SQLite shortcut + +Per-conf Store Routing (CVMFS Recipe) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Individual ``tool_conf`` files can opt into a *named* tool source store, +distinct from the global default. The typical use case is shipping a +read-only SQLite bundle on CVMFS alongside a tool_conf, so worker +processes can resolve every tool in that conf with local-cached lookups +instead of one network round-trip per JSON file. + +Declare the named stores under the new top-level ``tool_source_stores`` +key in ``galaxy.yml``. The ``sqlalchemy`` backend takes either a SQLAlchemy +``url`` or a ``path`` shortcut that builds a SQLite URL. SQLite is the +typical choice for CVMFS bundles (single self-contained file), but any +SQLAlchemy-supported database works: + +.. code-block:: yaml + + galaxy: + tool_source_store: database # the writable default + tool_source_stores: + cvmfs_main: + backend: sqlalchemy + path: /cvmfs/example.org/tools/sources.sqlite + read_only: true + site_shared: + backend: sqlalchemy + url: postgresql://galaxy_ro@db.example.org/tool_sources + read_only: true + +Then point the tool_conf at it via the root element's ``store`` attribute +(XML) or top-level key (YAML): + +.. code-block:: xml + + + +
+ + ... +
+ + +At startup, Galaxy inspects every ``tool_conf`` for the attribute, builds +the referenced stores, and wraps them with the writable default in a +composite store. Reads are tried in declared order (first hit wins) and +writes always go to the default store. If no tool_conf opts in, the +default store is used directly with zero overhead. + +**Building the bundle** + +Build the SQLite file from a writable host before shipping it: + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py -c galaxy.yml --target cvmfs_main + +Use ``--target`` to restrict population to a single named store; without +it, ``populate_store.py`` populates **every writable store** referenced +from a tool_conf in the same run. + +Once the bundle is in place on CVMFS (or any read-only mount), restart +Galaxy. + +Populating the Tool Source Store +-------------------------------- + +After configuring tool source storage, you need to populate it with your tools. +Use the ``populate_store.py`` script: + +Basic Usage +^^^^^^^^^^^ + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py --config /path/to/galaxy.yml + +This will: + +1. Discover tools from your tool configs (uses the same logic as Galaxy startup) +2. Parse each tool (with macro expansion) and compute a content hash +3. Store the tool sources in the configured backend (skipping unchanged tools) + +Note: ``--config`` is required; the script does not assume a default path. + +Command Line Options +^^^^^^^^^^^^^^^^^^^^ + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py --help + + Options: + --config, -c PATH Galaxy configuration file (required) + --dry-run Show what would be stored without storing + --incremental Only store new/changed tools (default) + --full Force re-store of all tools + --tool-id PATTERN Only process tools whose ID contains PATTERN + --parallel, -j N Number of parallel workers (default: 4) + --rebuild-index Rebuild the tool index after population + --target NAME Restrict to a single named store from + tool_source_stores (or '__default__'). Without + this, every writable store is populated. + --verbose, -v Verbose output + --watch, -w Watch tool directories and send reload notifications + --watch-polling Use polling observer (for NFS/CVMFS/network FS) + --debounce SECS Debounce time for watch mode (default: 2.0) + +Examples +^^^^^^^^ + +**Initial population:** + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py -c /path/to/galaxy.yml + +**Force re-store everything (e.g., after a parser change):** + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py -c galaxy.yml --full + +**Process only a subset of tools:** + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py -c galaxy.yml --tool-id samtools + +Automation with Cron +^^^^^^^^^^^^^^^^^^^^ + +For installations where tools are frequently updated, you can run the population +script on a schedule: + +.. code-block:: cron + + # Update tool source store every hour (incremental is the default) + 0 * * * * /path/to/galaxy/.venv/bin/python /path/to/galaxy/scripts/tool_source/populate_store.py -c /path/to/galaxy.yml >> /var/log/galaxy/tool_source_update.log 2>&1 + +Watch Mode (Live Updates) +^^^^^^^^^^^^^^^^^^^^^^^^^ + +For development environments or installations where tools change frequently, you can run +the population script in watch mode. This uses ``watchdog`` to monitor tool directories +for changes and automatically updates the store, then sends a notification via Kombu +to trigger cache reloads in all Galaxy processes. + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py --config galaxy.yml --watch + +Watch mode options: + +- ``--watch, -w`` - Enable watch mode +- ``--watch-polling`` - Use polling observer (required for network filesystems like NFS/CVMFS) +- ``--debounce SECS`` - Debounce time for file changes (default: 2.0 seconds) + +Example with polling for network filesystem: + +.. code-block:: console + + $ python scripts/tool_source/populate_store.py -c galaxy.yml --watch --watch-polling --debounce 5.0 + +**Requirements:** + +- The ``watchdog`` library must be installed: ``pip install watchdog`` +- Galaxy must have ``amqp_internal_connection`` configured for Kombu notifications +- All Galaxy processes must be connected to the same AMQP broker + +When a tool XML file changes, watch mode will: + +1. Detect the file change (with debouncing to handle rapid edits) +2. Re-parse the tool and update the store +3. Send a ``reload_tool_source_cache`` control message via Kombu +4. All Galaxy processes will invalidate their local caches + +This is useful for: + +- Development environments where tools are being actively edited +- CI/CD pipelines that deploy tool updates +- Installations using shared storage where tools may be updated externally + +Troubleshooting +--------------- + +Tools not appearing in the index +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. Re-run the population script with verbose output: + + .. code-block:: console + + $ python scripts/tool_source/populate_store.py -c galaxy.yml -v + +2. Check for parsing errors in the Galaxy log + +Populating an existing installation +----------------------------------- + +To set up tool source storage on an existing Galaxy installation: + +1. Add the configuration to ``galaxy.yml``: + + .. code-block:: yaml + + galaxy: + tool_source_store: database + +2. Run the population script: + + .. code-block:: console + + $ python scripts/tool_source/populate_store.py -c /path/to/galaxy.yml + +3. Restart Galaxy diff --git a/doc/source/dev/index.rst b/doc/source/dev/index.rst index 23e212d0a63..067764584d9 100644 --- a/doc/source/dev/index.rst +++ b/doc/source/dev/index.rst @@ -20,6 +20,7 @@ A multi-hour long video playlist covering these slides can be found at data_managers data_source data_types + tool_source_storage faq writing_tests debugging_tests diff --git a/doc/source/dev/tool_source_storage.rst b/doc/source/dev/tool_source_storage.rst new file mode 100644 index 00000000000..bd8111b1fb8 --- /dev/null +++ b/doc/source/dev/tool_source_storage.rst @@ -0,0 +1,206 @@ +Tool Source Storage Architecture +================================ + +This document describes the architecture of the tool source storage subsystem: +the store backends, the populator, and the index they build. For operator-facing +setup and configuration, see :doc:`/admin/tool_source_storage`. + +A toolbox that consumes this store to load tools on demand is planned as +follow-up work; the pieces documented here are the storage layer it will build on. + +Goals +----- + +The traditional ``ToolBox`` parses every tool XML at startup, builds full +``Tool`` objects, and keeps them all in memory. With thousands of tools that +scales poorly: slow boot, large per-process RSS, and expensive worker reloads. + +The tool source storage subsystem moves that parsing work out of the request +path: + +- A separate process (``populate_store.py``) parses tools once and persists + the canonical, macro-expanded source plus a lightweight metadata index. +- The store and index are laid out so a consumer can load only the index at + startup and materialize ``Tool`` objects on demand, instead of parsing the + full tree in-process. That consumer is the planned follow-up toolbox. + +Module Layout +------------- + +:: + + lib/galaxy/tool_source_store/ + __init__.py ToolSourceStore ABC, StoredToolSource, build_tool_source_store() + database.py DatabaseToolSourceStore (uses tool_source_record + tool_index tables) + sqlalchemy.py SqlAlchemyToolSourceStore (any SA URL; sqlite shortcut) + composite.py CompositeToolSourceStore (per-conf routing, merged index) + index.py ToolIndex, ToolIndexEntry (the lightweight metadata) + search.py ToolWhooshIndex (Whoosh search index built from a ToolIndex) + discover.py discover_tools() — conf walk without booting a ToolBox + populator.py Population + watch logic (parse, store, index, broadcast) + models.py Pydantic response models + + scripts/tool_source/populate_store.py Thin CLI wrapper over populator.main + +Data Model +---------- + +Two persistence concepts: + +**StoredToolSource** — the canonical macro-expanded XML/YAML for a tool, +keyed by SHA-256 of the expanded content. Multiple versions of the same +``tool_id`` coexist as separate hashes. The database backend persists these in +the store-owned ``tool_source_record`` table (the ``tool_source`` table belongs +to the job-request path and has a different payload contract); the sqlalchemy +backend uses the same schema in a standalone database. + +**ToolIndex** — a single dataclass containing one ``ToolIndexEntry`` per tool, +holding everything a store consumer needs (id, name, description, panel section, +labels, EDAM, requirements, container info, test counts, hidden/disabled, +shed metadata). The index is serialized and gzip-compressed as a blob. + +The database backend gets the ``tool_index`` and ``tool_source_record`` tables +(migration ``f5a73c8b9d12``); ``tool_index`` holds a single row per index +version. The sqlalchemy backend creates the same tables in its own database. + +Backend Abstraction +------------------- + +``ToolSourceStore`` (in ``tool_source_store/__init__.py``) is an ABC defining: + +- ``store/get/exists/delete/list_all/get_by_tool_id/count`` — per-tool source + operations, all keyed by content hash. +- ``store_index/load_index/update_index_entry`` — index operations. +- ``get_stats()`` — backend-specific stats (count, size, backend name). + +``build_tool_source_store(app)`` is the only entry point used by Galaxy. It +inspects ``config.tool_source_store`` and lazily imports the chosen backend +so deployments only pay for the dependencies they use. +``ConfigurationError`` is raised for unknown backends or missing required +settings; it is allowed to propagate up so misconfiguration fails fast at +startup. + +The ABC defines a ``read_only: bool`` class attribute (default ``False``). +``ReadOnlyStoreError`` is raised by mutating methods of stores that opted +in. The populator, watch reload, and composite all consult this flag to +route around read-only members rather than crashing. + +Per-conf composition +^^^^^^^^^^^^^^^^^^^^ + +If any tool_conf carries a top-level ``store="..."`` attribute (XML root) +or ``store: ...`` key (YAML), ``build_tool_source_store`` instantiates +the referenced named stores from ``config.tool_source_stores`` and wraps +them with the writable default in a :class:`CompositeToolSourceStore`. + +The composite implements the same ``ToolSourceStore`` interface, so store +consumers stay completely unaware of the multi-store layout: + +- **Reads** iterate ``[per-conf members..., default]`` in order; first + hit wins. ``count`` and ``list_all`` dedupe across members. +- **Writes** always land on the designated default. The default may not + itself be ``read_only``; that's enforced at construction. +- ``load_index()`` calls each member's ``load_index()`` and folds the + entries into a single :class:`ToolIndex`. Earlier members shadow later + ones on tool-id collisions; ``by_section`` is unioned; ``built_at`` + takes the most recent value. +- ``invalidate_index_cache()`` fans out so a single Kombu reload hits + every member. +- ``store_to(name, ...)`` lets the populator address a specific member by + name without going through composite write routing. + +When no tool_conf opts in, ``build_tool_source_store`` returns the +default store directly — the composite path is zero-cost for the common +case. + +The ``sqlalchemy`` backend (``sqlalchemy.py``) was added to make this +useful for CVMFS: a single self-contained ``.sqlite`` file, opened with +its own SQLAlchemy ``MetaData`` (independent of ``galaxy.model``) so the +file is portable, and openable with ``mode=ro&uri=true`` for read-only +mounts. Despite the name, the backend is not sqlite-specific — pass any +SQLAlchemy ``url`` (Postgres, MySQL, …) instead of ``path``. Auto schema +creation runs on first open; on remote backends operators may prefer to +manage migrations explicitly. + +Per-conf populator routing +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``scripts/tool_source/populate_store.py`` is per-conf aware. It reads +``parse_store_name()`` for each tool_conf, builds every named store plus +the default, and routes each ``DiscoveredTool.path`` to the store its +conf points at. By default it populates *every* writable store in one +run; ``--target NAME`` restricts to a single store and raises +``ReadOnlyStoreError`` if that store is read-only. Tools whose target is +read-only in default mode are silently skipped (the bundle is treated as +authoritative for those entries). + +Discovery +--------- + +``galaxy.tool_source_store.discover.discover_tools`` walks tool config files +and yields ``DiscoveredTool`` records without booting a full ``ToolBox``. It is +used by: + +- the populator to find tools to parse and store. +- watch mode to know which directories to monitor. +- callers that compare on-disk confs against the indexed tool set. + +Pulling discovery out of ``ToolBox`` was deliberate: the populator must run +*without* a full app (or even a running Galaxy), and the watch mode must run in +a long-lived loop with no Galaxy process at all. + +Population Script +----------------- + +``scripts/tool_source/populate_store.py`` is a thin CLI wrapper over +``galaxy.tool_source_store.populator.main``. It builds a minimal app context +(datatypes registry + SQLAlchemy model + config) and calls +``build_tool_source_store`` with that context. Tools are parsed in a +``ThreadPoolExecutor`` (``--parallel``, default 4 workers); each tool is +hashed and skipped if an entry with the same hash already exists +(``--incremental``, the default). Once the JSON index is committed the +populator rebuilds the Whoosh search index (``search.py``) so ranked tool +search stays in sync with the stored sources. + +Watch mode (``--watch``) uses ``watchdog`` to monitor every directory yielded +by ``discover_tools``. File events are debounced (default 2 s), the changed +files are re-parsed, the store is updated, and a single +``reload_tool_source_cache`` Kombu control task is published on the Galaxy +exchange. ``--watch-polling`` switches to ``PollingObserver`` for +NFS/CVMFS/network filesystems where inotify is unreliable. + +The broadcast is the populator's half of the contract: it publishes +``reload_tool_source_cache`` so peer processes can drop their stale index +view. The control-task handler that consumes the message lands with the +follow-up toolbox. + +Design Notes +------------ + +**Why a separate index instead of always querying the store?** A consumer +needs O(N) access to N entries; doing that against the backing store on every +request is a latency hit. Keeping the index in-process and only paying for +invalidation on reload is the better tradeoff. + +**Why an out-of-process populator?** Parsing tools and computing macro +expansions is expensive and shouldn't block worker startup. Keeping the +populator separate also lets it run on a single host while many web workers +share the resulting store. + +**Why hash-keyed storage?** Content-addressed storage gives us cheap +deduplication across versions and shed installations, and idempotent +incremental updates: re-running the populator over an unchanged tree is +effectively a no-op. + +Testing +------- + +- Store unit tests: ``test/unit/tool_source_store/`` exercises each backend + through the ``ToolSourceStore`` interface (``test_stores.py``, + ``test_sqlite_store.py``, ``test_composite_store.py``, + ``test_index_versions.py``). +- Populator/discovery unit tests: ``test/unit/scripts/tool_source/`` + (``test_populate_store.py``, ``test_discover.py``, + ``test_build_index_entry.py``, ``test_whoosh_dir.py``). These use fakes + (not mocks) of ``ToolSourceStore`` so behavior is verified against the real + interface. diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index 89575823f38..74dd4609f4b 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -290,6 +290,8 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.track_jobs_in_database = False self.amqp_internal_connection = None self.tool_configs = [] + self.tool_source_store = "database" + self.tool_source_stores = None self.manage_dependency_relationships = False self.enable_tool_shed_check = False self.monitor_thread_join_timeout = 1 diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index bbc57c7ecdd..ddf1401375d 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -322,6 +322,60 @@ mapping: Other tool config files must include the tool_path as an attribute in the tag. + tool_source_store: + type: str + default: database + required: false + desc: | + Backend for storing pre-parsed tool sources. Options are 'database' + or 'sqlalchemy'. + + - 'database': Stores tool sources in the Galaxy database. The default; + no additional setup required. + + - 'sqlalchemy': Stores tool sources in a separate SQLAlchemy-managed + database (typically a SQLite file). Useful for shipping read-only + tool source bundles via per-conf ``tool_source_stores`` entries. + + To populate the store, run: python scripts/tool_source/populate_store.py + + tool_source_disk_path: + type: str + default: tool_sources + path_resolves_to: data_dir + required: false + desc: | + Filesystem path used by the ``sqlalchemy`` backend — the path is + converted to a SQLite URL. Full SQLAlchemy URLs are available for + named stores via ``tool_source_stores``. + + tool_source_stores: + type: map + required: false + desc: | + Optional named tool source stores referenced from individual + tool_conf files via a top-level ``store=""`` attribute (XML) + or ``store: `` key (YAML). When any tool_conf opts in, the + process composes its named store with the default + (``tool_source_store``) backend at runtime, with reads tried in + declared order and writes always landing on the default. + + Each entry takes a ``backend`` (``database`` or ``sqlalchemy``) + plus the backend-specific settings, and an optional + ``read_only: true`` flag (typically used for CVMFS-shipped sqlite + bundles). + + The ``sqlalchemy`` backend accepts either a SQLAlchemy ``url`` or + a ``path`` shortcut that builds a SQLite URL. + + Example:: + + tool_source_stores: + cvmfs_main: + backend: sqlalchemy + path: /cvmfs/example.org/tools/sources.sqlite + read_only: true + tool_dependency_dir: type: str default: dependencies diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index edfc8feed19..9ef16623f6f 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -81,6 +81,7 @@ from sqlalchemy import ( Integer, join, JSON, + LargeBinary, literal, MetaData, not_, @@ -102,6 +103,7 @@ from sqlalchemy import ( update, VARCHAR, ) +from sqlalchemy.dialects import mysql from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.engine import CursorResult from sqlalchemy.exc import ( @@ -1418,6 +1420,63 @@ class ToolSource(Base, Dictifiable, RepresentById): dynamic_tool: Mapped[Optional["DynamicTool"]] = relationship() +class ToolSourceRecord(Base, Dictifiable, RepresentById): + """ + A tool source persisted by the tool source store. + + Deliberately separate from :class:`ToolSource`, whose rows are created + per executed tool by the job-request path and whose ``source`` column + carries the raw source string contract that path deserializes. Store + rows are content-addressed by ``hash`` and carry populator metadata; + the populator may prune them freely without affecting job records. + """ + + __tablename__ = "tool_source_record" + + dict_collection_visible_keys = ("id", "hash", "tool_id", "tool_version", "create_time", "update_time") + dict_element_visible_keys = ("id", "hash", "tool_id", "tool_version", "create_time", "update_time") + + id: Mapped[int] = mapped_column(primary_key=True) + hash: Mapped[str] = mapped_column(String(255), unique=True, index=True, nullable=False) + source: Mapped[str] = mapped_column(Text().with_variant(mysql.LONGTEXT(), "mysql"), nullable=False) + source_class: Mapped[str] = mapped_column(TrimmedString(255)) + tool_id: Mapped[str | None] = mapped_column(String(255), index=True) + tool_version: Mapped[str | None] = mapped_column(String(255)) + tool_dir: Mapped[str | None] = mapped_column(Text, nullable=True) + source_path: Mapped[str | None] = mapped_column(Text, nullable=True) + stored_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + source_metadata: Mapped[dict | None] = mapped_column(JSONType, nullable=True) + create_time: Mapped[datetime] = mapped_column(DateTime, default=now, nullable=False) + update_time: Mapped[datetime] = mapped_column(DateTime, default=now, onupdate=now, nullable=False) + + +class ToolIndexCache(Base, Dictifiable, RepresentById): + """ + Stores pre-computed tool index for fast API responses. + + The tool index contains lightweight metadata about all tools for + efficient batch endpoint access without loading full tool sources. + + The ``data`` column holds a gzipped JSON serialization of the index; + on MySQL the variant promotion to LONGBLOB is required because a real + tool index easily exceeds the 64KB BLOB default. Concurrent writers + must serialize their updates externally — the unique constraint on + ``version`` is intentionally singleton-like (delete-then-insert). + """ + + __tablename__ = "tool_index" + + dict_collection_visible_keys = ("id", "version", "built_at", "create_time", "update_time") + dict_element_visible_keys = ("id", "version", "built_at", "create_time", "update_time") + + id: Mapped[int] = mapped_column(primary_key=True) + version: Mapped[str] = mapped_column(String(64), unique=True, nullable=False) + data: Mapped[bytes] = mapped_column(LargeBinary().with_variant(mysql.LONGBLOB(), "mysql"), nullable=False) + built_at: Mapped[datetime | None] = mapped_column(DateTime, nullable=True) + create_time: Mapped[datetime] = mapped_column(DateTime, default=now, nullable=False) + update_time: Mapped[datetime] = mapped_column(DateTime, default=now, onupdate=now, nullable=False) + + class ToolRequest(Base, Dictifiable, RepresentById): """A captured request_internal payload for a tool execution. diff --git a/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py b/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py new file mode 100644 index 00000000000..b35c2b91247 --- /dev/null +++ b/lib/galaxy/model/migrations/alembic/versions_gxy/f5a73c8b9d12_add_tool_index_table.py @@ -0,0 +1,77 @@ +"""Add tool source store tables (tool_index, tool_source_record) + +Revision ID: f5a73c8b9d12 +Revises: 28885b317f78 +Create Date: 2026-01-25 10:00:00.000000 + +""" + +import sqlalchemy as sa +from sqlalchemy.dialects import mysql + +from galaxy.model.custom_types import JSONType +from galaxy.model.migrations.util import ( + create_table, + drop_table, +) + +# revision identifiers, used by Alembic. +revision = "f5a73c8b9d12" +down_revision = "28885b317f78" +branch_labels = None +depends_on = None + +INDEX_TABLE_NAME = "tool_index" +SOURCE_TABLE_NAME = "tool_source_record" + + +def upgrade(): + """Create the tool source store's tables. + + ``tool_index`` stores a serialized ToolIndex object that provides fast + access to tool metadata for API responses without loading full tool + sources. ``tool_source_record`` stores the content-addressed tool + sources themselves — separate from ``tool_source``, whose rows belong + to the job-request path and carry a different payload contract. + """ + create_table( + INDEX_TABLE_NAME, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("version", sa.String(64), nullable=False, unique=True), + sa.Column("data", sa.LargeBinary().with_variant(mysql.LONGBLOB(), "mysql"), nullable=False), + sa.Column("built_at", sa.DateTime, nullable=True), + sa.Column("create_time", sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.Column( + "update_time", + sa.DateTime, + nullable=False, + server_default=sa.func.now(), + onupdate=sa.func.now(), + ), + ) + create_table( + SOURCE_TABLE_NAME, + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("hash", sa.String(255), nullable=False, unique=True, index=True), + sa.Column("source", sa.Text().with_variant(mysql.LONGTEXT(), "mysql"), nullable=False), + sa.Column("source_class", sa.String(255)), + sa.Column("tool_id", sa.String(255), index=True), + sa.Column("tool_version", sa.String(255)), + sa.Column("tool_dir", sa.Text, nullable=True), + sa.Column("source_path", sa.Text, nullable=True), + sa.Column("stored_at", sa.DateTime, nullable=True), + sa.Column("source_metadata", JSONType, nullable=True), + sa.Column("create_time", sa.DateTime, nullable=False, server_default=sa.func.now()), + sa.Column( + "update_time", + sa.DateTime, + nullable=False, + server_default=sa.func.now(), + onupdate=sa.func.now(), + ), + ) + + +def downgrade(): + drop_table(SOURCE_TABLE_NAME) + drop_table(INDEX_TABLE_NAME) diff --git a/lib/galaxy/tool_source_store/__init__.py b/lib/galaxy/tool_source_store/__init__.py new file mode 100644 index 00000000000..b53bc4c9f97 --- /dev/null +++ b/lib/galaxy/tool_source_store/__init__.py @@ -0,0 +1,33 @@ +""" +Tool Source Store - Pluggable storage backends for Galaxy tool sources. + +This package provides a configurable, pluggable tool source storage system +that enables storing and retrieving tool sources from multiple backends +(currently ``database`` and ``sqlalchemy``). +""" + +from .factory import ( + build_named_store, + build_tool_source_store, +) +from .index import ( + ToolIndex, + ToolIndexEntry, +) +from .interface import ( + ConfigurationError, + ReadOnlyStoreError, + StoredToolSource, + ToolSourceStore, +) + +__all__ = [ + "StoredToolSource", + "ToolSourceStore", + "ToolIndex", + "ToolIndexEntry", + "build_tool_source_store", + "build_named_store", + "ConfigurationError", + "ReadOnlyStoreError", +] diff --git a/lib/galaxy/tool_source_store/composite.py b/lib/galaxy/tool_source_store/composite.py new file mode 100644 index 00000000000..f75ddbcf822 --- /dev/null +++ b/lib/galaxy/tool_source_store/composite.py @@ -0,0 +1,179 @@ +""" +Composite tool source store. + +Lets a single Galaxy process serve tools from multiple per-tool-conf +stores. Reads are tried in declared order (first hit wins), writes go to +a designated *default* store. Used to layer e.g. a CVMFS-resident +read-only sqlite bundle on top of the local writable store. + +The composite is invisible to the rest of Galaxy: it implements the same +:class:`ToolSourceStore` interface, and consumers / the populator +keep working unchanged. +""" + +import logging +from collections.abc import Iterator +from typing import ( + Any, +) + +from .index import ( + ToolIndex, + ToolIndexEntry, +) +from .interface import ( + StoredToolSource, + ToolSourceStore, +) + +log = logging.getLogger(__name__) + + +class CompositeToolSourceStore(ToolSourceStore): + """A read-priority store that fans out across several backends. + + Args: + members: Ordered list of ``(name, store)`` pairs consulted for + reads in order. Earlier entries shadow later ones on id/hash + collisions. + default: The store that receives all writes. Must be present in + ``members`` (its name is the value used for write routing). + Must not be ``read_only``. + """ + + def __init__( + self, + members: list[tuple[str, ToolSourceStore]], + default: str, + ) -> None: + if not members: + raise ValueError("CompositeToolSourceStore requires at least one member") + names = [n for n, _ in members] + if default not in names: + raise ValueError(f"default store {default!r} not in members {names!r}") + self._members: list[tuple[str, ToolSourceStore]] = list(members) + self._default_name = default + self._default_store = dict(members)[default] + if self._default_store.read_only: + raise ValueError(f"default store {default!r} is read-only") + # Composite as a whole is writable iff its default store is writable. + self.read_only = False + + # --- write ops: always default ------------------------------------ + + def store(self, tool_source: StoredToolSource) -> str: + return self._default_store.store(tool_source) + + def delete(self, hash: str) -> bool: + return self._default_store.delete(hash) + + def store_index(self, index: ToolIndex) -> None: + self._default_store.store_index(index) + + def update_index_entry(self, entry: ToolIndexEntry) -> None: + self._default_store.update_index_entry(entry) + + # --- read ops: priority order -------------------------------------- + + def get(self, hash: str) -> StoredToolSource | None: + for _name, member in self._members: + found = member.get(hash) + if found is not None: + return found + return None + + def exists(self, hash: str) -> bool: + return any(m.exists(hash) for _, m in self._members) + + def get_by_tool_id(self, tool_id: str, version: str | None = None) -> list[StoredToolSource]: + # Union across members, deduped by hash, preserving member order. + seen: set[str] = set() + out: list[StoredToolSource] = [] + for _name, member in self._members: + for src in member.get_by_tool_id(tool_id, version): + if src.hash in seen: + continue + seen.add(src.hash) + out.append(src) + return out + + def get_by_source_path(self, source_path: str) -> StoredToolSource | None: + for _name, member in self._members: + found = member.get_by_source_path(source_path) + if found is not None: + return found + return None + + def list_all(self) -> Iterator[str]: + seen: set[str] = set() + for _name, member in self._members: + for h in member.list_all(): + if h in seen: + continue + seen.add(h) + yield h + + def count(self) -> int: + # Distinct hashes across the composite. + return sum(1 for _ in self.list_all()) + + def get_stats(self) -> dict[str, Any]: + return { + "backend": "composite", + "count": self.count(), + "default": self._default_name, + "members": [{"name": name, **member.get_stats()} for name, member in self._members], + } + + # --- index --------------------------------------------------------- + + def load_index(self) -> ToolIndex | None: + merged = ToolIndex() + any_loaded = False + for name, member in self._members: + try: + idx = member.load_index() + except Exception as e: + log.warning(f"Failed to load index from store {name!r}: {e}") + continue + if idx is None: + continue + any_loaded = True + for tool_id, entry in idx.entries.items(): + # Earlier members win on collision. + if tool_id in merged.entries: + continue + merged.entries[tool_id] = entry + for section_id, ids in idx.by_section.items(): + bucket = merged.by_section.setdefault(section_id, []) + for tid in ids: + if tid not in bucket: + bucket.append(tid) + for view_name, view in idx.panel_views.items(): + merged.panel_views.setdefault(view_name, view) + if idx.built_at and (merged.built_at is None or idx.built_at > merged.built_at): + merged.built_at = idx.built_at + if not any_loaded: + return None + merged.version = merged.compute_version() + return merged + + def invalidate_index_cache(self) -> None: + for _name, member in self._members: + member.invalidate_index_cache() + + def commit(self) -> None: + """Propagate commit() to every writable member store.""" + for _name, member in self._members: + try: + member.commit() + except Exception as e: + log.warning(f"Composite store commit failed for member '{_name}': {e}") + + def close(self) -> None: + """Propagate close() to every member store.""" + for _name, member in self._members: + try: + member.close() + except Exception as e: + log.warning(f"Composite store close failed for member '{_name}': {e}") diff --git a/lib/galaxy/tool_source_store/database.py b/lib/galaxy/tool_source_store/database.py new file mode 100644 index 00000000000..27019e952e2 --- /dev/null +++ b/lib/galaxy/tool_source_store/database.py @@ -0,0 +1,327 @@ +""" +Database backend for Tool Source Store. + +This module provides a database-backed implementation of the ToolSourceStore +backed by the store-owned ``tool_source_record`` table (content-addressed +sources) and the ``tool_index`` table (serialized ToolIndex). The unrelated +``tool_source`` table belongs to the job-request path and is never touched. +""" + +import gzip +import json +import logging +from collections.abc import Iterator +from contextlib import contextmanager +from typing import ( + cast, +) + +from sqlalchemy import ( + func, + select, +) +from sqlalchemy.orm import Session + +from galaxy.model import ( + ToolIndexCache, + ToolSourceRecord, +) +from galaxy.model.scoped_session import galaxy_scoped_session +from .index import ( + ToolIndex, + ToolIndexEntry, +) +from .interface import ( + StoredToolSource, + ToolSourceStore, +) + +log = logging.getLogger(__name__) + + +class DatabaseToolSourceStore(ToolSourceStore): + """ + Database-backed tool source store. + + Uses the ``tool_source_record`` table for full tool sources and the + ``tool_index`` table for the serialized index. + """ + + def __init__(self, sa_session: galaxy_scoped_session): + """ + Initialize the database tool source store. + + Args: + sa_session: Galaxy's scoped SQLAlchemy session (``app.model.context``). + """ + self._sa_session = sa_session + self._cached_index: ToolIndex | None = None + + def _get_session(self) -> Session: + """Get the shared scoped session — used for writes that must commit + in the caller's context (request, queue worker, populator).""" + return cast(Session, self._sa_session) + + @contextmanager + def _read_session(self) -> Iterator[Session]: + """Yield a private Session for reads. + + Reads must not run on the shared scoped session: this code can + be called mid-request, and a ``rollback()`` issued afterwards to + release the implicit read transaction would expire the caller's + request-scoped objects. A private ``Session`` bound to the same + engine sees the same committed data but its lifecycle is isolated + from the caller's. + """ + bind = self._sa_session.get_bind() + session = Session(bind=bind, autoflush=False, expire_on_commit=False) + try: + yield session + finally: + try: + session.close() + except Exception as e: + log.debug("read session close raised: %s", e) + + def store(self, tool_source: StoredToolSource) -> str: + """Store a tool source in the database.""" + session = self._get_session() + + existing = session.execute( + select(ToolSourceRecord.id).where(ToolSourceRecord.hash == tool_source.hash) + ).scalar_one_or_none() + if existing: + return tool_source.hash + + model = ToolSourceRecord( + hash=tool_source.hash, + source=tool_source.raw_source, + source_class=tool_source.tool_source_class, + tool_id=tool_source.tool_id, + tool_version=tool_source.tool_version, + tool_dir=tool_source.tool_dir, + source_path=tool_source.source_path, + stored_at=tool_source.stored_at, + source_metadata=tool_source.metadata or None, + ) + session.add(model) + session.flush() + + return tool_source.hash + + def get(self, hash: str) -> StoredToolSource | None: + """Retrieve a tool source by hash.""" + with self._read_session() as session: + model = session.execute(select(ToolSourceRecord).where(ToolSourceRecord.hash == hash)).scalar_one_or_none() + if not model: + return None + return self._model_to_stored(model) + + def _model_to_stored(self, model: ToolSourceRecord) -> StoredToolSource: + """Convert database model to StoredToolSource.""" + return StoredToolSource( + hash=model.hash, + tool_source_class=model.source_class or "XmlToolSource", + raw_source=model.source or "", + tool_id=model.tool_id, + tool_version=model.tool_version, + tool_dir=model.tool_dir, + source_path=model.source_path, + stored_at=model.stored_at, + metadata=model.source_metadata or {}, + ) + + def exists(self, hash: str) -> bool: + """Check if a tool source exists.""" + with self._read_session() as session: + result = session.execute( + select(ToolSourceRecord.id).where(ToolSourceRecord.hash == hash) + ).scalar_one_or_none() + return result is not None + + def delete(self, hash: str) -> bool: + """Delete a tool source by hash.""" + session = self._get_session() + + model = session.execute(select(ToolSourceRecord).where(ToolSourceRecord.hash == hash)).scalar_one_or_none() + + if not model: + return False + + session.delete(model) + session.flush() + return True + + def list_all(self) -> Iterator[str]: + """List all stored tool source hashes.""" + # Materialise eagerly so the private session can close before we + # yield — otherwise an outer caller could keep the session open + # indefinitely while iterating. + with self._read_session() as session: + result = session.execute(select(ToolSourceRecord.hash)).all() + for (hash_value,) in result: + if hash_value: + yield hash_value + + def get_by_tool_id(self, tool_id: str, version: str | None = None) -> list[StoredToolSource]: + """Get tool sources by tool ID and optional version.""" + with self._read_session() as session: + stmt = select(ToolSourceRecord).where(ToolSourceRecord.tool_id == tool_id) + if version is not None: + stmt = stmt.where(ToolSourceRecord.tool_version == version) + return [self._model_to_stored(model) for model in session.scalars(stmt)] + + def get_by_source_path(self, source_path: str) -> StoredToolSource | None: + """Get the stored source for a given on-disk file path. + + The populator writes one entry per file, so there is at most one match. + """ + with self._read_session() as session: + model = session.execute( + select(ToolSourceRecord).where(ToolSourceRecord.source_path == source_path).limit(1) + ).scalar_one_or_none() + if not model: + return None + return self._model_to_stored(model) + + def count(self) -> int: + """Return the total number of stored tool sources.""" + with self._read_session() as session: + result = session.execute(select(func.count(ToolSourceRecord.id))) + return result.scalar() or 0 + + def get_stats(self) -> dict: + """Return storage statistics.""" + return { + "count": self.count(), + "backend": "database", + } + + # Index operations + + def store_index(self, index: ToolIndex) -> None: + """ + Store the complete tool index. + + Stores the index as a gzip-compressed JSON blob in the tool_index table. + Uses versioning for cache invalidation. + """ + session = self._get_session() + + # Serialize and compress index + index_data = index.to_dict() + json_bytes = json.dumps(index_data).encode("utf-8") + compressed = gzip.compress(json_bytes) + + version = index.compute_version() + + # Singleton-style upsert: keep at most one row, identified by version. We + # update an existing row in place rather than DELETE+INSERT to avoid a + # window where the unique-version constraint can be violated by a + # concurrent writer. + model = session.execute(select(ToolIndexCache).order_by(ToolIndexCache.id)).scalar_one_or_none() + if model is None: + model = ToolIndexCache( + version=version, + data=compressed, + built_at=index.built_at, + ) + session.add(model) + else: + model.version = version + model.data = compressed + model.built_at = index.built_at + + session.flush() + self._cached_index = index + + def load_index(self) -> ToolIndex | None: + """Load the tool index from the tool_index table. + + Uses a private session so the implicit read transaction is + scoped to this call — the shared scoped session (which may be + request-bound or driving the cold-start populator) keeps its + own transaction state. + """ + if self._cached_index is not None: + return self._cached_index + + with self._read_session() as session: + # Try to load from new tool_index table first + model = session.execute(select(ToolIndexCache).order_by(ToolIndexCache.id.desc())).scalar_one_or_none() + + if model and model.data: + try: + json_bytes = gzip.decompress(model.data) + index_data = json.loads(json_bytes.decode("utf-8")) + self._cached_index = ToolIndex.from_dict(index_data) + return self._cached_index + except Exception as e: + log.warning(f"Failed to load index from tool_index table: {e}") + + return None + + def update_index_entry(self, entry: ToolIndexEntry) -> None: + """Update a single index entry.""" + index = self.load_index() + if index is None: + index = ToolIndex() + + index.entries[entry.id] = entry + index.invalidate_caches() + + # Update section mapping + if entry.panel_section_id: + if entry.panel_section_id not in index.by_section: + index.by_section[entry.panel_section_id] = [] + if entry.id not in index.by_section[entry.panel_section_id]: + index.by_section[entry.panel_section_id].append(entry.id) + + self.store_index(index) + + def invalidate_index_cache(self) -> None: + """Invalidate the cached index.""" + self._cached_index = None + + def commit(self) -> None: + """Commit pending writes on the shared scoped session.""" + if self._sa_session is not None: + try: + self._sa_session.commit() + except Exception as e: + log.warning(f"DatabaseToolSourceStore.commit raised: {e}") + + def close(self) -> None: + """Commit pending writes and drop in-memory state at app shutdown. + + ``store`` and ``store_index`` only ``flush()`` — they don't + ``commit()`` (so Galaxy's request-scoped session stays in + control of when its work lands on disk). On + ``IntegrationTestCase.restart()`` the prior Galaxy then disposes + its engine via ``_shutdown_model``, which forcibly closes + in-flight transactions; psycopg's abort path rolls them back. + Result: every shed-installed tool / bootstrapped index entry + the prior Galaxy wrote is gone when the next Galaxy starts — + the next boot sees an empty store and re-bootstraps from + configs (484 tool sources × XML parse + DB insert). + + On CI the second bootstrap consistently stalls a few seconds in + and never completes, hanging the test (``test_recovery``'s + post-restart Galaxy is the most obvious victim — its first + Galaxy bootstrapped fine, the second got stuck part-way through + ``discover_tools``). + + Commit on close so the next embedded Galaxy sees the + already-bootstrapped index and skips the second bootstrap + entirely. Outside the test driver this is a no-op for the + common case (production Galaxy doesn't restart in-process). + """ + if self._sa_session is not None: + try: + self._sa_session.commit() + except Exception as e: + log.debug(f"DatabaseToolSourceStore.close commit raised: {e}") + self._cached_index = None + # Don't null out the session — it's a scoped session shared with + # the rest of Galaxy. Just stop holding a strong reference to the + # cached index. diff --git a/lib/galaxy/tool_source_store/discover.py b/lib/galaxy/tool_source_store/discover.py new file mode 100644 index 00000000000..cb7604e012e --- /dev/null +++ b/lib/galaxy/tool_source_store/discover.py @@ -0,0 +1,403 @@ +""" +Tool discovery utilities. + +Walks Galaxy's tool configuration files to enumerate every ```` referenced +from any tool_conf without booting a full ``ToolBox``. Used by the populator +(``galaxy.tool_source_store.populator``) and by callers that need to compare +on-disk confs against the indexed tool set (cold-start auto-populate, +``reset_shed_tools``). +""" + +import logging +import os +import string +from collections.abc import ( + Iterable, + Iterator, +) +from dataclasses import ( + dataclass, + field, +) +from pathlib import Path +from typing import ( + TYPE_CHECKING, +) + +from galaxy.model import _get_datatypes_registry +from galaxy.tool_util.loader_directory import looks_like_a_tool +from galaxy.tool_util.toolbox.parser import ( + get_toolbox_parser, + ToolConfItem, + ToolConfSection, +) + +if TYPE_CHECKING: + from galaxy.config import GalaxyAppConfiguration + +log = logging.getLogger(__name__) + + +@dataclass +class DiscoveredTool: + """Information about a discovered tool file.""" + + path: str # Absolute path to tool file + tool_conf: str # Path to the tool_conf file that referenced this tool + tool_path: str | None # The tool_path from the tool_conf + guid: str | None = None # GUID for shed tools + is_shed_tool: bool = False + # Conf-level ``hidden="true"`` on the ```` element (NOT the XML + # body's ``