diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index b48a4514921..1bd2ecb3867 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -425,16 +425,19 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ :Description: - SQLAlchemy connection string for the tool source store, a - rebuildable cache of pre-parsed tool sources kept outside Galaxy's - main database. Multi-host deployments should point every Galaxy - process at the same URI, such as a SQLite file on a shared - filesystem. + SQLAlchemy connection string for storing pre-parsed tool sources. + The store is a rebuildable cache that lives outside Galaxy's + database. This URI is used by tool source storage code paths, + including the population script and lazy toolbox consumers. + Runtime use also requires a populated store and a toolbox consumer + configured to read from tool source storage. + By default, Galaxy uses a SQLite database at + ``/tool_sources.sqlite``. Multi-host deployments should + point every process at the same SQLAlchemy URI, such as a SQLite + file on a shared filesystem or a shared PostgreSQL database. Sample default ``sqlite:////tool_sources.sqlite``. - Populate the store with: python + To populate the store, run: python scripts/tool_source/populate_store.py - For details see - https://docs.galaxyproject.org/en/master/admin/tool_source_storage.html :Default: ``None`` :Type: str @@ -453,8 +456,19 @@ Each entry takes a SQLAlchemy ``url`` and an optional ``read_only: true`` flag. For SQLite connection-level read-only, use a SQLite URI with ``mode=ro&uri=true``. - For details see - https://docs.galaxyproject.org/en/master/admin/tool_source_storage.html + An entry may also declare a freshness probe via ``freshness``. The + populator stamps the probe's value into the store's persisted + index; at boot a matching value proves the store still covers the + tool tree and skips the per-tool coverage scan entirely. + ``freshness: cvmfs`` reads the CernVM-FS repository revision (one + extended-attribute syscall) from ``freshness_path``, defaulting to + the store's SQLite file location — the right probe for a store + published on CVMFS in the same transaction as the tools it + indexes. ``freshness: tool_confs`` hashes the local tool conf + files plus tool_dir directory mtimes; the default + (``tool_source_database_connection``) store always uses this + probe. Without ``freshness``, a named store is verified by the + coverage scan as before. :Default: ``None`` :Type: map @@ -5775,18 +5789,7 @@ maximum output retries"). custom_tool's producer keeps a budget of 0 because it runs its own reflection loop; a shared ``default`` block does not change that -- set ``custom_tool.retries`` - explicitly to override it. custom_tool also accepts - ``quality_critic_enabled`` (default false) to turn on the LLM - clarity/idiomaticity critic, and - ``container_recommendation_enabled`` (default false) to resolve - the produced tool's container to a verified quay.io biocontainer. - Container recommendation runs a dedicated container critic that - infers the tool's conda packages from its command and config - files, independently of ``quality_critic_enabled``; it adds an - extra model call plus an outbound network call to quay.io during - the agent turn. Example: inference_services: { custom_tool: { - quality_critic_enabled: true, container_recommendation_enabled: - true } } + explicitly to override it. :Default: ``None`` :Type: any diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index 87fd65a4459..a7ea56ea6f7 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -588,16 +588,19 @@ galaxy: # an attribute in the tag. #tool_path: tools - # SQLAlchemy connection string for the tool source store, a - # rebuildable cache of pre-parsed tool sources kept outside Galaxy's - # main database. Multi-host deployments should point every Galaxy - # process at the same URI, such as a SQLite file on a shared - # filesystem. + # SQLAlchemy connection string for storing pre-parsed tool sources. + # The store is a rebuildable cache that lives outside Galaxy's + # database. This URI is used by tool source storage code paths, + # including the population script and lazy toolbox consumers. Runtime + # use also requires a populated store and a toolbox consumer + # configured to read from tool source storage. + # By default, Galaxy uses a SQLite database at + # ``/tool_sources.sqlite``. Multi-host deployments should + # point every process at the same SQLAlchemy URI, such as a SQLite + # file on a shared filesystem or a shared PostgreSQL database. # Sample default ``sqlite:////tool_sources.sqlite``. - # Populate the store with: python + # To populate the store, run: python # scripts/tool_source/populate_store.py - # For details see - # https://docs.galaxyproject.org/en/master/admin/tool_source_storage.html #tool_source_database_connection: null # Optional named tool source stores referenced from individual @@ -609,8 +612,18 @@ galaxy: # Each entry takes a SQLAlchemy ``url`` and an optional ``read_only: # true`` flag. For SQLite connection-level read-only, use a SQLite URI # with ``mode=ro&uri=true``. - # For details see - # https://docs.galaxyproject.org/en/master/admin/tool_source_storage.html + # An entry may also declare a freshness probe via ``freshness``. The + # populator stamps the probe's value into the store's persisted index; + # at boot a matching value proves the store still covers the tool tree + # and skips the per-tool coverage scan entirely. ``freshness: cvmfs`` + # reads the CernVM-FS repository revision (one extended-attribute + # syscall) from ``freshness_path``, defaulting to the store's SQLite + # file location — the right probe for a store published on CVMFS in + # the same transaction as the tools it indexes. ``freshness: + # tool_confs`` hashes the local tool conf files plus tool_dir + # directory mtimes; the default (``tool_source_database_connection``) + # store always uses this probe. Without ``freshness``, a named store + # is verified by the coverage scan as before. #tool_source_stores: null # When true, use the LazyToolBox which loads tools on demand from the @@ -3135,17 +3148,7 @@ galaxy: # retries"). custom_tool's producer keeps a budget of 0 because it # runs its own reflection loop; a shared ``default`` block does not # change that -- set ``custom_tool.retries`` explicitly to override - # it. custom_tool also accepts ``quality_critic_enabled`` (default - # false) to turn on the LLM clarity/idiomaticity critic, and - # ``container_recommendation_enabled`` (default false) to resolve the - # produced tool's container to a verified quay.io biocontainer. - # Container recommendation runs a dedicated container critic that - # infers the tool's conda packages from its command and config files, - # independently of ``quality_critic_enabled``; it adds an extra model - # call plus an outbound network call to quay.io during the agent turn. - # Example: inference_services: { custom_tool: { - # quality_critic_enabled: true, container_recommendation_enabled: true - # } } + # it. #inference_services: null # YAML file with capability hints for agent inference models. Maps diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 9d0fd6f89b7..91fa452922a 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -357,6 +357,39 @@ mapping: For details see https://docs.galaxyproject.org/en/master/admin/tool_source_storage.html + An entry may also declare a freshness probe via ``freshness``. + The populator stamps the probe's value into the store's persisted + index; at boot a matching value proves the store still covers the + tool tree and skips the per-tool coverage scan entirely. + ``freshness: cvmfs`` reads the CernVM-FS repository revision (one + extended-attribute syscall) from ``freshness_path``, defaulting + to the store's SQLite file location — the right probe for a store + published on CVMFS in the same transaction as the tools it + indexes. ``freshness: tool_confs`` hashes the local tool conf + files plus tool_dir directory mtimes; the default + (``tool_source_database_connection``) store always uses this + probe. Without ``freshness``, a named store is verified by the + coverage scan as before. + + use_lazy_toolbox: + type: bool + required: false + desc: | + When true, use the LazyToolBox which loads tools on demand from the + tool source store. Otherwise (the default), the traditional eager + ToolBox is used and any per-conf ``store="..."`` attributes on + tool_conf files are ignored. Opt-in is explicit: a populated tool + source store does not flip a default deployment to lazy mode. + + lazy_toolbox_cache_size: + type: int + default: 500 + required: false + desc: | + Maximum number of fully constructed Tool objects the LazyToolBox keeps + in its in-memory LRU cache. Larger values reduce repeat parsing cost + for popular tools at the expense of memory. + tool_dependency_dir: type: str default: dependencies diff --git a/lib/galaxy/tools/lazy_toolbox.py b/lib/galaxy/tools/lazy_toolbox.py index 6be6751e822..03f908c6b84 100644 --- a/lib/galaxy/tools/lazy_toolbox.py +++ b/lib/galaxy/tools/lazy_toolbox.py @@ -572,6 +572,18 @@ class LazyToolBox(ToolBox): return False if not self._tool_index or not self._tool_index.entries: return True + # Freshness fast path: when every member store carries a probe and + # every probe value matches the token its populator stamped, the + # store provably covers the current tree — skip the conf walk (and + # its per-file existence stats) entirely. See + # ``galaxy.tools.source_store.freshness``. + fresh = self._store.index_is_fresh() + if fresh is True: + log.info("Tool source store freshness token matches; skipping index coverage scan") + return False + if fresh is False: + log.info("Tool source store freshness token mismatch; running populator") + return True try: stored_paths = self._store.list_source_paths() read_only_stores: set[str] = set() diff --git a/lib/galaxy/tools/source_store/factory.py b/lib/galaxy/tools/source_store/factory.py index 4469b115e35..411b906ea9f 100644 --- a/lib/galaxy/tools/source_store/factory.py +++ b/lib/galaxy/tools/source_store/factory.py @@ -3,9 +3,12 @@ import logging from typing import TYPE_CHECKING +from sqlalchemy.engine import make_url + from galaxy.tool_util.toolbox.parser import get_toolbox_parser from .composite import CompositeToolSourceStore from .freshness import ( + cvmfs_probe, FreshnessProbe, tool_confs_probe, ) @@ -36,6 +39,19 @@ def _build_default_store( return SqlAlchemyToolSourceStore(url=url, read_only=False, freshness_probe=tool_confs_probe(config)) +def _sqlite_database_path(url: str) -> str | None: + """On-disk file behind a sqlite URL, or None for other backends.""" + parsed = make_url(url) + if parsed.drivername.split("+")[0] != "sqlite": + return None + database = parsed.database + if not database or database == ":memory:": + return None + if database.startswith("file:"): + database = database[len("file:") :].split("?", 1)[0] + return database or None + + def _build_freshness_probe( name: str, spec: dict, @@ -46,8 +62,15 @@ def _build_freshness_probe( return None if freshness == "tool_confs": return tool_confs_probe(config) + if freshness == "cvmfs": + probe_path = spec.get("freshness_path") or _sqlite_database_path(spec["url"]) + if not probe_path: + raise ConfigurationError( + f"tool_source_stores[{name!r}] uses freshness: cvmfs with a non-file url; set freshness_path" + ) + return cvmfs_probe(probe_path) raise ConfigurationError( - f"tool_source_stores[{name!r}] freshness must be 'tool_confs' or 'none' (got {freshness!r})" + f"tool_source_stores[{name!r}] freshness must be 'cvmfs', 'tool_confs', or 'none' (got {freshness!r})" ) @@ -59,7 +82,7 @@ def build_named_store( """Build a single named store from a ``tool_source_stores`` entry. ``spec`` is the dict from galaxy.yml - a SQLAlchemy ``url`` plus - optional ``read_only`` and ``freshness`` keys. + optional ``read_only``, ``freshness``, and ``freshness_path`` keys. Named stores get no probe unless one is declared: a store populated on a different host (the CVMFS publishing model) would never match a locally-computed conf hash, so ``tool_confs`` cannot be the default @@ -86,7 +109,7 @@ def _collect_per_conf_store_names(config: "GalaxyAppConfiguration") -> set[str]: try: parser = get_toolbox_parser(path) except Exception as e: - log.debug(f"skipping tool conf {path}: {e}") + log.error(f"skipping tool conf {path}: {e}") continue store = parser.parse_store_name() if store: diff --git a/lib/galaxy/tools/source_store/freshness.py b/lib/galaxy/tools/source_store/freshness.py index 62b6b6ee1e7..2e78a563500 100644 --- a/lib/galaxy/tools/source_store/freshness.py +++ b/lib/galaxy/tools/source_store/freshness.py @@ -7,18 +7,21 @@ compares. A match certifies the store still covers the current tree, so boot skips the per-path coverage scan (and the populate it would trigger). A mismatch is always safe — it only falls back to scanning/populating. -The built-in probe is ``tool_confs``: md5 over the tool and data-manager -conf file contents, plus the (recursive) directory mtimes of any -``tool_dir`` entries they declare. This captures tool -additions/removals/renames — the same class of drift the coverage scan -detects — without touching individual tool files. In-place edits to a -tool's XML are invisible to both, by design: content changes are the -incremental populate's job (raw-file md5), not the coverage check's. -Wired to the default (writable) store automatically. +Two probe kinds exist: -Read-only stores need no probe for boot freshness — they are trusted as -published (see ``SqlAlchemyToolSourceStore.index_is_fresh``); a probe on -such a store only feeds the watcher's change detection. +- ``tool_confs``: md5 over the tool and data-manager conf file contents, + plus the (recursive) directory mtimes of any ``tool_dir`` entries they + declare. This captures tool additions/removals/renames — the same class + of drift the coverage scan detects — without touching individual tool + files. In-place edits to a tool's XML are invisible to both, by design: + content changes are the incremental populate's job (raw-file md5), not + the coverage check's. Wired to the default (writable) store + automatically. +- ``cvmfs``: the CernVM-FS repository revision, read from the + ``user.revision`` extended attribute the CVMFS client exposes on the + mount point. One syscall covers every file in the repository. For a + store whose sqlite bundle is published in the same CVMFS transaction as + the tools it indexes, a matching revision is a hard consistency proof. """ import hashlib @@ -96,3 +99,38 @@ def tool_confs_token(config: "GalaxyAppConfiguration") -> str: def tool_confs_probe(config: "GalaxyAppConfiguration") -> FreshnessProbe: return lambda: tool_confs_token(config) + + +def _os_getxattr(path: str, attribute: str) -> bytes: + # ``os.getxattr`` only exists on Linux; CVMFS deployments are Linux. + getxattr = getattr(os, "getxattr", None) + if getxattr is None: + raise FreshnessProbeError("extended attributes are not supported on this platform") + return getxattr(path, attribute) + + +def cvmfs_revision_token(path: str, _getxattr: Callable[[str, str], bytes] = _os_getxattr) -> str: + """CVMFS repository revision token for the repository containing ``path``. + + The CVMFS client exposes repository metadata as extended attributes on + the mount point, so ascend from ``path`` until ``user.revision`` + answers. Raises :class:`FreshnessProbeError` when no ancestor exposes + it — ``path`` isn't on CVMFS, or the repository isn't mounted (in + which case its tools are unreadable anyway, and "not fresh" is the + right verdict). + """ + probe_path = os.path.abspath(path) + while True: + try: + revision = _getxattr(probe_path, "user.revision") + except OSError: + parent = os.path.dirname(probe_path) + if parent == probe_path: + raise FreshnessProbeError(f"no CVMFS revision xattr found on any ancestor of {path}") + probe_path = parent + continue + return f"cvmfs:{os.path.basename(probe_path)}:{revision.decode()}" + + +def cvmfs_probe(path: str) -> FreshnessProbe: + return lambda: cvmfs_revision_token(path) diff --git a/test/unit/app/tools/source_store/test_freshness.py b/test/unit/app/tools/source_store/test_freshness.py index d991ddc4aca..e25028740d7 100644 --- a/test/unit/app/tools/source_store/test_freshness.py +++ b/test/unit/app/tools/source_store/test_freshness.py @@ -2,8 +2,11 @@ import logging +import pytest + from galaxy.tools.source_store.composite import CompositeToolSourceStore from galaxy.tools.source_store.freshness import ( + cvmfs_revision_token, FreshnessProbeError, tool_confs_token, ) @@ -50,6 +53,24 @@ def test_tool_confs_token_sees_tool_dir_membership_changes(tmp_path): assert tool_confs_token(cfg) != after_top_level +def test_cvmfs_revision_token_ascends_to_the_mount_root(): + def fake_getxattr(path, attribute): + if path == "/cvmfs/main.galaxyproject.org" and attribute == "user.revision": + return b"1042" + raise OSError(61, "no attribute") + + token = cvmfs_revision_token("/cvmfs/main.galaxyproject.org/galaxy/store.sqlite", _getxattr=fake_getxattr) + assert token == "cvmfs:main.galaxyproject.org:1042" + + +def test_cvmfs_revision_token_raises_off_cvmfs(): + def fake_getxattr(path, attribute): + raise OSError(61, "no attribute") + + with pytest.raises(FreshnessProbeError): + cvmfs_revision_token("/plain/local/path", _getxattr=fake_getxattr) + + def test_index_is_fresh_tracks_probe(tmp_path): current = {"token": "confs:a"} store = SqlAlchemyToolSourceStore(url=f"sqlite:///{tmp_path}/a.sqlite", freshness_probe=lambda: current["token"]) @@ -71,7 +92,7 @@ def test_index_is_fresh_false_when_probe_fails(tmp_path): raise FreshnessProbeError("repo not mounted") store = SqlAlchemyToolSourceStore(url=f"sqlite:///{tmp_path}/a.sqlite", freshness_probe=broken_probe) - store.store_index(ToolIndex(freshness_token="bundle:1")) + store.store_index(ToolIndex(freshness_token="cvmfs:r:1")) assert store.index_is_fresh() is False @@ -81,46 +102,30 @@ def _stamped_store(path, token, probe_token, read_only=False): def test_composite_fresh_when_all_members_fresh(tmp_path): - ro = _stamped_store(tmp_path / "ro.sqlite", "bundle:1", "bundle:1", read_only=True) + ro = _stamped_store(tmp_path / "ro.sqlite", "cvmfs:r:1", "cvmfs:r:1", read_only=True) rw = _stamped_store(tmp_path / "rw.sqlite", "confs:x", "confs:x") composite = CompositeToolSourceStore(members=[("ro", ro), ("rw", rw)], default="rw") assert composite.index_is_fresh() is True def test_composite_stale_writable_member_wins(tmp_path): - ro = _stamped_store(tmp_path / "ro.sqlite", "bundle:1", "bundle:1", read_only=True) + ro = _stamped_store(tmp_path / "ro.sqlite", "cvmfs:r:1", "cvmfs:r:1", read_only=True) rw = _stamped_store(tmp_path / "rw.sqlite", "confs:x", "confs:y") composite = CompositeToolSourceStore(members=[("ro", ro), ("rw", rw)], default="rw") assert composite.index_is_fresh() is False -def test_read_only_store_trusts_schema_valid_index_over_probe(tmp_path): - # Stamped token and probe value disagree, but a read-only store is - # trusted whenever its index loads — the probe only feeds the watcher. - ro = _stamped_store(tmp_path / "ro.sqlite", "bundle:1", "bundle:2", read_only=True) - assert ro.index_is_fresh() is True - - -def test_read_only_store_without_index_is_not_fresh(tmp_path): - SqlAlchemyToolSourceStore(url=f"sqlite:///{tmp_path}/ro.sqlite") - ro = SqlAlchemyToolSourceStore( - url=f"sqlite:///{tmp_path}/ro.sqlite", read_only=True, freshness_probe=lambda: "bundle:1" - ) - assert ro.index_is_fresh() is False - - -def test_composite_read_only_member_without_index_warns_but_stays_fresh(tmp_path, caplog): - SqlAlchemyToolSourceStore(url=f"sqlite:///{tmp_path}/ro.sqlite") - ro = SqlAlchemyToolSourceStore(url=f"sqlite:///{tmp_path}/ro.sqlite", read_only=True) +def test_composite_stale_read_only_member_warns_but_stays_fresh(tmp_path, caplog): + ro = _stamped_store(tmp_path / "ro.sqlite", "cvmfs:r:1", "cvmfs:r:2", read_only=True) rw = _stamped_store(tmp_path / "rw.sqlite", "confs:x", "confs:x") composite = CompositeToolSourceStore(members=[("ro", ro), ("rw", rw)], default="rw") with caplog.at_level(logging.WARNING): assert composite.index_is_fresh() is True - assert "no loadable index" in caplog.text + assert "repopulated upstream" in caplog.text def test_composite_member_without_probe_downgrades_to_none(tmp_path): - ro = _stamped_store(tmp_path / "ro.sqlite", "bundle:1", "bundle:1", read_only=True) + ro = _stamped_store(tmp_path / "ro.sqlite", "cvmfs:r:1", "cvmfs:r:1", read_only=True) rw = SqlAlchemyToolSourceStore(url=f"sqlite:///{tmp_path}/rw.sqlite") composite = CompositeToolSourceStore(members=[("ro", ro), ("rw", rw)], default="rw") assert composite.index_is_fresh() is None