diff --git a/doc/source/admin/galaxy_options.rst b/doc/source/admin/galaxy_options.rst index 2f9ab86a54d..833afcd1133 100644 --- a/doc/source/admin/galaxy_options.rst +++ b/doc/source/admin/galaxy_options.rst @@ -420,6 +420,46 @@ :Type: str +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +``tool_source_database_connection`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + SQLAlchemy connection string for storing pre-parsed tool sources. + The store is a rebuildable cache that lives outside Galaxy's + database. Defining this URI does not make Galaxy's eager toolbox + startup use the store; it is opened by code paths that explicitly + use tool source storage, such as the population script and lazy + toolbox consumers. + By default, Galaxy uses a SQLite database at + ``/tool_sources.sqlite``. Multi-host deployments should + point every process at the same SQLAlchemy URI, such as a SQLite + file on a shared filesystem or a shared PostgreSQL database. + Sample default ``sqlite:////tool_sources.sqlite``. + To populate the store, run: python + scripts/tool_source/populate_store.py +:Default: ``None`` +:Type: str + + +~~~~~~~~~~~~~~~~~~~~~~ +``tool_source_stores`` +~~~~~~~~~~~~~~~~~~~~~~ + +:Description: + Optional named tool source stores referenced from individual + tool_conf files via a top-level ``store=""`` attribute (XML) + or ``store: `` key (YAML). When any tool_conf opts in, the + process composes its named store with the default + (``tool_source_database_connection``) store at runtime, with reads + tried in declared order and writes always landing on the default. + Each entry takes a SQLAlchemy ``url`` and an optional ``read_only: + true`` flag. For SQLite connection-level read-only, use a SQLite + URI with ``mode=ro&uri=true``. +:Default: ``None`` +:Type: map + + ~~~~~~~~~~~~~~~~~~~~~~~ ``tool_dependency_dir`` ~~~~~~~~~~~~~~~~~~~~~~~ diff --git a/doc/source/admin/tool_source_storage.rst b/doc/source/admin/tool_source_storage.rst index 5249f8d64da..4ccf6243715 100644 --- a/doc/source/admin/tool_source_storage.rst +++ b/doc/source/admin/tool_source_storage.rst @@ -28,31 +28,30 @@ Configuration Tool source storage is configured in ``galaxy.yml``. The following options are available: -Backend Selection -^^^^^^^^^^^^^^^^^ +Default Store +^^^^^^^^^^^^^ .. code-block:: yaml galaxy: - # Backend for storing tool sources ('sqlite', alias 'sqlalchemy') - tool_source_store: sqlite + # SQLAlchemy URI for storing tool sources. + tool_source_database_connection: sqlite:////srv/galaxy/tool_sources.sqlite -The store lives in a standalone database — a SQLite file under -``tool_source_disk_path`` (default: ``/tool_sources``) — never in -Galaxy's own database. It is a rebuildable cache: deleting it costs one -populator run. Nothing is initialized unless ``use_lazy_toolbox`` is -enabled. +The store lives in a standalone database - a SQLite file under +``/tool_sources.sqlite`` by default - never in Galaxy's own database. It is +a rebuildable cache: deleting it costs one populator run. Defining this URI +does not make Galaxy's eager toolbox startup use the store; it is opened by +code paths that explicitly use tool source storage, such as the population +script and lazy toolbox consumers. .. code-block:: yaml galaxy: - tool_source_store: sqlite - tool_source_disk_path: /path/to/tool_sources.sqlite + tool_source_database_connection: postgresql://galaxy@db.example.org/tool_sources Multi-host deployments must point every Galaxy process (web workers *and* -job handlers) at the same store: either a ``tool_source_disk_path`` on a -shared filesystem, or a named read-only store with a full SQLAlchemy -``url`` layered via ``tool_source_stores`` (see below). +job handlers) at the same store, such as a SQLite file on a shared filesystem +or a shared database URI. Per-conf Store Routing (CVMFS Recipe) ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -63,23 +62,20 @@ read-only SQLite bundle on CVMFS alongside a tool_conf, so worker processes can resolve every tool in that conf with local-cached lookups instead of one network round-trip per JSON file. -Declare the named stores under the new top-level ``tool_source_stores`` -key in ``galaxy.yml``. The ``sqlalchemy`` backend takes either a SQLAlchemy -``url`` or a ``path`` shortcut that builds a SQLite URL. SQLite is the -typical choice for CVMFS bundles (single self-contained file), but any -SQLAlchemy-supported database works: +Declare the named stores under the top-level ``tool_source_stores`` key in +``galaxy.yml``. Each entry takes a SQLAlchemy ``url`` and optional +``read_only`` flag. SQLite is the typical choice for CVMFS bundles (single +self-contained file), but any SQLAlchemy-supported database works: .. code-block:: yaml galaxy: - tool_source_store: sqlite # the writable default + tool_source_database_connection: sqlite:////srv/galaxy/tool_sources.sqlite tool_source_stores: cvmfs_main: - backend: sqlalchemy - path: /cvmfs/example.org/tools/sources.sqlite + url: sqlite:///file:/cvmfs/example.org/tools/sources.sqlite?mode=ro&uri=true read_only: true site_shared: - backend: sqlalchemy url: postgresql://galaxy_ro@db.example.org/tool_sources read_only: true @@ -115,7 +111,9 @@ it, ``populate_store.py`` populates **every writable store** referenced from a tool_conf in the same run. Once the bundle is in place on CVMFS (or any read-only mount), restart -Galaxy. +Galaxy. The ``read_only: true`` flag prevents Galaxy from writing through that +store. For SQLite connection-level read-only, use ``mode=ro&uri=true`` in the +SQLite URI as shown above. Populating the Tool Source Store -------------------------------- diff --git a/doc/source/dev/tool_source_storage.rst b/doc/source/dev/tool_source_storage.rst index 323271fadb8..6d8025482f6 100644 --- a/doc/source/dev/tool_source_storage.rst +++ b/doc/source/dev/tool_source_storage.rst @@ -31,7 +31,7 @@ Module Layout lib/galaxy/tools/source_store/ __init__.py ToolSourceStore ABC, StoredToolSource, build_tool_source_store() - sqlalchemy.py SqlAlchemyToolSourceStore (any SA URL; sqlite shortcut) + sqlalchemy.py SqlAlchemyToolSourceStore (any SQLAlchemy URL) composite.py CompositeToolSourceStore (per-conf routing, merged index) index.py ToolIndex, ToolIndexEntry (the lightweight metadata) search.py ToolWhooshIndex (Whoosh search index built from a ToolIndex) @@ -73,11 +73,11 @@ Backend Abstraction - ``get_stats()`` — backend-specific stats (count, size, backend name). ``build_tool_source_store(config)`` is the only entry point used -by Galaxy. It inspects ``config.tool_source_store`` to pick the backend -(currently ``sqlalchemy``, alias ``sqlite``). -``ConfigurationError`` is raised for unknown backends or missing required -settings; it is allowed to propagate up so misconfiguration fails fast at -startup. +by Galaxy. It builds the default store from +``config.tool_source_database_connection`` and uses the same SQLAlchemy-backed +store implementation for all configured URIs. ``ConfigurationError`` is raised +for missing required settings and is allowed to propagate up so +misconfiguration fails fast at startup. The ABC defines a ``read_only: bool`` class attribute (default ``False``). ``ReadOnlyStoreError`` is raised by mutating methods of stores that opted @@ -115,11 +115,12 @@ case. The ``sqlalchemy`` backend (``sqlalchemy.py``) was added to make this useful for CVMFS: a single self-contained ``.sqlite`` file, opened with its own SQLAlchemy ``MetaData`` (independent of ``galaxy.model``) so the -file is portable, and openable with ``mode=ro&uri=true`` for read-only -mounts. Despite the name, the backend is not sqlite-specific — pass any -SQLAlchemy ``url`` (Postgres, MySQL, …) instead of ``path``. Auto schema -creation runs on first open; on remote backends operators may prefer to -manage migrations explicitly. +file is portable, and openable with a SQLite URI such as +``sqlite:///file:/cvmfs/example.org/tools/sources.sqlite?mode=ro&uri=true`` +for read-only mounts. Despite the name, the backend is not sqlite-specific - +pass any SQLAlchemy URL (Postgres, MySQL, ...). Auto schema creation runs on +first open; on remote backends operators may prefer to manage migrations +explicitly. Per-conf populator routing ^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index a84847149ba..3a421fb1a0a 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -290,8 +290,7 @@ class MockAppConfig(GalaxyDataTestConfig, CommonConfigurationMixin): self.track_jobs_in_database = False self.amqp_internal_connection = None self.tool_configs = [] - self.tool_source_store = "sqlite" - self.tool_source_disk_path = os.path.join(self.data_dir, "tool_sources") + self.tool_source_database_connection = f"sqlite:///{os.path.join(self.data_dir, 'tool_sources.sqlite')}" self.tool_source_stores = None self.manage_dependency_relationships = False self.enable_tool_shed_check = False diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index 8d0b78f7628..322905a2994 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -850,6 +850,8 @@ class GalaxyAppConfiguration(GalaxyAppConfigurationAttributes, BaseAppConfigurat if not self.database_connection: # Provide default if not supplied by user db_path = self._in_data_dir("universe.sqlite") self.database_connection = f"sqlite:///{db_path}?isolation_level=IMMEDIATE" + if not self.tool_source_database_connection: + self.tool_source_database_connection = f"sqlite:///{self._in_data_dir('tool_sources.sqlite')}" self.database_engine_options = get_database_engine_options(kwargs) self.database_encoding = kwargs.get("database_encoding") # Create new databases with this encoding self.thread_local_log = None @@ -1357,6 +1359,7 @@ class GalaxyAppConfiguration(GalaxyAppConfigurationAttributes, BaseAppConfigurat try_parsing(self.database_connection, "database_connection") try_parsing(self.install_database_connection, "install_database_connection") + try_parsing(self.tool_source_database_connection, "tool_source_database_connection") if self.interactivetoolsproxy_map is not None: try_parsing(self.interactivetoolsproxy_map, "interactivetoolsproxy_map") try_parsing(self.amqp_internal_connection, "amqp_internal_connection") diff --git a/lib/galaxy/config/_galaxy_config_schema_attributes.py b/lib/galaxy/config/_galaxy_config_schema_attributes.py index 965af791522..0aa51434645 100644 --- a/lib/galaxy/config/_galaxy_config_schema_attributes.py +++ b/lib/galaxy/config/_galaxy_config_schema_attributes.py @@ -37,6 +37,8 @@ class GalaxyAppConfigurationAttributes: migrated_tools_config: str integrated_tool_panel_config: str tool_path: str + tool_source_database_connection: str | None + tool_source_stores: Any tool_dependency_dir: str | None dependency_resolvers_config_file: str conda_prefix: str | None diff --git a/lib/galaxy/config/sample/galaxy.yml.sample b/lib/galaxy/config/sample/galaxy.yml.sample index a818492b721..6e5a8d2613b 100644 --- a/lib/galaxy/config/sample/galaxy.yml.sample +++ b/lib/galaxy/config/sample/galaxy.yml.sample @@ -588,6 +588,32 @@ galaxy: # an attribute in the tag. #tool_path: tools + # SQLAlchemy connection string for storing pre-parsed tool sources. + # The store is a rebuildable cache that lives outside Galaxy's + # database. Defining this URI does not make Galaxy's eager toolbox + # startup use the store; it is opened by code paths that explicitly + # use tool source storage, such as the population script and lazy + # toolbox consumers. + # By default, Galaxy uses a SQLite database at + # ``/tool_sources.sqlite``. Multi-host deployments should + # point every process at the same SQLAlchemy URI, such as a SQLite + # file on a shared filesystem or a shared PostgreSQL database. + # Sample default ``sqlite:////tool_sources.sqlite``. + # To populate the store, run: python + # scripts/tool_source/populate_store.py + #tool_source_database_connection: null + + # Optional named tool source stores referenced from individual + # tool_conf files via a top-level ``store=""`` attribute (XML) + # or ``store: `` key (YAML). When any tool_conf opts in, the + # process composes its named store with the default + # (``tool_source_database_connection``) store at runtime, with reads + # tried in declared order and writes always landing on the default. + # Each entry takes a SQLAlchemy ``url`` and an optional ``read_only: + # true`` flag. For SQLite connection-level read-only, use a SQLite URI + # with ``mode=ro&uri=true``. + #tool_source_stores: null + # Various dependency resolver configuration parameters will have # defaults set relative to this path, such as the default conda # prefix, default Galaxy packages path, legacy tool shed dependencies diff --git a/lib/galaxy/config/schemas/config_schema.yml b/lib/galaxy/config/schemas/config_schema.yml index 43d7b04d50d..66ebc74c75b 100644 --- a/lib/galaxy/config/schemas/config_schema.yml +++ b/lib/galaxy/config/schemas/config_schema.yml @@ -322,33 +322,26 @@ mapping: Other tool config files must include the tool_path as an attribute in the tag. - tool_source_store: + tool_source_database_connection: type: str - default: sqlite required: false desc: | - Backend for storing pre-parsed tool sources. The store is a - rebuildable cache that lives outside Galaxy's database — it is - only initialized when ``use_lazy_toolbox`` is enabled. + SQLAlchemy connection string for storing pre-parsed tool sources. + The store is a rebuildable cache that lives outside Galaxy's + database. Defining this URI does not make Galaxy's eager toolbox + startup use the store; it is opened by code paths that explicitly + use tool source storage, such as the population script and lazy + toolbox consumers. - The 'sqlite' (alias 'sqlalchemy') backend keeps the store in a - standalone database at ``tool_source_disk_path``. Multi-host - deployments should point every process at a shared store — either - a ``tool_source_disk_path`` on a shared filesystem or a named - store with a full SQLAlchemy ``url`` via ``tool_source_stores``. + By default, Galaxy uses a SQLite database at + ``/tool_sources.sqlite``. Multi-host deployments should point + every process at the same SQLAlchemy URI, such as a SQLite file on + a shared filesystem or a shared PostgreSQL database. + + Sample default ``sqlite:////tool_sources.sqlite``. To populate the store, run: python scripts/tool_source/populate_store.py - tool_source_disk_path: - type: str - default: tool_sources - path_resolves_to: data_dir - required: false - desc: | - Filesystem path used by the ``sqlalchemy`` backend — the path is - converted to a SQLite URL. Full SQLAlchemy URLs are available for - named stores via ``tool_source_stores``. - tool_source_stores: type: map required: false @@ -357,24 +350,12 @@ mapping: tool_conf files via a top-level ``store=""`` attribute (XML) or ``store: `` key (YAML). When any tool_conf opts in, the process composes its named store with the default - (``tool_source_store``) backend at runtime, with reads tried in - declared order and writes always landing on the default. + (``tool_source_database_connection``) store at runtime, with reads + tried in declared order and writes always landing on the default. - Each entry takes a ``backend`` (``sqlalchemy``, alias - ``sqlite``) plus the backend-specific settings, and an optional - ``read_only: true`` flag (typically used for CVMFS-shipped sqlite - bundles). - - The ``sqlalchemy`` backend accepts either a SQLAlchemy ``url`` or - a ``path`` shortcut that builds a SQLite URL. - - Example:: - - tool_source_stores: - cvmfs_main: - backend: sqlalchemy - path: /cvmfs/example.org/tools/sources.sqlite - read_only: true + Each entry takes a SQLAlchemy ``url`` and an optional + ``read_only: true`` flag. For SQLite connection-level read-only, + use a SQLite URI with ``mode=ro&uri=true``. tool_dependency_dir: type: str diff --git a/lib/galaxy/tools/source_store/factory.py b/lib/galaxy/tools/source_store/factory.py index 5de04fbc6ca..2b001370a08 100644 --- a/lib/galaxy/tools/source_store/factory.py +++ b/lib/galaxy/tools/source_store/factory.py @@ -20,16 +20,11 @@ log = logging.getLogger(__name__) def _build_default_store( config: "GalaxyAppConfiguration", ) -> ToolSourceStore: - """Build the default store from top-level ``tool_source_*`` config.""" - backend = config.tool_source_store - - if backend in ("sqlalchemy", "sqlite"): - path = config.tool_source_disk_path - if path: - return SqlAlchemyToolSourceStore(path=path, read_only=False) - raise ConfigurationError(f"{backend!r} backend requires tool_source_disk_path") - - raise ConfigurationError(f"Unknown tool source store backend: {backend}") + """Build the default store from the top-level SQLAlchemy URI config.""" + url = config.tool_source_database_connection + if not url: + raise ConfigurationError("tool_source_database_connection is required") + return SqlAlchemyToolSourceStore(url=url, read_only=False) def build_named_store( @@ -38,22 +33,20 @@ def build_named_store( ) -> ToolSourceStore: """Build a single named store from a ``tool_source_stores`` entry. - ``spec`` is the dict from galaxy.yml — a ``backend`` plus its options - plus an optional ``read_only`` flag. + ``spec`` is the dict from galaxy.yml - a SQLAlchemy ``url`` plus an + optional ``read_only`` flag. """ if not isinstance(spec, dict): raise ConfigurationError(f"tool_source_stores[{name!r}] must be a mapping") - backend = spec.get("backend") + if "backend" in spec or "path" in spec: + raise ConfigurationError( + f"tool_source_stores[{name!r}] must use 'url'; 'backend' and 'path' are no longer supported" + ) + url = spec.get("url") + if not url: + raise ConfigurationError(f"tool_source_stores[{name!r}] requires a 'url'") read_only = bool(spec.get("read_only", False)) - - if backend in ("sqlalchemy", "sqlite"): - url = spec.get("url") - path = spec.get("path") - if not url and not path: - raise ConfigurationError(f"tool_source_stores[{name!r}] requires a 'url' or 'path'") - return SqlAlchemyToolSourceStore(url=url, path=path, read_only=read_only) - - raise ConfigurationError(f"tool_source_stores[{name!r}] has unknown backend {backend!r}") + return SqlAlchemyToolSourceStore(url=url, read_only=read_only) def _collect_per_conf_store_names(config: "GalaxyAppConfiguration") -> set[str]: diff --git a/lib/galaxy/tools/source_store/populator.py b/lib/galaxy/tools/source_store/populator.py index c0511c02cfd..bd3cfb80406 100644 --- a/lib/galaxy/tools/source_store/populator.py +++ b/lib/galaxy/tools/source_store/populator.py @@ -554,7 +554,7 @@ def populate_store_inline( after every store write succeeds, so peer Galaxy processes refresh their cached index. Shed-install and ``reset_shed_tools`` set this. """ - log.info(f"Building tool source stores (default backend: {config.tool_source_store})...") + log.info("Building tool source stores...") stores = _build_stores(config) conf_to_store = _build_conf_to_store_map(config) @@ -817,7 +817,7 @@ def watch_mode( properties = load_app_properties(config_file=config_file, config_section="galaxy") config = GalaxyAppConfiguration(**properties) - log.info(f"Building tool source store (backend: {config.tool_source_store})...") + log.info("Building tool source store...") store = build_tool_source_store(config) diff --git a/lib/galaxy/tools/source_store/sqlalchemy.py b/lib/galaxy/tools/source_store/sqlalchemy.py index 77ee7759e2e..ef8eeb5849e 100644 --- a/lib/galaxy/tools/source_store/sqlalchemy.py +++ b/lib/galaxy/tools/source_store/sqlalchemy.py @@ -5,9 +5,6 @@ A self-contained store that owns its own engine + metadata, decoupled from ``galaxy.model``. Works with any SQLAlchemy URL (sqlite, postgres, mysql, …); the SQLite single-file path is the typical use case (shippable on CVMFS) but nothing about the schema is sqlite-specific. - -For convenience the constructor accepts a ``path`` shortcut that builds -a SQLite URL — set ``url`` directly to use any other backend. """ import gzip @@ -28,6 +25,7 @@ from sqlalchemy import ( String, Text, ) +from sqlalchemy.engine import make_url from sqlalchemy.orm import ( DeclarativeBase, Mapped, @@ -92,55 +90,34 @@ class SqlAlchemyToolSourceStore(ToolSourceStore): openable without booting Galaxy. Args: - url: SQLAlchemy URL. Mutually exclusive with ``path``. - path: Convenience shortcut for SQLite — builds - ``sqlite:///{path}`` (or its read-only variant). + url: SQLAlchemy URL. read_only: If True, refuse all mutating operations with - :class:`ReadOnlyStoreError`. For SQLite, additionally opens - the file with ``mode=ro&uri=true`` (and requires the file to - exist). For other backends, read-only is enforced at the - Python level only — make sure the connection user lacks - write privileges if that matters. - - Either ``url`` or ``path`` must be provided. + :class:`ReadOnlyStoreError`. This is enforced at the Python + level only - make sure the connection user lacks write + privileges if that matters. For SQLite connection-level + read-only, use a URI such as + ``sqlite:///file:/path/to/store.sqlite?mode=ro&uri=true``. """ def __init__( self, - url: str | None = None, - path: str | None = None, + url: str, read_only: bool = False, ) -> None: - if (url is None) == (path is None): - raise ValueError("provide exactly one of url= or path=") self.url = url - self.path = path self.read_only = read_only self._cached_index: ToolIndex | None = None - if path is not None: - # SQLite shortcut — file-on-disk; honor the read-only flag at - # the connection level so accidental writes fail loudly. - if read_only: - if not os.path.exists(path): - raise FileNotFoundError(f"Read-only sqlite tool source store not found: {path}") - resolved_url = f"sqlite:///file:{path}?mode=ro&uri=true" - else: - os.makedirs(os.path.dirname(os.path.abspath(path)) or ".", exist_ok=True) - resolved_url = f"sqlite:///{path}" - else: - assert url is not None - resolved_url = url + self._ensure_sqlite_parent_directory(url) - self._engine = create_engine(resolved_url, future=True) - self._is_sqlite_file = path is not None + self._engine = create_engine(url, future=True) if not read_only and not self._is_remote_engine(): # Only auto-create schema on local/file backends. For shared # databases the operator should manage migrations explicitly # to avoid surprises. _metadata.create_all(self._engine) elif not read_only: - # Best-effort create_all on remote backends — harmless if + # Best-effort create_all on remote backends - harmless if # tables already exist. try: _metadata.create_all(self._engine) @@ -151,6 +128,17 @@ class SqlAlchemyToolSourceStore(ToolSourceStore): def _is_remote_engine(self) -> bool: return self._engine.url.drivername.split("+")[0] not in {"sqlite"} + def _ensure_sqlite_parent_directory(self, url: str) -> None: + parsed_url = make_url(url) + if parsed_url.drivername.split("+")[0] != "sqlite": + return + if str(parsed_url.query.get("uri", "")).lower() in {"true", "1", "yes"}: + return + database = parsed_url.database + if not database or database == ":memory:": + return + os.makedirs(os.path.dirname(os.path.abspath(database)) or ".", exist_ok=True) + # --- helpers -------------------------------------------------------- def _session(self) -> Session: @@ -158,8 +146,7 @@ class SqlAlchemyToolSourceStore(ToolSourceStore): def _ensure_writable(self) -> None: if self.read_only: - target = self.path or self.url - raise ReadOnlyStoreError(f"tool source store at {target} is read-only") + raise ReadOnlyStoreError(f"tool source store at {self.url} is read-only") # --- ToolSourceStore: per-source ops -------------------------------- @@ -303,8 +290,7 @@ class SqlAlchemyToolSourceStore(ToolSourceStore): self._cached_index = ToolIndex.from_dict(payload) return self._cached_index except Exception as e: - target = self.path or self.url - log.warning(f"Failed to decode tool index from store {target}: {e}") + log.warning(f"Failed to decode tool index from store {self.url}: {e}") return None def update_index_entry(self, entry: ToolIndexEntry) -> None: diff --git a/test/unit/app/tools/source_store/test_composite_store.py b/test/unit/app/tools/source_store/test_composite_store.py index 1ee4694538b..7a4df9fcc1c 100644 --- a/test/unit/app/tools/source_store/test_composite_store.py +++ b/test/unit/app/tools/source_store/test_composite_store.py @@ -30,10 +30,14 @@ def _src(hash, tool_id="t", version="1"): ) +def _sqlite_url(path): + return f"sqlite:///{path}" + + def test_priority_order_first_hit_wins(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) # Same hash, different tool_id payloads, to prove which member answered. a.store(_src("dup", tool_id="from_a")) b.store(_src("dup", tool_id="from_b")) @@ -45,8 +49,8 @@ def test_priority_order_first_hit_wins(two_paths): def test_writes_go_to_default(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) composite = CompositeToolSourceStore(members=[("a", a), ("b", b)], default="b") composite.store(_src("h1")) assert b.exists("h1") @@ -55,17 +59,17 @@ def test_writes_go_to_default(two_paths): def test_default_must_not_be_read_only(two_paths): pa, pb = two_paths - rw = SqliteToolSourceStore(path=pa) + rw = SqliteToolSourceStore(url=_sqlite_url(pa)) rw.store(_src("seed")) # so the file exists - ro = SqliteToolSourceStore(path=pa, read_only=True) + ro = SqliteToolSourceStore(url=_sqlite_url(pa), read_only=True) with pytest.raises(ValueError): CompositeToolSourceStore(members=[("ro", ro), ("rw", rw)], default="ro") def test_list_all_dedupes_across_members(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) a.store(_src("h1")) a.store(_src("dup")) b.store(_src("dup")) @@ -77,8 +81,8 @@ def test_list_all_dedupes_across_members(two_paths): def test_load_index_merges_and_dedupes(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) a.store_index( ToolIndex( entries={ @@ -105,8 +109,8 @@ def test_load_index_merges_and_dedupes(two_paths): def test_load_index_returns_none_when_no_member_has_one(two_paths): pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) composite = CompositeToolSourceStore(members=[("a", a), ("b", b)], default="b") assert composite.load_index() is None @@ -119,8 +123,8 @@ def test_invalidate_fans_out(two_paths): # cache hides the new entry; with composite.invalidate_index_cache() # the next load surfaces it. pa, pb = two_paths - a = SqliteToolSourceStore(path=pa) - b = SqliteToolSourceStore(path=pb) + a = SqliteToolSourceStore(url=_sqlite_url(pa)) + b = SqliteToolSourceStore(url=_sqlite_url(pb)) a.store_index(ToolIndex(entries={"x": ToolIndexEntry(id="x")})) b.store_index(ToolIndex(entries={"y": ToolIndexEntry(id="y")})) a.load_index() @@ -128,10 +132,10 @@ def test_invalidate_fans_out(two_paths): # Out-of-band update via a fresh handle so the existing instance's # cache stays primed with the old value. - SqliteToolSourceStore(path=pa).store_index( + SqliteToolSourceStore(url=_sqlite_url(pa)).store_index( ToolIndex(entries={"x": ToolIndexEntry(id="x"), "x2": ToolIndexEntry(id="x2")}) ) - SqliteToolSourceStore(path=pb).store_index( + SqliteToolSourceStore(url=_sqlite_url(pb)).store_index( ToolIndex(entries={"y": ToolIndexEntry(id="y"), "y2": ToolIndexEntry(id="y2")}) ) diff --git a/test/unit/app/tools/source_store/test_sqlite_store.py b/test/unit/app/tools/source_store/test_sqlite_store.py index 6ef5341512b..6695f7ddd91 100644 --- a/test/unit/app/tools/source_store/test_sqlite_store.py +++ b/test/unit/app/tools/source_store/test_sqlite_store.py @@ -33,8 +33,12 @@ def _source(hash="h1", tool_id="t1", version="1.0"): ) +def _sqlite_url(path): + return f"sqlite:///{path}" + + def test_store_and_retrieve_round_trip(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) store.store(_source()) got = store.get("h1") assert got is not None @@ -45,7 +49,7 @@ def test_store_and_retrieve_round_trip(sqlite_path): def test_get_by_tool_id_filters_by_version(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) store.store(_source(hash="h1", tool_id="t1", version="1.0")) store.store(_source(hash="h2", tool_id="t1", version="2.0")) assert {s.tool_version for s in store.get_by_tool_id("t1")} == {"1.0", "2.0"} @@ -53,7 +57,7 @@ def test_get_by_tool_id_filters_by_version(sqlite_path): def test_delete_returns_false_for_missing(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) assert store.delete("nope") is False store.store(_source()) assert store.delete("h1") is True @@ -61,7 +65,7 @@ def test_delete_returns_false_for_missing(sqlite_path): def test_index_round_trip(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) idx = ToolIndex(entries={"t1": ToolIndexEntry(id="t1", name="T1")}) store.store_index(idx) store.invalidate_index_cache() @@ -72,11 +76,11 @@ def test_index_round_trip(sqlite_path): def test_read_only_refuses_writes(sqlite_path): - rw = SqliteToolSourceStore(path=sqlite_path) + rw = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) rw.store(_source()) rw.store_index(ToolIndex(entries={"t1": ToolIndexEntry(id="t1", name="T1")})) - ro = SqliteToolSourceStore(path=sqlite_path, read_only=True) + ro = SqliteToolSourceStore(url=_sqlite_url(sqlite_path), read_only=True) assert ro.read_only is True fetched = ro.get("h1") assert fetched is not None @@ -89,14 +93,8 @@ def test_read_only_refuses_writes(sqlite_path): ro.store_index(ToolIndex()) -def test_read_only_missing_file_raises(tmp_path): - missing = tmp_path / "nope.sqlite" - with pytest.raises(FileNotFoundError): - SqliteToolSourceStore(path=str(missing), read_only=True) - - def test_get_stats_reports_backend_and_url(sqlite_path): - store = SqliteToolSourceStore(path=sqlite_path) + store = SqliteToolSourceStore(url=_sqlite_url(sqlite_path)) stats = store.get_stats() assert stats["backend"] == "sqlalchemy" assert stats["url"].startswith("sqlite:///") diff --git a/test/unit/app/tools/source_store/test_stores.py b/test/unit/app/tools/source_store/test_stores.py index 9dbb97de9e9..e5630d51393 100644 --- a/test/unit/app/tools/source_store/test_stores.py +++ b/test/unit/app/tools/source_store/test_stores.py @@ -7,6 +7,7 @@ factory / per-conf routing on top of it. import pytest from galaxy.tools.source_store import ( + build_named_store, build_tool_source_store, ConfigurationError, StoredToolSource, @@ -26,11 +27,15 @@ class FakeConfig: setattr(self, key, value) +def _sqlite_url(path): + return f"sqlite:///{path}" + + class TestSqlAlchemyBackend: """Tests for the sqlalchemy/sqlite backend.""" def test_sqlalchemy_store_basic_operations(self, tmp_path): - store = SqlAlchemyToolSourceStore(path=str(tmp_path / "ts.sqlite")) + store = SqlAlchemyToolSourceStore(url=_sqlite_url(tmp_path / "ts.sqlite")) tool_source = StoredToolSource( hash="sa_test_hash_123", @@ -50,7 +55,7 @@ class TestSqlAlchemyBackend: assert not store.exists("sa_test_hash_123") def test_sqlalchemy_identical_content_keeps_row_per_source_path(self, tmp_path): - store = SqlAlchemyToolSourceStore(path=str(tmp_path / "twins.sqlite")) + store = SqlAlchemyToolSourceStore(url=_sqlite_url(tmp_path / "twins.sqlite")) for path in ("/galaxy/tools/a/upload.xml", "/galaxy/tools/b/upload.xml"): store.store( StoredToolSource( @@ -68,7 +73,7 @@ class TestSqlAlchemyBackend: assert store.get_by_source_path("/galaxy/tools/a/upload.xml") is None def test_count_tracks_store_and_delete(self, tmp_path): - store = SqlAlchemyToolSourceStore(path=str(tmp_path / "count.sqlite")) + store = SqlAlchemyToolSourceStore(url=_sqlite_url(tmp_path / "count.sqlite")) assert store.count() == 0 store.store( StoredToolSource( @@ -84,7 +89,7 @@ class TestSqlAlchemyBackend: assert store.count() == 0 def test_changed_content_updates_path_row_in_place(self, tmp_path): - store = SqlAlchemyToolSourceStore(path=str(tmp_path / "edited.sqlite")) + store = SqlAlchemyToolSourceStore(url=_sqlite_url(tmp_path / "edited.sqlite")) path = "/galaxy/tools/edited.xml" def _stored(hash, raw): @@ -105,7 +110,7 @@ class TestSqlAlchemyBackend: assert not store.exists("edited_hash_v1") def test_pathless_sources_dedupe_on_hash(self, tmp_path): - store = SqlAlchemyToolSourceStore(path=str(tmp_path / "pathless.sqlite")) + store = SqlAlchemyToolSourceStore(url=_sqlite_url(tmp_path / "pathless.sqlite")) for _ in range(2): store.store( StoredToolSource( @@ -121,7 +126,7 @@ class TestSqlAlchemyBackend: assert not store.exists("pathless_hash") def test_remove_index_entry_persists_removal(self, tmp_path): - store = SqlAlchemyToolSourceStore(path=str(tmp_path / "ridx.sqlite")) + store = SqlAlchemyToolSourceStore(url=_sqlite_url(tmp_path / "ridx.sqlite")) index = ToolIndex() index.add_entry(ToolIndexEntry(id="removable", version="1.0", name="Removable", panel_section_id="sec1")) index.by_section["sec1"] = ["removable"] @@ -136,7 +141,7 @@ class TestSqlAlchemyBackend: assert "removable" not in reloaded.by_section.get("sec1", []) def test_update_index_entry_reaches_versioned_lookups(self, tmp_path): - store = SqlAlchemyToolSourceStore(path=str(tmp_path / "uidx.sqlite")) + store = SqlAlchemyToolSourceStore(url=_sqlite_url(tmp_path / "uidx.sqlite")) index = ToolIndex() index.add_entry(ToolIndexEntry(id="dm_tool", version="1.0", name="DM")) store.store_index(index) @@ -151,8 +156,8 @@ class TestSqlAlchemyBackend: assert versioned.description == "updated" def test_sqlalchemy_store_persistence(self, tmp_path): - path = str(tmp_path / "ts.sqlite") - store1 = SqlAlchemyToolSourceStore(path=path) + url = _sqlite_url(tmp_path / "ts.sqlite") + store1 = SqlAlchemyToolSourceStore(url=url) store1.store( StoredToolSource( hash="persist_test_hash", @@ -162,7 +167,7 @@ class TestSqlAlchemyBackend: tool_version="1.0", ) ) - store2 = SqlAlchemyToolSourceStore(path=path) + store2 = SqlAlchemyToolSourceStore(url=url) assert store2.exists("persist_test_hash") retrieved = store2.get("persist_test_hash") assert retrieved is not None @@ -174,8 +179,7 @@ class TestBuildToolSourceStore: def test_build_default_sqlite_store(self, tmp_path): config = FakeConfig( - tool_source_store="sqlite", - tool_source_disk_path=str(tmp_path / "default.sqlite"), + tool_source_database_connection=_sqlite_url(tmp_path / "default.sqlite"), tool_configs=[], tool_source_stores=None, use_lazy_toolbox=False, @@ -183,10 +187,9 @@ class TestBuildToolSourceStore: store = build_tool_source_store(config) # type: ignore[arg-type] assert isinstance(store, SqlAlchemyToolSourceStore) - def test_build_sqlalchemy_store(self, tmp_path): + def test_build_default_in_memory_sqlite_store(self): config = FakeConfig( - tool_source_store="sqlalchemy", - tool_source_disk_path=str(tmp_path / "ts.sqlite"), + tool_source_database_connection="sqlite:///:memory:", tool_configs=[], tool_source_stores=None, use_lazy_toolbox=False, @@ -194,10 +197,9 @@ class TestBuildToolSourceStore: store = build_tool_source_store(config) # type: ignore[arg-type] assert isinstance(store, SqlAlchemyToolSourceStore) - def test_build_sqlalchemy_store_missing_path_raises(self): + def test_build_store_missing_connection_raises(self): config = FakeConfig( - tool_source_store="sqlalchemy", - tool_source_disk_path=None, + tool_source_database_connection=None, tool_configs=[], tool_source_stores=None, use_lazy_toolbox=False, @@ -205,16 +207,34 @@ class TestBuildToolSourceStore: with pytest.raises(ConfigurationError): build_tool_source_store(config) # type: ignore[arg-type] - def test_build_unknown_backend_raises(self): + def test_build_non_sqlite_url_passes_through(self, monkeypatch): + from galaxy.tools.source_store import factory as factory_module + + class CapturingStore: + read_only = False + + def __init__(self, url, read_only): + self.url = url + self.read_only = read_only + + monkeypatch.setattr(factory_module, "SqlAlchemyToolSourceStore", CapturingStore) config = FakeConfig( - tool_source_store="not-a-backend", - tool_source_disk_path=None, + tool_source_database_connection="postgresql://galaxy@example.org/tool_sources", tool_configs=[], tool_source_stores=None, use_lazy_toolbox=False, ) + store = build_tool_source_store(config) # type: ignore[arg-type] + assert store.url == "postgresql://galaxy@example.org/tool_sources" + assert store.read_only is False + + def test_named_store_missing_url_raises(self): with pytest.raises(ConfigurationError): - build_tool_source_store(config) # type: ignore[arg-type] + build_named_store("missing", {"read_only": True}) + + def test_named_store_old_backend_path_spec_raises(self, tmp_path): + with pytest.raises(ConfigurationError): + build_named_store("old", {"backend": "sqlalchemy", "path": str(tmp_path / "old.sqlite")}) class TestPerConfStoreRouting: @@ -222,8 +242,7 @@ class TestPerConfStoreRouting: def _config(self, tmp_path, **overrides): defaults = dict( - tool_source_store="sqlalchemy", - tool_source_disk_path=str(tmp_path / "ts.sqlite"), + tool_source_database_connection=_sqlite_url(tmp_path / "ts.sqlite"), tool_configs=[], tool_source_stores={}, ) diff --git a/test/unit/config/test_config_values.py b/test/unit/config/test_config_values.py index 06f6e70d0db..561e6e73dc9 100644 --- a/test/unit/config/test_config_values.py +++ b/test/unit/config/test_config_values.py @@ -19,6 +19,7 @@ def test_root(appconfig): def test_common_base_config(appconfig): assert appconfig.shed_tools_dir == os.path.join(appconfig.data_dir, "shed_tools") + assert appconfig.tool_source_database_connection == f"sqlite:///{os.path.join(appconfig.data_dir, 'tool_sources.sqlite')}" if running_from_source: expected_path = os.path.join(appconfig.root, "lib", "galaxy", "config", "sample") else: @@ -64,6 +65,9 @@ def test_error_if_database_connection_contains_brackets(bracket): with pytest.raises(ConfigurationError): config.GalaxyAppConfiguration(override_tempdir=False, install_database_connection=uri) + with pytest.raises(ConfigurationError): + config.GalaxyAppConfiguration(override_tempdir=False, tool_source_database_connection=uri) + with pytest.raises(ConfigurationError): config.GalaxyAppConfiguration(override_tempdir=False, amqp_internal_connection=uri)