diff --git a/lib/galaxy/tool_source_store/populator.py b/lib/galaxy/tool_source_store/populator.py index 30452d7c121..3bde1718688 100644 --- a/lib/galaxy/tool_source_store/populator.py +++ b/lib/galaxy/tool_source_store/populator.py @@ -688,6 +688,15 @@ def populate_store_inline( if not dry_run: target_store.store(stored) + # Commit per tool so each write is a short transaction. + # When the populator runs inside a Galaxy process (cold + # start, shed install) the shared SQLAlchemy session is + # under concurrent pressure from request handlers and the + # queue worker; holding 484 inserts in one open transaction + # is enough to push SQLite past its 5s busy timeout. The + # CLI path takes the same hit but on its own model.context + # with no concurrent readers, so the cost is invisible. + target_store.commit() return ("stored", d, store_name, stored, tool_source, None) except Exception as e: log.error(f"Error processing {path}: {e}") @@ -759,6 +768,15 @@ def populate_store_inline( ) except Exception as e: log.warning("store_index for %s raised: %s", store_name, e) + # The flush failed — the session is now in PendingRollback + # state, which would surface to the very next caller as + # SQLAlchemyError. Clear it so the rest of the boot path + # (and the eventual test client) sees a clean session. + if sa_session is not None: + try: + sa_session.rollback() + except Exception as rb_e: + log.debug("session.rollback after store_index failure raised: %s", rb_e) continue # Rebuild the whoosh search index from the persisted ToolIndex. # Single-writer principle: the toolbox stops re-building this in @@ -798,6 +816,84 @@ def populate_for_paths( ) +def populate_single_path(config, sa_session, path: str) -> bool: + """Index a single tool file by path, bypassing ``discover_tools``. + + Used by ``LazyToolBox.create_tool`` to recover from index misses on + paths the populator's conf walk doesn't cover — typically lib tools + loaded via ``load_hidden_lib_tool`` (``set_metadata_tool.xml`` and + friends), which live under ``lib/galaxy/`` rather than any tool_conf. + + Parses the file, builds a ``StoredToolSource`` and ``ToolIndexEntry`` + (with no panel section, since the tool isn't in any conf), persists to + every writable store, and updates the cached index. Returns True on + success. + """ + import os + + from galaxy.tool_source_store import StoredToolSource + from galaxy.tool_source_store.discover import DiscoveredTool + from galaxy.tool_source_store.index import ToolIndex + from galaxy.tool_util.parser import get_tool_source + from galaxy.util import xml_to_string + + if not os.path.exists(path): + return False + try: + tool_source = get_tool_source(config_file=path) + root = tool_source.xml_tree.getroot() + expanded_content = xml_to_string(root, pretty=True) + except Exception as e: + log.warning("populate_single_path: parse of %s failed: %s", path, e) + return False + + content_hash = compute_hash(expanded_content) + stored = StoredToolSource( + hash=content_hash, + tool_source_class=type(tool_source).__name__, + raw_source=expanded_content, + tool_id=tool_source.parse_id(), + tool_version=tool_source.parse_version(), + tool_dir=str(Path(path).parent), + source_path=str(path), + stored_at=datetime.now(timezone.utc), + ) + discovered = DiscoveredTool(path=str(path), tool_conf="", tool_path=None) + entry = build_index_entry_from_source(discovered, stored, tool_source) + if entry is None: + return False + + stores = _build_stores(config, sa_session) + for store_name, store in stores.items(): + if store.read_only: + continue + try: + store.store(stored) + store.commit() + except Exception as e: + log.warning("populate_single_path: store(%s) for %s raised: %s", store_name, path, e) + if sa_session is not None: + try: + sa_session.rollback() + except Exception: + pass + continue + # Merge entry into the existing index (partial-update semantics). + try: + index = store.load_index() or ToolIndex() + index.add_entry(entry) + store.store_index(index) + store.commit() + except Exception as e: + log.warning("populate_single_path: store_index for %s raised: %s", store_name, e) + if sa_session is not None: + try: + sa_session.rollback() + except Exception: + pass + return True + + def reconcile_index( config, sa_session, diff --git a/lib/galaxy/tools/lazy_toolbox.py b/lib/galaxy/tools/lazy_toolbox.py index 609f83e1a19..33c05a6cb53 100644 --- a/lib/galaxy/tools/lazy_toolbox.py +++ b/lib/galaxy/tools/lazy_toolbox.py @@ -176,6 +176,20 @@ class LazyTool: def lineage(self): return self._lineage + @property + def tool_shed_repository(self): + # The eager pipeline sets this on the real ``Tool`` for shed-installed + # tools (passing ``tool_shed_repository=`` to ``create_tool``); + # the LazyTool stub doesn't carry the repo object. Default to None — + # callers that need a real ``ToolShedRepository`` go through + # materialisation via ``_materialize_for_lazy_tool``, which routes the + # repo lookup via ``_lookup_tool_shed_repository``. + return self._overrides.get("tool_shed_repository") + + @tool_shed_repository.setter + def tool_shed_repository(self, value): + self._overrides["tool_shed_repository"] = value + @property def tool_errors(self): return self._overrides.get("tool_errors") @@ -621,11 +635,24 @@ class LazyToolBox(ToolBox): The populator (cold-start in :meth:`_init_tools_from_configs`, shed installs via ``tool_panel_manager.add_to_tool_panel``) is the single - writer of the index; a miss here means the operator added a tool to - a conf without re-running the populator. Raise rather than parsing - in-toolbox so the contract failure is loud and addressable. + writer of the index. On an unexpected miss we attempt a one-shot + ``populate_single_path`` to cover ad-hoc loads (``load_hidden_lib_tool`` + for ``set_metadata_tool.xml`` and friends — Galaxy-internal tools + loaded after boot from outside any tool_conf). Only fall through to + a hard raise if even that recovery doesn't yield an entry. """ entry = self._resolve_index_entry(config_file, guid) + if entry is None and config_file is not None and self._store is not None: + from galaxy.tool_source_store.populator import populate_single_path + + if populate_single_path(self.app.config, self.app.model.context, str(config_file)): + # Drop the cached index so the next resolve sees the new row. + try: + self._store.invalidate_index_cache() + except Exception as e: + log.debug("invalidate_index_cache after populate_single_path raised: %s", e) + self._tool_index = self._store.load_index() or self._tool_index + entry = self._resolve_index_entry(config_file, guid) if entry is None: raise RuntimeError( "LazyToolBox.create_tool: no index entry for "