From ead11c99000a4a1506234727b4b38aa2e869a709 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 8 Jul 2026 17:29:46 +0200 Subject: [PATCH] lazy lineage: precompute guid sibling versions (boot was quadratic) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _index_versions_for prefix-scanned every index entry per lineage lookup, and the EDAM panel views resolve a lineage per shed tool — ~11.5k scans over ~9k entries. Profiled as the single largest toolbox boot cost: 18.0s cumulative, and _configure_toolbox drops from 25.8s to 11.7s (under cProfile) with the scan replaced by one precomputed versionless-guid -> (entry_id, version) map. The map is cached against the ToolIndex object identity plus entry count — reloads swap the object, registrations/removals mutate membership in place — and in-place removals also reset it explicitly (a pop-then-add could keep the count stable while changing membership). Claude-Session: https://claude.ai/code/session_018L7ZmCv2ubKA3JNeSL8Pkr --- lib/galaxy/tools/lazy_toolbox.py | 46 +++++++++++++++++++++++---- test/unit/app/tools/test_lazy_tool.py | 25 +++++++++++++++ 2 files changed, 65 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tools/lazy_toolbox.py b/lib/galaxy/tools/lazy_toolbox.py index 2ce91394ef5..68f932928f0 100644 --- a/lib/galaxy/tools/lazy_toolbox.py +++ b/lib/galaxy/tools/lazy_toolbox.py @@ -500,6 +500,11 @@ class LazyToolBox(ToolBox): # ``_index_source_paths``. Set before ``super().__init__`` because # the eager walk consults it through ``_tool_file_on_disk``. self._index_source_paths_cache: tuple[ToolIndex, set[str]] | None = None + # Versionless-guid → sibling versions map, keyed on index identity + # plus entry count — see ``_guid_sibling_versions``. Also consulted + # during ``super().__init__`` (panel views resolve lineages + # mid-walk). + self._guid_sibling_versions_cache: tuple[ToolIndex, int, dict[str, list[tuple[str, str]]]] | None = None # Eager init — its ``_init_tools_from_configs`` is overridden so the # walk goes through our ``create_tool`` seam and hands back LazyTool @@ -704,14 +709,39 @@ class LazyToolBox(ToolBox): if "/repos/" in tool_id: versionless = remove_version_from_guid(tool_id) if versionless: - prefix = f"{versionless}/" - for entry_id, entry in self._tool_index.entries.items(): - if entry_id == tool_id: - continue - if entry_id.startswith(prefix) and entry.version and entry.version not in result: - result.append(entry.version) + for entry_id, version in self._guid_sibling_versions().get(versionless, ()): + if entry_id != tool_id and version not in result: + result.append(version) return result + def _guid_sibling_versions(self) -> dict[str, list[tuple[str, str]]]: + """Versionless guid → ``(entry_id, version)`` of every indexed sibling. + + ``_index_versions_for`` used to prefix-scan all index entries per + lineage lookup; the EDAM panel views resolve a lineage per shed + tool, which made boot quadratic in installed-tool count (~11k + scans over ~9k entries profiled as the single largest toolbox + cost). Keyed on index identity *and* entry count — reloads swap + the ``ToolIndex`` object, while registrations and removals mutate + membership in place; in-place removals additionally reset the + cache explicitly (a pop-then-add could keep the count stable). + """ + index = self._tool_index + if index is None: + return {} + cached = self._guid_sibling_versions_cache + if cached is not None and cached[0] is index and cached[1] == len(index.entries): + return cached[2] + siblings: dict[str, list[tuple[str, str]]] = {} + for entry_id, entry in index.entries.items(): + if "/repos/" not in entry_id or not entry.version: + continue + versionless = remove_version_from_guid(entry_id) + if versionless: + siblings.setdefault(versionless, []).append((entry_id, entry.version)) + self._guid_sibling_versions_cache = (index, len(index.entries), siblings) + return siblings + def _rebuild_shed_short_id_map(self) -> None: """Walk the index and rebuild short-id → guid mappings for shed installs. @@ -1501,6 +1531,10 @@ class LazyToolBox(ToolBox): if self._tool_index is not None: self._tool_index.entries.pop(tool_id, None) self._tool_index.entries_by_version.pop(tool_id, None) + # In-place membership change on the same ToolIndex object: the + # identity-keyed sibling-versions cache would otherwise keep + # serving the removed version to lineage lookups. + self._guid_sibling_versions_cache = None # ``super().remove_tool_by_id`` clears ``_tools_by_id`` but leaves # ``_tool_versions_by_id`` and the lineage map intact. ``get_tool``'s # fall-through walks lineage versions via ``_tool_from_lineage_version`` diff --git a/test/unit/app/tools/test_lazy_tool.py b/test/unit/app/tools/test_lazy_tool.py index 7a042e81d0b..d3f93805821 100644 --- a/test/unit/app/tools/test_lazy_tool.py +++ b/test/unit/app/tools/test_lazy_tool.py @@ -541,3 +541,28 @@ def test_index_source_paths_refresh_on_index_swap(): swapped.add_entry(_entry(id="t2", source_path="/b/t2.xml")) box._tool_index = swapped assert box._index_source_paths() == {"/b/t2.xml"} + + +def test_index_versions_for_collects_guid_sibling_versions(): + box = _seam_box() + box._guid_sibling_versions_cache = None + prefix = "toolshed.g2.bx.psu.edu/repos/iuc/fastp/fastp" + for version in ("0.20.1", "0.23.2"): + box._tool_index.add_entry(_entry(id=f"{prefix}/{version}", version=version)) + box._tool_index.add_entry(_entry(id="unrelated", version="1.0")) + versions = box._index_versions_for(f"{prefix}/0.20.1") + assert set(versions) == {"0.20.1", "0.23.2"} + assert box._index_versions_for("unrelated") == ["1.0"] + + +def test_guid_sibling_versions_reset_on_in_place_removal(): + box = _registry_box() + box._guid_sibling_versions_cache = None + prefix = "toolshed.g2.bx.psu.edu/repos/iuc/fastp/fastp" + for version in ("0.20.1", "0.23.2"): + entry = _entry(id=f"{prefix}/{version}", version=version) + box._tool_index.add_entry(entry) + box._register_lazy_entry(entry) + assert set(box._index_versions_for(f"{prefix}/0.20.1")) == {"0.20.1", "0.23.2"} + box._remove_tool_in_memory(f"{prefix}/0.23.2") + assert box._index_versions_for(f"{prefix}/0.20.1") == ["0.20.1"]