From c638757db9bcfdc7a62f624ac8b075554c05faa7 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 19 Oct 2022 15:52:54 -0400 Subject: [PATCH] Refactor ToolShedRepositoryCache into galaxy.tool_shed.cache Things required to perform tool shed installs should be galaxy.tool_shed - this module doesn't require anything in galaxy-app - galaxy-data is sufficient. --- lib/galaxy/app.py | 6 +-- lib/galaxy/structured_app.py | 7 ++-- lib/galaxy/tool_shed/cache.py | 70 +++++++++++++++++++++++++++++++++ lib/galaxy/tools/cache.py | 65 ------------------------------ mypy.ini | 2 + test/unit/app/tools/conftest.py | 2 +- 6 files changed, 79 insertions(+), 73 deletions(-) create mode 100644 lib/galaxy/tool_shed/cache.py diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index da829e80739..c091b144401 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -91,6 +91,7 @@ from galaxy.security.vault import ( Vault, VaultFactory, ) +from galaxy.tool_shed.cache import ToolShedRepositoryCache from galaxy.tool_shed.galaxy_install.installed_repository_manager import InstalledRepositoryManager from galaxy.tool_shed.galaxy_install.update_repository_manager import UpdateRepositoryManager from galaxy.tool_util.deps import containers @@ -98,10 +99,7 @@ from galaxy.tool_util.deps.dependencies import AppInfo from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify.test_data import TestDataResolver from galaxy.tools.biotools import get_galaxy_biotools_metadata_source -from galaxy.tools.cache import ( - ToolCache, - ToolShedRepositoryCache, -) +from galaxy.tools.cache import ToolCache from galaxy.tools.data import ToolDataTableManager from galaxy.tools.data_manager.manager import DataManagers from galaxy.tools.error_reports import ErrorReports diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index d14f3db63a2..00c12384c75 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -30,6 +30,7 @@ from galaxy.objectstore import ( from galaxy.quota import QuotaAgent from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.vault import Vault +from galaxy.tool_shed.cache import ToolShedRepositoryCache from galaxy.tool_util.deps.views import DependencyResolversView from galaxy.tool_util.verify import test_data from galaxy.util.dbkeys import GenomeBuilds @@ -46,7 +47,7 @@ if TYPE_CHECKING: from galaxy.managers.histories import HistoryManager from galaxy.managers.workflows import WorkflowsManager from galaxy.tools import ToolBox - from galaxy.tools.cache import ToolShedRepositoryCache + from galaxy.tools.cache import ToolCache from galaxy.tools.data import ToolDataTableManager from galaxy.tools.error_reports import ErrorReports from galaxy.visualization.genomes import Genomes @@ -142,9 +143,9 @@ class StructuredApp(MinimalManagerApp): queue_worker: Any # 'galaxy.queue_worker.GalaxyQueueWorker' data_provider_registry: Any # 'galaxy.visualization.data_providers.registry.DataProviderRegistry' tool_data_tables: "ToolDataTableManager" - tool_cache: Any # 'galaxy.tools.cache.ToolCache' + tool_cache: "ToolCache" tool_shed_registry: ToolShedRegistry - tool_shed_repository_cache: Optional["ToolShedRepositoryCache"] + tool_shed_repository_cache: Optional[ToolShedRepositoryCache] watchers: "ConfigWatchers" workflow_scheduling_manager: Any # 'galaxy.workflow.scheduling_manager.WorkflowSchedulingManager' interactivetool_manager: Any diff --git a/lib/galaxy/tool_shed/cache.py b/lib/galaxy/tool_shed/cache.py new file mode 100644 index 00000000000..7380fcb7f30 --- /dev/null +++ b/lib/galaxy/tool_shed/cache.py @@ -0,0 +1,70 @@ +import logging +from collections import defaultdict +from typing import ( + Dict, + List, + Tuple, +) + +from sqlalchemy.orm import defer + +from galaxy.model.scoped_session import install_model_scoped_session +from galaxy.model.tool_shed_install import ToolShedRepository +from galaxy.tool_util.toolbox.base import ToolConfRepository + +log = logging.getLogger(__name__) + + +class ToolShedRepositoryCache: + """ + Cache installed ToolShedRepository objects. + """ + + local_repositories: List[ToolConfRepository] + repositories: List[ToolShedRepository] + repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]] + + def __init__(self, session: install_model_scoped_session): + self.session = session() + # Contains ToolConfRepository objects created from shed_tool_conf.xml entries + self.local_repositories = [] + # Repositories loaded from database + self.repositories = [] + self.repos_by_tuple = defaultdict(list) + self._build() + self.session.close() + + def add_local_repository(self, repository): + self.local_repositories.append(repository) + self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) + + def _build(self): + self.repositories = self.session.query(ToolShedRepository).options(defer(ToolShedRepository.metadata_)).all() + repos_by_tuple = defaultdict(list) + for repository in self.repositories + self.local_repositories: + repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) + self.repos_by_tuple = repos_by_tuple + + def get_installed_repository( + self, + tool_shed=None, + name=None, + owner=None, + installed_changeset_revision=None, + changeset_revision=None, + repository_id=None, + ): + if repository_id: + repos = [repo for repo in self.repositories if repo.id == repository_id] + if repos: + return repos[0] + else: + return None + repos = self.repos_by_tuple[(tool_shed, owner, name)] + for repo in repos: + if installed_changeset_revision and repo.installed_changeset_revision != installed_changeset_revision: + continue + if changeset_revision and repo.changeset_revision != changeset_revision: + continue + return repo + return None diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 69036f486b3..f491ca298f3 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -5,20 +5,10 @@ import shutil import sqlite3 import tempfile import zlib -from collections import defaultdict from threading import Lock -from typing import ( - Dict, - List, - Tuple, -) -from sqlalchemy.orm import defer from sqlitedict import SqliteDict -from galaxy.model.scoped_session import install_model_scoped_session -from galaxy.model.tool_shed_install import ToolShedRepository -from galaxy.tool_util.toolbox.base import ToolConfRepository from galaxy.util import unicodify from galaxy.util.hash_util import md5_hash_file @@ -278,58 +268,3 @@ class ToolHash: if self._tool_hash is None: self._tool_hash = md5_hash_file(self.path) return self._tool_hash - - -class ToolShedRepositoryCache: - """ - Cache installed ToolShedRepository objects. - """ - - local_repositories: List[ToolConfRepository] - repositories: List[ToolShedRepository] - repos_by_tuple: Dict[Tuple[str, str, str], List[ToolConfRepository]] - - def __init__(self, session: install_model_scoped_session): - self.session = session() - # Contains ToolConfRepository objects created from shed_tool_conf.xml entries - self.local_repositories = [] - # Repositories loaded from database - self.repositories = [] - self.repos_by_tuple = defaultdict(list) - self._build() - self.session.close() - - def add_local_repository(self, repository): - self.local_repositories.append(repository) - self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) - - def _build(self): - self.repositories = self.session.query(ToolShedRepository).options(defer(ToolShedRepository.metadata_)).all() - repos_by_tuple = defaultdict(list) - for repository in self.repositories + self.local_repositories: - repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) - self.repos_by_tuple = repos_by_tuple - - def get_installed_repository( - self, - tool_shed=None, - name=None, - owner=None, - installed_changeset_revision=None, - changeset_revision=None, - repository_id=None, - ): - if repository_id: - repos = [repo for repo in self.repositories if repo.id == repository_id] - if repos: - return repos[0] - else: - return None - repos = self.repos_by_tuple[(tool_shed, owner, name)] - for repo in repos: - if installed_changeset_revision and repo.installed_changeset_revision != installed_changeset_revision: - continue - if changeset_revision and repo.changeset_revision != changeset_revision: - continue - return repo - return None diff --git a/mypy.ini b/mypy.ini index d1e7d000a0b..4df2bf80ba6 100644 --- a/mypy.ini +++ b/mypy.ini @@ -454,6 +454,8 @@ check_untyped_defs = False check_untyped_defs = False [mypy-galaxy.tools.cache] check_untyped_defs = False +[mypy-galaxy.tool_shed.cache] +check_untyped_defs = False [mypy-galaxy.tool_util.deps.containers] check_untyped_defs = False [mypy-galaxy.managers.users] diff --git a/test/unit/app/tools/conftest.py b/test/unit/app/tools/conftest.py index 3c150e9b44f..be14067442a 100644 --- a/test/unit/app/tools/conftest.py +++ b/test/unit/app/tools/conftest.py @@ -5,8 +5,8 @@ import pytest from galaxy.model import tool_shed_install from galaxy.model.tool_shed_install import mapping +from galaxy.tool_shed.cache import ToolShedRepositoryCache from galaxy.tool_util.toolbox.base import ToolConfRepository -from galaxy.tools.cache import ToolShedRepositoryCache @pytest.fixture