From 2e7f174e818d3d0ce174329dfdcdacb1f3671843 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 26 Oct 2019 19:30:01 +0200 Subject: [PATCH 1/3] Limit lazy="joined" to tool_dependencies relation, defer repository.metadata loading --- lib/galaxy/model/tool_shed_install/mapping.py | 6 ++---- lib/galaxy/tools/cache.py | 4 +++- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/model/tool_shed_install/mapping.py b/lib/galaxy/model/tool_shed_install/mapping.py index 48a596e6373..630b3084037 100644 --- a/lib/galaxy/model/tool_shed_install/mapping.py +++ b/lib/galaxy/model/tool_shed_install/mapping.py @@ -89,16 +89,14 @@ install_model.MigrateTools.table = Table("migrate_tools", metadata, mapper(install_model.ToolShedRepository, install_model.ToolShedRepository.table, properties=dict(tool_versions=relation(install_model.ToolVersion, primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.ToolVersion.table.c.tool_shed_repository_id), - backref='tool_shed_repository', - lazy='joined'), + backref='tool_shed_repository'), tool_dependencies=relation(install_model.ToolDependency, primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.ToolDependency.table.c.tool_shed_repository_id), order_by=install_model.ToolDependency.table.c.name, backref='tool_shed_repository', lazy='joined'), required_repositories=relation(install_model.RepositoryRepositoryDependencyAssociation, - primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id), - lazy='joined'))) + primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id)))) mapper(install_model.RepositoryRepositoryDependencyAssociation, install_model.RepositoryRepositoryDependencyAssociation.table, properties=dict(repository=relation(install_model.ToolShedRepository, diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 6604303e43a..0c3c22dc843 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -3,6 +3,8 @@ import os from collections import defaultdict from threading import Lock +from sqlalchemy.orm import defer + from galaxy.util import unicodify from galaxy.util.hash_util import md5_hash_file @@ -144,7 +146,7 @@ class ToolShedRepositoryCache(object): self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) def rebuild(self): - self.repositories = self.app.install_model.context.current.query(self.app.install_model.ToolShedRepository).all() + self.repositories = self.app.install_model.context.current.query(self.app.install_model.ToolShedRepository).options(defer(self.app.install_model.ToolShedRepository.metadata)).all() repos_by_tuple = defaultdict(list) for repository in self.repositories + self.local_repositories: repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) From 3a329488182b0e050831556901eaf31bcb7cfb84 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 26 Oct 2019 20:55:47 +0200 Subject: [PATCH 2/3] Fix metadata reset --- lib/tool_shed/metadata/metadata_generator.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/tool_shed/metadata/metadata_generator.py b/lib/tool_shed/metadata/metadata_generator.py index 30026ab8c46..d7fb77fd866 100644 --- a/lib/tool_shed/metadata/metadata_generator.py +++ b/lib/tool_shed/metadata/metadata_generator.py @@ -1064,7 +1064,7 @@ class MetadataGenerator(object): self.set_changeset_revision(self.repository.changeset_revision) else: self.set_changeset_revision(changeset_revision) - self.shed_config_dict = repository.get_shed_config_dict(self.app) + self.shed_config_dict = repository.get_shed_config_dict(self.app, {}) self.metadata_dict = {'shed_config_filename': self.shed_config_dict.get('config_filename', None)} else: if relative_install_dir is None and self.repository is not None: From e4bc813a4f1cd8b5bfcd517667bf28ebb5c83f9c Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 26 Oct 2019 21:44:16 +0200 Subject: [PATCH 3/3] Use joined-loading only in ToolShedRepositoryCache query --- lib/galaxy/model/tool_shed_install/mapping.py | 3 +-- lib/galaxy/tools/cache.py | 10 ++++++++-- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/tool_shed_install/mapping.py b/lib/galaxy/model/tool_shed_install/mapping.py index 630b3084037..268103833b4 100644 --- a/lib/galaxy/model/tool_shed_install/mapping.py +++ b/lib/galaxy/model/tool_shed_install/mapping.py @@ -93,8 +93,7 @@ mapper(install_model.ToolShedRepository, install_model.ToolShedRepository.table, tool_dependencies=relation(install_model.ToolDependency, primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.ToolDependency.table.c.tool_shed_repository_id), order_by=install_model.ToolDependency.table.c.name, - backref='tool_shed_repository', - lazy='joined'), + backref='tool_shed_repository'), required_repositories=relation(install_model.RepositoryRepositoryDependencyAssociation, primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id)))) diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py index 0c3c22dc843..21fe1585654 100644 --- a/lib/galaxy/tools/cache.py +++ b/lib/galaxy/tools/cache.py @@ -3,7 +3,10 @@ import os from collections import defaultdict from threading import Lock -from sqlalchemy.orm import defer +from sqlalchemy.orm import ( + defer, + joinedload, +) from galaxy.util import unicodify from galaxy.util.hash_util import md5_hash_file @@ -146,7 +149,10 @@ class ToolShedRepositoryCache(object): self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository) def rebuild(self): - self.repositories = self.app.install_model.context.current.query(self.app.install_model.ToolShedRepository).options(defer(self.app.install_model.ToolShedRepository.metadata)).all() + self.repositories = self.app.install_model.context.current.query(self.app.install_model.ToolShedRepository).options( + defer(self.app.install_model.ToolShedRepository.metadata), + joinedload('tool_dependencies'), + ).all() repos_by_tuple = defaultdict(list) for repository in self.repositories + self.local_repositories: repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)