Merge pull request #8887 from mvdbeek/dont_eagerload_unused_metadata

[19.09] Fix excessive memory consumption by not loading repository.metadata column

lib/galaxy/model/tool_shed_install/mapping.py

@@ -89,16 +89,13 @@ install_model.MigrateTools.table = Table("migrate_tools", metadata,
 mapper(install_model.ToolShedRepository, install_model.ToolShedRepository.table,
        properties=dict(tool_versions=relation(install_model.ToolVersion,
                                               primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.ToolVersion.table.c.tool_shed_repository_id),
-                                              backref='tool_shed_repository',
-                                              lazy='joined'),
+                                              backref='tool_shed_repository'),
                        tool_dependencies=relation(install_model.ToolDependency,
                                                   primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.ToolDependency.table.c.tool_shed_repository_id),
                                                   order_by=install_model.ToolDependency.table.c.name,
-                                                  backref='tool_shed_repository',
-                                                  lazy='joined'),
+                                                  backref='tool_shed_repository'),
                        required_repositories=relation(install_model.RepositoryRepositoryDependencyAssociation,
-                                                      primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id),
-                                                      lazy='joined')))
+                                                      primaryjoin=(install_model.ToolShedRepository.table.c.id == install_model.RepositoryRepositoryDependencyAssociation.table.c.tool_shed_repository_id))))
 
 mapper(install_model.RepositoryRepositoryDependencyAssociation, install_model.RepositoryRepositoryDependencyAssociation.table,
        properties=dict(repository=relation(install_model.ToolShedRepository,

lib/galaxy/tools/cache.py

@@ -3,6 +3,11 @@ import os
 from collections import defaultdict
 from threading import Lock
 
+from sqlalchemy.orm import (
+    defer,
+    joinedload,
+)
+
 from galaxy.util import unicodify
 from galaxy.util.hash_util import md5_hash_file
 
@@ -144,7 +149,10 @@ class ToolShedRepositoryCache(object):
         self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
 
     def rebuild(self):
-        self.repositories = self.app.install_model.context.current.query(self.app.install_model.ToolShedRepository).all()
+        self.repositories = self.app.install_model.context.current.query(self.app.install_model.ToolShedRepository).options(
+            defer(self.app.install_model.ToolShedRepository.metadata),
+            joinedload('tool_dependencies'),
+        ).all()
         repos_by_tuple = defaultdict(list)
         for repository in self.repositories + self.local_repositories:
             repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)

lib/tool_shed/metadata/metadata_generator.py

@@ -1064,7 +1064,7 @@ class MetadataGenerator(object):
                 self.set_changeset_revision(self.repository.changeset_revision)
             else:
                 self.set_changeset_revision(changeset_revision)
-            self.shed_config_dict = repository.get_shed_config_dict(self.app)
+            self.shed_config_dict = repository.get_shed_config_dict(self.app, {})
             self.metadata_dict = {'shed_config_filename': self.shed_config_dict.get('config_filename', None)}
         else:
             if relative_install_dir is None and self.repository is not None: