Models: Allow core Galaxy models and install tool dependencies to target different databases.

Now app.model and app.install_model may be two distinct objects not just two references to the same object. Add ModelMapping class for galaxy.model.tool_shed_install.

If install_database_connection is set in Galaxy configuration, a separate database will be targetted for the persistence of the tool shed repository information. Additional database options may likewise be specified in this case: install_database_engine_option_pool_size, etc....

lib/galaxy/app.py

@@ -43,6 +43,10 @@ class UniverseApplication( object ):
             db_url = self.config.database_connection
         else:
             db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
+        install_db_url = self.config.install_database_connection
+        # TODO: Consider more aggressive check here that this is not the same
+        # database file under the hood.
+        combined_install_database = not( install_db_url and install_db_url != db_url )
         # Set up the tool sheds registry
         if os.path.isfile( self.config.tool_sheds_config ):
             self.tool_shed_registry = tool_shed.tool_shed_registry.Registry( self.config.root, self.config.tool_sheds_config )
@@ -52,6 +56,10 @@ class UniverseApplication( object ):
         # is a new installation, we'll restrict the tool migration messaging.
         from galaxy.model.migrate.check import create_or_verify_database
         create_or_verify_database( db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options, app=self )
+        if not combined_install_database:
+            from galaxy.model.tool_shed_install.migrate.check import create_or_verify_database as tsi_create_or_verify_database
+            tsi_create_or_verify_database( install_db_url, self.config.install_database_engine_options, app=self )
+
         # Alert the Galaxy admin to tools that have been moved from the distribution to the tool shed.
         from tool_shed.galaxy_install.migrate.check import verify_tools
         verify_tools( self, db_url, kwargs.get( 'global_conf', {} ).get( '__file__', None ), self.config.database_engine_options )
@@ -62,14 +70,20 @@ class UniverseApplication( object ):
         self.model = mapping.init( self.config.file_path,
                                    db_url,
                                    self.config.database_engine_options,
+                                   map_install_models=combined_install_database,
                                    database_query_profiling_proxy = self.config.database_query_profiling_proxy,
                                    object_store = self.object_store,
                                    trace_logger=self.trace_logger,
                                    use_pbkdf2=self.config.get_bool( 'use_pbkdf2', True ) )
-        # Want tool_shed_install models accessed through new attribute
-        # (install_model). This is the same object for now, but should ultimately
-        # be allowed to be a separate ModelMapping instances.
-        self.install_model = self.model
+
+        if combined_install_database:
+            self.install_model = self.model
+        else:
+            from galaxy.model.tool_shed_install import mapping as install_mapping
+            install_db_url = self.config.install_database_connection
+            install_db_engine_options = self.config.install_database_engine_options
+            self.install_model = install_mapping.init( install_db_url,
+                                                       install_db_engine_options )
         # Manage installed tool shed repositories.
         self.installed_repository_manager = tool_shed.galaxy_install.InstalledRepositoryManager( self )
         # Create an empty datatypes registry.

lib/galaxy/config.py

@@ -33,15 +33,22 @@ class Configuration( object ):
         self.umask = os.umask( 077 ) # get the current umask
         os.umask( self.umask ) # can't get w/o set, so set it back
         self.gid = os.getgid() # if running under newgrp(1) we'll need to fix the group of data created on the cluster
+
         # Database related configuration
         self.database = resolve_path( kwargs.get( "database_file", "database/universe.sqlite" ), self.root )
         self.database_connection = kwargs.get( "database_connection", False )
         self.database_engine_options = get_database_engine_options( kwargs )
         self.database_create_tables = string_as_bool( kwargs.get( "database_create_tables", "True" ) )
         self.database_query_profiling_proxy = string_as_bool( kwargs.get( "database_query_profiling_proxy", "False" ) )
+
         # Don't set this to true for production databases, but probably should
         # default to True for sqlite databases.
         self.database_auto_migrate = string_as_bool( kwargs.get( "database_auto_migrate", "False" ) )
+
+        # Install database related configuration (if different).
+        self.install_database_connection = kwargs.get( "install_database_connection", None )
+        self.install_database_engine_options = get_database_engine_options( kwargs, model_prefix="install_" )
+
         # Where dataset files are stored
         self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
         self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
@@ -439,7 +446,7 @@ class Configuration( object ):
         admin_users = [ x.strip() for x in self.get( "admin_users", "" ).split( "," ) ]
         return ( user is not None and user.email in admin_users )
 
-def get_database_engine_options( kwargs ):
+def get_database_engine_options( kwargs, model_prefix='' ):
     """
     Allow options for the SQLAlchemy database engine to be passed by using
     the prefix "database_engine_option".
@@ -455,7 +462,7 @@ def get_database_engine_options( kwargs ):
         'pool_threadlocal': string_as_bool,
         'server_side_cursors': string_as_bool
     }
-    prefix = "database_engine_option_"
+    prefix = "%sdatabase_engine_option_" % model_prefix
     prefix_len = len( prefix )
     rval = {}
     for key, value in kwargs.iteritems():

lib/galaxy/model/__init__.py

@@ -43,11 +43,6 @@ datatypes_registry = galaxy.datatypes.registry.Registry()
 # Default Value Required for unit tests
 datatypes_registry.load_datatypes()
 
-# TODO: Refactor references to these classes to eliminate need for this
-# import.
-from .tool_shed_install import ToolShedRepository, RepositoryRepositoryDependencyAssociation, RepositoryDependency
-from .tool_shed_install import ToolDependency, ToolVersion, ToolVersionAssociation, MigrateTools
-
 
 class NoConverterException(Exception):
     def __init__(self, value):

lib/galaxy/model/mapping.py

@@ -14,7 +14,6 @@ from sqlalchemy.orm import backref, object_session, relation, mapper, class_mapp
 from sqlalchemy.orm.collections import attribute_mapped_collection
 
 from galaxy import model
-from galaxy.model import tool_shed_install
 from galaxy.model.orm.engine_factory import build_engine
 from galaxy.model.orm.now import now
 from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString, UUIDType
@@ -25,10 +24,6 @@ log = logging.getLogger( __name__ )
 
 metadata = MetaData()
 
-# import tool shed mappings, TODO: update all references to eliminate
-# need for this import.
-from .tool_shed_install.mapping import *
-
 
 model.User.table = Table( "galaxy_user", metadata,
     Column( "id", Integer, primary_key=True),
@@ -1894,7 +1889,7 @@ def db_next_hid( self ):
 model.History._next_hid = db_next_hid
 
 
-def init( file_path, url, engine_options={}, create_tables=False, database_query_profiling_proxy=False, object_store=None, trace_logger=None, use_pbkdf2=True ):
+def init( file_path, url, engine_options={}, create_tables=False, map_install_models=False, database_query_profiling_proxy=False, object_store=None, trace_logger=None, use_pbkdf2=True ):
     """Connect mappings to the database"""
     # Connect dataset to the file path
     model.Dataset.file_path = file_path
@@ -1908,7 +1903,13 @@ def init( file_path, url, engine_options={}, create_tables=False, database_query
     # Connect the metadata to the database.
     metadata.bind = engine
 
-    result = ModelMapping([model, tool_shed_install], engine=engine)
+    model_modules = [model]
+    if map_install_models:
+        import galaxy.model.tool_shed_install.mapping
+        from galaxy.model import tool_shed_install
+        model_modules.append(tool_shed_install)
+
+    result = ModelMapping(model_modules, engine=engine)
 
     # Create tables if needed
     if create_tables:

lib/galaxy/model/tool_shed_install/mapping.py

@@ -1,11 +1,14 @@
-#from galaxy.model import tool_shed_models as install_model
-from galaxy import model as install_model
-#from sqlalchemy import MetaData
+from galaxy.model import tool_shed_install as install_model
+from sqlalchemy import MetaData
 from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, String, Table, TEXT
 from sqlalchemy.orm import relation, mapper
 from galaxy.model.custom_types import JSONType, TrimmedString
 from galaxy.model.orm.now import now
-from galaxy.model.mapping import metadata
+from galaxy.model.base import ModelMapping
+from galaxy.model.orm.engine_factory import build_engine
+
+
+metadata = MetaData()
 
 
 install_model.ToolShedRepository.table = Table( "tool_shed_repository", metadata,
@@ -103,3 +106,23 @@ mapper( install_model.ToolVersion, install_model.ToolVersion.table,
  )
 
 mapper( install_model.ToolVersionAssociation, install_model.ToolVersionAssociation.table )
+
+
+def init( url, engine_options={}, create_tables=False ):
+    """Connect mappings to the database"""
+    # Load the appropriate db module
+    engine = build_engine( url, engine_options )
+
+    # Connect the metadata to the database.
+    metadata.bind = engine
+
+    result = ModelMapping([install_model], engine=engine)
+
+    # Create tables if needed
+    if create_tables:
+        metadata.create_all()
+        # metadata.engine.commit()
+
+    result.create_tables = create_tables
+    #load local galaxy security policy
+    return result

lib/galaxy/webapps/galaxy/buildapp.py

@@ -221,6 +221,12 @@ def app_factory( global_conf, **kwargs ):
         galaxy.model.mapping.metadata.engine.connection_provider._pool.dispose()
     except:
         pass
+    # Close any pooled database connections before forking
+    try:
+        galaxy.model.tool_shed_install.mapping.metadata.engine.connection_provider._pool.dispose()
+    except:
+        pass
+
     # Return
     return webapp