Merged from central

eggs.ini

@@ -42,6 +42,7 @@ PasteScript = 1.3.6
 Routes = 1.6.3
 simplejson = 1.5
 SQLAlchemy = 0.4.7p1
+sqlalchemy_migrate = 0.4.5
 Tempita = 0.1
 twill = 0.9
 WebError = 0.8a
@@ -90,6 +91,7 @@ PasteScript = http://cheeseshop.python.org/packages/source/P/PasteScript/PasteSc
 Routes = http://pypi.python.org/packages/source/R/Routes/Routes-1.6.3.tar.gz
 simplejson = http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.tar.gz
 SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.4.7p1.tar.gz
+sqlalchemy_migrate = http://pypi.python.org/packages/source/s/sqlalchemy-migrate/sqlalchemy-migrate-0.4.5.tar.gz
 Tempita = http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz
 twill = http://darcs.idyll.org/~t/projects/twill-0.9.tar.gz
 WebError = http://pypi.python.org/packages/source/W/WebError/WebError-0.8a.tar.gz

lib/galaxy/app.py

@@ -4,7 +4,6 @@ from galaxy import config, jobs, util, tools, web
 from galaxy.tracks import store
 from galaxy.web import security
 import galaxy.model
-import galaxy.model.mapping
 import galaxy.datatypes.registry
 import galaxy.security
 
@@ -24,11 +23,14 @@ class UniverseApplication( object ):
             db_url = self.config.database_connection
         else:
             db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database
+        # Initialize database / check for appropriate schema version
+        from galaxy.model.migrate.check import create_or_verify_database
+        create_or_verify_database( db_url, self.config.database_engine_options )
         # Setup the database engine and ORM
-        self.model = galaxy.model.mapping.init( self.config.file_path,
-                                                db_url,
-                                                self.config.database_engine_options,
-                                                create_tables = self.config.database_create_tables )
+        from galaxy.model import mapping
+        self.model = mapping.init( self.config.file_path,
+                                   db_url,
+                                   self.config.database_engine_options )
         # Security helper
         self.security = security.SecurityHelper( id_secret=self.config.id_secret )
         # Initialize the tools

lib/galaxy/model/migrate/check.py

@@ -0,0 +1,112 @@
+import os.path, logging
+
+from galaxy import eggs
+
+import pkg_resources
+pkg_resources.require( "sqlalchemy-migrate" )
+
+from migrate.versioning import repository, schema
+from sqlalchemy import *
+from sqlalchemy.exceptions import NoSuchTableError
+
+log = logging.getLogger( __name__ )
+
+# path relative to galaxy
+migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 )
+migrate_repository = repository.Repository( migrate_repository_directory )
+dialect_to_egg = { 
+    "sqlite" : "pysqlite>=2",
+    "postgres" : "psycopg2",
+    "mysql" : "MySQL_python"
+}
+
+def create_or_verify_database( url, engine_options={} ):
+    """
+    Check that the database is use-able, possibly creating it if empty (this is
+    the only time we automatically create tables, otherwise we force the
+    user to do it using the management script so they can create backups).
+    
+    1) Empty database --> initialize with latest version and return
+    2) Database older than migration support --> fail and require manual update
+    3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update)
+    4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade"
+    
+    """
+
+    dialect = ( url.split( ':', 1 ) )[0]
+    try:
+        egg = dialect_to_egg[dialect]
+        try:
+            pkg_resources.require( egg )
+            log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
+        except:
+            # If the module is in the path elsewhere (i.e. non-egg), it'll still load.
+            log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
+    except KeyError:
+        # Let this go, it could possibly work with db's we don't support
+        log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
+
+    # Create engine and metadata
+    engine = create_engine( url, **engine_options )
+    meta = MetaData( bind=engine )
+    # Try to load dataset table
+    try:
+        dataset_table = Table( "dataset", meta, autoload=True )
+    except NoSuchTableError:
+        # No 'dataset' table means a completely uninitialized database, which
+        # is fine, init the database in a versioned state
+        log.info( "No database, initializing" )
+        # Database might or might not be versioned
+        try:
+            # Declare the database to be under a repository's version control
+            db_schema = schema.ControlledSchema.create( engine, migrate_repository )
+        except:
+            # The database is already under version control
+            db_schema = schema.ControlledSchema( engine, migrate_repository )
+        # Apply all scripts to get to current version
+        migrate_to_current_version( engine, db_schema )
+        return
+    try:
+        hda_table = Table( "history_dataset_association", meta, autoload=True )
+    except NoSuchTableError:
+        raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" )
+    # There is a 'history_dataset_association' table, so we (hopefully) have
+    # version 1 of the database, but without the migrate_version table. This
+    # happens if the user has a build from right before migration was added.
+    # Verify that this is true, if it is any older they'll have to update
+    # manually
+    if 'copied_from_history_dataset_association_id' not in hda_table.c:
+        # The 'copied_from_history_dataset_association_id' column was added in
+        # rev 1464:c7acaa1bb88f.  This is the oldest revision we currently do
+        # automated versioning for, so stop here
+        raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" )
+    # At revision 1464:c7acaa1bb88f or greater (database version 1), make sure
+    # that the db has version information. This is the trickiest case -- we
+    # have a database but no version control, and are assuming it is a certain
+    # version. If the user has postion version 1 changes this could cause
+    # problems
+    try:
+        version_table = Table( "migrate_version", meta, autoload=True )
+    except NoSuchTableError:
+        # The database exists but is not yet under migrate version control, so init with version 1
+        log.info( "Adding version control to existing database" )
+        try:
+            metadata_file_table = Table( "metadata_file", meta, autoload=True )
+            schema.ControlledSchema.create( engine, migrate_repository, version=2 )
+        except NoSuchTableError:
+            schema.ControlledSchema.create( engine, migrate_repository, version=1 )
+    # Verify that the code and the DB are in sync
+    db_schema = schema.ControlledSchema( engine, migrate_repository )
+    if migrate_repository.versions.latest != db_schema.version:
+        raise Exception( "Your database has version '%d' but this code expects version '%d'.  Please backup your database and then migrate the schema by running 'sh manage_db.sh upgrade'."
+                            % ( db_schema.version, migrate_repository.versions.latest ) )
+    else:
+        log.info( "At database version %d" % db_schema.version )
+        
+def migrate_to_current_version( engine, schema ):
+    # Changes to get to current version
+    changeset = schema.changeset( None )
+    for ver, change in changeset:
+        nextver = ver + changeset.step
+        log.info( 'Migrating %s -> %s... ' % ( ver, nextver ) )
+        schema.runchange( ver, change, changeset.step )

lib/galaxy/model/migrate/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=Galaxy
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to 
+# change the table name in each database too. 
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the 
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the 
+# commit continues, perhaps ending successfully. 
+# Databases in this list MUST compile successfully during a commit, or the 
+# entire commit will fail. List the databases your application will actually 
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

lib/galaxy/model/migrate/versions/0001_initial_tables.py

@@ -0,0 +1,211 @@
+from sqlalchemy import *
+from migrate import *
+
+import datetime
+now = datetime.datetime.utcnow
+
+# Need our custom types, but don't import anything else from model
+from galaxy.model.custom_types import *
+
+import logging
+log = logging.getLogger( __name__ )
+
+metadata = MetaData( migrate_engine )
+
+# Tables as of changeset 1464:c7acaa1bb88f
+User_table = Table( "galaxy_user", metadata,
+    Column( "id", Integer, primary_key=True),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "email", TrimmedString( 255 ), nullable=False ),
+    Column( "password", TrimmedString( 40 ), nullable=False ),
+    Column( "external", Boolean, default=False ) )
+            
+History_table = Table( "history", metadata,
+    Column( "id", Integer, primary_key=True),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
+    Column( "name", TrimmedString( 255 ) ),
+    Column( "hid_counter", Integer, default=1 ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "purged", Boolean, index=True, default=False ),
+    Column( "genome_build", TrimmedString( 40 ) ) )
+
+HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, 
+    Column( "id", Integer, primary_key=True ),
+    Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
+    Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "copied_from_history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ),
+    Column( "hid", Integer ),
+    Column( "name", TrimmedString( 255 ) ),
+    Column( "info", TrimmedString( 255 ) ),
+    Column( "blurb", TrimmedString( 255 ) ),
+    Column( "peek" , TEXT ),
+    Column( "extension", TrimmedString( 64 ) ),
+    Column( "metadata", MetadataType(), key="_metadata" ),
+    Column( "parent_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ),
+    Column( "designation", TrimmedString( 255 ) ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "visible", Boolean ) )
+
+Dataset_table = Table( "dataset", metadata, 
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
+    Column( "state", TrimmedString( 64 ) ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "purged", Boolean, index=True, default=False ),
+    Column( "purgable", Boolean, default=True ),
+    Column( "external_filename" , TEXT ),
+    Column( "_extra_files_path", TEXT ),
+    Column( 'file_size', Numeric( 15, 0 ) ) )
+
+ImplicitlyConvertedDatasetAssociation_table = Table( "implicitly_converted_dataset_association", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ),
+    Column( "hda_parent_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "metadata_safe", Boolean, index=True, default=True ),
+    Column( "type", TrimmedString( 255 ) ) )
+
+ValidationError_table = Table( "validation_error", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
+    Column( "message", TrimmedString( 255 ) ),
+    Column( "err_type", TrimmedString( 64 ) ),
+    Column( "attributes", TEXT ) )
+
+Job_table = Table( "job", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
+    Column( "tool_id", String( 255 ) ),
+    Column( "tool_version", TEXT, default="1.0.0" ),
+    Column( "state", String( 64 ) ),
+    Column( "info", TrimmedString( 255 ) ),
+    Column( "command_line", TEXT ), 
+    Column( "param_filename", String( 1024 ) ),
+    Column( "runner_name", String( 255 ) ),
+    Column( "stdout", TEXT ),
+    Column( "stderr", TEXT ),
+    Column( "traceback", TEXT ),
+    Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True, nullable=True ),
+    Column( "job_runner_name", String( 255 ) ),
+    Column( "job_runner_external_id", String( 255 ) ) )
+    
+JobParameter_table = Table( "job_parameter", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
+    Column( "name", String(255) ),
+    Column( "value", TEXT ) )
+    
+JobToInputDatasetAssociation_table = Table( "job_to_input_dataset", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
+    Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
+    Column( "name", String(255) ) )
+    
+JobToOutputDatasetAssociation_table = Table( "job_to_output_dataset", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
+    Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ),
+    Column( "name", String(255) ) )
+    
+Event_table = Table( "event", metadata, 
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "history_id", Integer, ForeignKey( "history.id" ), index=True, nullable=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=True ),
+    Column( "message", TrimmedString( 1024 ) ),
+    Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True, nullable=True ),
+    Column( "tool_id", String( 255 ) ) )
+
+GalaxySession_table = Table( "galaxy_session", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=True ),
+    Column( "remote_host", String( 255 ) ),
+    Column( "remote_addr", String( 255 ) ),
+    Column( "referer", TEXT ),
+    Column( "current_history_id", Integer, ForeignKey( "history.id" ), nullable=True ),
+    Column( "session_key", TrimmedString( 255 ), index=True, unique=True ),
+    Column( "is_valid", Boolean, default=False ),
+    Column( "prev_session_id", Integer )
+    )
+
+GalaxySessionToHistoryAssociation_table = Table( "galaxy_session_to_history", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ),
+    Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) )
+
+StoredWorkflow_table = Table( "stored_workflow", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ),
+    Column( "latest_workflow_id", Integer,
+            ForeignKey( "workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk' ), index=True ),
+    Column( "name", TEXT ),
+    Column( "deleted", Boolean, default=False ),
+    )
+
+Workflow_table = Table( "workflow", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True, nullable=False ),
+    Column( "name", TEXT ),
+    Column( "has_cycles", Boolean ),
+    Column( "has_errors", Boolean )
+    )
+
+WorkflowStep_table = Table( "workflow_step", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, default=now, onupdate=now ),
+    Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ),
+    Column( "type", String(64) ),
+    Column( "tool_id", TEXT ),
+    Column( "tool_version", TEXT ),
+    Column( "tool_inputs", JSONType ),
+    Column( "tool_errors", JSONType ),
+    Column( "position", JSONType ),
+    Column( "config", JSONType ),
+    Column( "order_index", Integer )
+    )
+
+WorkflowStepConnection_table = Table( "workflow_step_connection", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "output_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ),
+    Column( "input_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ),
+    Column( "output_name", TEXT ),
+    Column( "input_name", TEXT)
+    )
+
+StoredWorkflowUserShareAssociation_table = Table( "stored_workflow_user_share_connection", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
+    )
+
+StoredWorkflowMenuEntry_table = Table( "stored_workflow_menu_entry", metadata,
+    Column( "id", Integer, primary_key=True ),
+    Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ),
+    Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),                              
+    Column( "order_index", Integer ) )
+
+def upgrade():
+    metadata.create_all()
+
+## def downgrade():
+##     # Operations to reverse the above upgrade go here.
+##     pass

lib/galaxy/model/migrate/versions/0002_metadata_file_table.py

@@ -0,0 +1,31 @@
+from sqlalchemy import *
+from migrate import *
+
+import datetime
+now = datetime.datetime.utcnow
+
+# Need our custom types, but don't import anything else from model
+from galaxy.model.custom_types import *
+
+import logging
+log = logging.getLogger( __name__ )
+
+metadata = MetaData( migrate_engine )
+
+# New table in changeset 1568:0b022adfdc34
+MetadataFile_table = Table( "metadata_file", metadata, 
+    Column( "id", Integer, primary_key=True ),
+    Column( "name", TEXT ),
+    Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ),
+    Column( "create_time", DateTime, default=now ),
+    Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
+    Column( "deleted", Boolean, index=True, default=False ),
+    Column( "purged", Boolean, index=True, default=False ) )
+
+def upgrade():
+    metadata.reflect()
+    MetadataFile_table.create()
+
+def downgrade():
+    metadata.reflect()
+    MetadataFile_table.drop()

manage_db.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+
+cd `dirname $0`
+python -ES ./scripts/manage_db.py $@

scripts/manage_db.py

@@ -0,0 +1,45 @@
+import sys, os.path, logging
+
+new_path = [ os.path.join( os.getcwd(), "lib" ) ]
+new_path.extend( sys.path[1:] ) # remove scripts/ from the path
+sys.path = new_path
+
+from galaxy import eggs
+
+import pkg_resources
+pkg_resources.require( "sqlalchemy-migrate" )
+
+from migrate.versioning.shell import main
+from ConfigParser import SafeConfigParser
+
+log = logging.getLogger( __name__ )
+
+cp = SafeConfigParser()
+cp.read( "universe_wsgi.ini" )
+
+if cp.has_option( "app:main", "database_connection" ):
+    db_url = cp.get( "app:main", "database_connection" )
+elif cp.has_option( "app:main", "database_file" ):
+    db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % cp.get( "app:main", "database_file" )
+else:
+    db_url = None
+
+dialect_to_egg = {
+    "sqlite" : "pysqlite>=2",
+    "postgres" : "psycopg2",
+    "mysql" : "MySQL_python"
+}
+dialect = ( db_url.split( ':', 1 ) )[0]
+try:
+    egg = dialect_to_egg[dialect]
+    try:
+        pkg_resources.require( egg )
+        log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) )
+    except:
+        # If the module is in the path elsewhere (i.e. non-egg), it'll still load.
+        log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) )
+except KeyError:
+    # Let this go, it could possibly work with db's we don't support
+    log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect )
+
+main( repository='lib/galaxy/model/migrate', url=db_url )