diff --git a/eggs.ini b/eggs.ini index c0180418895..a00f6576630 100644 --- a/eggs.ini +++ b/eggs.ini @@ -42,6 +42,7 @@ PasteScript = 1.3.6 Routes = 1.6.3 simplejson = 1.5 SQLAlchemy = 0.4.7p1 +sqlalchemy_migrate = 0.4.5 Tempita = 0.1 twill = 0.9 WebError = 0.8a @@ -90,6 +91,7 @@ PasteScript = http://cheeseshop.python.org/packages/source/P/PasteScript/PasteSc Routes = http://pypi.python.org/packages/source/R/Routes/Routes-1.6.3.tar.gz simplejson = http://cheeseshop.python.org/packages/source/s/simplejson/simplejson-1.5.tar.gz SQLAlchemy = http://pypi.python.org/packages/source/S/SQLAlchemy/SQLAlchemy-0.4.7p1.tar.gz +sqlalchemy_migrate = http://pypi.python.org/packages/source/s/sqlalchemy-migrate/sqlalchemy-migrate-0.4.5.tar.gz Tempita = http://pypi.python.org/packages/source/T/Tempita/Tempita-0.1.tar.gz twill = http://darcs.idyll.org/~t/projects/twill-0.9.tar.gz WebError = http://pypi.python.org/packages/source/W/WebError/WebError-0.8a.tar.gz diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 211f28ad79b..a0f0ff9f115 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -4,7 +4,6 @@ from galaxy import config, jobs, util, tools, web from galaxy.tracks import store from galaxy.web import security import galaxy.model -import galaxy.model.mapping import galaxy.datatypes.registry import galaxy.security @@ -24,11 +23,14 @@ class UniverseApplication( object ): db_url = self.config.database_connection else: db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config.database + # Initialize database / check for appropriate schema version + from galaxy.model.migrate.check import create_or_verify_database + create_or_verify_database( db_url, self.config.database_engine_options ) # Setup the database engine and ORM - self.model = galaxy.model.mapping.init( self.config.file_path, - db_url, - self.config.database_engine_options, - create_tables = self.config.database_create_tables ) + from galaxy.model import mapping + self.model = mapping.init( self.config.file_path, + db_url, + self.config.database_engine_options ) # Security helper self.security = security.SecurityHelper( id_secret=self.config.id_secret ) # Initialize the tools diff --git a/lib/galaxy/model/migrate/__init__.py b/lib/galaxy/model/migrate/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/model/migrate/check.py b/lib/galaxy/model/migrate/check.py new file mode 100644 index 00000000000..20bd7f4d4bf --- /dev/null +++ b/lib/galaxy/model/migrate/check.py @@ -0,0 +1,112 @@ +import os.path, logging + +from galaxy import eggs + +import pkg_resources +pkg_resources.require( "sqlalchemy-migrate" ) + +from migrate.versioning import repository, schema +from sqlalchemy import * +from sqlalchemy.exceptions import NoSuchTableError + +log = logging.getLogger( __name__ ) + +# path relative to galaxy +migrate_repository_directory = os.path.dirname( __file__ ).replace( os.getcwd() + os.path.sep, '', 1 ) +migrate_repository = repository.Repository( migrate_repository_directory ) +dialect_to_egg = { + "sqlite" : "pysqlite>=2", + "postgres" : "psycopg2", + "mysql" : "MySQL_python" +} + +def create_or_verify_database( url, engine_options={} ): + """ + Check that the database is use-able, possibly creating it if empty (this is + the only time we automatically create tables, otherwise we force the + user to do it using the management script so they can create backups). + + 1) Empty database --> initialize with latest version and return + 2) Database older than migration support --> fail and require manual update + 3) Database at state where migrate support introduced --> add version control information but make no changes (might still require manual update) + 4) Database versioned but out of date --> fail with informative message, user must run "sh manage_db.sh upgrade" + + """ + + dialect = ( url.split( ':', 1 ) )[0] + try: + egg = dialect_to_egg[dialect] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) + except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + + # Create engine and metadata + engine = create_engine( url, **engine_options ) + meta = MetaData( bind=engine ) + # Try to load dataset table + try: + dataset_table = Table( "dataset", meta, autoload=True ) + except NoSuchTableError: + # No 'dataset' table means a completely uninitialized database, which + # is fine, init the database in a versioned state + log.info( "No database, initializing" ) + # Database might or might not be versioned + try: + # Declare the database to be under a repository's version control + db_schema = schema.ControlledSchema.create( engine, migrate_repository ) + except: + # The database is already under version control + db_schema = schema.ControlledSchema( engine, migrate_repository ) + # Apply all scripts to get to current version + migrate_to_current_version( engine, db_schema ) + return + try: + hda_table = Table( "history_dataset_association", meta, autoload=True ) + except NoSuchTableError: + raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) + # There is a 'history_dataset_association' table, so we (hopefully) have + # version 1 of the database, but without the migrate_version table. This + # happens if the user has a build from right before migration was added. + # Verify that this is true, if it is any older they'll have to update + # manually + if 'copied_from_history_dataset_association_id' not in hda_table.c: + # The 'copied_from_history_dataset_association_id' column was added in + # rev 1464:c7acaa1bb88f. This is the oldest revision we currently do + # automated versioning for, so stop here + raise Exception( "Your database is older than hg revision 1464:c7acaa1bb88f and will need to be updated manually" ) + # At revision 1464:c7acaa1bb88f or greater (database version 1), make sure + # that the db has version information. This is the trickiest case -- we + # have a database but no version control, and are assuming it is a certain + # version. If the user has postion version 1 changes this could cause + # problems + try: + version_table = Table( "migrate_version", meta, autoload=True ) + except NoSuchTableError: + # The database exists but is not yet under migrate version control, so init with version 1 + log.info( "Adding version control to existing database" ) + try: + metadata_file_table = Table( "metadata_file", meta, autoload=True ) + schema.ControlledSchema.create( engine, migrate_repository, version=2 ) + except NoSuchTableError: + schema.ControlledSchema.create( engine, migrate_repository, version=1 ) + # Verify that the code and the DB are in sync + db_schema = schema.ControlledSchema( engine, migrate_repository ) + if migrate_repository.versions.latest != db_schema.version: + raise Exception( "Your database has version '%d' but this code expects version '%d'. Please backup your database and then migrate the schema by running 'sh manage_db.sh upgrade'." + % ( db_schema.version, migrate_repository.versions.latest ) ) + else: + log.info( "At database version %d" % db_schema.version ) + +def migrate_to_current_version( engine, schema ): + # Changes to get to current version + changeset = schema.changeset( None ) + for ver, change in changeset: + nextver = ver + changeset.step + log.info( 'Migrating %s -> %s... ' % ( ver, nextver ) ) + schema.runchange( ver, change, changeset.step ) diff --git a/lib/galaxy/model/migrate/migrate.cfg b/lib/galaxy/model/migrate/migrate.cfg new file mode 100644 index 00000000000..3fd7400ff42 --- /dev/null +++ b/lib/galaxy/model/migrate/migrate.cfg @@ -0,0 +1,20 @@ +[db_settings] +# Used to identify which repository this database is versioned under. +# You can use the name of your project. +repository_id=Galaxy + +# The name of the database table used to track the schema version. +# This name shouldn't already be used by your project. +# If this is changed once a database is under version control, you'll need to +# change the table name in each database too. +version_table=migrate_version + +# When committing a change script, Migrate will attempt to generate the +# sql for all supported databases; normally, if one of them fails - probably +# because you don't have that database installed - it is ignored and the +# commit continues, perhaps ending successfully. +# Databases in this list MUST compile successfully during a commit, or the +# entire commit will fail. List the databases your application will actually +# be using to ensure your updates to that database work properly. +# This must be a list; example: ['postgres','sqlite'] +required_dbs=[] diff --git a/lib/galaxy/model/migrate/versions/0001_initial_tables.py b/lib/galaxy/model/migrate/versions/0001_initial_tables.py new file mode 100644 index 00000000000..c52537706e7 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0001_initial_tables.py @@ -0,0 +1,211 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +# Tables as of changeset 1464:c7acaa1bb88f +User_table = Table( "galaxy_user", metadata, + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "email", TrimmedString( 255 ), nullable=False ), + Column( "password", TrimmedString( 40 ), nullable=False ), + Column( "external", Boolean, default=False ) ) + +History_table = Table( "history", metadata, + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "name", TrimmedString( 255 ) ), + Column( "hid_counter", Integer, default=1 ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ), + Column( "genome_build", TrimmedString( 40 ) ) ) + +HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "copied_from_history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ), + Column( "hid", Integer ), + Column( "name", TrimmedString( 255 ) ), + Column( "info", TrimmedString( 255 ) ), + Column( "blurb", TrimmedString( 255 ) ), + Column( "peek" , TEXT ), + Column( "extension", TrimmedString( 64 ) ), + Column( "metadata", MetadataType(), key="_metadata" ), + Column( "parent_id", Integer, ForeignKey( "history_dataset_association.id" ), nullable=True ), + Column( "designation", TrimmedString( 255 ) ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "visible", Boolean ) ) + +Dataset_table = Table( "dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "state", TrimmedString( 64 ) ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ), + Column( "purgable", Boolean, default=True ), + Column( "external_filename" , TEXT ), + Column( "_extra_files_path", TEXT ), + Column( 'file_size', Numeric( 15, 0 ) ) ) + +ImplicitlyConvertedDatasetAssociation_table = Table( "implicitly_converted_dataset_association", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ), + Column( "hda_parent_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "metadata_safe", Boolean, index=True, default=True ), + Column( "type", TrimmedString( 255 ) ) ) + +ValidationError_table = Table( "validation_error", metadata, + Column( "id", Integer, primary_key=True ), + Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "message", TrimmedString( 255 ) ), + Column( "err_type", TrimmedString( 64 ) ), + Column( "attributes", TEXT ) ) + +Job_table = Table( "job", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), + Column( "tool_id", String( 255 ) ), + Column( "tool_version", TEXT, default="1.0.0" ), + Column( "state", String( 64 ) ), + Column( "info", TrimmedString( 255 ) ), + Column( "command_line", TEXT ), + Column( "param_filename", String( 1024 ) ), + Column( "runner_name", String( 255 ) ), + Column( "stdout", TEXT ), + Column( "stderr", TEXT ), + Column( "traceback", TEXT ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True, nullable=True ), + Column( "job_runner_name", String( 255 ) ), + Column( "job_runner_external_id", String( 255 ) ) ) + +JobParameter_table = Table( "job_parameter", metadata, + Column( "id", Integer, primary_key=True ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "name", String(255) ), + Column( "value", TEXT ) ) + +JobToInputDatasetAssociation_table = Table( "job_to_input_dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "name", String(255) ) ) + +JobToOutputDatasetAssociation_table = Table( "job_to_output_dataset", metadata, + Column( "id", Integer, primary_key=True ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), + Column( "dataset_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True ), + Column( "name", String(255) ) ) + +Event_table = Table( "event", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "history_id", Integer, ForeignKey( "history.id" ), index=True, nullable=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=True ), + Column( "message", TrimmedString( 1024 ) ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True, nullable=True ), + Column( "tool_id", String( 255 ) ) ) + +GalaxySession_table = Table( "galaxy_session", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=True ), + Column( "remote_host", String( 255 ) ), + Column( "remote_addr", String( 255 ) ), + Column( "referer", TEXT ), + Column( "current_history_id", Integer, ForeignKey( "history.id" ), nullable=True ), + Column( "session_key", TrimmedString( 255 ), index=True, unique=True ), + Column( "is_valid", Boolean, default=False ), + Column( "prev_session_id", Integer ) + ) + +GalaxySessionToHistoryAssociation_table = Table( "galaxy_session_to_history", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), + Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ) ) + +StoredWorkflow_table = Table( "stored_workflow", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True, nullable=False ), + Column( "latest_workflow_id", Integer, + ForeignKey( "workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk' ), index=True ), + Column( "name", TEXT ), + Column( "deleted", Boolean, default=False ), + ) + +Workflow_table = Table( "workflow", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True, nullable=False ), + Column( "name", TEXT ), + Column( "has_cycles", Boolean ), + Column( "has_errors", Boolean ) + ) + +WorkflowStep_table = Table( "workflow_step", metadata, + Column( "id", Integer, primary_key=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ), + Column( "type", String(64) ), + Column( "tool_id", TEXT ), + Column( "tool_version", TEXT ), + Column( "tool_inputs", JSONType ), + Column( "tool_errors", JSONType ), + Column( "position", JSONType ), + Column( "config", JSONType ), + Column( "order_index", Integer ) + ) + +WorkflowStepConnection_table = Table( "workflow_step_connection", metadata, + Column( "id", Integer, primary_key=True ), + Column( "output_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), + Column( "input_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), + Column( "output_name", TEXT ), + Column( "input_name", TEXT) + ) + +StoredWorkflowUserShareAssociation_table = Table( "stored_workflow_user_share_connection", metadata, + Column( "id", Integer, primary_key=True ), + Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) + ) + +StoredWorkflowMenuEntry_table = Table( "stored_workflow_menu_entry", metadata, + Column( "id", Integer, primary_key=True ), + Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "order_index", Integer ) ) + +def upgrade(): + metadata.create_all() + +## def downgrade(): +## # Operations to reverse the above upgrade go here. +## pass diff --git a/lib/galaxy/model/migrate/versions/0002_metadata_file_table.py b/lib/galaxy/model/migrate/versions/0002_metadata_file_table.py new file mode 100644 index 00000000000..acd67e16e27 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0002_metadata_file_table.py @@ -0,0 +1,31 @@ +from sqlalchemy import * +from migrate import * + +import datetime +now = datetime.datetime.utcnow + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import * + +import logging +log = logging.getLogger( __name__ ) + +metadata = MetaData( migrate_engine ) + +# New table in changeset 1568:0b022adfdc34 +MetadataFile_table = Table( "metadata_file", metadata, + Column( "id", Integer, primary_key=True ), + Column( "name", TEXT ), + Column( "hda_id", Integer, ForeignKey( "history_dataset_association.id" ), index=True, nullable=True ), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, index=True, default=now, onupdate=now ), + Column( "deleted", Boolean, index=True, default=False ), + Column( "purged", Boolean, index=True, default=False ) ) + +def upgrade(): + metadata.reflect() + MetadataFile_table.create() + +def downgrade(): + metadata.reflect() + MetadataFile_table.drop() diff --git a/manage_db.sh b/manage_db.sh new file mode 100644 index 00000000000..3201862a21e --- /dev/null +++ b/manage_db.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd `dirname $0` +python -ES ./scripts/manage_db.py $@ diff --git a/scripts/manage_db.py b/scripts/manage_db.py new file mode 100644 index 00000000000..5cd8893317f --- /dev/null +++ b/scripts/manage_db.py @@ -0,0 +1,45 @@ +import sys, os.path, logging + +new_path = [ os.path.join( os.getcwd(), "lib" ) ] +new_path.extend( sys.path[1:] ) # remove scripts/ from the path +sys.path = new_path + +from galaxy import eggs + +import pkg_resources +pkg_resources.require( "sqlalchemy-migrate" ) + +from migrate.versioning.shell import main +from ConfigParser import SafeConfigParser + +log = logging.getLogger( __name__ ) + +cp = SafeConfigParser() +cp.read( "universe_wsgi.ini" ) + +if cp.has_option( "app:main", "database_connection" ): + db_url = cp.get( "app:main", "database_connection" ) +elif cp.has_option( "app:main", "database_file" ): + db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % cp.get( "app:main", "database_file" ) +else: + db_url = None + +dialect_to_egg = { + "sqlite" : "pysqlite>=2", + "postgres" : "psycopg2", + "mysql" : "MySQL_python" +} +dialect = ( db_url.split( ':', 1 ) )[0] +try: + egg = dialect_to_egg[dialect] + try: + pkg_resources.require( egg ) + log.debug( "%s egg successfully loaded for %s dialect" % ( egg, dialect ) ) + except: + # If the module is in the path elsewhere (i.e. non-egg), it'll still load. + log.warning( "%s egg not found, but an attempt will be made to use %s anyway" % ( egg, dialect ) ) +except KeyError: + # Let this go, it could possibly work with db's we don't support + log.error( "database_connection contains an unknown SQLAlchemy database dialect: %s" % dialect ) + +main( repository='lib/galaxy/model/migrate', url=db_url )