opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

flake8 lib/galaxy/model/migrate/versions/ .

Browse Source 
Fix 1429 linting errors.
This commit is contained in:
Nicola Soranzo
2015-10-02 18:07:57 +01:00
parent 02962ae646
commit fd1fd5e73c
129 changed files with 1367 additions and 1714 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.ci/flake8_blacklist.txt
-2
View File
@@ -3,8 +3,6 @@ client/
database/
doc/patch.py
doc/source/conf.py
lib/galaxy/model/migrate/versions/
lib/galaxy/model/tool_shed_install/migrate/versions/
lib/galaxy/util/jstree.py
lib/pkg_resources.py
scripts/api/
lib/galaxy/model/migrate/versions/0001_initial_tables.py
+12 -24
View File
@@ -1,15 +1,13 @@
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
import logging
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Numeric, String, Table, TEXT
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString
import logging
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
# Tables as of changeset 1464:c7acaa1bb88f
@@ -138,8 +136,7 @@ GalaxySession_table = Table( "galaxy_session", metadata,
Column( "current_history_id", Integer, ForeignKey( "history.id" ), nullable=True ),
Column( "session_key", TrimmedString( 255 ), index=True, unique=True ),
Column( "is_valid", Boolean, default=False ),
Column( "prev_session_id", Integer )
)
Column( "prev_session_id", Integer ) )
GalaxySessionToHistoryAssociation_table = Table( "galaxy_session_to_history", metadata,
Column( "id", Integer, primary_key=True ),
@@ -155,8 +152,7 @@ StoredWorkflow_table = Table( "stored_workflow", metadata,
Column( "latest_workflow_id", Integer,
ForeignKey( "workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk' ), index=True ),
Column( "name", TEXT ),
Column( "deleted", Boolean, default=False ),
)
Column( "deleted", Boolean, default=False ) )
Workflow_table = Table( "workflow", metadata,
Column( "id", Integer, primary_key=True ),
@@ -165,8 +161,7 @@ Workflow_table = Table( "workflow", metadata,
Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True, nullable=False ),
Column( "name", TEXT ),
Column( "has_cycles", Boolean ),
Column( "has_errors", Boolean )
)
Column( "has_errors", Boolean ) )
WorkflowStep_table = Table( "workflow_step", metadata,
Column( "id", Integer, primary_key=True ),
@@ -180,22 +175,19 @@ WorkflowStep_table = Table( "workflow_step", metadata,
Column( "tool_errors", JSONType ),
Column( "position", JSONType ),
Column( "config", JSONType ),
Column( "order_index", Integer )
)
Column( "order_index", Integer ) )
WorkflowStepConnection_table = Table( "workflow_step_connection", metadata,
Column( "id", Integer, primary_key=True ),
Column( "output_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ),
Column( "input_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ),
Column( "output_name", TEXT ),
Column( "input_name", TEXT)
)
Column( "input_name", TEXT) )
StoredWorkflowUserShareAssociation_table = Table( "stored_workflow_user_share_connection", metadata,
Column( "id", Integer, primary_key=True ),
Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
)
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) )
StoredWorkflowMenuEntry_table = Table( "stored_workflow_menu_entry", metadata,
Column( "id", Integer, primary_key=True ),
@@ -203,11 +195,7 @@ StoredWorkflowMenuEntry_table = Table( "stored_workflow_menu_entry", metadata,
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "order_index", Integer ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.create_all()
## def downgrade(migrate_engine):
## metadata.bind = migrate_engine
## # Operations to reverse the above upgrade go here.
## pass
lib/galaxy/model/migrate/versions/0002_metadata_file_table.py
+6 -9
View File
@@ -1,15 +1,10 @@
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
# New table in changeset 1568:0b022adfdc34
@@ -22,11 +17,13 @@ MetadataFile_table = Table( "metadata_file", metadata,
Column( "deleted", Boolean, index=True, default=False ),
Column( "purged", Boolean, index=True, default=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
MetadataFile_table.create()
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
+40 -53
View File
@@ -1,13 +1,15 @@
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
import sys, logging
from migrate import ForeignKeyConstraint
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Index, Integer, MetaData, String, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -16,9 +18,6 @@ formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData()
# New tables as of changeset 2341:5498ac35eedd
@@ -131,13 +130,13 @@ DefaultHistoryPermissions_table = Table( "default_history_permissions", metadata
LibraryDataset_table = Table( "library_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ), # current version of dataset, if null, there is not a current version selected
Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ),
Column( "order_id", Integer ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), key="_name" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "info", TrimmedString( 255 ), key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "name", TrimmedString( 255 ), key="_name" ), # when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "info", TrimmedString( 255 ), key="_info" ), # when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "deleted", Boolean, index=True, default=False ) )
LibraryDatasetDatasetAssociation_table = Table( "library_dataset_dataset_association", metadata,
@@ -173,7 +172,7 @@ Library_table = Table( "library", metadata,
LibraryFolder_table = Table( "library_folder", metadata,
Column( "id", Integer, primary_key=True ),
Column( "parent_id", Integer, ForeignKey( "library_folder.id" ), nullable = True, index=True ),
Column( "parent_id", Integer, ForeignKey( "library_folder.id" ), nullable=True, index=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TEXT ),
@@ -304,35 +303,35 @@ JobExternalOutputMetadata_table = Table( "job_external_output_metadata", metadat
Column( "job_runner_external_pid", String( 255 ) ) )
Index( "ix_jeom_library_dataset_dataset_association_id", JobExternalOutputMetadata_table.c.library_dataset_dataset_association_id )
def upgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
# Load existing tables
metadata.reflect()
def nextval( table, col='id' ):
if migrate_engine.name == 'postgres':
if migrate_engine.name in ['postgres', 'postgresql']:
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
elif migrate_engine.name in ['mysql', 'sqlite']:
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def localtimestamp():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def boolean_false():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
# Add 2 new columns to the galaxy_user table
try:
@@ -441,7 +440,7 @@ def upgrade(migrate_engine):
LibraryDatasetDatasetAssociation_table = None
log.debug( "Failed loading table library_dataset_dataset_association" )
if migrate_engine.name != 'sqlite':
#Sqlite can't alter table add foreign key.
# Sqlite can't alter table add foreign key.
if MetadataFile_table is not None and LibraryDatasetDatasetAssociation_table is not None:
try:
cons = ForeignKeyConstraint( [MetadataFile_table.c.lda_id],
@@ -453,10 +452,10 @@ def upgrade(migrate_engine):
log.debug( "Adding foreign key constraint 'metadata_file_lda_id_fkey' to table 'metadata_file' failed: %s" % ( str( e ) ) )
# Make sure we have at least 1 user
cmd = "SELECT * FROM galaxy_user;"
users = db_session.execute( cmd ).fetchall()
users = migrate_engine.execute( cmd ).fetchall()
if users:
cmd = "SELECT * FROM role;"
roles = db_session.execute( cmd ).fetchall()
roles = migrate_engine.execute( cmd ).fetchall()
if not roles:
# Create private roles for each user - pass 1
cmd = \
@@ -471,13 +470,13 @@ def upgrade(migrate_engine):
"FROM galaxy_user " + \
"ORDER BY id;"
cmd = cmd % ( nextval('role'), localtimestamp(), localtimestamp(), boolean_false() )
db_session.execute( cmd )
migrate_engine.execute( cmd )
# Create private roles for each user - pass 2
if migrate_engine.name == 'postgres' or migrate_engine.name == 'sqlite':
if migrate_engine.name in ['postgres', 'postgresql', 'sqlite']:
cmd = "UPDATE role SET description = 'Private role for ' || description;"
elif migrate_engine.name == 'mysql':
cmd = "UPDATE role SET description = CONCAT( 'Private role for ', description );"
db_session.execute( cmd )
migrate_engine.execute( cmd )
# Create private roles for each user - pass 3
cmd = \
"INSERT INTO user_role_association " + \
@@ -490,7 +489,7 @@ def upgrade(migrate_engine):
"WHERE galaxy_user.email = role.name " + \
"ORDER BY galaxy_user.id;"
cmd = cmd % ( nextval('user_role_association'), localtimestamp(), localtimestamp() )
db_session.execute( cmd )
migrate_engine.execute( cmd )
# Create default permissions for each user
cmd = \
"INSERT INTO default_user_permissions " + \
@@ -502,7 +501,7 @@ def upgrade(migrate_engine):
"JOIN user_role_association ON user_role_association.user_id = galaxy_user.id " + \
"ORDER BY galaxy_user.id;"
cmd = cmd % nextval('default_user_permissions')
db_session.execute( cmd )
migrate_engine.execute( cmd )
# Create default history permissions for each active history associated with a user
cmd = \
@@ -515,7 +514,7 @@ def upgrade(migrate_engine):
"JOIN user_role_association ON user_role_association.user_id = history.user_id " + \
"WHERE history.purged = %s AND history.user_id IS NOT NULL;"
cmd = cmd % ( nextval('default_history_permissions'), boolean_false() )
db_session.execute( cmd )
migrate_engine.execute( cmd )
# Create "manage permissions" dataset_permissions for all activate-able datasets
cmd = \
"INSERT INTO dataset_permissions " + \
@@ -531,23 +530,11 @@ def upgrade(migrate_engine):
"JOIN user_role_association ON user_role_association.user_id = history.user_id " + \
"WHERE dataset.purged = %s AND history.user_id IS NOT NULL;"
cmd = cmd % ( nextval('dataset_permissions'), localtimestamp(), localtimestamp(), boolean_false() )
db_session.execute( cmd )
migrate_engine.execute( cmd )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
if migrate_engine.name == 'postgres':
# http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html
from sqlalchemy.databases import postgres
class PGCascadeSchemaDropper(postgres.PGSchemaDropper):
def visit_table(self, table):
for column in table.columns:
if column.default is not None:
self.traverse_single(column.default)
self.append("\nDROP TABLE " +
self.preparer.format_table(table) +
" CASCADE")
self.execute()
postgres.dialect.schemadropper = PGCascadeSchemaDropper
# Load existing tables
metadata.reflect()
# NOTE: all new data added in the upgrade method is eliminated here via table drops
lib/galaxy/model/migrate/versions/0004_indexes_and_defaults.py
+16 -13
View File
@@ -1,7 +1,7 @@
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
import sys, logging
import logging
import sys
from sqlalchemy import Index, MetaData, Table
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
@@ -13,18 +13,19 @@ log.addHandler( handler )
metadata = MetaData()
def upgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
User_table = Table( "galaxy_user", metadata, autoload=True )
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True )
def boolean_false():
if migrate_engine.name == 'postgresql' or migrate_engine.name == 'mysql':
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name)
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name)
# Load existing tables
metadata.reflect()
# Add 2 indexes to the galaxy_user table
@@ -42,13 +43,13 @@ def upgrade(migrate_engine):
cmd = "UPDATE galaxy_user SET deleted = %s WHERE deleted is null"
cmd = cmd % boolean_false()
try:
db_session.execute( cmd )
migrate_engine.execute( cmd )
except Exception, e:
log.debug( "Setting default data for galaxy_user.deleted column failed: %s" % ( str( e ) ) )
cmd = "UPDATE galaxy_user SET purged = %s WHERE purged is null"
cmd = cmd % boolean_false()
try:
db_session.execute( cmd )
migrate_engine.execute( cmd )
except Exception, e:
log.debug( "Setting default data for galaxy_user.purged column failed: %s" % ( str( e ) ) )
# Add 1 index to the history_dataset_association table
@@ -57,6 +58,8 @@ def upgrade(migrate_engine):
i.create()
except Exception, e:
log.debug( "Adding index 'ix_hda_copied_from_library_dataset_dataset_association_id' to history_dataset_association table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py
+133 -104
View File
@@ -2,31 +2,18 @@ import datetime
import errno
import logging
import os
import sys
import time
from sqlalchemy import and_, Boolean, Column, DateTime, false, ForeignKey, Integer, MetaData, not_, Numeric, Table, TEXT, true
from sqlalchemy.orm import backref, mapper, relation, scoped_session, sessionmaker
from galaxy.datatypes.metadata import MetadataCollection
from galaxy.model.custom_types import MetadataType, TrimmedString
from galaxy.util.bunch import Bunch
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
from sqlalchemy import and_, false, true
from sqlalchemy import *
now = datetime.datetime.utcnow
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
from galaxy.util.bunch import Bunch
metadata = MetaData()
context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) )
@@ -35,6 +22,7 @@ context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) )
def get_permitted_actions( **kwds ):
return Bunch()
def directory_hash_id( id ):
s = str( id )
l = len( s )
@@ -46,21 +34,22 @@ def directory_hash_id( id ):
# Drop the last three digits -- 1000 files per directory
padded = padded[:-3]
# Break into chunks of three
return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ]
return [ padded[i * 3:(i + 1) * 3] for i in range( len( padded ) // 3 ) ]
class Dataset( object ):
states = Bunch( NEW = 'new',
UPLOAD = 'upload',
QUEUED = 'queued',
RUNNING = 'running',
OK = 'ok',
EMPTY = 'empty',
ERROR = 'error',
DISCARDED = 'discarded' )
states = Bunch( NEW='new',
UPLOAD='upload',
QUEUED='queued',
RUNNING='running',
OK='ok',
EMPTY='empty',
ERROR='error',
DISCARDED='discarded' )
permitted_actions = get_permitted_actions( filter='DATASET' )
file_path = "/tmp/"
engine = None
def __init__( self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True ):
self.id = id
self.state = state
@@ -70,6 +59,7 @@ class Dataset( object ):
self.external_filename = external_filename
self._extra_files_path = extra_files_path
self.file_size = file_size
def get_file_name( self ):
if not self.external_filename:
assert self.id is not None, "ID must be set before filename used (commit the object)"
@@ -92,23 +82,26 @@ class Dataset( object ):
filename = self.external_filename
# Make filename absolute
return os.path.abspath( filename )
def set_file_name ( self, filename ):
def set_file_name( self, filename ):
if not filename:
self.external_filename = None
else:
self.external_filename = filename
file_name = property( get_file_name, set_file_name )
@property
def extra_files_path( self ):
if self._extra_files_path:
path = self._extra_files_path
else:
path = os.path.join( self.file_path, "dataset_%d_files" % self.id )
#only use path directly under self.file_path if it exists
# only use path directly under self.file_path if it exists
if not os.path.exists( path ):
path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id )
# Make path absolute
return os.path.abspath( path )
def get_size( self ):
"""Returns the size of the data on disk"""
if self.file_size:
@@ -118,6 +111,7 @@ class Dataset( object ):
return os.path.getsize( self.file_name )
except OSError:
return 0
def set_size( self ):
"""Returns the size of the data on disk"""
try:
@@ -125,12 +119,15 @@ class Dataset( object ):
self.file_size = os.path.getsize( self.file_name )
except OSError:
self.file_size = 0
def has_data( self ):
"""Detects whether there is any data"""
return self.get_size() > 0
def mark_deleted( self, include_children=True ):
self.deleted = True
# FIXME: sqlalchemy will replace this
def _delete(self):
"""Remove the file that corresponds to this data"""
try:
@@ -138,13 +135,15 @@ class Dataset( object ):
except OSError, e:
log.critical('%s delete error %s' % (self.__class__.__name__, e))
class DatasetInstance( object ):
"""A base class for all 'dataset instances', HDAs, LDAs, etc"""
states = Dataset.states
permitted_actions = Dataset.permitted_actions
def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None,
dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None,
parent_id=None, validation_errors=None, visible=True, create_dataset = False ):
parent_id=None, validation_errors=None, visible=True, create_dataset=False ):
self.name = name or "Unnamed dataset"
self.id = id
self.info = info
@@ -153,7 +152,7 @@ class DatasetInstance( object ):
self.extension = extension
self.designation = designation
self.metadata = metadata or dict()
if dbkey: #dbkey is stored in metadata, only set if non-zero, or else we could clobber one supplied by input 'metadata'
if dbkey: # dbkey is stored in metadata, only set if non-zero, or else we could clobber one supplied by input 'metadata'
self.dbkey = dbkey
self.deleted = deleted
self.visible = visible
@@ -165,42 +164,51 @@ class DatasetInstance( object ):
self.dataset = dataset
self.parent_id = parent_id
self.validation_errors = validation_errors
@property
def ext( self ):
return self.extension
def get_dataset_state( self ):
return self.dataset.state
def set_dataset_state ( self, state ):
def set_dataset_state( self, state ):
self.dataset.state = state
context.add( self.dataset )
context.flush() #flush here, because hda.flush() won't flush the Dataset object
context.flush() # flush here, because hda.flush() won't flush the Dataset object
state = property( get_dataset_state, set_dataset_state )
def get_file_name( self ):
return self.dataset.get_file_name()
def set_file_name (self, filename):
def set_file_name(self, filename):
return self.dataset.set_file_name( filename )
file_name = property( get_file_name, set_file_name )
@property
def extra_files_path( self ):
return self.dataset.extra_files_path
@property
def datatype( self ):
return datatypes_registry.get_datatype_by_extension( self.extension )
def get_metadata( self ):
if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent != self: #using weakref to store parent (to prevent circ ref), does a context.clear() cause parent to be invalidated, while still copying over this non-database attribute?
if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent != self: # using weakref to store parent (to prevent circ ref), does a context.clear() cause parent to be invalidated, while still copying over this non-database attribute?
self._metadata_collection = MetadataCollection( self )
return self._metadata_collection
def set_metadata( self, bunch ):
# Needs to accept a MetadataCollection, a bunch, or a dict
self._metadata = self.metadata.make_dict_copy( bunch )
metadata = property( get_metadata, set_metadata )
# This provide backwards compatibility with using the old dbkey
# field in the database. That field now maps to "old_dbkey" (see mapping.py).
def get_dbkey( self ):
dbkey = self.metadata.dbkey
if not isinstance(dbkey, list): dbkey = [dbkey]
if dbkey in [[None], []]: return "?"
if not isinstance(dbkey, list):
dbkey = [dbkey]
if dbkey in [[None], []]:
return "?"
return dbkey[0]
def set_dbkey( self, value ):
if "dbkey" in self.datatype.metadata_spec:
if not isinstance(value, list):
@@ -208,83 +216,95 @@ class DatasetInstance( object ):
else:
self.metadata.dbkey = value
dbkey = property( get_dbkey, set_dbkey )
def change_datatype( self, new_ext ):
self.clear_associated_files()
datatypes_registry.change_datatype( self, new_ext )
def get_size( self ):
"""Returns the size of the data on disk"""
return self.dataset.get_size()
def set_size( self ):
"""Returns the size of the data on disk"""
return self.dataset.set_size()
def has_data( self ):
"""Detects whether there is any data"""
return self.dataset.has_data()
def get_raw_data( self ):
"""Returns the full data. To stream it open the file_name and read/write as needed"""
return self.datatype.get_raw_data( self )
def write_from_stream( self, stream ):
"""Writes data from a stream"""
self.datatype.write_from_stream(self, stream)
def set_raw_data( self, data ):
"""Saves the data on the disc"""
self.datatype.set_raw_data(self, data)
def get_mime( self ):
"""Returns the mime type of the data"""
return datatypes_registry.get_mimetype_by_extension( self.extension.lower() )
def set_peek( self, is_multi_byte=False ):
return self.datatype.set_peek( self, is_multi_byte=is_multi_byte )
def init_meta( self, copy_from=None ):
return self.datatype.init_meta( self, copy_from=copy_from )
def set_meta( self, **kwd ):
self.clear_associated_files( metadata_safe = True )
self.clear_associated_files( metadata_safe=True )
return self.datatype.set_meta( self, **kwd )
def missing_meta( self, **kwd ):
return self.datatype.missing_meta( self, **kwd )
def as_display_type( self, type, **kwd ):
return self.datatype.as_display_type( self, type, **kwd )
def display_peek( self ):
return self.datatype.display_peek( self )
def display_name( self ):
return self.datatype.display_name( self )
def display_info( self ):
return self.datatype.display_info( self )
def get_converted_files_by_type( self, file_type ):
valid = []
for assoc in self.implicitly_converted_datasets:
if not assoc.deleted and assoc.type == file_type:
valid.append( assoc.dataset )
return valid
def clear_associated_files( self, metadata_safe = False, purge = False ):
def clear_associated_files( self, metadata_safe=False, purge=False ):
raise 'Unimplemented'
def get_child_by_designation(self, designation):
for child in self.children:
if child.designation == designation:
return child
return None
def get_converter_types(self):
return self.datatype.get_converter_types( self, datatypes_registry)
def find_conversion_destination( self, accepted_formats, **kwd ):
"""Returns ( target_ext, exisiting converted dataset )"""
return self.datatype.find_conversion_destination( self, accepted_formats, datatypes_registry, **kwd )
def add_validation_error( self, validation_error ):
self.validation_errors.append( validation_error )
def extend_validation_errors( self, validation_errors ):
self.validation_errors.extend(validation_errors)
def mark_deleted( self, include_children=True ):
self.deleted = True
if include_children:
for child in self.children:
child.mark_deleted()
def mark_undeleted( self, include_children=True ):
self.deleted = False
if include_children:
for child in self.children:
child.mark_undeleted()
def undeletable( self ):
if self.purged:
return False
return True
@property
def source_library_dataset( self ):
def get_source( dataset ):
@@ -305,10 +325,10 @@ class DatasetInstance( object ):
class HistoryDatasetAssociation( DatasetInstance ):
def __init__( self,
hid = None,
history = None,
copied_from_history_dataset_association = None,
copied_from_library_dataset_dataset_association = None,
hid=None,
history=None,
copied_from_history_dataset_association=None,
copied_from_library_dataset_dataset_association=None,
**kwd ):
DatasetInstance.__init__( self, **kwd )
self.hid = hid
@@ -316,7 +336,8 @@ class HistoryDatasetAssociation( DatasetInstance ):
self.history = history
self.copied_from_history_dataset_association = copied_from_history_dataset_association
self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association
def copy( self, copy_children = False, parent_id = None, target_history = None ):
def copy( self, copy_children=False, parent_id=None, target_history=None ):
hda = HistoryDatasetAssociation( hid=self.hid,
name=self.name,
info=self.info,
@@ -324,12 +345,12 @@ class HistoryDatasetAssociation( DatasetInstance ):
peek=self.peek,
extension=self.extension,
dbkey=self.dbkey,
dataset = self.dataset,
dataset=self.dataset,
visible=self.visible,
deleted=self.deleted,
parent_id=parent_id,
copied_from_history_dataset_association=self,
history = target_history )
history=target_history )
context.add( hda )
context.flush()
hda.set_size()
@@ -337,12 +358,13 @@ class HistoryDatasetAssociation( DatasetInstance ):
hda.metadata = self.metadata
if copy_children:
for child in self.children:
child_copy = child.copy( copy_children = copy_children, parent_id = hda.id )
child.copy( copy_children=copy_children, parent_id=hda.id )
if not self.datatype.copy_safe_peek:
# In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
hda.set_peek()
context.flush()
return hda
def to_library_dataset_dataset_association( self, target_folder, replace_dataset=None, parent_id=None ):
if replace_dataset:
# The replace_dataset param ( when not None ) refers to a LibraryDataset that is being replaced with a new version.
@@ -379,18 +401,18 @@ class HistoryDatasetAssociation( DatasetInstance ):
context.add( library_dataset )
context.flush()
for child in self.children:
child_copy = child.to_library_dataset_dataset_association( target_folder=target_folder, replace_dataset=replace_dataset, parent_id=ldda.id )
child.to_library_dataset_dataset_association( target_folder=target_folder, replace_dataset=replace_dataset, parent_id=ldda.id )
if not self.datatype.copy_safe_peek:
# In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
ldda.set_peek()
context.flush()
return ldda
def clear_associated_files( self, metadata_safe = False, purge = False ):
def clear_associated_files( self, metadata_safe=False, purge=False ):
# metadata_safe = True means to only clear when assoc.metadata_safe == False
for assoc in self.implicitly_converted_datasets:
if not metadata_safe or not assoc.metadata_safe:
assoc.clear( purge = purge )
assoc.clear( purge=purge )
class LibraryDatasetDatasetAssociation( DatasetInstance ):
@@ -405,6 +427,7 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association
self.library_dataset = library_dataset
self.user = user
def to_history_dataset_association( self, target_history, parent_id=None ):
hid = target_history._next_hid()
hda = HistoryDatasetAssociation( name=self.name,
@@ -420,16 +443,17 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
copied_from_library_dataset_dataset_association=self,
history=target_history,
hid=hid )
context.flush()
hda.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id
for child in self.children:
child_copy = child.to_history_dataset_association( target_history=target_history, parent_id=hda.id )
if not self.datatype.copy_safe_peek:
hda.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
context.add( hda )
context.flush()
hda.metadata = self.metadata # need to set after flushed, as MetadataFiles require dataset.id
for child in self.children:
child.to_history_dataset_association( target_history=target_history, parent_id=hda.id )
if not self.datatype.copy_safe_peek:
hda.set_peek() # in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
context.flush()
return hda
def copy( self, copy_children = False, parent_id = None, target_folder = None ):
def copy( self, copy_children=False, parent_id=None, target_folder=None ):
ldda = LibraryDatasetDatasetAssociation( name=self.name,
info=self.info,
blurb=self.blurb,
@@ -444,18 +468,20 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
folder=target_folder )
context.add( ldda )
context.flush()
# Need to set after flushed, as MetadataFiles require dataset.id
# Need to set after flushed, as MetadataFiles require dataset.id
ldda.metadata = self.metadata
if copy_children:
for child in self.children:
child_copy = child.copy( copy_children = copy_children, parent_id = ldda.id )
child.copy( copy_children=copy_children, parent_id=ldda.id )
if not self.datatype.copy_safe_peek:
# In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
# In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs
ldda.set_peek()
context.flush()
return ldda
def clear_associated_files( self, metadata_safe = False, purge = False ):
def clear_associated_files( self, metadata_safe=False, purge=False ):
return
def get_library_item_info_templates( self, template_list=[], restrict=False ):
# If restrict is True, we'll return only those templates directly associated with this LibraryDatasetDatasetAssociation
if self.library_dataset_dataset_info_template_associations:
@@ -464,7 +490,6 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ):
return template_list
class LibraryDataset( object ):
# This class acts as a proxy to the currently selected LDDA
def __init__( self, folder=None, order_id=None, name=None, info=None, library_dataset_dataset_association=None, **kwd ):
@@ -473,11 +498,13 @@ class LibraryDataset( object ):
self.name = name
self.info = info
self.library_dataset_dataset_association = library_dataset_dataset_association
def set_library_dataset_dataset_association( self, ldda ):
self.library_dataset_dataset_association = ldda
ldda.library_dataset = self
context.add_all( ( self, ldda ) )
context.flush()
def get_info( self ):
if self.library_dataset_dataset_association:
return self.library_dataset_dataset_association.info
@@ -485,9 +512,11 @@ class LibraryDataset( object ):
return self._info
else:
return 'no info'
def set_info( self, info ):
self._info = info
info = property( get_info, set_info )
def get_name( self ):
if self.library_dataset_dataset_association:
return self.library_dataset_dataset_association.name
@@ -495,19 +524,24 @@ class LibraryDataset( object ):
return self._name
else:
return 'Unnamed dataset'
def set_name( self, name ):
self._name = name
name = property( get_name, set_name )
def display_name( self ):
self.library_dataset_dataset_association.display_name()
def get_purged( self ):
return self.library_dataset_dataset_association.dataset.purged
def set_purged( self, purged ):
if purged:
raise Exception( "Not implemented" )
if not purged and self.purged:
raise Exception( "Cannot unpurge once purged" )
purged = property( get_purged, set_purged )
def get_library_item_info_templates( self, template_list=[], restrict=False ):
# If restrict is True, we'll return only those templates directly associated with this LibraryDataset
if self.library_dataset_info_template_associations:
@@ -516,7 +550,7 @@ class LibraryDataset( object ):
self.folder.get_library_item_info_templates( template_list, restrict )
return template_list
##tables
# tables
Dataset.table = Table( "dataset", metadata,
@@ -532,7 +566,6 @@ Dataset.table = Table( "dataset", metadata,
Column( 'file_size', Numeric( 15, 0 ) ) )
HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ),
@@ -575,17 +608,16 @@ LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_associa
LibraryDataset.table = Table( "library_dataset", metadata,
Column( "id", Integer, primary_key=True ),
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected
Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ), # current version of dataset, if null, there is not a current version selected
Column( "order_id", Integer ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), key="_name" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "info", TrimmedString( 255 ), key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "name", TrimmedString( 255 ), key="_name" ), # when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "info", TrimmedString( 255 ), key="_info" ), # when not None/null this will supercede display in library (but not when imported into user's history?)
Column( "deleted", Boolean, index=True, default=False ) )
##mappers
# mappers
mapper( Dataset, Dataset.table,
@@ -601,8 +633,7 @@ mapper( Dataset, Dataset.table,
primaryjoin=( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) ),
active_library_associations=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( ( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( LibraryDatasetDatasetAssociation.table.c.deleted == false() ) ) )
) )
primaryjoin=( ( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( LibraryDatasetDatasetAssociation.table.c.deleted == false() ) ) ) ) )
mapper( HistoryDatasetAssociation, HistoryDatasetAssociation.table,
@@ -625,13 +656,12 @@ mapper( HistoryDatasetAssociation, HistoryDatasetAssociation.table,
backref=backref( "parent", primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ),
visible_children=relation(
HistoryDatasetAssociation,
primaryjoin=( ( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ) & ( HistoryDatasetAssociation.table.c.visible == true() ) ) )
) )
primaryjoin=( ( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ) & ( HistoryDatasetAssociation.table.c.visible == true() ) ) ) ) )
mapper( LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table,
properties=dict(
dataset=relation( Dataset ),
library_dataset = relation( LibraryDataset,
library_dataset=relation( LibraryDataset,
primaryjoin=( LibraryDatasetDatasetAssociation.table.c.library_dataset_id == LibraryDataset.table.c.id ) ),
copied_to_library_dataset_dataset_associations=relation(
LibraryDatasetDatasetAssociation,
@@ -647,14 +677,12 @@ mapper( LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table
backref=backref( "parent", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ),
visible_children=relation(
LibraryDatasetDatasetAssociation,
primaryjoin=( ( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ) & ( LibraryDatasetDatasetAssociation.table.c.visible == true() ) ) )
) )
primaryjoin=( ( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ) & ( LibraryDatasetDatasetAssociation.table.c.visible == true() ) ) ) ) )
mapper( LibraryDataset, LibraryDataset.table,
properties=dict(
library_dataset_dataset_association=relation( LibraryDatasetDatasetAssociation, primaryjoin=( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ),
expired_datasets = relation( LibraryDatasetDatasetAssociation, foreign_keys=[LibraryDataset.table.c.id,LibraryDataset.table.c.library_dataset_dataset_association_id ], primaryjoin=( ( LibraryDataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.library_dataset_id ) & ( not_( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ) ), viewonly=True, uselist=True )
) )
expired_datasets=relation( LibraryDatasetDatasetAssociation, foreign_keys=[LibraryDataset.table.c.id, LibraryDataset.table.c.library_dataset_dataset_association_id ], primaryjoin=( ( LibraryDataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.library_dataset_id ) & ( not_( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ) ), viewonly=True, uselist=True ) ) )
def __guess_dataset_by_filename( filename ):
@@ -662,12 +690,13 @@ def __guess_dataset_by_filename( filename ):
try:
fields = os.path.split( filename )
if fields:
if fields[-1].startswith( 'dataset_' ) and fields[-1].endswith( '.dat' ): #dataset_%d.dat
if fields[-1].startswith( 'dataset_' ) and fields[-1].endswith( '.dat' ): # dataset_%d.dat
return Dataset.get( int( fields[-1][ len( 'dataset_' ): -len( '.dat' ) ] ) )
except:
pass #some parsing error, we can't guess Dataset
pass # some parsing error, we can't guess Dataset
return None
def upgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Fixing a discrepancy concerning deleted shared history items." )
@@ -678,14 +707,14 @@ def upgrade(migrate_engine):
if not dataset_instance.deleted:
dataset.deleted = False
if dataset.file_size in [ None, 0 ]:
dataset.set_size() #Restore filesize
dataset.set_size() # Restore filesize
affected_items += 1
break
context.flush()
log.debug( "%i items affected, and restored." % ( affected_items ) )
log.debug( "Time elapsed: %s" % ( time.time() - start_time ) )
#fix share before hda
# fix share before hda
log.debug( "Fixing a discrepancy concerning cleaning up deleted history items shared before HDAs." )
dataset_by_filename = {}
changed_associations = 0
@@ -695,23 +724,23 @@ def upgrade(migrate_engine):
guessed_dataset = dataset_by_filename[ dataset.file_name ]
else:
guessed_dataset = __guess_dataset_by_filename( dataset.file_name )
if guessed_dataset and dataset.file_name != guessed_dataset.file_name:#not os.path.samefile( dataset.file_name, guessed_dataset.file_name ):
if guessed_dataset and dataset.file_name != guessed_dataset.file_name: # not os.path.samefile( dataset.file_name, guessed_dataset.file_name ):
guessed_dataset = None
dataset_by_filename[ dataset.file_name ] = guessed_dataset
if guessed_dataset is not None and guessed_dataset.id != dataset.id: #could we have a self referential dataset?
if guessed_dataset is not None and guessed_dataset.id != dataset.id: # could we have a self referential dataset?
for dataset_instance in dataset.history_associations + dataset.library_associations:
dataset_instance.dataset = guessed_dataset
changed_associations += 1
#mark original Dataset as deleted and purged, it is no longer in use, but do not delete file_name contents
# mark original Dataset as deleted and purged, it is no longer in use, but do not delete file_name contents
dataset.deleted = True
dataset.external_filename = "Dataset was result of share before HDA, and has been replaced: %s mapped to Dataset %s" % ( dataset.external_filename, guessed_dataset.id )
dataset.purged = True #we don't really purge the file here, but we mark it as purged, since this dataset is now defunct
dataset.purged = True # we don't really purge the file here, but we mark it as purged, since this dataset is now defunct
context.flush()
log.debug( "%i items affected, and restored." % ( changed_associations ) )
log.debug( "Time elapsed: %s" % ( time.time() - start_time ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Downgrade is not possible." )
lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py
+7 -4
View File
@@ -2,10 +2,11 @@
This migration script changes certain values in the history_dataset_association.extension
column, specifically 'qual' is chaged to be 'qual454'.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
import sys, logging
import logging
import sys
from sqlalchemy import Index, MetaData, Table
from sqlalchemy.orm import scoped_session, sessionmaker
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
@@ -17,6 +18,7 @@ log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script changes certain values in the history_dataset_association.extension"
@@ -55,6 +57,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Dropping index 'ix_hda_extension' to history_dataset_association table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0007_sharing_histories.py
+9 -7
View File
@@ -3,11 +3,11 @@ This migration script creates the new history_user_share_association table, and
a new boolean type column to the history table. This provides support for sharing
histories in the same way that workflows are shared.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import sys, logging
import logging
import sys
from sqlalchemy import Boolean, Column, ForeignKey, Integer, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
@@ -19,6 +19,7 @@ log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script creates the new history_user_share_association table, and adds"
@@ -29,8 +30,8 @@ def display_migration_details():
HistoryUserShareAssociation_table = Table( "history_user_share_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
)
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -56,6 +57,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Adding column 'importable' to history table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# Load existing tables
lib/galaxy/model/migrate/versions/0008_galaxy_forms.py
+12 -14
View File
@@ -9,16 +9,17 @@ This migration script adds the following new tables for supporting Galaxy forms:
7) sample_state
8) sample_event
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
import sys, logging
from migrate import ForeignKeyConstraint
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import JSONType, TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -26,12 +27,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script adds the following new tables for supporting Galaxy forms:"
@@ -115,8 +113,8 @@ SampleEvent_table = Table('sample_event', metadata,
Column( "sample_state_id", Integer, ForeignKey( "sample_state.id" ), index=True ),
Column( "comment", TEXT ) )
def upgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
@@ -166,8 +164,8 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating sample_event table failed: %s" % str( e ) )
def downgrade(migrate_engine):
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.bind = migrate_engine
# Load existing tables
metadata.reflect()
lib/galaxy/model/migrate/versions/0009_request_table.py
+9 -8
View File
@@ -3,13 +3,12 @@ This migration script adds a new column to 2 tables:
1) a new boolean type column named 'submitted' to the 'request' table
2) a new string type column named 'bar_code' to the 'sample' table
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import sys, logging
from galaxy.model.custom_types import *
from sqlalchemy.exc import *
import logging
import sys
from sqlalchemy import Boolean, Column, MetaData, Table
from galaxy.model.custom_types import TrimmedString
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
@@ -21,6 +20,7 @@ log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script adds a new column to 2 tables:"
@@ -28,6 +28,7 @@ def display_migration_details():
print "2) a new string type column named 'bar_code' to the 'sample' table"
print "========================================"
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -40,7 +41,6 @@ def upgrade(migrate_engine):
try:
col = Column( 'submitted', Boolean, default=False )
col.create( Request_table)
#col.create( Request_table, index_name='ix_request_submitted')
assert col is Request_table.c.submitted
except Exception, e:
log.debug( "Adding column 'submitted' to request table failed: %s" % ( str( e ) ) )
@@ -54,6 +54,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Adding column 'bar_code' to sample table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py
+11 -12
View File
@@ -8,16 +8,16 @@ history_dataset_association_display_at_authorization table failed: (Operational
(1059, "Identifier name 'ix_history_dataset_association_display_at_authorization_update_time'
is too long
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
import sys, logging
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -25,12 +25,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script adds the history_dataset_association_display_at_authorization table, which"
@@ -50,6 +47,7 @@ HistoryDatasetAssociationDisplayAtAuthorization_table = Table( "history_dataset_
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "site", TrimmedString( 255 ) ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -60,6 +58,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating history_dataset_association_display_at_authorization table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# Load existing tables
lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py
+11 -12
View File
@@ -3,16 +3,16 @@ This script fixes a problem introduced in 0010_hda_display_at_atuhz_table.py. M
name length limit and thus the index "ix_hdadaa_history_dataset_association_id" has to be
manually created.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
import sys, logging
from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -20,12 +20,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This script fixes a problem introduced in the previous migration script ( 9->10 ). MySQL"
@@ -41,6 +38,7 @@ HistoryDatasetAssociationDisplayAtAuthorization_table = Table( "history_dataset_
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "site", TrimmedString( 255 ) ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -53,6 +51,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Adding index 'ix_hdadaa_history_dataset_association_id' to table 'history_dataset_association_display_at_authorization' table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
if migrate_engine.name == 'mysql':
lib/galaxy/model/migrate/versions/0012_user_address.py
+13 -12
View File
@@ -4,17 +4,17 @@ a user can select from a list of his addresses to associate with the request. T
drops the request.submitted column which was boolean and replaces it with a request.state column
which is a string, allowing for more flexibility with request states.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import sys, logging
# Need our custom types, but don't import anything else from model
import logging
import sys
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -22,9 +22,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This script adds a new user_address table that is currently only used with sample requests, where"
@@ -50,8 +50,8 @@ UserAddress_table = Table( "user_address", metadata,
Column( "deleted", Boolean, index=True, default=False ),
Column( "purged", Boolean, index=True, default=False ) )
def upgrade(migrate_engine):
#raise Exception
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
@@ -84,12 +84,13 @@ def upgrade(migrate_engine):
log.debug( "Failed loading table request" )
if Request_table is not None:
if migrate_engine.name != 'sqlite':
#DBTODO drop from table doesn't work in sqlite w/ sqlalchemy-migrate .6+
# DBTODO drop from table doesn't work in sqlite w/ sqlalchemy-migrate .6+
Request_table.c.submitted.drop()
col = Column( "state", TrimmedString( 255 ), index=True )
col.create( Request_table, index_name='ix_request_state')
assert col is Request_table.c.state
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0013_change_lib_item_templates_to_forms.py
+8 -20
View File
@@ -15,12 +15,11 @@ If using mysql, this script will throw an (OperationalError) exception due to a
the library_dataset_dataset_info_association table, which is OK because the script creates an index
with a shortened name.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import sys, logging
import logging
import sys
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
@@ -29,9 +28,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script eliminates all of the tables that were used for the 1st version of the"
@@ -68,24 +67,12 @@ LibraryDatasetDatasetInfoAssociation_table = Table( 'library_dataset_dataset_inf
Column( "form_definition_id", Integer, ForeignKey( "form_definition.id" ), index=True ),
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
# Load existing tables
metadata.reflect()
if migrate_engine.name == 'postgres':
# http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html
from sqlalchemy.databases import postgres
class PGCascadeSchemaDropper(postgres.PGSchemaDropper):
def visit_table(self, table):
for column in table.columns:
if column.default is not None:
self.traverse_single(column.default)
self.append("\nDROP TABLE " +
self.preparer.format_table(table) +
" CASCADE")
self.execute()
postgres.dialect.schemadropper = PGCascadeSchemaDropper
# Drop all of the original library_item_info tables
# NOTE: all existing library item into template data is eliminated here via table drops
try:
@@ -251,6 +238,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Adding index 'ix_lddaia_ldda_id' to table 'library_dataset_dataset_info_association' table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Downgrade is not possible." )
lib/galaxy/model/migrate/versions/0014_pages.py
+9 -13
View File
@@ -3,17 +3,13 @@ Migration script to add support for "Pages".
1) Creates Page and PageRevision tables
2) Adds username column to User table
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, MetaData, String, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
Page_table = Table( "page", metadata,
@@ -24,8 +20,7 @@ Page_table = Table( "page", metadata,
Column( "latest_revision_id", Integer,
ForeignKey( "page_revision.id", use_alter=True, name='page_latest_revision_id_fk' ), index=True ),
Column( "title", TEXT ),
Column( "slug", TEXT, unique=True, index=True ),
)
Column( "slug", TEXT, unique=True, index=True ) )
PageRevision_table = Table( "page_revision", metadata,
Column( "id", Integer, primary_key=True ),
@@ -33,8 +28,8 @@ PageRevision_table = Table( "page_revision", metadata,
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "page_id", Integer, ForeignKey( "page.id" ), index=True, nullable=False ),
Column( "title", TEXT ),
Column( "content", TEXT )
)
Column( "content", TEXT ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -51,7 +46,7 @@ def upgrade(migrate_engine):
Page_table.create()
if migrate_engine.name == 'mysql':
# Create slug index manually afterward.
i = Index( "ix_page_slug", Page_table.c.slug, mysql_length = 200)
i = Index( "ix_page_slug", Page_table.c.slug, mysql_length=200)
i.create()
except Exception, ex:
log.debug(ex)
@@ -67,6 +62,7 @@ def upgrade(migrate_engine):
col.create( User_table, index_name='ix_user_username', unique_name='username' )
assert col is User_table.c.username
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0015_tagging.py
+9 -8
View File
@@ -9,21 +9,20 @@ history_dataset_association_tag_association table failed: (OperationalError)
(1059, "Identifier name 'ix_history_dataset_association_tag_association_history_dataset_association_id'
is too long)
"""
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
import logging
from migrate import UniqueConstraint
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import TrimmedString
import logging
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def display_migration_details():
print ""
print "This migration script adds the tables necessary to support tagging of histories,"
@@ -66,6 +65,7 @@ HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_associat
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -91,6 +91,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating history_dataset_association_tag_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py
+8 -8
View File
@@ -3,21 +3,19 @@ This script fixes a problem introduced in 0015_tagging.py. MySQL has a name leng
limit and thus the index "ix_hda_ta_history_dataset_association_id" has to be
manually created.
"""
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
import logging
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import TrimmedString
import logging
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def display_migration_details():
print ""
print "This script fixes a problem introduced in 0015_tagging.py. MySQL has a"
@@ -31,6 +29,7 @@ HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_associat
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -42,6 +41,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Adding index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0017_library_item_indexes.py
+8 -5
View File
@@ -2,10 +2,10 @@
This script adds 3 indexes to table columns: library_folder.name,
library_dataset.name, library_dataset_dataset_association.name.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
import sys, logging
import logging
import sys
from sqlalchemy import Index, MetaData, Table
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
@@ -14,15 +14,16 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This script adds 3 indexes to table columns: library_folder.name,"
print "library_dataset.name, library_dataset_dataset_association.name."
print "========================================"
def upgrade(migrate_engine):
metadata.bind = migrate_engine
LibraryFolder_table = Table( "library_folder", metadata, autoload=True )
@@ -49,6 +50,8 @@ def upgrade(migrate_engine):
i.create()
except Exception, e:
log.debug( "Adding index 'ix_library_dataset_name' to library_dataset table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
log.debug( "Downgrade is not possible." )
lib/galaxy/model/migrate/versions/0018_ordered_tags_and_page_tags.py
+9 -11
View File
@@ -2,24 +2,20 @@
This migration script provides support for (a) ordering tags by recency and
(b) tagging pages. This script deletes all existing tags.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
import migrate.changeset
import datetime
now = datetime.datetime.utcnow
import logging
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table
from sqlalchemy.exc import OperationalError
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import TrimmedString
import logging
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def display_migration_details():
print ""
print "This migration script provides support for (a) ordering tags by recency and"
@@ -57,6 +53,7 @@ PageTagAssociation_table = Table( "page_tag_association", metadata,
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -104,6 +101,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating page_tag_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0019_request_library_folder.py
+12 -10
View File
@@ -1,14 +1,15 @@
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import sys, logging
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
from migrate import ForeignKeyConstraint
from sqlalchemy import Column, Integer, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import JSONType, TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -16,9 +17,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print """This script creates a request.folder_id column which is a foreign
@@ -26,6 +27,7 @@ key to the library_folder table. This also adds a 'type' and 'layout' column
to the form_definition table."""
print "========================================"
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
lib/galaxy/model/migrate/versions/0020_library_upload_job.py
+11 -10
View File
@@ -1,14 +1,12 @@
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import sys, logging
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
from migrate import ForeignKeyConstraint
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, String, Table
from sqlalchemy.exc import NoSuchTableError
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -19,6 +17,7 @@ log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print ""
print "========================================"
@@ -35,6 +34,7 @@ JobToOutputLibraryDatasetAssociation_table = Table( "job_to_output_library_datas
Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ),
Column( "name", String(255) ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -66,7 +66,7 @@ def upgrade(migrate_engine):
log.debug( "Failed loading table library_folder" )
# Add 1 foreign key constraint to the job table
if migrate_engine.name != 'sqlite':
#Sqlite can't alter-table-add-foreign-key
# Sqlite can't alter-table-add-foreign-key
if Job_table is not None and LibraryFolder_table is not None:
try:
cons = ForeignKeyConstraint( [Job_table.c.library_folder_id],
@@ -89,6 +89,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Adding index 'ix_dataset_state' to dataset table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0021_user_prefs.py
+7 -7
View File
@@ -1,18 +1,16 @@
"""
This migration script adds a user preferences table to Galaxy.
"""
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, Unicode
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def display_migration_details():
print ""
print "This migration script adds a user preferences table to Galaxy."
@@ -27,6 +25,7 @@ UserPreference_table = Table( "user_preference", metadata,
Column( "name", Unicode( 255 ), index=True),
Column( "value", Unicode( 1024 ) ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -37,6 +36,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating user_preference table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0022_visualization_tables.py
+7 -13
View File
@@ -2,17 +2,13 @@
Migration script to add support for storing visualizations.
1) Creates Visualization and VisualizationRevision tables
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
Visualization_table = Table( "visualization", metadata,
@@ -23,8 +19,7 @@ Visualization_table = Table( "visualization", metadata,
Column( "latest_revision_id", Integer,
ForeignKey( "visualization_revision.id", use_alter=True, name='visualization_latest_revision_id_fk' ), index=True ),
Column( "title", TEXT ),
Column( "type", TEXT )
)
Column( "type", TEXT ) )
VisualizationRevision_table = Table( "visualization_revision", metadata,
Column( "id", Integer, primary_key=True ),
@@ -32,8 +27,8 @@ VisualizationRevision_table = Table( "visualization_revision", metadata,
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "visualization_id", Integer, ForeignKey( "visualization.id" ), index=True, nullable=False ),
Column( "title", TEXT ),
Column( "config", TEXT )
)
Column( "config", TEXT ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -54,4 +49,3 @@ def downgrade(migrate_engine):
metadata.reflect()
Visualization_table.drop()
VisualizationRevision_table.drop()
lib/galaxy/model/migrate/versions/0023_page_published_and_deleted_columns.py
+6 -7
View File
@@ -2,16 +2,14 @@
Migration script to add columns for tracking whether pages are deleted and
publicly accessible.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -21,13 +19,14 @@ def upgrade(migrate_engine):
Page_table = Table( "page", metadata, autoload=True )
c = Column( "published", Boolean, index=True, default=False )
c.create( Page_table, index_name = 'ix_page_published' )
c.create( Page_table, index_name='ix_page_published' )
assert c is Page_table.c.published
c = Column( "deleted", Boolean, index=True, default=False )
c.create( Page_table, index_name='ix_page_deleted')
assert c is Page_table.c.deleted
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0024_page_slug_unique_constraint.py
+6 -10
View File
@@ -2,19 +2,16 @@
Remove unique constraint from page slugs to allow creating a page with
the same slug as a deleted page.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Index, MetaData, Table
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -38,8 +35,7 @@ def upgrade(migrate_engine):
Page_table.c.slug.alter( unique=False )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
#Page_table = Table( "page", metadata, autoload=True )
#Page_table.c.slug.alter( unique=True )
lib/galaxy/model/migrate/versions/0025_user_info.py
+12 -8
View File
@@ -1,16 +1,15 @@
"""
This script adds a foreign key to the form_values table in the galaxy_user table
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import sys, logging
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
from migrate import ForeignKeyConstraint
from sqlalchemy import Column, Integer, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -21,10 +20,13 @@ log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This script adds a foreign key to the form_values table in the galaxy_user table"
print "========================================"
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -58,6 +60,8 @@ def upgrade(migrate_engine):
cons.create()
except Exception, e:
log.debug( "Adding foreign key constraint 'user_form_values_id_fk' to table 'galaxy_user' failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0026_cloud_tables.py
+7 -9
View File
@@ -1,17 +1,13 @@
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def display_migration_details():
print
print "========================================"
@@ -127,6 +123,7 @@ CloudProvider_table = Table( "cloud_provider", metadata,
Column( "path", TEXT ),
Column( "deleted", Boolean, default=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -143,6 +140,7 @@ def upgrade(migrate_engine):
CloudStore_table.create()
CloudSnapshot_table.create()
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0027_request_events.py
+19 -20
View File
@@ -2,21 +2,17 @@
This migration script adds the request_event table and
removes the state field in the request table
"""
# Need our custom types, but don't import anything else from model
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
import datetime
import sys
import logging
import sys
now = datetime.datetime.utcnow
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -24,10 +20,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This migration script adds the request_event table and"
@@ -40,23 +35,26 @@ RequestEvent_table = Table('request_event', metadata,
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "request_id", Integer, ForeignKey( "request.id" ), index=True ),
Column( "state", TrimmedString( 255 ), index=True ),
Column( "state", TrimmedString( 255 ), index=True ),
Column( "comment", TEXT ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
def localtimestamp():
if migrate_engine.name == 'postgresql' or migrate_engine.name == 'mysql':
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def nextval( table, col='id' ):
if migrate_engine.name == 'postgresql':
if migrate_engine.name in ['postgres', 'postgresql']:
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
elif migrate_engine.name in ['mysql', 'sqlite']:
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
@@ -94,6 +92,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Deleting column 'state' to request table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py
+8 -8
View File
@@ -3,16 +3,13 @@ This script adds the filename_override_metadata column to the JobExternalOutputM
allowing existing metadata files to be written when using external metadata and a cluster
set up with read-only access to database/files
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import sys, logging
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
from sqlalchemy import Column, MetaData, String, Table
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -23,6 +20,7 @@ log.addHandler( handler )
metadata = MetaData()
def display_migration_details():
print "========================================"
print "This script adds the filename_override_metadata column to the JobExternalOutputMetadata table,"
@@ -30,6 +28,7 @@ def display_migration_details():
print "set up with read-only access to database/files"
print "========================================"
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -43,6 +42,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Adding column 'filename_override_metadata' to job_external_output_metadata table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0029_user_actions.py
+7 -7
View File
@@ -1,18 +1,16 @@
"""
This migration script adds a user actions table to Galaxy.
"""
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, Unicode
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def display_migration_details():
print ""
print "This migration script adds a user actions table to Galaxy."
@@ -29,6 +27,7 @@ UserAction_table = Table( "user_action", metadata,
Column( "context", Unicode( 512 ) ),
Column( "params", Unicode( 1024 ) ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
@@ -39,6 +38,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating user_action table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0030_history_slug_column.py
+5 -17
View File
@@ -1,16 +1,14 @@
"""
Migration script to add column for a history slug.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, Index, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -19,7 +17,6 @@ def upgrade(migrate_engine):
History_table = Table( "history", metadata, autoload=True )
# Mysql needs manual index creation because of max length index.
if migrate_engine.name != 'mysql':
# Create slug column.
@@ -28,20 +25,11 @@ def upgrade(migrate_engine):
else:
c = Column( "slug", TEXT )
c.create( History_table , index_name='')
i = Index( "ix_history_slug", History_table.c.slug, mysql_length = 200)
i = Index( "ix_history_slug", History_table.c.slug, mysql_length=200)
i.create()
assert c is History_table.c.slug
## Create slug index.
#try:
#i = Index( "ix_history_slug", History_table.c.slug )
#i.create()
#except:
## Mysql doesn't have a named index, but alter should work
#History_table.c.slug.alter( unique=False )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0031_community_and_workflow_tags.py
+9 -15
View File
@@ -5,15 +5,11 @@ Migration script to (a) add and populate necessary columns for doing community t
SQLite does not support 'ALTER TABLE ADD FOREIGN KEY', so this script will generate error messages when run against \
SQLite; however, script does execute successfully against SQLite.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, Unicode
log = logging.getLogger( __name__ )
metadata = MetaData()
StoredWorkflowTagAssociation_table = Table( "stored_workflow_tag_association", metadata,
@@ -34,6 +30,7 @@ WorkflowTagAssociation_table = Table( "workflow_tag_association", metadata,
Column( "value", Unicode(255), index=True),
Column( "user_value", Unicode(255), index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -63,8 +60,7 @@ def upgrade(migrate_engine):
# Populate column so that user_id is the id of the user who owns the history (and, up to now, was the only person able to tag the history).
if c is HistoryTagAssociation_table.c.user_id:
migrate_engine.execute(
"UPDATE history_tag_association SET user_id=( SELECT user_id FROM history WHERE history_tag_association.history_id = history.id )"
)
"UPDATE history_tag_association SET user_id=( SELECT user_id FROM history WHERE history_tag_association.history_id = history.id )" )
if migrate_engine.name != 'sqlite':
# Create user_id column in history_dataset_association_tag_association table.
@@ -78,7 +74,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Adding user_id column to history_dataset_association_tag_association table failed: %s" % str( e ) )
else:
#In sqlite, we can no longer quietly fail to add foreign key.
# In sqlite, we can no longer quietly fail to add foreign key.
# Create user_id column in history_dataset_association_tag_association table.
HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_association_tag_association", metadata, autoload=True )
c = Column( "user_id", Integer)
@@ -93,8 +89,7 @@ def upgrade(migrate_engine):
# Populate column so that user_id is the id of the user who owns the history_dataset_association (and, up to now, was the only person able to tag the page).
if c is HistoryDatasetAssociationTagAssociation_table.c.user_id:
migrate_engine.execute(
"UPDATE history_dataset_association_tag_association SET user_id=( SELECT history.user_id FROM history, history_dataset_association WHERE history_dataset_association.history_id = history.id AND history_dataset_association.id = history_dataset_association_tag_association.history_dataset_association_id)"
)
"UPDATE history_dataset_association_tag_association SET user_id=( SELECT history.user_id FROM history, history_dataset_association WHERE history_dataset_association.history_id = history.id AND history_dataset_association.id = history_dataset_association_tag_association.history_dataset_association_id)" )
if migrate_engine.name != 'sqlite':
# Create user_id column in page_tag_association table.
PageTagAssociation_table = Table( "page_tag_association", metadata, autoload=True )
@@ -121,8 +116,7 @@ def upgrade(migrate_engine):
# Populate column so that user_id is the id of the user who owns the page (and, up to now, was the only person able to tag the page).
if c is PageTagAssociation_table.c.user_id:
migrate_engine.execute(
"UPDATE page_tag_association SET user_id=( SELECT user_id FROM page WHERE page_tag_association.page_id = page.id )"
)
"UPDATE page_tag_association SET user_id=( SELECT user_id FROM page WHERE page_tag_association.page_id = page.id )" )
# Create stored_workflow_tag_association table.
try:
@@ -138,6 +132,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating workflow_tag_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
@@ -150,7 +145,6 @@ def downgrade(migrate_engine):
print str(e)
log.debug( "Dropping column user_id from history_tag_association table failed: %s" % str( e ) )
# Drop user_id column from history_dataset_association_tag_association table.
HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_association_tag_association", metadata, autoload=True )
try:
lib/galaxy/model/migrate/versions/0032_stored_workflow_slug_column.py
+6 -7
View File
@@ -1,16 +1,14 @@
"""
Migration script to add slug column for stored workflow.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, Index, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -28,12 +26,13 @@ def upgrade(migrate_engine):
# Create slug index.
if migrate_engine.name != 'sqlite':
try:
i = Index( "ix_stored_workflow_slug", StoredWorkflow_table.c.slug, mysql_length = 200 )
i = Index( "ix_stored_workflow_slug", StoredWorkflow_table.c.slug, mysql_length=200 )
i.create()
except:
# Mysql doesn't have a named index, but alter should work
StoredWorkflow_table.c.slug.alter( unique=False )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0033_published_cols_for_histories_and_workflows.py
+5 -8
View File
@@ -2,17 +2,14 @@
Migration script to add necessary columns for distinguishing between viewing/importing and publishing histories, \
workflows, and pages. Script adds published column to histories and workflows and importable column to pages.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, Index, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -28,7 +25,6 @@ def upgrade(migrate_engine):
print "Adding published column to history table failed: %s" % str( e )
log.debug( "Adding published column to history table failed: %s" % str( e ) )
if migrate_engine.name != 'sqlite':
# Create index for published column in history table.
try:
@@ -76,6 +72,7 @@ def upgrade(migrate_engine):
# Mysql doesn't have a named index, but alter should work
Page_table.c.importable.alter( unique=False )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0034_page_user_share_association.py
+6 -9
View File
@@ -1,22 +1,18 @@
"""
Migration script to create a table for page-user share association.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
PageUserShareAssociation_table = Table( "page_user_share_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "page_id", Integer, ForeignKey( "page.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
)
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -30,6 +26,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating page_user_share_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0035_item_annotations_and_workflow_step_tags.py
+23 -27
View File
@@ -1,15 +1,11 @@
"""
Migration script to (a) create tables for annotating objects and (b) create tags for workflow steps.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table, TEXT, Unicode
log = logging.getLogger( __name__ )
metadata = MetaData()
# Annotation tables.
@@ -90,10 +86,10 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating workflow_step_tag_association table failed: %s" % str( e ) )
haaa = Index( "ix_history_anno_assoc_annotation", HistoryAnnotationAssociation_table.c.annotation, mysql_length = 200)
hdaaa = Index( "ix_history_dataset_anno_assoc_annotation", HistoryDatasetAssociationAnnotationAssociation_table.c.annotation, mysql_length = 200)
swaaa = Index( "ix_stored_workflow_ann_assoc_annotation", StoredWorkflowAnnotationAssociation_table.c.annotation, mysql_length = 200)
wsaaa = Index( "ix_workflow_step_ann_assoc_annotation", WorkflowStepAnnotationAssociation_table.c.annotation, mysql_length = 200)
haaa = Index( "ix_history_anno_assoc_annotation", HistoryAnnotationAssociation_table.c.annotation, mysql_length=200)
hdaaa = Index( "ix_history_dataset_anno_assoc_annotation", HistoryDatasetAssociationAnnotationAssociation_table.c.annotation, mysql_length=200)
swaaa = Index( "ix_stored_workflow_ann_assoc_annotation", StoredWorkflowAnnotationAssociation_table.c.annotation, mysql_length=200)
wsaaa = Index( "ix_workflow_step_ann_assoc_annotation", WorkflowStepAnnotationAssociation_table.c.annotation, mysql_length=200)
try:
haaa.create()
@@ -104,42 +100,42 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating annotation indices failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
# Drop history_annotation_association table.
try:
HistoryAnnotationAssociation_table.drop()
HistoryAnnotationAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping history_annotation_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping history_annotation_association table failed: %s" % str( e ) )
# Drop history_dataset_association_annotation_association table.
try:
HistoryDatasetAssociationAnnotationAssociation_table.drop()
HistoryDatasetAssociationAnnotationAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping history_dataset_association_annotation_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping history_dataset_association_annotation_association table failed: %s" % str( e ) )
# Drop stored_workflow_annotation_association table.
try:
StoredWorkflowAnnotationAssociation_table.drop()
StoredWorkflowAnnotationAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping stored_workflow_annotation_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping stored_workflow_annotation_association table failed: %s" % str( e ) )
# Drop workflow_step_annotation_association table.
try:
WorkflowStepAnnotationAssociation_table.drop()
WorkflowStepAnnotationAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping workflow_step_annotation_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping workflow_step_annotation_association table failed: %s" % str( e ) )
# Drop workflow_step_tag_association table.
try:
WorkflowStepTagAssociation_table.drop()
WorkflowStepTagAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping workflow_step_tag_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping workflow_step_tag_association table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0036_add_deleted_column_to_library_template_assoc_tables.py
+6 -6
View File
@@ -2,22 +2,21 @@
Migration script to add a deleted column to the following tables:
library_info_association, library_folder_info_association, library_dataset_dataset_info_association.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def get_false_value(migrate_engine):
if migrate_engine.name == 'sqlite':
return '0'
else:
return 'false'
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -53,6 +52,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "deleted to false in library_dataset_dataset_info_association failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0037_samples_library.py
+22 -29
View File
@@ -4,16 +4,16 @@ adds the same to the 'sample' table. This also adds a 'datatx' column to request
to store the sequencer login information. Finally, this adds a 'dataset_files' column to
the sample table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import sys, logging
from galaxy.model.custom_types import *
from sqlalchemy.exc import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
from galaxy.model.custom_types import JSONType, TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -24,6 +24,7 @@ log.addHandler( handler )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -56,32 +57,23 @@ def upgrade(migrate_engine):
if migrate_engine.name == 'sqlite':
# create a temporary table
RequestTemp_table = Table( 'request_temp', metadata,
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), nullable=False ),
Column( "desc", TEXT ),
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ),
Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), nullable=False ),
Column( "desc", TEXT ),
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ),
Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
try:
RequestTemp_table.create()
except Exception, e:
log.debug( "Creating request_temp table failed: %s" % str( e ) )
# insert all the rows from the request table to the request_temp table
cmd = \
"INSERT INTO request_temp " + \
"SELECT id," + \
"create_time," + \
"update_time," + \
"name," + \
"desc," + \
"form_values_id," + \
"request_type_id," + \
"user_id," + \
"deleted " + \
"FROM request;"
cmd = "INSERT INTO request_temp SELECT id, create_time, " + \
"update_time, name, desc, form_values_id, request_type_id, " + \
"user_id, deleted FROM request;"
migrate_engine.execute( cmd )
# delete the 'request' table
try:
@@ -151,6 +143,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Adding column 'folder_id' to sample table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0038_add_inheritable_column_to_library_template_assoc_tables.py
+18 -16
View File
@@ -4,24 +4,25 @@ library_info_association, library_folder_info_association.
Also, in case of sqlite check if the previous migration script deleted the
request table and if so, restore the table.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def get_false_value(migrate_engine):
if migrate_engine.name == 'sqlite':
return '0'
else:
return 'false'
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -36,15 +37,15 @@ def upgrade(migrate_engine):
metadata.reflect(only=['form_values', 'request_type', 'galaxy_user'])
# create a temporary table
Request_table = Table( 'request', metadata,
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), nullable=False ),
Column( "desc", TEXT ),
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ),
Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), nullable=False ),
Column( "desc", TEXT ),
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ),
Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
try:
Request_table.create()
except Exception, e:
@@ -72,6 +73,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Setting value of column inheritable to false in library_folder_info_association failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0039_add_synopsis_column_to_library_table.py
+5 -6
View File
@@ -1,16 +1,14 @@
"""
Migration script to add a synopsis column to the library table.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -21,6 +19,7 @@ def upgrade(migrate_engine):
c.create( Library_table )
assert c is Library_table.c.synopsis
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0040_page_annotations.py
+7 -10
View File
@@ -1,15 +1,11 @@
"""
Migration script to (a) create tables for annotating pages.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
PageAnnotationAssociation_table = Table( "page_annotation_association", metadata,
@@ -18,6 +14,7 @@ PageAnnotationAssociation_table = Table( "page_annotation_association", metadata
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "annotation", TEXT, index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -30,6 +27,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating page_annotation_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
@@ -38,6 +36,5 @@ def downgrade(migrate_engine):
try:
PageAnnotationAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping page_annotation_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping page_annotation_association table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0041_workflow_invocation.py
+9 -14
View File
@@ -1,39 +1,33 @@
"""
Migration script to create tables for tracking workflow invocations.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
import logging
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table
logging.basicConfig( level=logging.DEBUG )
log = logging.getLogger( __name__ )
import datetime
now = datetime.datetime.utcnow
metadata = MetaData()
WorkflowInvocation_table = Table( "workflow_invocation", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False )
)
Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ) )
WorkflowInvocationStep_table = Table( "workflow_invocation_step", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "workflow_invocation_id", Integer, ForeignKey( "workflow_invocation.id" ), index=True, nullable=False ),
Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=False )
)
Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=False ) )
tables = [ WorkflowInvocation_table, WorkflowInvocationStep_table ]
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -45,6 +39,7 @@ def upgrade(migrate_engine):
except:
log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0042_workflow_invocation_fix.py
+10 -14
View File
@@ -1,20 +1,17 @@
"""
Drop and readd workflow invocation tables, allowing null jobs
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
import logging
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table
logging.basicConfig( level=logging.DEBUG )
log = logging.getLogger( __name__ )
import datetime
now = datetime.datetime.utcnow
metadata = MetaData()
def upgrade(migrate_engine):
print __doc__
metadata.bind = migrate_engine
@@ -34,17 +31,15 @@ def upgrade(migrate_engine):
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ),
)
Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ) )
WorkflowInvocationStep_table = Table( "workflow_invocation_step", metadata,
Column( "id", Integer, primary_key=True ),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "workflow_invocation_id", Integer, ForeignKey( "workflow_invocation.id" ), index=True, nullable=False ),
Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=True ),
)
Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=True ) )
for table in [ WorkflowInvocation_table, WorkflowInvocationStep_table ]:
try:
@@ -52,6 +47,7 @@ def upgrade(migrate_engine):
except:
log.exception( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# No downgrade
lib/galaxy/model/migrate/versions/0043_visualization_sharing_tagging_annotating.py
+21 -26
View File
@@ -1,15 +1,11 @@
"""
Migration script to create tables and columns for sharing visualizations.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, MetaData, Table, TEXT, Unicode
log = logging.getLogger( __name__ )
metadata = MetaData()
# Sharing visualizations.
@@ -17,8 +13,7 @@ metadata = MetaData()
VisualizationUserShareAssociation_table = Table( "visualization_user_share_association", metadata,
Column( "id", Integer, primary_key=True ),
Column( "visualization_id", Integer, ForeignKey( "visualization.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True )
)
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) )
# Tagging visualizations.
@@ -43,7 +38,6 @@ VisualizationAnnotationAssociation_table = Table( "visualization_annotation_asso
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
metadata.reflect()
Visualiation_table = Table( "visualization", metadata, autoload=True )
@@ -55,9 +49,9 @@ def upgrade(migrate_engine):
log.debug( "Creating visualization_user_share_association table failed: %s" % str( e ) )
# Get default boolean value 'false' so that columns can be initialized.
if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
if migrate_engine.name in ['mysql', 'sqlite']:
default_false = "0"
elif migrate_engine.name == 'postgresql':
elif migrate_engine.name in ['postgres', 'postgresql']:
default_false = "false"
# Add columns & create indices for supporting sharing to visualization table.
@@ -68,12 +62,12 @@ def upgrade(migrate_engine):
try:
# Add column.
deleted_column.create( Visualiation_table, index_name = "ix_visualization_deleted")
deleted_column.create( Visualiation_table, index_name="ix_visualization_deleted")
assert deleted_column is Visualiation_table.c.deleted
# Fill column with default value.
cmd = "UPDATE visualization SET deleted = %s" % default_false
db_session.execute( cmd )
migrate_engine.execute( cmd )
except Exception, e:
print "Adding deleted column to visualization table failed: %s" % str( e )
log.debug( "Adding deleted column to visualization table failed: %s" % str( e ) )
@@ -85,7 +79,7 @@ def upgrade(migrate_engine):
# Fill column with default value.
cmd = "UPDATE visualization SET importable = %s" % default_false
db_session.execute( cmd )
migrate_engine.execute( cmd )
except Exception, e:
print "Adding importable column to visualization table failed: %s" % str( e )
log.debug( "Adding importable column to visualization table failed: %s" % str( e ) )
@@ -101,7 +95,7 @@ def upgrade(migrate_engine):
if migrate_engine.name == 'mysql':
# Have to create index manually.
cmd = "CREATE INDEX ix_visualization_slug ON visualization ( slug ( 100 ) )"
db_session.execute( cmd )
migrate_engine.execute( cmd )
else:
i = Index( "ix_visualization_slug", Visualiation_table.c.slug )
i.create()
@@ -116,7 +110,7 @@ def upgrade(migrate_engine):
# Fill column with default value.
cmd = "UPDATE visualization SET published = %s" % default_false
db_session.execute( cmd )
migrate_engine.execute( cmd )
except Exception, e:
print "Adding published column to visualization table failed: %s" % str( e )
log.debug( "Adding published column to visualization table failed: %s" % str( e ) )
@@ -137,16 +131,17 @@ def upgrade(migrate_engine):
# Need to create index for visualization annotation manually to deal with errors.
try:
if migrate_engine.name == 'mysql':
# Have to create index manually.
cmd = "CREATE INDEX ix_visualization_annotation_association_annotation ON visualization_annotation_association ( annotation ( 100 ) )"
db_session.execute( cmd )
else:
i = Index( "ix_visualization_annotation_association_annotation", VisualizationAnnotationAssociation_table.c.annotation )
i.create()
if migrate_engine.name == 'mysql':
# Have to create index manually.
cmd = "CREATE INDEX ix_visualization_annotation_association_annotation ON visualization_annotation_association ( annotation ( 100 ) )"
migrate_engine.execute( cmd )
else:
i = Index( "ix_visualization_annotation_association_annotation", VisualizationAnnotationAssociation_table.c.annotation )
i.create()
except Exception, e:
print "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e )
log.debug( "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e ) )
print "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e )
log.debug( "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
lib/galaxy/model/migrate/versions/0044_add_notify_column_to_request_table.py
+5 -6
View File
@@ -1,16 +1,14 @@
"""
Migration script to add a notify column to the request table.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -21,6 +19,7 @@ def upgrade(migrate_engine):
c.create( Request_table )
assert c is Request_table.c.notify
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0045_request_type_permissions_table.py
+6 -8
View File
@@ -1,17 +1,13 @@
"""
Migration script to add the request_type_permissions table.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
RequestTypePermissions_table = Table( "request_type_permissions", metadata,
@@ -22,6 +18,7 @@ RequestTypePermissions_table = Table( "request_type_permissions", metadata,
Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), nullable=True, index=True ),
Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -31,6 +28,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating request_type_permissions table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0046_post_job_actions.py
+9 -12
View File
@@ -1,22 +1,17 @@
"""
Migration script to create tables for handling post-job actions.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
import logging
logging.basicConfig( level=logging.DEBUG )
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import JSONType
import datetime
logging.basicConfig( level=logging.DEBUG )
log = logging.getLogger( __name__ )
now = datetime.datetime.utcnow
metadata = MetaData()
PostJobAction_table = Table("post_job_action", metadata,
@@ -31,7 +26,8 @@ PostJobAction_table = Table("post_job_action", metadata,
# Column("post_job_action_id", Integer, ForeignKey("post_job_action.id"), index=True, nullable=False),
# Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False))
tables = [PostJobAction_table]#, PostJobActionAssociation_table]
tables = [PostJobAction_table] # , PostJobActionAssociation_table]
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -43,6 +39,7 @@ def upgrade(migrate_engine):
except:
log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0047_job_table_user_id_column.py
+8 -10
View File
@@ -1,18 +1,14 @@
"""
Add a user_id column to the job table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
import sys, logging
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -23,6 +19,7 @@ log.addHandler( handler )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -68,6 +65,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Updating job.user_id column failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0048_dataset_instance_state_column.py
+11 -11
View File
@@ -1,18 +1,16 @@
"""
Add a state column to the history_dataset_association and library_dataset_dataset_association table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from sqlalchemy.exc import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
import sys, logging
from sqlalchemy import Column, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -25,6 +23,7 @@ metadata = MetaData()
DATASET_INSTANCE_TABLE_NAMES = [ 'history_dataset_association', 'library_dataset_dataset_association' ]
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -40,11 +39,12 @@ def upgrade(migrate_engine):
index_name = "ix_%s_state" % table_name
try:
col = Column( "state", TrimmedString( 64 ), index=True, nullable=True )
col.create( dataset_instance_table, index_name = index_name)
col.create( dataset_instance_table, index_name=index_name)
assert col is dataset_instance_table.c.state
except Exception, e:
log.debug( "Adding column 'state' to %s table failed: %s" % ( table_name, str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0049_api_keys_table.py
+8 -9
View File
@@ -1,18 +1,15 @@
"""
Migration script to add the api_keys table.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
APIKeys_table = Table( "api_keys", metadata,
@@ -21,6 +18,7 @@ APIKeys_table = Table( "api_keys", metadata,
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "key", TrimmedString( 32 ), index=True, unique=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -30,6 +28,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating api_keys table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# Load existing tables
lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py
+6 -9
View File
@@ -1,17 +1,13 @@
from sqlalchemy import *
from migrate import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def display_migration_details():
print
print "========================================"
@@ -127,6 +123,7 @@ CloudProvider_table = Table( "cloud_provider", metadata,
Column( "path", TEXT ),
Column( "deleted", Boolean, default=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
display_migration_details()
lib/galaxy/model/migrate/versions/0051_imported_col_for_jobs_table.py
+6 -8
View File
@@ -1,17 +1,14 @@
"""
Migration script to add imported column for jobs table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -26,7 +23,7 @@ def upgrade(migrate_engine):
assert c is Jobs_table.c.imported
# Initialize.
if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
if migrate_engine.name in ['mysql', 'sqlite']:
default_false = "0"
elif migrate_engine.name in ['postgres', 'postgresql']:
default_false = "false"
@@ -36,6 +33,7 @@ def upgrade(migrate_engine):
print "Adding imported column to job table failed: %s" % str( e )
log.debug( "Adding imported column to job table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py
+23 -26
View File
@@ -2,40 +2,36 @@
Migration script to add the sample_dataset table and remove the 'dataset_files' column
from the 'sample' table
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
from galaxy.util.json import loads, dumps
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from json import loads
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def nextval( table, col='id' ):
if migrate_engine.name == 'postgres':
def nextval( migrate_engine, table, col='id' ):
if migrate_engine.name in ['postgres', 'postgresql']:
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
elif migrate_engine.name in ['mysql', 'sqlite']:
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def localtimestamp():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
def localtimestamp(migrate_engine):
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
SampleDataset_table = Table('sample_dataset', metadata,
Column( "id", Integer, primary_key=True ),
@@ -48,6 +44,7 @@ SampleDataset_table = Table('sample_dataset', metadata,
Column( "error_msg", TEXT ),
Column( "size", TrimmedString( 255 ) ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -66,9 +63,9 @@ def upgrade(migrate_engine):
for df in dataset_files:
if type(df) == type(dict()):
cmd = "INSERT INTO sample_dataset VALUES (%s, %s, %s, %s, '%s', '%s', '%s', '%s', '%s')"
cmd = cmd % ( nextval('sample_dataset'),
localtimestamp(),
localtimestamp(),
cmd = cmd % ( nextval(migrate_engine, 'sample_dataset'),
localtimestamp(migrate_engine),
localtimestamp(migrate_engine),
str(sample_id),
df.get('name', ''),
df.get('filepath', ''),
lib/galaxy/model/migrate/versions/0053_item_ratings.py
+21 -23
View File
@@ -1,15 +1,11 @@
"""
Migration script to create tables for rating histories, datasets, workflows, pages, and visualizations.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
# Rating tables.
@@ -44,6 +40,7 @@ VisualizationRatingAssociation_table = Table( "visualization_rating_association"
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "rating", Integer, index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -62,7 +59,7 @@ def upgrade(migrate_engine):
except Exception, e:
# MySQL cannot handle long index names; when we see this error, create the index name manually.
if migrate_engine.name == 'mysql' and \
str(e).lower().find("identifier name 'ix_history_dataset_association_rating_association_history_dataset_association_id' is too long"):
str(e).lower().find("identifier name 'ix_history_dataset_association_rating_association_history_dataset_association_id' is too long"):
i = Index( "ix_hda_rating_association_hda_id", HistoryDatasetAssociationRatingAssociation_table.c.history_dataset_association_id )
try:
i.create()
@@ -94,41 +91,42 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating visualization_rating_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
# Drop history_rating_association table.
try:
HistoryRatingAssociation_table.drop()
HistoryRatingAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping history_rating_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping history_rating_association table failed: %s" % str( e ) )
# Drop history_dataset_association_rating_association table.
try:
HistoryDatasetAssociationRatingAssociation_table.drop()
HistoryDatasetAssociationRatingAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping history_dataset_association_rating_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping history_dataset_association_rating_association table failed: %s" % str( e ) )
# Drop stored_workflow_rating_association table.
try:
StoredWorkflowRatingAssociation_table.drop()
StoredWorkflowRatingAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping stored_workflow_rating_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping stored_workflow_rating_association table failed: %s" % str( e ) )
# Drop page_rating_association table.
try:
PageRatingAssociation_table.drop()
PageRatingAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping page_rating_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping page_rating_association table failed: %s" % str( e ) )
# Drop visualization_rating_association table.
try:
VisualizationRatingAssociation_table.drop()
VisualizationRatingAssociation_table.drop()
except Exception, e:
print str(e)
log.debug( "Dropping visualization_rating_association table failed: %s" % str( e ) )
print str(e)
log.debug( "Dropping visualization_rating_association table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py
+8 -10
View File
@@ -1,18 +1,15 @@
"""
Migration script to add dbkey column for visualization.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.util.json import loads
import logging
log = logging.getLogger( __name__ )
from json import loads
from sqlalchemy import Column, Index, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -28,9 +25,9 @@ def upgrade(migrate_engine):
x.create( Visualization_table )
y.create( Visualization_revision_table )
# Manually create indexes for compatability w/ mysql_length.
xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length = 200)
xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length=200)
xi.create()
yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length = 200)
yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length=200)
yi.create()
assert x is Visualization_table.c.dbkey
assert y is Visualization_revision_table.c.dbkey
@@ -45,6 +42,7 @@ def upgrade(migrate_engine):
migrate_engine.execute("UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" % (dbkey, viz_rev_id))
migrate_engine.execute("UPDATE visualization SET dbkey='%s' WHERE id=%s" % (dbkey, viz_id))
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0055_add_pja_assoc_for_jobs.py
+6 -9
View File
@@ -1,18 +1,13 @@
"""
Migration script to add the post_job_action_association table.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
PostJobActionAssociation_table = Table("post_job_action_association", metadata,
@@ -20,6 +15,7 @@ PostJobActionAssociation_table = Table("post_job_action_association", metadata,
Column("post_job_action_id", Integer, ForeignKey("post_job_action.id"), index=True, nullable=False),
Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False))
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -29,6 +25,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating PostJobActionAssociation table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# Load existing tables
lib/galaxy/model/migrate/versions/0056_workflow_outputs.py
+5 -9
View File
@@ -1,19 +1,13 @@
"""
Migration script to create tables for adding explicit workflow outputs.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table
logging.basicConfig( level=logging.DEBUG )
log = logging.getLogger( __name__ )
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
metadata = MetaData()
WorkflowOutput_table = Table( "workflow_output", metadata,
@@ -23,6 +17,7 @@ WorkflowOutput_table = Table( "workflow_output", metadata,
tables = [WorkflowOutput_table]
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -33,6 +28,7 @@ def upgrade(migrate_engine):
except:
log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0057_request_notify.py
+10 -22
View File
@@ -2,24 +2,20 @@
Migration script to modify the 'notify' field in the 'request' table from a boolean
to a JSONType
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
from galaxy.util.json import loads, dumps
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from json import dumps
from sqlalchemy import Column, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
from galaxy.model.custom_types import JSONType
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -30,7 +26,6 @@ def upgrade(migrate_engine):
Request_table = None
log.debug( "Failed loading table 'request'" )
if Request_table is not None:
# create the column again as JSONType
try:
@@ -44,16 +39,10 @@ def upgrade(migrate_engine):
result = migrate_engine.execute( cmd )
for r in result:
id = int(r[0])
notify_old = r[1]
notify_new = dict(email=[], sample_states=[], body='', subject='')
cmd = "update request set notification='%s' where id=%i" % (dumps(notify_new), id)
cmd = "UPDATE request SET notification='%s' WHERE id=%i" % (dumps(notify_new), id)
migrate_engine.execute( cmd )
cmd = "SELECT id, notification FROM request"
result = migrate_engine.execute( cmd )
for r in result:
rr = loads(str(r[1]))
# remove the 'notify' column for non-sqlite databases.
if migrate_engine.name != 'sqlite':
try:
@@ -62,7 +51,6 @@ def upgrade(migrate_engine):
log.debug( "Deleting column 'notify' from the 'request' table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0058_history_import_export.py
+6 -9
View File
@@ -1,15 +1,11 @@
"""
Migration script to create table for exporting histories to archives.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, ForeignKey, Integer, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
# Table to add.
@@ -22,8 +18,8 @@ JobExportHistoryArchive_table = Table( "job_export_history_archive", metadata,
Column( "compressed", Boolean, index=True, default=False ),
Column( "history_attrs_filename", TEXT ),
Column( "datasets_attrs_filename", TEXT ),
Column( "jobs_attrs_filename", TEXT )
)
Column( "jobs_attrs_filename", TEXT ) )
def upgrade(migrate_engine):
print __doc__
@@ -37,6 +33,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Creating job_export_history_archive table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py
+7 -13
View File
@@ -2,24 +2,17 @@
Migration script to modify the 'file_path' field type in 'sample_dataset' table
to 'TEXT' so that it can support large file paths exceeding 255 characters
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
from galaxy.util.json import loads, dumps
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -54,6 +47,7 @@ def upgrade(migrate_engine):
cmd = "update sample_dataset set file_path='%s' where id=%i" % (file_path, id)
migrate_engine.execute( cmd )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0060_history_archive_import.py
+7 -10
View File
@@ -2,15 +2,11 @@
Migration script to create column and table for importing histories from
file archives.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, ForeignKey, Integer, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
# Columns to add.
@@ -24,8 +20,8 @@ JobImportHistoryArchive_table = Table( "job_import_history_archive", metadata,
Column( "id", Integer, primary_key=True ),
Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ),
Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ),
Column( "archive_dir", TEXT )
)
Column( "archive_dir", TEXT ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -39,7 +35,7 @@ def upgrade(migrate_engine):
assert importing_col is History_table.c.importing
# Initialize column to false.
if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
if migrate_engine.name in ['mysql', 'sqlite']:
default_false = "0"
elif migrate_engine.name in ['postgres', 'postgresql']:
default_false = "false"
@@ -54,6 +50,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating job_import_history_archive table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0061_tasks.py
+5 -7
View File
@@ -1,16 +1,12 @@
"""
Migration script to create tables task management.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
now = datetime.datetime.utcnow
@@ -33,6 +29,7 @@ Task_table = Table( "task", metadata,
tables = [Task_table]
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -43,6 +40,7 @@ def upgrade(migrate_engine):
except:
log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0062_user_openid_table.py
+7 -11
View File
@@ -2,18 +2,13 @@
Migration script to create table for associating sessions and users with
OpenIDs.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, MetaData, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
# Table to add
@@ -24,8 +19,8 @@ UserOpenID_table = Table( "galaxy_user_openid", metadata,
Column( "update_time", DateTime, index=True, default=now, onupdate=now ),
Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
Column( "openid", TEXT ),
)
Column( "openid", TEXT ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -49,6 +44,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Adding index '%s' failed: %s" % ( ix_name, str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0063_sequencer_table.py
+8 -13
View File
@@ -1,21 +1,16 @@
"""
Migration script to create a new 'sequencer' table
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
# Table to add
@@ -31,6 +26,7 @@ Sequencer_table = Table( 'sequencer', metadata,
Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -56,4 +52,3 @@ def downgrade(migrate_engine):
Sequencer_table.drop()
except Exception, e:
log.debug( "Deleting 'sequencer' table failed: %s" % str( e ) )
lib/galaxy/model/migrate/versions/0064_add_run_and_sample_run_association_tables.py
+6 -9
View File
@@ -1,18 +1,13 @@
"""
Migration script to add the run and sample_run_association tables.
"""
from sqlalchemy import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
Run_table = Table( "run", metadata,
@@ -33,6 +28,7 @@ SampleRunAssociation_table = Table( "sample_run_association", metadata,
Column( "sample_id", Integer, ForeignKey( "sample.id" ), index=True, nullable=False ),
Column( "run_id", Integer, ForeignKey( "run.id" ), index=True, nullable=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -50,6 +46,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating SampleRunAssociation table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
# Load existing tables
lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py
+20 -22
View File
@@ -3,19 +3,16 @@ Migration script to add 'name' attribute to the JSON dict which describes
a form definition field and the form values in the database. In the 'form_values'
table, the 'content' column is now a JSON dict instead of a list.
"""
import datetime
import logging
import sys
from json import loads, dumps
from sqlalchemy import MetaData, Table
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.util.json import loads, dumps
from galaxy.model.custom_types import _sniffnfix_pg9_hex
import datetime
now = datetime.datetime.utcnow
import sys, logging
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -23,7 +20,6 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s"
formatter = logging.Formatter( format )
handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData()
@@ -32,17 +28,18 @@ def upgrade(migrate_engine):
print __doc__
metadata.reflect()
try:
FormDefinition_table = Table( "form_definition", metadata, autoload=True )
Table( "form_definition", metadata, autoload=True )
except Exception, e:
log.debug( "Loading 'form_definition' table failed: %s" % str( e ) )
try:
FormValues_table = Table( "form_values", metadata, autoload=True )
Table( "form_values", metadata, autoload=True )
except Exception, e:
log.debug( "Loading 'form_values' table failed: %s" % str( e ) )
def get_value(lst, index):
try:
return str(lst[index]).replace("'", "''")
except IndexError,e:
except IndexError:
return ''
# Go through the entire table and add a 'name' attribute for each field
# in the list of fields for each form definition
@@ -61,9 +58,9 @@ def upgrade(migrate_engine):
field[ 'label' ] = field[ 'label' ].replace("'", "''")
fields_json = dumps( fields_list )
if migrate_engine.name == 'mysql':
cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( fields_json, form_definition_id )
cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" % ( fields_json, form_definition_id )
else:
cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( fields_json, form_definition_id )
cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" % ( fields_json, form_definition_id )
migrate_engine.execute( cmd )
# replace the values list in the content field of the form_values table with a name:value dict
cmd = "SELECT form_values.id, form_values.content, form_definition.fields" \
@@ -85,18 +82,19 @@ def upgrade(migrate_engine):
for field_index, field in enumerate( fields_list ):
field_name = field[ 'name' ]
values_dict[ field_name ] = get_value(values_list, field_index )
cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( dumps( values_dict ), form_values_id )
cmd = "UPDATE form_values SET content='%s' WHERE id=%i" % ( dumps( values_dict ), form_values_id )
migrate_engine.execute( cmd )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
try:
FormDefinition_table = Table( "form_definition", metadata, autoload=True )
Table( "form_definition", metadata, autoload=True )
except Exception, e:
log.debug( "Loading 'form_definition' table failed: %s" % str( e ) )
try:
FormValues_table = Table( "form_values", metadata, autoload=True )
Table( "form_values", metadata, autoload=True )
except Exception, e:
log.debug( "Loading 'form_values' table failed: %s" % str( e ) )
# remove the name attribute in the content column JSON dict in the form_values table
@@ -120,7 +118,7 @@ def downgrade(migrate_engine):
field_name = field[ 'name' ]
field_value = values_dict[ field_name ]
values_list.append( field_value )
cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( dumps( values_list ), form_values_id )
cmd = "UPDATE form_values SET content='%s' WHERE id=%i" % ( dumps( values_list ), form_values_id )
migrate_engine.execute( cmd )
# remove name attribute from the field column of the form_definition table
cmd = "SELECT f.id, f.fields FROM form_definition AS f"
@@ -133,10 +131,10 @@ def downgrade(migrate_engine):
fields_list = loads( _sniffnfix_pg9_hex( fields ) )
if len( fields_list ):
for index, field in enumerate( fields_list ):
if field.has_key( 'name' ):
if 'name' in field:
del field[ 'name' ]
if migrate_engine.name == 'mysql':
cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( dumps( fields_list ), form_definition_id )
cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" % ( dumps( fields_list ), form_definition_id )
else:
cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( dumps( fields_list ), form_definition_id )
cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" % ( dumps( fields_list ), form_definition_id )
migrate_engine.execute( cmd )
lib/galaxy/model/migrate/versions/0066_deferred_job_and_transfer_job_tables.py
+8 -11
View File
@@ -2,20 +2,15 @@
Migration script to create table for storing deferred job and managed transfer
information.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table
from galaxy.model.custom_types import JSONType
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
# Table to add
@@ -36,6 +31,7 @@ TransferJob_table = Table( "transfer_job", metadata,
Column( "path", String( 1024 ) ),
Column( "params", JSONType ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -53,6 +49,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.error( "Creating transfer_job table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py
+76 -93
View File
@@ -4,42 +4,39 @@ entries in the 'datatx_info' column in the 'request_type' table. It also deletes
column in the 'request_type' table and adds a foreign key to the 'sequencer' table. The
actual contents of the datatx_info column are stored as form_values.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
from galaxy.util.json import loads, dumps
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from json import dumps, loads
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
from sqlalchemy.exc import NoSuchTableError
from galaxy.model.custom_types import JSONType
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def nextval( table, col='id' ):
if migrate_engine.name == 'postgres':
def nextval( migrate_engine, table, col='id' ):
if migrate_engine.name in ['postgres', 'postgresql']:
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
elif migrate_engine.name in ['mysql', 'sqlite']:
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def localtimestamp():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
def get_latest_id( table ):
def localtimestamp( migrate_engine ):
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def get_latest_id( migrate_engine, table ):
result = migrate_engine.execute( "select id from %s order by id desc" % table )
row = result.fetchone()
if row:
@@ -47,17 +44,19 @@ def get_latest_id( table ):
else:
raise Exception( 'Unable to get the latest id in the %s table.' % table )
def boolean( value ):
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return value
elif migrate_engine.name == 'sqlite':
if value in [ 'True, true' ]:
return 1
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
def create_sequencer_form_definition():
def boolean( migrate_engine, value ):
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return value
elif migrate_engine.name == 'sqlite':
if value in [ 'True', 'true' ]:
return 1
return 0
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def create_sequencer_form_definition( migrate_engine ):
'''
Create a new form_definition containing 5 fields (host, username, password,
data_dir & rename_datasets) which described the existing datatx_info json
@@ -65,14 +64,14 @@ def create_sequencer_form_definition():
'''
# create new form_definition_current in the db
cmd = "INSERT INTO form_definition_current VALUES ( %s, %s, %s, %s, %s )"
cmd = cmd % ( nextval( 'form_definition_current' ),
localtimestamp(),
localtimestamp(),
cmd = cmd % ( nextval( migrate_engine, 'form_definition_current' ),
localtimestamp( migrate_engine ),
localtimestamp( migrate_engine ),
'NULL',
boolean( 'false' ) )
boolean( migrate_engine, 'false' ) )
migrate_engine.execute( cmd )
# get this form_definition_current id
form_definition_current_id = get_latest_id( 'form_definition_current' )
form_definition_current_id = get_latest_id( migrate_engine, 'form_definition_current' )
# create new form_definition in the db
form_definition_name = 'Generic sequencer form'
form_definition_desc = ''
@@ -106,9 +105,9 @@ def create_sequencer_form_definition():
form_definition_type = 'Sequencer Information Form'
form_definition_layout = dumps('[]')
cmd = "INSERT INTO form_definition VALUES ( %s, %s, %s, '%s', '%s', %s, '%s', '%s', '%s' )"
cmd = cmd % ( nextval( 'form_definition' ),
localtimestamp(),
localtimestamp(),
cmd = cmd % ( nextval( migrate_engine, 'form_definition' ),
localtimestamp( migrate_engine ),
localtimestamp( migrate_engine ),
form_definition_name,
form_definition_desc,
form_definition_current_id,
@@ -117,13 +116,14 @@ def create_sequencer_form_definition():
form_definition_layout )
migrate_engine.execute( cmd )
# get this form_definition id
form_definition_id = get_latest_id( 'form_definition' )
form_definition_id = get_latest_id( migrate_engine, 'form_definition' )
# update the form_definition_id column in form_definition_current
cmd = "UPDATE form_definition_current SET latest_form_id=%i WHERE id=%i" %( form_definition_id, form_definition_current_id )
cmd = "UPDATE form_definition_current SET latest_form_id=%i WHERE id=%i" % ( form_definition_id, form_definition_current_id )
migrate_engine.execute( cmd )
return form_definition_id
def get_sequencer_id( sequencer_info ):
def get_sequencer_id( migrate_engine, sequencer_info ):
'''Get the sequencer id corresponding to the sequencer information'''
# Check if there is any existing sequencer which have the same sequencer
# information fields & values
@@ -145,46 +145,48 @@ def get_sequencer_id( sequencer_info ):
return sequencer_id
return None
def add_sequencer( sequencer_index, sequencer_form_definition_id, sequencer_info ):
def add_sequencer( migrate_engine, sequencer_index, sequencer_form_definition_id, sequencer_info ):
'''Adds a new sequencer to the sequencer table along with its form values.'''
# Create a new form values record with the supplied sequencer information
values = dumps( { 'field_0': sequencer_info.get( 'host', '' ),
'field_1': sequencer_info.get( 'username', '' ),
'field_2': sequencer_info.get( 'password', '' ),
'field_3': sequencer_info.get( 'data_dir', '' ),
'field_4': sequencer_info.get( 'rename_dataset', '' ) } )
cmd = "INSERT INTO form_values VALUES ( %s, %s, %s, %s, '%s' )" % ( nextval( 'form_values' ),
localtimestamp(),
localtimestamp(),
'field_1': sequencer_info.get( 'username', '' ),
'field_2': sequencer_info.get( 'password', '' ),
'field_3': sequencer_info.get( 'data_dir', '' ),
'field_4': sequencer_info.get( 'rename_dataset', '' ) } )
cmd = "INSERT INTO form_values VALUES ( %s, %s, %s, %s, '%s' )" % ( nextval( migrate_engine, 'form_values' ),
localtimestamp( migrate_engine ),
localtimestamp( migrate_engine ),
sequencer_form_definition_id,
values )
migrate_engine.execute(cmd)
sequencer_form_values_id = get_latest_id( 'form_values' )
sequencer_form_values_id = get_latest_id( migrate_engine, 'form_values' )
# Create a new sequencer record with reference to the form value created above.
name = 'Sequencer_%i' % sequencer_index
desc = ''
version = ''
result_datasets = dict()
sequencer_type_id = 'simple_unknown_sequencer'
cmd = "INSERT INTO sequencer VALUES ( %s, %s, %s, '%s', '%s', '%s', '%s', %s, %s, %s )"
cmd = cmd % ( nextval('sequencer'),
localtimestamp(),
localtimestamp(),
cmd = cmd % ( nextval( migrate_engine, 'sequencer'),
localtimestamp( migrate_engine ),
localtimestamp( migrate_engine ),
name,
desc,
sequencer_type_id,
version,
sequencer_form_definition_id,
sequencer_form_values_id,
boolean( 'false' ) )
boolean( migrate_engine, 'false' ) )
migrate_engine.execute(cmd)
return get_latest_id( 'sequencer' )
return get_latest_id( migrate_engine, 'sequencer' )
def update_sequencer_id_in_request_type( request_type_id, sequencer_id ):
def update_sequencer_id_in_request_type( migrate_engine, request_type_id, sequencer_id ):
'''Update the foreign key to the sequencer table in the request_type table'''
cmd = "UPDATE request_type SET sequencer_id=%i WHERE id=%i" %( sequencer_id, request_type_id )
cmd = "UPDATE request_type SET sequencer_id=%i WHERE id=%i" % ( sequencer_id, request_type_id )
migrate_engine.execute( cmd )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -224,12 +226,11 @@ def upgrade(migrate_engine):
# For the existing request_types in the database, we add a new form_definition
# with these 4 fields. Then we populate the sequencer table with unique datatx_info
# column from the existing request_types.
sequencer_form_definition_id = create_sequencer_form_definition()
sequencer_form_definition_id = create_sequencer_form_definition( migrate_engine )
sequencer_index = 1
for row in results_list:
request_type_id = row[0]
request_type_name = row[1]
sequencer_info = str( row[2] ) # datatx_info column
sequencer_info = str( row[2] ) # datatx_info column
# skip if sequencer_info is empty
if not sequencer_info.strip() or sequencer_info in ['None', 'null']:
continue
@@ -237,12 +238,12 @@ def upgrade(migrate_engine):
# proceed only if sequencer_info is a valid dict
if sequencer_info and type( sequencer_info ) == type( dict() ):
# check if this sequencer has already been added to the sequencer table
sequencer_id = get_sequencer_id( sequencer_info )
sequencer_id = get_sequencer_id( migrate_engine, sequencer_info )
if not sequencer_id:
# add to the sequencer table
sequencer_id = add_sequencer( sequencer_index, sequencer_form_definition_id, sequencer_info )
sequencer_id = add_sequencer( migrate_engine, sequencer_index, sequencer_form_definition_id, sequencer_info )
# now update the sequencer_id column in request_type table
update_sequencer_id_in_request_type( request_type_id, sequencer_id )
update_sequencer_id_in_request_type( migrate_engine, request_type_id, sequencer_id )
sequencer_index = sequencer_index + 1
# Finally delete the 'datatx_info' column from the request_type table
@@ -279,34 +280,16 @@ def downgrade(migrate_engine):
request_type_id = row[0]
seq_values = loads( str( row[1] ) )
# create the datatx_info json dict
datatx_info = dumps( dict( host = seq_values.get( 'field_0', '' ),
username = seq_values.get( 'field_1', '' ),
password = seq_values.get( 'field_2', '' ),
data_dir = seq_values.get( 'field_3', '' ),
rename_dataset = seq_values.get( 'field_4', '' ) ) )
datatx_info = dumps( dict( host=seq_values.get( 'field_0', '' ),
username=seq_values.get( 'field_1', '' ),
password=seq_values.get( 'field_2', '' ),
data_dir=seq_values.get( 'field_3', '' ),
rename_dataset=seq_values.get( 'field_4', '' ) ) )
# update the column
cmd = "UPDATE request_type SET datatx_info='%s' WHERE id=%i" %( datatx_info, request_type_id )
cmd = "UPDATE request_type SET datatx_info='%s' WHERE id=%i" % ( datatx_info, request_type_id )
migrate_engine.execute( cmd )
# delete foreign key field to the sequencer table in the request_type table
try:
RequestType_table.c.sequencer_id.drop()
except Exception, e:
log.debug( "Deleting column 'sequencer_id' in the 'request_type' table failed: %s" % ( str( e ) ) )
lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py
+26 -42
View File
@@ -6,30 +6,25 @@ The 'sequencer_type_id' column is renamed to 'external_service_type_id' in the r
table 'external_service'. Finally, adds a foreign key to the external_service table in the
sample_dataset table and populates it.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
from galaxy.util.json import loads, dumps
import datetime
now = datetime.datetime.utcnow
import logging
from migrate import ForeignKeyConstraint
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT
from sqlalchemy.exc import NoSuchTableError
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
#migrate_engine = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
def nextval( table, col='id' ):
if migrate_engine.name == 'postgres':
def nextval( migrate_engine, table, col='id' ):
if migrate_engine.name in ['postgres', 'postgresql']:
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
elif migrate_engine.name in ['mysql', 'sqlite']:
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
@@ -150,7 +145,7 @@ def upgrade(migrate_engine):
if not sequencer_id:
sequencer_id = 'null'
cmd = "INSERT INTO request_type_external_service_association VALUES ( %s, %s, %s )"
cmd = cmd % ( nextval( 'request_type_external_service_association' ),
cmd = cmd % ( nextval( migrate_engine, 'request_type_external_service_association' ),
request_type_id,
sequencer_id )
migrate_engine.execute( cmd )
@@ -160,30 +155,22 @@ def upgrade(migrate_engine):
# In sqlite, create a temp table without the column that needs to be removed.
# then copy all the rows from the original table and finally rename the temp table
RequestTypeTemp_table = Table( 'request_type_temp', metadata,
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), nullable=False ),
Column( "desc", TEXT ),
Column( "request_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ),
Column( "sample_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
Column( "id", Integer, primary_key=True),
Column( "create_time", DateTime, default=now ),
Column( "update_time", DateTime, default=now, onupdate=now ),
Column( "name", TrimmedString( 255 ), nullable=False ),
Column( "desc", TEXT ),
Column( "request_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ),
Column( "sample_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
try:
RequestTypeTemp_table.create()
except Exception, e:
log.debug( "Creating request_type_temp table failed: %s" % str( e ) )
# insert all the rows from the request table to the request_temp table
cmd = \
"INSERT INTO request_type_temp " + \
"SELECT id," + \
"create_time," + \
"update_time," + \
"name," + \
"desc," + \
"request_form_id," + \
"sample_form_id," + \
"deleted " + \
"FROM request_type;"
cmd = "INSERT INTO request_type_temp SELECT id, create_time," + \
"update_time, name, desc, request_form_id, sample_form_id," + \
"deleted FROM request_type;"
migrate_engine.execute( cmd )
# delete the 'request_type' table
try:
@@ -269,7 +256,7 @@ def downgrade(migrate_engine):
cmd = "ALTER TABLE external_service RENAME TO sequencer"
migrate_engine.execute( cmd )
# if running postgres then rename the primary key sequence too
if migrate_engine.name == 'postgres':
if migrate_engine.name in ['postgres', 'postgresql']:
cmd = "ALTER SEQUENCE external_service_id_seq RENAME TO sequencer_id_seq"
migrate_engine.execute( cmd )
# drop the 'external_service_id' column in the 'sample_dataset' table
@@ -284,6 +271,3 @@ def downgrade(migrate_engine):
SampleDataset_table.c.external_service_id.drop()
except Exception, e:
log.debug( "Deleting column 'external_service_id' from the 'sample_dataset' table failed: %s" % ( str( e ) ) )
lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py
+5 -13
View File
@@ -1,23 +1,16 @@
"""
Migration script to rename the sequencer information form type to external service information form
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import MetaData
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -28,7 +21,6 @@ def upgrade(migrate_engine):
migrate_engine.execute( cmd )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0070_add_info_column_to_deferred_job_table.py
+5 -7
View File
@@ -1,17 +1,14 @@
"""
Migration script to add 'info' column to the transfer_job table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -25,6 +22,7 @@ def upgrade(migrate_engine):
print "Adding info column to transfer_job table failed: %s" % str( e )
log.debug( "Adding info column to transfer_job table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0071_add_history_and_workflow_to_sample.py
+7 -8
View File
@@ -1,18 +1,16 @@
"""
Migration script to add 'workflow' and 'history' columns for a sample.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
from galaxy.model.custom_types import JSONType
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -29,6 +27,7 @@ def upgrade(migrate_engine):
print "Adding history and workflow columns to sample table failed: %s" % str( e )
log.debug( "Adding history and workflow columns to sample table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0072_add_pid_and_socket_columns_to_transfer_job_table.py
+5 -7
View File
@@ -1,17 +1,14 @@
"""
Migration script to add 'pid' and 'socket' columns to the transfer_job table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, Integer, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -28,6 +25,7 @@ def upgrade(migrate_engine):
print "Adding columns to transfer_job table failed: %s" % str( e )
log.debug( "Adding columns to transfer_job table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0073_add_ldda_to_implicit_conversion_table.py
+6 -8
View File
@@ -1,17 +1,14 @@
"""
Migration script to add 'ldda_parent_id' column to the implicitly_converted_dataset_association table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -21,7 +18,7 @@ def upgrade(migrate_engine):
if migrate_engine.name != 'sqlite':
c = Column( "ldda_parent_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True, nullable=True )
else:
#Can't use the ForeignKey in sqlite.
# Can't use the ForeignKey in sqlite.
c = Column( "ldda_parent_id", Integer, index=True, nullable=True )
c.create( Implicitly_converted_table, index_name="ix_implicitly_converted_dataset_assoc_ldda_parent_id")
assert c is Implicitly_converted_table.c.ldda_parent_id
@@ -29,6 +26,7 @@ def upgrade(migrate_engine):
print "Adding ldda_parent_id column to implicitly_converted_dataset_association table failed: %s" % str( e )
log.debug( "Adding ldda_parent_id column to implicitly_converted_dataset_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py
+19 -19
View File
@@ -1,32 +1,31 @@
"""
Migration script to add 'purged' column to the library_dataset table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def boolean_false(migrate_engine):
if migrate_engine.name in ['postgres','mysql', 'postgresql']:
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to set True data value for unknown database type: %s' % str( migrate_engine.name ) )
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return False
elif migrate_engine.name == 'sqlite':
return 0
else:
raise Exception( 'Unable to set True data value for unknown database type: %s' % str( migrate_engine.name ) )
def boolean_true(migrate_engine):
if migrate_engine.name in ['postgres','mysql', 'postgresql']:
return True
elif migrate_engine.name == 'sqlite':
return 1
else:
raise Exception( 'Unable to set False data value for unknown database type: %s' % str( migrate_engine.name ) )
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return True
elif migrate_engine.name == 'sqlite':
return 1
else:
raise Exception( 'Unable to set False data value for unknown database type: %s' % str( migrate_engine.name ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -58,6 +57,7 @@ def upgrade(migrate_engine):
cmd = "UPDATE library_dataset SET purged = %s WHERE id = %d;" % ( boolean_true(migrate_engine), int( row.id ) )
migrate_engine.execute( cmd )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0075_add_subindex_column_to_run_table.py
+4 -6
View File
@@ -1,16 +1,13 @@
"""
Migration script to add a 'subindex' column to the run table.
"""
from sqlalchemy import Column, MetaData, Table
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
from galaxy.model.custom_types import TrimmedString
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -23,6 +20,7 @@ def upgrade(migrate_engine):
except Exception, e:
print "Adding the subindex column to the run table failed: ", str( e )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py
+15 -30
View File
@@ -2,31 +2,16 @@
This migration script fixes the data corruption caused in the form_values
table (content json field) by migrate script 65.
'''
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from sqlalchemy.exc import *
import binascii
from galaxy.util.json import loads, dumps
import logging
log = logging.getLogger( __name__ )
from json import dumps, loads
from sqlalchemy import MetaData
from galaxy.model.custom_types import _sniffnfix_pg9_hex
log = logging.getLogger( __name__ )
metadata = MetaData()
def _sniffnfix_pg9_hex(value):
"""
Sniff for and fix postgres 9 hex decoding issue
"""
try:
if value[0] == 'x':
return binascii.unhexlify(value[1:])
else:
return value
except Exception, ex:
return value
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -43,13 +28,13 @@ def upgrade(migrate_engine):
# if that fails, it means that the content field is corrupted.
try:
field_values_dict = loads( _sniffnfix_pg9_hex( str( row['field_values'] ) ) )
except Exception, e:
except Exception:
corrupted_rows = corrupted_rows + 1
# content field is corrupted
fields_list = loads( _sniffnfix_pg9_hex( str( row['fdfields'] ) ) )
field_values_str = _sniffnfix_pg9_hex( str( row['field_values'] ) )
try:
#Encoding errors? Just to be safe.
# Encoding errors? Just to be safe.
print "Attempting to fix row %s" % row['id']
print "Prior to replacement: %s" % field_values_str
except:
@@ -66,16 +51,16 @@ def upgrade(migrate_engine):
print "The 'content' field of row 'id' %i does not have the field '%s' in the 'form_values' table and could not be fixed by this migration script." % ( int( field['id'] ), field['name'] )
else:
# check if this is the last field
if index == len( fields_list )-1:
if index == len( fields_list ) - 1:
# since this is the last field, the value string lies between the
# field name and the '"}' string at the end, hence len(field_values_str)-2
value = field_values_str[ field_index+len( field_name_key ):len( field_values_str )-2 ]
# field name and the '"}' string at the end, hence len(field_values_str) - 2
value = field_values_str[ field_index + len( field_name_key ):len( field_values_str ) - 2 ]
else:
# if this is not the last field then the value string lies between
# this field name and the next field name
next_field = fields_list[index+1]
next_field = fields_list[index + 1]
next_field_index = field_values_str.find( '", "%s": "' % next_field['name'] )
value = field_values_str[ field_index+len( field_name_key ):next_field_index ]
value = field_values_str[ field_index + len( field_name_key ):next_field_index ]
# clean up the value string, escape the required quoutes and newline characters
value = value.replace( "'", "\''" )\
.replace( '"', '\\\\"' )\
@@ -86,7 +71,7 @@ def upgrade(migrate_engine):
field_values_dict[ field['name'] ] = value
# update the db
json_values = dumps(field_values_dict)
cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( json_values, int( row['id'] ) )
cmd = "UPDATE form_values SET content='%s' WHERE id=%i" % ( json_values, int( row['id'] ) )
migrate_engine.execute( cmd )
try:
print "Post replacement: %s" % json_values
@@ -97,7 +82,7 @@ def upgrade(migrate_engine):
else:
print 'No corrupted rows found.'
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0077_create_tool_tag_association_table.py
+8 -11
View File
@@ -1,20 +1,15 @@
"""
Migration script to create table for storing tool tag associations.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
# Table to add
@@ -28,6 +23,7 @@ ToolTagAssociation_table = Table( "tool_tag_association", metadata,
Column( "value", TrimmedString(255), index=True),
Column( "user_value", TrimmedString(255), index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -39,6 +35,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.error( "Creating tool_tag_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py
+5 -7
View File
@@ -3,17 +3,14 @@ Migration script to add 'total_size' column to the dataset table, 'purged'
column to the HDA table, and 'disk_usage' column to the User and GalaxySession
tables.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Boolean, Column, MetaData, Numeric, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -56,6 +53,7 @@ def upgrade(migrate_engine):
print "Adding disk_usage column to galaxy_session table failed: %s" % str( e )
log.debug( "Adding disk_usage column to galaxy_session table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py
+5 -6
View File
@@ -1,13 +1,10 @@
"""
Migration script to add the job_to_input_library_dataset table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
@@ -18,6 +15,7 @@ JobToInputLibraryDatasetAssociation_table = Table( "job_to_input_library_dataset
Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ),
Column( "name", String(255) ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -30,6 +28,7 @@ def upgrade(migrate_engine):
print "Creating job_to_input_library_dataset table failed: %s" % str( e )
log.debug( "Creating job_to_input_library_dataset table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0080_quota_tables.py
+5 -23
View File
@@ -1,20 +1,13 @@
"""
Migration script to create tables for disk quotas.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
# from galaxy.model.orm.ext.assignmapper import *
# from galaxy.model.custom_types import *
import datetime
now = datetime.datetime.utcnow
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import BigInteger, Boolean, Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
metadata = MetaData()
# Tables to add
@@ -50,6 +43,7 @@ DefaultQuotaAssociation_table = Table( "default_quota_association", metadata,
Column( "type", String( 32 ), index=True, unique=True ),
Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -79,18 +73,6 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating default_quota_association table failed: %s" % str( e ) )
# Create the default quota record
#class Quota( object ):
# def __init__( self, name, description, bytes, operation ):
# self.name = name
# self.description = description
# self.bytes = bytes
# self.operation = operation
#assign_mapper( db_session, Quota, Quota_table )
#default_quota = Quota( 'Default Quota', 'The base quota applied to all users', -1, '=' )
#db_session.add( default_quota )
#db_session.flush()
def downgrade(migrate_engine):
metadata.bind = migrate_engine
lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py
+3 -7
View File
@@ -1,16 +1,11 @@
"""
Migration script to add a 'tool_version' column to the hda/ldda tables.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import *
from sqlalchemy import Column, MetaData, Table, TEXT
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -29,6 +24,7 @@ def upgrade(migrate_engine):
except Exception, e:
print "Adding the tool_version column to the hda/ldda tables failed: ", str( e )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py
+10 -8
View File
@@ -1,16 +1,16 @@
"""
Migration script to add the tool_shed_repository table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import sys, logging
from galaxy.model.custom_types import *
from sqlalchemy.exc import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, Table, TEXT
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -33,6 +33,7 @@ ToolShedRepository_table = Table( "tool_shed_repository", metadata,
Column( "changeset_revision", TrimmedString( 255 ), index=True ),
Column( "deleted", Boolean, index=True, default=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -42,6 +43,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating tool_shed_repository table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0083_add_prepare_files_to_task.py
+7 -9
View File
@@ -1,17 +1,14 @@
"""
Migration script to add 'prepare_input_files_cmd' column to the task table and to rename a column.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, String, Table, TEXT
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -26,7 +23,7 @@ def upgrade(migrate_engine):
log.debug( "Adding prepare_input_files_cmd column to task table failed: %s" % str( e ) )
try:
task_table = Table( "task", metadata, autoload=True )
c = Column( "working_directory", String ( 1024 ), nullable=True )
c = Column( "working_directory", String( 1024 ), nullable=True )
c.create( task_table )
assert c is task_table.c.working_directory
except Exception, e:
@@ -39,6 +36,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Deleting column 'part_file' from the 'task' table failed: %s" % ( str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
@@ -56,7 +54,7 @@ def downgrade(migrate_engine):
log.debug( "Dropping working_directory column from task table failed: %s" % str( e ) )
try:
task_table = Table( "task", metadata, autoload=True )
c = Column( "part_file", String ( 1024 ), nullable=True )
c = Column( "part_file", String( 1024 ), nullable=True )
c.create( task_table )
assert c is task_table.c.part_file
except Exception, e:
lib/galaxy/model/migrate/versions/0084_add_ldda_id_to_implicit_conversion_table.py
+5 -7
View File
@@ -1,17 +1,14 @@
"""
Migration script to add 'ldda_id' column to the implicitly_converted_dataset_association table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -28,6 +25,7 @@ def upgrade(migrate_engine):
print "Adding ldda_id column to implicitly_converted_dataset_association table failed: %s" % str( e )
log.debug( "Adding ldda_id column to implicitly_converted_dataset_association table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0085_add_task_info.py
+7 -8
View File
@@ -1,31 +1,30 @@
"""
Migration script to add 'info' column to the task table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, Table
from galaxy.model.custom_types import TrimmedString
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
metadata.reflect()
try:
task_table = Table( "task", metadata, autoload=True )
c = Column( "info", TrimmedString (255) , nullable=True )
c = Column( "info", TrimmedString(255) , nullable=True )
c.create( task_table )
assert c is task_table.c.info
except Exception, e:
print "Adding info column to table table failed: %s" % str( e )
log.debug( "Adding info column to task table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py
+13 -12
View File
@@ -1,18 +1,16 @@
"""
Migration script to add the metadata, update_available and includes_datatypes columns to the tool_shed_repository table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
import sys, logging
from sqlalchemy import Boolean, Column, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import JSONType
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -23,12 +21,14 @@ log.addHandler( handler )
metadata = MetaData()
def get_default_false(migrate_engine):
if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
if migrate_engine.name in ['mysql', 'sqlite']:
return "0"
elif migrate_engine.name in ['postgresql', 'postgres']:
elif migrate_engine.name in ['postgres', 'postgresql']:
return "false"
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -58,6 +58,7 @@ def upgrade(migrate_engine):
print "Adding update_available column to the tool_shed_repository table failed: %s" % str( e )
log.debug( "Adding update_available column to the tool_shed_repository table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0087_tool_id_guid_map_table.py
+10 -10
View File
@@ -1,18 +1,16 @@
"""
Migration script to create the tool_id_guid_map table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
import sys, logging
from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table, TEXT
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -34,6 +32,7 @@ ToolIdGuidMap_table = Table( "tool_id_guid_map", metadata,
Column( "repository_name", TrimmedString( 255 ) ),
Column( "guid", TEXT, index=True, unique=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -43,6 +42,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating tool_id_guid_map table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py
+12 -11
View File
@@ -1,18 +1,16 @@
"""
Migration script to add the installed_changeset_revision column to the tool_shed_repository table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
import sys, logging
from sqlalchemy import Column, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -23,6 +21,7 @@ log.addHandler( handler )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -48,11 +47,13 @@ def upgrade(migrate_engine):
update_count = 0
for row in tool_shed_repositories:
cmd = "UPDATE tool_shed_repository " \
+ "SET installed_changeset_revision = '%s' " % row.changeset_revision \
+ "SET installed_changeset_revision = '%s' " % row.changeset_revision \
+ "WHERE changeset_revision = '%s';" % row.changeset_revision
migrate_engine.execute( cmd )
update_count += 1
print "Updated the installed_changeset_revision column for ", update_count, " rows in the tool_shed_repository table. "
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0089_add_object_store_id_columns.py
+6 -7
View File
@@ -1,18 +1,16 @@
"""
Migration script to add 'object_store_id' column to various tables
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, Table
from galaxy.model.custom_types import TrimmedString
log = logging.getLogger( __name__ )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -27,6 +25,7 @@ def upgrade(migrate_engine):
print "Adding object_store_id column to %s table failed: %s" % ( t_name, str( e ) )
log.debug( "Adding object_store_id column to %s table failed: %s" % ( t_name, str( e ) ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0090_add_tool_shed_repository_table_columns.py
+10 -12
View File
@@ -1,18 +1,13 @@
"""
Migration script to add the uninstalled and dist_to_shed columns to the tool_shed_repository table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
import sys, logging
from sqlalchemy import Boolean, Column, MetaData, Table
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -23,12 +18,14 @@ log.addHandler( handler )
metadata = MetaData()
def default_false(migrate_engine):
if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
if migrate_engine.name in ['mysql', 'sqlite']:
return "0"
elif migrate_engine.name in ['postgresql', 'postgres']:
elif migrate_engine.name in ['postgres', 'postgresql']:
return "false"
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -49,6 +46,7 @@ def upgrade(migrate_engine):
except Exception, e:
print "Adding dist_to_shed column to the tool_shed_repository table failed: %s" % str( e )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py
+24 -24
View File
@@ -1,20 +1,17 @@
"""
Migration script to create the tool_version and tool_version_association tables and drop the tool_id_guid_map table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import _sniffnfix_pg9_hex
from galaxy.util.json import loads, dumps
import logging
import sys
from json import loads
import sys, logging
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import _sniffnfix_pg9_hex, TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -24,23 +21,24 @@ handler.setFormatter( formatter )
log.addHandler( handler )
metadata = MetaData()
#migrate_engine = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
def nextval( table, col='id' ):
if migrate_engine.name == 'postgres':
def nextval( migrate_engine, table, col='id' ):
if migrate_engine.name in ['postgres', 'postgresql']:
return "nextval('%s_%s_seq')" % ( table, col )
elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite':
elif migrate_engine.name in ['mysql', 'sqlite']:
return "null"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
def localtimestamp():
if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql':
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % db )
def localtimestamp(migrate_engine):
if migrate_engine.name in ['mysql', 'postgres', 'postgresql']:
return "LOCALTIMESTAMP"
elif migrate_engine.name == 'sqlite':
return "current_date || ' ' || current_time"
else:
raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name )
ToolVersion_table = Table( "tool_version", metadata,
Column( "id", Integer, primary_key=True ),
@@ -54,6 +52,7 @@ ToolVersionAssociation_table = Table( "tool_version_association", metadata,
Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ),
Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -83,7 +82,7 @@ def upgrade(migrate_engine):
tools = repository_metadata.get( 'tools', [] )
for tool_dict in tools:
cmd = "INSERT INTO tool_version VALUES (%s, %s, %s, '%s', %s)" % \
( nextval( 'tool_version' ), localtimestamp(), localtimestamp(), tool_dict[ 'guid' ], tool_shed_repository_id )
( nextval( migrate_engine, 'tool_version' ), localtimestamp( migrate_engine ), localtimestamp( migrate_engine ), tool_dict[ 'guid' ], tool_shed_repository_id )
migrate_engine.execute( cmd )
count += 1
print "Added %d rows to the new tool_version table." % count
@@ -93,6 +92,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Dropping tool_id_guid_map table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py
+10 -10
View File
@@ -1,18 +1,16 @@
"""
Migration script to create the migrate_tools table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
import sys, logging
from sqlalchemy import Column, Integer, MetaData, Table, TEXT
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -28,6 +26,7 @@ MigrateTools_table = Table( "migrate_tools", metadata,
Column( "repository_path", TEXT ),
Column( "version", Integer ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -41,6 +40,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating migrate_tools table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0093_add_job_params_col.py
+7 -8
View File
@@ -1,27 +1,25 @@
"""
Migration script to create "params" column in job table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import TrimmedString
log = logging.getLogger( __name__ )
metadata = MetaData()
# Column to add.
params_col = Column( "params", TrimmedString(255), index=True )
def display_migration_details():
print ""
print "This migration script adds a 'params' column to the Job table."
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -37,6 +35,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Adding column 'params' to job table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py
+7 -8
View File
@@ -1,27 +1,25 @@
"""
Migration script to create "handler" column in job table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
from galaxy.model.custom_types import TrimmedString
log = logging.getLogger( __name__ )
metadata = MetaData()
# Column to add.
handler_col = Column( "handler", TrimmedString(255), index=True )
def display_migration_details():
print ""
print "This migration script adds a 'handler' column to the Job table."
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -37,6 +35,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Adding column 'handler' to job table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0095_hda_subsets.py
+6 -9
View File
@@ -1,15 +1,11 @@
"""
Migration script to create table for tracking history_dataset_association subsets.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import logging
log = logging.getLogger( __name__ )
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table, Unicode
log = logging.getLogger( __name__ )
metadata = MetaData()
# Table to add.
@@ -18,8 +14,8 @@ HistoryDatasetAssociationSubset_table = Table( "history_dataset_association_subs
Column( "id", Integer, primary_key=True ),
Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ) ),
Column( "history_dataset_association_subset_id", Integer, ForeignKey( "history_dataset_association.id" ) ),
Column( "location", Unicode(255), index=True)
)
Column( "location", Unicode(255), index=True) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -43,6 +39,7 @@ def upgrade(migrate_engine):
print str(e)
log.debug( "Adding indices to table 'history_dataset_association_subset' table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0096_openid_provider.py
+6 -7
View File
@@ -2,19 +2,17 @@
Migration script to add column to openid table for provider.
Remove any OpenID entries with nonunique GenomeSpace Identifier
"""
import logging
from sqlalchemy import Column, MetaData, Table
BAD_IDENTIFIER = 'https://identity.genomespace.org/identityServer/xrd.jsp'
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
from galaxy.model.custom_types import TrimmedString
import logging
log = logging.getLogger( __name__ )
BAD_IDENTIFIER = 'https://identity.genomespace.org/identityServer/xrd.jsp'
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -35,6 +33,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Deleting bad Identifiers from galaxy_user_openid failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py
+11 -10
View File
@@ -1,18 +1,16 @@
"""
Migration script to add the ctx_rev column to the tool_shed_repository table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import datetime
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import logging
import sys
import sys, logging
from sqlalchemy import Column, MetaData, Table
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -23,6 +21,7 @@ log.addHandler( handler )
metadata = MetaData()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -34,6 +33,8 @@ def upgrade(migrate_engine):
assert col is ToolShedRepository_table.c.ctx_rev
except Exception, e:
print "Adding ctx_rev column to the tool_shed_repository table failed: %s" % str( e )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py
+8 -11
View File
@@ -1,17 +1,13 @@
"""
Migration script to create the genome_index_tool_data table.
"""
from sqlalchemy import *
from migrate import *
import datetime
import logging
import sys
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table
now = datetime.datetime.utcnow
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import *
import sys, logging
log = logging.getLogger( __name__ )
log.setLevel(logging.DEBUG)
handler = logging.StreamHandler( sys.stdout )
@@ -33,8 +29,8 @@ GenomeIndexToolData_table = Table( "genome_index_tool_data", metadata,
Column( "created_time", DateTime, default=now ),
Column( "modified_time", DateTime, default=now, onupdate=now ),
Column( "indexer", String( 64 ) ),
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ),
)
Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
@@ -46,6 +42,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating genome_index_tool_data table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
lib/galaxy/model/migrate/versions/0099_add_tool_dependency_table.py
+9 -8
View File
@@ -1,16 +1,15 @@
"""
Migration script to add the tool_dependency table.
"""
from sqlalchemy import *
from sqlalchemy.orm import *
from migrate import *
from migrate.changeset import *
import sys, logging
from galaxy.model.custom_types import *
from sqlalchemy.exc import *
import datetime
now = datetime.datetime.utcnow
import logging
import sys
from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table
from galaxy.model.custom_types import TrimmedString
now = datetime.datetime.utcnow
log = logging.getLogger( __name__ )
log.setLevel( logging.DEBUG )
handler = logging.StreamHandler( sys.stdout )
@@ -33,6 +32,7 @@ ToolDependency_table = Table( "tool_dependency", metadata,
Column( "type", TrimmedString( 40 ) ),
Column( "uninstalled", Boolean, default=False ) )
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print __doc__
@@ -42,6 +42,7 @@ def upgrade(migrate_engine):
except Exception, e:
log.debug( "Creating tool_dependency table failed: %s" % str( e ) )
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()

Some files were not shown because too many files have changed in this diff Show More