From fd1fd5e73c1828f79526ac712e3a4d9d5e9afa1c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 22 Sep 2015 20:24:55 +0100 Subject: [PATCH] flake8 lib/galaxy/model/migrate/versions/ . Fix 1429 linting errors. --- .ci/flake8_blacklist.txt | 2 - .../migrate/versions/0001_initial_tables.py | 36 +-- .../versions/0002_metadata_file_table.py | 15 +- .../versions/0003_security_and_libraries.py | 93 +++---- .../versions/0004_indexes_and_defaults.py | 29 ++- .../versions/0005_cleanup_datasets_fix.py | 237 ++++++++++-------- .../versions/0006_change_qual_datatype.py | 11 +- .../versions/0007_sharing_histories.py | 16 +- .../migrate/versions/0008_galaxy_forms.py | 26 +- .../migrate/versions/0009_request_table.py | 17 +- .../0010_hda_display_at_authz_table.py | 23 +- .../versions/0011_v0010_mysql_index_fix.py | 23 +- .../migrate/versions/0012_user_address.py | 25 +- ...0013_change_lib_item_templates_to_forms.py | 28 +-- .../model/migrate/versions/0014_pages.py | 22 +- .../model/migrate/versions/0015_tagging.py | 17 +- .../versions/0016_v0015_mysql_index_fix.py | 16 +- .../versions/0017_library_item_indexes.py | 13 +- .../0018_ordered_tags_and_page_tags.py | 20 +- .../versions/0019_request_library_folder.py | 22 +- .../versions/0020_library_upload_job.py | 21 +- .../model/migrate/versions/0021_user_prefs.py | 14 +- .../versions/0022_visualization_tables.py | 20 +- ...0023_page_published_and_deleted_columns.py | 13 +- .../0024_page_slug_unique_constraint.py | 16 +- .../model/migrate/versions/0025_user_info.py | 20 +- .../migrate/versions/0026_cloud_tables.py | 16 +- .../migrate/versions/0027_request_events.py | 39 ++- .../0028_external_metadata_file_override.py | 16 +- .../migrate/versions/0029_user_actions.py | 14 +- .../versions/0030_history_slug_column.py | 22 +- .../0031_community_and_workflow_tags.py | 24 +- .../0032_stored_workflow_slug_column.py | 13 +- ...lished_cols_for_histories_and_workflows.py | 13 +- .../0034_page_user_share_association.py | 15 +- ...item_annotations_and_workflow_step_tags.py | 50 ++-- ...column_to_library_template_assoc_tables.py | 12 +- .../migrate/versions/0037_samples_library.py | 51 ++-- ...column_to_library_template_assoc_tables.py | 34 +-- ...39_add_synopsis_column_to_library_table.py | 11 +- .../migrate/versions/0040_page_annotations.py | 17 +- .../versions/0041_workflow_invocation.py | 23 +- .../versions/0042_workflow_invocation_fix.py | 24 +- ...isualization_sharing_tagging_annotating.py | 47 ++-- ...0044_add_notify_column_to_request_table.py | 11 +- .../0045_request_type_permissions_table.py | 14 +- .../migrate/versions/0046_post_job_actions.py | 21 +- .../versions/0047_job_table_user_id_column.py | 18 +- .../0048_dataset_instance_state_column.py | 22 +- .../migrate/versions/0049_api_keys_table.py | 17 +- .../versions/0050_drop_cloud_tables.py | 15 +- .../0051_imported_col_for_jobs_table.py | 14 +- .../versions/0052_sample_dataset_table.py | 49 ++-- .../migrate/versions/0053_item_ratings.py | 44 ++-- .../versions/0054_visualization_dbkey.py | 18 +- .../versions/0055_add_pja_assoc_for_jobs.py | 15 +- .../migrate/versions/0056_workflow_outputs.py | 14 +- .../migrate/versions/0057_request_notify.py | 32 +-- .../versions/0058_history_import_export.py | 15 +- .../versions/0059_sample_dataset_file_path.py | 20 +- .../versions/0060_history_archive_import.py | 17 +- .../model/migrate/versions/0061_tasks.py | 12 +- .../versions/0062_user_openid_table.py | 18 +- .../migrate/versions/0063_sequencer_table.py | 21 +- ...d_run_and_sample_run_association_tables.py | 15 +- ...0065_add_name_to_form_fields_and_values.py | 42 ++-- ...66_deferred_job_and_transfer_job_tables.py | 19 +- .../versions/0067_populate_sequencer_table.py | 169 ++++++------- ...8_rename_sequencer_to_external_services.py | 68 ++--- .../0069_rename_sequencer_form_type.py | 18 +- ...0_add_info_column_to_deferred_job_table.py | 12 +- ...0071_add_history_and_workflow_to_sample.py | 15 +- ...nd_socket_columns_to_transfer_job_table.py | 12 +- ...3_add_ldda_to_implicit_conversion_table.py | 14 +- ..._purged_column_to_library_dataset_table.py | 38 +-- .../0075_add_subindex_column_to_run_table.py | 10 +- .../0076_fix_form_values_data_corruption.py | 45 ++-- .../0077_create_tool_tag_association_table.py | 19 +- ...8_add_columns_for_disk_usage_accounting.py | 12 +- .../0079_input_library_to_job_table.py | 11 +- .../migrate/versions/0080_quota_tables.py | 28 +-- .../0081_add_tool_version_to_hda_ldda.py | 10 +- .../0082_add_tool_shed_repository_table.py | 18 +- .../0083_add_prepare_files_to_task.py | 16 +- ...dd_ldda_id_to_implicit_conversion_table.py | 12 +- .../migrate/versions/0085_add_task_info.py | 15 +- ..._add_tool_shed_repository_table_columns.py | 25 +- .../versions/0087_tool_id_guid_map_table.py | 20 +- ..._add_installed_changeset_revison_column.py | 23 +- .../0089_add_object_store_id_columns.py | 13 +- ..._add_tool_shed_repository_table_columns.py | 22 +- .../versions/0091_add_tool_version_tables.py | 48 ++-- .../versions/0092_add_migrate_tools_table.py | 20 +- .../versions/0093_add_job_params_col.py | 15 +- .../versions/0094_add_job_handler_col.py | 15 +- .../migrate/versions/0095_hda_subsets.py | 15 +- .../migrate/versions/0096_openid_provider.py | 13 +- .../versions/0097_add_ctx_rev_column.py | 21 +- .../0098_genome_index_tool_data_table.py | 19 +- .../0099_add_tool_dependency_table.py | 17 +- ...er_tool_dependency_table_version_column.py | 23 +- ...rop_installed_changeset_revision_column.py | 17 +- ...0102_add_tool_dependency_status_columns.py | 23 +- ...add_tool_shed_repository_status_columns.py | 18 +- ...update_genome_downloader_job_parameters.py | 34 +-- .../versions/0105_add_cleanup_event_table.py | 17 +- .../versions/0106_add_missing_indexes.py | 20 +- .../0107_add_exit_code_to_job_and_task.py | 17 +- .../versions/0108_add_extended_metadata.py | 14 +- .../0109_add_repository_dependency_tables.py | 15 +- .../migrate/versions/0110_add_dataset_uuid.py | 12 +- .../versions/0111_add_job_destinations.py | 11 +- ...and_data_manager_job_association_tables.py | 26 +- .../0113_update_migrate_tools_table.py | 15 +- .../0114_update_migrate_tools_table_again.py | 15 +- .../0115_longer_user_password_field.py | 9 +- ..._available_col_add_tool_shed_status_col.py | 24 +- .../versions/0117_add_user_activation.py | 9 +- .../0118_add_hda_extended_metadata.py | 19 +- .../migrate/versions/0119_job_metrics.py | 13 +- .../versions/0120_dataset_collections.py | 52 ++-- .../migrate/versions/0121_workflow_uuids.py | 21 +- .../0123_add_workflow_request_tables.py | 17 +- .../versions/0124_job_state_history.py | 20 +- .../versions/0125_workflow_step_tracking.py | 15 +- .../migrate/versions/0126_password_reset.py | 14 +- .../0127_output_collection_adjustments.py | 16 +- .../migrate/versions/0128_session_timeout.py | 13 +- ...9_job_external_output_metadata_validity.py | 15 +- 129 files changed, 1367 insertions(+), 1714 deletions(-) diff --git a/.ci/flake8_blacklist.txt b/.ci/flake8_blacklist.txt index 362ac7c8ff3..eecd11846a7 100644 --- a/.ci/flake8_blacklist.txt +++ b/.ci/flake8_blacklist.txt @@ -3,8 +3,6 @@ client/ database/ doc/patch.py doc/source/conf.py -lib/galaxy/model/migrate/versions/ -lib/galaxy/model/tool_shed_install/migrate/versions/ lib/galaxy/util/jstree.py lib/pkg_resources.py scripts/api/ diff --git a/lib/galaxy/model/migrate/versions/0001_initial_tables.py b/lib/galaxy/model/migrate/versions/0001_initial_tables.py index b3371ecb36a..50eec0e8f90 100644 --- a/lib/galaxy/model/migrate/versions/0001_initial_tables.py +++ b/lib/galaxy/model/migrate/versions/0001_initial_tables.py @@ -1,15 +1,13 @@ -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow +import logging + +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Numeric, String, Table, TEXT # Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString -import logging +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) - metadata = MetaData() # Tables as of changeset 1464:c7acaa1bb88f @@ -138,8 +136,7 @@ GalaxySession_table = Table( "galaxy_session", metadata, Column( "current_history_id", Integer, ForeignKey( "history.id" ), nullable=True ), Column( "session_key", TrimmedString( 255 ), index=True, unique=True ), Column( "is_valid", Boolean, default=False ), - Column( "prev_session_id", Integer ) - ) + Column( "prev_session_id", Integer ) ) GalaxySessionToHistoryAssociation_table = Table( "galaxy_session_to_history", metadata, Column( "id", Integer, primary_key=True ), @@ -155,8 +152,7 @@ StoredWorkflow_table = Table( "stored_workflow", metadata, Column( "latest_workflow_id", Integer, ForeignKey( "workflow.id", use_alter=True, name='stored_workflow_latest_workflow_id_fk' ), index=True ), Column( "name", TEXT ), - Column( "deleted", Boolean, default=False ), - ) + Column( "deleted", Boolean, default=False ) ) Workflow_table = Table( "workflow", metadata, Column( "id", Integer, primary_key=True ), @@ -165,8 +161,7 @@ Workflow_table = Table( "workflow", metadata, Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True, nullable=False ), Column( "name", TEXT ), Column( "has_cycles", Boolean ), - Column( "has_errors", Boolean ) - ) + Column( "has_errors", Boolean ) ) WorkflowStep_table = Table( "workflow_step", metadata, Column( "id", Integer, primary_key=True ), @@ -180,22 +175,19 @@ WorkflowStep_table = Table( "workflow_step", metadata, Column( "tool_errors", JSONType ), Column( "position", JSONType ), Column( "config", JSONType ), - Column( "order_index", Integer ) - ) + Column( "order_index", Integer ) ) WorkflowStepConnection_table = Table( "workflow_step_connection", metadata, Column( "id", Integer, primary_key=True ), Column( "output_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), Column( "input_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True ), Column( "output_name", TEXT ), - Column( "input_name", TEXT) - ) + Column( "input_name", TEXT) ) StoredWorkflowUserShareAssociation_table = Table( "stored_workflow_user_share_connection", metadata, Column( "id", Integer, primary_key=True ), Column( "stored_workflow_id", Integer, ForeignKey( "stored_workflow.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) - ) + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) StoredWorkflowMenuEntry_table = Table( "stored_workflow_menu_entry", metadata, Column( "id", Integer, primary_key=True ), @@ -203,11 +195,7 @@ StoredWorkflowMenuEntry_table = Table( "stored_workflow_menu_entry", metadata, Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "order_index", Integer ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine metadata.create_all() - -## def downgrade(migrate_engine): -## metadata.bind = migrate_engine -## # Operations to reverse the above upgrade go here. -## pass diff --git a/lib/galaxy/model/migrate/versions/0002_metadata_file_table.py b/lib/galaxy/model/migrate/versions/0002_metadata_file_table.py index aef94d00334..d1b20f0f2bf 100644 --- a/lib/galaxy/model/migrate/versions/0002_metadata_file_table.py +++ b/lib/galaxy/model/migrate/versions/0002_metadata_file_table.py @@ -1,15 +1,10 @@ -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() # New table in changeset 1568:0b022adfdc34 @@ -22,11 +17,13 @@ MetadataFile_table = Table( "metadata_file", metadata, Column( "deleted", Boolean, index=True, default=False ), Column( "purged", Boolean, index=True, default=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() MetadataFile_table.create() + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py index 3e3597a51eb..8bb6ff0366d 100644 --- a/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py +++ b/lib/galaxy/model/migrate/versions/0003_security_and_libraries.py @@ -1,13 +1,15 @@ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow +import logging +import sys -import sys, logging +from migrate import ForeignKeyConstraint +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Index, Integer, MetaData, String, Table, TEXT +from sqlalchemy.exc import NoSuchTableError + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -16,9 +18,6 @@ formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - metadata = MetaData() # New tables as of changeset 2341:5498ac35eedd @@ -131,13 +130,13 @@ DefaultHistoryPermissions_table = Table( "default_history_permissions", metadata LibraryDataset_table = Table( "library_dataset", metadata, Column( "id", Integer, primary_key=True ), - Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected + Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ), # current version of dataset, if null, there is not a current version selected Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ), Column( "order_id", Integer ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "name", TrimmedString( 255 ), key="_name" ), #when not None/null this will supercede display in library (but not when imported into user's history?) - Column( "info", TrimmedString( 255 ), key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?) + Column( "name", TrimmedString( 255 ), key="_name" ), # when not None/null this will supercede display in library (but not when imported into user's history?) + Column( "info", TrimmedString( 255 ), key="_info" ), # when not None/null this will supercede display in library (but not when imported into user's history?) Column( "deleted", Boolean, index=True, default=False ) ) LibraryDatasetDatasetAssociation_table = Table( "library_dataset_dataset_association", metadata, @@ -173,7 +172,7 @@ Library_table = Table( "library", metadata, LibraryFolder_table = Table( "library_folder", metadata, Column( "id", Integer, primary_key=True ), - Column( "parent_id", Integer, ForeignKey( "library_folder.id" ), nullable = True, index=True ), + Column( "parent_id", Integer, ForeignKey( "library_folder.id" ), nullable=True, index=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "name", TEXT ), @@ -304,35 +303,35 @@ JobExternalOutputMetadata_table = Table( "job_external_output_metadata", metadat Column( "job_runner_external_pid", String( 255 ) ) ) Index( "ix_jeom_library_dataset_dataset_association_id", JobExternalOutputMetadata_table.c.library_dataset_dataset_association_id ) + def upgrade(migrate_engine): - db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) metadata.bind = migrate_engine # Load existing tables metadata.reflect() def nextval( table, col='id' ): - if migrate_engine.name == 'postgres': + if migrate_engine.name in ['postgres', 'postgresql']: return "nextval('%s_%s_seq')" % ( table, col ) - elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + elif migrate_engine.name in ['mysql', 'sqlite']: return "null" else: raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) def localtimestamp(): - if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': - return "LOCALTIMESTAMP" - elif migrate_engine.name == 'sqlite': - return "current_date || ' ' || current_time" - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % db ) + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) def boolean_false(): - if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': - return False - elif migrate_engine.name == 'sqlite': - return 0 - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % db ) + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return False + elif migrate_engine.name == 'sqlite': + return 0 + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) # Add 2 new columns to the galaxy_user table try: @@ -441,7 +440,7 @@ def upgrade(migrate_engine): LibraryDatasetDatasetAssociation_table = None log.debug( "Failed loading table library_dataset_dataset_association" ) if migrate_engine.name != 'sqlite': - #Sqlite can't alter table add foreign key. + # Sqlite can't alter table add foreign key. if MetadataFile_table is not None and LibraryDatasetDatasetAssociation_table is not None: try: cons = ForeignKeyConstraint( [MetadataFile_table.c.lda_id], @@ -453,10 +452,10 @@ def upgrade(migrate_engine): log.debug( "Adding foreign key constraint 'metadata_file_lda_id_fkey' to table 'metadata_file' failed: %s" % ( str( e ) ) ) # Make sure we have at least 1 user cmd = "SELECT * FROM galaxy_user;" - users = db_session.execute( cmd ).fetchall() + users = migrate_engine.execute( cmd ).fetchall() if users: cmd = "SELECT * FROM role;" - roles = db_session.execute( cmd ).fetchall() + roles = migrate_engine.execute( cmd ).fetchall() if not roles: # Create private roles for each user - pass 1 cmd = \ @@ -471,13 +470,13 @@ def upgrade(migrate_engine): "FROM galaxy_user " + \ "ORDER BY id;" cmd = cmd % ( nextval('role'), localtimestamp(), localtimestamp(), boolean_false() ) - db_session.execute( cmd ) + migrate_engine.execute( cmd ) # Create private roles for each user - pass 2 - if migrate_engine.name == 'postgres' or migrate_engine.name == 'sqlite': + if migrate_engine.name in ['postgres', 'postgresql', 'sqlite']: cmd = "UPDATE role SET description = 'Private role for ' || description;" elif migrate_engine.name == 'mysql': cmd = "UPDATE role SET description = CONCAT( 'Private role for ', description );" - db_session.execute( cmd ) + migrate_engine.execute( cmd ) # Create private roles for each user - pass 3 cmd = \ "INSERT INTO user_role_association " + \ @@ -490,7 +489,7 @@ def upgrade(migrate_engine): "WHERE galaxy_user.email = role.name " + \ "ORDER BY galaxy_user.id;" cmd = cmd % ( nextval('user_role_association'), localtimestamp(), localtimestamp() ) - db_session.execute( cmd ) + migrate_engine.execute( cmd ) # Create default permissions for each user cmd = \ "INSERT INTO default_user_permissions " + \ @@ -502,7 +501,7 @@ def upgrade(migrate_engine): "JOIN user_role_association ON user_role_association.user_id = galaxy_user.id " + \ "ORDER BY galaxy_user.id;" cmd = cmd % nextval('default_user_permissions') - db_session.execute( cmd ) + migrate_engine.execute( cmd ) # Create default history permissions for each active history associated with a user cmd = \ @@ -515,7 +514,7 @@ def upgrade(migrate_engine): "JOIN user_role_association ON user_role_association.user_id = history.user_id " + \ "WHERE history.purged = %s AND history.user_id IS NOT NULL;" cmd = cmd % ( nextval('default_history_permissions'), boolean_false() ) - db_session.execute( cmd ) + migrate_engine.execute( cmd ) # Create "manage permissions" dataset_permissions for all activate-able datasets cmd = \ "INSERT INTO dataset_permissions " + \ @@ -531,23 +530,11 @@ def upgrade(migrate_engine): "JOIN user_role_association ON user_role_association.user_id = history.user_id " + \ "WHERE dataset.purged = %s AND history.user_id IS NOT NULL;" cmd = cmd % ( nextval('dataset_permissions'), localtimestamp(), localtimestamp(), boolean_false() ) - db_session.execute( cmd ) + migrate_engine.execute( cmd ) + def downgrade(migrate_engine): metadata.bind = migrate_engine - if migrate_engine.name == 'postgres': - # http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html - from sqlalchemy.databases import postgres - class PGCascadeSchemaDropper(postgres.PGSchemaDropper): - def visit_table(self, table): - for column in table.columns: - if column.default is not None: - self.traverse_single(column.default) - self.append("\nDROP TABLE " + - self.preparer.format_table(table) + - " CASCADE") - self.execute() - postgres.dialect.schemadropper = PGCascadeSchemaDropper # Load existing tables metadata.reflect() # NOTE: all new data added in the upgrade method is eliminated here via table drops diff --git a/lib/galaxy/model/migrate/versions/0004_indexes_and_defaults.py b/lib/galaxy/model/migrate/versions/0004_indexes_and_defaults.py index 907f46410b1..1039c36fad8 100644 --- a/lib/galaxy/model/migrate/versions/0004_indexes_and_defaults.py +++ b/lib/galaxy/model/migrate/versions/0004_indexes_and_defaults.py @@ -1,7 +1,7 @@ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -import sys, logging +import logging +import sys + +from sqlalchemy import Index, MetaData, Table log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) @@ -13,18 +13,19 @@ log.addHandler( handler ) metadata = MetaData() + def upgrade(migrate_engine): - db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) metadata.bind = migrate_engine User_table = Table( "galaxy_user", metadata, autoload=True ) HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True ) + def boolean_false(): - if migrate_engine.name == 'postgresql' or migrate_engine.name == 'mysql': - return False - elif migrate_engine.name == 'sqlite': - return 0 - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name) + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return False + elif migrate_engine.name == 'sqlite': + return 0 + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name) # Load existing tables metadata.reflect() # Add 2 indexes to the galaxy_user table @@ -42,13 +43,13 @@ def upgrade(migrate_engine): cmd = "UPDATE galaxy_user SET deleted = %s WHERE deleted is null" cmd = cmd % boolean_false() try: - db_session.execute( cmd ) + migrate_engine.execute( cmd ) except Exception, e: log.debug( "Setting default data for galaxy_user.deleted column failed: %s" % ( str( e ) ) ) cmd = "UPDATE galaxy_user SET purged = %s WHERE purged is null" cmd = cmd % boolean_false() try: - db_session.execute( cmd ) + migrate_engine.execute( cmd ) except Exception, e: log.debug( "Setting default data for galaxy_user.purged column failed: %s" % ( str( e ) ) ) # Add 1 index to the history_dataset_association table @@ -57,6 +58,8 @@ def upgrade(migrate_engine): i.create() except Exception, e: log.debug( "Adding index 'ix_hda_copied_from_library_dataset_dataset_association_id' to history_dataset_association table failed: %s" % ( str( e ) ) ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py index ea62c5b5c9f..2a8042eb25e 100644 --- a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py +++ b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py @@ -2,31 +2,18 @@ import datetime import errno import logging import os -import sys import time +from sqlalchemy import and_, Boolean, Column, DateTime, false, ForeignKey, Integer, MetaData, not_, Numeric, Table, TEXT, true +from sqlalchemy.orm import backref, mapper, relation, scoped_session, sessionmaker + +from galaxy.datatypes.metadata import MetadataCollection +from galaxy.model.custom_types import MetadataType, TrimmedString +from galaxy.util.bunch import Bunch + log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) -handler = logging.StreamHandler( sys.stdout ) -format = "%(name)s %(levelname)s %(asctime)s %(message)s" -formatter = logging.Formatter( format ) -handler.setFormatter( formatter ) -log.addHandler( handler ) - -from sqlalchemy import and_, false, true - -from sqlalchemy import * now = datetime.datetime.utcnow -from sqlalchemy.orm import * - -from migrate import * -from migrate.changeset import * - -from galaxy.model.custom_types import * - -from galaxy.util.bunch import Bunch - - metadata = MetaData() context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) ) @@ -35,6 +22,7 @@ context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) ) def get_permitted_actions( **kwds ): return Bunch() + def directory_hash_id( id ): s = str( id ) l = len( s ) @@ -46,21 +34,22 @@ def directory_hash_id( id ): # Drop the last three digits -- 1000 files per directory padded = padded[:-3] # Break into chunks of three - return [ padded[i*3:(i+1)*3] for i in range( len( padded ) // 3 ) ] + return [ padded[i * 3:(i + 1) * 3] for i in range( len( padded ) // 3 ) ] class Dataset( object ): - states = Bunch( NEW = 'new', - UPLOAD = 'upload', - QUEUED = 'queued', - RUNNING = 'running', - OK = 'ok', - EMPTY = 'empty', - ERROR = 'error', - DISCARDED = 'discarded' ) + states = Bunch( NEW='new', + UPLOAD='upload', + QUEUED='queued', + RUNNING='running', + OK='ok', + EMPTY='empty', + ERROR='error', + DISCARDED='discarded' ) permitted_actions = get_permitted_actions( filter='DATASET' ) file_path = "/tmp/" engine = None + def __init__( self, id=None, state=None, external_filename=None, extra_files_path=None, file_size=None, purgable=True ): self.id = id self.state = state @@ -70,6 +59,7 @@ class Dataset( object ): self.external_filename = external_filename self._extra_files_path = extra_files_path self.file_size = file_size + def get_file_name( self ): if not self.external_filename: assert self.id is not None, "ID must be set before filename used (commit the object)" @@ -92,23 +82,26 @@ class Dataset( object ): filename = self.external_filename # Make filename absolute return os.path.abspath( filename ) - def set_file_name ( self, filename ): + + def set_file_name( self, filename ): if not filename: self.external_filename = None else: self.external_filename = filename file_name = property( get_file_name, set_file_name ) + @property def extra_files_path( self ): if self._extra_files_path: path = self._extra_files_path else: path = os.path.join( self.file_path, "dataset_%d_files" % self.id ) - #only use path directly under self.file_path if it exists + # only use path directly under self.file_path if it exists if not os.path.exists( path ): path = os.path.join( os.path.join( self.file_path, *directory_hash_id( self.id ) ), "dataset_%d_files" % self.id ) # Make path absolute return os.path.abspath( path ) + def get_size( self ): """Returns the size of the data on disk""" if self.file_size: @@ -118,6 +111,7 @@ class Dataset( object ): return os.path.getsize( self.file_name ) except OSError: return 0 + def set_size( self ): """Returns the size of the data on disk""" try: @@ -125,12 +119,15 @@ class Dataset( object ): self.file_size = os.path.getsize( self.file_name ) except OSError: self.file_size = 0 + def has_data( self ): """Detects whether there is any data""" return self.get_size() > 0 + def mark_deleted( self, include_children=True ): self.deleted = True # FIXME: sqlalchemy will replace this + def _delete(self): """Remove the file that corresponds to this data""" try: @@ -138,13 +135,15 @@ class Dataset( object ): except OSError, e: log.critical('%s delete error %s' % (self.__class__.__name__, e)) + class DatasetInstance( object ): """A base class for all 'dataset instances', HDAs, LDAs, etc""" states = Dataset.states permitted_actions = Dataset.permitted_actions + def __init__( self, id=None, hid=None, name=None, info=None, blurb=None, peek=None, extension=None, dbkey=None, metadata=None, history=None, dataset=None, deleted=False, designation=None, - parent_id=None, validation_errors=None, visible=True, create_dataset = False ): + parent_id=None, validation_errors=None, visible=True, create_dataset=False ): self.name = name or "Unnamed dataset" self.id = id self.info = info @@ -153,7 +152,7 @@ class DatasetInstance( object ): self.extension = extension self.designation = designation self.metadata = metadata or dict() - if dbkey: #dbkey is stored in metadata, only set if non-zero, or else we could clobber one supplied by input 'metadata' + if dbkey: # dbkey is stored in metadata, only set if non-zero, or else we could clobber one supplied by input 'metadata' self.dbkey = dbkey self.deleted = deleted self.visible = visible @@ -165,42 +164,51 @@ class DatasetInstance( object ): self.dataset = dataset self.parent_id = parent_id self.validation_errors = validation_errors + @property def ext( self ): return self.extension + def get_dataset_state( self ): return self.dataset.state - def set_dataset_state ( self, state ): + + def set_dataset_state( self, state ): self.dataset.state = state context.add( self.dataset ) - context.flush() #flush here, because hda.flush() won't flush the Dataset object + context.flush() # flush here, because hda.flush() won't flush the Dataset object state = property( get_dataset_state, set_dataset_state ) + def get_file_name( self ): return self.dataset.get_file_name() - def set_file_name (self, filename): + + def set_file_name(self, filename): return self.dataset.set_file_name( filename ) file_name = property( get_file_name, set_file_name ) + @property def extra_files_path( self ): return self.dataset.extra_files_path - @property - def datatype( self ): - return datatypes_registry.get_datatype_by_extension( self.extension ) + def get_metadata( self ): - if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent != self: #using weakref to store parent (to prevent circ ref), does a context.clear() cause parent to be invalidated, while still copying over this non-database attribute? + if not hasattr( self, '_metadata_collection' ) or self._metadata_collection.parent != self: # using weakref to store parent (to prevent circ ref), does a context.clear() cause parent to be invalidated, while still copying over this non-database attribute? self._metadata_collection = MetadataCollection( self ) return self._metadata_collection + def set_metadata( self, bunch ): # Needs to accept a MetadataCollection, a bunch, or a dict self._metadata = self.metadata.make_dict_copy( bunch ) metadata = property( get_metadata, set_metadata ) # This provide backwards compatibility with using the old dbkey # field in the database. That field now maps to "old_dbkey" (see mapping.py). + def get_dbkey( self ): dbkey = self.metadata.dbkey - if not isinstance(dbkey, list): dbkey = [dbkey] - if dbkey in [[None], []]: return "?" + if not isinstance(dbkey, list): + dbkey = [dbkey] + if dbkey in [[None], []]: + return "?" return dbkey[0] + def set_dbkey( self, value ): if "dbkey" in self.datatype.metadata_spec: if not isinstance(value, list): @@ -208,83 +216,95 @@ class DatasetInstance( object ): else: self.metadata.dbkey = value dbkey = property( get_dbkey, set_dbkey ) - def change_datatype( self, new_ext ): - self.clear_associated_files() - datatypes_registry.change_datatype( self, new_ext ) + def get_size( self ): """Returns the size of the data on disk""" return self.dataset.get_size() + def set_size( self ): """Returns the size of the data on disk""" return self.dataset.set_size() + def has_data( self ): """Detects whether there is any data""" return self.dataset.has_data() + def get_raw_data( self ): """Returns the full data. To stream it open the file_name and read/write as needed""" return self.datatype.get_raw_data( self ) + def write_from_stream( self, stream ): """Writes data from a stream""" self.datatype.write_from_stream(self, stream) + def set_raw_data( self, data ): """Saves the data on the disc""" self.datatype.set_raw_data(self, data) - def get_mime( self ): - """Returns the mime type of the data""" - return datatypes_registry.get_mimetype_by_extension( self.extension.lower() ) + def set_peek( self, is_multi_byte=False ): return self.datatype.set_peek( self, is_multi_byte=is_multi_byte ) + def init_meta( self, copy_from=None ): return self.datatype.init_meta( self, copy_from=copy_from ) + def set_meta( self, **kwd ): - self.clear_associated_files( metadata_safe = True ) + self.clear_associated_files( metadata_safe=True ) return self.datatype.set_meta( self, **kwd ) + def missing_meta( self, **kwd ): return self.datatype.missing_meta( self, **kwd ) + def as_display_type( self, type, **kwd ): return self.datatype.as_display_type( self, type, **kwd ) + def display_peek( self ): return self.datatype.display_peek( self ) + def display_name( self ): return self.datatype.display_name( self ) + def display_info( self ): return self.datatype.display_info( self ) + def get_converted_files_by_type( self, file_type ): valid = [] for assoc in self.implicitly_converted_datasets: if not assoc.deleted and assoc.type == file_type: valid.append( assoc.dataset ) return valid - def clear_associated_files( self, metadata_safe = False, purge = False ): + + def clear_associated_files( self, metadata_safe=False, purge=False ): raise 'Unimplemented' + def get_child_by_designation(self, designation): for child in self.children: if child.designation == designation: return child return None - def get_converter_types(self): - return self.datatype.get_converter_types( self, datatypes_registry) - def find_conversion_destination( self, accepted_formats, **kwd ): - """Returns ( target_ext, exisiting converted dataset )""" - return self.datatype.find_conversion_destination( self, accepted_formats, datatypes_registry, **kwd ) + def add_validation_error( self, validation_error ): self.validation_errors.append( validation_error ) + def extend_validation_errors( self, validation_errors ): self.validation_errors.extend(validation_errors) + def mark_deleted( self, include_children=True ): self.deleted = True if include_children: for child in self.children: child.mark_deleted() + def mark_undeleted( self, include_children=True ): self.deleted = False if include_children: for child in self.children: child.mark_undeleted() + def undeletable( self ): if self.purged: return False return True + @property def source_library_dataset( self ): def get_source( dataset ): @@ -305,10 +325,10 @@ class DatasetInstance( object ): class HistoryDatasetAssociation( DatasetInstance ): def __init__( self, - hid = None, - history = None, - copied_from_history_dataset_association = None, - copied_from_library_dataset_dataset_association = None, + hid=None, + history=None, + copied_from_history_dataset_association=None, + copied_from_library_dataset_dataset_association=None, **kwd ): DatasetInstance.__init__( self, **kwd ) self.hid = hid @@ -316,7 +336,8 @@ class HistoryDatasetAssociation( DatasetInstance ): self.history = history self.copied_from_history_dataset_association = copied_from_history_dataset_association self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association - def copy( self, copy_children = False, parent_id = None, target_history = None ): + + def copy( self, copy_children=False, parent_id=None, target_history=None ): hda = HistoryDatasetAssociation( hid=self.hid, name=self.name, info=self.info, @@ -324,12 +345,12 @@ class HistoryDatasetAssociation( DatasetInstance ): peek=self.peek, extension=self.extension, dbkey=self.dbkey, - dataset = self.dataset, + dataset=self.dataset, visible=self.visible, deleted=self.deleted, parent_id=parent_id, copied_from_history_dataset_association=self, - history = target_history ) + history=target_history ) context.add( hda ) context.flush() hda.set_size() @@ -337,12 +358,13 @@ class HistoryDatasetAssociation( DatasetInstance ): hda.metadata = self.metadata if copy_children: for child in self.children: - child_copy = child.copy( copy_children = copy_children, parent_id = hda.id ) + child.copy( copy_children=copy_children, parent_id=hda.id ) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs hda.set_peek() context.flush() return hda + def to_library_dataset_dataset_association( self, target_folder, replace_dataset=None, parent_id=None ): if replace_dataset: # The replace_dataset param ( when not None ) refers to a LibraryDataset that is being replaced with a new version. @@ -379,18 +401,18 @@ class HistoryDatasetAssociation( DatasetInstance ): context.add( library_dataset ) context.flush() for child in self.children: - child_copy = child.to_library_dataset_dataset_association( target_folder=target_folder, replace_dataset=replace_dataset, parent_id=ldda.id ) + child.to_library_dataset_dataset_association( target_folder=target_folder, replace_dataset=replace_dataset, parent_id=ldda.id ) if not self.datatype.copy_safe_peek: # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs ldda.set_peek() context.flush() return ldda - def clear_associated_files( self, metadata_safe = False, purge = False ): + + def clear_associated_files( self, metadata_safe=False, purge=False ): # metadata_safe = True means to only clear when assoc.metadata_safe == False for assoc in self.implicitly_converted_datasets: if not metadata_safe or not assoc.metadata_safe: - assoc.clear( purge = purge ) - + assoc.clear( purge=purge ) class LibraryDatasetDatasetAssociation( DatasetInstance ): @@ -405,6 +427,7 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): self.copied_from_library_dataset_dataset_association = copied_from_library_dataset_dataset_association self.library_dataset = library_dataset self.user = user + def to_history_dataset_association( self, target_history, parent_id=None ): hid = target_history._next_hid() hda = HistoryDatasetAssociation( name=self.name, @@ -420,16 +443,17 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): copied_from_library_dataset_dataset_association=self, history=target_history, hid=hid ) - context.flush() - hda.metadata = self.metadata #need to set after flushed, as MetadataFiles require dataset.id - for child in self.children: - child_copy = child.to_history_dataset_association( target_history=target_history, parent_id=hda.id ) - if not self.datatype.copy_safe_peek: - hda.set_peek() #in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs context.add( hda ) context.flush() + hda.metadata = self.metadata # need to set after flushed, as MetadataFiles require dataset.id + for child in self.children: + child.to_history_dataset_association( target_history=target_history, parent_id=hda.id ) + if not self.datatype.copy_safe_peek: + hda.set_peek() # in some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs + context.flush() return hda - def copy( self, copy_children = False, parent_id = None, target_folder = None ): + + def copy( self, copy_children=False, parent_id=None, target_folder=None ): ldda = LibraryDatasetDatasetAssociation( name=self.name, info=self.info, blurb=self.blurb, @@ -444,18 +468,20 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): folder=target_folder ) context.add( ldda ) context.flush() - # Need to set after flushed, as MetadataFiles require dataset.id + # Need to set after flushed, as MetadataFiles require dataset.id ldda.metadata = self.metadata if copy_children: for child in self.children: - child_copy = child.copy( copy_children = copy_children, parent_id = ldda.id ) + child.copy( copy_children=copy_children, parent_id=ldda.id ) if not self.datatype.copy_safe_peek: - # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs + # In some instances peek relies on dataset_id, i.e. gmaj.zip for viewing MAFs ldda.set_peek() context.flush() return ldda - def clear_associated_files( self, metadata_safe = False, purge = False ): + + def clear_associated_files( self, metadata_safe=False, purge=False ): return + def get_library_item_info_templates( self, template_list=[], restrict=False ): # If restrict is True, we'll return only those templates directly associated with this LibraryDatasetDatasetAssociation if self.library_dataset_dataset_info_template_associations: @@ -464,7 +490,6 @@ class LibraryDatasetDatasetAssociation( DatasetInstance ): return template_list - class LibraryDataset( object ): # This class acts as a proxy to the currently selected LDDA def __init__( self, folder=None, order_id=None, name=None, info=None, library_dataset_dataset_association=None, **kwd ): @@ -473,11 +498,13 @@ class LibraryDataset( object ): self.name = name self.info = info self.library_dataset_dataset_association = library_dataset_dataset_association + def set_library_dataset_dataset_association( self, ldda ): self.library_dataset_dataset_association = ldda ldda.library_dataset = self context.add_all( ( self, ldda ) ) context.flush() + def get_info( self ): if self.library_dataset_dataset_association: return self.library_dataset_dataset_association.info @@ -485,9 +512,11 @@ class LibraryDataset( object ): return self._info else: return 'no info' + def set_info( self, info ): self._info = info info = property( get_info, set_info ) + def get_name( self ): if self.library_dataset_dataset_association: return self.library_dataset_dataset_association.name @@ -495,19 +524,24 @@ class LibraryDataset( object ): return self._name else: return 'Unnamed dataset' + def set_name( self, name ): self._name = name name = property( get_name, set_name ) + def display_name( self ): self.library_dataset_dataset_association.display_name() + def get_purged( self ): return self.library_dataset_dataset_association.dataset.purged + def set_purged( self, purged ): if purged: raise Exception( "Not implemented" ) if not purged and self.purged: raise Exception( "Cannot unpurge once purged" ) purged = property( get_purged, set_purged ) + def get_library_item_info_templates( self, template_list=[], restrict=False ): # If restrict is True, we'll return only those templates directly associated with this LibraryDataset if self.library_dataset_info_template_associations: @@ -516,7 +550,7 @@ class LibraryDataset( object ): self.folder.get_library_item_info_templates( template_list, restrict ) return template_list -##tables +# tables Dataset.table = Table( "dataset", metadata, @@ -532,7 +566,6 @@ Dataset.table = Table( "dataset", metadata, Column( 'file_size', Numeric( 15, 0 ) ) ) - HistoryDatasetAssociation.table = Table( "history_dataset_association", metadata, Column( "id", Integer, primary_key=True ), Column( "dataset_id", Integer, ForeignKey( "dataset.id" ), index=True ), @@ -575,17 +608,16 @@ LibraryDatasetDatasetAssociation.table = Table( "library_dataset_dataset_associa LibraryDataset.table = Table( "library_dataset", metadata, Column( "id", Integer, primary_key=True ), - Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ),#current version of dataset, if null, there is not a current version selected + Column( "library_dataset_dataset_association_id", Integer, ForeignKey( "library_dataset_dataset_association.id", use_alter=True, name="library_dataset_dataset_association_id_fk" ), nullable=True, index=True ), # current version of dataset, if null, there is not a current version selected Column( "order_id", Integer ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "name", TrimmedString( 255 ), key="_name" ), #when not None/null this will supercede display in library (but not when imported into user's history?) - Column( "info", TrimmedString( 255 ), key="_info" ), #when not None/null this will supercede display in library (but not when imported into user's history?) + Column( "name", TrimmedString( 255 ), key="_name" ), # when not None/null this will supercede display in library (but not when imported into user's history?) + Column( "info", TrimmedString( 255 ), key="_info" ), # when not None/null this will supercede display in library (but not when imported into user's history?) Column( "deleted", Boolean, index=True, default=False ) ) - -##mappers +# mappers mapper( Dataset, Dataset.table, @@ -601,8 +633,7 @@ mapper( Dataset, Dataset.table, primaryjoin=( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) ), active_library_associations=relation( LibraryDatasetDatasetAssociation, - primaryjoin=( ( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( LibraryDatasetDatasetAssociation.table.c.deleted == false() ) ) ) - ) ) + primaryjoin=( ( Dataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.dataset_id ) & ( LibraryDatasetDatasetAssociation.table.c.deleted == false() ) ) ) ) ) mapper( HistoryDatasetAssociation, HistoryDatasetAssociation.table, @@ -625,13 +656,12 @@ mapper( HistoryDatasetAssociation, HistoryDatasetAssociation.table, backref=backref( "parent", primaryjoin=( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ), remote_side=[HistoryDatasetAssociation.table.c.id], uselist=False ) ), visible_children=relation( HistoryDatasetAssociation, - primaryjoin=( ( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ) & ( HistoryDatasetAssociation.table.c.visible == true() ) ) ) - ) ) + primaryjoin=( ( HistoryDatasetAssociation.table.c.parent_id == HistoryDatasetAssociation.table.c.id ) & ( HistoryDatasetAssociation.table.c.visible == true() ) ) ) ) ) mapper( LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table, properties=dict( dataset=relation( Dataset ), - library_dataset = relation( LibraryDataset, + library_dataset=relation( LibraryDataset, primaryjoin=( LibraryDatasetDatasetAssociation.table.c.library_dataset_id == LibraryDataset.table.c.id ) ), copied_to_library_dataset_dataset_associations=relation( LibraryDatasetDatasetAssociation, @@ -647,14 +677,12 @@ mapper( LibraryDatasetDatasetAssociation, LibraryDatasetDatasetAssociation.table backref=backref( "parent", primaryjoin=( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ), remote_side=[LibraryDatasetDatasetAssociation.table.c.id] ) ), visible_children=relation( LibraryDatasetDatasetAssociation, - primaryjoin=( ( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ) & ( LibraryDatasetDatasetAssociation.table.c.visible == true() ) ) ) - ) ) + primaryjoin=( ( LibraryDatasetDatasetAssociation.table.c.parent_id == LibraryDatasetDatasetAssociation.table.c.id ) & ( LibraryDatasetDatasetAssociation.table.c.visible == true() ) ) ) ) ) mapper( LibraryDataset, LibraryDataset.table, properties=dict( library_dataset_dataset_association=relation( LibraryDatasetDatasetAssociation, primaryjoin=( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ), - expired_datasets = relation( LibraryDatasetDatasetAssociation, foreign_keys=[LibraryDataset.table.c.id,LibraryDataset.table.c.library_dataset_dataset_association_id ], primaryjoin=( ( LibraryDataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.library_dataset_id ) & ( not_( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ) ), viewonly=True, uselist=True ) - ) ) + expired_datasets=relation( LibraryDatasetDatasetAssociation, foreign_keys=[LibraryDataset.table.c.id, LibraryDataset.table.c.library_dataset_dataset_association_id ], primaryjoin=( ( LibraryDataset.table.c.id == LibraryDatasetDatasetAssociation.table.c.library_dataset_id ) & ( not_( LibraryDataset.table.c.library_dataset_dataset_association_id == LibraryDatasetDatasetAssociation.table.c.id ) ) ), viewonly=True, uselist=True ) ) ) def __guess_dataset_by_filename( filename ): @@ -662,12 +690,13 @@ def __guess_dataset_by_filename( filename ): try: fields = os.path.split( filename ) if fields: - if fields[-1].startswith( 'dataset_' ) and fields[-1].endswith( '.dat' ): #dataset_%d.dat + if fields[-1].startswith( 'dataset_' ) and fields[-1].endswith( '.dat' ): # dataset_%d.dat return Dataset.get( int( fields[-1][ len( 'dataset_' ): -len( '.dat' ) ] ) ) except: - pass #some parsing error, we can't guess Dataset + pass # some parsing error, we can't guess Dataset return None + def upgrade(migrate_engine): metadata.bind = migrate_engine log.debug( "Fixing a discrepancy concerning deleted shared history items." ) @@ -678,14 +707,14 @@ def upgrade(migrate_engine): if not dataset_instance.deleted: dataset.deleted = False if dataset.file_size in [ None, 0 ]: - dataset.set_size() #Restore filesize + dataset.set_size() # Restore filesize affected_items += 1 break context.flush() log.debug( "%i items affected, and restored." % ( affected_items ) ) log.debug( "Time elapsed: %s" % ( time.time() - start_time ) ) - #fix share before hda + # fix share before hda log.debug( "Fixing a discrepancy concerning cleaning up deleted history items shared before HDAs." ) dataset_by_filename = {} changed_associations = 0 @@ -695,23 +724,23 @@ def upgrade(migrate_engine): guessed_dataset = dataset_by_filename[ dataset.file_name ] else: guessed_dataset = __guess_dataset_by_filename( dataset.file_name ) - if guessed_dataset and dataset.file_name != guessed_dataset.file_name:#not os.path.samefile( dataset.file_name, guessed_dataset.file_name ): + if guessed_dataset and dataset.file_name != guessed_dataset.file_name: # not os.path.samefile( dataset.file_name, guessed_dataset.file_name ): guessed_dataset = None dataset_by_filename[ dataset.file_name ] = guessed_dataset - if guessed_dataset is not None and guessed_dataset.id != dataset.id: #could we have a self referential dataset? + if guessed_dataset is not None and guessed_dataset.id != dataset.id: # could we have a self referential dataset? for dataset_instance in dataset.history_associations + dataset.library_associations: dataset_instance.dataset = guessed_dataset changed_associations += 1 - #mark original Dataset as deleted and purged, it is no longer in use, but do not delete file_name contents + # mark original Dataset as deleted and purged, it is no longer in use, but do not delete file_name contents dataset.deleted = True dataset.external_filename = "Dataset was result of share before HDA, and has been replaced: %s mapped to Dataset %s" % ( dataset.external_filename, guessed_dataset.id ) - dataset.purged = True #we don't really purge the file here, but we mark it as purged, since this dataset is now defunct + dataset.purged = True # we don't really purge the file here, but we mark it as purged, since this dataset is now defunct context.flush() log.debug( "%i items affected, and restored." % ( changed_associations ) ) log.debug( "Time elapsed: %s" % ( time.time() - start_time ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine log.debug( "Downgrade is not possible." ) - diff --git a/lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py b/lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py index ff4521ca60f..3a84009d96e 100644 --- a/lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py +++ b/lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py @@ -2,10 +2,11 @@ This migration script changes certain values in the history_dataset_association.extension column, specifically 'qual' is chaged to be 'qual454'. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -import sys, logging +import logging +import sys + +from sqlalchemy import Index, MetaData, Table +from sqlalchemy.orm import scoped_session, sessionmaker log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) @@ -17,6 +18,7 @@ log.addHandler( handler ) metadata = MetaData() + def display_migration_details(): print "========================================" print "This migration script changes certain values in the history_dataset_association.extension" @@ -55,6 +57,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Dropping index 'ix_hda_extension' to history_dataset_association table failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0007_sharing_histories.py b/lib/galaxy/model/migrate/versions/0007_sharing_histories.py index 6fadb323ae4..c83bd51ad4d 100644 --- a/lib/galaxy/model/migrate/versions/0007_sharing_histories.py +++ b/lib/galaxy/model/migrate/versions/0007_sharing_histories.py @@ -3,11 +3,11 @@ This migration script creates the new history_user_share_association table, and a new boolean type column to the history table. This provides support for sharing histories in the same way that workflows are shared. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging +import logging +import sys + +from sqlalchemy import Boolean, Column, ForeignKey, Integer, MetaData, Table +from sqlalchemy.exc import NoSuchTableError log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) @@ -19,6 +19,7 @@ log.addHandler( handler ) metadata = MetaData() + def display_migration_details(): print "========================================" print "This migration script creates the new history_user_share_association table, and adds" @@ -29,8 +30,8 @@ def display_migration_details(): HistoryUserShareAssociation_table = Table( "history_user_share_association", metadata, Column( "id", Integer, primary_key=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) - ) + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -56,6 +57,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Adding column 'importable' to history table failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine # Load existing tables diff --git a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py index 9128370639b..4cc3e1cbbcf 100644 --- a/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py +++ b/lib/galaxy/model/migrate/versions/0008_galaxy_forms.py @@ -9,16 +9,17 @@ This migration script adds the following new tables for supporting Galaxy forms: 7) sample_state 8) sample_event """ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow +import logging +import sys -import sys, logging +from migrate import ForeignKeyConstraint +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import JSONType, TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -26,12 +27,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - metadata = MetaData() + def display_migration_details(): print "========================================" print "This migration script adds the following new tables for supporting Galaxy forms:" @@ -115,8 +113,8 @@ SampleEvent_table = Table('sample_event', metadata, Column( "sample_state_id", Integer, ForeignKey( "sample_state.id" ), index=True ), Column( "comment", TEXT ) ) + def upgrade(migrate_engine): - db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) metadata.bind = migrate_engine display_migration_details() # Load existing tables @@ -166,8 +164,8 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating sample_event table failed: %s" % str( e ) ) + def downgrade(migrate_engine): - db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) metadata.bind = migrate_engine # Load existing tables metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0009_request_table.py b/lib/galaxy/model/migrate/versions/0009_request_table.py index d89a28bb060..c51375f1c3c 100644 --- a/lib/galaxy/model/migrate/versions/0009_request_table.py +++ b/lib/galaxy/model/migrate/versions/0009_request_table.py @@ -3,13 +3,12 @@ This migration script adds a new column to 2 tables: 1) a new boolean type column named 'submitted' to the 'request' table 2) a new string type column named 'bar_code' to the 'sample' table """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * +import logging +import sys + +from sqlalchemy import Boolean, Column, MetaData, Table + +from galaxy.model.custom_types import TrimmedString log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) @@ -21,6 +20,7 @@ log.addHandler( handler ) metadata = MetaData() + def display_migration_details(): print "========================================" print "This migration script adds a new column to 2 tables:" @@ -28,6 +28,7 @@ def display_migration_details(): print "2) a new string type column named 'bar_code' to the 'sample' table" print "========================================" + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -40,7 +41,6 @@ def upgrade(migrate_engine): try: col = Column( 'submitted', Boolean, default=False ) col.create( Request_table) - #col.create( Request_table, index_name='ix_request_submitted') assert col is Request_table.c.submitted except Exception, e: log.debug( "Adding column 'submitted' to request table failed: %s" % ( str( e ) ) ) @@ -54,6 +54,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Adding column 'bar_code' to sample table failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py b/lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py index 9234e73185f..453dbfd36e0 100644 --- a/lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py +++ b/lib/galaxy/model/migrate/versions/0010_hda_display_at_authz_table.py @@ -8,16 +8,16 @@ history_dataset_association_display_at_authorization table failed: (Operational (1059, "Identifier name 'ix_history_dataset_association_display_at_authorization_update_time' is too long """ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow +import logging +import sys -import sys, logging +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -25,12 +25,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - metadata = MetaData() + def display_migration_details(): print "========================================" print "This migration script adds the history_dataset_association_display_at_authorization table, which" @@ -50,6 +47,7 @@ HistoryDatasetAssociationDisplayAtAuthorization_table = Table( "history_dataset_ Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "site", TrimmedString( 255 ) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -60,6 +58,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating history_dataset_association_display_at_authorization table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine # Load existing tables diff --git a/lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py b/lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py index 580f2b1ddf9..964c9685caa 100644 --- a/lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py +++ b/lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py @@ -3,16 +3,16 @@ This script fixes a problem introduced in 0010_hda_display_at_atuhz_table.py. M name length limit and thus the index "ix_hdadaa_history_dataset_association_id" has to be manually created. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow +import logging +import sys -import sys, logging +from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, MetaData, Table + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -20,12 +20,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - metadata = MetaData() + def display_migration_details(): print "========================================" print "This script fixes a problem introduced in the previous migration script ( 9->10 ). MySQL" @@ -41,6 +38,7 @@ HistoryDatasetAssociationDisplayAtAuthorization_table = Table( "history_dataset_ Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "site", TrimmedString( 255 ) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -53,6 +51,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Adding index 'ix_hdadaa_history_dataset_association_id' to table 'history_dataset_association_display_at_authorization' table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine if migrate_engine.name == 'mysql': diff --git a/lib/galaxy/model/migrate/versions/0012_user_address.py b/lib/galaxy/model/migrate/versions/0012_user_address.py index 4f0250a0ec1..31095ee6850 100644 --- a/lib/galaxy/model/migrate/versions/0012_user_address.py +++ b/lib/galaxy/model/migrate/versions/0012_user_address.py @@ -4,17 +4,17 @@ a user can select from a list of his addresses to associate with the request. T drops the request.submitted column which was boolean and replaces it with a request.state column which is a string, allowing for more flexibility with request states. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from galaxy.model.custom_types import * -from migrate import * -from migrate.changeset import * import datetime -now = datetime.datetime.utcnow -import sys, logging -# Need our custom types, but don't import anything else from model +import logging +import sys +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT +from sqlalchemy.exc import NoSuchTableError + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -22,9 +22,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - metadata = MetaData() + def display_migration_details(): print "========================================" print "This script adds a new user_address table that is currently only used with sample requests, where" @@ -50,8 +50,8 @@ UserAddress_table = Table( "user_address", metadata, Column( "deleted", Boolean, index=True, default=False ), Column( "purged", Boolean, index=True, default=False ) ) + def upgrade(migrate_engine): - #raise Exception metadata.bind = migrate_engine display_migration_details() # Load existing tables @@ -84,12 +84,13 @@ def upgrade(migrate_engine): log.debug( "Failed loading table request" ) if Request_table is not None: if migrate_engine.name != 'sqlite': - #DBTODO drop from table doesn't work in sqlite w/ sqlalchemy-migrate .6+ + # DBTODO drop from table doesn't work in sqlite w/ sqlalchemy-migrate .6+ Request_table.c.submitted.drop() col = Column( "state", TrimmedString( 255 ), index=True ) col.create( Request_table, index_name='ix_request_state') assert col is Request_table.c.state + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0013_change_lib_item_templates_to_forms.py b/lib/galaxy/model/migrate/versions/0013_change_lib_item_templates_to_forms.py index 1b9a127e4d6..bfefeb07ba6 100644 --- a/lib/galaxy/model/migrate/versions/0013_change_lib_item_templates_to_forms.py +++ b/lib/galaxy/model/migrate/versions/0013_change_lib_item_templates_to_forms.py @@ -15,12 +15,11 @@ If using mysql, this script will throw an (OperationalError) exception due to a the library_dataset_dataset_info_association table, which is OK because the script creates an index with a shortened name. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * -import sys, logging +import logging +import sys + +from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table +from sqlalchemy.exc import NoSuchTableError log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) @@ -29,9 +28,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - metadata = MetaData() + def display_migration_details(): print "========================================" print "This migration script eliminates all of the tables that were used for the 1st version of the" @@ -68,24 +67,12 @@ LibraryDatasetDatasetInfoAssociation_table = Table( 'library_dataset_dataset_inf Column( "form_definition_id", Integer, ForeignKey( "form_definition.id" ), index=True ), Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() # Load existing tables metadata.reflect() - if migrate_engine.name == 'postgres': - # http://blog.pythonisito.com/2008/01/cascading-drop-table-with-sqlalchemy.html - from sqlalchemy.databases import postgres - class PGCascadeSchemaDropper(postgres.PGSchemaDropper): - def visit_table(self, table): - for column in table.columns: - if column.default is not None: - self.traverse_single(column.default) - self.append("\nDROP TABLE " + - self.preparer.format_table(table) + - " CASCADE") - self.execute() - postgres.dialect.schemadropper = PGCascadeSchemaDropper # Drop all of the original library_item_info tables # NOTE: all existing library item into template data is eliminated here via table drops try: @@ -251,6 +238,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Adding index 'ix_lddaia_ldda_id' to table 'library_dataset_dataset_info_association' table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine log.debug( "Downgrade is not possible." ) diff --git a/lib/galaxy/model/migrate/versions/0014_pages.py b/lib/galaxy/model/migrate/versions/0014_pages.py index c7fb1db2295..42fa44f56af 100644 --- a/lib/galaxy/model/migrate/versions/0014_pages.py +++ b/lib/galaxy/model/migrate/versions/0014_pages.py @@ -3,17 +3,13 @@ Migration script to add support for "Pages". 1) Creates Page and PageRevision tables 2) Adds username column to User table """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, MetaData, String, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() Page_table = Table( "page", metadata, @@ -24,8 +20,7 @@ Page_table = Table( "page", metadata, Column( "latest_revision_id", Integer, ForeignKey( "page_revision.id", use_alter=True, name='page_latest_revision_id_fk' ), index=True ), Column( "title", TEXT ), - Column( "slug", TEXT, unique=True, index=True ), - ) + Column( "slug", TEXT, unique=True, index=True ) ) PageRevision_table = Table( "page_revision", metadata, Column( "id", Integer, primary_key=True ), @@ -33,8 +28,8 @@ PageRevision_table = Table( "page_revision", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "page_id", Integer, ForeignKey( "page.id" ), index=True, nullable=False ), Column( "title", TEXT ), - Column( "content", TEXT ) - ) + Column( "content", TEXT ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -51,7 +46,7 @@ def upgrade(migrate_engine): Page_table.create() if migrate_engine.name == 'mysql': # Create slug index manually afterward. - i = Index( "ix_page_slug", Page_table.c.slug, mysql_length = 200) + i = Index( "ix_page_slug", Page_table.c.slug, mysql_length=200) i.create() except Exception, ex: log.debug(ex) @@ -67,6 +62,7 @@ def upgrade(migrate_engine): col.create( User_table, index_name='ix_user_username', unique_name='username' ) assert col is User_table.c.username + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0015_tagging.py b/lib/galaxy/model/migrate/versions/0015_tagging.py index 10f682ec0de..6eafbf9c57e 100644 --- a/lib/galaxy/model/migrate/versions/0015_tagging.py +++ b/lib/galaxy/model/migrate/versions/0015_tagging.py @@ -9,21 +9,20 @@ history_dataset_association_tag_association table failed: (OperationalError) (1059, "Identifier name 'ix_history_dataset_association_tag_association_history_dataset_association_id' is too long) """ - -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow +import logging + +from migrate import UniqueConstraint +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table # Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from galaxy.model.custom_types import TrimmedString -import logging +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) - metadata = MetaData() + def display_migration_details(): print "" print "This migration script adds the tables necessary to support tagging of histories," @@ -66,6 +65,7 @@ HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_associat Column( "value", TrimmedString(255), index=True), Column( "user_value", TrimmedString(255), index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -91,6 +91,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating history_dataset_association_tag_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py b/lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py index 1b9b9deec8d..742ce73571d 100644 --- a/lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py +++ b/lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py @@ -3,21 +3,19 @@ This script fixes a problem introduced in 0015_tagging.py. MySQL has a name leng limit and thus the index "ix_hda_ta_history_dataset_association_id" has to be manually created. """ - -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow +import logging + +from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table # Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from galaxy.model.custom_types import TrimmedString -import logging +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) - metadata = MetaData() + def display_migration_details(): print "" print "This script fixes a problem introduced in 0015_tagging.py. MySQL has a" @@ -31,6 +29,7 @@ HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_associat Column( "value", TrimmedString(255), index=True), Column( "user_value", TrimmedString(255), index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -42,6 +41,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Adding index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0017_library_item_indexes.py b/lib/galaxy/model/migrate/versions/0017_library_item_indexes.py index a1733fcd17d..4e6174dcd43 100644 --- a/lib/galaxy/model/migrate/versions/0017_library_item_indexes.py +++ b/lib/galaxy/model/migrate/versions/0017_library_item_indexes.py @@ -2,10 +2,10 @@ This script adds 3 indexes to table columns: library_folder.name, library_dataset.name, library_dataset_dataset_association.name. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -import sys, logging +import logging +import sys + +from sqlalchemy import Index, MetaData, Table log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) @@ -14,15 +14,16 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - metadata = MetaData() + def display_migration_details(): print "========================================" print "This script adds 3 indexes to table columns: library_folder.name," print "library_dataset.name, library_dataset_dataset_association.name." print "========================================" + def upgrade(migrate_engine): metadata.bind = migrate_engine LibraryFolder_table = Table( "library_folder", metadata, autoload=True ) @@ -49,6 +50,8 @@ def upgrade(migrate_engine): i.create() except Exception, e: log.debug( "Adding index 'ix_library_dataset_name' to library_dataset table failed: %s" % ( str( e ) ) ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine log.debug( "Downgrade is not possible." ) diff --git a/lib/galaxy/model/migrate/versions/0018_ordered_tags_and_page_tags.py b/lib/galaxy/model/migrate/versions/0018_ordered_tags_and_page_tags.py index 6ffa9351ab9..062c77b8283 100644 --- a/lib/galaxy/model/migrate/versions/0018_ordered_tags_and_page_tags.py +++ b/lib/galaxy/model/migrate/versions/0018_ordered_tags_and_page_tags.py @@ -2,24 +2,20 @@ This migration script provides support for (a) ordering tags by recency and (b) tagging pages. This script deletes all existing tags. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -import migrate.changeset - import datetime -now = datetime.datetime.utcnow +import logging + +from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table +from sqlalchemy.exc import OperationalError # Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from galaxy.model.custom_types import TrimmedString -import logging +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) - metadata = MetaData() + def display_migration_details(): print "" print "This migration script provides support for (a) ordering tags by recency and" @@ -57,6 +53,7 @@ PageTagAssociation_table = Table( "page_tag_association", metadata, Column( "value", TrimmedString(255), index=True), Column( "user_value", TrimmedString(255), index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -104,6 +101,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating page_tag_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py index 2e596c30ca8..b60b265ee13 100644 --- a/lib/galaxy/model/migrate/versions/0019_request_library_folder.py +++ b/lib/galaxy/model/migrate/versions/0019_request_library_folder.py @@ -1,14 +1,15 @@ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * import datetime -now = datetime.datetime.utcnow -import sys, logging -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys +from migrate import ForeignKeyConstraint +from sqlalchemy import Column, Integer, MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import JSONType, TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -16,9 +17,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - metadata = MetaData() + def display_migration_details(): print "========================================" print """This script creates a request.folder_id column which is a foreign @@ -26,6 +27,7 @@ key to the library_folder table. This also adds a 'type' and 'layout' column to the form_definition table.""" print "========================================" + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() diff --git a/lib/galaxy/model/migrate/versions/0020_library_upload_job.py b/lib/galaxy/model/migrate/versions/0020_library_upload_job.py index 22fd33b2d8b..a68950b569a 100644 --- a/lib/galaxy/model/migrate/versions/0020_library_upload_job.py +++ b/lib/galaxy/model/migrate/versions/0020_library_upload_job.py @@ -1,14 +1,12 @@ -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * import datetime -now = datetime.datetime.utcnow -import sys, logging -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys +from migrate import ForeignKeyConstraint +from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, String, Table +from sqlalchemy.exc import NoSuchTableError + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -19,6 +17,7 @@ log.addHandler( handler ) metadata = MetaData() + def display_migration_details(): print "" print "========================================" @@ -35,6 +34,7 @@ JobToOutputLibraryDatasetAssociation_table = Table( "job_to_output_library_datas Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ), Column( "name", String(255) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -66,7 +66,7 @@ def upgrade(migrate_engine): log.debug( "Failed loading table library_folder" ) # Add 1 foreign key constraint to the job table if migrate_engine.name != 'sqlite': - #Sqlite can't alter-table-add-foreign-key + # Sqlite can't alter-table-add-foreign-key if Job_table is not None and LibraryFolder_table is not None: try: cons = ForeignKeyConstraint( [Job_table.c.library_folder_id], @@ -89,6 +89,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Adding index 'ix_dataset_state' to dataset table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0021_user_prefs.py b/lib/galaxy/model/migrate/versions/0021_user_prefs.py index 192c89d04ea..8f4a34e2ae6 100644 --- a/lib/galaxy/model/migrate/versions/0021_user_prefs.py +++ b/lib/galaxy/model/migrate/versions/0021_user_prefs.py @@ -1,18 +1,16 @@ """ This migration script adds a user preferences table to Galaxy. """ - -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, Unicode + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def display_migration_details(): print "" print "This migration script adds a user preferences table to Galaxy." @@ -27,6 +25,7 @@ UserPreference_table = Table( "user_preference", metadata, Column( "name", Unicode( 255 ), index=True), Column( "value", Unicode( 1024 ) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -37,6 +36,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating user_preference table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0022_visualization_tables.py b/lib/galaxy/model/migrate/versions/0022_visualization_tables.py index ebd0e78ba1c..47191e43dad 100644 --- a/lib/galaxy/model/migrate/versions/0022_visualization_tables.py +++ b/lib/galaxy/model/migrate/versions/0022_visualization_tables.py @@ -2,17 +2,13 @@ Migration script to add support for storing visualizations. 1) Creates Visualization and VisualizationRevision tables """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() Visualization_table = Table( "visualization", metadata, @@ -23,8 +19,7 @@ Visualization_table = Table( "visualization", metadata, Column( "latest_revision_id", Integer, ForeignKey( "visualization_revision.id", use_alter=True, name='visualization_latest_revision_id_fk' ), index=True ), Column( "title", TEXT ), - Column( "type", TEXT ) - ) + Column( "type", TEXT ) ) VisualizationRevision_table = Table( "visualization_revision", metadata, Column( "id", Integer, primary_key=True ), @@ -32,8 +27,8 @@ VisualizationRevision_table = Table( "visualization_revision", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "visualization_id", Integer, ForeignKey( "visualization.id" ), index=True, nullable=False ), Column( "title", TEXT ), - Column( "config", TEXT ) - ) + Column( "config", TEXT ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -54,4 +49,3 @@ def downgrade(migrate_engine): metadata.reflect() Visualization_table.drop() VisualizationRevision_table.drop() - diff --git a/lib/galaxy/model/migrate/versions/0023_page_published_and_deleted_columns.py b/lib/galaxy/model/migrate/versions/0023_page_published_and_deleted_columns.py index 8de454e4a82..f5edaa28cb5 100644 --- a/lib/galaxy/model/migrate/versions/0023_page_published_and_deleted_columns.py +++ b/lib/galaxy/model/migrate/versions/0023_page_published_and_deleted_columns.py @@ -2,16 +2,14 @@ Migration script to add columns for tracking whether pages are deleted and publicly accessible. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -21,13 +19,14 @@ def upgrade(migrate_engine): Page_table = Table( "page", metadata, autoload=True ) c = Column( "published", Boolean, index=True, default=False ) - c.create( Page_table, index_name = 'ix_page_published' ) + c.create( Page_table, index_name='ix_page_published' ) assert c is Page_table.c.published c = Column( "deleted", Boolean, index=True, default=False ) c.create( Page_table, index_name='ix_page_deleted') assert c is Page_table.c.deleted + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0024_page_slug_unique_constraint.py b/lib/galaxy/model/migrate/versions/0024_page_slug_unique_constraint.py index 861d2a3e45d..272e6b29f65 100644 --- a/lib/galaxy/model/migrate/versions/0024_page_slug_unique_constraint.py +++ b/lib/galaxy/model/migrate/versions/0024_page_slug_unique_constraint.py @@ -2,19 +2,16 @@ Remove unique constraint from page slugs to allow creating a page with the same slug as a deleted page. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Index, MetaData, Table + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -38,8 +35,7 @@ def upgrade(migrate_engine): Page_table.c.slug.alter( unique=False ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - #Page_table = Table( "page", metadata, autoload=True ) - #Page_table.c.slug.alter( unique=True ) diff --git a/lib/galaxy/model/migrate/versions/0025_user_info.py b/lib/galaxy/model/migrate/versions/0025_user_info.py index 633e6bdc1da..c687e06ed67 100644 --- a/lib/galaxy/model/migrate/versions/0025_user_info.py +++ b/lib/galaxy/model/migrate/versions/0025_user_info.py @@ -1,16 +1,15 @@ """ This script adds a foreign key to the form_values table in the galaxy_user table """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * import datetime -now = datetime.datetime.utcnow -import sys, logging -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys +from migrate import ForeignKeyConstraint +from sqlalchemy import Column, Integer, MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -21,10 +20,13 @@ log.addHandler( handler ) metadata = MetaData() + def display_migration_details(): print "========================================" print "This script adds a foreign key to the form_values table in the galaxy_user table" print "========================================" + + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -58,6 +60,8 @@ def upgrade(migrate_engine): cons.create() except Exception, e: log.debug( "Adding foreign key constraint 'user_form_values_id_fk' to table 'galaxy_user' failed: %s" % ( str( e ) ) ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0026_cloud_tables.py b/lib/galaxy/model/migrate/versions/0026_cloud_tables.py index 8c44aa2d1f6..816c1c00ef9 100644 --- a/lib/galaxy/model/migrate/versions/0026_cloud_tables.py +++ b/lib/galaxy/model/migrate/versions/0026_cloud_tables.py @@ -1,17 +1,13 @@ -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def display_migration_details(): print print "========================================" @@ -127,6 +123,7 @@ CloudProvider_table = Table( "cloud_provider", metadata, Column( "path", TEXT ), Column( "deleted", Boolean, default=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -143,6 +140,7 @@ def upgrade(migrate_engine): CloudStore_table.create() CloudSnapshot_table.create() + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0027_request_events.py b/lib/galaxy/model/migrate/versions/0027_request_events.py index dc683f6e337..5ab18f00b75 100644 --- a/lib/galaxy/model/migrate/versions/0027_request_events.py +++ b/lib/galaxy/model/migrate/versions/0027_request_events.py @@ -2,21 +2,17 @@ This migration script adds the request_event table and removes the state field in the request table """ -# Need our custom types, but don't import anything else from model -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * - import datetime -import sys import logging +import sys -now = datetime.datetime.utcnow +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT +from sqlalchemy.exc import NoSuchTableError +# Need our custom types, but don't import anything else from model from galaxy.model.custom_types import TrimmedString +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -24,10 +20,9 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - - metadata = MetaData() + def display_migration_details(): print "========================================" print "This migration script adds the request_event table and" @@ -40,23 +35,26 @@ RequestEvent_table = Table('request_event', metadata, Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "request_id", Integer, ForeignKey( "request.id" ), index=True ), - Column( "state", TrimmedString( 255 ), index=True ), + Column( "state", TrimmedString( 255 ), index=True ), Column( "comment", TEXT ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() + def localtimestamp(): - if migrate_engine.name == 'postgresql' or migrate_engine.name == 'mysql': - return "LOCALTIMESTAMP" - elif migrate_engine.name == 'sqlite': - return "current_date || ' ' || current_time" - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) + def nextval( table, col='id' ): - if migrate_engine.name == 'postgresql': + if migrate_engine.name in ['postgres', 'postgresql']: return "nextval('%s_%s_seq')" % ( table, col ) - elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + elif migrate_engine.name in ['mysql', 'sqlite']: return "null" else: raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) @@ -94,6 +92,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Deleting column 'state' to request table failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py b/lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py index fbcce08f7d2..096a4a4c786 100644 --- a/lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py +++ b/lib/galaxy/model/migrate/versions/0028_external_metadata_file_override.py @@ -3,16 +3,13 @@ This script adds the filename_override_metadata column to the JobExternalOutputM allowing existing metadata files to be written when using external metadata and a cluster set up with read-only access to database/files """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * import datetime -now = datetime.datetime.utcnow -import sys, logging -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys +from sqlalchemy import Column, MetaData, String, Table + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,6 +20,7 @@ log.addHandler( handler ) metadata = MetaData() + def display_migration_details(): print "========================================" print "This script adds the filename_override_metadata column to the JobExternalOutputMetadata table," @@ -30,6 +28,7 @@ def display_migration_details(): print "set up with read-only access to database/files" print "========================================" + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -43,6 +42,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Adding column 'filename_override_metadata' to job_external_output_metadata table failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0029_user_actions.py b/lib/galaxy/model/migrate/versions/0029_user_actions.py index 3504fb407fb..49d8a8143ab 100644 --- a/lib/galaxy/model/migrate/versions/0029_user_actions.py +++ b/lib/galaxy/model/migrate/versions/0029_user_actions.py @@ -1,18 +1,16 @@ """ This migration script adds a user actions table to Galaxy. """ - -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, Unicode + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def display_migration_details(): print "" print "This migration script adds a user actions table to Galaxy." @@ -29,6 +27,7 @@ UserAction_table = Table( "user_action", metadata, Column( "context", Unicode( 512 ) ), Column( "params", Unicode( 1024 ) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() @@ -39,6 +38,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating user_action table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0030_history_slug_column.py b/lib/galaxy/model/migrate/versions/0030_history_slug_column.py index 3486df23c4d..5e180c82ef3 100644 --- a/lib/galaxy/model/migrate/versions/0030_history_slug_column.py +++ b/lib/galaxy/model/migrate/versions/0030_history_slug_column.py @@ -1,16 +1,14 @@ """ Migration script to add column for a history slug. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, Index, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -19,7 +17,6 @@ def upgrade(migrate_engine): History_table = Table( "history", metadata, autoload=True ) - # Mysql needs manual index creation because of max length index. if migrate_engine.name != 'mysql': # Create slug column. @@ -28,20 +25,11 @@ def upgrade(migrate_engine): else: c = Column( "slug", TEXT ) c.create( History_table , index_name='') - i = Index( "ix_history_slug", History_table.c.slug, mysql_length = 200) + i = Index( "ix_history_slug", History_table.c.slug, mysql_length=200) i.create() assert c is History_table.c.slug - - ## Create slug index. - #try: - #i = Index( "ix_history_slug", History_table.c.slug ) - #i.create() - #except: - ## Mysql doesn't have a named index, but alter should work - #History_table.c.slug.alter( unique=False ) - def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0031_community_and_workflow_tags.py b/lib/galaxy/model/migrate/versions/0031_community_and_workflow_tags.py index 1ed576e1ec9..f483919bb02 100644 --- a/lib/galaxy/model/migrate/versions/0031_community_and_workflow_tags.py +++ b/lib/galaxy/model/migrate/versions/0031_community_and_workflow_tags.py @@ -5,15 +5,11 @@ Migration script to (a) add and populate necessary columns for doing community t SQLite does not support 'ALTER TABLE ADD FOREIGN KEY', so this script will generate error messages when run against \ SQLite; however, script does execute successfully against SQLite. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, Unicode + +log = logging.getLogger( __name__ ) metadata = MetaData() StoredWorkflowTagAssociation_table = Table( "stored_workflow_tag_association", metadata, @@ -34,6 +30,7 @@ WorkflowTagAssociation_table = Table( "workflow_tag_association", metadata, Column( "value", Unicode(255), index=True), Column( "user_value", Unicode(255), index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -63,8 +60,7 @@ def upgrade(migrate_engine): # Populate column so that user_id is the id of the user who owns the history (and, up to now, was the only person able to tag the history). if c is HistoryTagAssociation_table.c.user_id: migrate_engine.execute( - "UPDATE history_tag_association SET user_id=( SELECT user_id FROM history WHERE history_tag_association.history_id = history.id )" - ) + "UPDATE history_tag_association SET user_id=( SELECT user_id FROM history WHERE history_tag_association.history_id = history.id )" ) if migrate_engine.name != 'sqlite': # Create user_id column in history_dataset_association_tag_association table. @@ -78,7 +74,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Adding user_id column to history_dataset_association_tag_association table failed: %s" % str( e ) ) else: - #In sqlite, we can no longer quietly fail to add foreign key. + # In sqlite, we can no longer quietly fail to add foreign key. # Create user_id column in history_dataset_association_tag_association table. HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_association_tag_association", metadata, autoload=True ) c = Column( "user_id", Integer) @@ -93,8 +89,7 @@ def upgrade(migrate_engine): # Populate column so that user_id is the id of the user who owns the history_dataset_association (and, up to now, was the only person able to tag the page). if c is HistoryDatasetAssociationTagAssociation_table.c.user_id: migrate_engine.execute( - "UPDATE history_dataset_association_tag_association SET user_id=( SELECT history.user_id FROM history, history_dataset_association WHERE history_dataset_association.history_id = history.id AND history_dataset_association.id = history_dataset_association_tag_association.history_dataset_association_id)" - ) + "UPDATE history_dataset_association_tag_association SET user_id=( SELECT history.user_id FROM history, history_dataset_association WHERE history_dataset_association.history_id = history.id AND history_dataset_association.id = history_dataset_association_tag_association.history_dataset_association_id)" ) if migrate_engine.name != 'sqlite': # Create user_id column in page_tag_association table. PageTagAssociation_table = Table( "page_tag_association", metadata, autoload=True ) @@ -121,8 +116,7 @@ def upgrade(migrate_engine): # Populate column so that user_id is the id of the user who owns the page (and, up to now, was the only person able to tag the page). if c is PageTagAssociation_table.c.user_id: migrate_engine.execute( - "UPDATE page_tag_association SET user_id=( SELECT user_id FROM page WHERE page_tag_association.page_id = page.id )" - ) + "UPDATE page_tag_association SET user_id=( SELECT user_id FROM page WHERE page_tag_association.page_id = page.id )" ) # Create stored_workflow_tag_association table. try: @@ -138,6 +132,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating workflow_tag_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() @@ -150,7 +145,6 @@ def downgrade(migrate_engine): print str(e) log.debug( "Dropping column user_id from history_tag_association table failed: %s" % str( e ) ) - # Drop user_id column from history_dataset_association_tag_association table. HistoryDatasetAssociationTagAssociation_table = Table( "history_dataset_association_tag_association", metadata, autoload=True ) try: diff --git a/lib/galaxy/model/migrate/versions/0032_stored_workflow_slug_column.py b/lib/galaxy/model/migrate/versions/0032_stored_workflow_slug_column.py index 4f4cac3efdf..331e61189ea 100644 --- a/lib/galaxy/model/migrate/versions/0032_stored_workflow_slug_column.py +++ b/lib/galaxy/model/migrate/versions/0032_stored_workflow_slug_column.py @@ -1,16 +1,14 @@ """ Migration script to add slug column for stored workflow. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, Index, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -28,12 +26,13 @@ def upgrade(migrate_engine): # Create slug index. if migrate_engine.name != 'sqlite': try: - i = Index( "ix_stored_workflow_slug", StoredWorkflow_table.c.slug, mysql_length = 200 ) + i = Index( "ix_stored_workflow_slug", StoredWorkflow_table.c.slug, mysql_length=200 ) i.create() except: # Mysql doesn't have a named index, but alter should work StoredWorkflow_table.c.slug.alter( unique=False ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0033_published_cols_for_histories_and_workflows.py b/lib/galaxy/model/migrate/versions/0033_published_cols_for_histories_and_workflows.py index dc580d67140..9ed0e866b69 100644 --- a/lib/galaxy/model/migrate/versions/0033_published_cols_for_histories_and_workflows.py +++ b/lib/galaxy/model/migrate/versions/0033_published_cols_for_histories_and_workflows.py @@ -2,17 +2,14 @@ Migration script to add necessary columns for distinguishing between viewing/importing and publishing histories, \ workflows, and pages. Script adds published column to histories and workflows and importable column to pages. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, Index, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -28,7 +25,6 @@ def upgrade(migrate_engine): print "Adding published column to history table failed: %s" % str( e ) log.debug( "Adding published column to history table failed: %s" % str( e ) ) - if migrate_engine.name != 'sqlite': # Create index for published column in history table. try: @@ -76,6 +72,7 @@ def upgrade(migrate_engine): # Mysql doesn't have a named index, but alter should work Page_table.c.importable.alter( unique=False ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0034_page_user_share_association.py b/lib/galaxy/model/migrate/versions/0034_page_user_share_association.py index 306a933edba..8ef1b460345 100644 --- a/lib/galaxy/model/migrate/versions/0034_page_user_share_association.py +++ b/lib/galaxy/model/migrate/versions/0034_page_user_share_association.py @@ -1,22 +1,18 @@ """ Migration script to create a table for page-user share association. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() PageUserShareAssociation_table = Table( "page_user_share_association", metadata, Column( "id", Integer, primary_key=True ), Column( "page_id", Integer, ForeignKey( "page.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) - ) + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -30,6 +26,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating page_user_share_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0035_item_annotations_and_workflow_step_tags.py b/lib/galaxy/model/migrate/versions/0035_item_annotations_and_workflow_step_tags.py index c95909c8b39..7a5b84499f2 100644 --- a/lib/galaxy/model/migrate/versions/0035_item_annotations_and_workflow_step_tags.py +++ b/lib/galaxy/model/migrate/versions/0035_item_annotations_and_workflow_step_tags.py @@ -1,15 +1,11 @@ """ Migration script to (a) create tables for annotating objects and (b) create tags for workflow steps. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table, TEXT, Unicode + +log = logging.getLogger( __name__ ) metadata = MetaData() # Annotation tables. @@ -90,10 +86,10 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating workflow_step_tag_association table failed: %s" % str( e ) ) - haaa = Index( "ix_history_anno_assoc_annotation", HistoryAnnotationAssociation_table.c.annotation, mysql_length = 200) - hdaaa = Index( "ix_history_dataset_anno_assoc_annotation", HistoryDatasetAssociationAnnotationAssociation_table.c.annotation, mysql_length = 200) - swaaa = Index( "ix_stored_workflow_ann_assoc_annotation", StoredWorkflowAnnotationAssociation_table.c.annotation, mysql_length = 200) - wsaaa = Index( "ix_workflow_step_ann_assoc_annotation", WorkflowStepAnnotationAssociation_table.c.annotation, mysql_length = 200) + haaa = Index( "ix_history_anno_assoc_annotation", HistoryAnnotationAssociation_table.c.annotation, mysql_length=200) + hdaaa = Index( "ix_history_dataset_anno_assoc_annotation", HistoryDatasetAssociationAnnotationAssociation_table.c.annotation, mysql_length=200) + swaaa = Index( "ix_stored_workflow_ann_assoc_annotation", StoredWorkflowAnnotationAssociation_table.c.annotation, mysql_length=200) + wsaaa = Index( "ix_workflow_step_ann_assoc_annotation", WorkflowStepAnnotationAssociation_table.c.annotation, mysql_length=200) try: haaa.create() @@ -104,42 +100,42 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating annotation indices failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() # Drop history_annotation_association table. try: - HistoryAnnotationAssociation_table.drop() + HistoryAnnotationAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping history_annotation_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping history_annotation_association table failed: %s" % str( e ) ) # Drop history_dataset_association_annotation_association table. try: - HistoryDatasetAssociationAnnotationAssociation_table.drop() + HistoryDatasetAssociationAnnotationAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping history_dataset_association_annotation_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping history_dataset_association_annotation_association table failed: %s" % str( e ) ) # Drop stored_workflow_annotation_association table. try: - StoredWorkflowAnnotationAssociation_table.drop() + StoredWorkflowAnnotationAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping stored_workflow_annotation_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping stored_workflow_annotation_association table failed: %s" % str( e ) ) # Drop workflow_step_annotation_association table. try: - WorkflowStepAnnotationAssociation_table.drop() + WorkflowStepAnnotationAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping workflow_step_annotation_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping workflow_step_annotation_association table failed: %s" % str( e ) ) # Drop workflow_step_tag_association table. try: - WorkflowStepTagAssociation_table.drop() + WorkflowStepTagAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping workflow_step_tag_association table failed: %s" % str( e ) ) - + print str(e) + log.debug( "Dropping workflow_step_tag_association table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/migrate/versions/0036_add_deleted_column_to_library_template_assoc_tables.py b/lib/galaxy/model/migrate/versions/0036_add_deleted_column_to_library_template_assoc_tables.py index a4aafd55ba0..2863b068695 100644 --- a/lib/galaxy/model/migrate/versions/0036_add_deleted_column_to_library_template_assoc_tables.py +++ b/lib/galaxy/model/migrate/versions/0036_add_deleted_column_to_library_template_assoc_tables.py @@ -2,22 +2,21 @@ Migration script to add a deleted column to the following tables: library_info_association, library_folder_info_association, library_dataset_dataset_info_association. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def get_false_value(migrate_engine): if migrate_engine.name == 'sqlite': return '0' else: return 'false' + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -53,6 +52,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "deleted to false in library_dataset_dataset_info_association failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0037_samples_library.py b/lib/galaxy/model/migrate/versions/0037_samples_library.py index 329837b197a..c41c0f2eb86 100644 --- a/lib/galaxy/model/migrate/versions/0037_samples_library.py +++ b/lib/galaxy/model/migrate/versions/0037_samples_library.py @@ -4,16 +4,16 @@ adds the same to the 'sample' table. This also adds a 'datatx' column to request to store the sequencer login information. Finally, this adds a 'dataset_files' column to the sample table. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +import sys +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT +from sqlalchemy.exc import NoSuchTableError + +from galaxy.model.custom_types import JSONType, TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -24,6 +24,7 @@ log.addHandler( handler ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -56,32 +57,23 @@ def upgrade(migrate_engine): if migrate_engine.name == 'sqlite': # create a temporary table RequestTemp_table = Table( 'request_temp', metadata, - Column( "id", Integer, primary_key=True), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "name", TrimmedString( 255 ), nullable=False ), - Column( "desc", TEXT ), - Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), - Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "deleted", Boolean, index=True, default=False ) ) + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "name", TrimmedString( 255 ), nullable=False ), + Column( "desc", TEXT ), + Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), + Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) try: RequestTemp_table.create() except Exception, e: log.debug( "Creating request_temp table failed: %s" % str( e ) ) # insert all the rows from the request table to the request_temp table - cmd = \ - "INSERT INTO request_temp " + \ - "SELECT id," + \ - "create_time," + \ - "update_time," + \ - "name," + \ - "desc," + \ - "form_values_id," + \ - "request_type_id," + \ - "user_id," + \ - "deleted " + \ - "FROM request;" + cmd = "INSERT INTO request_temp SELECT id, create_time, " + \ + "update_time, name, desc, form_values_id, request_type_id, " + \ + "user_id, deleted FROM request;" migrate_engine.execute( cmd ) # delete the 'request' table try: @@ -151,6 +143,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Adding column 'folder_id' to sample table failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0038_add_inheritable_column_to_library_template_assoc_tables.py b/lib/galaxy/model/migrate/versions/0038_add_inheritable_column_to_library_template_assoc_tables.py index 0c189ed970e..73caa15fd63 100644 --- a/lib/galaxy/model/migrate/versions/0038_add_inheritable_column_to_library_template_assoc_tables.py +++ b/lib/galaxy/model/migrate/versions/0038_add_inheritable_column_to_library_template_assoc_tables.py @@ -4,24 +4,25 @@ library_info_association, library_folder_info_association. Also, in case of sqlite check if the previous migration script deleted the request table and if so, restore the table. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * import datetime -now = datetime.datetime.utcnow import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def get_false_value(migrate_engine): if migrate_engine.name == 'sqlite': return '0' else: return 'false' + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -36,15 +37,15 @@ def upgrade(migrate_engine): metadata.reflect(only=['form_values', 'request_type', 'galaxy_user']) # create a temporary table Request_table = Table( 'request', metadata, - Column( "id", Integer, primary_key=True), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "name", TrimmedString( 255 ), nullable=False ), - Column( "desc", TEXT ), - Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), - Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "deleted", Boolean, index=True, default=False ) ) + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "name", TrimmedString( 255 ), nullable=False ), + Column( "desc", TEXT ), + Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), + Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), index=True ), + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) try: Request_table.create() except Exception, e: @@ -72,6 +73,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Setting value of column inheritable to false in library_folder_info_association failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0039_add_synopsis_column_to_library_table.py b/lib/galaxy/model/migrate/versions/0039_add_synopsis_column_to_library_table.py index 538fbc3e2c8..aafe6500feb 100644 --- a/lib/galaxy/model/migrate/versions/0039_add_synopsis_column_to_library_table.py +++ b/lib/galaxy/model/migrate/versions/0039_add_synopsis_column_to_library_table.py @@ -1,16 +1,14 @@ """ Migration script to add a synopsis column to the library table. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -21,6 +19,7 @@ def upgrade(migrate_engine): c.create( Library_table ) assert c is Library_table.c.synopsis + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0040_page_annotations.py b/lib/galaxy/model/migrate/versions/0040_page_annotations.py index d54b2af32f4..d4ca1803617 100644 --- a/lib/galaxy/model/migrate/versions/0040_page_annotations.py +++ b/lib/galaxy/model/migrate/versions/0040_page_annotations.py @@ -1,15 +1,11 @@ """ Migration script to (a) create tables for annotating pages. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() PageAnnotationAssociation_table = Table( "page_annotation_association", metadata, @@ -18,6 +14,7 @@ PageAnnotationAssociation_table = Table( "page_annotation_association", metadata Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "annotation", TEXT, index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -30,6 +27,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating page_annotation_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() @@ -38,6 +36,5 @@ def downgrade(migrate_engine): try: PageAnnotationAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping page_annotation_association table failed: %s" % str( e ) ) - + print str(e) + log.debug( "Dropping page_annotation_association table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/migrate/versions/0041_workflow_invocation.py b/lib/galaxy/model/migrate/versions/0041_workflow_invocation.py index 6118d9d7318..57699db3ecf 100644 --- a/lib/galaxy/model/migrate/versions/0041_workflow_invocation.py +++ b/lib/galaxy/model/migrate/versions/0041_workflow_invocation.py @@ -1,39 +1,33 @@ """ Migration script to create tables for tracking workflow invocations. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - +import datetime import logging + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table + logging.basicConfig( level=logging.DEBUG ) log = logging.getLogger( __name__ ) - -import datetime now = datetime.datetime.utcnow - metadata = MetaData() WorkflowInvocation_table = Table( "workflow_invocation", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ) - ) + Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ) ) WorkflowInvocationStep_table = Table( "workflow_invocation_step", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "workflow_invocation_id", Integer, ForeignKey( "workflow_invocation.id" ), index=True, nullable=False ), - Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ), - Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=False ) - ) + Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=False ) ) tables = [ WorkflowInvocation_table, WorkflowInvocationStep_table ] + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -45,6 +39,7 @@ def upgrade(migrate_engine): except: log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0042_workflow_invocation_fix.py b/lib/galaxy/model/migrate/versions/0042_workflow_invocation_fix.py index 50ca9969b03..a19598ef210 100644 --- a/lib/galaxy/model/migrate/versions/0042_workflow_invocation_fix.py +++ b/lib/galaxy/model/migrate/versions/0042_workflow_invocation_fix.py @@ -1,20 +1,17 @@ """ Drop and readd workflow invocation tables, allowing null jobs """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - +import datetime import logging + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table + logging.basicConfig( level=logging.DEBUG ) log = logging.getLogger( __name__ ) - -import datetime now = datetime.datetime.utcnow - metadata = MetaData() + + def upgrade(migrate_engine): print __doc__ metadata.bind = migrate_engine @@ -34,17 +31,15 @@ def upgrade(migrate_engine): Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ), - ) + Column( "workflow_id", Integer, ForeignKey( "workflow.id" ), index=True, nullable=False ) ) WorkflowInvocationStep_table = Table( "workflow_invocation_step", metadata, Column( "id", Integer, primary_key=True ), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, default=now, onupdate=now ), Column( "workflow_invocation_id", Integer, ForeignKey( "workflow_invocation.id" ), index=True, nullable=False ), - Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ), - Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=True ), - ) + Column( "workflow_step_id", Integer, ForeignKey( "workflow_step.id" ), index=True, nullable=False ), + Column( "job_id", Integer, ForeignKey( "job.id" ), index=True, nullable=True ) ) for table in [ WorkflowInvocation_table, WorkflowInvocationStep_table ]: try: @@ -52,6 +47,7 @@ def upgrade(migrate_engine): except: log.exception( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name ) + def downgrade(migrate_engine): metadata.bind = migrate_engine # No downgrade diff --git a/lib/galaxy/model/migrate/versions/0043_visualization_sharing_tagging_annotating.py b/lib/galaxy/model/migrate/versions/0043_visualization_sharing_tagging_annotating.py index ed677bf70d9..a8ebaa58b82 100644 --- a/lib/galaxy/model/migrate/versions/0043_visualization_sharing_tagging_annotating.py +++ b/lib/galaxy/model/migrate/versions/0043_visualization_sharing_tagging_annotating.py @@ -1,15 +1,11 @@ """ Migration script to create tables and columns for sharing visualizations. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, ForeignKey, Index, Integer, MetaData, Table, TEXT, Unicode + +log = logging.getLogger( __name__ ) metadata = MetaData() # Sharing visualizations. @@ -17,8 +13,7 @@ metadata = MetaData() VisualizationUserShareAssociation_table = Table( "visualization_user_share_association", metadata, Column( "id", Integer, primary_key=True ), Column( "visualization_id", Integer, ForeignKey( "visualization.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) - ) + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) # Tagging visualizations. @@ -43,7 +38,6 @@ VisualizationAnnotationAssociation_table = Table( "visualization_annotation_asso def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ - db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) metadata.reflect() Visualiation_table = Table( "visualization", metadata, autoload=True ) @@ -55,9 +49,9 @@ def upgrade(migrate_engine): log.debug( "Creating visualization_user_share_association table failed: %s" % str( e ) ) # Get default boolean value 'false' so that columns can be initialized. - if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + if migrate_engine.name in ['mysql', 'sqlite']: default_false = "0" - elif migrate_engine.name == 'postgresql': + elif migrate_engine.name in ['postgres', 'postgresql']: default_false = "false" # Add columns & create indices for supporting sharing to visualization table. @@ -68,12 +62,12 @@ def upgrade(migrate_engine): try: # Add column. - deleted_column.create( Visualiation_table, index_name = "ix_visualization_deleted") + deleted_column.create( Visualiation_table, index_name="ix_visualization_deleted") assert deleted_column is Visualiation_table.c.deleted # Fill column with default value. cmd = "UPDATE visualization SET deleted = %s" % default_false - db_session.execute( cmd ) + migrate_engine.execute( cmd ) except Exception, e: print "Adding deleted column to visualization table failed: %s" % str( e ) log.debug( "Adding deleted column to visualization table failed: %s" % str( e ) ) @@ -85,7 +79,7 @@ def upgrade(migrate_engine): # Fill column with default value. cmd = "UPDATE visualization SET importable = %s" % default_false - db_session.execute( cmd ) + migrate_engine.execute( cmd ) except Exception, e: print "Adding importable column to visualization table failed: %s" % str( e ) log.debug( "Adding importable column to visualization table failed: %s" % str( e ) ) @@ -101,7 +95,7 @@ def upgrade(migrate_engine): if migrate_engine.name == 'mysql': # Have to create index manually. cmd = "CREATE INDEX ix_visualization_slug ON visualization ( slug ( 100 ) )" - db_session.execute( cmd ) + migrate_engine.execute( cmd ) else: i = Index( "ix_visualization_slug", Visualiation_table.c.slug ) i.create() @@ -116,7 +110,7 @@ def upgrade(migrate_engine): # Fill column with default value. cmd = "UPDATE visualization SET published = %s" % default_false - db_session.execute( cmd ) + migrate_engine.execute( cmd ) except Exception, e: print "Adding published column to visualization table failed: %s" % str( e ) log.debug( "Adding published column to visualization table failed: %s" % str( e ) ) @@ -137,16 +131,17 @@ def upgrade(migrate_engine): # Need to create index for visualization annotation manually to deal with errors. try: - if migrate_engine.name == 'mysql': - # Have to create index manually. - cmd = "CREATE INDEX ix_visualization_annotation_association_annotation ON visualization_annotation_association ( annotation ( 100 ) )" - db_session.execute( cmd ) - else: - i = Index( "ix_visualization_annotation_association_annotation", VisualizationAnnotationAssociation_table.c.annotation ) - i.create() + if migrate_engine.name == 'mysql': + # Have to create index manually. + cmd = "CREATE INDEX ix_visualization_annotation_association_annotation ON visualization_annotation_association ( annotation ( 100 ) )" + migrate_engine.execute( cmd ) + else: + i = Index( "ix_visualization_annotation_association_annotation", VisualizationAnnotationAssociation_table.c.annotation ) + i.create() except Exception, e: - print "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e ) - log.debug( "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e ) ) + print "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e ) + log.debug( "Adding index 'ix_visualization_annotation_association_annotation' failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine diff --git a/lib/galaxy/model/migrate/versions/0044_add_notify_column_to_request_table.py b/lib/galaxy/model/migrate/versions/0044_add_notify_column_to_request_table.py index 26e8a086874..e887f69666f 100644 --- a/lib/galaxy/model/migrate/versions/0044_add_notify_column_to_request_table.py +++ b/lib/galaxy/model/migrate/versions/0044_add_notify_column_to_request_table.py @@ -1,16 +1,14 @@ """ Migration script to add a notify column to the request table. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -21,6 +19,7 @@ def upgrade(migrate_engine): c.create( Request_table ) assert c is Request_table.c.notify + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0045_request_type_permissions_table.py b/lib/galaxy/model/migrate/versions/0045_request_type_permissions_table.py index 773b02cf988..267f0efb85d 100644 --- a/lib/galaxy/model/migrate/versions/0045_request_type_permissions_table.py +++ b/lib/galaxy/model/migrate/versions/0045_request_type_permissions_table.py @@ -1,17 +1,13 @@ """ Migration script to add the request_type_permissions table. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() RequestTypePermissions_table = Table( "request_type_permissions", metadata, @@ -22,6 +18,7 @@ RequestTypePermissions_table = Table( "request_type_permissions", metadata, Column( "request_type_id", Integer, ForeignKey( "request_type.id" ), nullable=True, index=True ), Column( "role_id", Integer, ForeignKey( "role.id" ), index=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -31,6 +28,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating request_type_permissions table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0046_post_job_actions.py b/lib/galaxy/model/migrate/versions/0046_post_job_actions.py index 20e8eeee480..e9d1c170f40 100644 --- a/lib/galaxy/model/migrate/versions/0046_post_job_actions.py +++ b/lib/galaxy/model/migrate/versions/0046_post_job_actions.py @@ -1,22 +1,17 @@ """ Migration script to create tables for handling post-job actions. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - +import datetime import logging -logging.basicConfig( level=logging.DEBUG ) -log = logging.getLogger( __name__ ) + +from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table # Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from galaxy.model.custom_types import JSONType -import datetime +logging.basicConfig( level=logging.DEBUG ) +log = logging.getLogger( __name__ ) now = datetime.datetime.utcnow - metadata = MetaData() PostJobAction_table = Table("post_job_action", metadata, @@ -31,7 +26,8 @@ PostJobAction_table = Table("post_job_action", metadata, # Column("post_job_action_id", Integer, ForeignKey("post_job_action.id"), index=True, nullable=False), # Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False)) -tables = [PostJobAction_table]#, PostJobActionAssociation_table] +tables = [PostJobAction_table] # , PostJobActionAssociation_table] + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -43,6 +39,7 @@ def upgrade(migrate_engine): except: log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0047_job_table_user_id_column.py b/lib/galaxy/model/migrate/versions/0047_job_table_user_id_column.py index 35fb6a8c018..9cf9cc7022b 100644 --- a/lib/galaxy/model/migrate/versions/0047_job_table_user_id_column.py +++ b/lib/galaxy/model/migrate/versions/0047_job_table_user_id_column.py @@ -1,18 +1,14 @@ """ Add a user_id column to the job table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow +import logging +import sys -import sys, logging +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,6 +19,7 @@ log.addHandler( handler ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -68,6 +65,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Updating job.user_id column failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0048_dataset_instance_state_column.py b/lib/galaxy/model/migrate/versions/0048_dataset_instance_state_column.py index b6c858737ac..8136a6c0065 100644 --- a/lib/galaxy/model/migrate/versions/0048_dataset_instance_state_column.py +++ b/lib/galaxy/model/migrate/versions/0048_dataset_instance_state_column.py @@ -1,18 +1,16 @@ """ Add a state column to the history_dataset_association and library_dataset_dataset_association table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow +import logging +import sys -import sys, logging +from sqlalchemy import Column, MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -25,6 +23,7 @@ metadata = MetaData() DATASET_INSTANCE_TABLE_NAMES = [ 'history_dataset_association', 'library_dataset_dataset_association' ] + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -40,11 +39,12 @@ def upgrade(migrate_engine): index_name = "ix_%s_state" % table_name try: col = Column( "state", TrimmedString( 64 ), index=True, nullable=True ) - col.create( dataset_instance_table, index_name = index_name) + col.create( dataset_instance_table, index_name=index_name) assert col is dataset_instance_table.c.state except Exception, e: log.debug( "Adding column 'state' to %s table failed: %s" % ( table_name, str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0049_api_keys_table.py b/lib/galaxy/model/migrate/versions/0049_api_keys_table.py index 4fcb1f5d289..66c956a5a73 100644 --- a/lib/galaxy/model/migrate/versions/0049_api_keys_table.py +++ b/lib/galaxy/model/migrate/versions/0049_api_keys_table.py @@ -1,18 +1,15 @@ """ Migration script to add the api_keys table. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() APIKeys_table = Table( "api_keys", metadata, @@ -21,6 +18,7 @@ APIKeys_table = Table( "api_keys", metadata, Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "key", TrimmedString( 32 ), index=True, unique=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -30,6 +28,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating api_keys table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine # Load existing tables diff --git a/lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py b/lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py index df9264a0af6..a825555ca7d 100644 --- a/lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py +++ b/lib/galaxy/model/migrate/versions/0050_drop_cloud_tables.py @@ -1,17 +1,13 @@ -from sqlalchemy import * -from migrate import * - import datetime -now = datetime.datetime.utcnow - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def display_migration_details(): print print "========================================" @@ -127,6 +123,7 @@ CloudProvider_table = Table( "cloud_provider", metadata, Column( "path", TEXT ), Column( "deleted", Boolean, default=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine display_migration_details() diff --git a/lib/galaxy/model/migrate/versions/0051_imported_col_for_jobs_table.py b/lib/galaxy/model/migrate/versions/0051_imported_col_for_jobs_table.py index f9654cf63d4..fc40791b081 100644 --- a/lib/galaxy/model/migrate/versions/0051_imported_col_for_jobs_table.py +++ b/lib/galaxy/model/migrate/versions/0051_imported_col_for_jobs_table.py @@ -1,17 +1,14 @@ """ Migration script to add imported column for jobs table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -26,7 +23,7 @@ def upgrade(migrate_engine): assert c is Jobs_table.c.imported # Initialize. - if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + if migrate_engine.name in ['mysql', 'sqlite']: default_false = "0" elif migrate_engine.name in ['postgres', 'postgresql']: default_false = "false" @@ -36,6 +33,7 @@ def upgrade(migrate_engine): print "Adding imported column to job table failed: %s" % str( e ) log.debug( "Adding imported column to job table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py b/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py index 33a18b1520a..098ac204bb8 100644 --- a/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py +++ b/lib/galaxy/model/migrate/versions/0052_sample_dataset_table.py @@ -2,40 +2,36 @@ Migration script to add the sample_dataset table and remove the 'dataset_files' column from the 'sample' table """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * - -from galaxy.model.custom_types import * -from galaxy.util.json import loads, dumps - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from json import loads +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT +from sqlalchemy.exc import NoSuchTableError + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() -def nextval( table, col='id' ): - if migrate_engine.name == 'postgres': +def nextval( migrate_engine, table, col='id' ): + if migrate_engine.name in ['postgres', 'postgresql']: return "nextval('%s_%s_seq')" % ( table, col ) - elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + elif migrate_engine.name in ['mysql', 'sqlite']: return "null" else: raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) -def localtimestamp(): - if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': - return "LOCALTIMESTAMP" - elif migrate_engine.name == 'sqlite': - return "current_date || ' ' || current_time" - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % db ) + +def localtimestamp(migrate_engine): + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) SampleDataset_table = Table('sample_dataset', metadata, Column( "id", Integer, primary_key=True ), @@ -48,6 +44,7 @@ SampleDataset_table = Table('sample_dataset', metadata, Column( "error_msg", TEXT ), Column( "size", TrimmedString( 255 ) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -66,9 +63,9 @@ def upgrade(migrate_engine): for df in dataset_files: if type(df) == type(dict()): cmd = "INSERT INTO sample_dataset VALUES (%s, %s, %s, %s, '%s', '%s', '%s', '%s', '%s')" - cmd = cmd % ( nextval('sample_dataset'), - localtimestamp(), - localtimestamp(), + cmd = cmd % ( nextval(migrate_engine, 'sample_dataset'), + localtimestamp(migrate_engine), + localtimestamp(migrate_engine), str(sample_id), df.get('name', ''), df.get('filepath', ''), diff --git a/lib/galaxy/model/migrate/versions/0053_item_ratings.py b/lib/galaxy/model/migrate/versions/0053_item_ratings.py index 4f379ffef97..83b2df0d9b4 100644 --- a/lib/galaxy/model/migrate/versions/0053_item_ratings.py +++ b/lib/galaxy/model/migrate/versions/0053_item_ratings.py @@ -1,15 +1,11 @@ """ Migration script to create tables for rating histories, datasets, workflows, pages, and visualizations. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() # Rating tables. @@ -44,6 +40,7 @@ VisualizationRatingAssociation_table = Table( "visualization_rating_association" Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "rating", Integer, index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -62,7 +59,7 @@ def upgrade(migrate_engine): except Exception, e: # MySQL cannot handle long index names; when we see this error, create the index name manually. if migrate_engine.name == 'mysql' and \ - str(e).lower().find("identifier name 'ix_history_dataset_association_rating_association_history_dataset_association_id' is too long"): + str(e).lower().find("identifier name 'ix_history_dataset_association_rating_association_history_dataset_association_id' is too long"): i = Index( "ix_hda_rating_association_hda_id", HistoryDatasetAssociationRatingAssociation_table.c.history_dataset_association_id ) try: i.create() @@ -94,41 +91,42 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating visualization_rating_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() # Drop history_rating_association table. try: - HistoryRatingAssociation_table.drop() + HistoryRatingAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping history_rating_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping history_rating_association table failed: %s" % str( e ) ) # Drop history_dataset_association_rating_association table. try: - HistoryDatasetAssociationRatingAssociation_table.drop() + HistoryDatasetAssociationRatingAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping history_dataset_association_rating_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping history_dataset_association_rating_association table failed: %s" % str( e ) ) # Drop stored_workflow_rating_association table. try: - StoredWorkflowRatingAssociation_table.drop() + StoredWorkflowRatingAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping stored_workflow_rating_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping stored_workflow_rating_association table failed: %s" % str( e ) ) # Drop page_rating_association table. try: - PageRatingAssociation_table.drop() + PageRatingAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping page_rating_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping page_rating_association table failed: %s" % str( e ) ) # Drop visualization_rating_association table. try: - VisualizationRatingAssociation_table.drop() + VisualizationRatingAssociation_table.drop() except Exception, e: - print str(e) - log.debug( "Dropping visualization_rating_association table failed: %s" % str( e ) ) + print str(e) + log.debug( "Dropping visualization_rating_association table failed: %s" % str( e ) ) diff --git a/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py b/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py index 9cac2d61df5..a60768336f5 100644 --- a/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py +++ b/lib/galaxy/model/migrate/versions/0054_visualization_dbkey.py @@ -1,18 +1,15 @@ """ Migration script to add dbkey column for visualization. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.util.json import loads - import logging -log = logging.getLogger( __name__ ) +from json import loads +from sqlalchemy import Column, Index, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -28,9 +25,9 @@ def upgrade(migrate_engine): x.create( Visualization_table ) y.create( Visualization_revision_table ) # Manually create indexes for compatability w/ mysql_length. - xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length = 200) + xi = Index( "ix_visualization_dbkey", Visualization_table.c.dbkey, mysql_length=200) xi.create() - yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length = 200) + yi = Index( "ix_visualization_revision_dbkey", Visualization_revision_table.c.dbkey, mysql_length=200) yi.create() assert x is Visualization_table.c.dbkey assert y is Visualization_revision_table.c.dbkey @@ -45,6 +42,7 @@ def upgrade(migrate_engine): migrate_engine.execute("UPDATE visualization_revision SET dbkey='%s' WHERE id=%s" % (dbkey, viz_rev_id)) migrate_engine.execute("UPDATE visualization SET dbkey='%s' WHERE id=%s" % (dbkey, viz_id)) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0055_add_pja_assoc_for_jobs.py b/lib/galaxy/model/migrate/versions/0055_add_pja_assoc_for_jobs.py index ce49bf86fa0..777d913780a 100644 --- a/lib/galaxy/model/migrate/versions/0055_add_pja_assoc_for_jobs.py +++ b/lib/galaxy/model/migrate/versions/0055_add_pja_assoc_for_jobs.py @@ -1,18 +1,13 @@ """ Migration script to add the post_job_action_association table. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() PostJobActionAssociation_table = Table("post_job_action_association", metadata, @@ -20,6 +15,7 @@ PostJobActionAssociation_table = Table("post_job_action_association", metadata, Column("post_job_action_id", Integer, ForeignKey("post_job_action.id"), index=True, nullable=False), Column("job_id", Integer, ForeignKey("job.id"), index=True, nullable=False)) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -29,6 +25,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating PostJobActionAssociation table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine # Load existing tables diff --git a/lib/galaxy/model/migrate/versions/0056_workflow_outputs.py b/lib/galaxy/model/migrate/versions/0056_workflow_outputs.py index a495c23d7a9..2e52a773295 100644 --- a/lib/galaxy/model/migrate/versions/0056_workflow_outputs.py +++ b/lib/galaxy/model/migrate/versions/0056_workflow_outputs.py @@ -1,19 +1,13 @@ """ Migration script to create tables for adding explicit workflow outputs. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging + +from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table + logging.basicConfig( level=logging.DEBUG ) log = logging.getLogger( __name__ ) -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - metadata = MetaData() WorkflowOutput_table = Table( "workflow_output", metadata, @@ -23,6 +17,7 @@ WorkflowOutput_table = Table( "workflow_output", metadata, tables = [WorkflowOutput_table] + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -33,6 +28,7 @@ def upgrade(migrate_engine): except: log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0057_request_notify.py b/lib/galaxy/model/migrate/versions/0057_request_notify.py index e763f524d96..2ee057e3e86 100644 --- a/lib/galaxy/model/migrate/versions/0057_request_notify.py +++ b/lib/galaxy/model/migrate/versions/0057_request_notify.py @@ -2,24 +2,20 @@ Migration script to modify the 'notify' field in the 'request' table from a boolean to a JSONType """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * - -from galaxy.model.custom_types import * -from galaxy.util.json import loads, dumps - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from json import dumps +from sqlalchemy import Column, MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +from galaxy.model.custom_types import JSONType + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -30,7 +26,6 @@ def upgrade(migrate_engine): Request_table = None log.debug( "Failed loading table 'request'" ) - if Request_table is not None: # create the column again as JSONType try: @@ -44,16 +39,10 @@ def upgrade(migrate_engine): result = migrate_engine.execute( cmd ) for r in result: id = int(r[0]) - notify_old = r[1] notify_new = dict(email=[], sample_states=[], body='', subject='') - cmd = "update request set notification='%s' where id=%i" % (dumps(notify_new), id) + cmd = "UPDATE request SET notification='%s' WHERE id=%i" % (dumps(notify_new), id) migrate_engine.execute( cmd ) - cmd = "SELECT id, notification FROM request" - result = migrate_engine.execute( cmd ) - for r in result: - rr = loads(str(r[1])) - # remove the 'notify' column for non-sqlite databases. if migrate_engine.name != 'sqlite': try: @@ -62,7 +51,6 @@ def upgrade(migrate_engine): log.debug( "Deleting column 'notify' from the 'request' table failed: %s" % ( str( e ) ) ) - def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0058_history_import_export.py b/lib/galaxy/model/migrate/versions/0058_history_import_export.py index e430eb65f13..070bfc7474a 100644 --- a/lib/galaxy/model/migrate/versions/0058_history_import_export.py +++ b/lib/galaxy/model/migrate/versions/0058_history_import_export.py @@ -1,15 +1,11 @@ """ Migration script to create table for exporting histories to archives. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, ForeignKey, Integer, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() # Table to add. @@ -22,8 +18,8 @@ JobExportHistoryArchive_table = Table( "job_export_history_archive", metadata, Column( "compressed", Boolean, index=True, default=False ), Column( "history_attrs_filename", TEXT ), Column( "datasets_attrs_filename", TEXT ), - Column( "jobs_attrs_filename", TEXT ) - ) + Column( "jobs_attrs_filename", TEXT ) ) + def upgrade(migrate_engine): print __doc__ @@ -37,6 +33,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Creating job_export_history_archive table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py b/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py index 53ee2ef8840..aa78fa5c71f 100644 --- a/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py +++ b/lib/galaxy/model/migrate/versions/0059_sample_dataset_file_path.py @@ -2,24 +2,17 @@ Migration script to modify the 'file_path' field type in 'sample_dataset' table to 'TEXT' so that it can support large file paths exceeding 255 characters """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * - -from galaxy.model.custom_types import * -from galaxy.util.json import loads, dumps - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, MetaData, Table, TEXT +from sqlalchemy.exc import NoSuchTableError + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -54,6 +47,7 @@ def upgrade(migrate_engine): cmd = "update sample_dataset set file_path='%s' where id=%i" % (file_path, id) migrate_engine.execute( cmd ) + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0060_history_archive_import.py b/lib/galaxy/model/migrate/versions/0060_history_archive_import.py index c9576429646..6d6c17190af 100644 --- a/lib/galaxy/model/migrate/versions/0060_history_archive_import.py +++ b/lib/galaxy/model/migrate/versions/0060_history_archive_import.py @@ -2,15 +2,11 @@ Migration script to create column and table for importing histories from file archives. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, ForeignKey, Integer, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() # Columns to add. @@ -24,8 +20,8 @@ JobImportHistoryArchive_table = Table( "job_import_history_archive", metadata, Column( "id", Integer, primary_key=True ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), - Column( "archive_dir", TEXT ) - ) + Column( "archive_dir", TEXT ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -39,7 +35,7 @@ def upgrade(migrate_engine): assert importing_col is History_table.c.importing # Initialize column to false. - if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + if migrate_engine.name in ['mysql', 'sqlite']: default_false = "0" elif migrate_engine.name in ['postgres', 'postgresql']: default_false = "false" @@ -54,6 +50,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating job_import_history_archive table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0061_tasks.py b/lib/galaxy/model/migrate/versions/0061_tasks.py index 549c84fa979..cf834eedb85 100644 --- a/lib/galaxy/model/migrate/versions/0061_tasks.py +++ b/lib/galaxy/model/migrate/versions/0061_tasks.py @@ -1,16 +1,12 @@ """ Migration script to create tables task management. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() now = datetime.datetime.utcnow @@ -33,6 +29,7 @@ Task_table = Table( "task", metadata, tables = [Task_table] + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -43,6 +40,7 @@ def upgrade(migrate_engine): except: log.warn( "Failed to create table '%s', ignoring (might result in wrong schema)" % table.name ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0062_user_openid_table.py b/lib/galaxy/model/migrate/versions/0062_user_openid_table.py index fdc58e4736c..1430645880c 100644 --- a/lib/galaxy/model/migrate/versions/0062_user_openid_table.py +++ b/lib/galaxy/model/migrate/versions/0062_user_openid_table.py @@ -2,18 +2,13 @@ Migration script to create table for associating sessions and users with OpenIDs. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, MetaData, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() # Table to add @@ -24,8 +19,8 @@ UserOpenID_table = Table( "galaxy_user_openid", metadata, Column( "update_time", DateTime, index=True, default=now, onupdate=now ), Column( "session_id", Integer, ForeignKey( "galaxy_session.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "openid", TEXT ), - ) + Column( "openid", TEXT ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -49,6 +44,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Adding index '%s' failed: %s" % ( ix_name, str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0063_sequencer_table.py b/lib/galaxy/model/migrate/versions/0063_sequencer_table.py index 3270649160d..85ea0acebb5 100644 --- a/lib/galaxy/model/migrate/versions/0063_sequencer_table.py +++ b/lib/galaxy/model/migrate/versions/0063_sequencer_table.py @@ -1,21 +1,16 @@ """ Migration script to create a new 'sequencer' table """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * - -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT +from sqlalchemy.exc import NoSuchTableError + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() # Table to add @@ -31,6 +26,7 @@ Sequencer_table = Table( 'sequencer', metadata, Column( "form_values_id", Integer, ForeignKey( "form_values.id" ), index=True ), Column( "deleted", Boolean, index=True, default=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -56,4 +52,3 @@ def downgrade(migrate_engine): Sequencer_table.drop() except Exception, e: log.debug( "Deleting 'sequencer' table failed: %s" % str( e ) ) - diff --git a/lib/galaxy/model/migrate/versions/0064_add_run_and_sample_run_association_tables.py b/lib/galaxy/model/migrate/versions/0064_add_run_and_sample_run_association_tables.py index 41fd3e92cbc..efdbe4306c9 100644 --- a/lib/galaxy/model/migrate/versions/0064_add_run_and_sample_run_association_tables.py +++ b/lib/galaxy/model/migrate/versions/0064_add_run_and_sample_run_association_tables.py @@ -1,18 +1,13 @@ """ Migration script to add the run and sample_run_association tables. """ - -from sqlalchemy import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() Run_table = Table( "run", metadata, @@ -33,6 +28,7 @@ SampleRunAssociation_table = Table( "sample_run_association", metadata, Column( "sample_id", Integer, ForeignKey( "sample.id" ), index=True, nullable=False ), Column( "run_id", Integer, ForeignKey( "run.id" ), index=True, nullable=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -50,6 +46,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating SampleRunAssociation table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine # Load existing tables diff --git a/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py b/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py index 2776d859fd1..78de6e579d2 100644 --- a/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py +++ b/lib/galaxy/model/migrate/versions/0065_add_name_to_form_fields_and_values.py @@ -3,19 +3,16 @@ Migration script to add 'name' attribute to the JSON dict which describes a form definition field and the form values in the database. In the 'form_values' table, the 'content' column is now a JSON dict instead of a list. """ +import datetime +import logging +import sys +from json import loads, dumps + +from sqlalchemy import MetaData, Table -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * -from galaxy.util.json import loads, dumps from galaxy.model.custom_types import _sniffnfix_pg9_hex -import datetime now = datetime.datetime.utcnow - -import sys, logging log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,7 +20,6 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - metadata = MetaData() @@ -32,17 +28,18 @@ def upgrade(migrate_engine): print __doc__ metadata.reflect() try: - FormDefinition_table = Table( "form_definition", metadata, autoload=True ) + Table( "form_definition", metadata, autoload=True ) except Exception, e: log.debug( "Loading 'form_definition' table failed: %s" % str( e ) ) try: - FormValues_table = Table( "form_values", metadata, autoload=True ) + Table( "form_values", metadata, autoload=True ) except Exception, e: log.debug( "Loading 'form_values' table failed: %s" % str( e ) ) + def get_value(lst, index): try: return str(lst[index]).replace("'", "''") - except IndexError,e: + except IndexError: return '' # Go through the entire table and add a 'name' attribute for each field # in the list of fields for each form definition @@ -61,9 +58,9 @@ def upgrade(migrate_engine): field[ 'label' ] = field[ 'label' ].replace("'", "''") fields_json = dumps( fields_list ) if migrate_engine.name == 'mysql': - cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( fields_json, form_definition_id ) + cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" % ( fields_json, form_definition_id ) else: - cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( fields_json, form_definition_id ) + cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" % ( fields_json, form_definition_id ) migrate_engine.execute( cmd ) # replace the values list in the content field of the form_values table with a name:value dict cmd = "SELECT form_values.id, form_values.content, form_definition.fields" \ @@ -85,18 +82,19 @@ def upgrade(migrate_engine): for field_index, field in enumerate( fields_list ): field_name = field[ 'name' ] values_dict[ field_name ] = get_value(values_list, field_index ) - cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( dumps( values_dict ), form_values_id ) + cmd = "UPDATE form_values SET content='%s' WHERE id=%i" % ( dumps( values_dict ), form_values_id ) migrate_engine.execute( cmd ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() try: - FormDefinition_table = Table( "form_definition", metadata, autoload=True ) + Table( "form_definition", metadata, autoload=True ) except Exception, e: log.debug( "Loading 'form_definition' table failed: %s" % str( e ) ) try: - FormValues_table = Table( "form_values", metadata, autoload=True ) + Table( "form_values", metadata, autoload=True ) except Exception, e: log.debug( "Loading 'form_values' table failed: %s" % str( e ) ) # remove the name attribute in the content column JSON dict in the form_values table @@ -120,7 +118,7 @@ def downgrade(migrate_engine): field_name = field[ 'name' ] field_value = values_dict[ field_name ] values_list.append( field_value ) - cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( dumps( values_list ), form_values_id ) + cmd = "UPDATE form_values SET content='%s' WHERE id=%i" % ( dumps( values_list ), form_values_id ) migrate_engine.execute( cmd ) # remove name attribute from the field column of the form_definition table cmd = "SELECT f.id, f.fields FROM form_definition AS f" @@ -133,10 +131,10 @@ def downgrade(migrate_engine): fields_list = loads( _sniffnfix_pg9_hex( fields ) ) if len( fields_list ): for index, field in enumerate( fields_list ): - if field.has_key( 'name' ): + if 'name' in field: del field[ 'name' ] if migrate_engine.name == 'mysql': - cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" %( dumps( fields_list ), form_definition_id ) + cmd = "UPDATE form_definition AS f SET f.fields='%s' WHERE f.id=%i" % ( dumps( fields_list ), form_definition_id ) else: - cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" %( dumps( fields_list ), form_definition_id ) + cmd = "UPDATE form_definition SET fields='%s' WHERE id=%i" % ( dumps( fields_list ), form_definition_id ) migrate_engine.execute( cmd ) diff --git a/lib/galaxy/model/migrate/versions/0066_deferred_job_and_transfer_job_tables.py b/lib/galaxy/model/migrate/versions/0066_deferred_job_and_transfer_job_tables.py index d6fe7eb1dec..d731aec8ecc 100644 --- a/lib/galaxy/model/migrate/versions/0066_deferred_job_and_transfer_job_tables.py +++ b/lib/galaxy/model/migrate/versions/0066_deferred_job_and_transfer_job_tables.py @@ -2,20 +2,15 @@ Migration script to create table for storing deferred job and managed transfer information. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table + +from galaxy.model.custom_types import JSONType + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() # Table to add @@ -36,6 +31,7 @@ TransferJob_table = Table( "transfer_job", metadata, Column( "path", String( 1024 ) ), Column( "params", JSONType ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -53,6 +49,7 @@ def upgrade(migrate_engine): except Exception, e: log.error( "Creating transfer_job table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py index 3c790330a24..5083dc80e81 100644 --- a/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py +++ b/lib/galaxy/model/migrate/versions/0067_populate_sequencer_table.py @@ -4,42 +4,39 @@ entries in the 'datatx_info' column in the 'request_type' table. It also deletes column in the 'request_type' table and adds a foreign key to the 'sequencer' table. The actual contents of the datatx_info column are stored as form_values. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * - -from galaxy.model.custom_types import * - -from galaxy.util.json import loads, dumps - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from json import dumps, loads +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +from galaxy.model.custom_types import JSONType + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() -def nextval( table, col='id' ): - if migrate_engine.name == 'postgres': + +def nextval( migrate_engine, table, col='id' ): + if migrate_engine.name in ['postgres', 'postgresql']: return "nextval('%s_%s_seq')" % ( table, col ) - elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + elif migrate_engine.name in ['mysql', 'sqlite']: return "null" else: raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) -def localtimestamp(): - if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': - return "LOCALTIMESTAMP" - elif migrate_engine.name == 'sqlite': - return "current_date || ' ' || current_time" - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % db ) -def get_latest_id( table ): +def localtimestamp( migrate_engine ): + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) + + +def get_latest_id( migrate_engine, table ): result = migrate_engine.execute( "select id from %s order by id desc" % table ) row = result.fetchone() if row: @@ -47,17 +44,19 @@ def get_latest_id( table ): else: raise Exception( 'Unable to get the latest id in the %s table.' % table ) -def boolean( value ): - if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': - return value - elif migrate_engine.name == 'sqlite': - if value in [ 'True, true' ]: - return 1 - return 0 - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % db ) -def create_sequencer_form_definition(): +def boolean( migrate_engine, value ): + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return value + elif migrate_engine.name == 'sqlite': + if value in [ 'True', 'true' ]: + return 1 + return 0 + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) + + +def create_sequencer_form_definition( migrate_engine ): ''' Create a new form_definition containing 5 fields (host, username, password, data_dir & rename_datasets) which described the existing datatx_info json @@ -65,14 +64,14 @@ def create_sequencer_form_definition(): ''' # create new form_definition_current in the db cmd = "INSERT INTO form_definition_current VALUES ( %s, %s, %s, %s, %s )" - cmd = cmd % ( nextval( 'form_definition_current' ), - localtimestamp(), - localtimestamp(), + cmd = cmd % ( nextval( migrate_engine, 'form_definition_current' ), + localtimestamp( migrate_engine ), + localtimestamp( migrate_engine ), 'NULL', - boolean( 'false' ) ) + boolean( migrate_engine, 'false' ) ) migrate_engine.execute( cmd ) # get this form_definition_current id - form_definition_current_id = get_latest_id( 'form_definition_current' ) + form_definition_current_id = get_latest_id( migrate_engine, 'form_definition_current' ) # create new form_definition in the db form_definition_name = 'Generic sequencer form' form_definition_desc = '' @@ -106,9 +105,9 @@ def create_sequencer_form_definition(): form_definition_type = 'Sequencer Information Form' form_definition_layout = dumps('[]') cmd = "INSERT INTO form_definition VALUES ( %s, %s, %s, '%s', '%s', %s, '%s', '%s', '%s' )" - cmd = cmd % ( nextval( 'form_definition' ), - localtimestamp(), - localtimestamp(), + cmd = cmd % ( nextval( migrate_engine, 'form_definition' ), + localtimestamp( migrate_engine ), + localtimestamp( migrate_engine ), form_definition_name, form_definition_desc, form_definition_current_id, @@ -117,13 +116,14 @@ def create_sequencer_form_definition(): form_definition_layout ) migrate_engine.execute( cmd ) # get this form_definition id - form_definition_id = get_latest_id( 'form_definition' ) + form_definition_id = get_latest_id( migrate_engine, 'form_definition' ) # update the form_definition_id column in form_definition_current - cmd = "UPDATE form_definition_current SET latest_form_id=%i WHERE id=%i" %( form_definition_id, form_definition_current_id ) + cmd = "UPDATE form_definition_current SET latest_form_id=%i WHERE id=%i" % ( form_definition_id, form_definition_current_id ) migrate_engine.execute( cmd ) return form_definition_id -def get_sequencer_id( sequencer_info ): + +def get_sequencer_id( migrate_engine, sequencer_info ): '''Get the sequencer id corresponding to the sequencer information''' # Check if there is any existing sequencer which have the same sequencer # information fields & values @@ -145,46 +145,48 @@ def get_sequencer_id( sequencer_info ): return sequencer_id return None -def add_sequencer( sequencer_index, sequencer_form_definition_id, sequencer_info ): + +def add_sequencer( migrate_engine, sequencer_index, sequencer_form_definition_id, sequencer_info ): '''Adds a new sequencer to the sequencer table along with its form values.''' # Create a new form values record with the supplied sequencer information values = dumps( { 'field_0': sequencer_info.get( 'host', '' ), - 'field_1': sequencer_info.get( 'username', '' ), - 'field_2': sequencer_info.get( 'password', '' ), - 'field_3': sequencer_info.get( 'data_dir', '' ), - 'field_4': sequencer_info.get( 'rename_dataset', '' ) } ) - cmd = "INSERT INTO form_values VALUES ( %s, %s, %s, %s, '%s' )" % ( nextval( 'form_values' ), - localtimestamp(), - localtimestamp(), + 'field_1': sequencer_info.get( 'username', '' ), + 'field_2': sequencer_info.get( 'password', '' ), + 'field_3': sequencer_info.get( 'data_dir', '' ), + 'field_4': sequencer_info.get( 'rename_dataset', '' ) } ) + cmd = "INSERT INTO form_values VALUES ( %s, %s, %s, %s, '%s' )" % ( nextval( migrate_engine, 'form_values' ), + localtimestamp( migrate_engine ), + localtimestamp( migrate_engine ), sequencer_form_definition_id, values ) migrate_engine.execute(cmd) - sequencer_form_values_id = get_latest_id( 'form_values' ) + sequencer_form_values_id = get_latest_id( migrate_engine, 'form_values' ) # Create a new sequencer record with reference to the form value created above. name = 'Sequencer_%i' % sequencer_index desc = '' version = '' - result_datasets = dict() sequencer_type_id = 'simple_unknown_sequencer' cmd = "INSERT INTO sequencer VALUES ( %s, %s, %s, '%s', '%s', '%s', '%s', %s, %s, %s )" - cmd = cmd % ( nextval('sequencer'), - localtimestamp(), - localtimestamp(), + cmd = cmd % ( nextval( migrate_engine, 'sequencer'), + localtimestamp( migrate_engine ), + localtimestamp( migrate_engine ), name, desc, sequencer_type_id, version, sequencer_form_definition_id, sequencer_form_values_id, - boolean( 'false' ) ) + boolean( migrate_engine, 'false' ) ) migrate_engine.execute(cmd) - return get_latest_id( 'sequencer' ) + return get_latest_id( migrate_engine, 'sequencer' ) -def update_sequencer_id_in_request_type( request_type_id, sequencer_id ): + +def update_sequencer_id_in_request_type( migrate_engine, request_type_id, sequencer_id ): '''Update the foreign key to the sequencer table in the request_type table''' - cmd = "UPDATE request_type SET sequencer_id=%i WHERE id=%i" %( sequencer_id, request_type_id ) + cmd = "UPDATE request_type SET sequencer_id=%i WHERE id=%i" % ( sequencer_id, request_type_id ) migrate_engine.execute( cmd ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -224,12 +226,11 @@ def upgrade(migrate_engine): # For the existing request_types in the database, we add a new form_definition # with these 4 fields. Then we populate the sequencer table with unique datatx_info # column from the existing request_types. - sequencer_form_definition_id = create_sequencer_form_definition() + sequencer_form_definition_id = create_sequencer_form_definition( migrate_engine ) sequencer_index = 1 for row in results_list: request_type_id = row[0] - request_type_name = row[1] - sequencer_info = str( row[2] ) # datatx_info column + sequencer_info = str( row[2] ) # datatx_info column # skip if sequencer_info is empty if not sequencer_info.strip() or sequencer_info in ['None', 'null']: continue @@ -237,12 +238,12 @@ def upgrade(migrate_engine): # proceed only if sequencer_info is a valid dict if sequencer_info and type( sequencer_info ) == type( dict() ): # check if this sequencer has already been added to the sequencer table - sequencer_id = get_sequencer_id( sequencer_info ) + sequencer_id = get_sequencer_id( migrate_engine, sequencer_info ) if not sequencer_id: # add to the sequencer table - sequencer_id = add_sequencer( sequencer_index, sequencer_form_definition_id, sequencer_info ) + sequencer_id = add_sequencer( migrate_engine, sequencer_index, sequencer_form_definition_id, sequencer_info ) # now update the sequencer_id column in request_type table - update_sequencer_id_in_request_type( request_type_id, sequencer_id ) + update_sequencer_id_in_request_type( migrate_engine, request_type_id, sequencer_id ) sequencer_index = sequencer_index + 1 # Finally delete the 'datatx_info' column from the request_type table @@ -279,34 +280,16 @@ def downgrade(migrate_engine): request_type_id = row[0] seq_values = loads( str( row[1] ) ) # create the datatx_info json dict - datatx_info = dumps( dict( host = seq_values.get( 'field_0', '' ), - username = seq_values.get( 'field_1', '' ), - password = seq_values.get( 'field_2', '' ), - data_dir = seq_values.get( 'field_3', '' ), - rename_dataset = seq_values.get( 'field_4', '' ) ) ) + datatx_info = dumps( dict( host=seq_values.get( 'field_0', '' ), + username=seq_values.get( 'field_1', '' ), + password=seq_values.get( 'field_2', '' ), + data_dir=seq_values.get( 'field_3', '' ), + rename_dataset=seq_values.get( 'field_4', '' ) ) ) # update the column - cmd = "UPDATE request_type SET datatx_info='%s' WHERE id=%i" %( datatx_info, request_type_id ) + cmd = "UPDATE request_type SET datatx_info='%s' WHERE id=%i" % ( datatx_info, request_type_id ) migrate_engine.execute( cmd ) # delete foreign key field to the sequencer table in the request_type table try: RequestType_table.c.sequencer_id.drop() except Exception, e: log.debug( "Deleting column 'sequencer_id' in the 'request_type' table failed: %s" % ( str( e ) ) ) - - - - - - - - - - - - - - - - - - diff --git a/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py b/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py index ec05e516239..f9ea4312457 100644 --- a/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py +++ b/lib/galaxy/model/migrate/versions/0068_rename_sequencer_to_external_services.py @@ -6,30 +6,25 @@ The 'sequencer_type_id' column is renamed to 'external_service_type_id' in the r table 'external_service'. Finally, adds a foreign key to the external_service table in the sample_dataset table and populates it. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * - -from galaxy.model.custom_types import * - -from galaxy.util.json import loads, dumps - import datetime -now = datetime.datetime.utcnow - import logging + +from migrate import ForeignKeyConstraint +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT +from sqlalchemy.exc import NoSuchTableError + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) - metadata = MetaData() -#migrate_engine = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) -def nextval( table, col='id' ): - if migrate_engine.name == 'postgres': + +def nextval( migrate_engine, table, col='id' ): + if migrate_engine.name in ['postgres', 'postgresql']: return "nextval('%s_%s_seq')" % ( table, col ) - elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + elif migrate_engine.name in ['mysql', 'sqlite']: return "null" else: raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) @@ -150,7 +145,7 @@ def upgrade(migrate_engine): if not sequencer_id: sequencer_id = 'null' cmd = "INSERT INTO request_type_external_service_association VALUES ( %s, %s, %s )" - cmd = cmd % ( nextval( 'request_type_external_service_association' ), + cmd = cmd % ( nextval( migrate_engine, 'request_type_external_service_association' ), request_type_id, sequencer_id ) migrate_engine.execute( cmd ) @@ -160,30 +155,22 @@ def upgrade(migrate_engine): # In sqlite, create a temp table without the column that needs to be removed. # then copy all the rows from the original table and finally rename the temp table RequestTypeTemp_table = Table( 'request_type_temp', metadata, - Column( "id", Integer, primary_key=True), - Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), - Column( "name", TrimmedString( 255 ), nullable=False ), - Column( "desc", TEXT ), - Column( "request_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ), - Column( "sample_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ), - Column( "deleted", Boolean, index=True, default=False ) ) + Column( "id", Integer, primary_key=True), + Column( "create_time", DateTime, default=now ), + Column( "update_time", DateTime, default=now, onupdate=now ), + Column( "name", TrimmedString( 255 ), nullable=False ), + Column( "desc", TEXT ), + Column( "request_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ), + Column( "sample_form_id", Integer, ForeignKey( "form_definition.id" ), index=True ), + Column( "deleted", Boolean, index=True, default=False ) ) try: RequestTypeTemp_table.create() except Exception, e: log.debug( "Creating request_type_temp table failed: %s" % str( e ) ) # insert all the rows from the request table to the request_temp table - cmd = \ - "INSERT INTO request_type_temp " + \ - "SELECT id," + \ - "create_time," + \ - "update_time," + \ - "name," + \ - "desc," + \ - "request_form_id," + \ - "sample_form_id," + \ - "deleted " + \ - "FROM request_type;" + cmd = "INSERT INTO request_type_temp SELECT id, create_time," + \ + "update_time, name, desc, request_form_id, sample_form_id," + \ + "deleted FROM request_type;" migrate_engine.execute( cmd ) # delete the 'request_type' table try: @@ -269,7 +256,7 @@ def downgrade(migrate_engine): cmd = "ALTER TABLE external_service RENAME TO sequencer" migrate_engine.execute( cmd ) # if running postgres then rename the primary key sequence too - if migrate_engine.name == 'postgres': + if migrate_engine.name in ['postgres', 'postgresql']: cmd = "ALTER SEQUENCE external_service_id_seq RENAME TO sequencer_id_seq" migrate_engine.execute( cmd ) # drop the 'external_service_id' column in the 'sample_dataset' table @@ -284,6 +271,3 @@ def downgrade(migrate_engine): SampleDataset_table.c.external_service_id.drop() except Exception, e: log.debug( "Deleting column 'external_service_id' from the 'sample_dataset' table failed: %s" % ( str( e ) ) ) - - - diff --git a/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py b/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py index 337b3cd1742..c92c7a554bb 100644 --- a/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py +++ b/lib/galaxy/model/migrate/versions/0069_rename_sequencer_form_type.py @@ -1,23 +1,16 @@ """ Migration script to rename the sequencer information form type to external service information form """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * - -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import MetaData + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -28,7 +21,6 @@ def upgrade(migrate_engine): migrate_engine.execute( cmd ) - def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0070_add_info_column_to_deferred_job_table.py b/lib/galaxy/model/migrate/versions/0070_add_info_column_to_deferred_job_table.py index 3f7f72d243d..edaa0c7f554 100644 --- a/lib/galaxy/model/migrate/versions/0070_add_info_column_to_deferred_job_table.py +++ b/lib/galaxy/model/migrate/versions/0070_add_info_column_to_deferred_job_table.py @@ -1,17 +1,14 @@ """ Migration script to add 'info' column to the transfer_job table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, MetaData, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -25,6 +22,7 @@ def upgrade(migrate_engine): print "Adding info column to transfer_job table failed: %s" % str( e ) log.debug( "Adding info column to transfer_job table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0071_add_history_and_workflow_to_sample.py b/lib/galaxy/model/migrate/versions/0071_add_history_and_workflow_to_sample.py index 710f10e6bae..a8f2db7b0e2 100644 --- a/lib/galaxy/model/migrate/versions/0071_add_history_and_workflow_to_sample.py +++ b/lib/galaxy/model/migrate/versions/0071_add_history_and_workflow_to_sample.py @@ -1,18 +1,16 @@ """ Migration script to add 'workflow' and 'history' columns for a sample. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table + +from galaxy.model.custom_types import JSONType + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -29,6 +27,7 @@ def upgrade(migrate_engine): print "Adding history and workflow columns to sample table failed: %s" % str( e ) log.debug( "Adding history and workflow columns to sample table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0072_add_pid_and_socket_columns_to_transfer_job_table.py b/lib/galaxy/model/migrate/versions/0072_add_pid_and_socket_columns_to_transfer_job_table.py index 9dbeea7d91d..ef05650c0ba 100644 --- a/lib/galaxy/model/migrate/versions/0072_add_pid_and_socket_columns_to_transfer_job_table.py +++ b/lib/galaxy/model/migrate/versions/0072_add_pid_and_socket_columns_to_transfer_job_table.py @@ -1,17 +1,14 @@ """ Migration script to add 'pid' and 'socket' columns to the transfer_job table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, Integer, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -28,6 +25,7 @@ def upgrade(migrate_engine): print "Adding columns to transfer_job table failed: %s" % str( e ) log.debug( "Adding columns to transfer_job table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0073_add_ldda_to_implicit_conversion_table.py b/lib/galaxy/model/migrate/versions/0073_add_ldda_to_implicit_conversion_table.py index e6ba0ea03b3..4d599172ad1 100644 --- a/lib/galaxy/model/migrate/versions/0073_add_ldda_to_implicit_conversion_table.py +++ b/lib/galaxy/model/migrate/versions/0073_add_ldda_to_implicit_conversion_table.py @@ -1,17 +1,14 @@ """ Migration script to add 'ldda_parent_id' column to the implicitly_converted_dataset_association table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -21,7 +18,7 @@ def upgrade(migrate_engine): if migrate_engine.name != 'sqlite': c = Column( "ldda_parent_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True, nullable=True ) else: - #Can't use the ForeignKey in sqlite. + # Can't use the ForeignKey in sqlite. c = Column( "ldda_parent_id", Integer, index=True, nullable=True ) c.create( Implicitly_converted_table, index_name="ix_implicitly_converted_dataset_assoc_ldda_parent_id") assert c is Implicitly_converted_table.c.ldda_parent_id @@ -29,6 +26,7 @@ def upgrade(migrate_engine): print "Adding ldda_parent_id column to implicitly_converted_dataset_association table failed: %s" % str( e ) log.debug( "Adding ldda_parent_id column to implicitly_converted_dataset_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py b/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py index 16bbb6f088f..f2168d3cb0f 100644 --- a/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py +++ b/lib/galaxy/model/migrate/versions/0074_add_purged_column_to_library_dataset_table.py @@ -1,32 +1,31 @@ """ Migration script to add 'purged' column to the library_dataset table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def boolean_false(migrate_engine): - if migrate_engine.name in ['postgres','mysql', 'postgresql']: - return False - elif migrate_engine.name == 'sqlite': - return 0 - else: - raise Exception( 'Unable to set True data value for unknown database type: %s' % str( migrate_engine.name ) ) + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return False + elif migrate_engine.name == 'sqlite': + return 0 + else: + raise Exception( 'Unable to set True data value for unknown database type: %s' % str( migrate_engine.name ) ) + def boolean_true(migrate_engine): - if migrate_engine.name in ['postgres','mysql', 'postgresql']: - return True - elif migrate_engine.name == 'sqlite': - return 1 - else: - raise Exception( 'Unable to set False data value for unknown database type: %s' % str( migrate_engine.name ) ) + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return True + elif migrate_engine.name == 'sqlite': + return 1 + else: + raise Exception( 'Unable to set False data value for unknown database type: %s' % str( migrate_engine.name ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -58,6 +57,7 @@ def upgrade(migrate_engine): cmd = "UPDATE library_dataset SET purged = %s WHERE id = %d;" % ( boolean_true(migrate_engine), int( row.id ) ) migrate_engine.execute( cmd ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0075_add_subindex_column_to_run_table.py b/lib/galaxy/model/migrate/versions/0075_add_subindex_column_to_run_table.py index 172181ffbef..651246407ea 100644 --- a/lib/galaxy/model/migrate/versions/0075_add_subindex_column_to_run_table.py +++ b/lib/galaxy/model/migrate/versions/0075_add_subindex_column_to_run_table.py @@ -1,16 +1,13 @@ """ Migration script to add a 'subindex' column to the run table. """ +from sqlalchemy import Column, MetaData, Table -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - -from galaxy.model.custom_types import * +from galaxy.model.custom_types import TrimmedString metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -23,6 +20,7 @@ def upgrade(migrate_engine): except Exception, e: print "Adding the subindex column to the run table failed: ", str( e ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py b/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py index 84202ca2a4e..bc90ec4769a 100644 --- a/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py +++ b/lib/galaxy/model/migrate/versions/0076_fix_form_values_data_corruption.py @@ -2,31 +2,16 @@ This migration script fixes the data corruption caused in the form_values table (content json field) by migrate script 65. ''' -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from sqlalchemy.exc import * -import binascii - -from galaxy.util.json import loads, dumps - import logging -log = logging.getLogger( __name__ ) +from json import dumps, loads +from sqlalchemy import MetaData + +from galaxy.model.custom_types import _sniffnfix_pg9_hex + +log = logging.getLogger( __name__ ) metadata = MetaData() -def _sniffnfix_pg9_hex(value): - """ - Sniff for and fix postgres 9 hex decoding issue - """ - try: - if value[0] == 'x': - return binascii.unhexlify(value[1:]) - else: - return value - except Exception, ex: - return value def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -43,13 +28,13 @@ def upgrade(migrate_engine): # if that fails, it means that the content field is corrupted. try: field_values_dict = loads( _sniffnfix_pg9_hex( str( row['field_values'] ) ) ) - except Exception, e: + except Exception: corrupted_rows = corrupted_rows + 1 # content field is corrupted fields_list = loads( _sniffnfix_pg9_hex( str( row['fdfields'] ) ) ) field_values_str = _sniffnfix_pg9_hex( str( row['field_values'] ) ) try: - #Encoding errors? Just to be safe. + # Encoding errors? Just to be safe. print "Attempting to fix row %s" % row['id'] print "Prior to replacement: %s" % field_values_str except: @@ -66,16 +51,16 @@ def upgrade(migrate_engine): print "The 'content' field of row 'id' %i does not have the field '%s' in the 'form_values' table and could not be fixed by this migration script." % ( int( field['id'] ), field['name'] ) else: # check if this is the last field - if index == len( fields_list )-1: + if index == len( fields_list ) - 1: # since this is the last field, the value string lies between the - # field name and the '"}' string at the end, hence len(field_values_str)-2 - value = field_values_str[ field_index+len( field_name_key ):len( field_values_str )-2 ] + # field name and the '"}' string at the end, hence len(field_values_str) - 2 + value = field_values_str[ field_index + len( field_name_key ):len( field_values_str ) - 2 ] else: # if this is not the last field then the value string lies between # this field name and the next field name - next_field = fields_list[index+1] + next_field = fields_list[index + 1] next_field_index = field_values_str.find( '", "%s": "' % next_field['name'] ) - value = field_values_str[ field_index+len( field_name_key ):next_field_index ] + value = field_values_str[ field_index + len( field_name_key ):next_field_index ] # clean up the value string, escape the required quoutes and newline characters value = value.replace( "'", "\''" )\ .replace( '"', '\\\\"' )\ @@ -86,7 +71,7 @@ def upgrade(migrate_engine): field_values_dict[ field['name'] ] = value # update the db json_values = dumps(field_values_dict) - cmd = "UPDATE form_values SET content='%s' WHERE id=%i" %( json_values, int( row['id'] ) ) + cmd = "UPDATE form_values SET content='%s' WHERE id=%i" % ( json_values, int( row['id'] ) ) migrate_engine.execute( cmd ) try: print "Post replacement: %s" % json_values @@ -97,7 +82,7 @@ def upgrade(migrate_engine): else: print 'No corrupted rows found.' + def downgrade(migrate_engine): metadata.bind = migrate_engine pass - diff --git a/lib/galaxy/model/migrate/versions/0077_create_tool_tag_association_table.py b/lib/galaxy/model/migrate/versions/0077_create_tool_tag_association_table.py index fbb8b873600..c85a080d362 100644 --- a/lib/galaxy/model/migrate/versions/0077_create_tool_tag_association_table.py +++ b/lib/galaxy/model/migrate/versions/0077_create_tool_tag_association_table.py @@ -1,20 +1,15 @@ """ Migration script to create table for storing tool tag associations. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() # Table to add @@ -28,6 +23,7 @@ ToolTagAssociation_table = Table( "tool_tag_association", metadata, Column( "value", TrimmedString(255), index=True), Column( "user_value", TrimmedString(255), index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -39,6 +35,7 @@ def upgrade(migrate_engine): except Exception, e: log.error( "Creating tool_tag_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py b/lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py index babff8d59c9..cd780cc984d 100644 --- a/lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py +++ b/lib/galaxy/model/migrate/versions/0078_add_columns_for_disk_usage_accounting.py @@ -3,17 +3,14 @@ Migration script to add 'total_size' column to the dataset table, 'purged' column to the HDA table, and 'disk_usage' column to the User and GalaxySession tables. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, MetaData, Numeric, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -56,6 +53,7 @@ def upgrade(migrate_engine): print "Adding disk_usage column to galaxy_session table failed: %s" % str( e ) log.debug( "Adding disk_usage column to galaxy_session table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py b/lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py index b4c43290c20..d8dd928cde7 100644 --- a/lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py +++ b/lib/galaxy/model/migrate/versions/0079_input_library_to_job_table.py @@ -1,13 +1,10 @@ """ Migration script to add the job_to_input_library_dataset table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging + +from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table + log = logging.getLogger( __name__ ) metadata = MetaData() @@ -18,6 +15,7 @@ JobToInputLibraryDatasetAssociation_table = Table( "job_to_input_library_dataset Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True ), Column( "name", String(255) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -30,6 +28,7 @@ def upgrade(migrate_engine): print "Creating job_to_input_library_dataset table failed: %s" % str( e ) log.debug( "Creating job_to_input_library_dataset table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0080_quota_tables.py b/lib/galaxy/model/migrate/versions/0080_quota_tables.py index 4559f774f5b..352821797fe 100644 --- a/lib/galaxy/model/migrate/versions/0080_quota_tables.py +++ b/lib/galaxy/model/migrate/versions/0080_quota_tables.py @@ -1,20 +1,13 @@ """ Migration script to create tables for disk quotas. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -# from galaxy.model.orm.ext.assignmapper import * -# from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import BigInteger, Boolean, Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() # Tables to add @@ -50,6 +43,7 @@ DefaultQuotaAssociation_table = Table( "default_quota_association", metadata, Column( "type", String( 32 ), index=True, unique=True ), Column( "quota_id", Integer, ForeignKey( "quota.id" ), index=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -79,18 +73,6 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating default_quota_association table failed: %s" % str( e ) ) - # Create the default quota record - #class Quota( object ): - # def __init__( self, name, description, bytes, operation ): - # self.name = name - # self.description = description - # self.bytes = bytes - # self.operation = operation - #assign_mapper( db_session, Quota, Quota_table ) - #default_quota = Quota( 'Default Quota', 'The base quota applied to all users', -1, '=' ) - #db_session.add( default_quota ) - #db_session.flush() - def downgrade(migrate_engine): metadata.bind = migrate_engine diff --git a/lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py b/lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py index 795cfa7a073..d1d8bea507d 100644 --- a/lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py +++ b/lib/galaxy/model/migrate/versions/0081_add_tool_version_to_hda_ldda.py @@ -1,16 +1,11 @@ """ Migration script to add a 'tool_version' column to the hda/ldda tables. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - -from galaxy.model.custom_types import * +from sqlalchemy import Column, MetaData, Table, TEXT metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -29,6 +24,7 @@ def upgrade(migrate_engine): except Exception, e: print "Adding the tool_version column to the hda/ldda tables failed: ", str( e ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py b/lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py index dc4dead2198..2b2bac72d91 100644 --- a/lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py +++ b/lib/galaxy/model/migrate/versions/0082_add_tool_shed_repository_table.py @@ -1,16 +1,16 @@ """ Migration script to add the tool_shed_repository table. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +import sys +from sqlalchemy import Boolean, Column, DateTime, Integer, MetaData, Table, TEXT + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -33,6 +33,7 @@ ToolShedRepository_table = Table( "tool_shed_repository", metadata, Column( "changeset_revision", TrimmedString( 255 ), index=True ), Column( "deleted", Boolean, index=True, default=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -42,6 +43,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating tool_shed_repository table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0083_add_prepare_files_to_task.py b/lib/galaxy/model/migrate/versions/0083_add_prepare_files_to_task.py index 0a0b75f293e..5c0693a3860 100644 --- a/lib/galaxy/model/migrate/versions/0083_add_prepare_files_to_task.py +++ b/lib/galaxy/model/migrate/versions/0083_add_prepare_files_to_task.py @@ -1,17 +1,14 @@ """ Migration script to add 'prepare_input_files_cmd' column to the task table and to rename a column. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, MetaData, String, Table, TEXT + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -26,7 +23,7 @@ def upgrade(migrate_engine): log.debug( "Adding prepare_input_files_cmd column to task table failed: %s" % str( e ) ) try: task_table = Table( "task", metadata, autoload=True ) - c = Column( "working_directory", String ( 1024 ), nullable=True ) + c = Column( "working_directory", String( 1024 ), nullable=True ) c.create( task_table ) assert c is task_table.c.working_directory except Exception, e: @@ -39,6 +36,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Deleting column 'part_file' from the 'task' table failed: %s" % ( str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() @@ -56,7 +54,7 @@ def downgrade(migrate_engine): log.debug( "Dropping working_directory column from task table failed: %s" % str( e ) ) try: task_table = Table( "task", metadata, autoload=True ) - c = Column( "part_file", String ( 1024 ), nullable=True ) + c = Column( "part_file", String( 1024 ), nullable=True ) c.create( task_table ) assert c is task_table.c.part_file except Exception, e: diff --git a/lib/galaxy/model/migrate/versions/0084_add_ldda_id_to_implicit_conversion_table.py b/lib/galaxy/model/migrate/versions/0084_add_ldda_id_to_implicit_conversion_table.py index f3ae10e6a20..29f2d590688 100644 --- a/lib/galaxy/model/migrate/versions/0084_add_ldda_id_to_implicit_conversion_table.py +++ b/lib/galaxy/model/migrate/versions/0084_add_ldda_id_to_implicit_conversion_table.py @@ -1,17 +1,14 @@ """ Migration script to add 'ldda_id' column to the implicitly_converted_dataset_association table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table + +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -28,6 +25,7 @@ def upgrade(migrate_engine): print "Adding ldda_id column to implicitly_converted_dataset_association table failed: %s" % str( e ) log.debug( "Adding ldda_id column to implicitly_converted_dataset_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0085_add_task_info.py b/lib/galaxy/model/migrate/versions/0085_add_task_info.py index 92ccc6d2d78..2a39dae269e 100644 --- a/lib/galaxy/model/migrate/versions/0085_add_task_info.py +++ b/lib/galaxy/model/migrate/versions/0085_add_task_info.py @@ -1,31 +1,30 @@ """ Migration script to add 'info' column to the task table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) + +from sqlalchemy import Column, MetaData, Table + from galaxy.model.custom_types import TrimmedString +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ metadata.reflect() try: task_table = Table( "task", metadata, autoload=True ) - c = Column( "info", TrimmedString (255) , nullable=True ) + c = Column( "info", TrimmedString(255) , nullable=True ) c.create( task_table ) assert c is task_table.c.info except Exception, e: print "Adding info column to table table failed: %s" % str( e ) log.debug( "Adding info column to task table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py index 828bacd910b..da2ffda0802 100644 --- a/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py +++ b/lib/galaxy/model/migrate/versions/0086_add_tool_shed_repository_table_columns.py @@ -1,18 +1,16 @@ """ Migration script to add the metadata, update_available and includes_datatypes columns to the tool_shed_repository table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import Boolean, Column, MetaData, Table + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import JSONType + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,12 +21,14 @@ log.addHandler( handler ) metadata = MetaData() + def get_default_false(migrate_engine): - if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + if migrate_engine.name in ['mysql', 'sqlite']: return "0" - elif migrate_engine.name in ['postgresql', 'postgres']: + elif migrate_engine.name in ['postgres', 'postgresql']: return "false" + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -58,6 +58,7 @@ def upgrade(migrate_engine): print "Adding update_available column to the tool_shed_repository table failed: %s" % str( e ) log.debug( "Adding update_available column to the tool_shed_repository table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0087_tool_id_guid_map_table.py b/lib/galaxy/model/migrate/versions/0087_tool_id_guid_map_table.py index 7395772f03f..bfba9e0c6b8 100644 --- a/lib/galaxy/model/migrate/versions/0087_tool_id_guid_map_table.py +++ b/lib/galaxy/model/migrate/versions/0087_tool_id_guid_map_table.py @@ -1,18 +1,16 @@ """ Migration script to create the tool_id_guid_map table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table, TEXT + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -34,6 +32,7 @@ ToolIdGuidMap_table = Table( "tool_id_guid_map", metadata, Column( "repository_name", TrimmedString( 255 ) ), Column( "guid", TEXT, index=True, unique=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -43,6 +42,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating tool_id_guid_map table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py b/lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py index eff95ca0076..9e4641ee439 100644 --- a/lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py +++ b/lib/galaxy/model/migrate/versions/0088_add_installed_changeset_revison_column.py @@ -1,18 +1,16 @@ """ Migration script to add the installed_changeset_revision column to the tool_shed_repository table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import Column, MetaData, Table + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,6 +21,7 @@ log.addHandler( handler ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -48,11 +47,13 @@ def upgrade(migrate_engine): update_count = 0 for row in tool_shed_repositories: cmd = "UPDATE tool_shed_repository " \ - + "SET installed_changeset_revision = '%s' " % row.changeset_revision \ + + "SET installed_changeset_revision = '%s' " % row.changeset_revision \ + "WHERE changeset_revision = '%s';" % row.changeset_revision migrate_engine.execute( cmd ) update_count += 1 print "Updated the installed_changeset_revision column for ", update_count, " rows in the tool_shed_repository table. " + + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0089_add_object_store_id_columns.py b/lib/galaxy/model/migrate/versions/0089_add_object_store_id_columns.py index bd905bc648b..e146a126538 100644 --- a/lib/galaxy/model/migrate/versions/0089_add_object_store_id_columns.py +++ b/lib/galaxy/model/migrate/versions/0089_add_object_store_id_columns.py @@ -1,18 +1,16 @@ """ Migration script to add 'object_store_id' column to various tables """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) + +from sqlalchemy import Column, MetaData, Table + from galaxy.model.custom_types import TrimmedString +log = logging.getLogger( __name__ ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -27,6 +25,7 @@ def upgrade(migrate_engine): print "Adding object_store_id column to %s table failed: %s" % ( t_name, str( e ) ) log.debug( "Adding object_store_id column to %s table failed: %s" % ( t_name, str( e ) ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0090_add_tool_shed_repository_table_columns.py b/lib/galaxy/model/migrate/versions/0090_add_tool_shed_repository_table_columns.py index 5f412cb5a40..14d13bbf8ba 100644 --- a/lib/galaxy/model/migrate/versions/0090_add_tool_shed_repository_table_columns.py +++ b/lib/galaxy/model/migrate/versions/0090_add_tool_shed_repository_table_columns.py @@ -1,18 +1,13 @@ """ Migration script to add the uninstalled and dist_to_shed columns to the tool_shed_repository table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import Boolean, Column, MetaData, Table + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,12 +18,14 @@ log.addHandler( handler ) metadata = MetaData() + def default_false(migrate_engine): - if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + if migrate_engine.name in ['mysql', 'sqlite']: return "0" - elif migrate_engine.name in ['postgresql', 'postgres']: + elif migrate_engine.name in ['postgres', 'postgresql']: return "false" + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -49,6 +46,7 @@ def upgrade(migrate_engine): except Exception, e: print "Adding dist_to_shed column to the tool_shed_repository table failed: %s" % str( e ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py b/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py index 5d43fdc509a..50ed32c10a2 100644 --- a/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py +++ b/lib/galaxy/model/migrate/versions/0091_add_tool_version_tables.py @@ -1,20 +1,17 @@ """ Migration script to create the tool_version and tool_version_association tables and drop the tool_id_guid_map table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * -from galaxy.model.custom_types import _sniffnfix_pg9_hex -from galaxy.util.json import loads, dumps +import logging +import sys +from json import loads -import sys, logging +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table, TEXT + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import _sniffnfix_pg9_hex, TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -24,23 +21,24 @@ handler.setFormatter( formatter ) log.addHandler( handler ) metadata = MetaData() -#migrate_engine = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) ) -def nextval( table, col='id' ): - if migrate_engine.name == 'postgres': + +def nextval( migrate_engine, table, col='id' ): + if migrate_engine.name in ['postgres', 'postgresql']: return "nextval('%s_%s_seq')" % ( table, col ) - elif migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + elif migrate_engine.name in ['mysql', 'sqlite']: return "null" else: raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) -def localtimestamp(): - if migrate_engine.name == 'postgres' or migrate_engine.name == 'mysql': - return "LOCALTIMESTAMP" - elif migrate_engine.name == 'sqlite': - return "current_date || ' ' || current_time" - else: - raise Exception( 'Unable to convert data for unknown database type: %s' % db ) + +def localtimestamp(migrate_engine): + if migrate_engine.name in ['mysql', 'postgres', 'postgresql']: + return "LOCALTIMESTAMP" + elif migrate_engine.name == 'sqlite': + return "current_date || ' ' || current_time" + else: + raise Exception( 'Unable to convert data for unknown database type: %s' % migrate_engine.name ) ToolVersion_table = Table( "tool_version", metadata, Column( "id", Integer, primary_key=True ), @@ -54,6 +52,7 @@ ToolVersionAssociation_table = Table( "tool_version_association", metadata, Column( "tool_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ), Column( "parent_id", Integer, ForeignKey( "tool_version.id" ), index=True, nullable=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -83,7 +82,7 @@ def upgrade(migrate_engine): tools = repository_metadata.get( 'tools', [] ) for tool_dict in tools: cmd = "INSERT INTO tool_version VALUES (%s, %s, %s, '%s', %s)" % \ - ( nextval( 'tool_version' ), localtimestamp(), localtimestamp(), tool_dict[ 'guid' ], tool_shed_repository_id ) + ( nextval( migrate_engine, 'tool_version' ), localtimestamp( migrate_engine ), localtimestamp( migrate_engine ), tool_dict[ 'guid' ], tool_shed_repository_id ) migrate_engine.execute( cmd ) count += 1 print "Added %d rows to the new tool_version table." % count @@ -93,6 +92,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Dropping tool_id_guid_map table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine diff --git a/lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py b/lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py index 2615ed56a26..b8b00eafff0 100644 --- a/lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py +++ b/lib/galaxy/model/migrate/versions/0092_add_migrate_tools_table.py @@ -1,18 +1,16 @@ """ Migration script to create the migrate_tools table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import Column, Integer, MetaData, Table, TEXT + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -28,6 +26,7 @@ MigrateTools_table = Table( "migrate_tools", metadata, Column( "repository_path", TEXT ), Column( "version", Integer ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -41,6 +40,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating migrate_tools table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0093_add_job_params_col.py b/lib/galaxy/model/migrate/versions/0093_add_job_params_col.py index 20912f44323..34ae580da50 100644 --- a/lib/galaxy/model/migrate/versions/0093_add_job_params_col.py +++ b/lib/galaxy/model/migrate/versions/0093_add_job_params_col.py @@ -1,27 +1,25 @@ """ Migration script to create "params" column in job table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) + +from sqlalchemy import Column, MetaData, Table # Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from galaxy.model.custom_types import TrimmedString +log = logging.getLogger( __name__ ) metadata = MetaData() # Column to add. params_col = Column( "params", TrimmedString(255), index=True ) + def display_migration_details(): print "" print "This migration script adds a 'params' column to the Job table." + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -37,6 +35,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Adding column 'params' to job table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py b/lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py index f8ad97dadd6..ba074730f5d 100644 --- a/lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py +++ b/lib/galaxy/model/migrate/versions/0094_add_job_handler_col.py @@ -1,27 +1,25 @@ """ Migration script to create "handler" column in job table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) + +from sqlalchemy import Column, MetaData, Table # Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from galaxy.model.custom_types import TrimmedString +log = logging.getLogger( __name__ ) metadata = MetaData() # Column to add. handler_col = Column( "handler", TrimmedString(255), index=True ) + def display_migration_details(): print "" print "This migration script adds a 'handler' column to the Job table." + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -37,6 +35,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Adding column 'handler' to job table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0095_hda_subsets.py b/lib/galaxy/model/migrate/versions/0095_hda_subsets.py index 58069cde418..8dadc1dcc08 100644 --- a/lib/galaxy/model/migrate/versions/0095_hda_subsets.py +++ b/lib/galaxy/model/migrate/versions/0095_hda_subsets.py @@ -1,15 +1,11 @@ """ Migration script to create table for tracking history_dataset_association subsets. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table, Unicode + +log = logging.getLogger( __name__ ) metadata = MetaData() # Table to add. @@ -18,8 +14,8 @@ HistoryDatasetAssociationSubset_table = Table( "history_dataset_association_subs Column( "id", Integer, primary_key=True ), Column( "history_dataset_association_id", Integer, ForeignKey( "history_dataset_association.id" ) ), Column( "history_dataset_association_subset_id", Integer, ForeignKey( "history_dataset_association.id" ) ), - Column( "location", Unicode(255), index=True) -) + Column( "location", Unicode(255), index=True) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -43,6 +39,7 @@ def upgrade(migrate_engine): print str(e) log.debug( "Adding indices to table 'history_dataset_association_subset' table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0096_openid_provider.py b/lib/galaxy/model/migrate/versions/0096_openid_provider.py index 15c3d993a76..52d7a1df8b2 100644 --- a/lib/galaxy/model/migrate/versions/0096_openid_provider.py +++ b/lib/galaxy/model/migrate/versions/0096_openid_provider.py @@ -2,19 +2,17 @@ Migration script to add column to openid table for provider. Remove any OpenID entries with nonunique GenomeSpace Identifier """ +import logging + +from sqlalchemy import Column, MetaData, Table -BAD_IDENTIFIER = 'https://identity.genomespace.org/identityServer/xrd.jsp' -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * from galaxy.model.custom_types import TrimmedString -import logging log = logging.getLogger( __name__ ) - +BAD_IDENTIFIER = 'https://identity.genomespace.org/identityServer/xrd.jsp' metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -35,6 +33,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Deleting bad Identifiers from galaxy_user_openid failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py b/lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py index 4d2acc5551a..b2cf86cd98f 100644 --- a/lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py +++ b/lib/galaxy/model/migrate/versions/0097_add_ctx_rev_column.py @@ -1,18 +1,16 @@ """ Migration script to add the ctx_rev column to the tool_shed_repository table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import Column, MetaData, Table + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,6 +21,7 @@ log.addHandler( handler ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -34,6 +33,8 @@ def upgrade(migrate_engine): assert col is ToolShedRepository_table.c.ctx_rev except Exception, e: print "Adding ctx_rev column to the tool_shed_repository table failed: %s" % str( e ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py b/lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py index 826fcd612d0..8becd71717c 100644 --- a/lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py +++ b/lib/galaxy/model/migrate/versions/0098_genome_index_tool_data_table.py @@ -1,17 +1,13 @@ """ Migration script to create the genome_index_tool_data table. """ - -from sqlalchemy import * -from migrate import * - import datetime +import logging +import sys + +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table + now = datetime.datetime.utcnow - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - -import sys, logging log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -33,8 +29,8 @@ GenomeIndexToolData_table = Table( "genome_index_tool_data", metadata, Column( "created_time", DateTime, default=now ), Column( "modified_time", DateTime, default=now, onupdate=now ), Column( "indexer", String( 64 ) ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - ) + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -46,6 +42,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating genome_index_tool_data table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0099_add_tool_dependency_table.py b/lib/galaxy/model/migrate/versions/0099_add_tool_dependency_table.py index 3567b41513f..c36fd5856f0 100644 --- a/lib/galaxy/model/migrate/versions/0099_add_tool_dependency_table.py +++ b/lib/galaxy/model/migrate/versions/0099_add_tool_dependency_table.py @@ -1,16 +1,15 @@ """ Migration script to add the tool_dependency table. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +import sys +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel( logging.DEBUG ) handler = logging.StreamHandler( sys.stdout ) @@ -33,6 +32,7 @@ ToolDependency_table = Table( "tool_dependency", metadata, Column( "type", TrimmedString( 40 ) ), Column( "uninstalled", Boolean, default=False ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -42,6 +42,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating tool_dependency table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0100_alter_tool_dependency_table_version_column.py b/lib/galaxy/model/migrate/versions/0100_alter_tool_dependency_table_version_column.py index 95554ea2791..a79315c2778 100644 --- a/lib/galaxy/model/migrate/versions/0100_alter_tool_dependency_table_version_column.py +++ b/lib/galaxy/model/migrate/versions/0100_alter_tool_dependency_table_version_column.py @@ -1,18 +1,13 @@ """ Migration script to alter the type of the tool_dependency.version column from TrimmedString(40) to Text. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import MetaData, Table + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -20,16 +15,16 @@ format = "%(name)s %(levelname)s %(asctime)s %(message)s" formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) - metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ metadata.reflect() - ToolDependency_table = Table( "tool_dependency", metadata, autoload=True ) + Table( "tool_dependency", metadata, autoload=True ) # Change the tool_dependency table's version column from TrimmedString to Text. - if migrate_engine.name in ['postgresql', 'postgres']: + if migrate_engine.name in ['postgres', 'postgresql']: cmd = "ALTER TABLE tool_dependency ALTER COLUMN version TYPE Text;" elif migrate_engine.name == 'mysql': cmd = "ALTER TABLE tool_dependency MODIFY COLUMN version Text;" @@ -48,6 +43,8 @@ def upgrade(migrate_engine): migrate_engine.execute( cmd ) except Exception, e: log.debug( "Altering tool_dependency.version column from TrimmedString(40) to Text failed: %s" % str( e ) ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine # Not necessary to change column type Text to TrimmedString(40). diff --git a/lib/galaxy/model/migrate/versions/0101_drop_installed_changeset_revision_column.py b/lib/galaxy/model/migrate/versions/0101_drop_installed_changeset_revision_column.py index 9fc373d4560..982065ffb26 100644 --- a/lib/galaxy/model/migrate/versions/0101_drop_installed_changeset_revision_column.py +++ b/lib/galaxy/model/migrate/versions/0101_drop_installed_changeset_revision_column.py @@ -1,16 +1,14 @@ """ Migration script to drop the installed_changeset_revision column from the tool_dependency table. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +import sys +from sqlalchemy import MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel( logging.DEBUG ) handler = logging.StreamHandler( sys.stdout ) @@ -21,6 +19,7 @@ log.addHandler( handler ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -36,6 +35,8 @@ def upgrade(migrate_engine): col.drop() except Exception, e: log.debug( "Dropping column 'installed_changeset_revision' from tool_dependency table failed: %s" % ( str( e ) ) ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine pass diff --git a/lib/galaxy/model/migrate/versions/0102_add_tool_dependency_status_columns.py b/lib/galaxy/model/migrate/versions/0102_add_tool_dependency_status_columns.py index 54eb23d2fb0..24b3802c597 100644 --- a/lib/galaxy/model/migrate/versions/0102_add_tool_dependency_status_columns.py +++ b/lib/galaxy/model/migrate/versions/0102_add_tool_dependency_status_columns.py @@ -1,18 +1,16 @@ """ Migration script to add status and error_message columns to the tool_dependency table and drop the uninstalled column from the tool_dependency table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +from sqlalchemy import Boolean, Column, MetaData, Table, TEXT + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -23,6 +21,7 @@ log.addHandler( handler ) metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -45,12 +44,14 @@ def upgrade(migrate_engine): print "Adding error_message column to the tool_dependency table failed: %s" % str( e ) if migrate_engine.name != 'sqlite': - #This breaks in sqlite due to failure to drop check constraint. + # This breaks in sqlite due to failure to drop check constraint. # TODO move to alembic. try: ToolDependency_table.c.uninstalled.drop() except Exception, e: print "Dropping uninstalled column from the tool_dependency table failed: %s" % str( e ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0103_add_tool_shed_repository_status_columns.py b/lib/galaxy/model/migrate/versions/0103_add_tool_shed_repository_status_columns.py index ff45a2426b7..4e8a0e7dde1 100644 --- a/lib/galaxy/model/migrate/versions/0103_add_tool_shed_repository_status_columns.py +++ b/lib/galaxy/model/migrate/versions/0103_add_tool_shed_repository_status_columns.py @@ -1,17 +1,15 @@ """Migration script to add status and error_message columns to the tool_shed_repository table.""" - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +from sqlalchemy import Column, MetaData, Table, TEXT + +# Need our custom types, but don't import anything else from model +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow metadata = MetaData() + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -55,6 +53,8 @@ def upgrade(migrate_engine): print "Exception executing sql command: " print cmd print str( e ) + + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py b/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py index 9a713781018..11e2ae94f92 100644 --- a/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py +++ b/lib/galaxy/model/migrate/versions/0104_update_genome_downloader_job_parameters.py @@ -1,18 +1,16 @@ """ Migration script to update the deferred job parameters for liftover transfer jobs. """ - import datetime -import logging, sys +import logging +import sys + +from sqlalchemy import Column, DateTime, Integer, MetaData, String, Table +from sqlalchemy.orm import mapper, scoped_session, sessionmaker + +# Need our custom types, but don't import anything else from model from galaxy.model.custom_types import JSONType from galaxy.util.bunch import Bunch -from sqlalchemy import * -from sqlalchemy import Integer, Table, MetaData, Column -from sqlalchemy.orm import * -from sqlalchemy.orm import scoped_session, sessionmaker -from migrate import * -# Need our custom types, but don't import anything else from model - now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) @@ -26,18 +24,21 @@ log.addHandler( handler ) metadata = MetaData() context = scoped_session( sessionmaker( autoflush=False, autocommit=True ) ) + class DeferredJob( object ): - states = Bunch( NEW = 'new', - WAITING = 'waiting', - QUEUED = 'queued', - RUNNING = 'running', - OK = 'ok', - ERROR = 'error' ) + states = Bunch( NEW='new', + WAITING='waiting', + QUEUED='queued', + RUNNING='running', + OK='ok', + ERROR='error' ) + def __init__( self, state=None, plugin=None, params=None ): self.state = state self.plugin = plugin self.params = params + def upgrade(migrate_engine): metadata.bind = migrate_engine @@ -49,7 +50,7 @@ def upgrade(migrate_engine): Column( "plugin", String( 128 ), index=True ), Column( "params", JSONType ) ) - mapper( DeferredJob, DeferredJob.table, properties = {} ) + mapper( DeferredJob, DeferredJob.table, properties={} ) liftoverjobs = dict() @@ -67,6 +68,7 @@ def upgrade(migrate_engine): context.flush() + def downgrade(migrate_engine): metadata.bind = migrate_engine diff --git a/lib/galaxy/model/migrate/versions/0105_add_cleanup_event_table.py b/lib/galaxy/model/migrate/versions/0105_add_cleanup_event_table.py index 0216567cca2..0a99a18c6f2 100644 --- a/lib/galaxy/model/migrate/versions/0105_add_cleanup_event_table.py +++ b/lib/galaxy/model/migrate/versions/0105_add_cleanup_event_table.py @@ -1,16 +1,15 @@ """ Migration script to add the cleanup_event* tables. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +import sys +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel( logging.DEBUG ) handler = logging.StreamHandler( sys.stdout ) @@ -81,6 +80,7 @@ CleanupEventImplicitlyConvertedDatasetAssociationAssociation_table = Table( "cle Column( "cleanup_event_id", Integer, ForeignKey( "cleanup_event.id" ), index=True, nullable=True ), Column( "icda_id", Integer, ForeignKey( "implicitly_converted_dataset_association.id" ), index=True ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine print __doc__ @@ -99,6 +99,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0106_add_missing_indexes.py b/lib/galaxy/model/migrate/versions/0106_add_missing_indexes.py index 529eb93dbe9..0bf4922b4c4 100644 --- a/lib/galaxy/model/migrate/versions/0106_add_missing_indexes.py +++ b/lib/galaxy/model/migrate/versions/0106_add_missing_indexes.py @@ -1,23 +1,13 @@ """ Migration script to create missing indexes. Adding new columns to existing tables via SQLAlchemy does not create the index, even if the column definition includes index=True. """ +import logging -from sqlalchemy import * -from sqlalchemy.orm import * -from sqlalchemy.exc import ProgrammingError +from sqlalchemy import Index, MetaData, Table from sqlalchemy.engine import reflection -from migrate import * -from migrate.changeset import * -import sys, logging log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) -handler = logging.StreamHandler( sys.stdout ) -format = "%(name)s %(levelname)s %(asctime)s %(message)s" -formatter = logging.Formatter( format ) -handler.setFormatter( formatter ) -log.addHandler( handler ) - metadata = MetaData() indexes = ( @@ -42,7 +32,7 @@ indexes = ( ( "ix_job_imported", 'job', 'imported' ), # 0051 ( "ix_request_notification", 'request', 'notification' ), # 0057 ( "ix_sd_external_service_id", 'sample_dataset', 'external_service_id' ), # 0068 - ( "ix_icda_ldda_parent_id", 'implicitly_converted_dataset_association', 'ldda_parent_id' ), # 0073 + ( "ix_icda_ldda_parent_id", 'implicitly_converted_dataset_association', 'ldda_parent_id' ), # 0073 ( "ix_library_dataset_purged", 'library_dataset', 'purged' ), # 0074 ( "ix_run_subindex", 'run', 'subindex' ), # 0075 ( "ix_history_dataset_association_purged", 'history_dataset_association', 'purged' ), # 0078 @@ -57,6 +47,7 @@ indexes = ( ( "ix_galaxy_user_email", 'galaxy_user', 'email' ) # 0106 ) + def upgrade(migrate_engine): print __doc__ metadata.bind = migrate_engine @@ -70,10 +61,11 @@ def upgrade(migrate_engine): if ix not in [ins_ix.get('name', None) for ins_ix in insp.get_indexes(table)]: Index( ix, t.c[col] ).create() else: - pass #Index already exists, don't recreate. + pass # Index already exists, don't recreate. except Exception, e: log.error("Unable to create index '%s': %s" % (ix, str(e))) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0107_add_exit_code_to_job_and_task.py b/lib/galaxy/model/migrate/versions/0107_add_exit_code_to_job_and_task.py index 6c9b73b55b1..e68f355be30 100644 --- a/lib/galaxy/model/migrate/versions/0107_add_exit_code_to_job_and_task.py +++ b/lib/galaxy/model/migrate/versions/0107_add_exit_code_to_job_and_task.py @@ -1,18 +1,11 @@ """ Add the exit_code column to the Job and Task tables. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import logging + +from sqlalchemy import Column, Integer, MetaData, Table + log = logging.getLogger( __name__ ) - -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * - metadata = MetaData() # There was a bug when only one column was used for both tables, @@ -20,10 +13,12 @@ metadata = MetaData() exit_code_job_col = Column( "exit_code", Integer, nullable=True ) exit_code_task_col = Column( "exit_code", Integer, nullable=True ) + def display_migration_details(): print "" print "This migration script adds a 'handler' column to the Job table." + def upgrade(migrate_engine): print __doc__ metadata.bind = migrate_engine @@ -49,6 +44,7 @@ def upgrade(migrate_engine): log.error( "Adding column 'exit_code' to task table failed: %s" % str( e ) ) return + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() @@ -68,4 +64,3 @@ def downgrade(migrate_engine): exit_code_col.drop() except Exception, e: log.debug( "Dropping 'exit_code' column from task table failed: %s" % ( str( e ) ) ) - diff --git a/lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py b/lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py index f9d16dcb21a..5da8e18e51f 100644 --- a/lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py +++ b/lib/galaxy/model/migrate/versions/0108_add_extended_metadata.py @@ -1,16 +1,13 @@ """ Add the ExtendedMetadata and ExtendedMetadataIndex tables """ +import logging + +from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table, TEXT -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * from galaxy.model.custom_types import JSONType -import logging log = logging.getLogger( __name__ ) - metadata = MetaData() ExtendedMetadata_table = Table("extended_metadata", metadata, @@ -22,7 +19,7 @@ ExtendedMetadataIndex_table = Table("extended_metadata_index", metadata, Column( "extended_metadata_id", Integer, ForeignKey("extended_metadata.id", onupdate="CASCADE", ondelete="CASCADE" ), - index=True ), + index=True ), Column( "path", String( 255 )), Column( "value", TEXT)) @@ -32,6 +29,7 @@ extended_metadata_ldda_col = Column( "extended_metadata_id", Integer, ForeignKey def display_migration_details(): print "This migration script adds a ExtendedMetadata tables" + def upgrade(migrate_engine): print __doc__ metadata.bind = migrate_engine @@ -74,5 +72,3 @@ def downgrade(migrate_engine): extended_metadata_id.drop() except Exception, e: log.debug( "Dropping 'extended_metadata_id' column from library_dataset_dataset_association table failed: %s" % ( str( e ) ) ) - - diff --git a/lib/galaxy/model/migrate/versions/0109_add_repository_dependency_tables.py b/lib/galaxy/model/migrate/versions/0109_add_repository_dependency_tables.py index 3181648f9d1..32138cb8b12 100644 --- a/lib/galaxy/model/migrate/versions/0109_add_repository_dependency_tables.py +++ b/lib/galaxy/model/migrate/versions/0109_add_repository_dependency_tables.py @@ -1,16 +1,13 @@ """ Migration script to add the repository_dependency and repository_repository_dependency_association tables. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +import sys +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel( logging.DEBUG ) handler = logging.StreamHandler( sys.stdout ) @@ -34,6 +31,7 @@ RepositoryRepositoryDependencyAssociation_table = Table( "repository_repository_ Column( "tool_shed_repository_id", Integer, ForeignKey( "tool_shed_repository.id" ), index=True ), Column( "repository_dependency_id", Integer, ForeignKey( "repository_dependency.id" ), index=True ) ) + def upgrade(migrate_engine): print __doc__ metadata.bind = migrate_engine @@ -47,6 +45,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Creating repository_repository_dependency_association table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0110_add_dataset_uuid.py b/lib/galaxy/model/migrate/versions/0110_add_dataset_uuid.py index 366419e1b55..d6eed4e5413 100644 --- a/lib/galaxy/model/migrate/versions/0110_add_dataset_uuid.py +++ b/lib/galaxy/model/migrate/versions/0110_add_dataset_uuid.py @@ -1,16 +1,13 @@ """ Add UUID column to dataset table """ +import logging + +from sqlalchemy import Column, MetaData, Table -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * from galaxy.model.custom_types import UUIDType -import logging log = logging.getLogger( __name__ ) - dataset_uuid_column = Column( "uuid", UUIDType, nullable=True ) @@ -18,6 +15,7 @@ def display_migration_details(): print "" print "This migration adds uuid column to dataset table" + def upgrade(migrate_engine): print __doc__ metadata = MetaData() @@ -47,5 +45,3 @@ def downgrade(migrate_engine): dataset_uuid.drop() except Exception, e: log.debug( "Dropping 'uuid' column from dataset table failed: %s" % ( str( e ) ) ) - - diff --git a/lib/galaxy/model/migrate/versions/0111_add_job_destinations.py b/lib/galaxy/model/migrate/versions/0111_add_job_destinations.py index 0a28694bc6c..af2196a86f7 100644 --- a/lib/galaxy/model/migrate/versions/0111_add_job_destinations.py +++ b/lib/galaxy/model/migrate/versions/0111_add_job_destinations.py @@ -1,20 +1,20 @@ """ Add support for job destinations to the job table """ +import logging + +from sqlalchemy import Column, MetaData, String, Table -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * from galaxy.model.custom_types import JSONType -import logging log = logging.getLogger( __name__ ) + def display_migration_details(): print "" print "This migration script adds 'destination_id' and 'destination_params' columns to the Job table." + def upgrade(migrate_engine): print __doc__ metadata = MetaData() @@ -36,6 +36,7 @@ def upgrade(migrate_engine): except Exception, e: log.error( "Adding column 'destination_params' to job table failed: %s" % str( e ) ) + def downgrade(migrate_engine): metadata = MetaData() metadata.bind = migrate_engine diff --git a/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py b/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py index b16ef973dd1..9a37fdb5bb7 100644 --- a/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py +++ b/lib/galaxy/model/migrate/versions/0112_add_data_manager_history_association_and_data_manager_job_association_tables.py @@ -1,25 +1,14 @@ """ Migration script to add the data_manager_history_association table and data_manager_job_association. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel( logging.DEBUG ) -handler = logging.StreamHandler( sys.stdout ) -format = "%(name)s %(levelname)s %(asctime)s %(message)s" -formatter = logging.Formatter( format ) -handler.setFormatter( formatter ) -log.addHandler( handler ) - - metadata = MetaData() DataManagerHistoryAssociation_table = Table( "data_manager_history_association", metadata, @@ -27,16 +16,15 @@ DataManagerHistoryAssociation_table = Table( "data_manager_history_association", Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, index=True, default=now, onupdate=now ), Column( "history_id", Integer, ForeignKey( "history.id" ), index=True ), - Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) - ) + Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ) ) DataManagerJobAssociation_table = Table( "data_manager_job_association", metadata, Column( "id", Integer, primary_key=True), Column( "create_time", DateTime, default=now ), Column( "update_time", DateTime, index=True, default=now, onupdate=now ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), - Column( "data_manager_id", TEXT, index=True ) - ) + Column( "data_manager_id", TEXT, index=True ) ) + def upgrade(migrate_engine): print __doc__ diff --git a/lib/galaxy/model/migrate/versions/0113_update_migrate_tools_table.py b/lib/galaxy/model/migrate/versions/0113_update_migrate_tools_table.py index bc423a0a93c..e8835131667 100644 --- a/lib/galaxy/model/migrate/versions/0113_update_migrate_tools_table.py +++ b/lib/galaxy/model/migrate/versions/0113_update_migrate_tools_table.py @@ -1,18 +1,11 @@ """ Migration script to update the migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -21,6 +14,7 @@ formatter = logging.Formatter( format ) handler.setFormatter( formatter ) log.addHandler( handler ) + def upgrade(migrate_engine): print __doc__ # Create the table. @@ -30,6 +24,7 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Updating migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate failed: %s" % str( e ) ) + def downgrade(migrate_engine): try: cmd = "UPDATE migrate_tools set repository_path='lib/galaxy/tool_shed/migrate';" diff --git a/lib/galaxy/model/migrate/versions/0114_update_migrate_tools_table_again.py b/lib/galaxy/model/migrate/versions/0114_update_migrate_tools_table_again.py index aab8bbfc55d..55902c1c816 100644 --- a/lib/galaxy/model/migrate/versions/0114_update_migrate_tools_table_again.py +++ b/lib/galaxy/model/migrate/versions/0114_update_migrate_tools_table_again.py @@ -1,18 +1,11 @@ """ Migration script to update the migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow -# Need our custom types, but don't import anything else from model -from galaxy.model.custom_types import * +import logging +import sys -import sys, logging +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel(logging.DEBUG) handler = logging.StreamHandler( sys.stdout ) @@ -31,10 +24,10 @@ def upgrade(migrate_engine): except Exception, e: log.debug( "Updating migrate_tools.repository_path column to point to the new location lib/tool_shed/galaxy_install/migrate failed: %s" % str( e ) ) + def downgrade(migrate_engine): try: cmd = "UPDATE migrate_tools set repository_path='lib/galaxy/tool_shed/migrate';" migrate_engine.execute( cmd ) except Exception, e: log.debug( "Updating migrate_tools.repository_path column to point to the old location lib/galaxy/tool_shed/migrate failed: %s" % str( e ) ) - diff --git a/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py b/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py index 0efb653eb7c..c981e19ec6c 100644 --- a/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py +++ b/lib/galaxy/model/migrate/versions/0115_longer_user_password_field.py @@ -1,13 +1,13 @@ """ Expand the length of the password fields in the galaxy_user table to allow for other hasing schemes """ - -from sqlalchemy import * -from migrate import * - import logging + +from sqlalchemy import MetaData, String, Table + log = logging.getLogger( __name__ ) + def upgrade( migrate_engine ): meta = MetaData( bind=migrate_engine ) user = Table( 'galaxy_user', meta, autoload=True ) @@ -16,6 +16,7 @@ def upgrade( migrate_engine ): except: log.exception( "Altering password column failed" ) + def downgrade(migrate_engine): meta = MetaData( bind=migrate_engine ) user = Table( 'galaxy_user', meta, autoload=True ) diff --git a/lib/galaxy/model/migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py b/lib/galaxy/model/migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py index 5cec56f23cd..7d3017b0c07 100644 --- a/lib/galaxy/model/migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py +++ b/lib/galaxy/model/migrate/versions/0116_drop_update_available_col_add_tool_shed_status_col.py @@ -1,17 +1,16 @@ """ Migration script to drop the update_available Boolean column and replace it with the tool_shed_status JSONType column in the tool_shed_repository table. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -import sys, logging -from galaxy.model.custom_types import * -from sqlalchemy.exc import * import datetime -now = datetime.datetime.utcnow +import logging +import sys +from sqlalchemy import Boolean, Column, MetaData, Table +from sqlalchemy.exc import NoSuchTableError + +from galaxy.model.custom_types import JSONType + +now = datetime.datetime.utcnow log = logging.getLogger( __name__ ) log.setLevel( logging.DEBUG ) handler = logging.StreamHandler( sys.stdout ) @@ -22,12 +21,14 @@ log.addHandler( handler ) metadata = MetaData() + def default_false( migrate_engine ): - if migrate_engine.name == 'mysql' or migrate_engine.name == 'sqlite': + if migrate_engine.name in ['mysql', 'sqlite']: return "0" - elif migrate_engine.name in [ 'postgresql', 'postgres' ]: + elif migrate_engine.name in [ 'postgres', 'postgresql' ]: return "false" + def upgrade( migrate_engine ): metadata.bind = migrate_engine print __doc__ @@ -52,6 +53,7 @@ def upgrade( migrate_engine ): except Exception, e: print "Adding tool_shed_status column to the tool_shed_repository table failed: %s" % str( e ) + def downgrade( migrate_engine ): metadata.bind = migrate_engine metadata.reflect() diff --git a/lib/galaxy/model/migrate/versions/0117_add_user_activation.py b/lib/galaxy/model/migrate/versions/0117_add_user_activation.py index a55d26531ef..2b7b3c48f55 100644 --- a/lib/galaxy/model/migrate/versions/0117_add_user_activation.py +++ b/lib/galaxy/model/migrate/versions/0117_add_user_activation.py @@ -5,16 +5,13 @@ Created on Sep 10, 2013 Adds 'active' and 'activation_token' columns to the galaxy_user table. ''' +import logging + +from sqlalchemy import Boolean, Column, MetaData, Table -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * from galaxy.model.custom_types import TrimmedString -import logging log = logging.getLogger( __name__ ) - user_active_column = Column( "active", Boolean, default=True, nullable=True ) user_activation_token_column = Column( "activation_token", TrimmedString( 64 ), nullable=True ) diff --git a/lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py b/lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py index 879f5ad8a41..3642fe41810 100644 --- a/lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py +++ b/lib/galaxy/model/migrate/versions/0118_add_hda_extended_metadata.py @@ -1,29 +1,24 @@ """ Add link from history_dataset_association to the extended_metadata table """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import JSONType - import logging + +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table + log = logging.getLogger( __name__ ) - metadata = MetaData() - extended_metadata_hda_col = Column( "extended_metadata_id", Integer, ForeignKey("extended_metadata.id"), nullable=True ) def display_migration_details(): print "This migration script adds a ExtendedMetadata links to HistoryDatasetAssociation tables" + def upgrade(migrate_engine): print __doc__ metadata.bind = migrate_engine metadata.reflect() - + try: hda_table = Table( "history_dataset_association", metadata, autoload=True ) extended_metadata_hda_col.create( hda_table ) @@ -36,7 +31,7 @@ def upgrade(migrate_engine): def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - + # Drop the HDA table's extended metadata ID column. try: hda_table = Table( "history_dataset_association", metadata, autoload=True ) @@ -44,5 +39,3 @@ def downgrade(migrate_engine): extended_metadata_id.drop() except Exception, e: log.debug( "Dropping 'extended_metadata_id' column from history_dataset_association table failed: %s" % ( str( e ) ) ) - - diff --git a/lib/galaxy/model/migrate/versions/0119_job_metrics.py b/lib/galaxy/model/migrate/versions/0119_job_metrics.py index 8a34975ecc6..94607d1e0ba 100644 --- a/lib/galaxy/model/migrate/versions/0119_job_metrics.py +++ b/lib/galaxy/model/migrate/versions/0119_job_metrics.py @@ -1,18 +1,13 @@ """ Migration script for job metric plugins. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Numeric, Table, Unicode + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() TEXT_METRIC_MAX_LENGTH = 1023 diff --git a/lib/galaxy/model/migrate/versions/0120_dataset_collections.py b/lib/galaxy/model/migrate/versions/0120_dataset_collections.py index 8d4d89c1f7e..b6c0b533a0f 100644 --- a/lib/galaxy/model/migrate/versions/0120_dataset_collections.py +++ b/lib/galaxy/model/migrate/versions/0120_dataset_collections.py @@ -1,27 +1,22 @@ """ Migration script for tables related to dataset collections. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, DateTime, ForeignKey, Integer, MetaData, Table, TEXT, Unicode + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() DatasetCollection_table = Table( "dataset_collection", metadata, Column( "id", Integer, primary_key=True ), Column( "collection_type", Unicode(255), nullable=False, ), Column( "create_time", DateTime, default=now ), - Column( "update_time", DateTime, default=now, onupdate=now ), -) + Column( "update_time", DateTime, default=now, onupdate=now ) ) HistoryDatasetCollectionAssociation_table = Table( "history_dataset_collection_association", metadata, Column( "id", Integer, primary_key=True ), @@ -32,16 +27,14 @@ HistoryDatasetCollectionAssociation_table = Table( "history_dataset_collection_a Column( "deleted", Boolean, default=False ), Column( "visible", Boolean, default=True ), Column( "copied_from_history_dataset_collection_association_id", Integer, ForeignKey( "history_dataset_collection_association.id" ), nullable=True ), - Column( "implicit_output_name", Unicode(255), nullable=True ), -) + Column( "implicit_output_name", Unicode(255), nullable=True ) ) LibraryDatasetCollectionAssociation_table = Table( "library_dataset_collection_association", metadata, Column( "id", Integer, primary_key=True ), Column( "collection_id", Integer, ForeignKey( "dataset_collection.id" ), index=True ), Column( "name", TrimmedString( 255 ) ), Column( "deleted", Boolean, default=False ), - Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ), -) + Column( "folder_id", Integer, ForeignKey( "library_folder.id" ), index=True ) ) DatasetCollectionElement_table = Table( "dataset_collection_element", metadata, Column( "id", Integer, primary_key=True ), @@ -50,36 +43,31 @@ DatasetCollectionElement_table = Table( "dataset_collection_element", metadata, Column( "ldda_id", Integer, ForeignKey( "library_dataset_dataset_association.id" ), index=True, nullable=True ), Column( "child_collection_id", Integer, ForeignKey( "dataset_collection.id" ), index=True, nullable=True ), Column( "element_index", Integer, nullable=False ), - Column( "element_identifier", Unicode(255), nullable=False ), -) + Column( "element_identifier", Unicode(255), nullable=False ) ) HistoryDatasetCollectionAnnotationAssociation_table = Table( "history_dataset_collection_annotation_association", metadata, Column( "id", Integer, primary_key=True ), Column( "history_dataset_collection_id", Integer, ForeignKey( "history_dataset_collection_association.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "annotation", TEXT ) -) + Column( "annotation", TEXT ) ) LibraryDatasetCollectionAnnotationAssociation_table = Table( "library_dataset_collection_annotation_association", metadata, Column( "id", Integer, primary_key=True ), Column( "library_dataset_collection_id", Integer, ForeignKey( "library_dataset_collection_association.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "annotation", TEXT ) -) + Column( "annotation", TEXT ) ) HistoryDatasetCollectionRatingAssociation_table = Table( "history_dataset_collection_rating_association", metadata, Column( "id", Integer, primary_key=True ), Column( "history_dataset_collection_id", Integer, ForeignKey( "history_dataset_collection_association.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "rating", Integer, index=True) -) + Column( "rating", Integer, index=True) ) LibraryDatasetCollectionRatingAssociation_table = Table( "library_dataset_collection_rating_association", metadata, Column( "id", Integer, primary_key=True ), Column( "library_dataset_collection_id", Integer, ForeignKey( "library_dataset_collection_association.id" ), index=True ), Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), - Column( "rating", Integer, index=True) -) + Column( "rating", Integer, index=True) ) HistoryDatasetCollectionTagAssociation_table = Table( "history_dataset_collection_tag_association", metadata, Column( "id", Integer, primary_key=True ), @@ -88,8 +76,7 @@ HistoryDatasetCollectionTagAssociation_table = Table( "history_dataset_collectio Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "user_tname", Unicode(255), index=True), Column( "value", Unicode(255), index=True), - Column( "user_value", Unicode(255), index=True) -) + Column( "user_value", Unicode(255), index=True) ) LibraryDatasetCollectionTagAssociation_table = Table( "library_dataset_collection_tag_association", metadata, Column( "id", Integer, primary_key=True ), @@ -98,22 +85,19 @@ LibraryDatasetCollectionTagAssociation_table = Table( "library_dataset_collectio Column( "user_id", Integer, ForeignKey( "galaxy_user.id" ), index=True ), Column( "user_tname", Unicode(255), index=True), Column( "value", Unicode(255), index=True), - Column( "user_value", Unicode(255), index=True) -) + Column( "user_value", Unicode(255), index=True) ) JobToInputDatasetCollectionAssociation_table = Table( "job_to_input_dataset_collection", metadata, Column( "id", Integer, primary_key=True ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), Column( "dataset_collection_id", Integer, ForeignKey( "history_dataset_collection_association.id" ), index=True ), - Column( "name", Unicode(255) ) -) + Column( "name", Unicode(255) ) ) JobToOutputDatasetCollectionAssociation_table = Table( "job_to_output_dataset_collection", metadata, Column( "id", Integer, primary_key=True ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), Column( "dataset_collection_id", Integer, ForeignKey( "history_dataset_collection_association.id" ), index=True ), - Column( "name", Unicode(255) ) -) + Column( "name", Unicode(255) ) ) ImplicitlyCreatedDatasetCollectionInput_table = Table( "implicitly_created_dataset_collection_inputs", metadata, Column( "id", Integer, primary_key=True ), diff --git a/lib/galaxy/model/migrate/versions/0121_workflow_uuids.py b/lib/galaxy/model/migrate/versions/0121_workflow_uuids.py index 19f0c8e39f8..86df373d47d 100644 --- a/lib/galaxy/model/migrate/versions/0121_workflow_uuids.py +++ b/lib/galaxy/model/migrate/versions/0121_workflow_uuids.py @@ -1,24 +1,20 @@ """ Add UUIDs to workflows """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import UUIDType, TrimmedString - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, MetaData, Table + +from galaxy.model.custom_types import UUIDType + +log = logging.getLogger( __name__ ) metadata = MetaData() - """ -Because both workflow and job requests can be determined +Because both workflow and job requests can be determined based the a fixed data structure, their IDs are based on -hashing the data structure +hashing the data structure """ workflow_uuid_column = Column( "uuid", UUIDType, nullable=True ) @@ -26,6 +22,7 @@ workflow_uuid_column = Column( "uuid", UUIDType, nullable=True ) def display_migration_details(): print "This migration script adds a UUID column to workflows" + def upgrade(migrate_engine): print __doc__ metadata.bind = migrate_engine @@ -41,6 +38,7 @@ def upgrade(migrate_engine): log.error( "Adding column 'uuid' to workflow table failed: %s" % str( e ) ) return + def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() @@ -52,4 +50,3 @@ def downgrade(migrate_engine): workflow_uuid.drop() except Exception, e: log.debug( "Dropping 'uuid' column from workflow table failed: %s" % ( str( e ) ) ) - diff --git a/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py b/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py index 653220fad22..5dbc356387e 100644 --- a/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py +++ b/lib/galaxy/model/migrate/versions/0123_add_workflow_request_tables.py @@ -1,18 +1,15 @@ """ Migration script for workflow request tables. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, String, Table, TEXT, Unicode + +from galaxy.model.custom_types import JSONType, TrimmedString, UUIDType + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() @@ -93,7 +90,7 @@ def upgrade(migrate_engine): WorkflowInvocationStepAction_column = Column( "action", JSONType, nullable=True ) __add_column( WorkflowInvocationStepAction_column, "workflow_invocation_step", metadata ) - + def downgrade(migrate_engine): metadata.bind = migrate_engine diff --git a/lib/galaxy/model/migrate/versions/0124_job_state_history.py b/lib/galaxy/model/migrate/versions/0124_job_state_history.py index 38796fac69d..35189f61f59 100644 --- a/lib/galaxy/model/migrate/versions/0124_job_state_history.py +++ b/lib/galaxy/model/migrate/versions/0124_job_state_history.py @@ -1,19 +1,15 @@ """ Migration script for the job state history table """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() JobStateHistory_table = Table( "job_state_history", metadata, @@ -22,8 +18,8 @@ JobStateHistory_table = Table( "job_state_history", metadata, Column( "update_time", DateTime, default=now, onupdate=now ), Column( "job_id", Integer, ForeignKey( "job.id" ), index=True ), Column( "state", String( 64 ), index=True ), - Column( "info", TrimmedString( 255 ) ) -) + Column( "info", TrimmedString( 255 ) ) ) + def upgrade(migrate_engine): metadata.bind = migrate_engine diff --git a/lib/galaxy/model/migrate/versions/0125_workflow_step_tracking.py b/lib/galaxy/model/migrate/versions/0125_workflow_step_tracking.py index 5564c00ce78..2aebdc78c09 100644 --- a/lib/galaxy/model/migrate/versions/0125_workflow_step_tracking.py +++ b/lib/galaxy/model/migrate/versions/0125_workflow_step_tracking.py @@ -1,18 +1,15 @@ """ Migration script to enhance workflow step usability by adding labels and UUIDs. """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, MetaData, Table + +from galaxy.model.custom_types import TrimmedString, UUIDType + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() diff --git a/lib/galaxy/model/migrate/versions/0126_password_reset.py b/lib/galaxy/model/migrate/versions/0126_password_reset.py index 96be92b0258..63cacb803a3 100644 --- a/lib/galaxy/model/migrate/versions/0126_password_reset.py +++ b/lib/galaxy/model/migrate/versions/0126_password_reset.py @@ -1,19 +1,13 @@ """ Migration script for the password reset table """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, String, Table + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() PasswordResetToken_table = Table("password_reset_token", metadata, diff --git a/lib/galaxy/model/migrate/versions/0127_output_collection_adjustments.py b/lib/galaxy/model/migrate/versions/0127_output_collection_adjustments.py index 76673edeafc..65355aba28d 100644 --- a/lib/galaxy/model/migrate/versions/0127_output_collection_adjustments.py +++ b/lib/galaxy/model/migrate/versions/0127_output_collection_adjustments.py @@ -1,19 +1,15 @@ """ Migration script updating collections tables for output collections. """ - -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, ForeignKey, Integer, MetaData, Table, TEXT, Unicode + +from galaxy.model.custom_types import TrimmedString + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() JobToImplicitOutputDatasetCollectionAssociation_table = Table( diff --git a/lib/galaxy/model/migrate/versions/0128_session_timeout.py b/lib/galaxy/model/migrate/versions/0128_session_timeout.py index cf92c0f0e27..fc63674a88b 100644 --- a/lib/galaxy/model/migrate/versions/0128_session_timeout.py +++ b/lib/galaxy/model/migrate/versions/0128_session_timeout.py @@ -1,18 +1,13 @@ """ Migration script to add session update time (used for timeouts) """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Column, DateTime, MetaData, Table + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() diff --git a/lib/galaxy/model/migrate/versions/0129_job_external_output_metadata_validity.py b/lib/galaxy/model/migrate/versions/0129_job_external_output_metadata_validity.py index f24a99f61b2..b158ee82224 100644 --- a/lib/galaxy/model/migrate/versions/0129_job_external_output_metadata_validity.py +++ b/lib/galaxy/model/migrate/versions/0129_job_external_output_metadata_validity.py @@ -1,18 +1,13 @@ """ Migration script to allow invalidation of job external output metadata temp files """ -from sqlalchemy import * -from sqlalchemy.orm import * -from migrate import * -from migrate.changeset import * -from galaxy.model.custom_types import * - import datetime -now = datetime.datetime.utcnow - import logging -log = logging.getLogger( __name__ ) +from sqlalchemy import Boolean, Column, MetaData, Table + +now = datetime.datetime.utcnow +log = logging.getLogger( __name__ ) metadata = MetaData() @@ -29,7 +24,7 @@ def downgrade(migrate_engine): metadata.bind = migrate_engine metadata.reflect() - __drop_column( isvalid_column, "job_external_output_metadata", metadata ) + __drop_column( "is_valid", "job_external_output_metadata", metadata ) def __add_column(column, table_name, metadata, **kwds):