diff --git a/lib/galaxy/model/migrate/versions/0169_add_missing_indexes.py b/lib/galaxy/model/migrate/versions/0169_add_missing_indexes.py new file mode 100644 index 00000000000..a9b56294e59 --- /dev/null +++ b/lib/galaxy/model/migrate/versions/0169_add_missing_indexes.py @@ -0,0 +1,130 @@ +""" +Migration script to create missing indexes. Adding new columns to existing tables via SQLAlchemy does not create the index, even if the column definition includes index=True. +""" + +import logging + +from sqlalchemy import MetaData + +from galaxy.model.migrate.versions.util import ( + add_index, + drop_index +) + +log = logging.getLogger(__name__) +metadata = MetaData() + +indexes = [ + [ + "ix_galaxy_user_activation_token", + "galaxy_user", + "activation_token" + ], + [ + "ix_workflow_step_dynamic_tool_id", + "workflow_step", + "dynamic_tool_id" + ], + [ + "ix_history_dataset_association_version", + "history_dataset_association", + "version" + ], + [ + "ix_workflow_invocation_scheduler", + "workflow_invocation", + "scheduler" + ], + [ + "ix_page_slug", + "page", + "slug" + ], + [ + "ix_workflow_invocation_state", + "workflow_invocation", + "state" + ], + [ + "ix_history_dataset_collection_association_implicit_collection_jobs_id", + "history_dataset_collection_association", + "implicit_collection_jobs_id" + ], + [ + "ix_workflow_step_subworkflow_id", + "workflow_step", + "subworkflow_id" + ], + [ + "ix_dynamic_tool_update_time", + "dynamic_tool", + "update_time" + ], + [ + "ix_library_dataset_dataset_association_extended_metadata_id", + "library_dataset_dataset_association", + "extended_metadata_id" + ], + [ + "ix_workflow_invocation_step_implicit_collection_jobs_id", + "workflow_invocation_step", + "implicit_collection_jobs_id" + ], + [ + "ix_workflow_invocation_step_state", + "workflow_invocation_step", + "state" + ], + [ + "ix_workflow_invocation_history_id", + "workflow_invocation", + "history_id" + ], + [ + "ix_workflow_parent_workflow_id", + "workflow", + "parent_workflow_id" + ], + [ + "ix_metadata_file_uuid", + "metadata_file", + "uuid" + ], + [ + "ix_history_dataset_collection_association_job_id", + "history_dataset_collection_association", + "job_id" + ], + [ + "ix_galaxy_user_active", + "galaxy_user", + "active" + ], + [ + "ix_job_dynamic_tool_id", + "job", + "dynamic_tool_id" + ], + [ + "ix_history_dataset_association_extended_metadata_id", + "history_dataset_association", + "extended_metadata_id" + ] +] + + +def upgrade(migrate_engine): + print(__doc__) + metadata.bind = migrate_engine + metadata.reflect() + + for ix, table, col in indexes: + add_index(ix, table, col, metadata) + + +def downgrade(migrate_engine): + metadata.bind = migrate_engine + metadata.reflect() + + for ix, table, col in indexes: + drop_index(ix, table, col, metadata) diff --git a/lib/galaxy/model/migrate/versions/util.py b/lib/galaxy/model/migrate/versions/util.py index 273939147b2..9b9334ba9e9 100644 --- a/lib/galaxy/model/migrate/versions/util.py +++ b/lib/galaxy/model/migrate/versions/util.py @@ -1,3 +1,4 @@ +import hashlib import logging from sqlalchemy import ( @@ -47,6 +48,14 @@ def localtimestamp(migrate_engine): raise Exception('Unable to convert data for unknown database type: %s' % migrate_engine.name) +def truncate_index_name(index_name, engine): + # does what sqlalchemy does, see https://github.com/sqlalchemy/sqlalchemy/blob/8455a11bcc23e97afe666873cd872b0f204848d8/lib/sqlalchemy/sql/compiler.py#L4696 + max_index_name_length = engine.dialect.max_index_name_length or engine.dialect.max_identifier_length + if len(index_name) > max_index_name_length: + suffix = hashlib.md5(index_name.encode('utf-8')).hexdigest()[-4:] + index_name = "{trunc}_{suffix}".format(trunc=index_name[0 : max_index_name_length - 8], suffix=suffix) + return index_name + def create_table(table): try: table.create() @@ -135,10 +144,14 @@ def add_index(index_name, table, column_name, metadata=None, **kwds): :param metadata: Needed only if ``table`` is a table name :type metadata: :class:`Metadata` """ + if len(index_name) > 63 and metadata.bind.name in ('postgres', 'postgresql'): + suffix = hashlib.md5(index_name.encode('utf-8')).hexdigest()[-4:] + try: if not isinstance(table, Table): assert metadata is not None table = Table(table, metadata, autoload=True) + index_name = truncate_index_name(index_name, table.metadata.bind) if index_name not in [ix.name for ix in table.indexes]: column = table.c[column_name] # MySQL cannot index a TEXT/BLOB column without specifying mysql_length @@ -168,6 +181,7 @@ def drop_index(index, table, column_name=None, metadata=None): if not isinstance(table, Table): assert metadata is not None table = Table(table, metadata, autoload=True) + index = truncate_index_name(index, table.metadata.bind) if index in [ix.name for ix in table.indexes]: index = Index(index, table.c[column_name]) else: