Add missing indexes

lib/galaxy/model/migrate/versions/0169_add_missing_indexes.py

@@ -0,0 +1,130 @@
+"""
+Migration script to create missing indexes.  Adding new columns to existing tables via SQLAlchemy does not create the index, even if the column definition includes index=True.
+"""
+
+import logging
+
+from sqlalchemy import MetaData
+
+from galaxy.model.migrate.versions.util import (
+    add_index,
+    drop_index
+)
+
+log = logging.getLogger(__name__)
+metadata = MetaData()
+
+indexes = [
+    [
+        "ix_galaxy_user_activation_token",
+        "galaxy_user",
+        "activation_token"
+    ],
+    [
+        "ix_workflow_step_dynamic_tool_id",
+        "workflow_step",
+        "dynamic_tool_id"
+    ],
+    [
+        "ix_history_dataset_association_version",
+        "history_dataset_association",
+        "version"
+    ],
+    [
+        "ix_workflow_invocation_scheduler",
+        "workflow_invocation",
+        "scheduler"
+    ],
+    [
+        "ix_page_slug",
+        "page",
+        "slug"
+    ],
+    [
+        "ix_workflow_invocation_state",
+        "workflow_invocation",
+        "state"
+    ],
+    [
+        "ix_history_dataset_collection_association_implicit_collection_jobs_id",
+        "history_dataset_collection_association",
+        "implicit_collection_jobs_id"
+    ],
+    [
+        "ix_workflow_step_subworkflow_id",
+        "workflow_step",
+        "subworkflow_id"
+    ],
+    [
+        "ix_dynamic_tool_update_time",
+        "dynamic_tool",
+        "update_time"
+    ],
+    [
+        "ix_library_dataset_dataset_association_extended_metadata_id",
+        "library_dataset_dataset_association",
+        "extended_metadata_id"
+    ],
+    [
+        "ix_workflow_invocation_step_implicit_collection_jobs_id",
+        "workflow_invocation_step",
+        "implicit_collection_jobs_id"
+    ],
+    [
+        "ix_workflow_invocation_step_state",
+        "workflow_invocation_step",
+        "state"
+    ],
+    [
+        "ix_workflow_invocation_history_id",
+        "workflow_invocation",
+        "history_id"
+    ],
+    [
+        "ix_workflow_parent_workflow_id",
+        "workflow",
+        "parent_workflow_id"
+    ],
+    [
+        "ix_metadata_file_uuid",
+        "metadata_file",
+        "uuid"
+    ],
+    [
+        "ix_history_dataset_collection_association_job_id",
+        "history_dataset_collection_association",
+        "job_id"
+    ],
+    [
+        "ix_galaxy_user_active",
+        "galaxy_user",
+        "active"
+    ],
+    [
+        "ix_job_dynamic_tool_id",
+        "job",
+        "dynamic_tool_id"
+    ],
+    [
+        "ix_history_dataset_association_extended_metadata_id",
+        "history_dataset_association",
+        "extended_metadata_id"
+    ]
+]
+
+
+def upgrade(migrate_engine):
+    print(__doc__)
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    for ix, table, col in indexes:
+        add_index(ix, table, col, metadata)
+
+
+def downgrade(migrate_engine):
+    metadata.bind = migrate_engine
+    metadata.reflect()
+
+    for ix, table, col in indexes:
+        drop_index(ix, table, col, metadata)

lib/galaxy/model/migrate/versions/util.py

@@ -1,3 +1,4 @@
+import hashlib
 import logging
 
 from sqlalchemy import (
@@ -47,6 +48,14 @@ def localtimestamp(migrate_engine):
         raise Exception('Unable to convert data for unknown database type: %s' % migrate_engine.name)
 
 
+def truncate_index_name(index_name, engine):
+    # does what sqlalchemy does, see https://github.com/sqlalchemy/sqlalchemy/blob/8455a11bcc23e97afe666873cd872b0f204848d8/lib/sqlalchemy/sql/compiler.py#L4696
+    max_index_name_length = engine.dialect.max_index_name_length or engine.dialect.max_identifier_length
+    if len(index_name) > max_index_name_length:
+        suffix = hashlib.md5(index_name.encode('utf-8')).hexdigest()[-4:]
+        index_name = "{trunc}_{suffix}".format(trunc=index_name[0 : max_index_name_length - 8], suffix=suffix)
+    return index_name
+
 def create_table(table):
     try:
         table.create()
@@ -135,10 +144,14 @@ def add_index(index_name, table, column_name, metadata=None, **kwds):
     :param metadata: Needed only if ``table`` is a table name
     :type metadata: :class:`Metadata`
     """
+    if len(index_name) > 63 and metadata.bind.name in ('postgres', 'postgresql'):
+        suffix = hashlib.md5(index_name.encode('utf-8')).hexdigest()[-4:]
+
     try:
         if not isinstance(table, Table):
             assert metadata is not None
             table = Table(table, metadata, autoload=True)
+        index_name = truncate_index_name(index_name, table.metadata.bind)
         if index_name not in [ix.name for ix in table.indexes]:
             column = table.c[column_name]
             # MySQL cannot index a TEXT/BLOB column without specifying mysql_length
@@ -168,6 +181,7 @@ def drop_index(index, table, column_name=None, metadata=None):
             if not isinstance(table, Table):
                 assert metadata is not None
                 table = Table(table, metadata, autoload=True)
+            index = truncate_index_name(index, table.metadata.bind)
             if index in [ix.name for ix in table.indexes]:
                 index = Index(index, table.c[column_name])
             else: