opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-24 02:04:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor migrations dropping indexes

Browse Source
This commit is contained in:
Nicola Soranzo
2019-04-13 03:28:54 +01:00
parent 262d9ba0ea
commit 3a9991ec95
8 changed files with 78 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/model/migrate/versions/0003_security_and_libraries.py
+4 -22
View File
@@ -11,6 +11,8 @@ from sqlalchemy.exc import NoSuchTableError
from galaxy.model.custom_types import JSONType, MetadataType, TrimmedString
from galaxy.model.migrate.versions.util import (
add_column,
add_index,
drop_index,
engine_false,
localtimestamp,
nextval
@@ -369,17 +371,7 @@ def upgrade(migrate_engine):
except Exception:
log.exception("Adding column 'importable' to stored_workflow table failed.")
# Create an index on the Job.state column - changeset 2192
try:
Job_table = Table("job", metadata, autoload=True)
except NoSuchTableError:
Job_table = None
log.debug("Failed loading table job")
if Job_table is not None:
try:
i = Index('ix_job_state', Job_table.c.state)
i.create()
except Exception:
log.exception("Adding index to job.state column failed.")
add_index('ix_job_state', 'job', 'state', metadata)
# Add all of the new tables above
metadata.create_all()
# Add 1 foreign key constraint to the history_dataset_association table
@@ -678,17 +670,7 @@ def downgrade(migrate_engine):
except Exception:
log.exception("Dropping library_item_info_template table failed.")
# Drop the index on the Job.state column - changeset 2192
try:
Job_table = Table("job", metadata, autoload=True)
except NoSuchTableError:
Job_table = None
log.debug("Failed loading table job")
if Job_table is not None:
try:
i = Index('ix_job_state', Job_table.c.state)
i.drop()
except Exception:
log.exception("Dropping index from job.state column failed.")
drop_index('ix_job_state', 'job', 'state', metadata)
# Drop 1 column from the stored_workflow table - changeset 2328
try:
StoredWorkflow_table = Table("stored_workflow", metadata, autoload=True)
lib/galaxy/model/migrate/versions/0006_change_qual_datatype.py
+1 -15
View File
@@ -7,7 +7,7 @@ from __future__ import print_function
import logging
import sys
from sqlalchemy import Index, MetaData, Table
from sqlalchemy import MetaData
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
@@ -23,15 +23,7 @@ metadata = MetaData()
def upgrade(migrate_engine):
print(__doc__)
metadata.bind = migrate_engine
HistoryDatasetAssociation_table = Table("history_dataset_association", metadata, autoload=True)
# Load existing tables
metadata.reflect()
# Add 2 indexes to the galaxy_user table
i = Index('ix_hda_extension', HistoryDatasetAssociation_table.c.extension)
try:
i.create()
except Exception:
log.exception("Adding index 'ix_hda_extension' to history_dataset_association table failed.")
# Set the default data in the galaxy_user table, but only for null values
cmd = "UPDATE history_dataset_association SET extension = 'qual454' WHERE extension = 'qual' and peek like \'>%%\'"
@@ -44,13 +36,7 @@ def upgrade(migrate_engine):
migrate_engine.execute(cmd)
except Exception:
log.exception("Resetting extension qual to qualsolexa in history_dataset_association failed.")
# Add 1 index to the history_dataset_association table
try:
i.drop()
except Exception:
log.exception("Dropping index 'ix_hda_extension' to history_dataset_association table failed.")
def downgrade(migrate_engine):
metadata.bind = migrate_engine
pass
lib/galaxy/model/migrate/versions/0011_v0010_mysql_index_fix.py
+11 -25
View File
@@ -10,10 +10,12 @@ import datetime
import logging
import sys
from sqlalchemy import Column, DateTime, ForeignKey, Index, Integer, MetaData, Table
from sqlalchemy import MetaData
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
from galaxy.model.migrate.versions.util import (
add_index,
drop_index,
)
now = datetime.datetime.utcnow
log = logging.getLogger(__name__)
@@ -25,35 +27,19 @@ handler.setFormatter(formatter)
log.addHandler(handler)
metadata = MetaData()
HistoryDatasetAssociationDisplayAtAuthorization_table = Table("history_dataset_association_display_at_authorization", metadata,
Column("id", Integer, primary_key=True),
Column("create_time", DateTime, default=now),
Column("update_time", DateTime, index=True, default=now, onupdate=now),
Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
Column("site", TrimmedString(255)))
def upgrade(migrate_engine):
print(__doc__)
metadata.bind = migrate_engine
metadata.reflect()
if migrate_engine.name == 'mysql':
# Load existing tables
metadata.reflect()
i = Index("ix_hdadaa_history_dataset_association_id", HistoryDatasetAssociationDisplayAtAuthorization_table.c.history_dataset_association_id)
try:
i.create()
except Exception:
log.exception("Adding index 'ix_hdadaa_history_dataset_association_id' to table 'history_dataset_association_display_at_authorization' table failed.")
add_index('ix_hdadaa_history_dataset_association_id', 'history_dataset_association_display_at_authorization', 'history_dataset_association_id', metadata)
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
if migrate_engine.name == 'mysql':
# Load existing tables
metadata.reflect()
i = Index("ix_hdadaa_history_dataset_association_id", HistoryDatasetAssociationDisplayAtAuthorization_table.c.history_dataset_association_id)
try:
i.drop()
except Exception:
log.exception("Removing index 'ix_hdadaa_history_dataset_association_id' from table 'history_dataset_association_display_at_authorization' table failed.")
drop_index('ix_hdadaa_history_dataset_association_id', 'history_dataset_association_display_at_authorization', 'history_dataset_association_id', metadata)
lib/galaxy/model/migrate/versions/0016_v0015_mysql_index_fix.py
+10 -22
View File
@@ -7,39 +7,27 @@ from __future__ import print_function
import logging
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table
from sqlalchemy import MetaData
# Need our custom types, but don't import anything else from model
from galaxy.model.custom_types import TrimmedString
from galaxy.model.migrate.versions.util import (
add_index,
drop_index
)
log = logging.getLogger(__name__)
metadata = MetaData()
HistoryDatasetAssociationTagAssociation_table = Table("history_dataset_association_tag_association", metadata,
Column("history_dataset_association_id", Integer, ForeignKey("history_dataset_association.id"), index=True),
Column("tag_id", Integer, ForeignKey("tag.id"), index=True),
Column("user_tname", TrimmedString(255), index=True),
Column("value", TrimmedString(255), index=True),
Column("user_value", TrimmedString(255), index=True))
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print(__doc__)
metadata.bind = migrate_engine
metadata.reflect()
i = Index("ix_hda_ta_history_dataset_association_id", HistoryDatasetAssociationTagAssociation_table.c.history_dataset_association_id)
try:
i.create()
except Exception:
log.exception("Adding index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed.")
add_index('ix_hda_ta_history_dataset_association_id', 'history_dataset_association_tag_association', 'history_dataset_association_id', metadata)
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
i = Index("ix_hda_ta_history_dataset_association_id", HistoryDatasetAssociationTagAssociation_table.c.history_dataset_association_id)
try:
i.drop()
except Exception:
log.exception("Removing index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed.")
drop_index('ix_hda_ta_history_dataset_association_id', 'history_dataset_association_tag_association', 'history_dataset_association_id', metadata)
lib/galaxy/model/migrate/versions/0020_library_upload_job.py
+10 -7
View File
@@ -10,13 +10,21 @@ from __future__ import print_function
import logging
import sys
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, String, Table
from sqlalchemy import (
Column,
ForeignKey,
Integer,
MetaData,
String,
Table
)
from galaxy.model.migrate.versions.util import (
add_column,
add_index,
create_table,
drop_column,
drop_index,
drop_table
)
@@ -62,12 +70,7 @@ def downgrade(migrate_engine):
metadata.reflect()
# Drop the ix_dataset_state index
Dataset_table = Table("dataset", metadata, autoload=True)
i = Index("ix_dataset_state", Dataset_table.c.state)
try:
i.drop()
except Exception:
log.exception("Dropping index 'ix_dataset_state' from dataset table failed.")
drop_index('ix_dataset_state', 'dataset', 'state', metadata)
# Drop the library_folder_id column
drop_column('library_folder_id', 'job', metadata)
lib/galaxy/model/migrate/versions/0053_item_ratings.py
+13 -8
View File
@@ -5,15 +5,24 @@ from __future__ import print_function
import logging
from sqlalchemy import Column, ForeignKey, Index, Integer, MetaData, Table
from sqlalchemy import (
Column,
ForeignKey,
Integer,
MetaData,
Table
)
from galaxy.model.migrate.versions.util import create_table, drop_table
from galaxy.model.migrate.versions.util import (
add_index,
create_table,
drop_table
)
log = logging.getLogger(__name__)
metadata = MetaData()
# Rating tables.
HistoryRatingAssociation_table = Table("history_rating_association", metadata,
Column("id", Integer, primary_key=True),
Column("history_id", Integer, ForeignKey("history.id"), index=True),
@@ -59,11 +68,7 @@ def upgrade(migrate_engine):
# MySQL cannot handle long index names; when we see this error, create the index name manually.
if migrate_engine.name == 'mysql' and \
str(e).lower().find("identifier name 'ix_history_dataset_association_rating_association_history_dataset_association_id' is too long"):
i = Index("ix_hda_rating_association_hda_id", HistoryDatasetAssociationRatingAssociation_table.c.history_dataset_association_id)
try:
i.create()
except Exception:
log.exception("Adding index 'ix_hda_rating_association_hda_id' to table 'history_dataset_association_rating_association' table failed.")
add_index('ix_hda_rating_association_hda_id', HistoryDatasetAssociationRatingAssociation_table, 'history_dataset_association_id')
else:
log.exception("Creating history_dataset_association_rating_association table failed.")
lib/galaxy/model/migrate/versions/0106_add_missing_indexes.py
+6 -8
View File
@@ -5,9 +5,12 @@ from __future__ import print_function
import logging
from sqlalchemy import Index, MetaData, Table
from sqlalchemy import MetaData
from galaxy.model.migrate.versions.util import add_index
from galaxy.model.migrate.versions.util import (
add_index,
drop_index
)
log = logging.getLogger(__name__)
log.setLevel(logging.DEBUG)
@@ -64,10 +67,5 @@ def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
# Drop indexes
for ix, table, col in indexes:
try:
t = Table(table, metadata, autoload=True)
Index(ix, t.c[col]).drop()
except Exception:
log.exception("Unable to drop index '%s'.", ix)
drop_index(ix, table, col, metadata)
lib/galaxy/model/migrate/versions/util.py
+23 -2
View File
@@ -78,7 +78,7 @@ def add_column(column, table, metadata=None, **kwds):
def alter_column(column_name, table, metadata=None, **kwds):
"""
:param table: Table to drop the column from
:param table: Table to alter
:type table: :class:`Table` or str
:param metadata: Needed only if ``table`` is a table name
@@ -114,7 +114,7 @@ def drop_column(column_name, table, metadata=None):
def add_index(index_name, table, column_name, metadata=None):
"""
:param table: Table to drop the column from
:param table: Table to add the index to
:type table: :class:`Table` or str
:param metadata: Needed only if ``table`` is a table name
@@ -131,3 +131,24 @@ def add_index(index_name, table, column_name, metadata=None):
log.debug("Index '%s' on column '%s' in table '%s' already exists.", index_name, column_name, table)
except Exception:
log.exception("Adding index '%s' on column '%s' to table '%s' failed.", index_name, column_name, table)
def drop_index(index_name, table, column_name, metadata=None):
"""
:param table: Table to drop the index from
:type table: :class:`Table` or str
:param metadata: Needed only if ``table`` is a table name
:type metadata: :class:`Metadata`
"""
try:
if not isinstance(table, Table):
assert metadata is not None
table = Table(table, metadata, autoload=True)
if index_name in [ix.name for ix in table.indexes]:
index = Index(index_name, table.c[column_name])
index.drop()
else:
log.debug("Index '%s' in table '%s' does not exist.", index_name, table)
except Exception:
log.exception("Dropping index '%s' from table '%s' failed", index_name, table)