opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-19 10:51:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Apply black, isort

Browse Source
This commit is contained in:
Sergey Golitsynskiy
2022-03-01 14:44:38 -05:00
parent 8a26c3c7af
commit ebe36de5fc
27 changed files with 379 additions and 342 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/config/__init__.py
+15 -8
View File
@@ -727,7 +727,7 @@ class GalaxyAppConfiguration(BaseAppConfiguration, CommonConfigurationMixin):
db_path = self._in_data_dir("universe.sqlite")
self.database_connection = f"sqlite:///{db_path}?isolation_level=IMMEDIATE"
self.database_engine_options = get_database_engine_options(kwargs)
self.database_encoding = kwargs.get('database_encoding') # Create new databases with this encoding
self.database_encoding = kwargs.get("database_encoding") # Create new databases with this encoding
self.thread_local_log = None
if self.enable_per_request_sql_debugging:
self.thread_local_log = threading.local()
@@ -1487,7 +1487,7 @@ class ConfiguresGalaxyMixin:
trace_logger,
self.config.slow_query_log_threshold,
self.config.thread_local_log,
self.config.database_log_query_counts
self.config.database_log_query_counts,
)
install_engine = None
if not combined_install_database:
@@ -1499,6 +1499,7 @@ class ConfiguresGalaxyMixin:
# TODO this block doesn't seem to belong in this method
if getattr(self.config, "max_metadata_value_size", None):
from galaxy.model import custom_types
custom_types.MAX_METADATA_VALUE_SIZE = self.config.max_metadata_value_size
db_url = get_database_url(self.config)
@@ -1518,7 +1519,7 @@ class ConfiguresGalaxyMixin:
self.config.use_pbkdf2,
engine,
combined_install_database,
self.config.thread_local_log
self.config.thread_local_log,
)
if combined_install_database:
@@ -1526,20 +1527,26 @@ class ConfiguresGalaxyMixin:
self.install_model = self.model
else:
from galaxy.model.tool_shed_install import mapping as install_mapping
self.install_model = install_mapping.configure_model_mapping(install_engine)
log.info(f"Install database using its own connection {install_db_url}")
def _verify_databases(self, engine, install_engine, combined_install_database):
from galaxy.model.migrations import verify_databases
install_template, install_encoding = None, None
if not combined_install_database: # Otherwise these options are not used.
install_template = getattr(self.config, 'install_database_template', None)
install_encoding = getattr(self.config, 'install_database_encoding', None)
install_template = getattr(self.config, "install_database_template", None)
install_encoding = getattr(self.config, "install_database_encoding", None)
verify_databases(
engine, self.config.database_template, self.config.database_encoding,
install_engine, install_template, install_encoding,
self.config.database_auto_migrate
engine,
self.config.database_template,
self.config.database_encoding,
install_engine,
install_template,
install_encoding,
self.config.database_auto_migrate,
)
def _configure_signal_handlers(self, handlers):
lib/galaxy/model/database_utils.py
+1 -1
View File
@@ -129,4 +129,4 @@ def is_one_database(db1_url: str, db2_url: Optional[str]):
"""
# TODO: Consider more aggressive check here that this is not the same
# database file under the hood.
return not(db1_url and db2_url and db1_url != db2_url)
return not (db1_url and db2_url and db1_url != db2_url)
lib/galaxy/model/mapping.py
+25 -7
View File
@@ -25,13 +25,30 @@ class GalaxyModelMapping(SharedModelMapping):
GalaxySession: Type
def init(file_path, url, engine_options=None, create_tables=False, map_install_models=False,
database_query_profiling_proxy=False, object_store=None, trace_logger=None, use_pbkdf2=True,
slow_query_log_threshold=0, thread_local_log: Optional[local] = None, log_query_counts=False) -> GalaxyModelMapping:
def init(
file_path,
url,
engine_options=None,
create_tables=False,
map_install_models=False,
database_query_profiling_proxy=False,
object_store=None,
trace_logger=None,
use_pbkdf2=True,
slow_query_log_threshold=0,
thread_local_log: Optional[local] = None,
log_query_counts=False,
) -> GalaxyModelMapping:
# Build engine
engine = build_engine(
url, engine_options, database_query_profiling_proxy, trace_logger, slow_query_log_threshold,
thread_local_log=thread_local_log, log_query_counts=log_query_counts)
url,
engine_options,
database_query_profiling_proxy,
trace_logger,
slow_query_log_threshold,
thread_local_log=thread_local_log,
log_query_counts=log_query_counts,
)
# Create tables if needed
if create_tables:
@@ -39,11 +56,11 @@ def init(file_path, url, engine_options=None, create_tables=False, map_install_m
create_additional_database_objects(engine)
if map_install_models:
from galaxy.model.tool_shed_install import mapping as install_mapping # noqa: F401
install_mapping.create_database_objects(engine)
# Configure model, build ModelMapping
return configure_model_mapping(
file_path, object_store, use_pbkdf2, engine, map_install_models, thread_local_log)
return configure_model_mapping(file_path, object_store, use_pbkdf2, engine, map_install_models, thread_local_log)
def create_additional_database_objects(engine):
@@ -73,6 +90,7 @@ def _build_model_mapping(engine, map_install_models, thread_local_log):
model_modules = [model]
if map_install_models:
from galaxy.model import tool_shed_install
model_modules.append(tool_shed_install)
model_mapping = GalaxyModelMapping(model_modules, engine=engine)
lib/galaxy/model/migrate/versions/0165_add_content_update_time.py
+14 -3
View File
@@ -5,11 +5,22 @@ to update history.update_time when contents are changed.
import logging
from sqlalchemy import Column, DateTime, MetaData, Table
from sqlalchemy import (
Column,
DateTime,
MetaData,
Table,
)
from galaxy.model.migrate.versions.util import add_column, drop_column
from galaxy.model.migrate.versions.util import (
add_column,
drop_column,
)
from galaxy.model.orm.now import now
from galaxy.model.triggers.history_update_time_field import drop_timestamp_triggers, install_timestamp_triggers
from galaxy.model.triggers.history_update_time_field import (
drop_timestamp_triggers,
install_timestamp_triggers,
)
log = logging.getLogger(__name__)
metadata = MetaData()
lib/galaxy/model/migrate/versions/0175_history_audit.py
+11 -5
View File
@@ -5,16 +5,22 @@ Add history audit table and associated triggers
import datetime
import logging
from sqlalchemy import Column, DateTime, ForeignKey, Integer, MetaData, PrimaryKeyConstraint, Table
from sqlalchemy import (
Column,
DateTime,
ForeignKey,
Integer,
MetaData,
PrimaryKeyConstraint,
Table,
)
from galaxy.model.migrate.versions.util import (
create_table,
drop_table,
)
from galaxy.model.triggers import (
history_update_time_field as old_triggers, # rollback to old ones
update_audit_table as new_triggers, # install me
)
from galaxy.model.triggers import history_update_time_field as old_triggers # rollback to old ones
from galaxy.model.triggers import update_audit_table as new_triggers # install me
log = logging.getLogger(__name__)
now = datetime.datetime.utcnow
lib/galaxy/model/migrations/__init__.py
+50 -49
View File
@@ -29,19 +29,22 @@ from sqlalchemy.engine import (
)
from galaxy.model import Base as gxy_base
from galaxy.model.database_utils import create_database, database_exists
from galaxy.model.database_utils import (
create_database,
database_exists,
)
from galaxy.model.mapping import create_additional_database_objects
from galaxy.model.tool_shed_install import Base as tsi_base
ModelId = NewType('ModelId', str)
ModelId = NewType("ModelId", str)
# These identifiers are used throughout the migrations system to distinquish
# between the two models; they refer to version directories, branch labels, etc.
# (if you rename these, you need to rename branch labels in alembic version directories)
GXY = ModelId('gxy') # galaxy model identifier
TSI = ModelId('tsi') # tool_shed_install model identifier
GXY = ModelId("gxy") # galaxy model identifier
TSI = ModelId("tsi") # tool_shed_install model identifier
ALEMBIC_TABLE = 'alembic_version'
SQLALCHEMYMIGRATE_TABLE = 'migrate_version'
ALEMBIC_TABLE = "alembic_version"
SQLALCHEMYMIGRATE_TABLE = "migrate_version"
SQLALCHEMYMIGRATE_LAST_VERSION_GXY = 179
SQLALCHEMYMIGRATE_LAST_VERSION_TSI = 17
log = logging.getLogger(__name__)
@@ -57,7 +60,7 @@ class NoVersionTableError(Exception):
# The database has no version table (neither SQLAlchemy Migrate, nor Alembic), so it is
# impossible to automatically determine the state of the database. Manual update required.
def __init__(self, model: str) -> None:
super().__init__(f'Your {model} database has no version table; manual update is required')
super().__init__(f"Your {model} database has no version table; manual update is required")
class VersionTooOldError(Exception):
@@ -65,25 +68,26 @@ class VersionTooOldError(Exception):
# {SQLALCHEMYMIGRATE_LAST_VERSION_GXY/TSI}, so it cannot be upgraded with Alembic.
# Manual update required.
def __init__(self, model: str) -> None:
super().__init__(f'Your {model} database version is too old; manual update is required')
super().__init__(f"Your {model} database version is too old; manual update is required")
class OutdatedDatabaseError(Exception):
# The database is under Alembic version control, but is out-of-date. Automatic upgrade possible.
def __init__(self, model: str) -> None:
msg = f'Your {model} database is out-of-date; automatic update requires setting `database_auto_migrate`'
msg = f"Your {model} database is out-of-date; automatic update requires setting `database_auto_migrate`"
super().__init__(msg)
class InvalidModelIdError(Exception):
def __init__(self, model: str) -> None:
super().__init__(f'Invalid model: {model}')
super().__init__(f"Invalid model: {model}")
class AlembicManager:
"""
Alembic operations on one database.
"""
@staticmethod
def is_at_revision(engine: Engine, revision: Union[str, Iterable[str]]) -> bool:
"""
@@ -104,10 +108,10 @@ class AlembicManager:
def _load_config(self, config_dict: Optional[dict]) -> Config:
alembic_root = os.path.dirname(__file__)
_alembic_file = os.path.join(alembic_root, 'alembic.ini')
_alembic_file = os.path.join(alembic_root, "alembic.ini")
config = Config(_alembic_file)
url = get_url_string(self.engine)
config.set_main_option('sqlalchemy.url', url)
config.set_main_option("sqlalchemy.url", url)
if config_dict:
for key, value in config_dict.items():
config.set_main_option(key, value)
@@ -115,7 +119,7 @@ class AlembicManager:
def stamp_model_head(self, model: ModelId) -> None:
"""Partial proxy to alembic's stamp command."""
command.stamp(self.alembic_cfg, f'{model}@head')
command.stamp(self.alembic_cfg, f"{model}@head")
self._reset_db_heads()
def stamp_revision(self, revision: Union[str, Iterable[str]]) -> None:
@@ -126,7 +130,7 @@ class AlembicManager:
def upgrade(self, model: ModelId) -> None:
"""Partial proxy to alembic's upgrade command."""
# This works with or without an existing alembic version table.
command.upgrade(self.alembic_cfg, f'{model}@head')
command.upgrade(self.alembic_cfg, f"{model}@head")
self._reset_db_heads()
def is_under_version_control(self, model: ModelId) -> bool:
@@ -187,7 +191,7 @@ class AlembicManager:
try:
return self.script_directory.get_revision(revision_id)
except alembic.util.exc.CommandError as e:
log.error(f'Revision {revision_id} not found in the script directory')
log.error(f"Revision {revision_id} not found in the script directory")
raise e
def _reset_db_heads(self) -> None:
@@ -198,6 +202,7 @@ class DatabaseStateCache:
"""
Snapshot of database state.
"""
def __init__(self, engine: Engine) -> None:
self._load_db(engine)
@@ -246,9 +251,13 @@ def verify_databases_via_script(
tsi_engine = create_engine(tsi_config.url)
verify_databases(
gxy_engine, gxy_config.template, gxy_config.encoding,
tsi_engine, tsi_config.template, tsi_config.encoding,
is_auto_migrate
gxy_engine,
gxy_config.template,
gxy_config.encoding,
tsi_engine,
tsi_config.template,
tsi_config.encoding,
is_auto_migrate,
)
gxy_engine.dispose()
if tsi_engine:
@@ -265,8 +274,7 @@ def verify_databases(
is_auto_migrate: bool,
) -> None:
# Verify gxy model.
gxy_verifier = DatabaseStateVerifier(
gxy_engine, GXY, gxy_template, gxy_encoding, is_auto_migrate)
gxy_verifier = DatabaseStateVerifier(gxy_engine, GXY, gxy_template, gxy_encoding, is_auto_migrate)
gxy_verifier.run()
# New database = same engine, and gxy model has just been initialized.
@@ -276,21 +284,19 @@ def verify_databases(
tsi_engine = tsi_engine or gxy_engine
# Verify tsi model model.
tsi_verifier = DatabaseStateVerifier(
tsi_engine, TSI, tsi_template, tsi_encoding, is_auto_migrate, is_new_database)
tsi_verifier = DatabaseStateVerifier(tsi_engine, TSI, tsi_template, tsi_encoding, is_auto_migrate, is_new_database)
tsi_verifier.run()
class DatabaseStateVerifier:
def __init__(
self,
engine: Engine,
model: ModelId,
database_template: Optional[str],
database_encoding: Optional[str],
is_auto_migrate: bool,
is_new_database: Optional[bool] = False,
self,
engine: Engine,
model: ModelId,
database_template: Optional[str],
database_encoding: Optional[str],
is_auto_migrate: bool,
is_new_database: Optional[bool] = False,
) -> None:
self.engine = engine
self.model = model
@@ -365,7 +371,7 @@ class DatabaseStateVerifier:
model = self._get_model_name()
# first check if this model is up to date
if am.is_up_to_date(self.model):
log.info(f'Your {model} database is up-to-date')
log.info(f"Your {model} database is up-to-date")
return
# is outdated: try to upgrade
if not self.is_auto_migrate:
@@ -375,25 +381,20 @@ class DatabaseStateVerifier:
log.warning(msg)
raise OutdatedDatabaseError(model)
else:
log.info('Database is being upgraded to current version')
log.info("Database is being upgraded to current version")
am.upgrade(self.model)
return
def _get_upgrade_message(
self,
model: str,
db_version: str,
code_version: str
) -> str:
msg = f'Your {model} database has version {db_version}, but this code expects '
msg += f'version {code_version}. '
msg += 'This database can be upgraded automatically if database_auto_migrate is set. '
msg += 'To upgrade manually, run `run_alembic.sh` (see instructions in that file). '
msg += 'Please remember to backup your database before migrating.'
def _get_upgrade_message(self, model: str, db_version: str, code_version: str) -> str:
msg = f"Your {model} database has version {db_version}, but this code expects "
msg += f"version {code_version}. "
msg += "This database can be upgraded automatically if database_auto_migrate is set. "
msg += "To upgrade manually, run `run_alembic.sh` (see instructions in that file). "
msg += "Please remember to backup your database before migrating."
return msg
def _get_model_name(self) -> str:
return 'galaxy' if self.model == GXY else 'tool shed install'
return "galaxy" if self.model == GXY else "tool shed install"
def _no_model_tables_exist(self) -> bool:
# True if there are no tables from `self.model` in the database.
@@ -405,13 +406,13 @@ class DatabaseStateVerifier:
def _create_database(self, url: str) -> None:
create_kwds = {}
message = f'Creating database for URI [{url}]'
message = f"Creating database for URI [{url}]"
if self.database_template:
message += f' from template [{self.database_template}]'
create_kwds['template'] = self.database_template
message += f" from template [{self.database_template}]"
create_kwds["template"] = self.database_template
if self.database_encoding:
message += f' with encoding [{self.database_encoding}]'
create_kwds['encoding'] = self.database_encoding
message += f" with encoding [{self.database_encoding}]"
create_kwds["encoding"] = self.database_encoding
log.info(message)
create_database(url, **create_kwds)
lib/galaxy/model/migrations/alembic/env.py
+11 -11
View File
@@ -10,7 +10,10 @@ from alembic.script import ScriptDirectory
from alembic.script.base import Script
from sqlalchemy import create_engine
from galaxy.model.migrations import GXY, TSI
from galaxy.model.migrations import (
GXY,
TSI,
)
config = context.config
target_metadata = None # Not implemented: used for autogenerate, which we don't use here.
@@ -46,9 +49,9 @@ def _run_migrations_invoked_via_script(run_migrations: Callable[[str], None]) ->
revision_str = config.cmd_opts.revision # type: ignore[union-attr]
if revision_str.startswith(f'{GXY}@'):
if revision_str.startswith(f"{GXY}@"):
url = urls[GXY]
elif revision_str.startswith(f'{TSI}@'):
elif revision_str.startswith(f"{TSI}@"):
url = urls[TSI]
else:
revision = _get_revision(revision_str)
@@ -61,7 +64,7 @@ def _run_migrations_invoked_via_script(run_migrations: Callable[[str], None]) ->
def _process_cmd_current(urls: Dict[str, str]) -> bool:
if config.cmd_opts.cmd[0].__name__ == 'current': # type: ignore[union-attr]
if config.cmd_opts.cmd[0].__name__ == "current": # type: ignore[union-attr]
for url in urls.values():
_configure_and_run_migrations_online(url)
return True
@@ -79,7 +82,7 @@ def _get_revision(revision_str: str) -> Script:
def _get_revision_id(revision_str: str) -> str:
# Match a full or partial revision (GUID) or a relative migration identifier
p = re.compile(r'([0-9A-Fa-f]+)([+-]\d)?')
p = re.compile(r"([0-9A-Fa-f]+)([+-]\d)?")
m = p.match(revision_str)
if not m:
raise Exception(f'Invalid revision or migration identifier: "{revision_str}"')
@@ -100,10 +103,7 @@ def _configure_and_run_migrations_offline(url: str) -> None:
def _configure_and_run_migrations_online(url) -> None:
engine = create_engine(url)
with engine.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata
)
context.configure(connection=connection, target_metadata=target_metadata)
with context.begin_transaction():
context.run_migrations()
engine.dispose()
@@ -115,8 +115,8 @@ def _get_url_from_config() -> str:
def _load_urls() -> Dict[str, str]:
gxy_url = context.get_x_argument(as_dictionary=True).get(f'{GXY}_url')
tsi_url = context.get_x_argument(as_dictionary=True).get(f'{TSI}_url')
gxy_url = context.get_x_argument(as_dictionary=True).get(f"{GXY}_url")
tsi_url = context.get_x_argument(as_dictionary=True).get(f"{TSI}_url")
return {
GXY: gxy_url,
TSI: tsi_url,
lib/galaxy/model/migrations/alembic/versions_gxy/40aaada107df_create_table_vault.py
+10 -10
View File
@@ -11,10 +11,9 @@ import sqlalchemy as sa
from alembic import op
from sqlalchemy.engine.reflection import Inspector
# revision identifiers, used by Alembic.
revision = '40aaada107df'
down_revision = 'e7b6dcb09efd'
revision = "40aaada107df"
down_revision = "e7b6dcb09efd"
branch_labels = None
depends_on = None
@@ -23,12 +22,13 @@ def upgrade():
# This is the last revision to the database that was handled with SQLAlchemy Migrate prior to
# the move to Alembic. However, it was added to the dev branch after 21.09 was released, but
# before 22.01. Therefore, this table should be created when upgrading from 21.09, but not from dev.
if not _table_exists('vault'):
if not _table_exists("vault"):
now = datetime.datetime.utcnow
op.create_table('vault',
sa.Column('key', sa.Text, primary_key=True),
sa.Column('parent_key', sa.Text, sa.ForeignKey('vault.key'), index=True, nullable=True),
sa.Column('value', sa.Text, nullable=True),
op.create_table(
"vault",
sa.Column("key", sa.Text, primary_key=True),
sa.Column("parent_key", sa.Text, sa.ForeignKey("vault.key"), index=True, nullable=True),
sa.Column("value", sa.Text, nullable=True),
sa.Column("create_time", sa.DateTime, default=now),
sa.Column("update_time", sa.DateTime, default=now, onupdate=now),
)
@@ -36,8 +36,8 @@ def upgrade():
def downgrade():
# See comment in upgrade function
if _table_exists('vault'):
op.drop_table('vault')
if _table_exists("vault"):
op.drop_table("vault")
def _table_exists(table: str):
lib/galaxy/model/migrations/alembic/versions_gxy/e7b6dcb09efd_create_gxy_branch.py
+2 -2
View File
@@ -8,9 +8,9 @@ Create Date: 2021-11-05 16:32:43.243049
# revision identifiers, used by Alembic.
revision = 'e7b6dcb09efd'
revision = "e7b6dcb09efd"
down_revision = None
branch_labels = ('gxy',)
branch_labels = ("gxy",)
depends_on = None
lib/galaxy/model/migrations/alembic/versions_tsi/d4a650f47a3c_create_tsi_branch.py
+2 -2
View File
@@ -8,9 +8,9 @@ Create Date: 2021-11-05 16:32:25.113750
# revision identifiers, used by Alembic.
revision = 'd4a650f47a3c'
revision = "d4a650f47a3c"
down_revision = None
branch_labels = ('tsi',)
branch_labels = ("tsi",)
depends_on = None
lib/galaxy/model/migrations/scripts.py
+37 -37
View File
@@ -21,11 +21,11 @@ from galaxy.util.properties import (
load_app_properties,
)
DEFAULT_CONFIG_NAMES = ['galaxy', 'universe_wsgi']
CONFIG_FILE_ARG = '--galaxy-config'
CONFIG_DIR_NAME = 'config'
GXY_CONFIG_PREFIX = 'GALAXY_CONFIG_'
TSI_CONFIG_PREFIX = 'GALAXY_INSTALL_CONFIG_'
DEFAULT_CONFIG_NAMES = ["galaxy", "universe_wsgi"]
CONFIG_FILE_ARG = "--galaxy-config"
CONFIG_DIR_NAME = "config"
GXY_CONFIG_PREFIX = "GALAXY_CONFIG_"
TSI_CONFIG_PREFIX = "GALAXY_INSTALL_CONFIG_"
def get_configuration(argv: List[str], cwd: str) -> Tuple[DatabaseConfig, DatabaseConfig, bool]:
@@ -40,18 +40,18 @@ def get_configuration(argv: List[str], cwd: str) -> Tuple[DatabaseConfig, Databa
# load gxy properties and auto-migrate
properties = load_app_properties(config_file=config_file, config_prefix=GXY_CONFIG_PREFIX)
default_url = f"sqlite:///{os.path.join(get_data_dir(properties), 'universe.sqlite')}?isolation_level=IMMEDIATE"
url = properties.get('database_connection', default_url)
template = properties.get('database_template', None)
encoding = properties.get('database_encoding', None)
is_auto_migrate = properties.get('database_auto_migrate', False)
url = properties.get("database_connection", default_url)
template = properties.get("database_template", None)
encoding = properties.get("database_encoding", None)
is_auto_migrate = properties.get("database_auto_migrate", False)
gxy_config = DatabaseConfig(url, template, encoding)
# load tsi properties
properties = load_app_properties(config_file=config_file, config_prefix=TSI_CONFIG_PREFIX)
default_url = gxy_config.url
url = properties.get('install_database_connection', default_url)
template = properties.get('database_template', None)
encoding = properties.get('database_encoding', None)
url = properties.get("install_database_connection", default_url)
template = properties.get("database_template", None)
encoding = properties.get("database_encoding", None)
tsi_config = DatabaseConfig(url, template, encoding)
return (gxy_config, tsi_config, is_auto_migrate)
@@ -66,14 +66,14 @@ def _pop_config_file(argv: List[str]) -> Optional[str]:
def add_db_urls_to_command_arguments(argv: List[str], gxy_url: str, tsi_url: str) -> None:
_insert_x_argument(argv, 'tsi_url', tsi_url)
_insert_x_argument(argv, 'gxy_url', gxy_url)
_insert_x_argument(argv, "tsi_url", tsi_url)
_insert_x_argument(argv, "gxy_url", gxy_url)
def _insert_x_argument(argv, key: str, value: str) -> None:
# `_insert_x_argument('mykey', 'myval')` transforms `foo -a 1` into `foo -x mykey=myval -a 42`
argv.insert(1, f'{key}={value}')
argv.insert(1, '-x')
argv.insert(1, f"{key}={value}")
argv.insert(1, "-x")
def invoke_alembic() -> None:
@@ -85,11 +85,11 @@ def invoke_alembic() -> None:
separate gxy and tsi databases: we can't attach a database url to a revision after Alembic has been
invoked with the 'upgrade' command and the 'heads' argument. So, instead we invoke Alembic for each head.
"""
if 'heads' in sys.argv and 'upgrade' in sys.argv:
i = sys.argv.index('heads')
sys.argv[i] = f'{GXY}@head'
if "heads" in sys.argv and "upgrade" in sys.argv:
i = sys.argv.index("heads")
sys.argv[i] = f"{GXY}@head"
alembic.config.main()
sys.argv[i] = f'{TSI}@head'
sys.argv[i] = f"{TSI}@head"
alembic.config.main()
else:
alembic.config.main()
@@ -103,9 +103,9 @@ class LegacyScriptsException(Exception):
class LegacyScripts:
LEGACY_CONFIG_FILE_ARG_NAMES = ['-c', '--config', '--config-file']
ALEMBIC_CONFIG_FILE_ARG = '--alembic-config' # alembic config file, set in the calling script
DEFAULT_DB_ARG = 'default'
LEGACY_CONFIG_FILE_ARG_NAMES = ["-c", "--config", "--config-file"]
ALEMBIC_CONFIG_FILE_ARG = "--alembic-config" # alembic config file, set in the calling script
DEFAULT_DB_ARG = "default"
def __init__(self, argv: List[str], cwd: Optional[str] = None) -> None:
self.argv = argv
@@ -137,7 +137,7 @@ class LegacyScripts:
If last argument is a valid database name, pop and assign it; otherwise assign default.
"""
arg = self.argv[-1]
if arg in ['galaxy', 'install']:
if arg in ["galaxy", "install"]:
self.database = self.argv.pop()
def rename_config_argument(self) -> None:
@@ -153,21 +153,21 @@ class LegacyScripts:
"""
Rename argument name: `--alembic-config` to `-c`. There should be no `-c` argument present.
"""
if '-c' in self.argv:
raise LegacyScriptsException('Cannot rename alembic config argument: `-c` argument present.')
self._rename_arg(self.ALEMBIC_CONFIG_FILE_ARG, '-c')
if "-c" in self.argv:
raise LegacyScriptsException("Cannot rename alembic config argument: `-c` argument present.")
self._rename_arg(self.ALEMBIC_CONFIG_FILE_ARG, "-c")
def convert_version_argument(self) -> None:
"""
Convert legacy version argument to current spec required by Alembic.
"""
if '--version' in self.argv:
if "--version" in self.argv:
# Just remove it: the following argument should be the version/revision identifier.
pos = self.argv.index('--version')
pos = self.argv.index("--version")
self.argv.pop(pos)
else:
# If we find --version=foo, extract foo and replace arg with foo (which is the revision identifier)
p = re.compile(r'--version=([0-9A-Fa-f]+)')
p = re.compile(r"--version=([0-9A-Fa-f]+)")
for i, arg in enumerate(self.argv):
m = p.match(arg)
if m:
@@ -175,16 +175,16 @@ class LegacyScripts:
return
# No version argument found: construct argument for an upgrade operation.
# Raise exception otherwise.
if 'upgrade' not in self.argv:
raise LegacyScriptsException('If no `--version` argument supplied, `upgrade` argument is requried')
if "upgrade" not in self.argv:
raise LegacyScriptsException("If no `--version` argument supplied, `upgrade` argument is requried")
if self._is_one_database(): # upgrade both regardless of database argument
self.argv.append('heads')
self.argv.append("heads")
else: # for separate databases, choose one
if self.database in ['galaxy', self.DEFAULT_DB_ARG]:
self.argv.append('gxy@head')
elif self.database == 'install':
self.argv.append('tsi@head')
if self.database in ["galaxy", self.DEFAULT_DB_ARG]:
self.argv.append("gxy@head")
elif self.database == "install":
self.argv.append("tsi@head")
def _rename_arg(self, old_name, new_name) -> None:
pos = self.argv.index(old_name)
lib/galaxy/model/orm/engine_factory.py
+1 -1
View File
@@ -53,7 +53,7 @@ def build_engine(
trace_logger=None,
slow_query_log_threshold=0,
thread_local_log=None,
log_query_counts=False
log_query_counts=False,
):
if database_query_profiling_proxy or slow_query_log_threshold or thread_local_log or log_query_counts:
lib/galaxy/model/orm/scripts.py
+32 -32
View File
@@ -8,7 +8,10 @@ import sys
import alembic.config
from galaxy.model.migrations import GXY, TSI
from galaxy.model.migrations import (
GXY,
TSI,
)
from galaxy.model.migrations.scripts import get_configuration
from galaxy.util.path import get_ext
from galaxy.util.properties import (
@@ -26,25 +29,22 @@ DEFAULT_CONFIG_PREFIX = ""
DEFAULT_DATABASE = "galaxy"
DATABASE = {
"galaxy":
{
'default_sqlite_file': 'universe.sqlite',
'config_override': 'GALAXY_CONFIG_',
},
"tool_shed":
{
'repo': 'tool_shed/webapp/model/migrate',
'config_names': ['tool_shed', 'tool_shed_wsgi'],
'default_sqlite_file': 'community.sqlite',
'config_override': 'TOOL_SHED_CONFIG_',
'config_section': 'tool_shed',
},
"install":
{
'config_prefix': 'install_',
'default_sqlite_file': 'install.sqlite',
'config_override': 'GALAXY_INSTALL_CONFIG_',
},
"galaxy": {
"default_sqlite_file": "universe.sqlite",
"config_override": "GALAXY_CONFIG_",
},
"tool_shed": {
"repo": "tool_shed/webapp/model/migrate",
"config_names": ["tool_shed", "tool_shed_wsgi"],
"default_sqlite_file": "community.sqlite",
"config_override": "TOOL_SHED_CONFIG_",
"config_section": "tool_shed",
},
"install": {
"config_prefix": "install_",
"default_sqlite_file": "install.sqlite",
"config_override": "GALAXY_INSTALL_CONFIG_",
},
}
@@ -119,13 +119,13 @@ def get_config(argv, use_argparse=True, cwd=None):
cwd = [DEFAULT_CONFIG_DIR]
config_file = find_config_file(config_names, dirs=cwd)
repo = database_defaults.get('repo')
repo = database_defaults.get("repo")
if repo:
repo = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, repo)
config_prefix = database_defaults.get('config_prefix', DEFAULT_CONFIG_PREFIX)
config_override = database_defaults.get('config_override', 'GALAXY_CONFIG_')
default_sqlite_file = database_defaults['default_sqlite_file']
config_prefix = database_defaults.get("config_prefix", DEFAULT_CONFIG_PREFIX)
config_override = database_defaults.get("config_override", "GALAXY_CONFIG_")
default_sqlite_file = database_defaults["default_sqlite_file"]
if config_section is None:
if not config_file or get_ext(config_file, ignore="sample") == "yaml":
config_section = database_defaults.get("config_section", None)
@@ -155,18 +155,18 @@ def manage_db():
# This is a duplicate implementation of scripts/migrate_db.py.
# See run_alembic.sh for usage.
def _insert_x_argument(key, value):
sys.argv.insert(1, f'{key}={value}')
sys.argv.insert(1, '-x')
sys.argv.insert(1, f"{key}={value}")
sys.argv.insert(1, "-x")
gxy_config, tsi_config, _ = get_configuration(sys.argv, os.getcwd())
_insert_x_argument('tsi_url', tsi_config.url)
_insert_x_argument('gxy_url', gxy_config.url)
_insert_x_argument("tsi_url", tsi_config.url)
_insert_x_argument("gxy_url", gxy_config.url)
if 'heads' in sys.argv and 'upgrade' in sys.argv:
i = sys.argv.index('heads')
sys.argv[i] = f'{GXY}@head'
if "heads" in sys.argv and "upgrade" in sys.argv:
i = sys.argv.index("heads")
sys.argv[i] = f"{GXY}@head"
alembic.config.main()
sys.argv[i] = f'{TSI}@head'
sys.argv[i] = f"{TSI}@head"
alembic.config.main()
else:
alembic.config.main()
scripts/create_toolshed_db.py
+2 -2
View File
@@ -11,7 +11,7 @@ import logging
import os.path
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.model.orm.scripts import get_config
from tool_shed.webapp.model.migrate.check import create_or_verify_database as create_tool_shed_db
@@ -22,7 +22,7 @@ log = logging.getLogger(__name__)
def invoke_create():
config = get_config(sys.argv)
create_tool_shed_db(config['db_url'])
create_tool_shed_db(config["db_url"])
if __name__ == "__main__":
scripts/manage_db_adapter.py
+2 -2
View File
@@ -20,7 +20,7 @@ The optional `-c` argument name is renamed to `--galaxy-config`.
import os
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.model.migrations.scripts import (
invoke_alembic,
@@ -34,5 +34,5 @@ def run():
invoke_alembic()
if __name__ == '__main__':
if __name__ == "__main__":
run()
scripts/migrate_db.py
+2 -2
View File
@@ -8,7 +8,7 @@ import logging
import os.path
import sys
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.model.migrations.scripts import (
add_db_urls_to_command_arguments,
@@ -26,5 +26,5 @@ def run():
invoke_alembic()
if __name__ == '__main__':
if __name__ == "__main__":
run()
scripts/migrate_toolshed_db.py
+3 -3
View File
@@ -10,7 +10,7 @@ import sys
from migrate.versioning.shell import main
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, 'lib')))
sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib")))
from galaxy.model.orm.scripts import get_config
@@ -21,8 +21,8 @@ log = logging.getLogger(__name__)
def invoke_migrate_main():
# Migrate has its own args, so cannot use argparse
config = get_config(sys.argv, use_argparse=False, cwd=os.getcwd())
db_url = config['db_url']
repo = config['repo']
db_url = config["db_url"]
repo = config["repo"]
main(repository=repo, url=db_url)
test/unit/data/model/migrations/common.py
+10 -8
View File
@@ -18,7 +18,7 @@ from sqlalchemy.sql.compiler import IdentifierPreparer
from galaxy.model.database_utils import create_database
DbUrl = NewType('DbUrl', str)
DbUrl = NewType("DbUrl", str)
# Fixture and helper functions used to generate urls for postgresql and sqlite databases
@@ -30,6 +30,7 @@ def url_factory(tmp_directory: str) -> Callable[[], DbUrl]:
If _get_connection_url() returns a value, the database is postgresql; otherwise, it's
sqlite (referring to a location witin the /tmp directory).
"""
def url() -> DbUrl:
database = _generate_unique_database_name()
connection_url = _get_connection_url()
@@ -37,6 +38,7 @@ def url_factory(tmp_directory: str) -> Callable[[], DbUrl]:
return _make_postgres_db_url(DbUrl(connection_url), database)
else:
return _make_sqlite_db_url(tmp_directory, database)
return url
@@ -78,20 +80,20 @@ def drop_database(url: DbUrl) -> Iterator[None]:
def _generate_unique_database_name() -> str:
return f'galaxytest_{uuid.uuid4().hex}'
return f"galaxytest_{uuid.uuid4().hex}"
def _get_connection_url() -> Optional[str]:
return os.environ.get('GALAXY_TEST_DBURI')
return os.environ.get("GALAXY_TEST_DBURI")
def _is_postgres(url: DbUrl) -> bool:
return url.startswith('postgres')
return url.startswith("postgres")
def _make_sqlite_db_url(tmpdir: str, database: str) -> DbUrl:
path = os.path.join(tmpdir, database)
return DbUrl(f'sqlite:///{path}')
return DbUrl(f"sqlite:///{path}")
def _make_postgres_db_url(connection_url: DbUrl, database: str) -> DbUrl:
@@ -103,11 +105,11 @@ def _make_postgres_db_url(connection_url: DbUrl, database: str) -> DbUrl:
def _drop_postgres_database(url: DbUrl) -> None:
db_url = make_url(url)
database = db_url.database
connection_url = db_url.set(database='postgres')
engine = create_engine(connection_url, isolation_level='AUTOCOMMIT')
connection_url = db_url.set(database="postgres")
engine = create_engine(connection_url, isolation_level="AUTOCOMMIT")
preparer = IdentifierPreparer(engine.dialect)
database = preparer.quote(database)
stmt = f'DROP DATABASE IF EXISTS {database}'
stmt = f"DROP DATABASE IF EXISTS {database}"
with engine.connect() as conn:
conn.execute(stmt)
engine.dispose()
test/unit/data/model/migrations/conftest.py
+30 -26
View File
@@ -3,14 +3,15 @@ import tempfile
import pytest
import sqlalchemy as sa
# Helper fixtures
@pytest.fixture(scope='module')
@pytest.fixture(scope="module")
def tmp_directory():
with tempfile.TemporaryDirectory() as tmp_dir:
yield tmp_dir
# Fixtures: metadata containing one or more tables and representing database state.
# Used to load a database with a given state.
# Each state has 3 versions, distinguished by suffix:
@@ -107,9 +108,7 @@ def metadata_state3_tsi(tsi_table1, tsi_table2, sqlalchemymigrate_table):
@pytest.fixture
def metadata_state3_combined(
gxy_table1, gxy_table2, tsi_table1, tsi_table2, sqlalchemymigrate_table
):
def metadata_state3_combined(gxy_table1, gxy_table2, tsi_table1, tsi_table2, sqlalchemymigrate_table):
metadata = sa.MetaData()
gxy_table1(metadata)
gxy_table2(metadata)
@@ -141,9 +140,7 @@ def metadata_state4_tsi(tsi_table1, tsi_table2, sqlalchemymigrate_table, alembic
@pytest.fixture
def metadata_state4_combined(
gxy_table1, gxy_table2, tsi_table1, tsi_table2, sqlalchemymigrate_table, alembic_table
):
def metadata_state4_combined(gxy_table1, gxy_table2, tsi_table1, tsi_table2, sqlalchemymigrate_table, alembic_table):
metadata = sa.MetaData()
gxy_table1(metadata)
gxy_table2(metadata)
@@ -174,9 +171,7 @@ def metadata_state5_tsi(tsi_table1, tsi_table2, alembic_table):
@pytest.fixture
def metadata_state5_combined(
gxy_table1, gxy_table2, tsi_table1, tsi_table2, alembic_table
):
def metadata_state5_combined(gxy_table1, gxy_table2, tsi_table1, tsi_table2, alembic_table):
metadata = sa.MetaData()
gxy_table1(metadata)
gxy_table2(metadata)
@@ -208,9 +203,7 @@ def metadata_state6_tsi(tsi_table1, tsi_table2, tsi_table3, alembic_table):
@pytest.fixture
def metadata_state6_combined(
gxy_table1, gxy_table2, gxy_table3, tsi_table1, tsi_table2, tsi_table3, alembic_table
):
def metadata_state6_combined(gxy_table1, gxy_table2, gxy_table3, tsi_table1, tsi_table2, tsi_table3, alembic_table):
metadata = sa.MetaData()
gxy_table1(metadata)
gxy_table2(metadata)
@@ -241,63 +234,74 @@ def metadata_state6_gxy_state3_tsi_no_sam(
# Used to compose metadata representing database state.
# (The `_factory` suffix is ommitted to keep the code less verbose)
@pytest.fixture
def gxy_table1():
def make_table(metadata):
return sa.Table('gxy_table1', metadata, sa.Column('id', sa.Integer, primary_key=True))
return sa.Table("gxy_table1", metadata, sa.Column("id", sa.Integer, primary_key=True))
return make_table
@pytest.fixture
def gxy_table2():
def make_table(metadata):
return sa.Table('gxy_table2', metadata, sa.Column('id', sa.Integer, primary_key=True))
return sa.Table("gxy_table2", metadata, sa.Column("id", sa.Integer, primary_key=True))
return make_table
@pytest.fixture
def gxy_table3():
def make_table(metadata):
return sa.Table('gxy_table3', metadata, sa.Column('id', sa.Integer, primary_key=True))
return sa.Table("gxy_table3", metadata, sa.Column("id", sa.Integer, primary_key=True))
return make_table
@pytest.fixture
def tsi_table1():
def make_table(metadata):
return sa.Table('tsi_table1', metadata, sa.Column('id', sa.Integer, primary_key=True))
return sa.Table("tsi_table1", metadata, sa.Column("id", sa.Integer, primary_key=True))
return make_table
@pytest.fixture
def tsi_table2():
def make_table(metadata):
return sa.Table('tsi_table2', metadata, sa.Column('id', sa.Integer, primary_key=True))
return sa.Table("tsi_table2", metadata, sa.Column("id", sa.Integer, primary_key=True))
return make_table
@pytest.fixture
def tsi_table3():
def make_table(metadata):
return sa.Table('tsi_table3', metadata, sa.Column('id', sa.Integer, primary_key=True))
return sa.Table("tsi_table3", metadata, sa.Column("id", sa.Integer, primary_key=True))
return make_table
@pytest.fixture
def alembic_table():
def make_table(metadata):
table = sa.Table('alembic_version', metadata,
sa.Column('version_num', sa.String(250), primary_key=True))
table = sa.Table("alembic_version", metadata, sa.Column("version_num", sa.String(250), primary_key=True))
return table
return make_table
@pytest.fixture
def sqlalchemymigrate_table():
def make_table(metadata):
table = sa.Table('migrate_version', metadata,
sa.Column('repository_id', sa.String(250), primary_key=True),
sa.Column('repository_path', sa.Text),
sa.Column('version', sa.Integer),)
table = sa.Table(
"migrate_version",
metadata,
sa.Column("repository_id", sa.String(250), primary_key=True),
sa.Column("repository_path", sa.Text),
sa.Column("version", sa.Integer),
)
return table
return make_table
test/unit/data/model/migrations/test_migrations.py
+37 -62
View File
@@ -32,16 +32,15 @@ from .common import ( # noqa: F401 (url_factory is a fixture we have to import
)
# Revision numbers from test versions directories
GXY_REVISION_0 = '62695fac6cc0' # oldest/base
GXY_REVISION_1 = '2e8a580bc79a'
GXY_REVISION_2 = 'e02cef55763c' # current/head
TSI_REVISION_0 = '1bceec30363a' # oldest/base
TSI_REVISION_1 = '8364ef1cab05'
TSI_REVISION_2 = '0e28bf2fb7b5' # current/head
GXY_REVISION_0 = "62695fac6cc0" # oldest/base
GXY_REVISION_1 = "2e8a580bc79a"
GXY_REVISION_2 = "e02cef55763c" # current/head
TSI_REVISION_0 = "1bceec30363a" # oldest/base
TSI_REVISION_1 = "8364ef1cab05"
TSI_REVISION_2 = "0e28bf2fb7b5" # current/head
class TestAlembicManager:
def test_is_at_revision__one_head_one_revision(self, url_factory): # noqa: F811
# Use case: Check if separate tsi database is at a given revision.
db_url = url_factory()
@@ -147,7 +146,7 @@ class TestAlembicManager:
with disposing_engine(db_url) as engine:
am = AlembicManagerForTests(engine)
with pytest.raises(alembic.util.exc.CommandError):
am._get_revision('invalid')
am._get_revision("invalid")
def test_get_model_db_head(self, url_factory): # noqa: F811
db_url = url_factory()
@@ -169,7 +168,6 @@ class TestAlembicManager:
class TestDatabaseStateCache:
def test_is_empty(self, url_factory, metadata_state1_gxy): # noqa: F811
db_url, metadata = url_factory(), metadata_state1_gxy
with create_and_drop_database(db_url):
@@ -204,14 +202,15 @@ class TestDatabaseStateCache:
load_metadata(metadata_state2_gxy, engine)
load_sqlalchemymigrate_version(db_url, SQLALCHEMYMIGRATE_LAST_VERSION_GXY - 1)
assert not DatabaseStateCache(engine).is_last_sqlalchemymigrate_version(
SQLALCHEMYMIGRATE_LAST_VERSION_GXY)
SQLALCHEMYMIGRATE_LAST_VERSION_GXY
)
load_sqlalchemymigrate_version(db_url, SQLALCHEMYMIGRATE_LAST_VERSION_GXY)
assert DatabaseStateCache(engine).is_last_sqlalchemymigrate_version(
SQLALCHEMYMIGRATE_LAST_VERSION_GXY)
assert DatabaseStateCache(engine).is_last_sqlalchemymigrate_version(SQLALCHEMYMIGRATE_LAST_VERSION_GXY)
# Database fixture tests
class TestDatabaseFixtures:
# Verify that database fixtures have the expected state.
#
@@ -219,7 +218,6 @@ class TestDatabaseFixtures:
# to databases that HAVE BEEN CREATED. Thus, we are not wrapping them here
# in the `create_and_drop_database` context manager: they are wrapped already.
class TestState1:
def test_database_gxy(self, db_state1_gxy, metadata_state1_gxy):
self.verify_state(db_state1_gxy, metadata_state1_gxy)
@@ -237,7 +235,6 @@ class TestDatabaseFixtures:
assert not db.has_alembic_version_table()
class TestState2:
def test_database_gxy(self, db_state2_gxy, metadata_state2_gxy):
self.verify_state(db_state2_gxy, metadata_state2_gxy)
@@ -257,7 +254,6 @@ class TestDatabaseFixtures:
assert not db.has_alembic_version_table()
class TestState3:
def test_database_gxy(self, db_state3_gxy, metadata_state3_gxy):
self.verify_state(db_state3_gxy, metadata_state3_gxy, SQLALCHEMYMIGRATE_LAST_VERSION_GXY)
@@ -276,7 +272,6 @@ class TestDatabaseFixtures:
assert not db.has_alembic_version_table()
class TestState4:
def test_database_gxy(self, db_state4_gxy, metadata_state4_gxy):
self.verify_state(db_state4_gxy, metadata_state4_gxy, GXY_REVISION_0, SQLALCHEMYMIGRATE_LAST_VERSION_GXY)
@@ -285,7 +280,11 @@ class TestDatabaseFixtures:
def test_database_combined(self, db_state4_combined, metadata_state4_combined):
self.verify_state(
db_state4_combined, metadata_state4_combined, [GXY_REVISION_0, TSI_REVISION_0], SQLALCHEMYMIGRATE_LAST_VERSION_GXY)
db_state4_combined,
metadata_state4_combined,
[GXY_REVISION_0, TSI_REVISION_0],
SQLALCHEMYMIGRATE_LAST_VERSION_GXY,
)
def verify_state(self, db_url, metadata, revision, last_version):
assert is_metadata_loaded(db_url, metadata)
@@ -297,7 +296,6 @@ class TestDatabaseFixtures:
assert AlembicManagerForTests.is_at_revision(engine, revision)
class TestState5:
def test_database_gxy(self, db_state5_gxy, metadata_state5_gxy):
self.verify_state(db_state5_gxy, metadata_state5_gxy, GXY_REVISION_1)
@@ -305,8 +303,7 @@ class TestDatabaseFixtures:
self.verify_state(db_state5_tsi, metadata_state5_tsi, TSI_REVISION_1)
def test_database_combined(self, db_state5_combined, metadata_state5_combined):
self.verify_state(
db_state5_combined, metadata_state5_combined, [GXY_REVISION_1, TSI_REVISION_1])
self.verify_state(db_state5_combined, metadata_state5_combined, [GXY_REVISION_1, TSI_REVISION_1])
def verify_state(self, db_url, metadata, revision):
assert is_metadata_loaded(db_url, metadata)
@@ -317,7 +314,6 @@ class TestDatabaseFixtures:
assert AlembicManagerForTests.is_at_revision(engine, revision)
class TestState6:
def test_database_gxy(self, db_state6_gxy, metadata_state6_gxy):
self.verify_state(db_state6_gxy, metadata_state6_gxy, GXY_REVISION_2)
@@ -325,8 +321,7 @@ class TestDatabaseFixtures:
self.verify_state(db_state6_tsi, metadata_state6_tsi, TSI_REVISION_2)
def test_database_combined(self, db_state6_combined, metadata_state6_combined):
self.verify_state(
db_state6_combined, metadata_state6_combined, [GXY_REVISION_2, TSI_REVISION_2])
self.verify_state(db_state6_combined, metadata_state6_combined, [GXY_REVISION_2, TSI_REVISION_2])
def verify_state(self, db_url, metadata, revision):
assert is_metadata_loaded(db_url, metadata)
@@ -336,7 +331,6 @@ class TestDatabaseFixtures:
assert db.has_alembic_version_table()
class TestState6GxyState3TsiNoSam:
def test_database_combined(self, db_state6_gxy_state3_tsi_no_sam, metadata_state6_gxy_state3_tsi_no_sam):
db_url = db_state6_gxy_state3_tsi_no_sam
metadata = metadata_state6_gxy_state3_tsi_no_sam
@@ -540,12 +534,7 @@ class TestDatabaseStates:
# Expect:
# a) auto-migrate enabled: database upgraded to current version.
# b) auto-migrate disabled: fail with appropriate message.
def test_combined_database_automigrate(
self,
db_state5_combined,
metadata_state6_combined,
set_automigrate
):
def test_combined_database_automigrate(self, db_state5_combined, metadata_state6_combined, set_automigrate):
db_url = db_state5_combined
with disposing_engine(db_url) as engine:
_verify_databases(engine)
@@ -553,12 +542,7 @@ class TestDatabaseStates:
assert database_is_up_to_date(db_url, metadata_state6_combined, TSI)
def test_separate_databases_automigrate(
self,
db_state5_gxy,
db_state5_tsi,
metadata_state6_gxy,
metadata_state6_tsi,
set_automigrate
self, db_state5_gxy, db_state5_tsi, metadata_state6_gxy, metadata_state6_tsi, set_automigrate
):
db1_url, db2_url = db_state5_gxy, db_state5_tsi
with disposing_engine(db1_url) as engine1, disposing_engine(db2_url) as engine2:
@@ -591,13 +575,7 @@ class TestDatabaseStates:
assert database_is_up_to_date(db_url, metadata_state6_combined, GXY)
assert database_is_up_to_date(db_url, metadata_state6_combined, TSI)
def test_separate_databases(
self,
db_state6_gxy,
db_state6_tsi,
metadata_state6_gxy,
metadata_state6_tsi
):
def test_separate_databases(self, db_state6_gxy, db_state6_tsi, metadata_state6_gxy, metadata_state6_tsi):
db1_url, db2_url = db_state6_gxy, db_state6_tsi
with disposing_engine(db1_url) as engine1, disposing_engine(db2_url) as engine2:
_verify_databases(engine1, engine2)
@@ -670,47 +648,44 @@ class TestDatabaseStates:
# Test helpers + their tests, misc. fixtures
@pytest.fixture(autouse=True) # always override AlembicManager
def set_create_additional(monkeypatch):
monkeypatch.setattr(DatabaseStateVerifier, '_create_additional_database_objects', lambda *_: None)
monkeypatch.setattr(DatabaseStateVerifier, "_create_additional_database_objects", lambda *_: None)
@pytest.fixture
def set_automigrate(monkeypatch):
monkeypatch.setattr(DatabaseStateVerifier, 'is_auto_migrate', True)
monkeypatch.setattr(DatabaseStateVerifier, "is_auto_migrate", True)
@pytest.fixture(autouse=True) # always override AlembicManager
def set_alembic_manager(monkeypatch):
monkeypatch.setattr(
migrations, 'get_alembic_manager', lambda engine: AlembicManagerForTests(engine))
monkeypatch.setattr(migrations, "get_alembic_manager", lambda engine: AlembicManagerForTests(engine))
@pytest.fixture(autouse=True) # always override gxy_metadata
def set_gxy_metadata(monkeypatch, metadata_state6_gxy):
monkeypatch.setattr(
migrations, 'get_gxy_metadata', lambda: metadata_state6_gxy)
monkeypatch.setattr(migrations, "get_gxy_metadata", lambda: metadata_state6_gxy)
@pytest.fixture(autouse=True) # always override tsi_metadata
def set_tsi_metadata(monkeypatch, metadata_state6_tsi):
monkeypatch.setattr(
migrations, 'get_tsi_metadata', lambda: metadata_state6_tsi)
monkeypatch.setattr(migrations, "get_tsi_metadata", lambda: metadata_state6_tsi)
class AlembicManagerForTests(AlembicManager):
def __init__(self, engine):
path1, path2 = self._get_paths_to_version_locations()
config_dict = {'version_locations': f'{path1};{path2}'}
config_dict = {"version_locations": f"{path1};{path2}"}
super().__init__(engine, config_dict)
def _get_paths_to_version_locations(self):
# One does not simply use a relative path for both tests and package tests.
basepath = os.path.abspath(os.path.dirname(__file__))
basepath = os.path.join(basepath, 'versions')
path1 = os.path.join(basepath, 'db1')
path2 = os.path.join(basepath, 'db2')
basepath = os.path.join(basepath, "versions")
path1 = os.path.join(basepath, "db1")
path2 = os.path.join(basepath, "db2")
return path1, path2
@@ -769,9 +744,9 @@ def database_is_up_to_date(db_url, current_state_metadata, model):
# passed as an argument. That's why we ensure that the passed metadata is current
# (this guards againt an incorrect test).
if model == GXY:
current_tables = {'gxy_table1', 'gxy_table2', 'gxy_table3'}
current_tables = {"gxy_table1", "gxy_table2", "gxy_table3"}
elif model == TSI:
current_tables = {'tsi_table1', 'tsi_table2', 'tsi_table3'}
current_tables = {"tsi_table1", "tsi_table2", "tsi_table3"}
is_metadata_current = current_tables <= set(current_state_metadata.tables)
with disposing_engine(db_url) as engine:
@@ -787,7 +762,7 @@ def test_database_is_up_to_date(url_factory, metadata_state6_gxy): # noqa F811
assert not database_is_up_to_date(db_url, metadata, GXY)
load_metadata(metadata, engine)
am = AlembicManagerForTests(engine)
am.stamp_revision('heads')
am.stamp_revision("heads")
assert database_is_up_to_date(db_url, metadata, GXY)
@@ -799,7 +774,7 @@ def test_database_is_up_to_date_for_passed_model_only(url_factory, metadata_stat
assert not database_is_up_to_date(db_url, metadata, TSI)
load_metadata(metadata, engine)
am = AlembicManagerForTests(engine)
am.stamp_revision('heads')
am.stamp_revision("heads")
assert database_is_up_to_date(db_url, metadata, GXY)
assert not database_is_up_to_date(db_url, metadata, TSI)
@@ -810,7 +785,7 @@ def test_database_is_not_up_to_date_if_noncurrent_metadata_passed(url_factory, m
with disposing_engine(db_url) as engine:
load_metadata(metadata, engine)
am = AlembicManagerForTests(engine)
am.stamp_revision('heads')
am.stamp_revision("heads")
assert not database_is_up_to_date(db_url, metadata, GXY)
@@ -819,7 +794,7 @@ def test_database_is_not_up_to_date_if_metadata_not_loaded(url_factory, metadata
with create_and_drop_database(db_url):
with disposing_engine(db_url) as engine:
am = AlembicManagerForTests(engine)
am.stamp_revision('heads')
am.stamp_revision("heads")
assert not database_is_up_to_date(db_url, metadata, GXY)
test/unit/data/model/migrations/test_scripts.py
+52 -37
View File
@@ -10,123 +10,138 @@ from galaxy.model.migrations.scripts import (
def set_db_urls(monkeypatch):
# Do not try to access galaxy config; values not needed.
def no_config_call(self):
self.gxy_url = 'a string'
self.tsi_url = 'a stirng'
self.gxy_url = "a string"
self.tsi_url = "a stirng"
monkeypatch.setattr(LegacyScripts, '_get_db_urls', no_config_call)
monkeypatch.setattr(LegacyScripts, "_get_db_urls", no_config_call)
@pytest.fixture(autouse=True) # set combined db for all tests
def set_combined(monkeypatch):
monkeypatch.setattr(LegacyScripts, '_is_one_database', lambda self: True)
monkeypatch.setattr(LegacyScripts, "_is_one_database", lambda self: True)
@pytest.fixture
def set_separate(monkeypatch):
monkeypatch.setattr(LegacyScripts, '_is_one_database', lambda self: False)
monkeypatch.setattr(LegacyScripts, "_is_one_database", lambda self: False)
class TestLegacyScripts():
@pytest.mark.parametrize('database_arg', ['galaxy', 'install'])
class TestLegacyScripts:
@pytest.mark.parametrize("database_arg", ["galaxy", "install"])
def test_pop_database_name(self, database_arg):
# arg_value = 'install'
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc', database_arg]
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc", database_arg]
ls = LegacyScripts(argv)
ls.pop_database_argument()
assert ls.database == database_arg
assert argv == ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc']
assert argv == ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"]
def test_pop_database_name_use_default(self):
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"]
ls = LegacyScripts(argv)
ls.pop_database_argument()
assert ls.database == LegacyScripts.DEFAULT_DB_ARG
assert argv == ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc']
assert argv == ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"]
@pytest.mark.parametrize('arg_name', LegacyScripts.LEGACY_CONFIG_FILE_ARG_NAMES)
@pytest.mark.parametrize("arg_name", LegacyScripts.LEGACY_CONFIG_FILE_ARG_NAMES)
def test_rename_config_arg(self, arg_name):
# `-c|--config|__config-file` should be renamed to `--galaxy-config`
argv = ['caller', '--alembic-config', 'path-to-alembic', arg_name, 'path-to-galaxy', 'upgrade', '--version=abc']
argv = ["caller", "--alembic-config", "path-to-alembic", arg_name, "path-to-galaxy", "upgrade", "--version=abc"]
LegacyScripts(argv).rename_config_argument()
assert argv == ['caller', '--alembic-config', 'path-to-alembic', '--galaxy-config', 'path-to-galaxy', 'upgrade', '--version=abc']
assert argv == [
"caller",
"--alembic-config",
"path-to-alembic",
"--galaxy-config",
"path-to-galaxy",
"upgrade",
"--version=abc",
]
def test_rename_config_arg_reordered_args(self):
# `-c|--config|__config-file` should be renamed to `--galaxy-config`
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc', '-c', 'path-to-galaxy']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc", "-c", "path-to-galaxy"]
LegacyScripts(argv).rename_config_argument()
assert argv == ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc', '--galaxy-config', 'path-to-galaxy']
assert argv == [
"caller",
"--alembic-config",
"path-to-alembic",
"upgrade",
"--version=abc",
"--galaxy-config",
"path-to-galaxy",
]
def test_rename_alembic_config_arg(self):
# `--alembic-config` should be renamed to `-c`
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"]
LegacyScripts(argv).rename_alembic_config_argument()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', '--version=abc']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "--version=abc"]
def test_rename_alembic_config_arg_raises_error_if_c_arg_present(self):
# Ensure alembic config arg is renamed AFTER renaming the galaxy config arg. Raise error otherwise.
argv = ['caller', '--alembic-config', 'path-to-alembic', '-c', 'path-to-galaxy', 'upgrade', '--version=abc']
argv = ["caller", "--alembic-config", "path-to-alembic", "-c", "path-to-galaxy", "upgrade", "--version=abc"]
with pytest.raises(LegacyScriptsException):
LegacyScripts(argv).rename_alembic_config_argument()
def test_convert__version_arg_1(self):
# `sh manage_db.sh upgrade --version X` >> `... upgrade X`
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version', 'abc']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version", "abc"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'abc']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "abc"]
def test_convert__version_arg_2(self):
# `sh manage_db.sh upgrade --version=X` >> `... upgrade X`
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', '--version=abc']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'abc']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "abc"]
def test_convert__no_version_no_model_combined_database(self):
# `sh manage_db.sh upgrade` >> `... upgrade heads`
# No version and no model implies "upgrade the default db (which is galaxy) to its latest version".
# If it is combined, we upgrade both models: gxy and tsi.
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'heads']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"]
def test_convert__no_version_galaxy_model_combined_database_(self):
# `sh manage_db.sh upgrade galaxy` >> `... upgrade heads`
# same as no model: if combined we upgrade the whole database
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', 'galaxy']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "galaxy"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'heads']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"]
def test_convert__no_version_install_model_combined_database_(self):
# `sh manage_db.sh upgrade install` >> `... upgrade heads`
# same as no model: if combined we upgrade the whole database
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', 'install']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "install"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'heads']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"]
def test_convert__no_version_no_model_separate_databases(self, set_separate):
# `sh manage_db.sh upgrade` >> `... upgrade gxy@head`
# No version and no model implies "upgrade the default db (which is galaxy) to its latest version".
# Since the tsi model has its own db, we only upgrade the gxy model.
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'gxy@head']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "gxy@head"]
def test_convert__no_version_galaxy_model_separate_databases(self, set_separate):
# `sh manage_db.sh upgrade galaxy` >> `... upgrade gxy@head`
# No version + a model implies "upgrade the db for the specified model to its latest version".
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', 'galaxy']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "galaxy"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'gxy@head']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "gxy@head"]
def test_convert__no_version_install_model_separate_databases(self, set_separate):
# `sh manage_db.sh upgrade install` >> `... upgrade tsi@head`
# No version + a model implies "upgrade the db for the specified model to its latest version".
argv = ['caller', '--alembic-config', 'path-to-alembic', 'upgrade', 'install']
argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "install"]
LegacyScripts(argv).convert_args()
assert argv == ['caller', '-c', 'path-to-alembic', 'upgrade', 'tsi@head']
assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "tsi@head"]
def test_downgrade_with_no_version_argument_raises_error(self):
argv = ['caller', '--alembic-config', 'path-to-alembic', 'downgrade']
argv = ["caller", "--alembic-config", "path-to-alembic", "downgrade"]
with pytest.raises(LegacyScriptsException):
LegacyScripts(argv).convert_args()
test/unit/data/model/migrations/versions/db1/2e8a580bc79a_drop_sqlachemymigrate_table.py
+7 -8
View File
@@ -8,10 +8,9 @@ Create Date: 2021-11-05 16:29:19.123118
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '2e8a580bc79a'
down_revision = '62695fac6cc0'
revision = "2e8a580bc79a"
down_revision = "62695fac6cc0"
branch_labels = None
depends_on = None
@@ -21,15 +20,15 @@ def upgrade():
# this migration will be applied twice to the same database, so we ignore
# the error that happens on the second run when the table has been dropped.
try:
op.drop_table('migrate_version', must_exist=True)
op.drop_table("migrate_version", must_exist=True)
except sa.exc.InvalidRequestError:
pass
def downgrade():
op.create_table(
'migrate_version',
sa.Column('repository_id', sa.String(250), primary_key=True),
sa.Column('repository_path', sa.Text),
sa.Column('version', sa.Integer),
"migrate_version",
sa.Column("repository_id", sa.String(250), primary_key=True),
sa.Column("repository_path", sa.Text),
sa.Column("version", sa.Integer),
)
test/unit/data/model/migrations/versions/db1/62695fac6cc0_create_gxy_test_branch.py
+2 -2
View File
@@ -8,9 +8,9 @@ Create Date: 2021-11-05 16:28:30.497050
# revision identifiers, used by Alembic.
revision = '62695fac6cc0'
revision = "62695fac6cc0"
down_revision = None
branch_labels = ('gxy',)
branch_labels = ("gxy",)
depends_on = None
test/unit/data/model/migrations/versions/db1/e02cef55763c_add_gxy_table3.py
+6 -6
View File
@@ -8,19 +8,19 @@ Create Date: 2021-11-05 16:30:30.521436
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = 'e02cef55763c'
down_revision = '2e8a580bc79a'
revision = "e02cef55763c"
down_revision = "2e8a580bc79a"
branch_labels = None
depends_on = None
def upgrade():
op.create_table('gxy_table3',
sa.Column('id', sa.Integer, primary_key=True),
op.create_table(
"gxy_table3",
sa.Column("id", sa.Integer, primary_key=True),
)
def downgrade():
op.drop_table('gxy_table3')
op.drop_table("gxy_table3")
test/unit/data/model/migrations/versions/db2/0e28bf2fb7b5_add_tsi_table3.py
+6 -6
View File
@@ -8,19 +8,19 @@ Create Date: 2021-11-05 16:31:00.530235
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '0e28bf2fb7b5'
down_revision = '8364ef1cab05'
revision = "0e28bf2fb7b5"
down_revision = "8364ef1cab05"
branch_labels = None
depends_on = None
def upgrade():
op.create_table('tsi_table3',
sa.Column('id', sa.Integer, primary_key=True),
op.create_table(
"tsi_table3",
sa.Column("id", sa.Integer, primary_key=True),
)
def downgrade():
op.drop_table('tsi_table3')
op.drop_table("tsi_table3")
test/unit/data/model/migrations/versions/db2/1bceec30363a_create_tsi_test_branch.py
+2 -2
View File
@@ -8,9 +8,9 @@ Create Date: 2021-11-05 16:28:45.450830
# revision identifiers, used by Alembic.
revision = '1bceec30363a'
revision = "1bceec30363a"
down_revision = None
branch_labels = ('tsi',)
branch_labels = ("tsi",)
depends_on = None
test/unit/data/model/migrations/versions/db2/8364ef1cab05_drop_sqlachemymigrate_table.py
+7 -8
View File
@@ -8,10 +8,9 @@ Create Date: 2021-11-05 16:30:51.369967
import sqlalchemy as sa
from alembic import op
# revision identifiers, used by Alembic.
revision = '8364ef1cab05'
down_revision = '1bceec30363a'
revision = "8364ef1cab05"
down_revision = "1bceec30363a"
branch_labels = None
depends_on = None
@@ -21,15 +20,15 @@ def upgrade():
# this migration will be applied twice to the same database, so we ignore
# the error that happens on the second run when the table has been dropped.
try:
op.drop_table('migrate_version', must_exist=True)
op.drop_table("migrate_version", must_exist=True)
except sa.exc.InvalidRequestError:
pass
def downgrade():
op.create_table(
'migrate_version',
sa.Column('repository_id', sa.String(250), primary_key=True),
sa.Column('repository_path', sa.Text),
sa.Column('version', sa.Integer),
"migrate_version",
sa.Column("repository_id", sa.String(250), primary_key=True),
sa.Column("repository_path", sa.Text),
sa.Column("version", sa.Integer),
)