From 4cd3291598490a900185a3a286e632ddfe0a84c2 Mon Sep 17 00:00:00 2001 From: John Davis Date: Sun, 28 Aug 2022 17:53:11 -0400 Subject: [PATCH 01/34] Refactor get_alembic_cfg: make available to other code --- lib/galaxy/model/migrations/scripts.py | 14 ++++++++------ test/unit/data/model/migrations/test_migrations.py | 5 +++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py index 54f0ad774e1..487f77788f4 100644 --- a/lib/galaxy/model/migrations/scripts.py +++ b/lib/galaxy/model/migrations/scripts.py @@ -1,3 +1,4 @@ +import argparse import os import re import sys @@ -339,14 +340,9 @@ class LegacyManageDb: return dbcache.sqlalchemymigrate_version def _get_script_directory(self): - alembic_cfg = self._get_alembic_cfg() + alembic_cfg = get_alembic_cfg() return ScriptDirectory.from_config(alembic_cfg) - def _get_alembic_cfg(self): - config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") - config_file = os.path.abspath(config_file) - return Config(config_file) - def _get_gxy_alembic_db_version(self, engine): # We may get 2 values, one for each branch (gxy and tsi). So we need to # determine which one is the gxy head. @@ -369,5 +365,11 @@ class LegacyManageDb: return gxy_revisions +def get_alembic_cfg(): + config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") + config_file = os.path.abspath(config_file) + return Config(config_file) + + def get_alembic_manager(engine: Engine) -> AlembicManager: return AlembicManager(engine) diff --git a/test/unit/data/model/migrations/test_migrations.py b/test/unit/data/model/migrations/test_migrations.py index b8f96fe7f6a..c89bfb62a21 100644 --- a/test/unit/data/model/migrations/test_migrations.py +++ b/test/unit/data/model/migrations/test_migrations.py @@ -9,6 +9,7 @@ from sqlalchemy import ( text, ) +import galaxy.model.migrations.scripts from galaxy.model import migrations from galaxy.model.database_utils import database_exists from galaxy.model.migrations import ( @@ -1120,7 +1121,7 @@ def db_state6_gxy_state3_tsi_no_sam(url_factory, metadata_state6_gxy_state3_tsi_ @pytest.fixture(autouse=True) def legacy_manage_db(monkeypatch): - def get_alembic_cfg(self): + def get_alembic_cfg(): path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, os.pardir) path = os.path.normpath(path) # Adjust path when running from packages @@ -1133,7 +1134,7 @@ def legacy_manage_db(monkeypatch): config.set_main_option("version_locations", f"{path1};{path2}") return config - monkeypatch.setattr(LegacyManageDb, "_get_alembic_cfg", get_alembic_cfg) + monkeypatch.setattr(galaxy.model.migrations.scripts, "get_alembic_cfg", get_alembic_cfg) @pytest.fixture(autouse=True) From b64fda74783d5e5caf6f85482b6587835b66f1bd Mon Sep 17 00:00:00 2001 From: John Davis Date: Sun, 28 Aug 2022 18:49:51 -0400 Subject: [PATCH 02/34] Add initial draft --- db.sh | 10 + lib/galaxy/model/migrations/dbscript.py | 126 ++++++ lib/galaxy/model/migrations/scripts.py | 2 + scripts/db.py | 141 +++++++ .../data/model/migrations/test_dbscript.py | 364 ++++++++++++++++++ 5 files changed, 643 insertions(+) create mode 100755 db.sh create mode 100644 lib/galaxy/model/migrations/dbscript.py create mode 100644 scripts/db.py create mode 100644 test/unit/data/model/migrations/test_dbscript.py diff --git a/db.sh b/db.sh new file mode 100755 index 00000000000..0dfa2a8cfea --- /dev/null +++ b/db.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# draft of script to replace manage_db.sh +cd `dirname $0` + +. ./scripts/common_startup_functions.sh + +setup_python + +python ./scripts/db.py "$@" diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py new file mode 100644 index 00000000000..d268d4c602c --- /dev/null +++ b/lib/galaxy/model/migrations/dbscript.py @@ -0,0 +1,126 @@ +import argparse +import os +import sys +from typing import ( + List, + Optional, + Tuple, +) + +import alembic.config +from alembic import command +from alembic.config import Config +from alembic.runtime.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine +from sqlalchemy.engine import Engine + +from galaxy.model.database_utils import ( + database_exists, + is_one_database, +) +from galaxy.model.migrations import ( + AlembicManager, + DatabaseConfig, + DatabaseStateCache, + GXY, + IncorrectVersionError, + NoVersionTableError, + SQLALCHEMYMIGRATE_LAST_VERSION_GXY, + TSI, +) +from galaxy.util.properties import ( + find_config_file, + get_data_dir, + load_app_properties, +) + +DEFAULT_CONFIG_NAMES = ["galaxy", "universe_wsgi"] +CONFIG_FILE_ARG = "--galaxy-config" +CONFIG_DIR_NAME = "config" +GXY_CONFIG_PREFIX = "GALAXY_CONFIG_" +TSI_CONFIG_PREFIX = "GALAXY_INSTALL_CONFIG_" + + +class DbScript: + """ + Used to manage the gxy db. + The upgrade command is called on both: gxy and tsi. Reason: if this is the first alembic command on this branch, + the upgrade command will stamp the alembic_version table: we need that for both branches. + """ + + def __init__(self, config_file: Optional[str] = None) -> None: + self.alembic_config = self._get_alembic_cfg() + self._set_dburl(config_file) + + def upgrade(self, args: argparse.Namespace) -> None: + revision = self._parse_revision(args.revision) + command.upgrade(self.alembic_config, revision, args.sql) + + def downgrade(self, args: argparse.Namespace) -> None: + command.downgrade(self.alembic_config, args.revision, args.sql) + + def revision(self, args: argparse.Namespace) -> None: + """Create revision script for the gxy branch only.""" + command.revision(self.alembic_config, message=args.message, rev_id=args.rev_id, head="gxy@head") + + def version(self, args: argparse.Namespace) -> None: + command.heads(self.alembic_config, verbose=args.verbose) + + def dbversion(self, args: argparse.Namespace) -> None: + command.current(self.alembic_config, verbose=args.verbose) + + def history(self, args: argparse.Namespace) -> None: + command.history(self.alembic_config, verbose=args.verbose, indicate_current=args.indicate_current) + + def show(self, args: argparse.Namespace) -> None: + command.show(self.alembic_config, args.revision) + + def _get_alembic_cfg(self): + config_file = os.getenv("ALEMBIC_CONFIG") + if not config_file: + config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") + config_file = os.path.abspath(config_file) + return Config(config_file) + + def _set_dburl(self, config_file: Optional[str] = None) -> None: + gxy_config, tsi_config = self._get_configuration(config_file) + self.gxy_url = gxy_config.url + self.tsi_url = tsi_config.url + self._set_url(self.gxy_url) + + def _set_url(self, url: str) -> None: + self.alembic_config.set_main_option("sqlalchemy.url", url) + + def _parse_revision(self, rev): + # Relative revision identifier requires a branch label + if rev.startswith("+") or rev.startswith("-"): + return f"gxy@{rev}" + return rev + + def _get_configuration(self, config_file: Optional[str] = None) -> Tuple[DatabaseConfig, DatabaseConfig]: + """ + Return a 2-item-tuple with configuration values used for managing databases. + """ + if config_file is None: + cwd = os.getcwd() + cwds = [cwd, os.path.join(cwd, CONFIG_DIR_NAME)] + config_file = find_config_file(DEFAULT_CONFIG_NAMES, dirs=cwds) + + # load gxy properties and auto-migrate + properties = load_app_properties(config_file=config_file, config_prefix=GXY_CONFIG_PREFIX) + default_url = f"sqlite:///{os.path.join(get_data_dir(properties), 'universe.sqlite')}?isolation_level=IMMEDIATE" + url = properties.get("database_connection", default_url) + template = properties.get("database_template", None) + encoding = properties.get("database_encoding", None) + gxy_config = DatabaseConfig(url, template, encoding) + + # load tsi properties + properties = load_app_properties(config_file=config_file, config_prefix=TSI_CONFIG_PREFIX) + default_url = gxy_config.url + url = properties.get("install_database_connection", default_url) + template = properties.get("database_template", None) + encoding = properties.get("database_encoding", None) + tsi_config = DatabaseConfig(url, template, encoding) + + return (gxy_config, tsi_config) diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py index 487f77788f4..3b6f52ce58e 100644 --- a/lib/galaxy/model/migrations/scripts.py +++ b/lib/galaxy/model/migrations/scripts.py @@ -9,6 +9,7 @@ from typing import ( ) import alembic.config +from alembic import command from alembic.config import Config from alembic.runtime.migration import MigrationContext from alembic.script import ScriptDirectory @@ -88,6 +89,7 @@ def verify_database_is_initialized(db_url: str) -> None: def get_configuration(argv: List[str], cwd: str) -> Tuple[DatabaseConfig, DatabaseConfig, bool]: + # TODO i think is_auto-migrate is not used! """ Return a 3-item-tuple with configuration values used for managing databases. """ diff --git a/scripts/db.py b/scripts/db.py new file mode 100644 index 00000000000..9a79da6ae60 --- /dev/null +++ b/scripts/db.py @@ -0,0 +1,141 @@ +""" +This script is intended to be invoked by the db.sh script. +""" + +import os +import sys +from argparse import ( + ArgumentParser, + Namespace, +) + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) + +from galaxy.model.migrations.dbscript import DbScript + + +def exec_upgrade(args: Namespace) -> None: + _exec_command("upgrade", args) + + +def exec_downgrade(args: Namespace) -> None: + _exec_command("downgrade", args) + + +def exec_revision(args: Namespace) -> None: + _exec_command("revision", args) + + +def exec_version(args: Namespace) -> None: + _exec_command("version", args) + + +def exec_dbversion(args: Namespace) -> None: + _exec_command("dbversion", args) + + +def exec_history(args: Namespace) -> None: + _exec_command("history", args) + + +def exec_show(args: Namespace) -> None: + _exec_command("show", args) + + +def _exec_command(command, args): + dbscript = DbScript(args.config) + getattr(dbscript, command)(args) + + +def main() -> None: + def add_parser(command, func, help, aliases=None, parents=None): + aliases = aliases or [] + parents = parents or [] + parser = subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) + parser.set_defaults(func=func) + return parser + + config_arg_parser = ArgumentParser(add_help=False) + # TODO: after refactoring legacy scripts, this can be changed to "-c, --config" + config_arg_parser.add_argument("--galaxy-config", help="Alternate Galaxy configuration file", dest="config") + + verbose_arg_parser = ArgumentParser(add_help=False) + verbose_arg_parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") + + sql_arg_parser = ArgumentParser(add_help=False) + sql_arg_parser.add_argument( + "--sql", + action="store_true", + help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", + ) + + parser = ArgumentParser( + description="Common database schema migration operations", + epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." + " For migrating the `tsi` branch, use the `run_alembic.sh` script.", + ) + + subparsers = parser.add_subparsers(required=True) + + upgrade_cmd_parser = add_parser( + "upgrade", + exec_upgrade, + "Upgrade to a later version", + aliases=["u"], + parents=[config_arg_parser, sql_arg_parser], + ) + upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?", default="heads") + + downgrade_cmd_parser = add_parser( + "downgrade", + exec_downgrade, + "Revert to a previous version", + aliases=["d"], + parents=[config_arg_parser, sql_arg_parser], + ) + downgrade_cmd_parser.add_argument("revision", help="Revision identifier") + + add_parser( + "version", + exec_version, + "Show the head revision in the migrations script directory", + aliases=["v"], + parents=[config_arg_parser, verbose_arg_parser], + ) + + add_parser( + "dbversion", + exec_dbversion, + "Show the current revision for Galaxy's database", + aliases=["dbv"], + parents=[config_arg_parser, verbose_arg_parser], + ) + + history_cmd_parser = add_parser( + "history", + exec_history, + "List revision scripts in chronological order", + parents=[config_arg_parser, verbose_arg_parser], + ) + history_cmd_parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") + + show_cmd_parser = add_parser( + "show", + exec_show, + "Show the revision(s) denoted by the given symbol", + parents=[config_arg_parser], + ) + show_cmd_parser.add_argument("revision", help="Revision identifier") + + revision_cmd_parser = add_parser( + "revision", aliases=["r"], help="Create a new revision file", parents=[config_arg_parser], func=exec_revision + ) + revision_cmd_parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) + revision_cmd_parser.add_argument("--rev-id", help="Specify a revision id instead of generating one") + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py new file mode 100644 index 00000000000..8c95fd05fce --- /dev/null +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -0,0 +1,364 @@ +import os +import re +import subprocess +import tempfile +import uuid +from contextlib import contextmanager +from typing import ( + Callable, + Iterator, + List, + NewType, + Optional, +) + +import alembic +import pytest +from alembic import command +from alembic.config import Config +from alembic.runtime.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import ( + create_engine, + delete, + select, +) +from sqlalchemy.engine import ( + Engine, + make_url, +) +from sqlalchemy.sql.compiler import IdentifierPreparer + +from galaxy.model.database_utils import ( + create_database, + database_exists, +) +from ..testing_utils import ( # noqa: F401 (url_factory is a fixture we have to import explicitly) + create_and_drop_database, + disposing_engine, + drop_existing_database, + url_factory, +) + +DbUrl = NewType("DbUrl", str) + + +GXY_BRANCH_LABEL = "gxy" +TSI_BRANCH_LABEL = "tsi" +GXY_BASE_ID = "gxy0" +TSI_BASE_ID = "tsi0" + + +@pytest.fixture(scope="session") +def alembic_env_dir() -> str: + """[galaxy-root]/lib/galaxy/model/migrations/alembic/""" + galaxy_root = os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..") + return os.path.join(galaxy_root, "lib", "galaxy", "model", "migrations", "alembic") + + +@pytest.fixture(scope="session") +def alembic_config_text(alembic_env_dir) -> List[str]: + """Contents of production alembic.ini as list of lines""" + current_config_path = os.path.join(alembic_env_dir, "..", "alembic.ini") + with open(current_config_path, "r") as f: + return f.readlines() + + +@pytest.fixture() +def tmp_directory(): + with tempfile.TemporaryDirectory() as tmp_dir: + yield tmp_dir + + +@pytest.fixture() +def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, monkeypatch): + """ + Construct Config object for staging; setup staging env. + """ + gxy_versions_dir = os.path.join(tmp_directory, "versions_gxy") + tsi_versions_dir = os.path.join(tmp_directory, "versions_tsi") + version_locations = f"{gxy_versions_dir};{tsi_versions_dir}" + + dburl = url_factory() + config_file_path = os.path.join(tmp_directory, "alembic.ini") + update_config_for_staging(alembic_config_text, alembic_env_dir, version_locations, dburl) + write_config_file(config_file_path, alembic_config_text) + + alembic_cfg = Config(config_file_path) + create_alembic_branches(alembic_cfg, gxy_versions_dir, tsi_versions_dir) + + monkeypatch.setenv("ALEMBIC_CONFIG", config_file_path) + monkeypatch.setenv("GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION", dburl) + monkeypatch.setenv("GALAXY_CONFIG_OVERRIDE_INSTALL_DATABASE_CONNECTION", dburl) + + return alembic_cfg + + +def update_config_for_staging(config_text, script_location, version_locations, dburl) -> None: + """Set script_location, version_locations, sqlalchemy.url values.""" + alembic_section_index, url_set = -1, False + url_line = f"sqlalchemy.url = {dburl}\n" + for i, line in enumerate(config_text): + if line.strip() == "[alembic]": + alembic_section_index = i + elif line.startswith("script_location ="): + config_text[i] = f"script_location = {script_location}\n" + elif line.startswith("version_locations ="): + config_text[i] = f"version_locations = {version_locations}\n" + elif line.startswith("sqlalchemy.url ="): + config_text[i] = url_line + url_set = True + if not url_set: # True when executed for the first time + config_text.insert(alembic_section_index + 1, url_line) + + +def write_config_file(config_file_path, config_text): + with open(config_file_path, "w") as f: + f.write("".join(config_text)) + + +def create_alembic_branches(config, gxy_versions_dir, tsi_versions_dir): + """Create gxy and tsi branches""" + alembic.command.revision( + config, branch_label=GXY_BRANCH_LABEL, head="base", rev_id=GXY_BASE_ID, version_path=gxy_versions_dir + ) + alembic.command.revision( + config, branch_label=TSI_BRANCH_LABEL, head="base", rev_id=TSI_BASE_ID, version_path=tsi_versions_dir + ) + + +def stdout(capture): + return capture.readouterr().out + + +def dburl_from_config(config): + return config.get_main_option("sqlalchemy.url") + + +def run_command(cmd): + completed_process = subprocess.run(cmd.split(), capture_output=True, text=True) + return completed_process + + +def get_db_heads(config): + dburl = dburl_from_config(config) + engine = create_engine(dburl) + with engine.connect() as conn: + context = MigrationContext.configure(conn) + heads = context.get_current_heads() + engine.dispose() + return heads + + +class TestRevisionCommand: + def test_revision_cmd(self, config): + run_command(f"./db.sh revision --message foo1") + run_command(f"./db.sh revision --rev-id 2 --message foo2") + run_command(f"./db.sh revision --rev-id 3 --message foo3") + + script_dir = ScriptDirectory.from_config(config) + revisions = [rev for rev in script_dir.walk_revisions()] + assert len(revisions) == 5 # verify total revisions: 2 base + 3 new + + rev = script_dir.get_revision("3") + assert GXY_BRANCH_LABEL in rev.branch_labels # verify branch label + assert rev.down_revision == "2" # verify parent revision + assert rev.module.__name__ == "3_foo3_py" # verify message + + def test_revision_cmd_missing_message_arg_error(self): + completed = run_command(f"./db.sh revision --rev-id 1") + assert completed.returncode == 2 + assert "the following arguments are required: -m/--message" in completed.stderr + + +class TestShowCommand: + def test_show_cmd(self, config): + alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) + completed = run_command(f"./db.sh show 42") + assert "Revision ID: 42" in completed.stdout + + def test_show_cmd_invalid_revision_error(self, config): + alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) + completed = run_command(f"./db.sh show idonotexist") + assert completed.returncode == 1 + assert "Can't locate revision identified by 'idonotexist'" in completed.stderr + + def test_show_cmd_missing_revision_arg_error(self): + completed = run_command(f"./db.sh show") + assert completed.returncode == 2 + assert "the following arguments are required: revision" in completed.stderr + + +class TestHistoryCommand: + def test_history_cmd(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + + completed = run_command(f"./db.sh history") + assert completed.returncode == 0 + assert "2 -> 3 (gxy) (head), empty message" in completed.stdout + assert "1 -> 2 (gxy)" in completed.stdout + assert "gxy0 -> 1 (gxy)" in completed.stdout + + def test_history_cmd_verbose(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + + completed = run_command(f"./db.sh history --verbose") + assert "Revision ID: 2" in completed.stdout + assert "Revises: 1" in completed.stdout + + def test_history_cmd_indicate_current(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + alembic.command.upgrade(config, "heads") + + completed = run_command(f"./db.sh history --indicate-current") + assert completed.returncode == 0 + assert "2 -> 3 (gxy) (head) (current), empty message" in completed.stdout + assert "1 -> 2 (gxy)" in completed.stdout + assert "gxy0 -> 1 (gxy)" in completed.stdout + + +class TestVersionCommand: + def test_version_cmd(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"./db.sh version") + assert completed.returncode == 0 + assert "2 (gxy) (head)" in completed.stdout + + def test_version_cmd_verbose(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"./db.sh version --verbose") + assert completed.returncode == 0 + assert "Revision ID: 2" in completed.stdout + assert "Revises: 1" in completed.stdout + + +class TestUpgradeCommand: + def test_upgrade_cmd(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + # first upgrade: upgrades gxy to 2, tsi to base + completed = run_command(f"./db.sh upgrade") + assert completed.returncode == 0 + assert "Running upgrade gxy0 -> 1" in completed.stderr + assert "Running upgrade 1 -> 2" in completed.stderr + + heads = get_db_heads(config) + assert len(heads) == 2 + assert "2" in heads + assert TSI_BASE_ID in heads + + alembic.command.revision(config, rev_id="3", head="2") + + # next upgrade: upgrades gxy to 3, no effect on tsi + completed = run_command(f"./db.sh upgrade") + assert completed.returncode == 0 + assert "Running upgrade 2 -> 3" in completed.stderr + + heads = get_db_heads(config) + assert len(heads) == 2 + assert "3" in heads + assert TSI_BASE_ID in heads + + def test_upgrade_cmd_sql_only(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"./db.sh upgrade --sql") + assert completed.returncode == 0 + assert "UPDATE alembic_version SET version_num='2'" in completed.stdout + assert "UPDATE alembic_version SET version_num='3'" not in completed.stdout + + alembic.command.revision(config, rev_id="3", head="2") + + completed = run_command(f"./db.sh upgrade --sql") + assert completed.returncode == 0 + assert "UPDATE alembic_version SET version_num='2'" in completed.stdout + assert "UPDATE alembic_version SET version_num='3'" in completed.stdout + + def test_upgrade_cmd_with_revision_arg(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + # upgrades gxy to 1 + completed = run_command(f"./db.sh upgrade 1") + assert completed.returncode == 0 + assert "Running upgrade gxy0 -> 1" in completed.stderr + + heads = get_db_heads(config) + assert heads == ("1",) + + def test_upgrade_cmd_with_relative_revision_syntax(self, config): + alembic.command.revision(config, rev_id="a", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="b", head="a") + alembic.command.revision(config, rev_id="c", head="b") + alembic.command.revision(config, rev_id="d", head="c") + alembic.command.revision(config, rev_id="e", head="d") + + # upgrades gxy to b: none + 2 (none > base > a) + completed = run_command(f"./db.sh upgrade +3") + assert completed.returncode == 0 + assert "Running upgrade -> gxy0" in completed.stderr + assert "Running upgrade gxy0 -> a" in completed.stderr + assert "Running upgrade a -> b" in completed.stderr + + heads = get_db_heads(config) + assert heads == ("b",) + + # upgrades gxy to d relative to b: b + 2 (b > c > d) + completed = run_command(f"./db.sh upgrade b+2") + assert completed.returncode == 0 + assert "Running upgrade b -> c" in completed.stderr + assert "Running upgrade c -> d" in completed.stderr + + heads = get_db_heads(config) + assert heads == ("d",) + + +class TestDowngradeCommand: + def test_downgrade_cmd(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + alembic.command.upgrade(config, "heads") + + completed = run_command(f"./db.sh downgrade 1") # downgrade gxy to 1, no effect on tsi + assert completed.returncode == 0 + assert "Running downgrade 3 -> 2" in completed.stderr + assert "Running downgrade 2 -> 1" in completed.stderr + + heads = get_db_heads(config) + assert len(heads) == 2 + assert "1" in heads + + +# TODO add same type of test cases as in TestUpgradeCommand + + +class TestDbVersionCommand: + def test_dbversion_cmd(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"./db.sh dbversion") + assert completed.returncode == 0 + assert "(head)" not in completed.stdout # there has been no upgrade + + alembic.command.upgrade(config, "heads") + + completed = run_command(f"./db.sh dbversion") + assert completed.returncode == 0 + assert "2 (head)" in completed.stdout + + +# TODO test for 2 separate databases: gxy and tsi From d1c9342ecad03f059f3e45b970a812705a41e6ef Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 30 Aug 2022 15:52:13 -0400 Subject: [PATCH 03/34] Add logging to script --- scripts/db.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/db.py b/scripts/db.py index 9a79da6ae60..86a8ed2b8fc 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -2,6 +2,7 @@ This script is intended to be invoked by the db.sh script. """ +import logging import os import sys from argparse import ( @@ -13,6 +14,9 @@ sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pa from galaxy.model.migrations.dbscript import DbScript +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + def exec_upgrade(args: Namespace) -> None: _exec_command("upgrade", args) From 887b882ff091f3bdeb1975cf06d6e3fd0de723be Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 7 Sep 2022 10:39:22 -0400 Subject: [PATCH 04/34] Add inline testing documentation --- .../data/model/migrations/test_dbscript.py | 25 ++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index 8c95fd05fce..84373a25319 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -1,3 +1,24 @@ +""" +Testing approach: +- Use a test database, store revision scripts in a different location; leave the rest unchanged. +- Use alembic api for setup and accessing the database. +- Run command as subprocess, verify captured output + database state. + +1. Setup staging environment: + - Create staging location (/tmp) + - Create test database (sqlite in /tmp) + - Copy production alembic.ini to staging location, overwriting: + - sqlalchemy.url (url of test database) + - version_locations (staging location) + - script_location (lib/galaxy/model/migrations/alembic/) + - Create gxy and tsi branches + +2. For each test case: + - Optionally, use alembic api for any setup + - Run command as a subprocess, capture output + - Run assertions against captured output + - Optionally, use alembic api to access database; verify database state +""" import os import re import subprocess @@ -118,7 +139,9 @@ def write_config_file(config_file_path, config_text): def create_alembic_branches(config, gxy_versions_dir, tsi_versions_dir): - """Create gxy and tsi branches""" + """ + Create gxy and tsi branches (required for galaxy's alembic setup; included with 22.05 release) + """ alembic.command.revision( config, branch_label=GXY_BRANCH_LABEL, head="base", rev_id=GXY_BASE_ID, version_path=gxy_versions_dir ) From ab1854f3a7cd6d4980ac5b90c9a7035d64dcc58f Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 8 Sep 2022 01:18:36 -0400 Subject: [PATCH 05/34] Test against 1 or 2 databases --- lib/galaxy/model/migrations/dbscript.py | 20 +++-- scripts/db.py | 2 +- .../data/model/migrations/test_dbscript.py | 73 ++++++++++++++----- 3 files changed, 69 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index d268d4c602c..f8e2192a7f4 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -54,11 +54,21 @@ class DbScript: self._set_dburl(config_file) def upgrade(self, args: argparse.Namespace) -> None: - revision = self._parse_revision(args.revision) - command.upgrade(self.alembic_config, revision, args.sql) + def upgrade_to_revision(rev): + command.upgrade(self.alembic_config, rev, args.sql) + + if args.revision: + revision = self._parse_revision(args.revision) + upgrade_to_revision(revision) + else: # Run for each model + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) + upgrade_to_revision("gxy@head") + self.alembic_config.set_main_option("sqlalchemy.url", self.tsi_url) + upgrade_to_revision("tsi@head") def downgrade(self, args: argparse.Namespace) -> None: - command.downgrade(self.alembic_config, args.revision, args.sql) + revision = self._parse_revision(args.revision) + command.downgrade(self.alembic_config, revision, args.sql) def revision(self, args: argparse.Namespace) -> None: """Create revision script for the gxy branch only.""" @@ -87,10 +97,6 @@ class DbScript: gxy_config, tsi_config = self._get_configuration(config_file) self.gxy_url = gxy_config.url self.tsi_url = tsi_config.url - self._set_url(self.gxy_url) - - def _set_url(self, url: str) -> None: - self.alembic_config.set_main_option("sqlalchemy.url", url) def _parse_revision(self, rev): # Relative revision identifier requires a branch label diff --git a/scripts/db.py b/scripts/db.py index 86a8ed2b8fc..f4964d34a6a 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -88,7 +88,7 @@ def main() -> None: aliases=["u"], parents=[config_arg_parser, sql_arg_parser], ) - upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?", default="heads") + upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?") downgrade_cmd_parser = add_parser( "downgrade", diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index 84373a25319..8796382b957 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -91,8 +91,8 @@ def tmp_directory(): yield tmp_dir -@pytest.fixture() -def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, monkeypatch): +@pytest.fixture(params=["one database", "two databases"]) +def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, monkeypatch, request): """ Construct Config object for staging; setup staging env. """ @@ -100,17 +100,19 @@ def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, mon tsi_versions_dir = os.path.join(tmp_directory, "versions_tsi") version_locations = f"{gxy_versions_dir};{tsi_versions_dir}" - dburl = url_factory() + gxy_dburl = url_factory() + tsi_dburl = gxy_dburl if request.param == "one database" else url_factory() + config_file_path = os.path.join(tmp_directory, "alembic.ini") - update_config_for_staging(alembic_config_text, alembic_env_dir, version_locations, dburl) + update_config_for_staging(alembic_config_text, alembic_env_dir, version_locations, gxy_dburl) write_config_file(config_file_path, alembic_config_text) alembic_cfg = Config(config_file_path) create_alembic_branches(alembic_cfg, gxy_versions_dir, tsi_versions_dir) monkeypatch.setenv("ALEMBIC_CONFIG", config_file_path) - monkeypatch.setenv("GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION", dburl) - monkeypatch.setenv("GALAXY_CONFIG_OVERRIDE_INSTALL_DATABASE_CONNECTION", dburl) + monkeypatch.setenv("GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION", gxy_dburl) + monkeypatch.setenv("GALAXY_INSTALL_CONFIG_OVERRIDE_INSTALL_DATABASE_CONNECTION", tsi_dburl) return alembic_cfg @@ -275,23 +277,21 @@ class TestUpgradeCommand: assert completed.returncode == 0 assert "Running upgrade gxy0 -> 1" in completed.stderr assert "Running upgrade 1 -> 2" in completed.stderr + assert "Running upgrade -> tsi0" in completed.stderr heads = get_db_heads(config) - assert len(heads) == 2 assert "2" in heads - assert TSI_BASE_ID in heads alembic.command.revision(config, rev_id="3", head="2") - # next upgrade: upgrades gxy to 3, no effect on tsi + # next upgrade: upgrades gxy to 3 completed = run_command(f"./db.sh upgrade") assert completed.returncode == 0 assert "Running upgrade 2 -> 3" in completed.stderr + assert "tsi0" not in completed.stderr # no effect on tsi heads = get_db_heads(config) - assert len(heads) == 2 assert "3" in heads - assert TSI_BASE_ID in heads def test_upgrade_cmd_sql_only(self, config): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) @@ -328,7 +328,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="d", head="c") alembic.command.revision(config, rev_id="e", head="d") - # upgrades gxy to b: none + 2 (none > base > a) + # upgrades gxy to b: none + 2 (none -> base -> a) completed = run_command(f"./db.sh upgrade +3") assert completed.returncode == 0 assert "Running upgrade -> gxy0" in completed.stderr @@ -338,7 +338,7 @@ class TestUpgradeCommand: heads = get_db_heads(config) assert heads == ("b",) - # upgrades gxy to d relative to b: b + 2 (b > c > d) + # upgrades gxy to d relative to b: b + 2 (b -> c -> d) completed = run_command(f"./db.sh upgrade b+2") assert completed.returncode == 0 assert "Running upgrade b -> c" in completed.stderr @@ -355,7 +355,7 @@ class TestDowngradeCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command(f"./db.sh downgrade 1") # downgrade gxy to 1, no effect on tsi + completed = run_command(f"./db.sh downgrade 1") # downgrade gxy to 1 assert completed.returncode == 0 assert "Running downgrade 3 -> 2" in completed.stderr assert "Running downgrade 2 -> 1" in completed.stderr @@ -364,8 +364,48 @@ class TestDowngradeCommand: assert len(heads) == 2 assert "1" in heads + def test_downgrade_cmd_sql_only(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + alembic.command.upgrade(config, "heads") -# TODO add same type of test cases as in TestUpgradeCommand + completed = run_command(f"./db.sh downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi + assert completed.returncode == 0 + assert "UPDATE alembic_version SET version_num='2'" in completed.stdout + assert "UPDATE alembic_version SET version_num='1'" in completed.stdout + + def test_downgrade_cmd_missing_revision_arg_error(self): + completed = run_command(f"./db.sh downgrade") + assert completed.returncode == 2 + assert "the following arguments are required: revision" in completed.stderr + + def test_downgrade_cmd_with_relative_revision_syntax(self, config): + alembic.command.revision(config, rev_id="a", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="b", head="a") + alembic.command.revision(config, rev_id="c", head="b") + alembic.command.revision(config, rev_id="d", head="c") + alembic.command.revision(config, rev_id="e", head="d") + alembic.command.upgrade(config, "heads") + + # downgrades gxy to c: e - 2 (e -> d -> c) + completed = run_command(f"./db.sh downgrade -2") + + assert completed.returncode == 0 + assert "Running downgrade e -> d" in completed.stderr + assert "Running downgrade d -> c" in completed.stderr + + heads = get_db_heads(config) + assert "c" in heads + + # downgrades gxy to a relative to c: c - 2 (c -> b -> a) + completed = run_command(f"./db.sh downgrade c-2") + assert completed.returncode == 0 + assert "Running downgrade c -> b" in completed.stderr + assert "Running downgrade b -> a" in completed.stderr + + heads = get_db_heads(config) + assert "a" in heads class TestDbVersionCommand: @@ -382,6 +422,3 @@ class TestDbVersionCommand: completed = run_command(f"./db.sh dbversion") assert completed.returncode == 0 assert "2 (head)" in completed.stdout - - -# TODO test for 2 separate databases: gxy and tsi From 103210cc7317395f39bd189e573124976fa2e291 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 8 Sep 2022 11:49:15 -0400 Subject: [PATCH 06/34] Set dburl in alembic config --- lib/galaxy/model/migrations/dbscript.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index f8e2192a7f4..aaaa3e9c85a 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -44,14 +44,18 @@ TSI_CONFIG_PREFIX = "GALAXY_INSTALL_CONFIG_" class DbScript: """ - Used to manage the gxy db. - The upgrade command is called on both: gxy and tsi. Reason: if this is the first alembic command on this branch, - the upgrade command will stamp the alembic_version table: we need that for both branches. + Facade for common database schema migration operations on the gxy branch. + When the gxy and tsi branches are persisted in the same database, some + alembic commands will display output on the state on both branches (e.g. + history, version, dbversion). The upgrade command is executed on both + branches: gxy and tsi (the upgrade command ensures the branch has been + initialized by stamping its version in the alembic_version table). """ def __init__(self, config_file: Optional[str] = None) -> None: self.alembic_config = self._get_alembic_cfg() self._set_dburl(config_file) + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) def upgrade(self, args: argparse.Namespace) -> None: def upgrade_to_revision(rev): @@ -60,11 +64,14 @@ class DbScript: if args.revision: revision = self._parse_revision(args.revision) upgrade_to_revision(revision) - else: # Run for each model + else: self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) upgrade_to_revision("gxy@head") - self.alembic_config.set_main_option("sqlalchemy.url", self.tsi_url) - upgrade_to_revision("tsi@head") + try: + self.alembic_config.set_main_option("sqlalchemy.url", self.tsi_url) + upgrade_to_revision("tsi@head") + finally: + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) def downgrade(self, args: argparse.Namespace) -> None: revision = self._parse_revision(args.revision) From 0c9b0875430da7cb6fe79f9d0e13cf36af658c0c Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 8 Sep 2022 21:45:47 -0400 Subject: [PATCH 07/34] Add init command --- scripts/db.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/scripts/db.py b/scripts/db.py index f4964d34a6a..2365cae6758 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -12,7 +12,9 @@ from argparse import ( sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) +from galaxy.model.migrations import verify_databases_via_script from galaxy.model.migrations.dbscript import DbScript +from galaxy.model.migrations.scripts import get_configuration logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) @@ -46,6 +48,11 @@ def exec_show(args: Namespace) -> None: _exec_command("show", args) +def exec_init(args: Namespace) -> None: + gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) + verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) + + def _exec_command(command, args): dbscript = DbScript(args.config) getattr(dbscript, command)(args) @@ -137,6 +144,13 @@ def main() -> None: revision_cmd_parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) revision_cmd_parser.add_argument("--rev-id", help="Specify a revision id instead of generating one") + add_parser( + "init", + exec_init, + "Initialize empty database(s) for both branches (create database objects for gxy and tsi branch)", + parents=[config_arg_parser], + ) + args = parser.parse_args() args.func(args) From 3be7c53eb97d8380779a9d0e25532f0d76475bf5 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 8 Sep 2022 22:14:08 -0400 Subject: [PATCH 08/34] Fix linting --- lib/galaxy/model/migrations/dbscript.py | 22 +----- lib/galaxy/model/migrations/scripts.py | 2 - .../data/model/migrations/test_dbscript.py | 78 +++++++------------ 3 files changed, 30 insertions(+), 72 deletions(-) diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index aaaa3e9c85a..6f55fb834be 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -1,34 +1,14 @@ import argparse import os -import sys from typing import ( - List, Optional, Tuple, ) -import alembic.config from alembic import command from alembic.config import Config -from alembic.runtime.migration import MigrationContext -from alembic.script import ScriptDirectory -from sqlalchemy import create_engine -from sqlalchemy.engine import Engine -from galaxy.model.database_utils import ( - database_exists, - is_one_database, -) -from galaxy.model.migrations import ( - AlembicManager, - DatabaseConfig, - DatabaseStateCache, - GXY, - IncorrectVersionError, - NoVersionTableError, - SQLALCHEMYMIGRATE_LAST_VERSION_GXY, - TSI, -) +from galaxy.model.migrations import DatabaseConfig from galaxy.util.properties import ( find_config_file, get_data_dir, diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py index 3b6f52ce58e..c727884cab2 100644 --- a/lib/galaxy/model/migrations/scripts.py +++ b/lib/galaxy/model/migrations/scripts.py @@ -1,4 +1,3 @@ -import argparse import os import re import sys @@ -9,7 +8,6 @@ from typing import ( ) import alembic.config -from alembic import command from alembic.config import Config from alembic.runtime.migration import MigrationContext from alembic.script import ScriptDirectory diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index 8796382b957..8dcfc331eea 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -12,7 +12,7 @@ Testing approach: - version_locations (staging location) - script_location (lib/galaxy/model/migrations/alembic/) - Create gxy and tsi branches - + 2. For each test case: - Optionally, use alembic api for any setup - Run command as a subprocess, capture output @@ -20,40 +20,20 @@ Testing approach: - Optionally, use alembic api to access database; verify database state """ import os -import re import subprocess import tempfile -import uuid -from contextlib import contextmanager from typing import ( - Callable, - Iterator, List, NewType, - Optional, ) import alembic import pytest -from alembic import command from alembic.config import Config from alembic.runtime.migration import MigrationContext from alembic.script import ScriptDirectory -from sqlalchemy import ( - create_engine, - delete, - select, -) -from sqlalchemy.engine import ( - Engine, - make_url, -) -from sqlalchemy.sql.compiler import IdentifierPreparer +from sqlalchemy import create_engine -from galaxy.model.database_utils import ( - create_database, - database_exists, -) from ..testing_utils import ( # noqa: F401 (url_factory is a fixture we have to import explicitly) create_and_drop_database, disposing_engine, @@ -92,7 +72,7 @@ def tmp_directory(): @pytest.fixture(params=["one database", "two databases"]) -def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, monkeypatch, request): +def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, monkeypatch, request): # noqa: F811 """ Construct Config object for staging; setup staging env. """ @@ -177,9 +157,9 @@ def get_db_heads(config): class TestRevisionCommand: def test_revision_cmd(self, config): - run_command(f"./db.sh revision --message foo1") - run_command(f"./db.sh revision --rev-id 2 --message foo2") - run_command(f"./db.sh revision --rev-id 3 --message foo3") + run_command("./db.sh revision --message foo1") + run_command("./db.sh revision --rev-id 2 --message foo2") + run_command("./db.sh revision --rev-id 3 --message foo3") script_dir = ScriptDirectory.from_config(config) revisions = [rev for rev in script_dir.walk_revisions()] @@ -191,7 +171,7 @@ class TestRevisionCommand: assert rev.module.__name__ == "3_foo3_py" # verify message def test_revision_cmd_missing_message_arg_error(self): - completed = run_command(f"./db.sh revision --rev-id 1") + completed = run_command("./db.sh revision --rev-id 1") assert completed.returncode == 2 assert "the following arguments are required: -m/--message" in completed.stderr @@ -199,17 +179,17 @@ class TestRevisionCommand: class TestShowCommand: def test_show_cmd(self, config): alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) - completed = run_command(f"./db.sh show 42") + completed = run_command("./db.sh show 42") assert "Revision ID: 42" in completed.stdout def test_show_cmd_invalid_revision_error(self, config): alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) - completed = run_command(f"./db.sh show idonotexist") + completed = run_command("./db.sh show idonotexist") assert completed.returncode == 1 assert "Can't locate revision identified by 'idonotexist'" in completed.stderr def test_show_cmd_missing_revision_arg_error(self): - completed = run_command(f"./db.sh show") + completed = run_command("./db.sh show") assert completed.returncode == 2 assert "the following arguments are required: revision" in completed.stderr @@ -220,7 +200,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") - completed = run_command(f"./db.sh history") + completed = run_command("./db.sh history") assert completed.returncode == 0 assert "2 -> 3 (gxy) (head), empty message" in completed.stdout assert "1 -> 2 (gxy)" in completed.stdout @@ -231,7 +211,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") - completed = run_command(f"./db.sh history --verbose") + completed = run_command("./db.sh history --verbose") assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout @@ -241,7 +221,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command(f"./db.sh history --indicate-current") + completed = run_command("./db.sh history --indicate-current") assert completed.returncode == 0 assert "2 -> 3 (gxy) (head) (current), empty message" in completed.stdout assert "1 -> 2 (gxy)" in completed.stdout @@ -253,7 +233,7 @@ class TestVersionCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command(f"./db.sh version") + completed = run_command("./db.sh version") assert completed.returncode == 0 assert "2 (gxy) (head)" in completed.stdout @@ -261,7 +241,7 @@ class TestVersionCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command(f"./db.sh version --verbose") + completed = run_command("./db.sh version --verbose") assert completed.returncode == 0 assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout @@ -273,7 +253,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="2", head="1") # first upgrade: upgrades gxy to 2, tsi to base - completed = run_command(f"./db.sh upgrade") + completed = run_command("./db.sh upgrade") assert completed.returncode == 0 assert "Running upgrade gxy0 -> 1" in completed.stderr assert "Running upgrade 1 -> 2" in completed.stderr @@ -285,7 +265,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="3", head="2") # next upgrade: upgrades gxy to 3 - completed = run_command(f"./db.sh upgrade") + completed = run_command("./db.sh upgrade") assert completed.returncode == 0 assert "Running upgrade 2 -> 3" in completed.stderr assert "tsi0" not in completed.stderr # no effect on tsi @@ -297,14 +277,14 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command(f"./db.sh upgrade --sql") + completed = run_command("./db.sh upgrade --sql") assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='3'" not in completed.stdout alembic.command.revision(config, rev_id="3", head="2") - completed = run_command(f"./db.sh upgrade --sql") + completed = run_command("./db.sh upgrade --sql") assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='3'" in completed.stdout @@ -314,7 +294,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="2", head="1") # upgrades gxy to 1 - completed = run_command(f"./db.sh upgrade 1") + completed = run_command("./db.sh upgrade 1") assert completed.returncode == 0 assert "Running upgrade gxy0 -> 1" in completed.stderr @@ -329,7 +309,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="e", head="d") # upgrades gxy to b: none + 2 (none -> base -> a) - completed = run_command(f"./db.sh upgrade +3") + completed = run_command("./db.sh upgrade +3") assert completed.returncode == 0 assert "Running upgrade -> gxy0" in completed.stderr assert "Running upgrade gxy0 -> a" in completed.stderr @@ -339,7 +319,7 @@ class TestUpgradeCommand: assert heads == ("b",) # upgrades gxy to d relative to b: b + 2 (b -> c -> d) - completed = run_command(f"./db.sh upgrade b+2") + completed = run_command("./db.sh upgrade b+2") assert completed.returncode == 0 assert "Running upgrade b -> c" in completed.stderr assert "Running upgrade c -> d" in completed.stderr @@ -355,7 +335,7 @@ class TestDowngradeCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command(f"./db.sh downgrade 1") # downgrade gxy to 1 + completed = run_command("./db.sh downgrade 1") # downgrade gxy to 1 assert completed.returncode == 0 assert "Running downgrade 3 -> 2" in completed.stderr assert "Running downgrade 2 -> 1" in completed.stderr @@ -370,13 +350,13 @@ class TestDowngradeCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command(f"./db.sh downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi + completed = run_command("./db.sh downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='1'" in completed.stdout def test_downgrade_cmd_missing_revision_arg_error(self): - completed = run_command(f"./db.sh downgrade") + completed = run_command("./db.sh downgrade") assert completed.returncode == 2 assert "the following arguments are required: revision" in completed.stderr @@ -389,7 +369,7 @@ class TestDowngradeCommand: alembic.command.upgrade(config, "heads") # downgrades gxy to c: e - 2 (e -> d -> c) - completed = run_command(f"./db.sh downgrade -2") + completed = run_command("./db.sh downgrade -2") assert completed.returncode == 0 assert "Running downgrade e -> d" in completed.stderr @@ -399,7 +379,7 @@ class TestDowngradeCommand: assert "c" in heads # downgrades gxy to a relative to c: c - 2 (c -> b -> a) - completed = run_command(f"./db.sh downgrade c-2") + completed = run_command("./db.sh downgrade c-2") assert completed.returncode == 0 assert "Running downgrade c -> b" in completed.stderr assert "Running downgrade b -> a" in completed.stderr @@ -413,12 +393,12 @@ class TestDbVersionCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command(f"./db.sh dbversion") + completed = run_command("./db.sh dbversion") assert completed.returncode == 0 assert "(head)" not in completed.stdout # there has been no upgrade alembic.command.upgrade(config, "heads") - completed = run_command(f"./db.sh dbversion") + completed = run_command("./db.sh dbversion") assert completed.returncode == 0 assert "2 (head)" in completed.stdout From e0e30a57448ab4507bdbef52b3a8836f0bb6312b Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 8 Sep 2022 22:58:16 -0400 Subject: [PATCH 09/34] Fix packages tests: script is one level up --- test/unit/data/model/migrations/test_dbscript.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index 8dcfc331eea..93dd77fcfba 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -141,10 +141,19 @@ def dburl_from_config(config): def run_command(cmd): + if in_packages(): + cmd = f"../.{cmd}" # if this is run from `packages`, db.sh is in parent directory completed_process = subprocess.run(cmd.split(), capture_output=True, text=True) return completed_process +def in_packages(): + """Checks if test is run from the packages directory.""" + path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, os.pardir) + path = os.path.normpath(path) + return os.path.split(path)[1] == "packages" + + def get_db_heads(config): dburl = dburl_from_config(config) engine = create_engine(dburl) From bf83b4347acecc99944265e561440914a2164e9f Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 11:59:25 -0400 Subject: [PATCH 10/34] Fix mypy by asserting object before accessing its attributes --- test/unit/data/model/migrations/test_dbscript.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index 93dd77fcfba..bb8225bb471 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -175,6 +175,7 @@ class TestRevisionCommand: assert len(revisions) == 5 # verify total revisions: 2 base + 3 new rev = script_dir.get_revision("3") + assert rev assert GXY_BRANCH_LABEL in rev.branch_labels # verify branch label assert rev.down_revision == "2" # verify parent revision assert rev.module.__name__ == "3_foo3_py" # verify message From 889a8445bdc023315e650b6bff4512c73a4f1921 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 13:29:05 -0400 Subject: [PATCH 11/34] Change parsing of --galaxy-config option Rationale: alternate config file applies to all subcommands, so, I think, it is more correct to run this: > ./db.sh -c mygalaxy.yml upgrade some-revision-id than this: > ./db.sh upgrade some-revision-id -c mygalaxy.yml That's also consistent with how alembic handles this option. --- scripts/db.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index 2365cae6758..a847948c487 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -67,8 +67,7 @@ def main() -> None: return parser config_arg_parser = ArgumentParser(add_help=False) - # TODO: after refactoring legacy scripts, this can be changed to "-c, --config" - config_arg_parser.add_argument("--galaxy-config", help="Alternate Galaxy configuration file", dest="config") + config_arg_parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") verbose_arg_parser = ArgumentParser(add_help=False) verbose_arg_parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") @@ -84,6 +83,7 @@ def main() -> None: description="Common database schema migration operations", epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." " For migrating the `tsi` branch, use the `run_alembic.sh` script.", + parents=[config_arg_parser], ) subparsers = parser.add_subparsers(required=True) @@ -93,7 +93,7 @@ def main() -> None: exec_upgrade, "Upgrade to a later version", aliases=["u"], - parents=[config_arg_parser, sql_arg_parser], + parents=[sql_arg_parser], ) upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?") @@ -102,7 +102,7 @@ def main() -> None: exec_downgrade, "Revert to a previous version", aliases=["d"], - parents=[config_arg_parser, sql_arg_parser], + parents=[sql_arg_parser], ) downgrade_cmd_parser.add_argument("revision", help="Revision identifier") @@ -111,7 +111,7 @@ def main() -> None: exec_version, "Show the head revision in the migrations script directory", aliases=["v"], - parents=[config_arg_parser, verbose_arg_parser], + parents=[verbose_arg_parser], ) add_parser( @@ -119,14 +119,14 @@ def main() -> None: exec_dbversion, "Show the current revision for Galaxy's database", aliases=["dbv"], - parents=[config_arg_parser, verbose_arg_parser], + parents=[verbose_arg_parser], ) history_cmd_parser = add_parser( "history", exec_history, "List revision scripts in chronological order", - parents=[config_arg_parser, verbose_arg_parser], + parents=[verbose_arg_parser], ) history_cmd_parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") @@ -134,13 +134,10 @@ def main() -> None: "show", exec_show, "Show the revision(s) denoted by the given symbol", - parents=[config_arg_parser], ) show_cmd_parser.add_argument("revision", help="Revision identifier") - revision_cmd_parser = add_parser( - "revision", aliases=["r"], help="Create a new revision file", parents=[config_arg_parser], func=exec_revision - ) + revision_cmd_parser = add_parser("revision", aliases=["r"], help="Create a new revision file", func=exec_revision) revision_cmd_parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) revision_cmd_parser.add_argument("--rev-id", help="Specify a revision id instead of generating one") @@ -148,7 +145,6 @@ def main() -> None: "init", exec_init, "Initialize empty database(s) for both branches (create database objects for gxy and tsi branch)", - parents=[config_arg_parser], ) args = parser.parse_args() From 40249133cc6541ce79dbe29c84522bd021b6d0c7 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 13:40:24 -0400 Subject: [PATCH 12/34] Revise db.sh cli command aliases 1. Remove aliases from subcommands that change things (upgrade, downgrade, revision, init) to reduce risk of accidental usage. 2. Add aliases to readonly subcommands (history, show, version, dbversion) 3. Minor edits --- scripts/db.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index a847948c487..b8983b5d20f 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -92,7 +92,6 @@ def main() -> None: "upgrade", exec_upgrade, "Upgrade to a later version", - aliases=["u"], parents=[sql_arg_parser], ) upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?") @@ -101,7 +100,6 @@ def main() -> None: "downgrade", exec_downgrade, "Revert to a previous version", - aliases=["d"], parents=[sql_arg_parser], ) downgrade_cmd_parser.add_argument("revision", help="Revision identifier") @@ -118,7 +116,7 @@ def main() -> None: "dbversion", exec_dbversion, "Show the current revision for Galaxy's database", - aliases=["dbv"], + aliases=["dv"], parents=[verbose_arg_parser], ) @@ -126,6 +124,7 @@ def main() -> None: "history", exec_history, "List revision scripts in chronological order", + aliases=["h"], parents=[verbose_arg_parser], ) history_cmd_parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") @@ -134,12 +133,15 @@ def main() -> None: "show", exec_show, "Show the revision(s) denoted by the given symbol", + aliases=["s"], ) show_cmd_parser.add_argument("revision", help="Revision identifier") - revision_cmd_parser = add_parser("revision", aliases=["r"], help="Create a new revision file", func=exec_revision) + revision_cmd_parser = add_parser("revision", help="Create a new revision file", func=exec_revision) revision_cmd_parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) - revision_cmd_parser.add_argument("--rev-id", help="Specify a revision id instead of generating one") + revision_cmd_parser.add_argument( + "--rev-id", help="Specify a revision id instead of generating one (This option is for testing purposes only)" + ) add_parser( "init", From b4b76c91a28db2f03e5e706dec3975d4661896ab Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 13:51:40 -0400 Subject: [PATCH 13/34] Add missing tests --- .../data/model/migrations/test_dbscript.py | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index bb8225bb471..10bc7f4cd24 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -257,6 +257,33 @@ class TestVersionCommand: assert "Revises: 1" in completed.stdout +class TestDbVersionCommand: + def test_dbversion_cmd(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command("./db.sh dbversion") + assert completed.returncode == 0 + assert "(head)" not in completed.stdout # there has been no upgrade + + alembic.command.upgrade(config, "heads") + + completed = run_command("./db.sh dbversion") + assert completed.returncode == 0 + assert "2 (head)" in completed.stdout + + def test_dbversion_cmd_verbose(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + alembic.command.upgrade(config, "heads") + + completed = run_command("./db.sh dbversion --verbose") + assert completed.returncode == 0 + assert "Revision ID: 2" in completed.stdout + assert "Revises: 1" in completed.stdout + + class TestUpgradeCommand: def test_upgrade_cmd(self, config): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) @@ -396,19 +423,3 @@ class TestDowngradeCommand: heads = get_db_heads(config) assert "a" in heads - - -class TestDbVersionCommand: - def test_dbversion_cmd(self, config): - alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) - alembic.command.revision(config, rev_id="2", head="1") - - completed = run_command("./db.sh dbversion") - assert completed.returncode == 0 - assert "(head)" not in completed.stdout # there has been no upgrade - - alembic.command.upgrade(config, "heads") - - completed = run_command("./db.sh dbversion") - assert completed.returncode == 0 - assert "2 (head)" in completed.stdout From d8e8a8efb600f80a201689bb6c4c6695c3a326e1 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 14:48:06 -0400 Subject: [PATCH 14/34] Add --raiseerr option; hide error traceback by default --- scripts/db.py | 18 +++++++++++++----- .../data/model/migrations/test_dbscript.py | 7 +++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/scripts/db.py b/scripts/db.py index b8983b5d20f..50a4b76b365 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -10,6 +10,8 @@ from argparse import ( Namespace, ) +import alembic + sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) from galaxy.model.migrations import verify_databases_via_script @@ -55,7 +57,15 @@ def exec_init(args: Namespace) -> None: def _exec_command(command, args): dbscript = DbScript(args.config) - getattr(dbscript, command)(args) + try: + getattr(dbscript, command)(args) + except alembic.util.exc.CommandError as e: + if args.raiseerr: + raise + else: + log.error(e) + print(f"FAILED: {str(e)}") + sys.exit(1) def main() -> None: @@ -66,9 +76,6 @@ def main() -> None: parser.set_defaults(func=func) return parser - config_arg_parser = ArgumentParser(add_help=False) - config_arg_parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") - verbose_arg_parser = ArgumentParser(add_help=False) verbose_arg_parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") @@ -83,8 +90,9 @@ def main() -> None: description="Common database schema migration operations", epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." " For migrating the `tsi` branch, use the `run_alembic.sh` script.", - parents=[config_arg_parser], ) + parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") + parser.add_argument("--raiseerr", help="Raise a full stack trace on error", action="store_true") subparsers = parser.add_subparsers(required=True) diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index 10bc7f4cd24..f6acaad81d8 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -196,6 +196,13 @@ class TestShowCommand: alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) completed = run_command("./db.sh show idonotexist") assert completed.returncode == 1 + assert "Traceback" not in completed.stderr + assert "Can't locate revision identified by 'idonotexist'" in completed.stderr + + def test_show_cmd_invalid_revision_error_with_traceback(self): + completed = run_command("./db.sh --raiseerr show idonotexist") + assert completed.returncode == 1 + assert "Traceback" in completed.stderr assert "Can't locate revision identified by 'idonotexist'" in completed.stderr def test_show_cmd_missing_revision_arg_error(self): From dc823d589cde655732a06e95e6a2c2731016f166 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 16:48:45 -0400 Subject: [PATCH 15/34] Misc edits to db.sh --- db.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/db.sh b/db.sh index 0dfa2a8cfea..cc9df6105f0 100755 --- a/db.sh +++ b/db.sh @@ -1,7 +1,12 @@ #!/bin/sh -# draft of script to replace manage_db.sh -cd `dirname $0` +####### +# Use this script to manage Galaxy database schema migrations. +# For help, run `sh db.sh -h`. +# For detailed help, see documentation at lib/galaxy/model/migrations/README.md. +####### + +cd "$(dirname "$0")" || exit . ./scripts/common_startup_functions.sh From ed1f6702e550cbe1756d78e1f6dac2f77ed964ea Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 17:10:08 -0400 Subject: [PATCH 16/34] Rename manage_db.sh to manage_toolshed_db.sh. Limit to TS operations. Becuase everything else is handled via db.sh or run_alembic.sh --- manage_db.sh | 50 ------------------------------------------- manage_toolshed_db.sh | 14 ++++++++++++ 2 files changed, 14 insertions(+), 50 deletions(-) delete mode 100755 manage_db.sh create mode 100755 manage_toolshed_db.sh diff --git a/manage_db.sh b/manage_db.sh deleted file mode 100755 index 6200d4b98a8..00000000000 --- a/manage_db.sh +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/sh - -####### -# Use this script to manage Galaxy, Tool Shed Install, and Tool Shed database migrations. -# -# For advanced usage and access to the full scope of command line options provided by -# Alembic, you may use the run_alembic.sh script. However, for regular database management -# tasks, we encourage you to use the manage_db.sh script. -# -# NOTE: If your database is empty, use create_db.sh instead. -# -# Database options: galaxy (default), install, tool_shed -# To pass a galaxy config file, you may use `-c|--config|--config-file your-config-file` -# -# To upgrade or downgrade to some version X: -# sh manage_db.sh [upgrade|downgrade] --version=X [tool_shed|install|galaxy] -# -# You may also skip the version argument when upgrading, in which case the database -# will be upgraded to the latest version. -# -# Example 1: upgrade "galaxy" database to version "abc123" using default config: -# sh manage_db.sh upgrade --version=abc123 -# -# Example 2: downgrade "install" database to version "xyz789" passing config file "mygalaxy.yml": -# sh manage_db.sh downgrade --version=xyz789 -c mygalaxy.yml install -# -# Example 3: upgrade "galaxy" database to latest version using default config: -# sh manage_db.sh upgrade -# -# (Note: Tool Shed migrations use the legacy migrations system, so we check the -# last argument (the database) to invoke the appropriate script. Therefore, if -# you don't specify the database (galaxy is used by default) and pass a config -# file, your config file should not be named `tool_shed`.) -####### - -ALEMBIC_CONFIG='lib/galaxy/model/migrations/alembic.ini' - -cd `dirname $0` - -. ./scripts/common_startup_functions.sh - -setup_python - -for i; do :; done -if [ "$i" = "tool_shed" ]; then - python ./scripts/migrate_toolshed_db.py "$@" tool_shed -else - find lib/galaxy/model/migrations/alembic -name '*.pyc' -delete - python ./scripts/manage_db_adapter.py --alembic-config "$ALEMBIC_CONFIG" "$@" -fi diff --git a/manage_toolshed_db.sh b/manage_toolshed_db.sh new file mode 100755 index 00000000000..414fd47dfeb --- /dev/null +++ b/manage_toolshed_db.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +####### +# Use this script to manage Tool Shed database migrations. +# NOTE: If your database is empty, use create_toolshed_db.sh instead. +####### + +cd "$(dirname "$0")" || exit + +. ./scripts/common_startup_functions.sh + +setup_python + +python ./scripts/migrate_toolshed_db.py "$@" tool_shed From 5d376fdacc8334c36dcc5d02a300e35c570a6ec6 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 17:18:15 -0400 Subject: [PATCH 17/34] Drop create_db.sh, create_db.py (see note) For ToolShed, we have create_toolshed_db.sh. For galaxy and toolshed install databases (gxy and tsi branches under alembic), we use the `init` subcommand of the `db.sh` script (`./db.sh init`), which has exactly the same functionality as these deleted scripts. To the best of my knowledge, these are not used anywhere else (unlike, for example, scripts/manage_db.py, which is called from the ansible-galaxy role). --- create_db.sh | 25 ------------------------- scripts/create_db.py | 30 ------------------------------ scripts/db_shell.py | 1 - tox.ini | 2 +- 4 files changed, 1 insertion(+), 57 deletions(-) delete mode 100755 create_db.sh delete mode 100755 scripts/create_db.py diff --git a/create_db.sh b/create_db.sh deleted file mode 100755 index 690d0c6f8c8..00000000000 --- a/create_db.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -####### -# Use this script to verify the state of the Galaxy and Tool Shed Install -# database(s). If the database does not exist or is empty, it will be created -# and initialized. -# (Use create_toolshed_db.sh to create and initialize a new -# Tool Shed database.) -# -# To pass a galaxy config file, use `--galaxy-config` -# -# You may also override the galaxy database url and/or the -# tool shed install database url, as well as the database_template -# and database_encoding configuration options with env vars: -# GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-db-url ./create_db.sh -# GALAXY_INSTALL_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-other-db-url ./create_db.sh -####### - -cd "$(dirname "$0")" - -. ./scripts/common_startup_functions.sh - -setup_python - -python ./scripts/create_db.py "$@" diff --git a/scripts/create_db.py b/scripts/create_db.py deleted file mode 100755 index c9d5b750084..00000000000 --- a/scripts/create_db.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -This script retrieves relevant configuration values and verifies the state of -the Galaxy and Tool Shed Install database(s). -There may be one combined database (galaxy and tool shed install) or two -separate databases. -If the database does not exist or is empty, it will be created and initialized. -(See inline comments in lib/galaxy/model/migrations/__init__.py for details on -how other database states are handled). -It is wrapped by create_db.sh (see that file for usage). -""" -import logging -import os.path -import sys - -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) - -from galaxy.model.migrations import verify_databases_via_script -from galaxy.model.migrations.scripts import get_configuration - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) - - -def invoke_create(): - gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) - verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) - - -if __name__ == "__main__": - invoke_create() diff --git a/scripts/db_shell.py b/scripts/db_shell.py index a74bce329aa..66098309a5e 100644 --- a/scripts/db_shell.py +++ b/scripts/db_shell.py @@ -11,7 +11,6 @@ # % ipython -i scripts/db_shell.py -- -c config/galaxy.ini # # You can also use this script as a library, for instance see https://gist.github.com/1979583 -# TODO: This script overlaps a lot with manage_db.py and create_db.py, # these should maybe be refactored to remove duplication. import datetime diff --git a/tox.ini b/tox.ini index 588d6e53832..ee55e978ed6 100644 --- a/tox.ini +++ b/tox.ini @@ -58,5 +58,5 @@ commands = bash .ci/check_controller.sh [testenv:check_indexes] commands = bash scripts/common_startup.sh - bash create_db.sh + bash db.sh init bash check_model.sh From 5e8add0e46a963cfde5363be10c2a0dc434ec59a Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 17:35:20 -0400 Subject: [PATCH 18/34] Rename migrate_db.py > run_alembic.py This is consistent with other similar pairs of files: [script-name].sh invoking scripts/[script-name].py (check_model, create_roolshed_db, db) Also, the python file does not require exec permissions. --- run_alembic.sh | 4 ++-- scripts/{migrate_db.py => run_alembic.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename scripts/{migrate_db.py => run_alembic.py} (100%) mode change 100755 => 100644 diff --git a/run_alembic.sh b/run_alembic.sh index 0dfeddd85a0..15fc1555f4c 100755 --- a/run_alembic.sh +++ b/run_alembic.sh @@ -46,11 +46,11 @@ ALEMBIC_CONFIG='lib/galaxy/model/migrations/alembic.ini' -cd `dirname $0` +cd "$(dirname "$0")" || exit . ./scripts/common_startup_functions.sh setup_python find lib/galaxy/model/migrations/alembic -name '*.pyc' -delete -python ./scripts/migrate_db.py --config "$ALEMBIC_CONFIG" "$@" +python ./scripts/run_alembic.py --config "$ALEMBIC_CONFIG" "$@" diff --git a/scripts/migrate_db.py b/scripts/run_alembic.py old mode 100755 new mode 100644 similarity index 100% rename from scripts/migrate_db.py rename to scripts/run_alembic.py From b0a3c013ab22173cfb09962b9c2680299a03f6b1 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 17:44:53 -0400 Subject: [PATCH 19/34] Remove manage_db_adapter.py We no longer need to translate the arguments of manage_db.sh into arguments for alembic. The new facade script for alembic (db.sh) accesses the alembic api programmatically. --- scripts/manage_db_adapter.py | 41 ------------------------------------ 1 file changed, 41 deletions(-) delete mode 100644 scripts/manage_db_adapter.py diff --git a/scripts/manage_db_adapter.py b/scripts/manage_db_adapter.py deleted file mode 100644 index f19974486e8..00000000000 --- a/scripts/manage_db_adapter.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -This script is intended to be invoked by the manage_db.sh script. -It translates the arguments supplied to manage_db.sh into the format used -by migrate_db.py. - -INPUT: | OUTPUT: ----------------------------------------------------------- -upgrade --version=foo | upgrade foo -upgrade --version foo | upgrade foo -upgrade | upgrade heads (if using a combined db for galaxy and install) -upgrade | upgrade gxy@head (if using separate dbs for galaxy and install) -upgrade install | upgrade tsi@head -upgrade --version=bar install | upgrade bar -upgrade -c path-to-galaxy.yml | upgrade --galaxy-config path-to-galaxy.yml gxy@head - -The converted sys.argv will include `-c path-to-alembic.ini`. -The optional `-c` argument name is renamed to `--galaxy-config`. -""" - -import os -import sys - -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) - -from galaxy.model.migrations.scripts import ( - invoke_alembic, - LegacyScripts, - verify_database_is_initialized, -) - - -def run(): - ls = LegacyScripts(sys.argv, os.getcwd()) - ls.run() - db_url = ls.get_db_url() - verify_database_is_initialized(db_url) - invoke_alembic() - - -if __name__ == "__main__": - run() From 6fcc58a85c2998b08de0def7ba235793f6dbf39c Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 19:31:25 -0400 Subject: [PATCH 20/34] Drop LegacyScripts: no longer used --- lib/galaxy/model/migrations/scripts.py | 143 +++--------------- .../data/model/migrations/test_scripts.py | 141 ++--------------- 2 files changed, 37 insertions(+), 247 deletions(-) diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py index c727884cab2..7b41d9925ec 100644 --- a/lib/galaxy/model/migrations/scripts.py +++ b/lib/galaxy/model/migrations/scripts.py @@ -1,5 +1,4 @@ import os -import re import sys from typing import ( List, @@ -14,10 +13,7 @@ from alembic.script import ScriptDirectory from sqlalchemy import create_engine from sqlalchemy.engine import Engine -from galaxy.model.database_utils import ( - database_exists, - is_one_database, -) +from galaxy.model.database_utils import database_exists from galaxy.model.migrations import ( AlembicManager, DatabaseConfig, @@ -160,120 +156,10 @@ class LegacyScriptsException(Exception): super().__init__(message) -class LegacyScripts: +class LegacyManageDb: LEGACY_CONFIG_FILE_ARG_NAMES = ["-c", "--config", "--config-file"] - ALEMBIC_CONFIG_FILE_ARG = "--alembic-config" # alembic config file, set in the calling script - DEFAULT_DB_ARG = "default" - def __init__(self, argv: List[str], cwd: Optional[str] = None) -> None: - self.argv = argv - self.cwd = cwd or os.getcwd() - self._database: Optional[str] = None # Do not assign default value: `None` means we don't know yet. - - @property - def database(self): - if self._database is None: - raise LegacyScriptsException( - "Attempt to access identifier of database before processing the script arguments" - ) - return self._database - - def run(self) -> None: - """ - Convert legacy arguments to current spec required by Alembic, - then add db url arguments required by Alembic - """ - self.convert_args() - add_db_urls_to_command_arguments(self.argv, self.gxy_url, self.tsi_url) - - def convert_args(self) -> None: - """ - Convert legacy arguments to current spec required by Alembic. - - Note: The following method calls must be done in this sequence. - """ - self.pop_database_argument() - self.rename_config_argument() - self.rename_alembic_config_argument() - self.load_db_urls() - self.convert_version_argument() - - def pop_database_argument(self) -> None: - """ - If last argument is a valid database name, pop and assign it; otherwise assign default. - """ - arg = self.argv[-1] - self._database = self.DEFAULT_DB_ARG - if arg in ["galaxy", "install"]: - self._database = self.argv.pop() - - def rename_config_argument(self) -> None: - """ - Rename the optional config argument: we can't use '-c' because that option is used by Alembic. - """ - for arg in self.LEGACY_CONFIG_FILE_ARG_NAMES: - if arg in self.argv: - self._rename_arg(arg, CONFIG_FILE_ARG) - return - - def rename_alembic_config_argument(self) -> None: - """ - Rename argument name: `--alembic-config` to `-c`. There should be no `-c` argument present. - """ - if "-c" in self.argv: - raise LegacyScriptsException("Cannot rename alembic config argument: `-c` argument present.") - self._rename_arg(self.ALEMBIC_CONFIG_FILE_ARG, "-c") - - def convert_version_argument(self) -> None: - """ - Convert legacy version argument to current spec required by Alembic. - """ - if "--version" in self.argv: - # Just remove it: the following argument should be the version/revision identifier. - pos = self.argv.index("--version") - self.argv.pop(pos) - else: - # If we find --version=foo, extract foo and replace arg with foo (which is the revision identifier) - p = re.compile(r"--version=([0-9A-Fa-f]+)") - for i, arg in enumerate(self.argv): - m = p.match(arg) - if m: - self.argv[i] = m.group(1) - return - # No version argument found: construct argument for an upgrade operation. - # Raise exception otherwise. - if "upgrade" not in self.argv: - raise LegacyScriptsException("If no `--version` argument supplied, `upgrade` argument is requried") - - if self._is_one_database(): # upgrade both regardless of database argument - self.argv.append("heads") - else: # for separate databases, choose one - if self.database in ["galaxy", self.DEFAULT_DB_ARG]: - self.argv.append("gxy@head") - elif self.database == "install": - self.argv.append("tsi@head") - - def get_db_url(self): - if self.database in ["galaxy", self.DEFAULT_DB_ARG]: - return self.gxy_url - elif self.database == "install": - return self.tsi_url - - def _rename_arg(self, old_name, new_name) -> None: - pos = self.argv.index(old_name) - self.argv[pos] = new_name - - def load_db_urls(self) -> None: - gxy_config, tsi_config, _ = get_configuration(self.argv, self.cwd) - self.gxy_url = gxy_config.url - self.tsi_url = tsi_config.url - - def _is_one_database(self): - return is_one_database(self.gxy_url, self.tsi_url) - - -class LegacyManageDb: def __init__(self): self._set_db_urls() @@ -320,6 +206,19 @@ class LegacyManageDb: self._upgrade(gxy_db_url, GXY) self._upgrade(tsi_db_url, TSI) + def rename_config_argument(self, argv: List[str]) -> None: + """ + Rename the optional config argument: we can't use '-c' because that option is used by Alembic. + """ + for arg in self.LEGACY_CONFIG_FILE_ARG_NAMES: + if arg in argv: + self._rename_arg(argv, arg, CONFIG_FILE_ARG) + return + + def _rename_arg(self, argv, old_name, new_name) -> None: + pos = argv.index(old_name) + argv[pos] = new_name + def _upgrade(self, db_url, model): try: engine = create_engine(db_url) @@ -329,11 +228,13 @@ class LegacyManageDb: engine.dispose() def _set_db_urls(self): - ls = LegacyScripts(sys.argv, os.getcwd()) - ls.rename_config_argument() - ls.load_db_urls() - self.gxy_db_url = ls.gxy_url - self.tsi_db_url = ls.tsi_url + self.rename_config_argument(sys.argv) + self._load_db_urls() + + def _load_db_urls(self): + gxy_config, tsi_config, _ = get_configuration(sys.argv, os.getcwd()) + self.gxy_db_url = gxy_config.url + self.tsi_db_url = tsi_config.url def _get_gxy_sam_db_version(self, engine): dbcache = DatabaseStateCache(engine) diff --git a/test/unit/data/model/migrations/test_scripts.py b/test/unit/data/model/migrations/test_scripts.py index 0472b7503a7..b34a8cb3fc3 100644 --- a/test/unit/data/model/migrations/test_scripts.py +++ b/test/unit/data/model/migrations/test_scripts.py @@ -5,8 +5,7 @@ import pytest from galaxy.model.migrations.scripts import ( DatabaseDoesNotExistError, DatabaseNotInitializedError, - LegacyScripts, - LegacyScriptsException, + LegacyManageDb, verify_database_is_initialized, ) @@ -16,44 +15,17 @@ def set_db_urls(monkeypatch): # Do not try to access galaxy config; values not needed. def no_config_call(self): self.gxy_url = "a string" - self.tsi_url = "a stirng" + self.tsi_url = "a string" - monkeypatch.setattr(LegacyScripts, "load_db_urls", no_config_call) + monkeypatch.setattr(LegacyManageDb, "_load_db_urls", no_config_call) -@pytest.fixture(autouse=True) # set combined db for all tests -def set_combined(monkeypatch): - monkeypatch.setattr(LegacyScripts, "_is_one_database", lambda self: True) - - -@pytest.fixture -def set_separate(monkeypatch): - monkeypatch.setattr(LegacyScripts, "_is_one_database", lambda self: False) - - -class TestLegacyScripts: - @pytest.mark.parametrize("database_arg", ["galaxy", "install"]) - def test_pop_database_name(self, database_arg): - # arg_value = 'install' - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc", database_arg] - ls = LegacyScripts(argv) - - ls.pop_database_argument() - assert ls.database == database_arg - assert argv == ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - - def test_pop_database_name_use_default(self): - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - ls = LegacyScripts(argv) - ls.pop_database_argument() - assert ls.database == LegacyScripts.DEFAULT_DB_ARG - assert argv == ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - - @pytest.mark.parametrize("arg_name", LegacyScripts.LEGACY_CONFIG_FILE_ARG_NAMES) +class TestLegacyManageDb: + @pytest.mark.parametrize("arg_name", LegacyManageDb.LEGACY_CONFIG_FILE_ARG_NAMES) def test_rename_config_arg(self, arg_name): # `-c|--config|__config-file` should be renamed to `--galaxy-config` argv = ["caller", "--alembic-config", "path-to-alembic", arg_name, "path-to-galaxy", "upgrade", "--version=abc"] - LegacyScripts(argv).rename_config_argument() + LegacyManageDb().rename_config_argument(argv) assert argv == [ "caller", "--alembic-config", @@ -67,7 +39,7 @@ class TestLegacyScripts: def test_rename_config_arg_reordered_args(self): # `-c|--config|__config-file` should be renamed to `--galaxy-config` argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc", "-c", "path-to-galaxy"] - LegacyScripts(argv).rename_config_argument() + LegacyManageDb().rename_config_argument(argv) assert argv == [ "caller", "--alembic-config", @@ -78,97 +50,14 @@ class TestLegacyScripts: "path-to-galaxy", ] - def test_rename_alembic_config_arg(self): - # `--alembic-config` should be renamed to `-c` - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - LegacyScripts(argv).rename_alembic_config_argument() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "--version=abc"] - def test_rename_alembic_config_arg_raises_error_if_c_arg_present(self): - # Ensure alembic config arg is renamed AFTER renaming the galaxy config arg. Raise error otherwise. - argv = ["caller", "--alembic-config", "path-to-alembic", "-c", "path-to-galaxy", "upgrade", "--version=abc"] - with pytest.raises(LegacyScriptsException): - LegacyScripts(argv).rename_alembic_config_argument() +def test_verify_database_is_init_raises_error_if_no_database(): + nonexistant_path = str(random.random())[2:] + db_url = f"sqlite:////{nonexistant_path}" + with pytest.raises(DatabaseDoesNotExistError): + verify_database_is_initialized(db_url) - def test_convert__version_arg_1(self): - # `sh manage_db.sh upgrade --version X` >> `... upgrade X` - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version", "abc"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "abc"] - def test_convert__version_arg_2(self): - # `sh manage_db.sh upgrade --version=X` >> `... upgrade X` - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "abc"] - - def test_convert__no_version_no_model_combined_database(self): - # `sh manage_db.sh upgrade` >> `... upgrade heads` - # No version and no model implies "upgrade the default db (which is galaxy) to its latest version". - # If it is combined, we upgrade both models: gxy and tsi. - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"] - - def test_convert__no_version_galaxy_model_combined_database_(self): - # `sh manage_db.sh upgrade galaxy` >> `... upgrade heads` - # same as no model: if combined we upgrade the whole database - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "galaxy"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"] - - def test_convert__no_version_install_model_combined_database_(self): - # `sh manage_db.sh upgrade install` >> `... upgrade heads` - # same as no model: if combined we upgrade the whole database - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "install"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"] - - def test_convert__no_version_no_model_separate_databases(self, set_separate): - # `sh manage_db.sh upgrade` >> `... upgrade gxy@head` - # No version and no model implies "upgrade the default db (which is galaxy) to its latest version". - # Since the tsi model has its own db, we only upgrade the gxy model. - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "gxy@head"] - - def test_convert__no_version_galaxy_model_separate_databases(self, set_separate): - # `sh manage_db.sh upgrade galaxy` >> `... upgrade gxy@head` - # No version + a model implies "upgrade the db for the specified model to its latest version". - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "galaxy"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "gxy@head"] - - def test_convert__no_version_install_model_separate_databases(self, set_separate): - # `sh manage_db.sh upgrade install` >> `... upgrade tsi@head` - # No version + a model implies "upgrade the db for the specified model to its latest version". - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "install"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "tsi@head"] - - def test_downgrade_with_no_version_argument_raises_error(self): - argv = ["caller", "--alembic-config", "path-to-alembic", "downgrade"] - with pytest.raises(LegacyScriptsException): - LegacyScripts(argv).convert_args() - - def test_access_database_id(self): - db = "galaxy" - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", db] - ls = LegacyScripts(argv) - ls.run() - assert ls.database == db - - def test_access_database_id_before_processing_script_args_raises_error(self): - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"] - with pytest.raises(LegacyScriptsException): - LegacyScripts(argv).database - - def test_verify_database_is_init_raises_error_if_no_database(self): - nonexistant_path = str(random.random())[2:] - db_url = f"sqlite:////{nonexistant_path}" - with pytest.raises(DatabaseDoesNotExistError): - verify_database_is_initialized(db_url) - - def test_verify_database_is_init_raises_error_if_database_not_initialized(self, sqlite_memory_url): - with pytest.raises(DatabaseNotInitializedError): - verify_database_is_initialized(sqlite_memory_url) +def test_verify_database_is_init_raises_error_if_database_not_initialized(sqlite_memory_url): + with pytest.raises(DatabaseNotInitializedError): + verify_database_is_initialized(sqlite_memory_url) From 5902cadf7f8d853b0a67b23ee1d8bfc476f4e909 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 9 Sep 2022 22:54:51 -0400 Subject: [PATCH 21/34] Remove duplicate migration code --- lib/galaxy/model/migrations/dbscript.py | 41 ++----------------------- lib/galaxy/model/migrations/scripts.py | 8 ++++- 2 files changed, 10 insertions(+), 39 deletions(-) diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index 6f55fb834be..57474ddd26a 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -1,19 +1,11 @@ import argparse import os -from typing import ( - Optional, - Tuple, -) +from typing import Optional from alembic import command from alembic.config import Config -from galaxy.model.migrations import DatabaseConfig -from galaxy.util.properties import ( - find_config_file, - get_data_dir, - load_app_properties, -) +from galaxy.model.migrations.scripts import get_configuration_from_file DEFAULT_CONFIG_NAMES = ["galaxy", "universe_wsgi"] CONFIG_FILE_ARG = "--galaxy-config" @@ -81,7 +73,7 @@ class DbScript: return Config(config_file) def _set_dburl(self, config_file: Optional[str] = None) -> None: - gxy_config, tsi_config = self._get_configuration(config_file) + gxy_config, tsi_config, _ = get_configuration_from_file(os.getcwd(), config_file) self.gxy_url = gxy_config.url self.tsi_url = tsi_config.url @@ -90,30 +82,3 @@ class DbScript: if rev.startswith("+") or rev.startswith("-"): return f"gxy@{rev}" return rev - - def _get_configuration(self, config_file: Optional[str] = None) -> Tuple[DatabaseConfig, DatabaseConfig]: - """ - Return a 2-item-tuple with configuration values used for managing databases. - """ - if config_file is None: - cwd = os.getcwd() - cwds = [cwd, os.path.join(cwd, CONFIG_DIR_NAME)] - config_file = find_config_file(DEFAULT_CONFIG_NAMES, dirs=cwds) - - # load gxy properties and auto-migrate - properties = load_app_properties(config_file=config_file, config_prefix=GXY_CONFIG_PREFIX) - default_url = f"sqlite:///{os.path.join(get_data_dir(properties), 'universe.sqlite')}?isolation_level=IMMEDIATE" - url = properties.get("database_connection", default_url) - template = properties.get("database_template", None) - encoding = properties.get("database_encoding", None) - gxy_config = DatabaseConfig(url, template, encoding) - - # load tsi properties - properties = load_app_properties(config_file=config_file, config_prefix=TSI_CONFIG_PREFIX) - default_url = gxy_config.url - url = properties.get("install_database_connection", default_url) - template = properties.get("database_template", None) - encoding = properties.get("database_encoding", None) - tsi_config = DatabaseConfig(url, template, encoding) - - return (gxy_config, tsi_config) diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py index 7b41d9925ec..aef7ff0f297 100644 --- a/lib/galaxy/model/migrations/scripts.py +++ b/lib/galaxy/model/migrations/scripts.py @@ -83,11 +83,17 @@ def verify_database_is_initialized(db_url: str) -> None: def get_configuration(argv: List[str], cwd: str) -> Tuple[DatabaseConfig, DatabaseConfig, bool]: - # TODO i think is_auto-migrate is not used! """ Return a 3-item-tuple with configuration values used for managing databases. """ config_file = _pop_config_file(argv) + return get_configuration_from_file(cwd, config_file) + + +def get_configuration_from_file( + cwd: str, config_file: Optional[str] = None +) -> Tuple[DatabaseConfig, DatabaseConfig, bool]: + if config_file is None: cwds = [cwd, os.path.join(cwd, CONFIG_DIR_NAME)] config_file = find_config_file(DEFAULT_CONFIG_NAMES, dirs=cwds) From dfd32a2ee814470570041f4877ae11e65d9fe84e Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 12 Sep 2022 19:21:17 -0400 Subject: [PATCH 22/34] Add type hints --- lib/galaxy/model/migrations/dbscript.py | 4 ++-- scripts/db.py | 2 +- .../data/model/migrations/test_dbscript.py | 23 +++++++++---------- 3 files changed, 14 insertions(+), 15 deletions(-) diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index 57474ddd26a..b016f9d43b6 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -65,7 +65,7 @@ class DbScript: def show(self, args: argparse.Namespace) -> None: command.show(self.alembic_config, args.revision) - def _get_alembic_cfg(self): + def _get_alembic_cfg(self) -> Config: config_file = os.getenv("ALEMBIC_CONFIG") if not config_file: config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") @@ -77,7 +77,7 @@ class DbScript: self.gxy_url = gxy_config.url self.tsi_url = tsi_config.url - def _parse_revision(self, rev): + def _parse_revision(self, rev: str) -> str: # Relative revision identifier requires a branch label if rev.startswith("+") or rev.startswith("-"): return f"gxy@{rev}" diff --git a/scripts/db.py b/scripts/db.py index 50a4b76b365..551cb4fb5c4 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -55,7 +55,7 @@ def exec_init(args: Namespace) -> None: verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) -def _exec_command(command, args): +def _exec_command(command: str, args: Namespace) -> None: dbscript = DbScript(args.config) try: getattr(dbscript, command)(args) diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index f6acaad81d8..fd32f5922a5 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -25,6 +25,7 @@ import tempfile from typing import ( List, NewType, + Tuple, ) import alembic @@ -97,7 +98,7 @@ def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, mon return alembic_cfg -def update_config_for_staging(config_text, script_location, version_locations, dburl) -> None: +def update_config_for_staging(config_text: List[str], script_location: str, version_locations: str, dburl: str) -> None: """Set script_location, version_locations, sqlalchemy.url values.""" alembic_section_index, url_set = -1, False url_line = f"sqlalchemy.url = {dburl}\n" @@ -115,12 +116,12 @@ def update_config_for_staging(config_text, script_location, version_locations, d config_text.insert(alembic_section_index + 1, url_line) -def write_config_file(config_file_path, config_text): +def write_config_file(config_file_path: str, config_text: str) -> None: with open(config_file_path, "w") as f: f.write("".join(config_text)) -def create_alembic_branches(config, gxy_versions_dir, tsi_versions_dir): +def create_alembic_branches(config: Config, gxy_versions_dir: str, tsi_versions_dir: str) -> None: """ Create gxy and tsi branches (required for galaxy's alembic setup; included with 22.05 release) """ @@ -132,29 +133,27 @@ def create_alembic_branches(config, gxy_versions_dir, tsi_versions_dir): ) -def stdout(capture): - return capture.readouterr().out +def dburl_from_config(config: Config) -> str: + url = config.get_main_option("sqlalchemy.url") + assert url + return url -def dburl_from_config(config): - return config.get_main_option("sqlalchemy.url") - - -def run_command(cmd): +def run_command(cmd: str) -> subprocess.CompletedProcess: if in_packages(): cmd = f"../.{cmd}" # if this is run from `packages`, db.sh is in parent directory completed_process = subprocess.run(cmd.split(), capture_output=True, text=True) return completed_process -def in_packages(): +def in_packages() -> bool: """Checks if test is run from the packages directory.""" path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, os.pardir) path = os.path.normpath(path) return os.path.split(path)[1] == "packages" -def get_db_heads(config): +def get_db_heads(config: Config) -> Tuple[str, ...]: dburl = dburl_from_config(config) engine = create_engine(dburl) with engine.connect() as conn: From 0c348a575982e3aefeb23cf6c7a3bc55d7e43e77 Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 12 Sep 2022 19:57:16 -0400 Subject: [PATCH 23/34] Update test comments --- .../data/model/migrations/test_dbscript.py | 21 +++++-------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index fd32f5922a5..c0de5e59dd3 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -1,23 +1,8 @@ """ Testing approach: -- Use a test database, store revision scripts in a different location; leave the rest unchanged. +- Use test database(s), store revision scripts in a different location; leave the rest unchanged. - Use alembic api for setup and accessing the database. - Run command as subprocess, verify captured output + database state. - -1. Setup staging environment: - - Create staging location (/tmp) - - Create test database (sqlite in /tmp) - - Copy production alembic.ini to staging location, overwriting: - - sqlalchemy.url (url of test database) - - version_locations (staging location) - - script_location (lib/galaxy/model/migrations/alembic/) - - Create gxy and tsi branches - -2. For each test case: - - Optionally, use alembic api for any setup - - Run command as a subprocess, capture output - - Run assertions against captured output - - Optionally, use alembic api to access database; verify database state """ import os import subprocess @@ -77,13 +62,16 @@ def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, mon """ Construct Config object for staging; setup staging env. """ + # Create staging location for revision sctipts gxy_versions_dir = os.path.join(tmp_directory, "versions_gxy") tsi_versions_dir = os.path.join(tmp_directory, "versions_tsi") version_locations = f"{gxy_versions_dir};{tsi_versions_dir}" + # Create test database(s) gxy_dburl = url_factory() tsi_dburl = gxy_dburl if request.param == "one database" else url_factory() + # Copy production alembic.ini to staging location config_file_path = os.path.join(tmp_directory, "alembic.ini") update_config_for_staging(alembic_config_text, alembic_env_dir, version_locations, gxy_dburl) write_config_file(config_file_path, alembic_config_text) @@ -91,6 +79,7 @@ def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, mon alembic_cfg = Config(config_file_path) create_alembic_branches(alembic_cfg, gxy_versions_dir, tsi_versions_dir) + # Point tests to test database(s) monkeypatch.setenv("ALEMBIC_CONFIG", config_file_path) monkeypatch.setenv("GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION", gxy_dburl) monkeypatch.setenv("GALAXY_INSTALL_CONFIG_OVERRIDE_INSTALL_DATABASE_CONNECTION", tsi_dburl) From 503adcdf3bafdeb0abe120933d37cbd05d961f82 Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 12 Sep 2022 21:23:14 -0400 Subject: [PATCH 24/34] Override script name to display correct usage in cli help --- scripts/db.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/db.py b/scripts/db.py index 551cb4fb5c4..ddd4abc90b4 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -87,6 +87,7 @@ def main() -> None: ) parser = ArgumentParser( + prog="db.sh", description="Common database schema migration operations", epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." " For migrating the `tsi` branch, use the `run_alembic.sh` script.", From f049a386835b893506f6e5f2043a193c2c1f14f4 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 13 Sep 2022 02:37:56 -0400 Subject: [PATCH 25/34] Rewrite migrations documentation --- lib/galaxy/model/migrations/README.md | 400 +++++++++++++++++++++++++- run_alembic.sh | 18 +- 2 files changed, 406 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/model/migrations/README.md b/lib/galaxy/model/migrations/README.md index f95029e06bc..1486ec2a63d 100644 --- a/lib/galaxy/model/migrations/README.md +++ b/lib/galaxy/model/migrations/README.md @@ -1,5 +1,401 @@ # Galaxy Database Schema Migrations -Starting with release 22.05, to manage its database migrations, Galaxy uses [Alembic](https://alembic.sqlalchemy.org). +## Overview -For admin and development options, please see [Galaxy's documentation](https://docs.galaxyproject.org/en/master/admin/db_migration.html). +Galaxy's database schema migration system is built on top of [Alembic](https://alembic.sqlalchemy.org) - a lightweight database migration tool for usage with SQLAlchemy. +(This documentation applies to release 22.05 and up. Prior to 22.05, Galaxy has used SQLAlchemy Migrate.) + +The purpose of the database schema migration system is to support automated, incremental, reversible changes to Galaxy's database schema. Central to this is the concept of a database version (more specifically, database *schema* version). A version is represented by a revision script (the terms *version* and *revision* may be used interchangeably). Executing a revision script will upgrade, or downgrade, the database schema by applying the changes specified in the script. + +Galaxy keeps track of the database schema version in two places: one is a directory we refer to as "the migration environment" (located at `lib/galaxy/model/migrations/alembic`), the other is the `alembic_version` table in the database. For Galaxy to run, these versions must be the same, which, essentially, means that the version of the database matches the version expected by the codebase. + +On startup, the system checks if the version in the database matches the version in the codebase. If they do not match, and the `database_auto_migrate` configuration option is not set, Galaxy will fail with an error message explaining how to proceed. In most cases, you'll need to upgrade the database schema to the current version, which is represented by the latest revision script in the migration environment. + +## Administering Galaxy: upgrading and downgrading the database + +To initialize an empty database (or create a new SQLite database), simply start Galaxy. To upgrade or downgrade an existing database, you'll need to use a script. + +To manage database schema migrations, as well as to perform common migration-related operations, +Galaxy provides two scripts: `db.sh` and `run_alembic.sh`. + +The `db.sh` script is the recommended way to interact with Galaxy's database schema migration system. +It provides access to a subset of commands offered by Alembic's CLI, while hiding some of the +implementation complexity of Galaxy's model. The provided commands and options should be +sufficient for most use cases involving Galaxy development or system and database administration. + +The `run_alembic.sh` script is a thin wrapper around the Alembic CLI runner. It offers more +flexibility and the full scope of Alembic's CLI commands and options; however, it requires more detailed command arguments, as well as basic familiarity with [Alembic branches](https://alembic.sqlalchemy.org/en/latest/branches.html). + +#### Implementation detail: branches + +Galaxy's data model is split into the [galaxy model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/__init__.py) and the [install model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/tool_shed_install/__init__.py). +These two models may be persisted in one combined database (which is the default) or two separate databases (which is enabled by setting the +[`install_database_connection`](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/webapps/galaxy/config_schema.yml#L157) configuration option). + +To accommodate this setup, Galaxy uses [Alembic +branches](https://alembic.sqlalchemy.org/en/latest/branches.html#working-with-branches). A branch is +a versioning lineage that starts at a common base revision and represents part of Galaxy's data +model. These branches are identified by labels (***gxy*** for the galaxy model and ***tsi*** for the install +model) and may share the same Alembic version table (if they share the same database; otherwise, +each database has its own version table). Each branch has its own version history, represented by +revision scripts located in the branch version directory (`migrations/alembic/versions_gxy` for +*gxy* and `migrations/alembic/versions_tsi` for *tsi*). + +For a more detailed description of the system's internals, see pull request [#13108](https://github.com/galaxyproject/galaxy/pull/13108). + +## db.sh + +This script offers a set of common database schema migration operations that are executed on the *gxy* branch, which, in the vast majority of cases, is all you need to develop and administer Galaxy. +To run operations on the *tsi* branch, you need to use `run_alembic.sh`. + +``` +usage: db.sh [-h] [-c CONFIG] [--raiseerr] {upgrade,downgrade,version,v, + dbversion,dv,history,h,show,s,revision,init} ... + +positional arguments: + {upgrade,downgrade,version,v,dbversion,dv,history,h,show,s,revision,init} + upgrade Upgrade to a later version + downgrade Revert to a previous version + version (v) Show the head revision in the migrations script directory + dbversion (dv) Show the current revision for Galaxy's database + history (h) List revision scripts in chronological order + show (s) Show the revision(s) denoted by the given symbol + revision Create a new revision file + init Initialize empty database(s) for both branches + (create database objects for gxy and tsi branch) + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --galaxy-config CONFIG + Alternate Galaxy configuration file + --raiseerr Raise a full stack trace on error +``` + +#### Revision identifiers + +Some of the commands accept revision identifiers as arguments. A revision is usually identified by a +12-digit hexadecimal number (i.e., `6a67bf27e6a6`). Anytime you need to refer to a specific +revision, you have the option to use a partial number. As long as the partial number uniquely +identifies the revision, you may use that partial number in any command in place of the full +revision number. + +For example, you may use `./db.sh upgrade 6a` instead of `./db.sh upgrade 6a67bf27e6a6` if `6a` is sufficient to uniquely identify that revision. + +(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#partial-revision-identifiers)) + +#### Relative migration identifiers + +You may also use Alembic's syntax for relative migration identifiers for the upgrade/downgrade commands: + +To move 2 versions from the current version, a decimal value `+N` can be supplied: + +`./db.sh upgrade +2` + +Negative values are accepted for downgrades: + +`./db.sh downgrade -2` + +Relative identifiers may also be in terms of a specific revision. For example, to upgrade to +revision 6a67bf27e6a6 plus two additional steps: + +`.db.sh upgrade 6a67bf27e6a6+2`. + +You may also combine relative migration identifiers with partial revision identifiers: + +`.db.sh upgrade 6a+2`. + +(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#relative-migration-identifiers) + +### Subcommands + +#### upgrade + +Upgrade to a later version. The revision argument is optional: omitting it is equivalent to +specifying `heads` as the revision identifier; in that case, the database(s) will be upgraded to the +latest revisions in ***both*** branches, *gxy* and *tsi*. + +***If you are upgrading a database that has not been version-controlled by Alembic, you should run +this command without the revision argument: `./db.sh upgrade` - this will ensure that both branches, +`gxy` and `tsi`, are initialized.*** + +``` +usage: db.sh upgrade [-h] [--sql] [revision] + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). + +#### downgrade + +Revert to a previous version. + +``` +usage: db.sh downgrade [-h] [--sql] revision + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +Specifying `base` as the revision identifier will downgrade both branches, *gxy* and *tsi*, to their +initial state prior to any revisions; the `alembic_version` table will be empty. + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). +Note that in this mode, instead of specifying a revision identifier, you have to specify a range of revisions using the following format: `:`*. + +*You cannot use this script to downgrade past the initial Alembic revisions that created the *gxy* and *tsi* branches. + + +#### version + +Show the head revision in the migrations script directory. This will display the latest (i.e., head) revision in the migration environment. + +``` +Activating virtualenv at .venv +\usage: db.sh version [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output + +``` + +If your database is setup to host both branches (*gxy* and *tsi*), the head revisions for both branches will be displayed: + +``` +$ ./db.sh version +186d4835587b (gxy) (head) +d4a650f47a3c (tsi) (head) +``` + +#### dbversion + +Show the current revision for Galaxy's database. + +``` +usage: db.sh dbversion [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output +``` + +Similar to the version command, this command will display the revision(s) stored in the +`alembic_version` table in the database. If the database revision corresponds to the head revision +in the codebase, it will be marked as `(head)`. The output will be slightly more verbose and will vary +depending on the database. + +``` +$ ./db.sh dbversion +INFO:alembic.runtime.migration:Context impl PostgresqlImpl. +INFO:alembic.runtime.migration:Will assume transactional DDL. +d4a650f47a3c (head) +6a67bf27e6a6 +``` + +#### history + +List revision scripts in chronological order. + +``` +usage: db.sh history [-h] [-v] [-i] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output + -i, --indicate-current + Indicate current revision +``` + +Depending on your setup, the list may include revision histories for both branches. The oldest +revisions are the ones that created the `gxy` and `tsi` branches (introduced in 22.05). The +`--indicate-current` option is particularly useful when you need to determine how far behind (or +ahead) your database version is compared to the version expected by your codebase: + +``` +$ ./db.sh history --indicate-current +6a67bf27e6a6 -> 186d4835587b (gxy) (head), drop job_state_history.update_time column +b182f655505f -> 6a67bf27e6a6 (gxy) (current), deferred data tables +e7b6dcb09efd -> b182f655505f (gxy), add workflow.source_metadata column + -> e7b6dcb09efd (gxy), create gxy branch + -> d4a650f47a3c (tsi) (head), create tsi branch +``` + +#### show + +Show the revision(s) denoted by the given revision identifier. + +``` +usage: db.sh show [-h] revision + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit +``` + +#### revision + +Create a new revision file. + +``` +usage: db.sh revision [-h] -m MESSAGE [--rev-id REV_ID] + +optional arguments: + -h, --help show this help message and exit + -m MESSAGE, --message MESSAGE + Message string to use with 'revision' + --rev-id REV_ID Specify a revision id instead of generating one + (This option is for testing purposes only) +``` + +The `--message` argument is required: this ensures a readable revision history. The message is +appended to the new revision identifier to form the filename for the new revision script, so it +should be a succinct description of the change: + +``` +$ ./db.sh revision --message "add column foo to table bar" +[output omitted] + +$ ls lib/galaxy/model/migrations/alembic/versions_gxy/ +a2e418ad6a15_add_column_foo_to_table_bar.py +``` + +#### init + +Initialize an empty database (or create a new SQLite database) for both branches, `gxy` and `tsi` (creates database objects in one or two databases, depending on configuration settings). + +``` +usage: db.sh init [-h] + +optional arguments: + -h, --help show this help message and exit +``` + +## run_alembic.sh + +If you need to run operations on the *tsi* branch, or you need access to the full scope of command +line options provided by Alembic, you should use the `run_alembic.sh` script, which is a thin +wrapper around Alembic's CLI runner. The script modifies the path, initializes the Python virtual +environment, retrieves any necessary configuration values, and invokes the Alembic CLI runner with +appropriate arguments. + +Keep in mind that since Galaxy uses branch labels to distinguish between the galaxy and the install +models, in most cases, you'll need to identify the target branch to which your command should be +applied. + +### Examples of usage + +Remember to first backup your database(s). + +#### Upgrading + +Upgrade to the head revision (both, *gxy* and *tsi* branches): + +`./run_alembic.sh upgrade heads` + +Upgrade to the head revision (*gxy* branch): + +`./run_alembic.sh upgrade gxy@head` + +Upgrade to 1 revision above the current (*tsi* branch): + +`./run_alembic.sh upgrade tsi@+1` + +Upgrade to a specific revision: + +`./run_alembic.sh upgrade [revision identifier]` + +Upgrade to 1 revision above a specific revision: + +`./run_alembic.sh upgrade [revision identifier]+1` + +#### Downgrading + +Downgrade to base revision (*gxy* branch): + +`./run_alembic.sh downgrade gxy@base` + +Downgrade to 1 revision below current (*tsi* branch): + +`./run_alembic.sh downgrade tsi@-1` + +Downgrade to a specific revision: + +`./run_alembic.sh downgrade [revision identifier]` + +Downgrade to 1 revision below specific revision: + +`./run_alembic.sh downgrade [revision identifier]-1 ` + +#### Creating new revisions + +To create a revision for the galaxy model: + +`./run_alembic.sh revision --head=gxy@head -message "your description"` + +To create a revision for the install model: + +`./run_alembic.sh revision --head=tsi@head -message "your description"` + +Check [Alembic's documentation](https://alembic.sqlalchemy.org) for more examples. + +*Note: the `run_alembic.sh` script does not support relative upgrades and downgrades without a +revision identifier: you cannot `upgrade +1` or `downgrade -1` without providing a revision identifier.* + +## Upgrading from SQLAlchemy Migrate + +Galaxy no longer supports SQLAlchemy Migrate. To upgrade to Alembic, follow these steps: + +1. Backup your database(s). + +2. Make sure your codebase is in pre-22.05 state: you will need to use the old `manage_db.sh` script which invokes the SQLAlchemy Migrate tool, which is not available in 22.05 and up. Checking out the 22.01 release branch is the simplest step. + +3. Verify that your database is at the latest SQLAlchemy Migrate version. If you have a combined database, it should be version 180 (check the `migrate_version` table). If you have separate galaxy model and install model databases, your galaxy version should be 180, and your install model version should be 17. + + If your database is not current, run `manage_db.sh upgrade` to upgrade your database. + + Once your database has the latest SQLAlchemy Migrate version, switch back to your current branch (22.05 or more recent). + +5. Run `db.sh upgrade`. + +## Developing Galaxy: creating new revisions + +Make sure you have updated the model and have added appropriate tests before creating a new +revision. + +You create a new revision file by running the `revision` subcommand of the `db.sh` or the +`run_alembic.sh` script (see sections above for usage information). Alembic generates a revision +script in the appropriate version directory (`migrations/alembic/versions_gxy` for *gxy* and +`migrations/alembic/versions_tsi` for *tsi*). You'll need to fill out the `upgrade` and `downgrade` +functions. Use Alembic documentation for examples: + +- [https://alembic.sqlalchemy.org/en/latest/tutorial.html#create-a-migration-script](https://alembic.sqlalchemy.org/en/latest/tutorial.html#create-a-migration-script) +- [https://alembic.sqlalchemy.org/en/latest/ops.html](https://alembic.sqlalchemy.org/en/latest/ops.html) + +We encourage you to use Galaxy-specific utility functions (`galaxy/model/migrations/util.py`) when appropriate. Don't forget to provide tests for any modifications you make to the model (most likely, you will need to add them to the appropriate `test/unit/data/model/mapping/test_*model_mapping.py` module. + +After that, run the upgrade script: `./db.sh upgrade`. And you're done! + +## Troubleshooting + +### How to handle migrations.IncorrectVersionError + +If you see this error, you'll need to upgrade or downgrade your database *before* upgrading to +Alembic. Whether you need to upgrade or downgrade depends on what version number is stored in +the `migrate_version` table in your database. Alembic expects version 180, so you will need to +upgrade if your version is less than that or, in very rare circumstances, downgrade if your version +is 181. Please see [this issue](https://github.com/galaxyproject/galaxy/issues/13528) for more details. + +#### Please help us improve this page: +If you encounter any migration-related errors or issues, please [open an issue](https://github.com/galaxyproject/galaxy/issues/new?assignees=&labels=&template=bug_report.md&title=), and we will add the solution with any relevant context to this page. diff --git a/run_alembic.sh b/run_alembic.sh index 15fc1555f4c..3c0172e5045 100755 --- a/run_alembic.sh +++ b/run_alembic.sh @@ -1,14 +1,10 @@ #!/bin/sh ####### -# Use this script to manage Galaxy and Tool Shed Install migrations. -# (Use the manage_db.sh script to manage Tool Shed migrations.) -# -# This script provides access to Alembic's command line options and is -# intended for advanced use scenarios. For regular database management tasks, -# we encourage you to use the manage_db.sh script. -# -# NOTE: If your database is empty, use create_db.sh instead. +# Use this script if you need to run migration operations on the Tool Shed +# Install data model (the *tsi* migration branch), or if you need access to the +# full scope of command line options provided by Alembic. For regular migration +# tasks, uses the db.sh script. # # We use branch labels to distinguish between the galaxy and the tool_shed_install models, # so in most cases you'll need to identify the branch to which your command should be applied. @@ -28,7 +24,7 @@ # ./run_alembic.sh upgrade heads # upgrade gxy and tsi to head revisions # # To downgrade: -# ./run_alembic.sh downgrade gxy@base # downgrade gxy to base (empty db with empty alembic table) +# ./run_alembic.sh downgrade gxy@base # downgrade gxy to base (database with empty alembic table) # ./run_alembic.sh downgrade gxy@-1 # downgrade gxy to 1 revision below current # ./run_alembic.sh downgrade [revision identifier] # downgrade gxy to a specific revision # ./run_alembic.sh downgrade [revision identifier]-1 # downgrade gxy to 1 revision below specific revision @@ -41,7 +37,9 @@ # GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-db-url ./run_alembic.sh ... # GALAXY_INSTALL_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-other-db-url ./run_alembic.sh ... # -# For more options, see Alembic's documentation at https://alembic.sqlalchemy.org +# Further information: +# Galaxy migration documentation: lib/galaxy/model/migrations/README.md +# Alembic documentation: https://alembic.sqlalchemy.org ####### ALEMBIC_CONFIG='lib/galaxy/model/migrations/alembic.ini' From 170e4a4be22cc4b6694cd2ea88ecc449f94c262a Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 14 Sep 2022 08:18:11 -0400 Subject: [PATCH 26/34] Update lib/galaxy/model/migrations/README.md Co-authored-by: Marius van den Beek Update lib/galaxy/model/migrations/README.md Co-authored-by: Marius van den Beek Update lib/galaxy/model/migrations/README.md Co-authored-by: Marius van den Beek Update lib/galaxy/model/migrations/README.md Co-authored-by: Marius van den Beek --- lib/galaxy/model/migrations/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/model/migrations/README.md b/lib/galaxy/model/migrations/README.md index 1486ec2a63d..2f690a8a829 100644 --- a/lib/galaxy/model/migrations/README.md +++ b/lib/galaxy/model/migrations/README.md @@ -13,9 +13,8 @@ On startup, the system checks if the version in the database matches the version ## Administering Galaxy: upgrading and downgrading the database -To initialize an empty database (or create a new SQLite database), simply start Galaxy. To upgrade or downgrade an existing database, you'll need to use a script. +To initialize an empty database (or create a new SQLite database) start Galaxy. To upgrade or downgrade an existing database, you'll need to use a script. -To manage database schema migrations, as well as to perform common migration-related operations, Galaxy provides two scripts: `db.sh` and `run_alembic.sh`. The `db.sh` script is the recommended way to interact with Galaxy's database schema migration system. @@ -61,8 +60,7 @@ positional arguments: history (h) List revision scripts in chronological order show (s) Show the revision(s) denoted by the given symbol revision Create a new revision file - init Initialize empty database(s) for both branches - (create database objects for gxy and tsi branch) + init Initialize empty database(s) optional arguments: -h, --help show this help message and exit From e43552fdc5d887fb14383305af14c6ec0ebaf915 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 14 Sep 2022 20:32:15 -0400 Subject: [PATCH 27/34] Rename db.sh > manage_db.sh, following code review feedback --- lib/galaxy/model/migrations/README.md | 50 ++++++++-------- db.sh => manage_db.sh | 2 +- scripts/db.py | 4 +- .../data/model/migrations/test_dbscript.py | 58 +++++++++---------- tox.ini | 2 +- 5 files changed, 58 insertions(+), 58 deletions(-) rename db.sh => manage_db.sh (87%) diff --git a/lib/galaxy/model/migrations/README.md b/lib/galaxy/model/migrations/README.md index 2f690a8a829..c5ab8818414 100644 --- a/lib/galaxy/model/migrations/README.md +++ b/lib/galaxy/model/migrations/README.md @@ -15,9 +15,9 @@ On startup, the system checks if the version in the database matches the version To initialize an empty database (or create a new SQLite database) start Galaxy. To upgrade or downgrade an existing database, you'll need to use a script. -Galaxy provides two scripts: `db.sh` and `run_alembic.sh`. +Galaxy provides two scripts: `manage_db.sh` and `run_alembic.sh`. -The `db.sh` script is the recommended way to interact with Galaxy's database schema migration system. +The `manage_db.sh` script is the recommended way to interact with Galaxy's database schema migration system. It provides access to a subset of commands offered by Alembic's CLI, while hiding some of the implementation complexity of Galaxy's model. The provided commands and options should be sufficient for most use cases involving Galaxy development or system and database administration. @@ -42,13 +42,13 @@ revision scripts located in the branch version directory (`migrations/alembic/ve For a more detailed description of the system's internals, see pull request [#13108](https://github.com/galaxyproject/galaxy/pull/13108). -## db.sh +## manage_db.sh This script offers a set of common database schema migration operations that are executed on the *gxy* branch, which, in the vast majority of cases, is all you need to develop and administer Galaxy. To run operations on the *tsi* branch, you need to use `run_alembic.sh`. ``` -usage: db.sh [-h] [-c CONFIG] [--raiseerr] {upgrade,downgrade,version,v, +usage: manage_db.sh [-h] [-c CONFIG] [--raiseerr] {upgrade,downgrade,version,v, dbversion,dv,history,h,show,s,revision,init} ... positional arguments: @@ -77,7 +77,7 @@ revision, you have the option to use a partial number. As long as the partial nu identifies the revision, you may use that partial number in any command in place of the full revision number. -For example, you may use `./db.sh upgrade 6a` instead of `./db.sh upgrade 6a67bf27e6a6` if `6a` is sufficient to uniquely identify that revision. +For example, you may use `./manage_db.sh upgrade 6a` instead of `./manage_db.sh upgrade 6a67bf27e6a6` if `6a` is sufficient to uniquely identify that revision. (Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#partial-revision-identifiers)) @@ -87,20 +87,20 @@ You may also use Alembic's syntax for relative migration identifiers for the upg To move 2 versions from the current version, a decimal value `+N` can be supplied: -`./db.sh upgrade +2` +`./manage_db.sh upgrade +2` Negative values are accepted for downgrades: -`./db.sh downgrade -2` +`./manage_db.sh downgrade -2` Relative identifiers may also be in terms of a specific revision. For example, to upgrade to revision 6a67bf27e6a6 plus two additional steps: -`.db.sh upgrade 6a67bf27e6a6+2`. +`.manage_db.sh upgrade 6a67bf27e6a6+2`. You may also combine relative migration identifiers with partial revision identifiers: -`.db.sh upgrade 6a+2`. +`.manage_db.sh upgrade 6a+2`. (Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#relative-migration-identifiers) @@ -113,11 +113,11 @@ specifying `heads` as the revision identifier; in that case, the database(s) wil latest revisions in ***both*** branches, *gxy* and *tsi*. ***If you are upgrading a database that has not been version-controlled by Alembic, you should run -this command without the revision argument: `./db.sh upgrade` - this will ensure that both branches, +this command without the revision argument: `./manage_db.sh upgrade` - this will ensure that both branches, `gxy` and `tsi`, are initialized.*** ``` -usage: db.sh upgrade [-h] [--sql] [revision] +usage: manage_db.sh upgrade [-h] [--sql] [revision] positional arguments: revision Revision identifier @@ -134,7 +134,7 @@ For the `--sql` option, see [Alembic documentation on offline mode](https://alem Revert to a previous version. ``` -usage: db.sh downgrade [-h] [--sql] revision +usage: manage_db.sh downgrade [-h] [--sql] revision positional arguments: revision Revision identifier @@ -159,7 +159,7 @@ Show the head revision in the migrations script directory. This will display the ``` Activating virtualenv at .venv -\usage: db.sh version [-h] [-v] +\usage: manage_db.sh version [-h] [-v] optional arguments: -h, --help show this help message and exit @@ -170,7 +170,7 @@ optional arguments: If your database is setup to host both branches (*gxy* and *tsi*), the head revisions for both branches will be displayed: ``` -$ ./db.sh version +$ ./manage_db.sh version 186d4835587b (gxy) (head) d4a650f47a3c (tsi) (head) ``` @@ -180,7 +180,7 @@ d4a650f47a3c (tsi) (head) Show the current revision for Galaxy's database. ``` -usage: db.sh dbversion [-h] [-v] +usage: manage_db.sh dbversion [-h] [-v] optional arguments: -h, --help show this help message and exit @@ -193,7 +193,7 @@ in the codebase, it will be marked as `(head)`. The output will be slightly more depending on the database. ``` -$ ./db.sh dbversion +$ ./manage_db.sh dbversion INFO:alembic.runtime.migration:Context impl PostgresqlImpl. INFO:alembic.runtime.migration:Will assume transactional DDL. d4a650f47a3c (head) @@ -205,7 +205,7 @@ d4a650f47a3c (head) List revision scripts in chronological order. ``` -usage: db.sh history [-h] [-v] [-i] +usage: manage_db.sh history [-h] [-v] [-i] optional arguments: -h, --help show this help message and exit @@ -220,7 +220,7 @@ revisions are the ones that created the `gxy` and `tsi` branches (introduced in ahead) your database version is compared to the version expected by your codebase: ``` -$ ./db.sh history --indicate-current +$ ./manage_db.sh history --indicate-current 6a67bf27e6a6 -> 186d4835587b (gxy) (head), drop job_state_history.update_time column b182f655505f -> 6a67bf27e6a6 (gxy) (current), deferred data tables e7b6dcb09efd -> b182f655505f (gxy), add workflow.source_metadata column @@ -233,7 +233,7 @@ e7b6dcb09efd -> b182f655505f (gxy), add workflow.source_metadata column Show the revision(s) denoted by the given revision identifier. ``` -usage: db.sh show [-h] revision +usage: manage_db.sh show [-h] revision positional arguments: revision Revision identifier @@ -247,7 +247,7 @@ optional arguments: Create a new revision file. ``` -usage: db.sh revision [-h] -m MESSAGE [--rev-id REV_ID] +usage: manage_db.sh revision [-h] -m MESSAGE [--rev-id REV_ID] optional arguments: -h, --help show this help message and exit @@ -262,7 +262,7 @@ appended to the new revision identifier to form the filename for the new revisio should be a succinct description of the change: ``` -$ ./db.sh revision --message "add column foo to table bar" +$ ./manage_db.sh revision --message "add column foo to table bar" [output omitted] $ ls lib/galaxy/model/migrations/alembic/versions_gxy/ @@ -274,7 +274,7 @@ a2e418ad6a15_add_column_foo_to_table_bar.py Initialize an empty database (or create a new SQLite database) for both branches, `gxy` and `tsi` (creates database objects in one or two databases, depending on configuration settings). ``` -usage: db.sh init [-h] +usage: manage_db.sh init [-h] optional arguments: -h, --help show this help message and exit @@ -365,14 +365,14 @@ Galaxy no longer supports SQLAlchemy Migrate. To upgrade to Alembic, follow thes Once your database has the latest SQLAlchemy Migrate version, switch back to your current branch (22.05 or more recent). -5. Run `db.sh upgrade`. +5. Run `manage_db.sh upgrade`. ## Developing Galaxy: creating new revisions Make sure you have updated the model and have added appropriate tests before creating a new revision. -You create a new revision file by running the `revision` subcommand of the `db.sh` or the +You create a new revision file by running the `revision` subcommand of the `manage_db.sh` or the `run_alembic.sh` script (see sections above for usage information). Alembic generates a revision script in the appropriate version directory (`migrations/alembic/versions_gxy` for *gxy* and `migrations/alembic/versions_tsi` for *tsi*). You'll need to fill out the `upgrade` and `downgrade` @@ -383,7 +383,7 @@ functions. Use Alembic documentation for examples: We encourage you to use Galaxy-specific utility functions (`galaxy/model/migrations/util.py`) when appropriate. Don't forget to provide tests for any modifications you make to the model (most likely, you will need to add them to the appropriate `test/unit/data/model/mapping/test_*model_mapping.py` module. -After that, run the upgrade script: `./db.sh upgrade`. And you're done! +After that, run the upgrade script: `./manage_db.sh upgrade`. And you're done! ## Troubleshooting diff --git a/db.sh b/manage_db.sh similarity index 87% rename from db.sh rename to manage_db.sh index cc9df6105f0..10f1ad2500f 100755 --- a/db.sh +++ b/manage_db.sh @@ -2,7 +2,7 @@ ####### # Use this script to manage Galaxy database schema migrations. -# For help, run `sh db.sh -h`. +# For help, run `sh manage_db.sh -h`. # For detailed help, see documentation at lib/galaxy/model/migrations/README.md. ####### diff --git a/scripts/db.py b/scripts/db.py index ddd4abc90b4..e7e8baa7f2e 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -1,5 +1,5 @@ """ -This script is intended to be invoked by the db.sh script. +This script is intended to be invoked by the manage_db.sh script. """ import logging @@ -87,7 +87,7 @@ def main() -> None: ) parser = ArgumentParser( - prog="db.sh", + prog="manage_db.sh", description="Common database schema migration operations", epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." " For migrating the `tsi` branch, use the `run_alembic.sh` script.", diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index c0de5e59dd3..4f346e3944d 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -130,7 +130,7 @@ def dburl_from_config(config: Config) -> str: def run_command(cmd: str) -> subprocess.CompletedProcess: if in_packages(): - cmd = f"../.{cmd}" # if this is run from `packages`, db.sh is in parent directory + cmd = f"../.{cmd}" # if this is run from `packages`, manage_db.sh is in parent directory completed_process = subprocess.run(cmd.split(), capture_output=True, text=True) return completed_process @@ -154,9 +154,9 @@ def get_db_heads(config: Config) -> Tuple[str, ...]: class TestRevisionCommand: def test_revision_cmd(self, config): - run_command("./db.sh revision --message foo1") - run_command("./db.sh revision --rev-id 2 --message foo2") - run_command("./db.sh revision --rev-id 3 --message foo3") + run_command("./manage_db.sh revision --message foo1") + run_command("./manage_db.sh revision --rev-id 2 --message foo2") + run_command("./manage_db.sh revision --rev-id 3 --message foo3") script_dir = ScriptDirectory.from_config(config) revisions = [rev for rev in script_dir.walk_revisions()] @@ -169,7 +169,7 @@ class TestRevisionCommand: assert rev.module.__name__ == "3_foo3_py" # verify message def test_revision_cmd_missing_message_arg_error(self): - completed = run_command("./db.sh revision --rev-id 1") + completed = run_command("./manage_db.sh revision --rev-id 1") assert completed.returncode == 2 assert "the following arguments are required: -m/--message" in completed.stderr @@ -177,24 +177,24 @@ class TestRevisionCommand: class TestShowCommand: def test_show_cmd(self, config): alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) - completed = run_command("./db.sh show 42") + completed = run_command("./manage_db.sh show 42") assert "Revision ID: 42" in completed.stdout def test_show_cmd_invalid_revision_error(self, config): alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) - completed = run_command("./db.sh show idonotexist") + completed = run_command("./manage_db.sh show idonotexist") assert completed.returncode == 1 assert "Traceback" not in completed.stderr assert "Can't locate revision identified by 'idonotexist'" in completed.stderr def test_show_cmd_invalid_revision_error_with_traceback(self): - completed = run_command("./db.sh --raiseerr show idonotexist") + completed = run_command("./manage_db.sh --raiseerr show idonotexist") assert completed.returncode == 1 assert "Traceback" in completed.stderr assert "Can't locate revision identified by 'idonotexist'" in completed.stderr def test_show_cmd_missing_revision_arg_error(self): - completed = run_command("./db.sh show") + completed = run_command("./manage_db.sh show") assert completed.returncode == 2 assert "the following arguments are required: revision" in completed.stderr @@ -205,7 +205,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") - completed = run_command("./db.sh history") + completed = run_command("./manage_db.sh history") assert completed.returncode == 0 assert "2 -> 3 (gxy) (head), empty message" in completed.stdout assert "1 -> 2 (gxy)" in completed.stdout @@ -216,7 +216,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") - completed = run_command("./db.sh history --verbose") + completed = run_command("./manage_db.sh history --verbose") assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout @@ -226,7 +226,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command("./db.sh history --indicate-current") + completed = run_command("./manage_db.sh history --indicate-current") assert completed.returncode == 0 assert "2 -> 3 (gxy) (head) (current), empty message" in completed.stdout assert "1 -> 2 (gxy)" in completed.stdout @@ -238,7 +238,7 @@ class TestVersionCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./db.sh version") + completed = run_command("./manage_db.sh version") assert completed.returncode == 0 assert "2 (gxy) (head)" in completed.stdout @@ -246,7 +246,7 @@ class TestVersionCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./db.sh version --verbose") + completed = run_command("./manage_db.sh version --verbose") assert completed.returncode == 0 assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout @@ -257,13 +257,13 @@ class TestDbVersionCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./db.sh dbversion") + completed = run_command("./manage_db.sh dbversion") assert completed.returncode == 0 assert "(head)" not in completed.stdout # there has been no upgrade alembic.command.upgrade(config, "heads") - completed = run_command("./db.sh dbversion") + completed = run_command("./manage_db.sh dbversion") assert completed.returncode == 0 assert "2 (head)" in completed.stdout @@ -273,7 +273,7 @@ class TestDbVersionCommand: alembic.command.upgrade(config, "heads") - completed = run_command("./db.sh dbversion --verbose") + completed = run_command("./manage_db.sh dbversion --verbose") assert completed.returncode == 0 assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout @@ -285,7 +285,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="2", head="1") # first upgrade: upgrades gxy to 2, tsi to base - completed = run_command("./db.sh upgrade") + completed = run_command("./manage_db.sh upgrade") assert completed.returncode == 0 assert "Running upgrade gxy0 -> 1" in completed.stderr assert "Running upgrade 1 -> 2" in completed.stderr @@ -297,7 +297,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="3", head="2") # next upgrade: upgrades gxy to 3 - completed = run_command("./db.sh upgrade") + completed = run_command("./manage_db.sh upgrade") assert completed.returncode == 0 assert "Running upgrade 2 -> 3" in completed.stderr assert "tsi0" not in completed.stderr # no effect on tsi @@ -309,14 +309,14 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./db.sh upgrade --sql") + completed = run_command("./manage_db.sh upgrade --sql") assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='3'" not in completed.stdout alembic.command.revision(config, rev_id="3", head="2") - completed = run_command("./db.sh upgrade --sql") + completed = run_command("./manage_db.sh upgrade --sql") assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='3'" in completed.stdout @@ -326,7 +326,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="2", head="1") # upgrades gxy to 1 - completed = run_command("./db.sh upgrade 1") + completed = run_command("./manage_db.sh upgrade 1") assert completed.returncode == 0 assert "Running upgrade gxy0 -> 1" in completed.stderr @@ -341,7 +341,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="e", head="d") # upgrades gxy to b: none + 2 (none -> base -> a) - completed = run_command("./db.sh upgrade +3") + completed = run_command("./manage_db.sh upgrade +3") assert completed.returncode == 0 assert "Running upgrade -> gxy0" in completed.stderr assert "Running upgrade gxy0 -> a" in completed.stderr @@ -351,7 +351,7 @@ class TestUpgradeCommand: assert heads == ("b",) # upgrades gxy to d relative to b: b + 2 (b -> c -> d) - completed = run_command("./db.sh upgrade b+2") + completed = run_command("./manage_db.sh upgrade b+2") assert completed.returncode == 0 assert "Running upgrade b -> c" in completed.stderr assert "Running upgrade c -> d" in completed.stderr @@ -367,7 +367,7 @@ class TestDowngradeCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command("./db.sh downgrade 1") # downgrade gxy to 1 + completed = run_command("./manage_db.sh downgrade 1") # downgrade gxy to 1 assert completed.returncode == 0 assert "Running downgrade 3 -> 2" in completed.stderr assert "Running downgrade 2 -> 1" in completed.stderr @@ -382,13 +382,13 @@ class TestDowngradeCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command("./db.sh downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi + completed = run_command("./manage_db.sh downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='1'" in completed.stdout def test_downgrade_cmd_missing_revision_arg_error(self): - completed = run_command("./db.sh downgrade") + completed = run_command("./manage_db.sh downgrade") assert completed.returncode == 2 assert "the following arguments are required: revision" in completed.stderr @@ -401,7 +401,7 @@ class TestDowngradeCommand: alembic.command.upgrade(config, "heads") # downgrades gxy to c: e - 2 (e -> d -> c) - completed = run_command("./db.sh downgrade -2") + completed = run_command("./manage_db.sh downgrade -2") assert completed.returncode == 0 assert "Running downgrade e -> d" in completed.stderr @@ -411,7 +411,7 @@ class TestDowngradeCommand: assert "c" in heads # downgrades gxy to a relative to c: c - 2 (c -> b -> a) - completed = run_command("./db.sh downgrade c-2") + completed = run_command("./manage_db.sh downgrade c-2") assert completed.returncode == 0 assert "Running downgrade c -> b" in completed.stderr assert "Running downgrade b -> a" in completed.stderr diff --git a/tox.ini b/tox.ini index ee55e978ed6..0ea5082a227 100644 --- a/tox.ini +++ b/tox.ini @@ -58,5 +58,5 @@ commands = bash .ci/check_controller.sh [testenv:check_indexes] commands = bash scripts/common_startup.sh - bash db.sh init + bash manage_db.sh init bash check_model.sh From 6c64a44a10a1592aca5d616973d45da98735d2f1 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 14 Sep 2022 22:22:45 -0400 Subject: [PATCH 28/34] Remove run_alembic.sh from root; user scripts/run_alembic.py instead. Rationale: move developer-focused scripts out of galaxy root. --- run_alembic.sh | 54 ------------------------------------------ scripts/run_alembic.py | 53 ++++++++++++++++++++++++++++++++++++++--- 2 files changed, 50 insertions(+), 57 deletions(-) delete mode 100755 run_alembic.sh mode change 100644 => 100755 scripts/run_alembic.py diff --git a/run_alembic.sh b/run_alembic.sh deleted file mode 100755 index 3c0172e5045..00000000000 --- a/run_alembic.sh +++ /dev/null @@ -1,54 +0,0 @@ -#!/bin/sh - -####### -# Use this script if you need to run migration operations on the Tool Shed -# Install data model (the *tsi* migration branch), or if you need access to the -# full scope of command line options provided by Alembic. For regular migration -# tasks, uses the db.sh script. -# -# We use branch labels to distinguish between the galaxy and the tool_shed_install models, -# so in most cases you'll need to identify the branch to which your command should be applied. -# Use these identifiers: `gxy` for galaxy, and `tsi` for tool_shed_install. -# -# To create a revision for galaxy: -# ./run_alembic.sh revision --head=gxy@head -m "your description" -# -# To create a revision for tool_shed_install: -# ./run_alembic.sh revision --head=tsi@head -m "your description" -# -# To upgrade: -# ./run_alembic.sh upgrade gxy@head # upgrade gxy to head revision -# ./run_alembic.sh upgrade gxy@+1 # upgrade gxy to 1 revision above current -# ./run_alembic.sh upgrade [revision identifier] # upgrade gxy to a specific revision -# ./run_alembic.sh upgrade [revision identifier]+1 # upgrade gxy to 1 revision above specific revision -# ./run_alembic.sh upgrade heads # upgrade gxy and tsi to head revisions -# -# To downgrade: -# ./run_alembic.sh downgrade gxy@base # downgrade gxy to base (database with empty alembic table) -# ./run_alembic.sh downgrade gxy@-1 # downgrade gxy to 1 revision below current -# ./run_alembic.sh downgrade [revision identifier] # downgrade gxy to a specific revision -# ./run_alembic.sh downgrade [revision identifier]-1 # downgrade gxy to 1 revision below specific revision -# -# To pass a galaxy config file, use `--galaxy-config` -# -# You may also override the galaxy database url and/or the -# tool shed install database url, as well as the database_template -# and database_encoding configuration options with env vars: -# GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-db-url ./run_alembic.sh ... -# GALAXY_INSTALL_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-other-db-url ./run_alembic.sh ... -# -# Further information: -# Galaxy migration documentation: lib/galaxy/model/migrations/README.md -# Alembic documentation: https://alembic.sqlalchemy.org -####### - -ALEMBIC_CONFIG='lib/galaxy/model/migrations/alembic.ini' - -cd "$(dirname "$0")" || exit - -. ./scripts/common_startup_functions.sh - -setup_python - -find lib/galaxy/model/migrations/alembic -name '*.pyc' -delete -python ./scripts/run_alembic.py --config "$ALEMBIC_CONFIG" "$@" diff --git a/scripts/run_alembic.py b/scripts/run_alembic.py old mode 100644 new mode 100755 index 78f976ff158..b642c84e0c7 --- a/scripts/run_alembic.py +++ b/scripts/run_alembic.py @@ -1,8 +1,51 @@ +#!/usr/bin/env python + """ -This script retrieves relevant configuration values and invokes -the Alembic console runner. -It is wrapped by run_alembic.sh (see that file for usage). +Retrieves relevant configuration values and invokes the Alembic console runner. + +Must be executed from Galaxy's root directory. + +Use this script if you need to run migration operations on the Tool Shed +Install data model (the *tsi* migration branch), or if you need access to the +full scope of command line options provided by Alembic. For regular migration +tasks, uses the db.sh script. + +We use branch labels to distinguish between the galaxy and the tool_shed_install models, +so in most cases you'll need to identify the branch to which your command should be applied. +Use these identifiers: `gxy` for galaxy, and `tsi` for tool_shed_install. + +To create a revision for galaxy: +./scripts/run_alembic.py revision --head=gxy@head -m "your description" + +To create a revision for tool_shed_install: +./scripts/run_alembic.py revision --head=tsi@head -m "your description" + +To upgrade: +./scripts/run_alembic.py upgrade gxy@head # upgrade gxy to head revision +./scripts/run_alembic.py upgrade gxy@+1 # upgrade gxy to 1 revision above current +./scripts/run_alembic.py upgrade [revision identifier] # upgrade gxy to a specific revision +./scripts/run_alembic.py upgrade [revision identifier]+1 # upgrade gxy to 1 revision above specific revision +./scripts/run_alembic.py upgrade heads # upgrade gxy and tsi to head revisions + +To downgrade: +./scripts/run_alembic.py downgrade gxy@base # downgrade gxy to base (database with empty alembic table) +./scripts/run_alembic.py downgrade gxy@-1 # downgrade gxy to 1 revision below current +./scripts/run_alembic.py downgrade [revision identifier] # downgrade gxy to a specific revision +./scripts/run_alembic.py downgrade [revision identifier]-1 # downgrade gxy to 1 revision below specific revision + +To pass a galaxy config file, use `--galaxy-config` + +You may also override the galaxy database url and/or the +tool shed install database url, as well as the database_template +and database_encoding configuration options with env vars: +GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-db-url ./scripts/run_alembic.py ... +GALAXY_INSTALL_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-other-db-url ./scripts/run_alembic.py ... + +Further information: +Galaxy migration documentation: lib/galaxy/model/migrations/README.md +Alembic documentation: https://alembic.sqlalchemy.org """ + import logging import os.path import sys @@ -18,8 +61,12 @@ from galaxy.model.migrations.scripts import ( logging.basicConfig(level=logging.DEBUG) log = logging.getLogger(__name__) +ALEMBIC_CONFIG = "lib/galaxy/model/migrations/alembic.ini" + def run(): + sys.argv.insert(1, "--config") + sys.argv.insert(2, ALEMBIC_CONFIG) gxy_config, tsi_config, _ = get_configuration(sys.argv, os.getcwd()) add_db_urls_to_command_arguments(sys.argv, gxy_config.url, tsi_config.url) invoke_alembic() From bd56131ebd8eeb043bbc3858f5a4445b779594e7 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 15 Sep 2022 00:20:50 -0400 Subject: [PATCH 29/34] Split script into admin and dev TODO: remove duplication between db.py and db_dev.py (split into multiple commits is intentional) --- scripts/db.py | 38 ---- scripts/db_dev.py | 166 ++++++++++++++++++ scripts/db_dev.sh | 17 ++ .../data/model/migrations/test_dbscript.py | 90 +++++----- 4 files changed, 233 insertions(+), 78 deletions(-) create mode 100644 scripts/db_dev.py create mode 100755 scripts/db_dev.sh diff --git a/scripts/db.py b/scripts/db.py index e7e8baa7f2e..6578564d6c1 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -30,10 +30,6 @@ def exec_downgrade(args: Namespace) -> None: _exec_command("downgrade", args) -def exec_revision(args: Namespace) -> None: - _exec_command("revision", args) - - def exec_version(args: Namespace) -> None: _exec_command("version", args) @@ -42,14 +38,6 @@ def exec_dbversion(args: Namespace) -> None: _exec_command("dbversion", args) -def exec_history(args: Namespace) -> None: - _exec_command("history", args) - - -def exec_show(args: Namespace) -> None: - _exec_command("show", args) - - def exec_init(args: Namespace) -> None: gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) @@ -89,11 +77,8 @@ def main() -> None: parser = ArgumentParser( prog="manage_db.sh", description="Common database schema migration operations", - epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." - " For migrating the `tsi` branch, use the `run_alembic.sh` script.", ) parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") - parser.add_argument("--raiseerr", help="Raise a full stack trace on error", action="store_true") subparsers = parser.add_subparsers(required=True) @@ -129,29 +114,6 @@ def main() -> None: parents=[verbose_arg_parser], ) - history_cmd_parser = add_parser( - "history", - exec_history, - "List revision scripts in chronological order", - aliases=["h"], - parents=[verbose_arg_parser], - ) - history_cmd_parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") - - show_cmd_parser = add_parser( - "show", - exec_show, - "Show the revision(s) denoted by the given symbol", - aliases=["s"], - ) - show_cmd_parser.add_argument("revision", help="Revision identifier") - - revision_cmd_parser = add_parser("revision", help="Create a new revision file", func=exec_revision) - revision_cmd_parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) - revision_cmd_parser.add_argument( - "--rev-id", help="Specify a revision id instead of generating one (This option is for testing purposes only)" - ) - add_parser( "init", exec_init, diff --git a/scripts/db_dev.py b/scripts/db_dev.py new file mode 100644 index 00000000000..1a94ded1e29 --- /dev/null +++ b/scripts/db_dev.py @@ -0,0 +1,166 @@ +""" +This script is intended to be invoked by the scripts/db_dev.sh script. +""" + +import logging +import os +import sys +from argparse import ( + ArgumentParser, + Namespace, +) + +import alembic + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) + +from galaxy.model.migrations import verify_databases_via_script +from galaxy.model.migrations.dbscript import DbScript +from galaxy.model.migrations.scripts import get_configuration + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +def exec_upgrade(args: Namespace) -> None: + _exec_command("upgrade", args) + + +def exec_downgrade(args: Namespace) -> None: + _exec_command("downgrade", args) + + +def exec_revision(args: Namespace) -> None: + _exec_command("revision", args) + + +def exec_version(args: Namespace) -> None: + _exec_command("version", args) + + +def exec_dbversion(args: Namespace) -> None: + _exec_command("dbversion", args) + + +def exec_history(args: Namespace) -> None: + _exec_command("history", args) + + +def exec_show(args: Namespace) -> None: + _exec_command("show", args) + + +def exec_init(args: Namespace) -> None: + gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) + verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) + + +def _exec_command(command: str, args: Namespace) -> None: + dbscript = DbScript(args.config) + try: + getattr(dbscript, command)(args) + except alembic.util.exc.CommandError as e: + if args.raiseerr: + raise + else: + log.error(e) + print(f"FAILED: {str(e)}") + sys.exit(1) + + +def main() -> None: + def add_parser(command, func, help, aliases=None, parents=None): + aliases = aliases or [] + parents = parents or [] + parser = subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) + parser.set_defaults(func=func) + return parser + + verbose_arg_parser = ArgumentParser(add_help=False) + verbose_arg_parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") + + sql_arg_parser = ArgumentParser(add_help=False) + sql_arg_parser.add_argument( + "--sql", + action="store_true", + help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", + ) + + parser = ArgumentParser( + prog="db_dev.py", + description="Common database schema migration operations", + epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." + " For migrating the `tsi` branch, use the `run_alembic.sh` script.", + ) + parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") + parser.add_argument("--raiseerr", help="Raise a full stack trace on error", action="store_true") + + subparsers = parser.add_subparsers(required=True) + + upgrade_cmd_parser = add_parser( + "upgrade", + exec_upgrade, + "Upgrade to a later version", + parents=[sql_arg_parser], + ) + upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?") + + downgrade_cmd_parser = add_parser( + "downgrade", + exec_downgrade, + "Revert to a previous version", + parents=[sql_arg_parser], + ) + downgrade_cmd_parser.add_argument("revision", help="Revision identifier") + + add_parser( + "version", + exec_version, + "Show the head revision in the migrations script directory", + aliases=["v"], + parents=[verbose_arg_parser], + ) + + add_parser( + "dbversion", + exec_dbversion, + "Show the current revision for Galaxy's database", + aliases=["dv"], + parents=[verbose_arg_parser], + ) + + history_cmd_parser = add_parser( + "history", + exec_history, + "List revision scripts in chronological order", + aliases=["h"], + parents=[verbose_arg_parser], + ) + history_cmd_parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") + + show_cmd_parser = add_parser( + "show", + exec_show, + "Show the revision(s) denoted by the given symbol", + aliases=["s"], + ) + show_cmd_parser.add_argument("revision", help="Revision identifier") + + revision_cmd_parser = add_parser("revision", help="Create a new revision file", func=exec_revision) + revision_cmd_parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) + revision_cmd_parser.add_argument( + "--rev-id", help="Specify a revision id instead of generating one (This option is for testing purposes only)" + ) + + add_parser( + "init", + exec_init, + "Initialize empty database(s) for both branches (create database objects for gxy and tsi branch)", + ) + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/db_dev.sh b/scripts/db_dev.sh new file mode 100755 index 00000000000..c6c8a42685f --- /dev/null +++ b/scripts/db_dev.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +####### +# Extended set of database schema migration operaions. +# For help, run `sh db_dev.sh -h`. +# For detailed help, see documentation at lib/galaxy/model/migrations/README.md. +####### + +cd "$(dirname "$0")" || exit + +cd .. + +. ./scripts/common_startup_functions.sh + +setup_python + +python scripts/db_dev.py "$@" diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py index 4f346e3944d..d3367e84ce7 100644 --- a/test/unit/data/model/migrations/test_dbscript.py +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -35,6 +35,10 @@ TSI_BRANCH_LABEL = "tsi" GXY_BASE_ID = "gxy0" TSI_BASE_ID = "tsi0" +ADMIN_CMD = "./manage_db.sh" +DEV_CMD = "./scripts/db_dev.sh" +COMMANDS = [ADMIN_CMD, DEV_CMD] + @pytest.fixture(scope="session") def alembic_env_dir() -> str: @@ -131,7 +135,9 @@ def dburl_from_config(config: Config) -> str: def run_command(cmd: str) -> subprocess.CompletedProcess: if in_packages(): cmd = f"../.{cmd}" # if this is run from `packages`, manage_db.sh is in parent directory + completed_process = subprocess.run(cmd.split(), capture_output=True, text=True) + return completed_process @@ -154,9 +160,9 @@ def get_db_heads(config: Config) -> Tuple[str, ...]: class TestRevisionCommand: def test_revision_cmd(self, config): - run_command("./manage_db.sh revision --message foo1") - run_command("./manage_db.sh revision --rev-id 2 --message foo2") - run_command("./manage_db.sh revision --rev-id 3 --message foo3") + run_command(f"{DEV_CMD} revision --message foo1") + run_command(f"{DEV_CMD} revision --rev-id 2 --message foo2") + run_command(f"{DEV_CMD} revision --rev-id 3 --message foo3") script_dir = ScriptDirectory.from_config(config) revisions = [rev for rev in script_dir.walk_revisions()] @@ -169,7 +175,7 @@ class TestRevisionCommand: assert rev.module.__name__ == "3_foo3_py" # verify message def test_revision_cmd_missing_message_arg_error(self): - completed = run_command("./manage_db.sh revision --rev-id 1") + completed = run_command(f"{DEV_CMD} revision --rev-id 1") assert completed.returncode == 2 assert "the following arguments are required: -m/--message" in completed.stderr @@ -177,24 +183,24 @@ class TestRevisionCommand: class TestShowCommand: def test_show_cmd(self, config): alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) - completed = run_command("./manage_db.sh show 42") + completed = run_command(f"{DEV_CMD} show 42") assert "Revision ID: 42" in completed.stdout def test_show_cmd_invalid_revision_error(self, config): alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) - completed = run_command("./manage_db.sh show idonotexist") + completed = run_command(f"{DEV_CMD} show idonotexist") assert completed.returncode == 1 assert "Traceback" not in completed.stderr assert "Can't locate revision identified by 'idonotexist'" in completed.stderr def test_show_cmd_invalid_revision_error_with_traceback(self): - completed = run_command("./manage_db.sh --raiseerr show idonotexist") + completed = run_command(f"{DEV_CMD} --raiseerr show idonotexist") assert completed.returncode == 1 assert "Traceback" in completed.stderr assert "Can't locate revision identified by 'idonotexist'" in completed.stderr def test_show_cmd_missing_revision_arg_error(self): - completed = run_command("./manage_db.sh show") + completed = run_command(f"{DEV_CMD} show") assert completed.returncode == 2 assert "the following arguments are required: revision" in completed.stderr @@ -205,7 +211,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") - completed = run_command("./manage_db.sh history") + completed = run_command(f"{DEV_CMD} history") assert completed.returncode == 0 assert "2 -> 3 (gxy) (head), empty message" in completed.stdout assert "1 -> 2 (gxy)" in completed.stdout @@ -216,7 +222,7 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") - completed = run_command("./manage_db.sh history --verbose") + completed = run_command(f"{DEV_CMD} history --verbose") assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout @@ -226,66 +232,69 @@ class TestHistoryCommand: alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command("./manage_db.sh history --indicate-current") + completed = run_command(f"{DEV_CMD} history --indicate-current") assert completed.returncode == 0 assert "2 -> 3 (gxy) (head) (current), empty message" in completed.stdout assert "1 -> 2 (gxy)" in completed.stdout assert "gxy0 -> 1 (gxy)" in completed.stdout +@pytest.mark.parametrize("command", COMMANDS) class TestVersionCommand: - def test_version_cmd(self, config): + def test_version_cmd(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./manage_db.sh version") + completed = run_command(f"{command} version") assert completed.returncode == 0 assert "2 (gxy) (head)" in completed.stdout - def test_version_cmd_verbose(self, config): + def test_version_cmd_verbose(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./manage_db.sh version --verbose") + completed = run_command(f"{command} version --verbose") assert completed.returncode == 0 assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout +@pytest.mark.parametrize("command", COMMANDS) class TestDbVersionCommand: - def test_dbversion_cmd(self, config): + def test_dbversion_cmd(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./manage_db.sh dbversion") + completed = run_command(f"{command} dbversion") assert completed.returncode == 0 assert "(head)" not in completed.stdout # there has been no upgrade alembic.command.upgrade(config, "heads") - completed = run_command("./manage_db.sh dbversion") + completed = run_command(f"{command} dbversion") assert completed.returncode == 0 assert "2 (head)" in completed.stdout - def test_dbversion_cmd_verbose(self, config): + def test_dbversion_cmd_verbose(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") alembic.command.upgrade(config, "heads") - completed = run_command("./manage_db.sh dbversion --verbose") + completed = run_command(f"{command} dbversion --verbose") assert completed.returncode == 0 assert "Revision ID: 2" in completed.stdout assert "Revises: 1" in completed.stdout +@pytest.mark.parametrize("command", COMMANDS) class TestUpgradeCommand: - def test_upgrade_cmd(self, config): + def test_upgrade_cmd(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") # first upgrade: upgrades gxy to 2, tsi to base - completed = run_command("./manage_db.sh upgrade") + completed = run_command(f"{command} upgrade") assert completed.returncode == 0 assert "Running upgrade gxy0 -> 1" in completed.stderr assert "Running upgrade 1 -> 2" in completed.stderr @@ -297,7 +306,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="3", head="2") # next upgrade: upgrades gxy to 3 - completed = run_command("./manage_db.sh upgrade") + completed = run_command(f"{command} upgrade") assert completed.returncode == 0 assert "Running upgrade 2 -> 3" in completed.stderr assert "tsi0" not in completed.stderr # no effect on tsi @@ -305,35 +314,35 @@ class TestUpgradeCommand: heads = get_db_heads(config) assert "3" in heads - def test_upgrade_cmd_sql_only(self, config): + def test_upgrade_cmd_sql_only(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") - completed = run_command("./manage_db.sh upgrade --sql") + completed = run_command(f"{command} upgrade --sql") assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='3'" not in completed.stdout alembic.command.revision(config, rev_id="3", head="2") - completed = run_command("./manage_db.sh upgrade --sql") + completed = run_command(f"{command} upgrade --sql") assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='3'" in completed.stdout - def test_upgrade_cmd_with_revision_arg(self, config): + def test_upgrade_cmd_with_revision_arg(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") # upgrades gxy to 1 - completed = run_command("./manage_db.sh upgrade 1") + completed = run_command(f"{command} upgrade 1") assert completed.returncode == 0 assert "Running upgrade gxy0 -> 1" in completed.stderr heads = get_db_heads(config) assert heads == ("1",) - def test_upgrade_cmd_with_relative_revision_syntax(self, config): + def test_upgrade_cmd_with_relative_revision_syntax(self, config, command): alembic.command.revision(config, rev_id="a", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="b", head="a") alembic.command.revision(config, rev_id="c", head="b") @@ -341,7 +350,7 @@ class TestUpgradeCommand: alembic.command.revision(config, rev_id="e", head="d") # upgrades gxy to b: none + 2 (none -> base -> a) - completed = run_command("./manage_db.sh upgrade +3") + completed = run_command(f"{command} upgrade +3") assert completed.returncode == 0 assert "Running upgrade -> gxy0" in completed.stderr assert "Running upgrade gxy0 -> a" in completed.stderr @@ -351,7 +360,7 @@ class TestUpgradeCommand: assert heads == ("b",) # upgrades gxy to d relative to b: b + 2 (b -> c -> d) - completed = run_command("./manage_db.sh upgrade b+2") + completed = run_command(f"{command} upgrade b+2") assert completed.returncode == 0 assert "Running upgrade b -> c" in completed.stderr assert "Running upgrade c -> d" in completed.stderr @@ -360,14 +369,15 @@ class TestUpgradeCommand: assert heads == ("d",) +@pytest.mark.parametrize("command", COMMANDS) class TestDowngradeCommand: - def test_downgrade_cmd(self, config): + def test_downgrade_cmd(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command("./manage_db.sh downgrade 1") # downgrade gxy to 1 + completed = run_command(f"{command} downgrade 1") # downgrade gxy to 1 assert completed.returncode == 0 assert "Running downgrade 3 -> 2" in completed.stderr assert "Running downgrade 2 -> 1" in completed.stderr @@ -376,23 +386,23 @@ class TestDowngradeCommand: assert len(heads) == 2 assert "1" in heads - def test_downgrade_cmd_sql_only(self, config): + def test_downgrade_cmd_sql_only(self, config, command): alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="2", head="1") alembic.command.revision(config, rev_id="3", head="2") alembic.command.upgrade(config, "heads") - completed = run_command("./manage_db.sh downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi + completed = run_command(f"{command} downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi assert completed.returncode == 0 assert "UPDATE alembic_version SET version_num='2'" in completed.stdout assert "UPDATE alembic_version SET version_num='1'" in completed.stdout - def test_downgrade_cmd_missing_revision_arg_error(self): - completed = run_command("./manage_db.sh downgrade") + def test_downgrade_cmd_missing_revision_arg_error(self, command): + completed = run_command(f"{command} downgrade") assert completed.returncode == 2 assert "the following arguments are required: revision" in completed.stderr - def test_downgrade_cmd_with_relative_revision_syntax(self, config): + def test_downgrade_cmd_with_relative_revision_syntax(self, config, command): alembic.command.revision(config, rev_id="a", head=GXY_BASE_ID) alembic.command.revision(config, rev_id="b", head="a") alembic.command.revision(config, rev_id="c", head="b") @@ -401,7 +411,7 @@ class TestDowngradeCommand: alembic.command.upgrade(config, "heads") # downgrades gxy to c: e - 2 (e -> d -> c) - completed = run_command("./manage_db.sh downgrade -2") + completed = run_command(f"{command} downgrade -2") assert completed.returncode == 0 assert "Running downgrade e -> d" in completed.stderr @@ -411,7 +421,7 @@ class TestDowngradeCommand: assert "c" in heads # downgrades gxy to a relative to c: c - 2 (c -> b -> a) - completed = run_command("./manage_db.sh downgrade c-2") + completed = run_command(f"{command} downgrade c-2") assert completed.returncode == 0 assert "Running downgrade c -> b" in completed.stderr assert "Running downgrade b -> a" in completed.stderr From f1dc53e51ec95343ef694d769cf0f076aac4a18b Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 15 Sep 2022 15:21:27 -0400 Subject: [PATCH 30/34] Remove duplication --- lib/galaxy/model/migrations/dbscript.py | 187 ++++++++++++++++++++++-- scripts/db.py | 110 +------------- scripts/db_dev.py | 152 ++----------------- 3 files changed, 199 insertions(+), 250 deletions(-) diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index b016f9d43b6..4e841a2f106 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -1,11 +1,24 @@ -import argparse +import logging import os +import sys +from argparse import ( + ArgumentParser, + Namespace, +) from typing import Optional +import alembic from alembic import command from alembic.config import Config -from galaxy.model.migrations.scripts import get_configuration_from_file +from galaxy.model.migrations import verify_databases_via_script +from galaxy.model.migrations.scripts import ( + get_configuration, + get_configuration_from_file, +) + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) DEFAULT_CONFIG_NAMES = ["galaxy", "universe_wsgi"] CONFIG_FILE_ARG = "--galaxy-config" @@ -29,7 +42,7 @@ class DbScript: self._set_dburl(config_file) self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) - def upgrade(self, args: argparse.Namespace) -> None: + def upgrade(self, args: Namespace) -> None: def upgrade_to_revision(rev): command.upgrade(self.alembic_config, rev, args.sql) @@ -45,24 +58,24 @@ class DbScript: finally: self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) - def downgrade(self, args: argparse.Namespace) -> None: + def downgrade(self, args: Namespace) -> None: revision = self._parse_revision(args.revision) command.downgrade(self.alembic_config, revision, args.sql) - def revision(self, args: argparse.Namespace) -> None: + def revision(self, args: Namespace) -> None: """Create revision script for the gxy branch only.""" command.revision(self.alembic_config, message=args.message, rev_id=args.rev_id, head="gxy@head") - def version(self, args: argparse.Namespace) -> None: + def version(self, args: Namespace) -> None: command.heads(self.alembic_config, verbose=args.verbose) - def dbversion(self, args: argparse.Namespace) -> None: + def dbversion(self, args: Namespace) -> None: command.current(self.alembic_config, verbose=args.verbose) - def history(self, args: argparse.Namespace) -> None: + def history(self, args: Namespace) -> None: command.history(self.alembic_config, verbose=args.verbose, indicate_current=args.indicate_current) - def show(self, args: argparse.Namespace) -> None: + def show(self, args: Namespace) -> None: command.show(self.alembic_config, args.revision) def _get_alembic_cfg(self) -> Config: @@ -82,3 +95,159 @@ class DbScript: if rev.startswith("+") or rev.startswith("-"): return f"gxy@{rev}" return rev + + +class ParserBuilder: + """ + Assembler object that simplifies the construction of an argument parser for db/db_dev migration scripts. + """ + + def __init__(self, parser, subcommand_required=True): + self.parser = parser + self.subparsers = parser.add_subparsers(required=subcommand_required) + self._init_arg_parsers() + + def add_upgrade_command(self): + parser = self._add_parser( + "upgrade", + Command.upgrade, + "Upgrade to a later version", + parents=[self._sql_arg_parser], + ) + parser.add_argument("revision", help="Revision identifier", nargs="?") + + def add_downgrade_command(self): + parser = self._add_parser( + "downgrade", + Command.downgrade, + "Revert to a previous version", + parents=[self._sql_arg_parser], + ) + parser.add_argument("revision", help="Revision identifier") + + def add_version_command(self): + self._add_parser( + "version", + Command.version, + "Show the head revision in the migrations script directory", + aliases=["v"], + parents=[self._verbose_arg_parser], + ) + + def add_dbversion_command(self): + self._add_parser( + "dbversion", + Command.dbversion, + "Show the current revision for Galaxy's database", + aliases=["dv"], + parents=[self._verbose_arg_parser], + ) + + def add_init_command(self): + self._add_parser( + "init", + Command.init, + "Initialize empty database(s) for both branches (create database objects for gxy and tsi branch)", + ) + + def add_revision_command(self): + parser = self._add_parser("revision", help="Create a new revision file", func=Command.revision) + parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) + parser.add_argument( + "--rev-id", + help="Specify a revision id instead of generating one (This option is for testing purposes only)", + ) + + def add_history_command(self): + parser = self._add_parser( + "history", + Command.history, + "List revision scripts in chronological order", + aliases=["h"], + parents=[self._verbose_arg_parser], + ) + parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") + + def add_show_command(self): + parser = self._add_parser( + "show", + Command.show, + "Show the revision(s) denoted by the given symbol", + aliases=["s"], + ) + parser.add_argument("revision", help="Revision identifier") + + def _init_arg_parsers(self): + self._verbose_arg_parser = self._make_verbose_arg_parser() + self._sql_arg_parser = self._make_sql_arg_parser() + + def _make_verbose_arg_parser(self): + parser = ArgumentParser(add_help=False) + parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") + return parser + + def _make_sql_arg_parser(self): + parser = ArgumentParser(add_help=False) + parser.add_argument( + "--sql", + action="store_true", + help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", + ) + return parser + + def _add_parser(self, command, func, help, aliases=None, parents=None): + aliases = aliases or [] + parents = parents or [] + parser = self.subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) + parser.set_defaults(func=func) + return parser + + +class Command: + """Execute commands.""" + + @staticmethod + def upgrade(args: Namespace) -> None: + Command._exec_command("upgrade", args) + + @staticmethod + def downgrade(args: Namespace) -> None: + Command._exec_command("downgrade", args) + + @staticmethod + def version(args: Namespace) -> None: + Command._exec_command("version", args) + + @staticmethod + def dbversion(args: Namespace) -> None: + Command._exec_command("dbversion", args) + + @staticmethod + def init(args: Namespace) -> None: + gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) + verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) + + @staticmethod + def revision(args: Namespace) -> None: + Command._exec_command("revision", args) + + @staticmethod + def history(args: Namespace) -> None: + Command._exec_command("history", args) + + @staticmethod + def show(args: Namespace) -> None: + Command._exec_command("show", args) + + @staticmethod + def _exec_command(command: str, args: Namespace) -> None: + dbscript = DbScript(args.config) + try: + getattr(dbscript, command)(args) + except alembic.util.exc.CommandError as e: + if args.raiseerr: + raise + else: + log.error(e) + print(f"FAILED: {str(e)}") + sys.exit(1) diff --git a/scripts/db.py b/scripts/db.py index 6578564d6c1..f87b9b9d9d9 100644 --- a/scripts/db.py +++ b/scripts/db.py @@ -2,123 +2,29 @@ This script is intended to be invoked by the manage_db.sh script. """ -import logging import os import sys -from argparse import ( - ArgumentParser, - Namespace, -) - -import alembic +from argparse import ArgumentParser sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) -from galaxy.model.migrations import verify_databases_via_script -from galaxy.model.migrations.dbscript import DbScript -from galaxy.model.migrations.scripts import get_configuration - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) - - -def exec_upgrade(args: Namespace) -> None: - _exec_command("upgrade", args) - - -def exec_downgrade(args: Namespace) -> None: - _exec_command("downgrade", args) - - -def exec_version(args: Namespace) -> None: - _exec_command("version", args) - - -def exec_dbversion(args: Namespace) -> None: - _exec_command("dbversion", args) - - -def exec_init(args: Namespace) -> None: - gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) - verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) - - -def _exec_command(command: str, args: Namespace) -> None: - dbscript = DbScript(args.config) - try: - getattr(dbscript, command)(args) - except alembic.util.exc.CommandError as e: - if args.raiseerr: - raise - else: - log.error(e) - print(f"FAILED: {str(e)}") - sys.exit(1) +from galaxy.model.migrations.dbscript import ParserBuilder def main() -> None: - def add_parser(command, func, help, aliases=None, parents=None): - aliases = aliases or [] - parents = parents or [] - parser = subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) - parser.set_defaults(func=func) - return parser - - verbose_arg_parser = ArgumentParser(add_help=False) - verbose_arg_parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") - - sql_arg_parser = ArgumentParser(add_help=False) - sql_arg_parser.add_argument( - "--sql", - action="store_true", - help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", - ) - parser = ArgumentParser( prog="manage_db.sh", description="Common database schema migration operations", ) parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") - subparsers = parser.add_subparsers(required=True) + parser_builder = ParserBuilder(parser) - upgrade_cmd_parser = add_parser( - "upgrade", - exec_upgrade, - "Upgrade to a later version", - parents=[sql_arg_parser], - ) - upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?") - - downgrade_cmd_parser = add_parser( - "downgrade", - exec_downgrade, - "Revert to a previous version", - parents=[sql_arg_parser], - ) - downgrade_cmd_parser.add_argument("revision", help="Revision identifier") - - add_parser( - "version", - exec_version, - "Show the head revision in the migrations script directory", - aliases=["v"], - parents=[verbose_arg_parser], - ) - - add_parser( - "dbversion", - exec_dbversion, - "Show the current revision for Galaxy's database", - aliases=["dv"], - parents=[verbose_arg_parser], - ) - - add_parser( - "init", - exec_init, - "Initialize empty database(s) for both branches (create database objects for gxy and tsi branch)", - ) + parser_builder.add_upgrade_command() + parser_builder.add_downgrade_command() + parser_builder.add_version_command() + parser_builder.add_dbversion_command() + parser_builder.add_init_command() args = parser.parse_args() args.func(args) diff --git a/scripts/db_dev.py b/scripts/db_dev.py index 1a94ded1e29..e244c3822dd 100644 --- a/scripts/db_dev.py +++ b/scripts/db_dev.py @@ -2,161 +2,35 @@ This script is intended to be invoked by the scripts/db_dev.sh script. """ -import logging import os import sys -from argparse import ( - ArgumentParser, - Namespace, -) - -import alembic +from argparse import ArgumentParser sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) -from galaxy.model.migrations import verify_databases_via_script -from galaxy.model.migrations.dbscript import DbScript -from galaxy.model.migrations.scripts import get_configuration - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) - - -def exec_upgrade(args: Namespace) -> None: - _exec_command("upgrade", args) - - -def exec_downgrade(args: Namespace) -> None: - _exec_command("downgrade", args) - - -def exec_revision(args: Namespace) -> None: - _exec_command("revision", args) - - -def exec_version(args: Namespace) -> None: - _exec_command("version", args) - - -def exec_dbversion(args: Namespace) -> None: - _exec_command("dbversion", args) - - -def exec_history(args: Namespace) -> None: - _exec_command("history", args) - - -def exec_show(args: Namespace) -> None: - _exec_command("show", args) - - -def exec_init(args: Namespace) -> None: - gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) - verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) - - -def _exec_command(command: str, args: Namespace) -> None: - dbscript = DbScript(args.config) - try: - getattr(dbscript, command)(args) - except alembic.util.exc.CommandError as e: - if args.raiseerr: - raise - else: - log.error(e) - print(f"FAILED: {str(e)}") - sys.exit(1) +from galaxy.model.migrations.dbscript import ParserBuilder def main() -> None: - def add_parser(command, func, help, aliases=None, parents=None): - aliases = aliases or [] - parents = parents or [] - parser = subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) - parser.set_defaults(func=func) - return parser - - verbose_arg_parser = ArgumentParser(add_help=False) - verbose_arg_parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") - - sql_arg_parser = ArgumentParser(add_help=False) - sql_arg_parser.add_argument( - "--sql", - action="store_true", - help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", - ) - parser = ArgumentParser( - prog="db_dev.py", - description="Common database schema migration operations", + prog="db_dev.sh", + description="Extended database schema migration operations", epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." " For migrating the `tsi` branch, use the `run_alembic.sh` script.", ) parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") parser.add_argument("--raiseerr", help="Raise a full stack trace on error", action="store_true") - subparsers = parser.add_subparsers(required=True) + parser_builder = ParserBuilder(parser) - upgrade_cmd_parser = add_parser( - "upgrade", - exec_upgrade, - "Upgrade to a later version", - parents=[sql_arg_parser], - ) - upgrade_cmd_parser.add_argument("revision", help="Revision identifier", nargs="?") - - downgrade_cmd_parser = add_parser( - "downgrade", - exec_downgrade, - "Revert to a previous version", - parents=[sql_arg_parser], - ) - downgrade_cmd_parser.add_argument("revision", help="Revision identifier") - - add_parser( - "version", - exec_version, - "Show the head revision in the migrations script directory", - aliases=["v"], - parents=[verbose_arg_parser], - ) - - add_parser( - "dbversion", - exec_dbversion, - "Show the current revision for Galaxy's database", - aliases=["dv"], - parents=[verbose_arg_parser], - ) - - history_cmd_parser = add_parser( - "history", - exec_history, - "List revision scripts in chronological order", - aliases=["h"], - parents=[verbose_arg_parser], - ) - history_cmd_parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") - - show_cmd_parser = add_parser( - "show", - exec_show, - "Show the revision(s) denoted by the given symbol", - aliases=["s"], - ) - show_cmd_parser.add_argument("revision", help="Revision identifier") - - revision_cmd_parser = add_parser("revision", help="Create a new revision file", func=exec_revision) - revision_cmd_parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) - revision_cmd_parser.add_argument( - "--rev-id", help="Specify a revision id instead of generating one (This option is for testing purposes only)" - ) - - add_parser( - "init", - exec_init, - "Initialize empty database(s) for both branches (create database objects for gxy and tsi branch)", - ) + parser_builder.add_upgrade_command() + parser_builder.add_downgrade_command() + parser_builder.add_version_command() + parser_builder.add_dbversion_command() + parser_builder.add_init_command() + parser_builder.add_revision_command() + parser_builder.add_history_command() + parser_builder.add_show_command() args = parser.parse_args() args.func(args) From 61c4933947f257f0eab773041a428dcda54edc42 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 16 Sep 2022 12:17:21 -0400 Subject: [PATCH 31/34] Split documentation into admin and dev sections --- lib/galaxy/model/migrations/README.md | 342 ++++++++++++++++-------- lib/galaxy/model/migrations/dbscript.py | 2 +- 2 files changed, 228 insertions(+), 116 deletions(-) diff --git a/lib/galaxy/model/migrations/README.md b/lib/galaxy/model/migrations/README.md index c5ab8818414..5414881362f 100644 --- a/lib/galaxy/model/migrations/README.md +++ b/lib/galaxy/model/migrations/README.md @@ -1,120 +1,41 @@ # Galaxy Database Schema Migrations -## Overview - Galaxy's database schema migration system is built on top of [Alembic](https://alembic.sqlalchemy.org) - a lightweight database migration tool for usage with SQLAlchemy. -(This documentation applies to release 22.05 and up. Prior to 22.05, Galaxy has used SQLAlchemy Migrate.) - -The purpose of the database schema migration system is to support automated, incremental, reversible changes to Galaxy's database schema. Central to this is the concept of a database version (more specifically, database *schema* version). A version is represented by a revision script (the terms *version* and *revision* may be used interchangeably). Executing a revision script will upgrade, or downgrade, the database schema by applying the changes specified in the script. - -Galaxy keeps track of the database schema version in two places: one is a directory we refer to as "the migration environment" (located at `lib/galaxy/model/migrations/alembic`), the other is the `alembic_version` table in the database. For Galaxy to run, these versions must be the same, which, essentially, means that the version of the database matches the version expected by the codebase. - -On startup, the system checks if the version in the database matches the version in the codebase. If they do not match, and the `database_auto_migrate` configuration option is not set, Galaxy will fail with an error message explaining how to proceed. In most cases, you'll need to upgrade the database schema to the current version, which is represented by the latest revision script in the migration environment. +(This documentation applies to release 22.05 and up. Prior to 22.05, Galaxy used SQLAlchemy Migrate.) ## Administering Galaxy: upgrading and downgrading the database -To initialize an empty database (or create a new SQLite database) start Galaxy. To upgrade or downgrade an existing database, you'll need to use a script. - -Galaxy provides two scripts: `manage_db.sh` and `run_alembic.sh`. - -The `manage_db.sh` script is the recommended way to interact with Galaxy's database schema migration system. -It provides access to a subset of commands offered by Alembic's CLI, while hiding some of the -implementation complexity of Galaxy's model. The provided commands and options should be -sufficient for most use cases involving Galaxy development or system and database administration. - -The `run_alembic.sh` script is a thin wrapper around the Alembic CLI runner. It offers more -flexibility and the full scope of Alembic's CLI commands and options; however, it requires more detailed command arguments, as well as basic familiarity with [Alembic branches](https://alembic.sqlalchemy.org/en/latest/branches.html). - -#### Implementation detail: branches - -Galaxy's data model is split into the [galaxy model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/__init__.py) and the [install model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/tool_shed_install/__init__.py). -These two models may be persisted in one combined database (which is the default) or two separate databases (which is enabled by setting the -[`install_database_connection`](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/webapps/galaxy/config_schema.yml#L157) configuration option). - -To accommodate this setup, Galaxy uses [Alembic -branches](https://alembic.sqlalchemy.org/en/latest/branches.html#working-with-branches). A branch is -a versioning lineage that starts at a common base revision and represents part of Galaxy's data -model. These branches are identified by labels (***gxy*** for the galaxy model and ***tsi*** for the install -model) and may share the same Alembic version table (if they share the same database; otherwise, -each database has its own version table). Each branch has its own version history, represented by -revision scripts located in the branch version directory (`migrations/alembic/versions_gxy` for -*gxy* and `migrations/alembic/versions_tsi` for *tsi*). - -For a more detailed description of the system's internals, see pull request [#13108](https://github.com/galaxyproject/galaxy/pull/13108). +To initialize an empty database (or create a new SQLite database) start Galaxy. To upgrade or downgrade an existing database, you'll need to use the `manage_db.sh` script, which is the recommended way to interact with Galaxy's database schema migration system. ## manage_db.sh -This script offers a set of common database schema migration operations that are executed on the *gxy* branch, which, in the vast majority of cases, is all you need to develop and administer Galaxy. -To run operations on the *tsi* branch, you need to use `run_alembic.sh`. - ``` -usage: manage_db.sh [-h] [-c CONFIG] [--raiseerr] {upgrade,downgrade,version,v, - dbversion,dv,history,h,show,s,revision,init} ... +usage: manage_db.sh [-h] [-c CONFIG] {upgrade,downgrade,version,v,dbversion,dv,init} ... positional arguments: - {upgrade,downgrade,version,v,dbversion,dv,history,h,show,s,revision,init} + {upgrade,downgrade,version,v,dbversion,dv,init} upgrade Upgrade to a later version downgrade Revert to a previous version version (v) Show the head revision in the migrations script directory dbversion (dv) Show the current revision for Galaxy's database - history (h) List revision scripts in chronological order - show (s) Show the revision(s) denoted by the given symbol - revision Create a new revision file init Initialize empty database(s) optional arguments: -h, --help show this help message and exit -c CONFIG, --galaxy-config CONFIG Alternate Galaxy configuration file - --raiseerr Raise a full stack trace on error ``` -#### Revision identifiers - -Some of the commands accept revision identifiers as arguments. A revision is usually identified by a -12-digit hexadecimal number (i.e., `6a67bf27e6a6`). Anytime you need to refer to a specific -revision, you have the option to use a partial number. As long as the partial number uniquely -identifies the revision, you may use that partial number in any command in place of the full -revision number. - -For example, you may use `./manage_db.sh upgrade 6a` instead of `./manage_db.sh upgrade 6a67bf27e6a6` if `6a` is sufficient to uniquely identify that revision. - -(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#partial-revision-identifiers)) - -#### Relative migration identifiers - -You may also use Alembic's syntax for relative migration identifiers for the upgrade/downgrade commands: - -To move 2 versions from the current version, a decimal value `+N` can be supplied: - -`./manage_db.sh upgrade +2` - -Negative values are accepted for downgrades: - -`./manage_db.sh downgrade -2` - -Relative identifiers may also be in terms of a specific revision. For example, to upgrade to -revision 6a67bf27e6a6 plus two additional steps: - -`.manage_db.sh upgrade 6a67bf27e6a6+2`. - -You may also combine relative migration identifiers with partial revision identifiers: - -`.manage_db.sh upgrade 6a+2`. - -(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#relative-migration-identifiers) - ### Subcommands #### upgrade -Upgrade to a later version. The revision argument is optional: omitting it is equivalent to -specifying `heads` as the revision identifier; in that case, the database(s) will be upgraded to the -latest revisions in ***both*** branches, *gxy* and *tsi*. +Upgrade to a later version. The revision argument is optional. -***If you are upgrading a database that has not been version-controlled by Alembic, you should run -this command without the revision argument: `./manage_db.sh upgrade` - this will ensure that both branches, -`gxy` and `tsi`, are initialized.*** +If you are upgrading a database that +has not been version-controlled by Alembic, you should run this command for the fist time without +the revision argument: `./manage_db.sh upgrade` - this will ensure proper initialization of the +migration system for your database(s). ``` usage: manage_db.sh upgrade [-h] [--sql] [revision] @@ -144,18 +65,12 @@ optional arguments: --sql Don't emit SQL to database - dump to standard output/file instead. ``` -Specifying `base` as the revision identifier will downgrade both branches, *gxy* and *tsi*, to their -initial state prior to any revisions; the `alembic_version` table will be empty. - For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). -Note that in this mode, instead of specifying a revision identifier, you have to specify a range of revisions using the following format: `:`*. - -*You cannot use this script to downgrade past the initial Alembic revisions that created the *gxy* and *tsi* branches. - +Note that in this mode, instead of specifying a revision identifier, you have to specify a range of revisions using the following format: `:`. #### version -Show the head revision in the migrations script directory. This will display the latest (i.e., head) revision in the migration environment. +Show the latest (i.e., head) revisions in the codebase. ``` Activating virtualenv at .venv @@ -167,17 +82,10 @@ optional arguments: ``` -If your database is setup to host both branches (*gxy* and *tsi*), the head revisions for both branches will be displayed: - -``` -$ ./manage_db.sh version -186d4835587b (gxy) (head) -d4a650f47a3c (tsi) (head) -``` #### dbversion -Show the current revision for Galaxy's database. +Show the current revision for Galaxy's database. If the database revision corresponds to the head revision in the codebase, it will be marked as `(head)`. ``` usage: manage_db.sh dbversion [-h] [-v] @@ -187,13 +95,217 @@ optional arguments: -v, --verbose Display more detailed output ``` -Similar to the version command, this command will display the revision(s) stored in the -`alembic_version` table in the database. If the database revision corresponds to the head revision +#### init + +Initialize an empty database (or create a new SQLite database). + +``` +usage: manage_db.sh init [-h] + +optional arguments: + -h, --help show this help message and exit +``` + +# Advanced usage + +The following sections provide more details on the migration system and describe commands that are relevant for development scenarios. + +## Overview + +The purpose of the database schema migration system is to support automated, incremental, reversible changes to Galaxy's database schema. Central to this is the concept of a database version (more specifically, database *schema* version). A version is represented by a revision script (the terms *version* and *revision* may be used interchangeably). Executing a revision script will upgrade, or downgrade, the database schema by applying the changes specified in the script. + +Galaxy keeps track of the database schema version in two places: one is a directory we refer to as "the migration environment" (located at `lib/galaxy/model/migrations/alembic`), the other is the `alembic_version` table in the database. For Galaxy to run, these versions must be the same, which, essentially, means that the version of the database matches the version expected by the codebase. + +On startup, the system checks if the version in the database matches the version in the codebase. If they do not match, and the `database_auto_migrate` configuration option is not set, Galaxy will fail with an error message explaining how to proceed. In most cases, you'll need to upgrade the database schema to the current version, which is represented by the latest revision script in the migration environment. + +### A note on models and branch labels + +Galaxy's data model includes the [galaxy model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/__init__.py) and the [install model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/tool_shed_install/__init__.py). +These two models may be persisted in one combined database (which is the default) or two separate databases (which is enabled by setting the +[`install_database_connection`](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/webapps/galaxy/config_schema.yml#L157) configuration option). + +These models are represented by migration [branches](https://alembic.sqlalchemy.org/en/latest/branches.html#working-with-branches) (versioning lineages with a common base) labeled as *gxy* for the galaxy model and *tsi* for the install model. If both models are hosted in the same databases, the branches will share the same Alembic version table; otherwise, each database has its own version table. + +Each branch has its own version history, represented by revision scripts located in the branch version directory (`migrations/alembic/versions_gxy` for *gxy* and `migrations/alembic/versions_tsi` for *tsi*). + +For a more detailed description of the system's internals, see pull request [#13108](https://github.com/galaxyproject/galaxy/pull/13108). + +## Migration management scripts + +The **`manage_db.sh`** script is the recommended way to interact with Galaxy's database schema migration system. +It provides access to a subset of commands offered by Alembic's CLI, while hiding some of the +implementation complexity of Galaxy's model. The provided commands and options should be +sufficient for most use cases involving Galaxy development or system and database administration. + +Additionally, Galaxy provides two scripts for advanced usage: `db_dev.sh` and `run_alembic.sh`. Both are located in the `scripts` directory. + +The **`db_dev.sh`** script is similar to `manage_db.sh`, but provides additional commands and options. + +The **`run_alembic.sh`** script is a thin wrapper around the Alembic CLI runner. It offers more +flexibility and the full scope of Alembic's CLI commands and options; however, it requires more detailed command arguments, as well as basic familiarity with [Alembic branches](https://alembic.sqlalchemy.org/en/latest/branches.html). + +### Revision identifiers + +Some of the commands accept revision identifiers as arguments. A revision is usually identified by a +12-digit hexadecimal number (i.e., `6a67bf27e6a6`). Anytime you need to refer to a specific +revision, you have the option to use a partial number. + +For example, you may use `./db_dev.sh upgrade 6a` instead of `./db_dev.sh upgrade 6a67bf27e6a6` if `6a` is sufficient to uniquely identify that revision. + +(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#partial-revision-identifiers)) + +### Relative migration identifiers + +You may also use Alembic's syntax for relative migration identifiers for the upgrade/downgrade commands: + +To move 2 versions from the current version, a decimal value `+N` can be supplied: + +`./db_dev.sh upgrade +2` + +Negative values are accepted for downgrades: + +`./db_dev.sh downgrade -2` + +Relative identifiers may also be in terms of a specific revision. For example, to upgrade to +revision 6a67bf27e6a6 plus two additional steps: + +`./db_dev.sh upgrade 6a67bf27e6a6+2`. + +You may also combine relative migration identifiers with partial revision identifiers: + +`./db_dev.sh upgrade 6a+2`. + +(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#relative-migration-identifiers) + +*Revision identifiers and relative migration identifiers can be used with all the provided scripts.* + +## db_dev.sh + +This script offers a set of common database schema migration operations that are executed on the *gxy* branch, which, in the vast majority of cases, is all you need to develop and administer Galaxy. +To run operations on the *tsi* branch, you need to use `run_alembic.sh`. + +``` +usage: db_dev.sh [-h] [-c CONFIG] [--raiseerr] {upgrade,downgrade,version,v, + dbversion,dv,history,h,show,s,revision,init} ... + +positional arguments: + {upgrade,downgrade,version,v,dbversion,dv,history,h,show,s,revision,init} + upgrade Upgrade to a later version + downgrade Revert to a previous version + version (v) Show the head revision in the migrations script directory + dbversion (dv) Show the current revision for Galaxy's database + history (h) List revision scripts in chronological order + show (s) Show the revision(s) denoted by the given symbol + revision Create a new revision file + init Initialize empty database(s) + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --galaxy-config CONFIG + Alternate Galaxy configuration file + --raiseerr Raise a full stack trace on error +``` + +### Subcommands + +#### upgrade + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Upgrade to a later version. The revision argument is optional. + +Omitting the revision argument is equivalent to specifying `heads` as the revision identifier; in +that case, the database(s) will be upgraded to the latest revisions in both branches, *gxy* and +*tsi*. + +If you are upgrading a database that has not been version-controlled by Alembic, you should run this +command for the fist time without the revision argument: `./db_dev.sh upgrade` - this will ensure +proper initialization of the migration system for your database(s). + + +``` +usage: db_dev.sh upgrade [-h] [--sql] [revision] + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). + +#### downgrade + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Revert to a previous version. + +``` +usage: db_dev.sh downgrade [-h] [--sql] revision + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +Specifying `base` as the revision identifier will downgrade both branches, *gxy* and *tsi*, to their +initial state prior to any revisions; the `alembic_version` table will be empty. + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). +Note that in this mode, instead of specifying a revision identifier, you have to specify a range of revisions using the following format: `:`*. + +*You cannot use this script to downgrade past the initial Alembic revisions that created the *gxy* and *tsi* branches. + + +#### version + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Show the latest (i.e., head) revisions in the codebase. + +``` +Activating virtualenv at .venv +\usage: db_dev.sh version [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output + +``` + +The output of this command will include the head revisions for both branches: + +``` +$ .db_dev.sh version +186d4835587b (gxy) (head) +d4a650f47a3c (tsi) (head) +``` + +#### dbversion + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Show the current revision for Galaxy's database. + +``` +usage: db_dev.sh dbversion [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output +``` + +If the database revision corresponds to the head revision in the codebase, it will be marked as `(head)`. The output will be slightly more verbose and will vary depending on the database. ``` -$ ./manage_db.sh dbversion +$ ./db_dev.sh dbversion INFO:alembic.runtime.migration:Context impl PostgresqlImpl. INFO:alembic.runtime.migration:Will assume transactional DDL. d4a650f47a3c (head) @@ -205,7 +317,7 @@ d4a650f47a3c (head) List revision scripts in chronological order. ``` -usage: manage_db.sh history [-h] [-v] [-i] +usage: db_dev.sh history [-h] [-v] [-i] optional arguments: -h, --help show this help message and exit @@ -215,12 +327,12 @@ optional arguments: ``` Depending on your setup, the list may include revision histories for both branches. The oldest -revisions are the ones that created the `gxy` and `tsi` branches (introduced in 22.05). The +revisions are the ones that created the *gxy* and *tsi* branches (introduced in 22.05). The `--indicate-current` option is particularly useful when you need to determine how far behind (or ahead) your database version is compared to the version expected by your codebase: ``` -$ ./manage_db.sh history --indicate-current +$ ./db_dev.sh history --indicate-current 6a67bf27e6a6 -> 186d4835587b (gxy) (head), drop job_state_history.update_time column b182f655505f -> 6a67bf27e6a6 (gxy) (current), deferred data tables e7b6dcb09efd -> b182f655505f (gxy), add workflow.source_metadata column @@ -233,7 +345,7 @@ e7b6dcb09efd -> b182f655505f (gxy), add workflow.source_metadata column Show the revision(s) denoted by the given revision identifier. ``` -usage: manage_db.sh show [-h] revision +usage: db_dev.sh show [-h] revision positional arguments: revision Revision identifier @@ -247,7 +359,7 @@ optional arguments: Create a new revision file. ``` -usage: manage_db.sh revision [-h] -m MESSAGE [--rev-id REV_ID] +usage: db_dev.sh revision [-h] -m MESSAGE [--rev-id REV_ID] optional arguments: -h, --help show this help message and exit @@ -262,7 +374,7 @@ appended to the new revision identifier to form the filename for the new revisio should be a succinct description of the change: ``` -$ ./manage_db.sh revision --message "add column foo to table bar" +$ ./db_dev.sh revision --message "add column foo to table bar" [output omitted] $ ls lib/galaxy/model/migrations/alembic/versions_gxy/ @@ -271,10 +383,10 @@ a2e418ad6a15_add_column_foo_to_table_bar.py #### init -Initialize an empty database (or create a new SQLite database) for both branches, `gxy` and `tsi` (creates database objects in one or two databases, depending on configuration settings). +Initialize an empty database (or create a new SQLite database) for both branches, *gxy* and *tsi*. ``` -usage: manage_db.sh init [-h] +usage: db_dev.sh init [-h] optional arguments: -h, --help show this help message and exit diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index 4e841a2f106..c1686585605 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -147,7 +147,7 @@ class ParserBuilder: self._add_parser( "init", Command.init, - "Initialize empty database(s) for both branches (create database objects for gxy and tsi branch)", + "Initialize empty database(s)", ) def add_revision_command(self): From 98218ac23629770133ff72c52d3ca469393fb119 Mon Sep 17 00:00:00 2001 From: John Davis Date: Fri, 16 Sep 2022 12:30:55 -0400 Subject: [PATCH 32/34] Point to init command as safer way to initialize empty database --- lib/galaxy/model/migrations/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/migrations/README.md b/lib/galaxy/model/migrations/README.md index 5414881362f..7f51f07dd7c 100644 --- a/lib/galaxy/model/migrations/README.md +++ b/lib/galaxy/model/migrations/README.md @@ -5,7 +5,9 @@ Galaxy's database schema migration system is built on top of [Alembic](https://a ## Administering Galaxy: upgrading and downgrading the database -To initialize an empty database (or create a new SQLite database) start Galaxy. To upgrade or downgrade an existing database, you'll need to use the `manage_db.sh` script, which is the recommended way to interact with Galaxy's database schema migration system. +To initialize an empty database (or create a new SQLite database) start Galaxy. However, this approach is safe only if booting to a single process. A better approach is to use the [*init*](#init) command: `manage_db.sh init`. + +To upgrade or downgrade an existing database, you'll need to use the `manage_db.sh` script, which is the recommended way to interact with Galaxy's database schema migration system. ## manage_db.sh From 5ea65e6c0e5aa471993fe32b68d19c076684b82b Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 29 Sep 2022 12:04:57 -0400 Subject: [PATCH 33/34] Fix error in sample code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: David López <46503462+davelopez@users.noreply.github.com> --- lib/galaxy/model/migrations/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/model/migrations/README.md b/lib/galaxy/model/migrations/README.md index 7f51f07dd7c..6ef1ebf14d7 100644 --- a/lib/galaxy/model/migrations/README.md +++ b/lib/galaxy/model/migrations/README.md @@ -454,11 +454,11 @@ Downgrade to 1 revision below specific revision: To create a revision for the galaxy model: -`./run_alembic.sh revision --head=gxy@head -message "your description"` +`./run_alembic.sh revision --head=gxy@head --message "your description"` To create a revision for the install model: -`./run_alembic.sh revision --head=tsi@head -message "your description"` +`./run_alembic.sh revision --head=tsi@head --message "your description"` Check [Alembic's documentation](https://alembic.sqlalchemy.org) for more examples. From 8538ab7fcb45bb503a45d2a63938991fbcd2a0b2 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 29 Sep 2022 12:23:28 -0400 Subject: [PATCH 34/34] Fix same error in sample code in the rst doc file --- doc/source/admin/db_migration.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/source/admin/db_migration.md b/doc/source/admin/db_migration.md index c3c620fbfff..605bbc77501 100644 --- a/doc/source/admin/db_migration.md +++ b/doc/source/admin/db_migration.md @@ -454,11 +454,11 @@ Downgrade to 1 revision below specific revision: To create a revision for the galaxy model: -`./run_alembic.sh revision --head=gxy@head -message "your description"` +`./run_alembic.sh revision --head=gxy@head --message "your description"` To create a revision for the install model: -`./run_alembic.sh revision --head=tsi@head -message "your description"` +`./run_alembic.sh revision --head=tsi@head --message "your description"` Check [Alembic's documentation](https://alembic.sqlalchemy.org) for more examples.