diff --git a/lib/galaxy/model/migrations/base.py b/lib/galaxy/model/migrations/base.py new file mode 100644 index 00000000000..69deec085ed --- /dev/null +++ b/lib/galaxy/model/migrations/base.py @@ -0,0 +1,266 @@ +import abc +import logging +import os +import sys +from argparse import ( + ArgumentParser, + Namespace, +) +from typing import ( + List, + Optional, +) + +import alembic +from alembic import command +from alembic.config import Config + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + + +class BaseParserBuilder(abc.ABC): + """ + Assembler object that simplifies the construction of an argument parser for database migration scripts. + """ + + @abc.abstractmethod + def _get_command_object(self): + ... + + def __init__(self, parser, subcommand_required=True): + self._cmd = self._get_command_object() + self.parser = parser + self.subparsers = parser.add_subparsers(required=subcommand_required) + self._init_arg_parsers() + + def add_upgrade_command(self): + parser = self._add_parser( + "upgrade", + self._cmd.upgrade, + "Upgrade to a later version", + parents=[self._sql_arg_parser], + ) + parser.add_argument("revision", help="Revision identifier or release tag", nargs="?") + + def add_downgrade_command(self): + parser = self._add_parser( + "downgrade", + self._cmd.downgrade, + "Revert to a previous version", + parents=[self._sql_arg_parser], + ) + parser.add_argument("revision", help="Revision identifier or release tag") + + def add_version_command(self): + self._add_parser( + "version", + self._cmd.version, + "Show the head revision in the migrations script directory", + aliases=["v"], + parents=[self._verbose_arg_parser], + ) + + def add_dbversion_command(self): + self._add_parser( + "dbversion", + self._cmd.dbversion, + "Show the current revision for the database", + aliases=["dv"], + parents=[self._verbose_arg_parser], + ) + + def add_init_command(self): + self._add_parser( + "init", + self._cmd.init, + "Initialize empty database(s)", + ) + + def add_revision_command(self): + parser = self._add_parser("revision", help="Create a new revision file", func=self._cmd.revision) + parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) + parser.add_argument( + "--rev-id", + help="Specify a revision id instead of generating one (This option is for testing purposes only)", + ) + + def add_history_command(self): + parser = self._add_parser( + "history", + self._cmd.history, + "List revision scripts in chronological order", + aliases=["h"], + parents=[self._verbose_arg_parser], + ) + parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") + + def add_show_command(self): + parser = self._add_parser( + "show", + self._cmd.show, + "Show the revision(s) denoted by the given symbol", + aliases=["s"], + ) + parser.add_argument("revision", help="Revision identifier") + + def _init_arg_parsers(self): + self._verbose_arg_parser = self._make_verbose_arg_parser() + self._sql_arg_parser = self._make_sql_arg_parser() + + def _make_verbose_arg_parser(self): + parser = ArgumentParser(add_help=False) + parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") + return parser + + def _make_sql_arg_parser(self): + parser = ArgumentParser(add_help=False) + parser.add_argument( + "--sql", + action="store_true", + help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", + ) + return parser + + def _add_parser(self, command, func, help, aliases=None, parents=None): + aliases = aliases or [] + parents = parents or [] + parser = self.subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) + parser.set_defaults(func=func) + return parser + + +class BaseDbScript(abc.ABC): + """ + Facade for common database schema migration operations on the gxy branch. + When the gxy and tsi branches are persisted in the same database, some + alembic commands will display output on the state on both branches (e.g. + history, version, dbversion). The upgrade command is executed on both + branches: gxy and tsi (the upgrade command ensures the branch has been + initialized by stamping its version in the alembic_version table). + """ + + @abc.abstractmethod + def _set_dburl(self, config_file: Optional[str] = None) -> None: + ... + + @abc.abstractmethod + def _upgrade_to_head(self, is_sql_mode: bool): + ... + + def __init__(self, config_file: Optional[str] = None) -> None: + self.alembic_config = self._get_alembic_cfg() + self._set_dburl(config_file) + + def revision(self, args: Namespace) -> None: + head = self._add_branch_label("head") + command.revision(self.alembic_config, message=args.message, rev_id=args.rev_id, head=head) + + def upgrade(self, args: Namespace) -> None: + if args.revision: + revision = self._parse_revision(args.revision) + self._upgrade_to_revision(revision, args.sql) + else: + self._upgrade_to_head(args.sql) + + def downgrade(self, args: Namespace) -> None: + revision = self._parse_revision(args.revision) + command.downgrade(self.alembic_config, revision, args.sql) + + def version(self, args: Namespace) -> None: + command.heads(self.alembic_config, verbose=args.verbose) + + def dbversion(self, args: Namespace) -> None: + command.current(self.alembic_config, verbose=args.verbose) + + def history(self, args: Namespace) -> None: + command.history(self.alembic_config, verbose=args.verbose, indicate_current=args.indicate_current) + + def show(self, args: Namespace) -> None: + command.show(self.alembic_config, args.revision) + + def _get_alembic_cfg(self) -> Config: + config_file = os.getenv("ALEMBIC_CONFIG") + if not config_file: + # We need to reference the runtime module because this is an abstract class, + # so the path will be different for different concrete classes. + module_file = sys.modules[self.__module__].__file__ + assert module_file + config_file = os.path.join(os.path.dirname(module_file), "alembic.ini") + # config_file = os.path.join(os.path.dirname(module.__file__), "alembic.ini") + config_file = os.path.abspath(config_file) + return Config(config_file) + + def _upgrade_to_revision(self, rev, is_sql_mode: bool): + command.upgrade(self.alembic_config, rev, is_sql_mode) + + def _parse_revision(self, revision_id: str) -> str: + # If branches are used, relative revision identifier requires a branch label. + if revision_id.startswith("+") or revision_id.startswith("-"): + return self._add_branch_label(revision_id) + # Check if it's a tag, leave unchanged if not + revision_id = self._revision_tags().get(revision_id, revision_id) + return revision_id + + def _add_branch_label(self, revision_id: str) -> str: + # Subclasses that need to add a branch label should overwrite this method. + return revision_id + + def _revision_tags(self): + # Subclasses that have revision tags should overwrite this method. + return {} + + +class BaseCommand(abc.ABC): + @abc.abstractmethod + def init(self, args: Namespace) -> None: + ... + + @abc.abstractmethod + def _get_dbscript(self, config_file: str) -> BaseDbScript: + ... + + def upgrade(self, args: Namespace) -> None: + self._exec_command("upgrade", args) + + def downgrade(self, args: Namespace) -> None: + self._exec_command("downgrade", args) + + def version(self, args: Namespace) -> None: + self._exec_command("version", args) + + def dbversion(self, args: Namespace) -> None: + self._exec_command("dbversion", args) + + def revision(self, args: Namespace) -> None: + self._exec_command("revision", args) + + def history(self, args: Namespace) -> None: + self._exec_command("history", args) + + def show(self, args: Namespace) -> None: + self._exec_command("show", args) + + def _exec_command(self, command: str, args: Namespace) -> None: + dbscript = self._get_dbscript(args.config) + + try: + getattr(dbscript, command)(args) + except alembic.util.exc.CommandError as e: + if hasattr(args, "raiseerr") and args.raiseerr: + raise + else: + log.error(e) + print(f"FAILED: {str(e)}") + sys.exit(1) + + +def pop_arg_from_args(args: List[str], arg_name) -> Optional[str]: + """ + Pop and return argument name and value from args if arg_name is in args. + """ + if arg_name in args: + pos = args.index(arg_name) + args.pop(pos) # pop argument name + return args.pop(pos) # pop and return argument value + return None diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py index f1326fe39f0..4403f7fd634 100644 --- a/lib/galaxy/model/migrations/dbscript.py +++ b/lib/galaxy/model/migrations/dbscript.py @@ -1,17 +1,15 @@ import logging import os import sys -from argparse import ( - ArgumentParser, - Namespace, -) +from argparse import Namespace from typing import Optional -import alembic -from alembic import command -from alembic.config import Config - from galaxy.model.migrations import verify_databases_via_script +from galaxy.model.migrations.base import ( + BaseCommand, + BaseDbScript, + BaseParserBuilder, +) from galaxy.model.migrations.scripts import ( get_configuration, get_configuration_from_file, @@ -39,229 +37,38 @@ REVISION_TAGS = { } -class DbScript: - """ - Facade for common database schema migration operations on the gxy branch. - When the gxy and tsi branches are persisted in the same database, some - alembic commands will display output on the state on both branches (e.g. - history, version, dbversion). The upgrade command is executed on both - branches: gxy and tsi (the upgrade command ensures the branch has been - initialized by stamping its version in the alembic_version table). - """ +class ParserBuilder(BaseParserBuilder): + def _get_command_object(self): + return Command() - def __init__(self, config_file: Optional[str] = None) -> None: - self.alembic_config = self._get_alembic_cfg() - self._set_dburl(config_file) - self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) - def upgrade(self, args: Namespace) -> None: - def upgrade_to_revision(rev): - command.upgrade(self.alembic_config, rev, args.sql) +class Command(BaseCommand): + def init(self, args: Namespace) -> None: + gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) + verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) - if args.revision: - revision = self._parse_revision(args.revision) - upgrade_to_revision(revision) - else: - self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) - upgrade_to_revision("gxy@head") - try: - self.alembic_config.set_main_option("sqlalchemy.url", self.tsi_url) - upgrade_to_revision("tsi@head") - finally: - self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) + def _get_dbscript(self, config_file: str) -> BaseDbScript: + return DbScript(config_file) - def downgrade(self, args: Namespace) -> None: - revision = self._parse_revision(args.revision) - command.downgrade(self.alembic_config, revision, args.sql) - def revision(self, args: Namespace) -> None: - """Create revision script for the gxy branch only.""" - command.revision(self.alembic_config, message=args.message, rev_id=args.rev_id, head="gxy@head") +class DbScript(BaseDbScript): + def _add_branch_label(self, revision_id: str) -> str: + return f"gxy@{revision_id}" - def version(self, args: Namespace) -> None: - command.heads(self.alembic_config, verbose=args.verbose) - - def dbversion(self, args: Namespace) -> None: - command.current(self.alembic_config, verbose=args.verbose) - - def history(self, args: Namespace) -> None: - command.history(self.alembic_config, verbose=args.verbose, indicate_current=args.indicate_current) - - def show(self, args: Namespace) -> None: - command.show(self.alembic_config, args.revision) - - def _get_alembic_cfg(self) -> Config: - config_file = os.getenv("ALEMBIC_CONFIG") - if not config_file: - config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") - config_file = os.path.abspath(config_file) - return Config(config_file) + def _revision_tags(self): + return REVISION_TAGS def _set_dburl(self, config_file: Optional[str] = None) -> None: gxy_config, tsi_config, _ = get_configuration_from_file(os.getcwd(), config_file) self.gxy_url = gxy_config.url self.tsi_url = tsi_config.url + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) - def _parse_revision(self, rev: str) -> str: - # Relative revision identifier requires a branch label - if rev.startswith("+") or rev.startswith("-"): - return f"gxy@{rev}" - # Check if it's a tag, leave unchanged if not - rev = REVISION_TAGS.get(rev, rev) - return rev - - -class ParserBuilder: - """ - Assembler object that simplifies the construction of an argument parser for db/db_dev migration scripts. - """ - - def __init__(self, parser, subcommand_required=True): - self.parser = parser - self.subparsers = parser.add_subparsers(required=subcommand_required) - self._init_arg_parsers() - - def add_upgrade_command(self): - parser = self._add_parser( - "upgrade", - Command.upgrade, - "Upgrade to a later version", - parents=[self._sql_arg_parser], - ) - parser.add_argument("revision", help="Revision identifier or release tag", nargs="?") - - def add_downgrade_command(self): - parser = self._add_parser( - "downgrade", - Command.downgrade, - "Revert to a previous version", - parents=[self._sql_arg_parser], - ) - parser.add_argument("revision", help="Revision identifier or release tag") - - def add_version_command(self): - self._add_parser( - "version", - Command.version, - "Show the head revision in the migrations script directory", - aliases=["v"], - parents=[self._verbose_arg_parser], - ) - - def add_dbversion_command(self): - self._add_parser( - "dbversion", - Command.dbversion, - "Show the current revision for Galaxy's database", - aliases=["dv"], - parents=[self._verbose_arg_parser], - ) - - def add_init_command(self): - self._add_parser( - "init", - Command.init, - "Initialize empty database(s)", - ) - - def add_revision_command(self): - parser = self._add_parser("revision", help="Create a new revision file", func=Command.revision) - parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) - parser.add_argument( - "--rev-id", - help="Specify a revision id instead of generating one (This option is for testing purposes only)", - ) - - def add_history_command(self): - parser = self._add_parser( - "history", - Command.history, - "List revision scripts in chronological order", - aliases=["h"], - parents=[self._verbose_arg_parser], - ) - parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") - - def add_show_command(self): - parser = self._add_parser( - "show", - Command.show, - "Show the revision(s) denoted by the given symbol", - aliases=["s"], - ) - parser.add_argument("revision", help="Revision identifier") - - def _init_arg_parsers(self): - self._verbose_arg_parser = self._make_verbose_arg_parser() - self._sql_arg_parser = self._make_sql_arg_parser() - - def _make_verbose_arg_parser(self): - parser = ArgumentParser(add_help=False) - parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") - return parser - - def _make_sql_arg_parser(self): - parser = ArgumentParser(add_help=False) - parser.add_argument( - "--sql", - action="store_true", - help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", - ) - return parser - - def _add_parser(self, command, func, help, aliases=None, parents=None): - aliases = aliases or [] - parents = parents or [] - parser = self.subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) - parser.set_defaults(func=func) - return parser - - -class Command: - """Execute commands.""" - - @staticmethod - def upgrade(args: Namespace) -> None: - Command._exec_command("upgrade", args) - - @staticmethod - def downgrade(args: Namespace) -> None: - Command._exec_command("downgrade", args) - - @staticmethod - def version(args: Namespace) -> None: - Command._exec_command("version", args) - - @staticmethod - def dbversion(args: Namespace) -> None: - Command._exec_command("dbversion", args) - - @staticmethod - def init(args: Namespace) -> None: - gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) - verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) - - @staticmethod - def revision(args: Namespace) -> None: - Command._exec_command("revision", args) - - @staticmethod - def history(args: Namespace) -> None: - Command._exec_command("history", args) - - @staticmethod - def show(args: Namespace) -> None: - Command._exec_command("show", args) - - @staticmethod - def _exec_command(command: str, args: Namespace) -> None: - dbscript = DbScript(args.config) + def _upgrade_to_head(self, is_sql_mode: bool): + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) + self._upgrade_to_revision("gxy@head", is_sql_mode) try: - getattr(dbscript, command)(args) - except alembic.util.exc.CommandError as e: - if hasattr(args, "raiseerr") and args.raiseerr: - raise - else: - log.error(e) - print(f"FAILED: {str(e)}") - sys.exit(1) + self.alembic_config.set_main_option("sqlalchemy.url", self.tsi_url) + self._upgrade_to_revision("tsi@head", is_sql_mode) + finally: + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py index ab775eef7df..068b67a0e71 100644 --- a/lib/galaxy/model/migrations/scripts.py +++ b/lib/galaxy/model/migrations/scripts.py @@ -24,6 +24,7 @@ from galaxy.model.migrations import ( SQLALCHEMYMIGRATE_LAST_VERSION_GXY, TSI, ) +from galaxy.model.migrations.base import pop_arg_from_args from galaxy.util.properties import ( find_config_file, get_data_dir, @@ -86,7 +87,7 @@ def get_configuration(argv: List[str], cwd: str) -> Tuple[DatabaseConfig, Databa """ Return a 3-item-tuple with configuration values used for managing databases. """ - config_file = _pop_config_file(argv) + config_file = pop_arg_from_args(argv, CONFIG_FILE_ARG) return get_configuration_from_file(cwd, config_file) @@ -117,14 +118,6 @@ def get_configuration_from_file( return (gxy_config, tsi_config, is_auto_migrate) -def _pop_config_file(argv: List[str]) -> Optional[str]: - if CONFIG_FILE_ARG in argv: - pos = argv.index(CONFIG_FILE_ARG) - argv.pop(pos) # pop argument name - return argv.pop(pos) # pop and return argument value - return None - - def add_db_urls_to_command_arguments(argv: List[str], gxy_url: str, tsi_url: str) -> None: _insert_x_argument(argv, f"{TSI}_url", tsi_url) _insert_x_argument(argv, f"{GXY}_url", gxy_url) diff --git a/lib/tool_shed/webapp/model/migrations/__init__.py b/lib/tool_shed/webapp/model/migrations/__init__.py index b1f8d1bff84..a10c0fe79ab 100644 --- a/lib/tool_shed/webapp/model/migrations/__init__.py +++ b/lib/tool_shed/webapp/model/migrations/__init__.py @@ -55,6 +55,7 @@ def verify_database(url, engine_options=None): engine = create_engine(url, **engine_options) verifier = DatabaseStateVerifier(engine) verifier.run() + engine.dispose() class AlembicManager: diff --git a/lib/tool_shed/webapp/model/migrations/alembic/env.py b/lib/tool_shed/webapp/model/migrations/alembic/env.py index 71f40d7e724..4c05fabaf5d 100644 --- a/lib/tool_shed/webapp/model/migrations/alembic/env.py +++ b/lib/tool_shed/webapp/model/migrations/alembic/env.py @@ -42,7 +42,7 @@ def _run_migrations_invoked_via_script(run_migrations: Callable[[str], None]) -> run_migrations(url) -def _process_cmd_current(url: str) -> bool: # TODO check this! +def _process_cmd_current(url: str) -> bool: if config.cmd_opts.cmd[0].__name__ == "current": # type: ignore[union-attr] # Run command for each url only if urls are different; otherwise run once. _configure_and_run_migrations_online(url) diff --git a/lib/tool_shed/webapp/model/migrations/dbscript.py b/lib/tool_shed/webapp/model/migrations/dbscript.py new file mode 100644 index 00000000000..ec9de2c618d --- /dev/null +++ b/lib/tool_shed/webapp/model/migrations/dbscript.py @@ -0,0 +1,45 @@ +import logging +import os +import sys +from argparse import Namespace +from typing import Optional + +from galaxy.model.migrations.base import ( + BaseCommand, + BaseDbScript, + BaseParserBuilder, +) +from tool_shed.webapp.model.migrations import verify_database +from tool_shed.webapp.model.migrations.scripts import ( + get_dburl, + get_dburl_from_file, +) + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + +CONFIG_FILE_ARG = "--toolshed-config" + + +class ParserBuilder(BaseParserBuilder): + def _get_command_object(self): + return Command() + + +class Command(BaseCommand): + def init(self, args: Namespace) -> None: + url = get_dburl(sys.argv, os.getcwd()) + verify_database(url) + + def _get_dbscript(self, config_file: str) -> BaseDbScript: + return DbScript(config_file) + + +class DbScript(BaseDbScript): + def _set_dburl(self, config_file: Optional[str] = None) -> None: + self.url = get_dburl_from_file(os.getcwd(), config_file) + self.alembic_config.set_main_option("sqlalchemy.url", self.url) + + def _upgrade_to_head(self, is_sql_mode: bool): + self.alembic_config.set_main_option("sqlalchemy.url", self.url) + self._upgrade_to_revision("head", is_sql_mode) diff --git a/lib/tool_shed/webapp/model/migrations/scripts.py b/lib/tool_shed/webapp/model/migrations/scripts.py new file mode 100644 index 00000000000..7bd28000106 --- /dev/null +++ b/lib/tool_shed/webapp/model/migrations/scripts.py @@ -0,0 +1,34 @@ +import os +from typing import ( + List, + Optional, +) + +from galaxy.model.migrations.base import pop_arg_from_args +from galaxy.util.properties import ( + find_config_file, + get_data_dir, + load_app_properties, +) + +DEFAULT_CONFIG_NAMES = ["tool_shed", "tool_shed_wsgi"] +CONFIG_FILE_ARG = "--toolshed-config" +CONFIG_DIR_NAME = "config" +TOOLSHED_CONFIG_PREFIX = "TOOL_SHED_CONFIG_" + + +def get_dburl(argv: List[str], cwd: str) -> str: + """Return db url.""" + config_file = pop_arg_from_args(argv, CONFIG_FILE_ARG) + return get_dburl_from_file(cwd, config_file) + + +def get_dburl_from_file(cwd: str, config_file: Optional[str] = None) -> str: + if config_file is None: + cwds = [cwd, os.path.join(cwd, CONFIG_DIR_NAME)] + config_file = find_config_file(DEFAULT_CONFIG_NAMES, dirs=cwds) + + properties = load_app_properties(config_file=config_file, config_prefix=TOOLSHED_CONFIG_PREFIX) + default_url = f"sqlite:///{os.path.join(get_data_dir(properties), 'community.sqlite')}?isolation_level=IMMEDIATE" + url = properties.get("database_connection", default_url) + return url