diff --git a/create_db.sh b/create_db.sh deleted file mode 100755 index 690d0c6f8c8..00000000000 --- a/create_db.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh - -####### -# Use this script to verify the state of the Galaxy and Tool Shed Install -# database(s). If the database does not exist or is empty, it will be created -# and initialized. -# (Use create_toolshed_db.sh to create and initialize a new -# Tool Shed database.) -# -# To pass a galaxy config file, use `--galaxy-config` -# -# You may also override the galaxy database url and/or the -# tool shed install database url, as well as the database_template -# and database_encoding configuration options with env vars: -# GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-db-url ./create_db.sh -# GALAXY_INSTALL_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-other-db-url ./create_db.sh -####### - -cd "$(dirname "$0")" - -. ./scripts/common_startup_functions.sh - -setup_python - -python ./scripts/create_db.py "$@" diff --git a/doc/source/admin/db_migration.md b/doc/source/admin/db_migration.md index c3c620fbfff..605bbc77501 100644 --- a/doc/source/admin/db_migration.md +++ b/doc/source/admin/db_migration.md @@ -454,11 +454,11 @@ Downgrade to 1 revision below specific revision: To create a revision for the galaxy model: -`./run_alembic.sh revision --head=gxy@head -message "your description"` +`./run_alembic.sh revision --head=gxy@head --message "your description"` To create a revision for the install model: -`./run_alembic.sh revision --head=tsi@head -message "your description"` +`./run_alembic.sh revision --head=tsi@head --message "your description"` Check [Alembic's documentation](https://alembic.sqlalchemy.org) for more examples. diff --git a/lib/galaxy/model/migrations/README.md b/lib/galaxy/model/migrations/README.md index f95029e06bc..6ef1ebf14d7 100644 --- a/lib/galaxy/model/migrations/README.md +++ b/lib/galaxy/model/migrations/README.md @@ -1,5 +1,513 @@ # Galaxy Database Schema Migrations -Starting with release 22.05, to manage its database migrations, Galaxy uses [Alembic](https://alembic.sqlalchemy.org). +Galaxy's database schema migration system is built on top of [Alembic](https://alembic.sqlalchemy.org) - a lightweight database migration tool for usage with SQLAlchemy. +(This documentation applies to release 22.05 and up. Prior to 22.05, Galaxy used SQLAlchemy Migrate.) -For admin and development options, please see [Galaxy's documentation](https://docs.galaxyproject.org/en/master/admin/db_migration.html). +## Administering Galaxy: upgrading and downgrading the database + +To initialize an empty database (or create a new SQLite database) start Galaxy. However, this approach is safe only if booting to a single process. A better approach is to use the [*init*](#init) command: `manage_db.sh init`. + +To upgrade or downgrade an existing database, you'll need to use the `manage_db.sh` script, which is the recommended way to interact with Galaxy's database schema migration system. + +## manage_db.sh + +``` +usage: manage_db.sh [-h] [-c CONFIG] {upgrade,downgrade,version,v,dbversion,dv,init} ... + +positional arguments: + {upgrade,downgrade,version,v,dbversion,dv,init} + upgrade Upgrade to a later version + downgrade Revert to a previous version + version (v) Show the head revision in the migrations script directory + dbversion (dv) Show the current revision for Galaxy's database + init Initialize empty database(s) + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --galaxy-config CONFIG + Alternate Galaxy configuration file +``` + +### Subcommands + +#### upgrade + +Upgrade to a later version. The revision argument is optional. + +If you are upgrading a database that +has not been version-controlled by Alembic, you should run this command for the fist time without +the revision argument: `./manage_db.sh upgrade` - this will ensure proper initialization of the +migration system for your database(s). + +``` +usage: manage_db.sh upgrade [-h] [--sql] [revision] + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). + +#### downgrade + +Revert to a previous version. + +``` +usage: manage_db.sh downgrade [-h] [--sql] revision + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). +Note that in this mode, instead of specifying a revision identifier, you have to specify a range of revisions using the following format: `:`. + +#### version + +Show the latest (i.e., head) revisions in the codebase. + +``` +Activating virtualenv at .venv +\usage: manage_db.sh version [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output + +``` + + +#### dbversion + +Show the current revision for Galaxy's database. If the database revision corresponds to the head revision in the codebase, it will be marked as `(head)`. + +``` +usage: manage_db.sh dbversion [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output +``` + +#### init + +Initialize an empty database (or create a new SQLite database). + +``` +usage: manage_db.sh init [-h] + +optional arguments: + -h, --help show this help message and exit +``` + +# Advanced usage + +The following sections provide more details on the migration system and describe commands that are relevant for development scenarios. + +## Overview + +The purpose of the database schema migration system is to support automated, incremental, reversible changes to Galaxy's database schema. Central to this is the concept of a database version (more specifically, database *schema* version). A version is represented by a revision script (the terms *version* and *revision* may be used interchangeably). Executing a revision script will upgrade, or downgrade, the database schema by applying the changes specified in the script. + +Galaxy keeps track of the database schema version in two places: one is a directory we refer to as "the migration environment" (located at `lib/galaxy/model/migrations/alembic`), the other is the `alembic_version` table in the database. For Galaxy to run, these versions must be the same, which, essentially, means that the version of the database matches the version expected by the codebase. + +On startup, the system checks if the version in the database matches the version in the codebase. If they do not match, and the `database_auto_migrate` configuration option is not set, Galaxy will fail with an error message explaining how to proceed. In most cases, you'll need to upgrade the database schema to the current version, which is represented by the latest revision script in the migration environment. + +### A note on models and branch labels + +Galaxy's data model includes the [galaxy model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/__init__.py) and the [install model](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/model/tool_shed_install/__init__.py). +These two models may be persisted in one combined database (which is the default) or two separate databases (which is enabled by setting the +[`install_database_connection`](https://github.com/galaxyproject/galaxy/blob/dev/lib/galaxy/webapps/galaxy/config_schema.yml#L157) configuration option). + +These models are represented by migration [branches](https://alembic.sqlalchemy.org/en/latest/branches.html#working-with-branches) (versioning lineages with a common base) labeled as *gxy* for the galaxy model and *tsi* for the install model. If both models are hosted in the same databases, the branches will share the same Alembic version table; otherwise, each database has its own version table. + +Each branch has its own version history, represented by revision scripts located in the branch version directory (`migrations/alembic/versions_gxy` for *gxy* and `migrations/alembic/versions_tsi` for *tsi*). + +For a more detailed description of the system's internals, see pull request [#13108](https://github.com/galaxyproject/galaxy/pull/13108). + +## Migration management scripts + +The **`manage_db.sh`** script is the recommended way to interact with Galaxy's database schema migration system. +It provides access to a subset of commands offered by Alembic's CLI, while hiding some of the +implementation complexity of Galaxy's model. The provided commands and options should be +sufficient for most use cases involving Galaxy development or system and database administration. + +Additionally, Galaxy provides two scripts for advanced usage: `db_dev.sh` and `run_alembic.sh`. Both are located in the `scripts` directory. + +The **`db_dev.sh`** script is similar to `manage_db.sh`, but provides additional commands and options. + +The **`run_alembic.sh`** script is a thin wrapper around the Alembic CLI runner. It offers more +flexibility and the full scope of Alembic's CLI commands and options; however, it requires more detailed command arguments, as well as basic familiarity with [Alembic branches](https://alembic.sqlalchemy.org/en/latest/branches.html). + +### Revision identifiers + +Some of the commands accept revision identifiers as arguments. A revision is usually identified by a +12-digit hexadecimal number (i.e., `6a67bf27e6a6`). Anytime you need to refer to a specific +revision, you have the option to use a partial number. + +For example, you may use `./db_dev.sh upgrade 6a` instead of `./db_dev.sh upgrade 6a67bf27e6a6` if `6a` is sufficient to uniquely identify that revision. + +(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#partial-revision-identifiers)) + +### Relative migration identifiers + +You may also use Alembic's syntax for relative migration identifiers for the upgrade/downgrade commands: + +To move 2 versions from the current version, a decimal value `+N` can be supplied: + +`./db_dev.sh upgrade +2` + +Negative values are accepted for downgrades: + +`./db_dev.sh downgrade -2` + +Relative identifiers may also be in terms of a specific revision. For example, to upgrade to +revision 6a67bf27e6a6 plus two additional steps: + +`./db_dev.sh upgrade 6a67bf27e6a6+2`. + +You may also combine relative migration identifiers with partial revision identifiers: + +`./db_dev.sh upgrade 6a+2`. + +(Ref: [Alembic documentation](https://alembic.sqlalchemy.org/en/latest/tutorial.html#relative-migration-identifiers) + +*Revision identifiers and relative migration identifiers can be used with all the provided scripts.* + +## db_dev.sh + +This script offers a set of common database schema migration operations that are executed on the *gxy* branch, which, in the vast majority of cases, is all you need to develop and administer Galaxy. +To run operations on the *tsi* branch, you need to use `run_alembic.sh`. + +``` +usage: db_dev.sh [-h] [-c CONFIG] [--raiseerr] {upgrade,downgrade,version,v, + dbversion,dv,history,h,show,s,revision,init} ... + +positional arguments: + {upgrade,downgrade,version,v,dbversion,dv,history,h,show,s,revision,init} + upgrade Upgrade to a later version + downgrade Revert to a previous version + version (v) Show the head revision in the migrations script directory + dbversion (dv) Show the current revision for Galaxy's database + history (h) List revision scripts in chronological order + show (s) Show the revision(s) denoted by the given symbol + revision Create a new revision file + init Initialize empty database(s) + +optional arguments: + -h, --help show this help message and exit + -c CONFIG, --galaxy-config CONFIG + Alternate Galaxy configuration file + --raiseerr Raise a full stack trace on error +``` + +### Subcommands + +#### upgrade + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Upgrade to a later version. The revision argument is optional. + +Omitting the revision argument is equivalent to specifying `heads` as the revision identifier; in +that case, the database(s) will be upgraded to the latest revisions in both branches, *gxy* and +*tsi*. + +If you are upgrading a database that has not been version-controlled by Alembic, you should run this +command for the fist time without the revision argument: `./db_dev.sh upgrade` - this will ensure +proper initialization of the migration system for your database(s). + + +``` +usage: db_dev.sh upgrade [-h] [--sql] [revision] + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). + +#### downgrade + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Revert to a previous version. + +``` +usage: db_dev.sh downgrade [-h] [--sql] revision + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit + --sql Don't emit SQL to database - dump to standard output/file instead. +``` + +Specifying `base` as the revision identifier will downgrade both branches, *gxy* and *tsi*, to their +initial state prior to any revisions; the `alembic_version` table will be empty. + +For the `--sql` option, see [Alembic documentation on offline mode](https://alembic.sqlalchemy.org/en/latest/offline.html). +Note that in this mode, instead of specifying a revision identifier, you have to specify a range of revisions using the following format: `:`*. + +*You cannot use this script to downgrade past the initial Alembic revisions that created the *gxy* and *tsi* branches. + + +#### version + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Show the latest (i.e., head) revisions in the codebase. + +``` +Activating virtualenv at .venv +\usage: db_dev.sh version [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output + +``` + +The output of this command will include the head revisions for both branches: + +``` +$ .db_dev.sh version +186d4835587b (gxy) (head) +d4a650f47a3c (tsi) (head) +``` + +#### dbversion + +This command is identical to the *upgrade* command in the `manage_db.sh` script. + +Show the current revision for Galaxy's database. + +``` +usage: db_dev.sh dbversion [-h] [-v] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output +``` + +If the database revision corresponds to the head revision +in the codebase, it will be marked as `(head)`. The output will be slightly more verbose and will vary +depending on the database. + +``` +$ ./db_dev.sh dbversion +INFO:alembic.runtime.migration:Context impl PostgresqlImpl. +INFO:alembic.runtime.migration:Will assume transactional DDL. +d4a650f47a3c (head) +6a67bf27e6a6 +``` + +#### history + +List revision scripts in chronological order. + +``` +usage: db_dev.sh history [-h] [-v] [-i] + +optional arguments: + -h, --help show this help message and exit + -v, --verbose Display more detailed output + -i, --indicate-current + Indicate current revision +``` + +Depending on your setup, the list may include revision histories for both branches. The oldest +revisions are the ones that created the *gxy* and *tsi* branches (introduced in 22.05). The +`--indicate-current` option is particularly useful when you need to determine how far behind (or +ahead) your database version is compared to the version expected by your codebase: + +``` +$ ./db_dev.sh history --indicate-current +6a67bf27e6a6 -> 186d4835587b (gxy) (head), drop job_state_history.update_time column +b182f655505f -> 6a67bf27e6a6 (gxy) (current), deferred data tables +e7b6dcb09efd -> b182f655505f (gxy), add workflow.source_metadata column + -> e7b6dcb09efd (gxy), create gxy branch + -> d4a650f47a3c (tsi) (head), create tsi branch +``` + +#### show + +Show the revision(s) denoted by the given revision identifier. + +``` +usage: db_dev.sh show [-h] revision + +positional arguments: + revision Revision identifier + +optional arguments: + -h, --help show this help message and exit +``` + +#### revision + +Create a new revision file. + +``` +usage: db_dev.sh revision [-h] -m MESSAGE [--rev-id REV_ID] + +optional arguments: + -h, --help show this help message and exit + -m MESSAGE, --message MESSAGE + Message string to use with 'revision' + --rev-id REV_ID Specify a revision id instead of generating one + (This option is for testing purposes only) +``` + +The `--message` argument is required: this ensures a readable revision history. The message is +appended to the new revision identifier to form the filename for the new revision script, so it +should be a succinct description of the change: + +``` +$ ./db_dev.sh revision --message "add column foo to table bar" +[output omitted] + +$ ls lib/galaxy/model/migrations/alembic/versions_gxy/ +a2e418ad6a15_add_column_foo_to_table_bar.py +``` + +#### init + +Initialize an empty database (or create a new SQLite database) for both branches, *gxy* and *tsi*. + +``` +usage: db_dev.sh init [-h] + +optional arguments: + -h, --help show this help message and exit +``` + +## run_alembic.sh + +If you need to run operations on the *tsi* branch, or you need access to the full scope of command +line options provided by Alembic, you should use the `run_alembic.sh` script, which is a thin +wrapper around Alembic's CLI runner. The script modifies the path, initializes the Python virtual +environment, retrieves any necessary configuration values, and invokes the Alembic CLI runner with +appropriate arguments. + +Keep in mind that since Galaxy uses branch labels to distinguish between the galaxy and the install +models, in most cases, you'll need to identify the target branch to which your command should be +applied. + +### Examples of usage + +Remember to first backup your database(s). + +#### Upgrading + +Upgrade to the head revision (both, *gxy* and *tsi* branches): + +`./run_alembic.sh upgrade heads` + +Upgrade to the head revision (*gxy* branch): + +`./run_alembic.sh upgrade gxy@head` + +Upgrade to 1 revision above the current (*tsi* branch): + +`./run_alembic.sh upgrade tsi@+1` + +Upgrade to a specific revision: + +`./run_alembic.sh upgrade [revision identifier]` + +Upgrade to 1 revision above a specific revision: + +`./run_alembic.sh upgrade [revision identifier]+1` + +#### Downgrading + +Downgrade to base revision (*gxy* branch): + +`./run_alembic.sh downgrade gxy@base` + +Downgrade to 1 revision below current (*tsi* branch): + +`./run_alembic.sh downgrade tsi@-1` + +Downgrade to a specific revision: + +`./run_alembic.sh downgrade [revision identifier]` + +Downgrade to 1 revision below specific revision: + +`./run_alembic.sh downgrade [revision identifier]-1 ` + +#### Creating new revisions + +To create a revision for the galaxy model: + +`./run_alembic.sh revision --head=gxy@head --message "your description"` + +To create a revision for the install model: + +`./run_alembic.sh revision --head=tsi@head --message "your description"` + +Check [Alembic's documentation](https://alembic.sqlalchemy.org) for more examples. + +*Note: the `run_alembic.sh` script does not support relative upgrades and downgrades without a +revision identifier: you cannot `upgrade +1` or `downgrade -1` without providing a revision identifier.* + +## Upgrading from SQLAlchemy Migrate + +Galaxy no longer supports SQLAlchemy Migrate. To upgrade to Alembic, follow these steps: + +1. Backup your database(s). + +2. Make sure your codebase is in pre-22.05 state: you will need to use the old `manage_db.sh` script which invokes the SQLAlchemy Migrate tool, which is not available in 22.05 and up. Checking out the 22.01 release branch is the simplest step. + +3. Verify that your database is at the latest SQLAlchemy Migrate version. If you have a combined database, it should be version 180 (check the `migrate_version` table). If you have separate galaxy model and install model databases, your galaxy version should be 180, and your install model version should be 17. + + If your database is not current, run `manage_db.sh upgrade` to upgrade your database. + + Once your database has the latest SQLAlchemy Migrate version, switch back to your current branch (22.05 or more recent). + +5. Run `manage_db.sh upgrade`. + +## Developing Galaxy: creating new revisions + +Make sure you have updated the model and have added appropriate tests before creating a new +revision. + +You create a new revision file by running the `revision` subcommand of the `manage_db.sh` or the +`run_alembic.sh` script (see sections above for usage information). Alembic generates a revision +script in the appropriate version directory (`migrations/alembic/versions_gxy` for *gxy* and +`migrations/alembic/versions_tsi` for *tsi*). You'll need to fill out the `upgrade` and `downgrade` +functions. Use Alembic documentation for examples: + +- [https://alembic.sqlalchemy.org/en/latest/tutorial.html#create-a-migration-script](https://alembic.sqlalchemy.org/en/latest/tutorial.html#create-a-migration-script) +- [https://alembic.sqlalchemy.org/en/latest/ops.html](https://alembic.sqlalchemy.org/en/latest/ops.html) + +We encourage you to use Galaxy-specific utility functions (`galaxy/model/migrations/util.py`) when appropriate. Don't forget to provide tests for any modifications you make to the model (most likely, you will need to add them to the appropriate `test/unit/data/model/mapping/test_*model_mapping.py` module. + +After that, run the upgrade script: `./manage_db.sh upgrade`. And you're done! + +## Troubleshooting + +### How to handle migrations.IncorrectVersionError + +If you see this error, you'll need to upgrade or downgrade your database *before* upgrading to +Alembic. Whether you need to upgrade or downgrade depends on what version number is stored in +the `migrate_version` table in your database. Alembic expects version 180, so you will need to +upgrade if your version is less than that or, in very rare circumstances, downgrade if your version +is 181. Please see [this issue](https://github.com/galaxyproject/galaxy/issues/13528) for more details. + +#### Please help us improve this page: +If you encounter any migration-related errors or issues, please [open an issue](https://github.com/galaxyproject/galaxy/issues/new?assignees=&labels=&template=bug_report.md&title=), and we will add the solution with any relevant context to this page. diff --git a/lib/galaxy/model/migrations/dbscript.py b/lib/galaxy/model/migrations/dbscript.py new file mode 100644 index 00000000000..c1686585605 --- /dev/null +++ b/lib/galaxy/model/migrations/dbscript.py @@ -0,0 +1,253 @@ +import logging +import os +import sys +from argparse import ( + ArgumentParser, + Namespace, +) +from typing import Optional + +import alembic +from alembic import command +from alembic.config import Config + +from galaxy.model.migrations import verify_databases_via_script +from galaxy.model.migrations.scripts import ( + get_configuration, + get_configuration_from_file, +) + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + +DEFAULT_CONFIG_NAMES = ["galaxy", "universe_wsgi"] +CONFIG_FILE_ARG = "--galaxy-config" +CONFIG_DIR_NAME = "config" +GXY_CONFIG_PREFIX = "GALAXY_CONFIG_" +TSI_CONFIG_PREFIX = "GALAXY_INSTALL_CONFIG_" + + +class DbScript: + """ + Facade for common database schema migration operations on the gxy branch. + When the gxy and tsi branches are persisted in the same database, some + alembic commands will display output on the state on both branches (e.g. + history, version, dbversion). The upgrade command is executed on both + branches: gxy and tsi (the upgrade command ensures the branch has been + initialized by stamping its version in the alembic_version table). + """ + + def __init__(self, config_file: Optional[str] = None) -> None: + self.alembic_config = self._get_alembic_cfg() + self._set_dburl(config_file) + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) + + def upgrade(self, args: Namespace) -> None: + def upgrade_to_revision(rev): + command.upgrade(self.alembic_config, rev, args.sql) + + if args.revision: + revision = self._parse_revision(args.revision) + upgrade_to_revision(revision) + else: + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) + upgrade_to_revision("gxy@head") + try: + self.alembic_config.set_main_option("sqlalchemy.url", self.tsi_url) + upgrade_to_revision("tsi@head") + finally: + self.alembic_config.set_main_option("sqlalchemy.url", self.gxy_url) + + def downgrade(self, args: Namespace) -> None: + revision = self._parse_revision(args.revision) + command.downgrade(self.alembic_config, revision, args.sql) + + def revision(self, args: Namespace) -> None: + """Create revision script for the gxy branch only.""" + command.revision(self.alembic_config, message=args.message, rev_id=args.rev_id, head="gxy@head") + + def version(self, args: Namespace) -> None: + command.heads(self.alembic_config, verbose=args.verbose) + + def dbversion(self, args: Namespace) -> None: + command.current(self.alembic_config, verbose=args.verbose) + + def history(self, args: Namespace) -> None: + command.history(self.alembic_config, verbose=args.verbose, indicate_current=args.indicate_current) + + def show(self, args: Namespace) -> None: + command.show(self.alembic_config, args.revision) + + def _get_alembic_cfg(self) -> Config: + config_file = os.getenv("ALEMBIC_CONFIG") + if not config_file: + config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") + config_file = os.path.abspath(config_file) + return Config(config_file) + + def _set_dburl(self, config_file: Optional[str] = None) -> None: + gxy_config, tsi_config, _ = get_configuration_from_file(os.getcwd(), config_file) + self.gxy_url = gxy_config.url + self.tsi_url = tsi_config.url + + def _parse_revision(self, rev: str) -> str: + # Relative revision identifier requires a branch label + if rev.startswith("+") or rev.startswith("-"): + return f"gxy@{rev}" + return rev + + +class ParserBuilder: + """ + Assembler object that simplifies the construction of an argument parser for db/db_dev migration scripts. + """ + + def __init__(self, parser, subcommand_required=True): + self.parser = parser + self.subparsers = parser.add_subparsers(required=subcommand_required) + self._init_arg_parsers() + + def add_upgrade_command(self): + parser = self._add_parser( + "upgrade", + Command.upgrade, + "Upgrade to a later version", + parents=[self._sql_arg_parser], + ) + parser.add_argument("revision", help="Revision identifier", nargs="?") + + def add_downgrade_command(self): + parser = self._add_parser( + "downgrade", + Command.downgrade, + "Revert to a previous version", + parents=[self._sql_arg_parser], + ) + parser.add_argument("revision", help="Revision identifier") + + def add_version_command(self): + self._add_parser( + "version", + Command.version, + "Show the head revision in the migrations script directory", + aliases=["v"], + parents=[self._verbose_arg_parser], + ) + + def add_dbversion_command(self): + self._add_parser( + "dbversion", + Command.dbversion, + "Show the current revision for Galaxy's database", + aliases=["dv"], + parents=[self._verbose_arg_parser], + ) + + def add_init_command(self): + self._add_parser( + "init", + Command.init, + "Initialize empty database(s)", + ) + + def add_revision_command(self): + parser = self._add_parser("revision", help="Create a new revision file", func=Command.revision) + parser.add_argument("-m", "--message", help="Message string to use with 'revision'", required=True) + parser.add_argument( + "--rev-id", + help="Specify a revision id instead of generating one (This option is for testing purposes only)", + ) + + def add_history_command(self): + parser = self._add_parser( + "history", + Command.history, + "List revision scripts in chronological order", + aliases=["h"], + parents=[self._verbose_arg_parser], + ) + parser.add_argument("-i", "--indicate-current", help="Indicate current revision", action="store_true") + + def add_show_command(self): + parser = self._add_parser( + "show", + Command.show, + "Show the revision(s) denoted by the given symbol", + aliases=["s"], + ) + parser.add_argument("revision", help="Revision identifier") + + def _init_arg_parsers(self): + self._verbose_arg_parser = self._make_verbose_arg_parser() + self._sql_arg_parser = self._make_sql_arg_parser() + + def _make_verbose_arg_parser(self): + parser = ArgumentParser(add_help=False) + parser.add_argument("-v", "--verbose", action="store_true", help="Display more detailed output") + return parser + + def _make_sql_arg_parser(self): + parser = ArgumentParser(add_help=False) + parser.add_argument( + "--sql", + action="store_true", + help="Don't emit SQL to database - dump to standard output/file instead. See Alembic docs on offline mode.", + ) + return parser + + def _add_parser(self, command, func, help, aliases=None, parents=None): + aliases = aliases or [] + parents = parents or [] + parser = self.subparsers.add_parser(command, aliases=aliases, help=help, parents=parents) + parser.set_defaults(func=func) + return parser + + +class Command: + """Execute commands.""" + + @staticmethod + def upgrade(args: Namespace) -> None: + Command._exec_command("upgrade", args) + + @staticmethod + def downgrade(args: Namespace) -> None: + Command._exec_command("downgrade", args) + + @staticmethod + def version(args: Namespace) -> None: + Command._exec_command("version", args) + + @staticmethod + def dbversion(args: Namespace) -> None: + Command._exec_command("dbversion", args) + + @staticmethod + def init(args: Namespace) -> None: + gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) + verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) + + @staticmethod + def revision(args: Namespace) -> None: + Command._exec_command("revision", args) + + @staticmethod + def history(args: Namespace) -> None: + Command._exec_command("history", args) + + @staticmethod + def show(args: Namespace) -> None: + Command._exec_command("show", args) + + @staticmethod + def _exec_command(command: str, args: Namespace) -> None: + dbscript = DbScript(args.config) + try: + getattr(dbscript, command)(args) + except alembic.util.exc.CommandError as e: + if args.raiseerr: + raise + else: + log.error(e) + print(f"FAILED: {str(e)}") + sys.exit(1) diff --git a/lib/galaxy/model/migrations/scripts.py b/lib/galaxy/model/migrations/scripts.py index 54f0ad774e1..aef7ff0f297 100644 --- a/lib/galaxy/model/migrations/scripts.py +++ b/lib/galaxy/model/migrations/scripts.py @@ -1,5 +1,4 @@ import os -import re import sys from typing import ( List, @@ -14,10 +13,7 @@ from alembic.script import ScriptDirectory from sqlalchemy import create_engine from sqlalchemy.engine import Engine -from galaxy.model.database_utils import ( - database_exists, - is_one_database, -) +from galaxy.model.database_utils import database_exists from galaxy.model.migrations import ( AlembicManager, DatabaseConfig, @@ -91,6 +87,13 @@ def get_configuration(argv: List[str], cwd: str) -> Tuple[DatabaseConfig, Databa Return a 3-item-tuple with configuration values used for managing databases. """ config_file = _pop_config_file(argv) + return get_configuration_from_file(cwd, config_file) + + +def get_configuration_from_file( + cwd: str, config_file: Optional[str] = None +) -> Tuple[DatabaseConfig, DatabaseConfig, bool]: + if config_file is None: cwds = [cwd, os.path.join(cwd, CONFIG_DIR_NAME)] config_file = find_config_file(DEFAULT_CONFIG_NAMES, dirs=cwds) @@ -159,120 +162,10 @@ class LegacyScriptsException(Exception): super().__init__(message) -class LegacyScripts: +class LegacyManageDb: LEGACY_CONFIG_FILE_ARG_NAMES = ["-c", "--config", "--config-file"] - ALEMBIC_CONFIG_FILE_ARG = "--alembic-config" # alembic config file, set in the calling script - DEFAULT_DB_ARG = "default" - def __init__(self, argv: List[str], cwd: Optional[str] = None) -> None: - self.argv = argv - self.cwd = cwd or os.getcwd() - self._database: Optional[str] = None # Do not assign default value: `None` means we don't know yet. - - @property - def database(self): - if self._database is None: - raise LegacyScriptsException( - "Attempt to access identifier of database before processing the script arguments" - ) - return self._database - - def run(self) -> None: - """ - Convert legacy arguments to current spec required by Alembic, - then add db url arguments required by Alembic - """ - self.convert_args() - add_db_urls_to_command_arguments(self.argv, self.gxy_url, self.tsi_url) - - def convert_args(self) -> None: - """ - Convert legacy arguments to current spec required by Alembic. - - Note: The following method calls must be done in this sequence. - """ - self.pop_database_argument() - self.rename_config_argument() - self.rename_alembic_config_argument() - self.load_db_urls() - self.convert_version_argument() - - def pop_database_argument(self) -> None: - """ - If last argument is a valid database name, pop and assign it; otherwise assign default. - """ - arg = self.argv[-1] - self._database = self.DEFAULT_DB_ARG - if arg in ["galaxy", "install"]: - self._database = self.argv.pop() - - def rename_config_argument(self) -> None: - """ - Rename the optional config argument: we can't use '-c' because that option is used by Alembic. - """ - for arg in self.LEGACY_CONFIG_FILE_ARG_NAMES: - if arg in self.argv: - self._rename_arg(arg, CONFIG_FILE_ARG) - return - - def rename_alembic_config_argument(self) -> None: - """ - Rename argument name: `--alembic-config` to `-c`. There should be no `-c` argument present. - """ - if "-c" in self.argv: - raise LegacyScriptsException("Cannot rename alembic config argument: `-c` argument present.") - self._rename_arg(self.ALEMBIC_CONFIG_FILE_ARG, "-c") - - def convert_version_argument(self) -> None: - """ - Convert legacy version argument to current spec required by Alembic. - """ - if "--version" in self.argv: - # Just remove it: the following argument should be the version/revision identifier. - pos = self.argv.index("--version") - self.argv.pop(pos) - else: - # If we find --version=foo, extract foo and replace arg with foo (which is the revision identifier) - p = re.compile(r"--version=([0-9A-Fa-f]+)") - for i, arg in enumerate(self.argv): - m = p.match(arg) - if m: - self.argv[i] = m.group(1) - return - # No version argument found: construct argument for an upgrade operation. - # Raise exception otherwise. - if "upgrade" not in self.argv: - raise LegacyScriptsException("If no `--version` argument supplied, `upgrade` argument is requried") - - if self._is_one_database(): # upgrade both regardless of database argument - self.argv.append("heads") - else: # for separate databases, choose one - if self.database in ["galaxy", self.DEFAULT_DB_ARG]: - self.argv.append("gxy@head") - elif self.database == "install": - self.argv.append("tsi@head") - - def get_db_url(self): - if self.database in ["galaxy", self.DEFAULT_DB_ARG]: - return self.gxy_url - elif self.database == "install": - return self.tsi_url - - def _rename_arg(self, old_name, new_name) -> None: - pos = self.argv.index(old_name) - self.argv[pos] = new_name - - def load_db_urls(self) -> None: - gxy_config, tsi_config, _ = get_configuration(self.argv, self.cwd) - self.gxy_url = gxy_config.url - self.tsi_url = tsi_config.url - - def _is_one_database(self): - return is_one_database(self.gxy_url, self.tsi_url) - - -class LegacyManageDb: def __init__(self): self._set_db_urls() @@ -319,6 +212,19 @@ class LegacyManageDb: self._upgrade(gxy_db_url, GXY) self._upgrade(tsi_db_url, TSI) + def rename_config_argument(self, argv: List[str]) -> None: + """ + Rename the optional config argument: we can't use '-c' because that option is used by Alembic. + """ + for arg in self.LEGACY_CONFIG_FILE_ARG_NAMES: + if arg in argv: + self._rename_arg(argv, arg, CONFIG_FILE_ARG) + return + + def _rename_arg(self, argv, old_name, new_name) -> None: + pos = argv.index(old_name) + argv[pos] = new_name + def _upgrade(self, db_url, model): try: engine = create_engine(db_url) @@ -328,25 +234,22 @@ class LegacyManageDb: engine.dispose() def _set_db_urls(self): - ls = LegacyScripts(sys.argv, os.getcwd()) - ls.rename_config_argument() - ls.load_db_urls() - self.gxy_db_url = ls.gxy_url - self.tsi_db_url = ls.tsi_url + self.rename_config_argument(sys.argv) + self._load_db_urls() + + def _load_db_urls(self): + gxy_config, tsi_config, _ = get_configuration(sys.argv, os.getcwd()) + self.gxy_db_url = gxy_config.url + self.tsi_db_url = tsi_config.url def _get_gxy_sam_db_version(self, engine): dbcache = DatabaseStateCache(engine) return dbcache.sqlalchemymigrate_version def _get_script_directory(self): - alembic_cfg = self._get_alembic_cfg() + alembic_cfg = get_alembic_cfg() return ScriptDirectory.from_config(alembic_cfg) - def _get_alembic_cfg(self): - config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") - config_file = os.path.abspath(config_file) - return Config(config_file) - def _get_gxy_alembic_db_version(self, engine): # We may get 2 values, one for each branch (gxy and tsi). So we need to # determine which one is the gxy head. @@ -369,5 +272,11 @@ class LegacyManageDb: return gxy_revisions +def get_alembic_cfg(): + config_file = os.path.join(os.path.dirname(__file__), "alembic.ini") + config_file = os.path.abspath(config_file) + return Config(config_file) + + def get_alembic_manager(engine: Engine) -> AlembicManager: return AlembicManager(engine) diff --git a/manage_db.sh b/manage_db.sh index 6200d4b98a8..10f1ad2500f 100755 --- a/manage_db.sh +++ b/manage_db.sh @@ -1,50 +1,15 @@ #!/bin/sh ####### -# Use this script to manage Galaxy, Tool Shed Install, and Tool Shed database migrations. -# -# For advanced usage and access to the full scope of command line options provided by -# Alembic, you may use the run_alembic.sh script. However, for regular database management -# tasks, we encourage you to use the manage_db.sh script. -# -# NOTE: If your database is empty, use create_db.sh instead. -# -# Database options: galaxy (default), install, tool_shed -# To pass a galaxy config file, you may use `-c|--config|--config-file your-config-file` -# -# To upgrade or downgrade to some version X: -# sh manage_db.sh [upgrade|downgrade] --version=X [tool_shed|install|galaxy] -# -# You may also skip the version argument when upgrading, in which case the database -# will be upgraded to the latest version. -# -# Example 1: upgrade "galaxy" database to version "abc123" using default config: -# sh manage_db.sh upgrade --version=abc123 -# -# Example 2: downgrade "install" database to version "xyz789" passing config file "mygalaxy.yml": -# sh manage_db.sh downgrade --version=xyz789 -c mygalaxy.yml install -# -# Example 3: upgrade "galaxy" database to latest version using default config: -# sh manage_db.sh upgrade -# -# (Note: Tool Shed migrations use the legacy migrations system, so we check the -# last argument (the database) to invoke the appropriate script. Therefore, if -# you don't specify the database (galaxy is used by default) and pass a config -# file, your config file should not be named `tool_shed`.) +# Use this script to manage Galaxy database schema migrations. +# For help, run `sh manage_db.sh -h`. +# For detailed help, see documentation at lib/galaxy/model/migrations/README.md. ####### -ALEMBIC_CONFIG='lib/galaxy/model/migrations/alembic.ini' - -cd `dirname $0` +cd "$(dirname "$0")" || exit . ./scripts/common_startup_functions.sh setup_python -for i; do :; done -if [ "$i" = "tool_shed" ]; then - python ./scripts/migrate_toolshed_db.py "$@" tool_shed -else - find lib/galaxy/model/migrations/alembic -name '*.pyc' -delete - python ./scripts/manage_db_adapter.py --alembic-config "$ALEMBIC_CONFIG" "$@" -fi +python ./scripts/db.py "$@" diff --git a/manage_toolshed_db.sh b/manage_toolshed_db.sh new file mode 100755 index 00000000000..414fd47dfeb --- /dev/null +++ b/manage_toolshed_db.sh @@ -0,0 +1,14 @@ +#!/bin/sh + +####### +# Use this script to manage Tool Shed database migrations. +# NOTE: If your database is empty, use create_toolshed_db.sh instead. +####### + +cd "$(dirname "$0")" || exit + +. ./scripts/common_startup_functions.sh + +setup_python + +python ./scripts/migrate_toolshed_db.py "$@" tool_shed diff --git a/run_alembic.sh b/run_alembic.sh deleted file mode 100755 index 0dfeddd85a0..00000000000 --- a/run_alembic.sh +++ /dev/null @@ -1,56 +0,0 @@ -#!/bin/sh - -####### -# Use this script to manage Galaxy and Tool Shed Install migrations. -# (Use the manage_db.sh script to manage Tool Shed migrations.) -# -# This script provides access to Alembic's command line options and is -# intended for advanced use scenarios. For regular database management tasks, -# we encourage you to use the manage_db.sh script. -# -# NOTE: If your database is empty, use create_db.sh instead. -# -# We use branch labels to distinguish between the galaxy and the tool_shed_install models, -# so in most cases you'll need to identify the branch to which your command should be applied. -# Use these identifiers: `gxy` for galaxy, and `tsi` for tool_shed_install. -# -# To create a revision for galaxy: -# ./run_alembic.sh revision --head=gxy@head -m "your description" -# -# To create a revision for tool_shed_install: -# ./run_alembic.sh revision --head=tsi@head -m "your description" -# -# To upgrade: -# ./run_alembic.sh upgrade gxy@head # upgrade gxy to head revision -# ./run_alembic.sh upgrade gxy@+1 # upgrade gxy to 1 revision above current -# ./run_alembic.sh upgrade [revision identifier] # upgrade gxy to a specific revision -# ./run_alembic.sh upgrade [revision identifier]+1 # upgrade gxy to 1 revision above specific revision -# ./run_alembic.sh upgrade heads # upgrade gxy and tsi to head revisions -# -# To downgrade: -# ./run_alembic.sh downgrade gxy@base # downgrade gxy to base (empty db with empty alembic table) -# ./run_alembic.sh downgrade gxy@-1 # downgrade gxy to 1 revision below current -# ./run_alembic.sh downgrade [revision identifier] # downgrade gxy to a specific revision -# ./run_alembic.sh downgrade [revision identifier]-1 # downgrade gxy to 1 revision below specific revision -# -# To pass a galaxy config file, use `--galaxy-config` -# -# You may also override the galaxy database url and/or the -# tool shed install database url, as well as the database_template -# and database_encoding configuration options with env vars: -# GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-db-url ./run_alembic.sh ... -# GALAXY_INSTALL_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-other-db-url ./run_alembic.sh ... -# -# For more options, see Alembic's documentation at https://alembic.sqlalchemy.org -####### - -ALEMBIC_CONFIG='lib/galaxy/model/migrations/alembic.ini' - -cd `dirname $0` - -. ./scripts/common_startup_functions.sh - -setup_python - -find lib/galaxy/model/migrations/alembic -name '*.pyc' -delete -python ./scripts/migrate_db.py --config "$ALEMBIC_CONFIG" "$@" diff --git a/scripts/create_db.py b/scripts/create_db.py deleted file mode 100755 index c9d5b750084..00000000000 --- a/scripts/create_db.py +++ /dev/null @@ -1,30 +0,0 @@ -""" -This script retrieves relevant configuration values and verifies the state of -the Galaxy and Tool Shed Install database(s). -There may be one combined database (galaxy and tool shed install) or two -separate databases. -If the database does not exist or is empty, it will be created and initialized. -(See inline comments in lib/galaxy/model/migrations/__init__.py for details on -how other database states are handled). -It is wrapped by create_db.sh (see that file for usage). -""" -import logging -import os.path -import sys - -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) - -from galaxy.model.migrations import verify_databases_via_script -from galaxy.model.migrations.scripts import get_configuration - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) - - -def invoke_create(): - gxy_config, tsi_config, is_auto_migrate = get_configuration(sys.argv, os.getcwd()) - verify_databases_via_script(gxy_config, tsi_config, is_auto_migrate) - - -if __name__ == "__main__": - invoke_create() diff --git a/scripts/db.py b/scripts/db.py new file mode 100644 index 00000000000..f87b9b9d9d9 --- /dev/null +++ b/scripts/db.py @@ -0,0 +1,34 @@ +""" +This script is intended to be invoked by the manage_db.sh script. +""" + +import os +import sys +from argparse import ArgumentParser + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) + +from galaxy.model.migrations.dbscript import ParserBuilder + + +def main() -> None: + parser = ArgumentParser( + prog="manage_db.sh", + description="Common database schema migration operations", + ) + parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") + + parser_builder = ParserBuilder(parser) + + parser_builder.add_upgrade_command() + parser_builder.add_downgrade_command() + parser_builder.add_version_command() + parser_builder.add_dbversion_command() + parser_builder.add_init_command() + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/db_dev.py b/scripts/db_dev.py new file mode 100644 index 00000000000..e244c3822dd --- /dev/null +++ b/scripts/db_dev.py @@ -0,0 +1,40 @@ +""" +This script is intended to be invoked by the scripts/db_dev.sh script. +""" + +import os +import sys +from argparse import ArgumentParser + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) + +from galaxy.model.migrations.dbscript import ParserBuilder + + +def main() -> None: + parser = ArgumentParser( + prog="db_dev.sh", + description="Extended database schema migration operations", + epilog="Note: these operations are applied to the Galaxy model only (stored in the `gxy` branch)." + " For migrating the `tsi` branch, use the `run_alembic.sh` script.", + ) + parser.add_argument("-c", "--galaxy-config", help="Alternate Galaxy configuration file", dest="config") + parser.add_argument("--raiseerr", help="Raise a full stack trace on error", action="store_true") + + parser_builder = ParserBuilder(parser) + + parser_builder.add_upgrade_command() + parser_builder.add_downgrade_command() + parser_builder.add_version_command() + parser_builder.add_dbversion_command() + parser_builder.add_init_command() + parser_builder.add_revision_command() + parser_builder.add_history_command() + parser_builder.add_show_command() + + args = parser.parse_args() + args.func(args) + + +if __name__ == "__main__": + main() diff --git a/scripts/db_dev.sh b/scripts/db_dev.sh new file mode 100755 index 00000000000..c6c8a42685f --- /dev/null +++ b/scripts/db_dev.sh @@ -0,0 +1,17 @@ +#!/bin/sh + +####### +# Extended set of database schema migration operaions. +# For help, run `sh db_dev.sh -h`. +# For detailed help, see documentation at lib/galaxy/model/migrations/README.md. +####### + +cd "$(dirname "$0")" || exit + +cd .. + +. ./scripts/common_startup_functions.sh + +setup_python + +python scripts/db_dev.py "$@" diff --git a/scripts/db_shell.py b/scripts/db_shell.py index a74bce329aa..66098309a5e 100644 --- a/scripts/db_shell.py +++ b/scripts/db_shell.py @@ -11,7 +11,6 @@ # % ipython -i scripts/db_shell.py -- -c config/galaxy.ini # # You can also use this script as a library, for instance see https://gist.github.com/1979583 -# TODO: This script overlaps a lot with manage_db.py and create_db.py, # these should maybe be refactored to remove duplication. import datetime diff --git a/scripts/manage_db_adapter.py b/scripts/manage_db_adapter.py deleted file mode 100644 index f19974486e8..00000000000 --- a/scripts/manage_db_adapter.py +++ /dev/null @@ -1,41 +0,0 @@ -""" -This script is intended to be invoked by the manage_db.sh script. -It translates the arguments supplied to manage_db.sh into the format used -by migrate_db.py. - -INPUT: | OUTPUT: ----------------------------------------------------------- -upgrade --version=foo | upgrade foo -upgrade --version foo | upgrade foo -upgrade | upgrade heads (if using a combined db for galaxy and install) -upgrade | upgrade gxy@head (if using separate dbs for galaxy and install) -upgrade install | upgrade tsi@head -upgrade --version=bar install | upgrade bar -upgrade -c path-to-galaxy.yml | upgrade --galaxy-config path-to-galaxy.yml gxy@head - -The converted sys.argv will include `-c path-to-alembic.ini`. -The optional `-c` argument name is renamed to `--galaxy-config`. -""" - -import os -import sys - -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) - -from galaxy.model.migrations.scripts import ( - invoke_alembic, - LegacyScripts, - verify_database_is_initialized, -) - - -def run(): - ls = LegacyScripts(sys.argv, os.getcwd()) - ls.run() - db_url = ls.get_db_url() - verify_database_is_initialized(db_url) - invoke_alembic() - - -if __name__ == "__main__": - run() diff --git a/scripts/migrate_db.py b/scripts/migrate_db.py deleted file mode 100755 index 78f976ff158..00000000000 --- a/scripts/migrate_db.py +++ /dev/null @@ -1,29 +0,0 @@ -""" -This script retrieves relevant configuration values and invokes -the Alembic console runner. -It is wrapped by run_alembic.sh (see that file for usage). -""" -import logging -import os.path -import sys - -sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) - -from galaxy.model.migrations.scripts import ( - add_db_urls_to_command_arguments, - get_configuration, - invoke_alembic, -) - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger(__name__) - - -def run(): - gxy_config, tsi_config, _ = get_configuration(sys.argv, os.getcwd()) - add_db_urls_to_command_arguments(sys.argv, gxy_config.url, tsi_config.url) - invoke_alembic() - - -if __name__ == "__main__": - run() diff --git a/scripts/run_alembic.py b/scripts/run_alembic.py new file mode 100755 index 00000000000..b642c84e0c7 --- /dev/null +++ b/scripts/run_alembic.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python + +""" +Retrieves relevant configuration values and invokes the Alembic console runner. + +Must be executed from Galaxy's root directory. + +Use this script if you need to run migration operations on the Tool Shed +Install data model (the *tsi* migration branch), or if you need access to the +full scope of command line options provided by Alembic. For regular migration +tasks, uses the db.sh script. + +We use branch labels to distinguish between the galaxy and the tool_shed_install models, +so in most cases you'll need to identify the branch to which your command should be applied. +Use these identifiers: `gxy` for galaxy, and `tsi` for tool_shed_install. + +To create a revision for galaxy: +./scripts/run_alembic.py revision --head=gxy@head -m "your description" + +To create a revision for tool_shed_install: +./scripts/run_alembic.py revision --head=tsi@head -m "your description" + +To upgrade: +./scripts/run_alembic.py upgrade gxy@head # upgrade gxy to head revision +./scripts/run_alembic.py upgrade gxy@+1 # upgrade gxy to 1 revision above current +./scripts/run_alembic.py upgrade [revision identifier] # upgrade gxy to a specific revision +./scripts/run_alembic.py upgrade [revision identifier]+1 # upgrade gxy to 1 revision above specific revision +./scripts/run_alembic.py upgrade heads # upgrade gxy and tsi to head revisions + +To downgrade: +./scripts/run_alembic.py downgrade gxy@base # downgrade gxy to base (database with empty alembic table) +./scripts/run_alembic.py downgrade gxy@-1 # downgrade gxy to 1 revision below current +./scripts/run_alembic.py downgrade [revision identifier] # downgrade gxy to a specific revision +./scripts/run_alembic.py downgrade [revision identifier]-1 # downgrade gxy to 1 revision below specific revision + +To pass a galaxy config file, use `--galaxy-config` + +You may also override the galaxy database url and/or the +tool shed install database url, as well as the database_template +and database_encoding configuration options with env vars: +GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-db-url ./scripts/run_alembic.py ... +GALAXY_INSTALL_CONFIG_OVERRIDE_DATABASE_CONNECTION=my-other-db-url ./scripts/run_alembic.py ... + +Further information: +Galaxy migration documentation: lib/galaxy/model/migrations/README.md +Alembic documentation: https://alembic.sqlalchemy.org +""" + +import logging +import os.path +import sys + +sys.path.insert(1, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, "lib"))) + +from galaxy.model.migrations.scripts import ( + add_db_urls_to_command_arguments, + get_configuration, + invoke_alembic, +) + +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger(__name__) + +ALEMBIC_CONFIG = "lib/galaxy/model/migrations/alembic.ini" + + +def run(): + sys.argv.insert(1, "--config") + sys.argv.insert(2, ALEMBIC_CONFIG) + gxy_config, tsi_config, _ = get_configuration(sys.argv, os.getcwd()) + add_db_urls_to_command_arguments(sys.argv, gxy_config.url, tsi_config.url) + invoke_alembic() + + +if __name__ == "__main__": + run() diff --git a/test/unit/data/model/migrations/test_dbscript.py b/test/unit/data/model/migrations/test_dbscript.py new file mode 100644 index 00000000000..d3367e84ce7 --- /dev/null +++ b/test/unit/data/model/migrations/test_dbscript.py @@ -0,0 +1,430 @@ +""" +Testing approach: +- Use test database(s), store revision scripts in a different location; leave the rest unchanged. +- Use alembic api for setup and accessing the database. +- Run command as subprocess, verify captured output + database state. +""" +import os +import subprocess +import tempfile +from typing import ( + List, + NewType, + Tuple, +) + +import alembic +import pytest +from alembic.config import Config +from alembic.runtime.migration import MigrationContext +from alembic.script import ScriptDirectory +from sqlalchemy import create_engine + +from ..testing_utils import ( # noqa: F401 (url_factory is a fixture we have to import explicitly) + create_and_drop_database, + disposing_engine, + drop_existing_database, + url_factory, +) + +DbUrl = NewType("DbUrl", str) + + +GXY_BRANCH_LABEL = "gxy" +TSI_BRANCH_LABEL = "tsi" +GXY_BASE_ID = "gxy0" +TSI_BASE_ID = "tsi0" + +ADMIN_CMD = "./manage_db.sh" +DEV_CMD = "./scripts/db_dev.sh" +COMMANDS = [ADMIN_CMD, DEV_CMD] + + +@pytest.fixture(scope="session") +def alembic_env_dir() -> str: + """[galaxy-root]/lib/galaxy/model/migrations/alembic/""" + galaxy_root = os.path.join(os.path.dirname(__file__), "..", "..", "..", "..", "..") + return os.path.join(galaxy_root, "lib", "galaxy", "model", "migrations", "alembic") + + +@pytest.fixture(scope="session") +def alembic_config_text(alembic_env_dir) -> List[str]: + """Contents of production alembic.ini as list of lines""" + current_config_path = os.path.join(alembic_env_dir, "..", "alembic.ini") + with open(current_config_path, "r") as f: + return f.readlines() + + +@pytest.fixture() +def tmp_directory(): + with tempfile.TemporaryDirectory() as tmp_dir: + yield tmp_dir + + +@pytest.fixture(params=["one database", "two databases"]) +def config(url_factory, alembic_env_dir, alembic_config_text, tmp_directory, monkeypatch, request): # noqa: F811 + """ + Construct Config object for staging; setup staging env. + """ + # Create staging location for revision sctipts + gxy_versions_dir = os.path.join(tmp_directory, "versions_gxy") + tsi_versions_dir = os.path.join(tmp_directory, "versions_tsi") + version_locations = f"{gxy_versions_dir};{tsi_versions_dir}" + + # Create test database(s) + gxy_dburl = url_factory() + tsi_dburl = gxy_dburl if request.param == "one database" else url_factory() + + # Copy production alembic.ini to staging location + config_file_path = os.path.join(tmp_directory, "alembic.ini") + update_config_for_staging(alembic_config_text, alembic_env_dir, version_locations, gxy_dburl) + write_config_file(config_file_path, alembic_config_text) + + alembic_cfg = Config(config_file_path) + create_alembic_branches(alembic_cfg, gxy_versions_dir, tsi_versions_dir) + + # Point tests to test database(s) + monkeypatch.setenv("ALEMBIC_CONFIG", config_file_path) + monkeypatch.setenv("GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION", gxy_dburl) + monkeypatch.setenv("GALAXY_INSTALL_CONFIG_OVERRIDE_INSTALL_DATABASE_CONNECTION", tsi_dburl) + + return alembic_cfg + + +def update_config_for_staging(config_text: List[str], script_location: str, version_locations: str, dburl: str) -> None: + """Set script_location, version_locations, sqlalchemy.url values.""" + alembic_section_index, url_set = -1, False + url_line = f"sqlalchemy.url = {dburl}\n" + for i, line in enumerate(config_text): + if line.strip() == "[alembic]": + alembic_section_index = i + elif line.startswith("script_location ="): + config_text[i] = f"script_location = {script_location}\n" + elif line.startswith("version_locations ="): + config_text[i] = f"version_locations = {version_locations}\n" + elif line.startswith("sqlalchemy.url ="): + config_text[i] = url_line + url_set = True + if not url_set: # True when executed for the first time + config_text.insert(alembic_section_index + 1, url_line) + + +def write_config_file(config_file_path: str, config_text: str) -> None: + with open(config_file_path, "w") as f: + f.write("".join(config_text)) + + +def create_alembic_branches(config: Config, gxy_versions_dir: str, tsi_versions_dir: str) -> None: + """ + Create gxy and tsi branches (required for galaxy's alembic setup; included with 22.05 release) + """ + alembic.command.revision( + config, branch_label=GXY_BRANCH_LABEL, head="base", rev_id=GXY_BASE_ID, version_path=gxy_versions_dir + ) + alembic.command.revision( + config, branch_label=TSI_BRANCH_LABEL, head="base", rev_id=TSI_BASE_ID, version_path=tsi_versions_dir + ) + + +def dburl_from_config(config: Config) -> str: + url = config.get_main_option("sqlalchemy.url") + assert url + return url + + +def run_command(cmd: str) -> subprocess.CompletedProcess: + if in_packages(): + cmd = f"../.{cmd}" # if this is run from `packages`, manage_db.sh is in parent directory + + completed_process = subprocess.run(cmd.split(), capture_output=True, text=True) + + return completed_process + + +def in_packages() -> bool: + """Checks if test is run from the packages directory.""" + path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, os.pardir) + path = os.path.normpath(path) + return os.path.split(path)[1] == "packages" + + +def get_db_heads(config: Config) -> Tuple[str, ...]: + dburl = dburl_from_config(config) + engine = create_engine(dburl) + with engine.connect() as conn: + context = MigrationContext.configure(conn) + heads = context.get_current_heads() + engine.dispose() + return heads + + +class TestRevisionCommand: + def test_revision_cmd(self, config): + run_command(f"{DEV_CMD} revision --message foo1") + run_command(f"{DEV_CMD} revision --rev-id 2 --message foo2") + run_command(f"{DEV_CMD} revision --rev-id 3 --message foo3") + + script_dir = ScriptDirectory.from_config(config) + revisions = [rev for rev in script_dir.walk_revisions()] + assert len(revisions) == 5 # verify total revisions: 2 base + 3 new + + rev = script_dir.get_revision("3") + assert rev + assert GXY_BRANCH_LABEL in rev.branch_labels # verify branch label + assert rev.down_revision == "2" # verify parent revision + assert rev.module.__name__ == "3_foo3_py" # verify message + + def test_revision_cmd_missing_message_arg_error(self): + completed = run_command(f"{DEV_CMD} revision --rev-id 1") + assert completed.returncode == 2 + assert "the following arguments are required: -m/--message" in completed.stderr + + +class TestShowCommand: + def test_show_cmd(self, config): + alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) + completed = run_command(f"{DEV_CMD} show 42") + assert "Revision ID: 42" in completed.stdout + + def test_show_cmd_invalid_revision_error(self, config): + alembic.command.revision(config, rev_id="42", head=GXY_BASE_ID) + completed = run_command(f"{DEV_CMD} show idonotexist") + assert completed.returncode == 1 + assert "Traceback" not in completed.stderr + assert "Can't locate revision identified by 'idonotexist'" in completed.stderr + + def test_show_cmd_invalid_revision_error_with_traceback(self): + completed = run_command(f"{DEV_CMD} --raiseerr show idonotexist") + assert completed.returncode == 1 + assert "Traceback" in completed.stderr + assert "Can't locate revision identified by 'idonotexist'" in completed.stderr + + def test_show_cmd_missing_revision_arg_error(self): + completed = run_command(f"{DEV_CMD} show") + assert completed.returncode == 2 + assert "the following arguments are required: revision" in completed.stderr + + +class TestHistoryCommand: + def test_history_cmd(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + + completed = run_command(f"{DEV_CMD} history") + assert completed.returncode == 0 + assert "2 -> 3 (gxy) (head), empty message" in completed.stdout + assert "1 -> 2 (gxy)" in completed.stdout + assert "gxy0 -> 1 (gxy)" in completed.stdout + + def test_history_cmd_verbose(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + + completed = run_command(f"{DEV_CMD} history --verbose") + assert "Revision ID: 2" in completed.stdout + assert "Revises: 1" in completed.stdout + + def test_history_cmd_indicate_current(self, config): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + alembic.command.upgrade(config, "heads") + + completed = run_command(f"{DEV_CMD} history --indicate-current") + assert completed.returncode == 0 + assert "2 -> 3 (gxy) (head) (current), empty message" in completed.stdout + assert "1 -> 2 (gxy)" in completed.stdout + assert "gxy0 -> 1 (gxy)" in completed.stdout + + +@pytest.mark.parametrize("command", COMMANDS) +class TestVersionCommand: + def test_version_cmd(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"{command} version") + assert completed.returncode == 0 + assert "2 (gxy) (head)" in completed.stdout + + def test_version_cmd_verbose(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"{command} version --verbose") + assert completed.returncode == 0 + assert "Revision ID: 2" in completed.stdout + assert "Revises: 1" in completed.stdout + + +@pytest.mark.parametrize("command", COMMANDS) +class TestDbVersionCommand: + def test_dbversion_cmd(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"{command} dbversion") + assert completed.returncode == 0 + assert "(head)" not in completed.stdout # there has been no upgrade + + alembic.command.upgrade(config, "heads") + + completed = run_command(f"{command} dbversion") + assert completed.returncode == 0 + assert "2 (head)" in completed.stdout + + def test_dbversion_cmd_verbose(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + alembic.command.upgrade(config, "heads") + + completed = run_command(f"{command} dbversion --verbose") + assert completed.returncode == 0 + assert "Revision ID: 2" in completed.stdout + assert "Revises: 1" in completed.stdout + + +@pytest.mark.parametrize("command", COMMANDS) +class TestUpgradeCommand: + def test_upgrade_cmd(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + # first upgrade: upgrades gxy to 2, tsi to base + completed = run_command(f"{command} upgrade") + assert completed.returncode == 0 + assert "Running upgrade gxy0 -> 1" in completed.stderr + assert "Running upgrade 1 -> 2" in completed.stderr + assert "Running upgrade -> tsi0" in completed.stderr + + heads = get_db_heads(config) + assert "2" in heads + + alembic.command.revision(config, rev_id="3", head="2") + + # next upgrade: upgrades gxy to 3 + completed = run_command(f"{command} upgrade") + assert completed.returncode == 0 + assert "Running upgrade 2 -> 3" in completed.stderr + assert "tsi0" not in completed.stderr # no effect on tsi + + heads = get_db_heads(config) + assert "3" in heads + + def test_upgrade_cmd_sql_only(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + completed = run_command(f"{command} upgrade --sql") + assert completed.returncode == 0 + assert "UPDATE alembic_version SET version_num='2'" in completed.stdout + assert "UPDATE alembic_version SET version_num='3'" not in completed.stdout + + alembic.command.revision(config, rev_id="3", head="2") + + completed = run_command(f"{command} upgrade --sql") + assert completed.returncode == 0 + assert "UPDATE alembic_version SET version_num='2'" in completed.stdout + assert "UPDATE alembic_version SET version_num='3'" in completed.stdout + + def test_upgrade_cmd_with_revision_arg(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + + # upgrades gxy to 1 + completed = run_command(f"{command} upgrade 1") + assert completed.returncode == 0 + assert "Running upgrade gxy0 -> 1" in completed.stderr + + heads = get_db_heads(config) + assert heads == ("1",) + + def test_upgrade_cmd_with_relative_revision_syntax(self, config, command): + alembic.command.revision(config, rev_id="a", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="b", head="a") + alembic.command.revision(config, rev_id="c", head="b") + alembic.command.revision(config, rev_id="d", head="c") + alembic.command.revision(config, rev_id="e", head="d") + + # upgrades gxy to b: none + 2 (none -> base -> a) + completed = run_command(f"{command} upgrade +3") + assert completed.returncode == 0 + assert "Running upgrade -> gxy0" in completed.stderr + assert "Running upgrade gxy0 -> a" in completed.stderr + assert "Running upgrade a -> b" in completed.stderr + + heads = get_db_heads(config) + assert heads == ("b",) + + # upgrades gxy to d relative to b: b + 2 (b -> c -> d) + completed = run_command(f"{command} upgrade b+2") + assert completed.returncode == 0 + assert "Running upgrade b -> c" in completed.stderr + assert "Running upgrade c -> d" in completed.stderr + + heads = get_db_heads(config) + assert heads == ("d",) + + +@pytest.mark.parametrize("command", COMMANDS) +class TestDowngradeCommand: + def test_downgrade_cmd(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + alembic.command.upgrade(config, "heads") + + completed = run_command(f"{command} downgrade 1") # downgrade gxy to 1 + assert completed.returncode == 0 + assert "Running downgrade 3 -> 2" in completed.stderr + assert "Running downgrade 2 -> 1" in completed.stderr + + heads = get_db_heads(config) + assert len(heads) == 2 + assert "1" in heads + + def test_downgrade_cmd_sql_only(self, config, command): + alembic.command.revision(config, rev_id="1", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="2", head="1") + alembic.command.revision(config, rev_id="3", head="2") + alembic.command.upgrade(config, "heads") + + completed = run_command(f"{command} downgrade --sql 3:1") # downgrade gxy to 1, no effect on tsi + assert completed.returncode == 0 + assert "UPDATE alembic_version SET version_num='2'" in completed.stdout + assert "UPDATE alembic_version SET version_num='1'" in completed.stdout + + def test_downgrade_cmd_missing_revision_arg_error(self, command): + completed = run_command(f"{command} downgrade") + assert completed.returncode == 2 + assert "the following arguments are required: revision" in completed.stderr + + def test_downgrade_cmd_with_relative_revision_syntax(self, config, command): + alembic.command.revision(config, rev_id="a", head=GXY_BASE_ID) + alembic.command.revision(config, rev_id="b", head="a") + alembic.command.revision(config, rev_id="c", head="b") + alembic.command.revision(config, rev_id="d", head="c") + alembic.command.revision(config, rev_id="e", head="d") + alembic.command.upgrade(config, "heads") + + # downgrades gxy to c: e - 2 (e -> d -> c) + completed = run_command(f"{command} downgrade -2") + + assert completed.returncode == 0 + assert "Running downgrade e -> d" in completed.stderr + assert "Running downgrade d -> c" in completed.stderr + + heads = get_db_heads(config) + assert "c" in heads + + # downgrades gxy to a relative to c: c - 2 (c -> b -> a) + completed = run_command(f"{command} downgrade c-2") + assert completed.returncode == 0 + assert "Running downgrade c -> b" in completed.stderr + assert "Running downgrade b -> a" in completed.stderr + + heads = get_db_heads(config) + assert "a" in heads diff --git a/test/unit/data/model/migrations/test_migrations.py b/test/unit/data/model/migrations/test_migrations.py index b8f96fe7f6a..c89bfb62a21 100644 --- a/test/unit/data/model/migrations/test_migrations.py +++ b/test/unit/data/model/migrations/test_migrations.py @@ -9,6 +9,7 @@ from sqlalchemy import ( text, ) +import galaxy.model.migrations.scripts from galaxy.model import migrations from galaxy.model.database_utils import database_exists from galaxy.model.migrations import ( @@ -1120,7 +1121,7 @@ def db_state6_gxy_state3_tsi_no_sam(url_factory, metadata_state6_gxy_state3_tsi_ @pytest.fixture(autouse=True) def legacy_manage_db(monkeypatch): - def get_alembic_cfg(self): + def get_alembic_cfg(): path = os.path.join(os.path.dirname(__file__), os.pardir, os.pardir, os.pardir, os.pardir, os.pardir) path = os.path.normpath(path) # Adjust path when running from packages @@ -1133,7 +1134,7 @@ def legacy_manage_db(monkeypatch): config.set_main_option("version_locations", f"{path1};{path2}") return config - monkeypatch.setattr(LegacyManageDb, "_get_alembic_cfg", get_alembic_cfg) + monkeypatch.setattr(galaxy.model.migrations.scripts, "get_alembic_cfg", get_alembic_cfg) @pytest.fixture(autouse=True) diff --git a/test/unit/data/model/migrations/test_scripts.py b/test/unit/data/model/migrations/test_scripts.py index 0472b7503a7..b34a8cb3fc3 100644 --- a/test/unit/data/model/migrations/test_scripts.py +++ b/test/unit/data/model/migrations/test_scripts.py @@ -5,8 +5,7 @@ import pytest from galaxy.model.migrations.scripts import ( DatabaseDoesNotExistError, DatabaseNotInitializedError, - LegacyScripts, - LegacyScriptsException, + LegacyManageDb, verify_database_is_initialized, ) @@ -16,44 +15,17 @@ def set_db_urls(monkeypatch): # Do not try to access galaxy config; values not needed. def no_config_call(self): self.gxy_url = "a string" - self.tsi_url = "a stirng" + self.tsi_url = "a string" - monkeypatch.setattr(LegacyScripts, "load_db_urls", no_config_call) + monkeypatch.setattr(LegacyManageDb, "_load_db_urls", no_config_call) -@pytest.fixture(autouse=True) # set combined db for all tests -def set_combined(monkeypatch): - monkeypatch.setattr(LegacyScripts, "_is_one_database", lambda self: True) - - -@pytest.fixture -def set_separate(monkeypatch): - monkeypatch.setattr(LegacyScripts, "_is_one_database", lambda self: False) - - -class TestLegacyScripts: - @pytest.mark.parametrize("database_arg", ["galaxy", "install"]) - def test_pop_database_name(self, database_arg): - # arg_value = 'install' - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc", database_arg] - ls = LegacyScripts(argv) - - ls.pop_database_argument() - assert ls.database == database_arg - assert argv == ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - - def test_pop_database_name_use_default(self): - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - ls = LegacyScripts(argv) - ls.pop_database_argument() - assert ls.database == LegacyScripts.DEFAULT_DB_ARG - assert argv == ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - - @pytest.mark.parametrize("arg_name", LegacyScripts.LEGACY_CONFIG_FILE_ARG_NAMES) +class TestLegacyManageDb: + @pytest.mark.parametrize("arg_name", LegacyManageDb.LEGACY_CONFIG_FILE_ARG_NAMES) def test_rename_config_arg(self, arg_name): # `-c|--config|__config-file` should be renamed to `--galaxy-config` argv = ["caller", "--alembic-config", "path-to-alembic", arg_name, "path-to-galaxy", "upgrade", "--version=abc"] - LegacyScripts(argv).rename_config_argument() + LegacyManageDb().rename_config_argument(argv) assert argv == [ "caller", "--alembic-config", @@ -67,7 +39,7 @@ class TestLegacyScripts: def test_rename_config_arg_reordered_args(self): # `-c|--config|__config-file` should be renamed to `--galaxy-config` argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc", "-c", "path-to-galaxy"] - LegacyScripts(argv).rename_config_argument() + LegacyManageDb().rename_config_argument(argv) assert argv == [ "caller", "--alembic-config", @@ -78,97 +50,14 @@ class TestLegacyScripts: "path-to-galaxy", ] - def test_rename_alembic_config_arg(self): - # `--alembic-config` should be renamed to `-c` - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - LegacyScripts(argv).rename_alembic_config_argument() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "--version=abc"] - def test_rename_alembic_config_arg_raises_error_if_c_arg_present(self): - # Ensure alembic config arg is renamed AFTER renaming the galaxy config arg. Raise error otherwise. - argv = ["caller", "--alembic-config", "path-to-alembic", "-c", "path-to-galaxy", "upgrade", "--version=abc"] - with pytest.raises(LegacyScriptsException): - LegacyScripts(argv).rename_alembic_config_argument() +def test_verify_database_is_init_raises_error_if_no_database(): + nonexistant_path = str(random.random())[2:] + db_url = f"sqlite:////{nonexistant_path}" + with pytest.raises(DatabaseDoesNotExistError): + verify_database_is_initialized(db_url) - def test_convert__version_arg_1(self): - # `sh manage_db.sh upgrade --version X` >> `... upgrade X` - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version", "abc"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "abc"] - def test_convert__version_arg_2(self): - # `sh manage_db.sh upgrade --version=X` >> `... upgrade X` - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "--version=abc"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "abc"] - - def test_convert__no_version_no_model_combined_database(self): - # `sh manage_db.sh upgrade` >> `... upgrade heads` - # No version and no model implies "upgrade the default db (which is galaxy) to its latest version". - # If it is combined, we upgrade both models: gxy and tsi. - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"] - - def test_convert__no_version_galaxy_model_combined_database_(self): - # `sh manage_db.sh upgrade galaxy` >> `... upgrade heads` - # same as no model: if combined we upgrade the whole database - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "galaxy"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"] - - def test_convert__no_version_install_model_combined_database_(self): - # `sh manage_db.sh upgrade install` >> `... upgrade heads` - # same as no model: if combined we upgrade the whole database - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "install"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "heads"] - - def test_convert__no_version_no_model_separate_databases(self, set_separate): - # `sh manage_db.sh upgrade` >> `... upgrade gxy@head` - # No version and no model implies "upgrade the default db (which is galaxy) to its latest version". - # Since the tsi model has its own db, we only upgrade the gxy model. - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "gxy@head"] - - def test_convert__no_version_galaxy_model_separate_databases(self, set_separate): - # `sh manage_db.sh upgrade galaxy` >> `... upgrade gxy@head` - # No version + a model implies "upgrade the db for the specified model to its latest version". - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "galaxy"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "gxy@head"] - - def test_convert__no_version_install_model_separate_databases(self, set_separate): - # `sh manage_db.sh upgrade install` >> `... upgrade tsi@head` - # No version + a model implies "upgrade the db for the specified model to its latest version". - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", "install"] - LegacyScripts(argv).convert_args() - assert argv == ["caller", "-c", "path-to-alembic", "upgrade", "tsi@head"] - - def test_downgrade_with_no_version_argument_raises_error(self): - argv = ["caller", "--alembic-config", "path-to-alembic", "downgrade"] - with pytest.raises(LegacyScriptsException): - LegacyScripts(argv).convert_args() - - def test_access_database_id(self): - db = "galaxy" - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade", db] - ls = LegacyScripts(argv) - ls.run() - assert ls.database == db - - def test_access_database_id_before_processing_script_args_raises_error(self): - argv = ["caller", "--alembic-config", "path-to-alembic", "upgrade"] - with pytest.raises(LegacyScriptsException): - LegacyScripts(argv).database - - def test_verify_database_is_init_raises_error_if_no_database(self): - nonexistant_path = str(random.random())[2:] - db_url = f"sqlite:////{nonexistant_path}" - with pytest.raises(DatabaseDoesNotExistError): - verify_database_is_initialized(db_url) - - def test_verify_database_is_init_raises_error_if_database_not_initialized(self, sqlite_memory_url): - with pytest.raises(DatabaseNotInitializedError): - verify_database_is_initialized(sqlite_memory_url) +def test_verify_database_is_init_raises_error_if_database_not_initialized(sqlite_memory_url): + with pytest.raises(DatabaseNotInitializedError): + verify_database_is_initialized(sqlite_memory_url) diff --git a/tox.ini b/tox.ini index 588d6e53832..0ea5082a227 100644 --- a/tox.ini +++ b/tox.ini @@ -58,5 +58,5 @@ commands = bash .ci/check_controller.sh [testenv:check_indexes] commands = bash scripts/common_startup.sh - bash create_db.sh + bash manage_db.sh init bash check_model.sh