From e99adb5b0fc6c0573a95d70830f8617de73e3c67 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Feb 2015 00:38:06 -0500 Subject: [PATCH] Add --dockerize and --db [postgres|mysql|sqlite] options to run_tests.sh. Runs tests in a Dockerfile with pre-migrated databases and all testing dependencies (postgres, mysql, npm, casper, phantomjs, etc...). Hopefully this makes it easier to run more interesting tests - down the road I would like to add more options for things like condor, slurm, run as real user, etc.... --- run_tests.sh | 19 +++++++++ test/docker/README.md | 9 ++++ test/docker/base/Dockerfile | 63 ++++++++++++++++++++++++++++ test/docker/base/ansible_vars.yml | 21 ++++++++++ test/docker/base/provision.yml | 15 +++++++ test/docker/base/run_test_wrapper.sh | 23 ++++++++++ test/docker/base/start_mysql.sh | 39 +++++++++++++++++ 7 files changed, 189 insertions(+) create mode 100644 test/docker/README.md create mode 100644 test/docker/base/Dockerfile create mode 100644 test/docker/base/ansible_vars.yml create mode 100644 test/docker/base/provision.yml create mode 100644 test/docker/base/run_test_wrapper.sh create mode 100644 test/docker/base/start_mysql.sh diff --git a/run_tests.sh b/run_tests.sh index 91bf74ae532..8a72d95d916 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -29,6 +29,9 @@ Extra options: --no_cleanup Do not delete temp files for Python functional tests (-toolshed, -framework, etc...) --report_file Path of HTML report to produce (for Python Galaxy functional tests). --xunit_report_file Path of XUnit report to produce (for Python Galaxy functional tests). + --dockerize Run tests in a pre-configured Docker container (must be first argument if present). + --db For use with --dockerize, run tests using partially migrated 'postgres', 'mysql', + or 'sqlite' databases. EOF } @@ -60,6 +63,22 @@ with_framework_test_tools_arg="" driver="python" +if [ "$1" = "--dockerize" ]; +then + shift + if [ "$1" = "--db" ]; then + db_type=$2 + shift 2 + else + db_type="sqlite" + fi + DOCKER_EXTRA_ARGS=${DOCKER_ARGS:-""} + DOCKER_RUN_EXTRA_ARGS=${DOCKER_ARGS:-""} + DOCKER_IMAGE=${DOCKER_IMAGE:-"galaxyprojectdotorg/testing-base"} + docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS -e "GALAXY_TEST_DATABASE_TYPE=$db_type" --rm -v `pwd`:/galaxy $DOCKER_IMAGE "$@" + exit $? +fi + while : do case "$1" in diff --git a/test/docker/README.md b/test/docker/README.md new file mode 100644 index 00000000000..14a86fc9e4c --- /dev/null +++ b/test/docker/README.md @@ -0,0 +1,9 @@ +Setup Docker environment for use with testing using the following +commands (until this is on Docker hub). + + % cd test/base + % docker build -t galaxyprojectdotorg/testing-base . + +This will create a docker container ready to run most of Galaxy tests +(functional tests for specific bioinformatics tools may still fail for +now until they are migrated out of Galaxy and to the tool shed). diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile new file mode 100644 index 00000000000..eee363938ae --- /dev/null +++ b/test/docker/base/Dockerfile @@ -0,0 +1,63 @@ +FROM toolshed/requirements +MAINTAINER John Chilton + +ENV MYSQL_MAJOR=5.7 \ + MYSQL_VERSION=5.7.5-m15 + +# Pre-install a bunch of packages to speed up ansible steps. +RUN apt-get update -y && apt-get install -y software-properties-common curl && \ + apt-add-repository -y ppa:ansible/ansible && \ + curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ + echo 'deb https://deb.nodesource.com/node trusty main' > /etc/apt/sources.list.d/nodesource.list && \ + apt-key adv --keyserver pool.sks-keyservers.net --recv-keys A4A9406876FCBD3C456770C88C718D3B5072E1F5 && \ + echo "deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-${MYSQL_MAJOR}-dmr" > /etc/apt/sources.list.d/mysql.list && \ + { \ + echo mysql-community-server mysql-community-server/data-dir select ''; \ + echo mysql-community-server mysql-community-server/root-pass password ''; \ + echo mysql-community-server mysql-community-server/re-root-pass password ''; \ + echo mysql-community-server mysql-community-server/remove-test-db select false; \ + } | debconf-set-selections && \ + apt-get update -y && \ + apt-get install -y libpq-dev postgresql postgresql-client postgresql-plpython-9.3 \ + ansible postgresql-server-dev-9.3 wget mysql-server="${MYSQL_VERSION}"* \ + nodejs && \ + npm install -g grunt-contrib-qunit grunt grunt-cli casperjs phantomjs && \ + apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN mkdir -p /tmp/ansible && \ + mkdir -p /opt/galaxy/db && \ + chown -R postgres:postgres /opt/galaxy/db && \ + sed -Ei 's/^(bind-address|log)/#&/' /etc/mysql/my.cnf + +ADD start_mysql.sh /opt/galaxy/start_mysql.sh +ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml +ADD provision.yml /tmp/ansible/provision.yml + +ENV PYTHON_EGG_CACHE=/root/.python-eggs \ + GALAXY_ROOT=/opt/galaxy/galaxy-app + +RUN cd /tmp/ansible && \ + mkdir roles && \ + mkdir roles/galaxyprojectdotorg.galaxy-os && \ + wget -qO- https://github.com/galaxyproject/ansible-galaxy-os/archive/master.tar.gz | tar -xzvf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-os && \ + mkdir roles/galaxyprojectdotorg.cloudman-database && \ + wget -qO- https://github.com/galaxyproject/ansible-cloudman-database/archive/master.tar.gz | tar -xzvf- --strip-components=1 -C roles/galaxyprojectdotorg.cloudman-database && \ + mkdir roles/galaxyprojectdotorg.galaxy && \ + wget -qO- https://github.com/galaxyproject/ansible-galaxy/archive/master.tar.gz | tar -xzvf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy && \ + ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=image -c local -e "@ansible_vars.yml" && \ + ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=database -c local -e "@ansible_vars.yml" && \ + ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=galaxy -c local -e "@ansible_vars.yml" && \ + apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* + +RUN cd $GALAXY_ROOT && \ + echo "Prepopuating postgres database" && \ + su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ + GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://galaxy@localhost:5930/galaxy" sh create_db.sh && \ + echo "Prepopuating sqlite database" && \ + GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" sh create_db.sh && \ + sh /opt/galaxy/start_mysql.sh && \ + GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh + +ADD run_test_wrapper.sh /usr/local/bin/run_test_wrapper.sh + +ENTRYPOINT ["/bin/bash", "/usr/local/bin/run_test_wrapper.sh"] diff --git a/test/docker/base/ansible_vars.yml b/test/docker/base/ansible_vars.yml new file mode 100644 index 00000000000..ec967a280cd --- /dev/null +++ b/test/docker/base/ansible_vars.yml @@ -0,0 +1,21 @@ +--- +add_system_users: no +galaxyFS_base_dir: /opt/galaxy +# TODO: use GALAXY_ROOT via cmd line instead +galaxy_server_dir: "{{ galaxyFS_base_dir }}/galaxy-app" +galaxy_venv_dir: "{{ galaxy_server_dir }}/.venv" +galxy_egg_cache: "/home/{{ galaxy_user_name }}/.python-eggs" +galaxy_repo: https://bitbucket.org/galaxy/galaxy-central +galaxy_changeset_id: stable +galaxy_manage_clone: no +galaxy_manage_download: yes +galaxy_root: "{{ galaxy_server_dir}}" +galaxy_db_dir: "{{ galaxyFS_base_dir }}/db" +galaxy_db_port: 5930 +galaxy_db_log: /tmp/pSQL.log +psql_galaxyftp_password: ignoredthis +galaxy_config_file: "{{ galaxy_server_dir }}/config/galaxy.ini" +configure_docker: no +postgresql_bin_dir: /usr/lib/postgresql/9.3/bin +galaxy_manage_database: no + diff --git a/test/docker/base/provision.yml b/test/docker/base/provision.yml new file mode 100644 index 00000000000..79e4d1a5b03 --- /dev/null +++ b/test/docker/base/provision.yml @@ -0,0 +1,15 @@ +- hosts: localhost + connection: local + roles: + - role: galaxyprojectdotorg.galaxy-os + tags: image + sudo: yes + - role: galaxyprojectdotorg.cloudman-database + sudo: yes + sudo_user: postgres + tags: database + - role: galaxyprojectdotorg.galaxy + sudo: yes + tags: galaxy + + diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh new file mode 100644 index 00000000000..df8961f5792 --- /dev/null +++ b/test/docker/base/run_test_wrapper.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -e + +# TODO: switch on environment to boot mysql here... +if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; +then + su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres + sleep 3 + export GALAXY_TEST_DBURI="postgres://galaxy@localhost:5930/galaxy" +elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ]; +then + sh /opt/galaxy/start_mysql.sh + export GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" +elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ]; +then + export GALAXY_TEST_DBURI="sqlite:////opt/galaxy/galaxy.sqlite" +else + echo "Unknown database type" + exit 1 +fi +cd /galaxy +GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI" sh manage_db.sh upgrade +sh run_tests.sh $@ diff --git a/test/docker/base/start_mysql.sh b/test/docker/base/start_mysql.sh new file mode 100644 index 00000000000..b02c0e5742d --- /dev/null +++ b/test/docker/base/start_mysql.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +MYSQL_USER=galaxy +MYSQL_PASSWORD=galaxy +MYSQL_DATABASE=galaxy + +mkdir -p /var/lib/mysql +chown -R mysql:mysql /var/lib/mysql + +# Derived from +# https://github.com/docker-library/mysql/blob/master/5.7/docker-entrypoint.sh + +DATADIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')" + +tempSqlFile='/tmp/mysql-first-time.sql' +cat > "$tempSqlFile" <<-EOSQL + DELETE FROM mysql.user ; + CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ; + GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ; + DROP DATABASE IF EXISTS test ; +EOSQL + +if [ "$MYSQL_DATABASE" ]; then + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" +fi + +if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + + if [ "$MYSQL_DATABASE" ]; then + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" + fi +fi + +echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile" + +mysqld_safe --init-file="$tempSqlFile" & +sleep 5 \ No newline at end of file