mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Add --dockerize and --db [postgres|mysql|sqlite] options to run_tests.sh.
Runs tests in a Dockerfile with pre-migrated databases and all testing dependencies (postgres, mysql, npm, casper, phantomjs, etc...). Hopefully this makes it easier to run more interesting tests - down the road I would like to add more options for things like condor, slurm, run as real user, etc....
This commit is contained in:
John Chilton
committed by
Nate Coraor
Nate Coraor
parent
b922485487
commit
e99adb5b0f
@@ -29,6 +29,9 @@ Extra options:
|
||||
--no_cleanup Do not delete temp files for Python functional tests (-toolshed, -framework, etc...)
|
||||
--report_file Path of HTML report to produce (for Python Galaxy functional tests).
|
||||
--xunit_report_file Path of XUnit report to produce (for Python Galaxy functional tests).
|
||||
--dockerize Run tests in a pre-configured Docker container (must be first argument if present).
|
||||
--db <type> For use with --dockerize, run tests using partially migrated 'postgres', 'mysql',
|
||||
or 'sqlite' databases.
|
||||
EOF
|
||||
}
|
||||
|
||||
@@ -60,6 +63,22 @@ with_framework_test_tools_arg=""
|
||||
|
||||
driver="python"
|
||||
|
||||
if [ "$1" = "--dockerize" ];
|
||||
then
|
||||
shift
|
||||
if [ "$1" = "--db" ]; then
|
||||
db_type=$2
|
||||
shift 2
|
||||
else
|
||||
db_type="sqlite"
|
||||
fi
|
||||
DOCKER_EXTRA_ARGS=${DOCKER_ARGS:-""}
|
||||
DOCKER_RUN_EXTRA_ARGS=${DOCKER_ARGS:-""}
|
||||
DOCKER_IMAGE=${DOCKER_IMAGE:-"galaxyprojectdotorg/testing-base"}
|
||||
docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS -e "GALAXY_TEST_DATABASE_TYPE=$db_type" --rm -v `pwd`:/galaxy $DOCKER_IMAGE "$@"
|
||||
exit $?
|
||||
fi
|
||||
|
||||
while :
|
||||
do
|
||||
case "$1" in
|
||||
|
||||
@@ -0,0 +1,9 @@
|
||||
Setup Docker environment for use with testing using the following
|
||||
commands (until this is on Docker hub).
|
||||
|
||||
% cd test/base
|
||||
% docker build -t galaxyprojectdotorg/testing-base .
|
||||
|
||||
This will create a docker container ready to run most of Galaxy tests
|
||||
(functional tests for specific bioinformatics tools may still fail for
|
||||
now until they are migrated out of Galaxy and to the tool shed).
|
||||
@@ -0,0 +1,63 @@
|
||||
FROM toolshed/requirements
|
||||
MAINTAINER John Chilton <jmchilton@gmail.com>
|
||||
|
||||
ENV MYSQL_MAJOR=5.7 \
|
||||
MYSQL_VERSION=5.7.5-m15
|
||||
|
||||
# Pre-install a bunch of packages to speed up ansible steps.
|
||||
RUN apt-get update -y && apt-get install -y software-properties-common curl && \
|
||||
apt-add-repository -y ppa:ansible/ansible && \
|
||||
curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \
|
||||
echo 'deb https://deb.nodesource.com/node trusty main' > /etc/apt/sources.list.d/nodesource.list && \
|
||||
apt-key adv --keyserver pool.sks-keyservers.net --recv-keys A4A9406876FCBD3C456770C88C718D3B5072E1F5 && \
|
||||
echo "deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-${MYSQL_MAJOR}-dmr" > /etc/apt/sources.list.d/mysql.list && \
|
||||
{ \
|
||||
echo mysql-community-server mysql-community-server/data-dir select ''; \
|
||||
echo mysql-community-server mysql-community-server/root-pass password ''; \
|
||||
echo mysql-community-server mysql-community-server/re-root-pass password ''; \
|
||||
echo mysql-community-server mysql-community-server/remove-test-db select false; \
|
||||
} | debconf-set-selections && \
|
||||
apt-get update -y && \
|
||||
apt-get install -y libpq-dev postgresql postgresql-client postgresql-plpython-9.3 \
|
||||
ansible postgresql-server-dev-9.3 wget mysql-server="${MYSQL_VERSION}"* \
|
||||
nodejs && \
|
||||
npm install -g grunt-contrib-qunit grunt grunt-cli casperjs phantomjs && \
|
||||
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||
|
||||
RUN mkdir -p /tmp/ansible && \
|
||||
mkdir -p /opt/galaxy/db && \
|
||||
chown -R postgres:postgres /opt/galaxy/db && \
|
||||
sed -Ei 's/^(bind-address|log)/#&/' /etc/mysql/my.cnf
|
||||
|
||||
ADD start_mysql.sh /opt/galaxy/start_mysql.sh
|
||||
ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml
|
||||
ADD provision.yml /tmp/ansible/provision.yml
|
||||
|
||||
ENV PYTHON_EGG_CACHE=/root/.python-eggs \
|
||||
GALAXY_ROOT=/opt/galaxy/galaxy-app
|
||||
|
||||
RUN cd /tmp/ansible && \
|
||||
mkdir roles && \
|
||||
mkdir roles/galaxyprojectdotorg.galaxy-os && \
|
||||
wget -qO- https://github.com/galaxyproject/ansible-galaxy-os/archive/master.tar.gz | tar -xzvf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-os && \
|
||||
mkdir roles/galaxyprojectdotorg.cloudman-database && \
|
||||
wget -qO- https://github.com/galaxyproject/ansible-cloudman-database/archive/master.tar.gz | tar -xzvf- --strip-components=1 -C roles/galaxyprojectdotorg.cloudman-database && \
|
||||
mkdir roles/galaxyprojectdotorg.galaxy && \
|
||||
wget -qO- https://github.com/galaxyproject/ansible-galaxy/archive/master.tar.gz | tar -xzvf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy && \
|
||||
ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=image -c local -e "@ansible_vars.yml" && \
|
||||
ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=database -c local -e "@ansible_vars.yml" && \
|
||||
ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=galaxy -c local -e "@ansible_vars.yml" && \
|
||||
apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
|
||||
|
||||
RUN cd $GALAXY_ROOT && \
|
||||
echo "Prepopuating postgres database" && \
|
||||
su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://galaxy@localhost:5930/galaxy" sh create_db.sh && \
|
||||
echo "Prepopuating sqlite database" && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" sh create_db.sh && \
|
||||
sh /opt/galaxy/start_mysql.sh && \
|
||||
GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh
|
||||
|
||||
ADD run_test_wrapper.sh /usr/local/bin/run_test_wrapper.sh
|
||||
|
||||
ENTRYPOINT ["/bin/bash", "/usr/local/bin/run_test_wrapper.sh"]
|
||||
@@ -0,0 +1,21 @@
|
||||
---
|
||||
add_system_users: no
|
||||
galaxyFS_base_dir: /opt/galaxy
|
||||
# TODO: use GALAXY_ROOT via cmd line instead
|
||||
galaxy_server_dir: "{{ galaxyFS_base_dir }}/galaxy-app"
|
||||
galaxy_venv_dir: "{{ galaxy_server_dir }}/.venv"
|
||||
galxy_egg_cache: "/home/{{ galaxy_user_name }}/.python-eggs"
|
||||
galaxy_repo: https://bitbucket.org/galaxy/galaxy-central
|
||||
galaxy_changeset_id: stable
|
||||
galaxy_manage_clone: no
|
||||
galaxy_manage_download: yes
|
||||
galaxy_root: "{{ galaxy_server_dir}}"
|
||||
galaxy_db_dir: "{{ galaxyFS_base_dir }}/db"
|
||||
galaxy_db_port: 5930
|
||||
galaxy_db_log: /tmp/pSQL.log
|
||||
psql_galaxyftp_password: ignoredthis
|
||||
galaxy_config_file: "{{ galaxy_server_dir }}/config/galaxy.ini"
|
||||
configure_docker: no
|
||||
postgresql_bin_dir: /usr/lib/postgresql/9.3/bin
|
||||
galaxy_manage_database: no
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
- hosts: localhost
|
||||
connection: local
|
||||
roles:
|
||||
- role: galaxyprojectdotorg.galaxy-os
|
||||
tags: image
|
||||
sudo: yes
|
||||
- role: galaxyprojectdotorg.cloudman-database
|
||||
sudo: yes
|
||||
sudo_user: postgres
|
||||
tags: database
|
||||
- role: galaxyprojectdotorg.galaxy
|
||||
sudo: yes
|
||||
tags: galaxy
|
||||
|
||||
|
||||
@@ -0,0 +1,23 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
# TODO: switch on environment to boot mysql here...
|
||||
if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ];
|
||||
then
|
||||
su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres
|
||||
sleep 3
|
||||
export GALAXY_TEST_DBURI="postgres://galaxy@localhost:5930/galaxy"
|
||||
elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ];
|
||||
then
|
||||
sh /opt/galaxy/start_mysql.sh
|
||||
export GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock"
|
||||
elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ];
|
||||
then
|
||||
export GALAXY_TEST_DBURI="sqlite:////opt/galaxy/galaxy.sqlite"
|
||||
else
|
||||
echo "Unknown database type"
|
||||
exit 1
|
||||
fi
|
||||
cd /galaxy
|
||||
GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI" sh manage_db.sh upgrade
|
||||
sh run_tests.sh $@
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
MYSQL_USER=galaxy
|
||||
MYSQL_PASSWORD=galaxy
|
||||
MYSQL_DATABASE=galaxy
|
||||
|
||||
mkdir -p /var/lib/mysql
|
||||
chown -R mysql:mysql /var/lib/mysql
|
||||
|
||||
# Derived from
|
||||
# https://github.com/docker-library/mysql/blob/master/5.7/docker-entrypoint.sh
|
||||
|
||||
DATADIR="$(mysqld --verbose --help 2>/dev/null | awk '$1 == "datadir" { print $2; exit }')"
|
||||
|
||||
tempSqlFile='/tmp/mysql-first-time.sql'
|
||||
cat > "$tempSqlFile" <<-EOSQL
|
||||
DELETE FROM mysql.user ;
|
||||
CREATE USER 'root'@'%' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
|
||||
GRANT ALL ON *.* TO 'root'@'%' WITH GRANT OPTION ;
|
||||
DROP DATABASE IF EXISTS test ;
|
||||
EOSQL
|
||||
|
||||
if [ "$MYSQL_DATABASE" ]; then
|
||||
echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile"
|
||||
fi
|
||||
|
||||
if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then
|
||||
echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile"
|
||||
|
||||
if [ "$MYSQL_DATABASE" ]; then
|
||||
echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile"
|
||||
fi
|
||||
fi
|
||||
|
||||
echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile"
|
||||
|
||||
mysqld_safe --init-file="$tempSqlFile" &
|
||||
sleep 5
|
||||
Reference in New Issue
Block a user