From ffda4aa2cd033e8953ef780fb553afbc77e4a7f4 Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 19 Oct 2015 13:13:34 -0400 Subject: [PATCH 1/2] Allow dockerized toolshed tests. --- lib/galaxy/model/orm/scripts.py | 6 +++- lib/galaxy/webapps/tool_shed/buildapp.py | 5 ++++ test/docker/base/Dockerfile | 37 ++++++++++++++++++------ test/docker/base/ansible_vars.yml | 11 ++++++- test/docker/base/provision.yml | 5 ++-- test/docker/base/run_test_wrapper.sh | 10 +++++++ test/docker/base/start_mysql.sh | 18 ++++++++---- 7 files changed, 74 insertions(+), 18 deletions(-) diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index fb4c332d5a4..e98711dbe84 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -30,6 +30,7 @@ DATABASE = { 'repo': 'lib/galaxy/model/migrate', 'old_config_file': 'universe_wsgi.ini', 'default_sqlite_file': './database/universe.sqlite', + 'config_override': 'GALAXY_CONFIG_', }, "tool_shed": { @@ -37,6 +38,7 @@ DATABASE = { 'config_file': 'config/tool_shed.ini', 'old_config_file': 'tool_shed_wsgi.ini', 'default_sqlite_file': './database/community.sqlite', + 'config_override': 'TOOL_SHED_CONFIG_', }, "install": { @@ -44,6 +46,7 @@ DATABASE = { 'old_config_file': 'universe_wsgi.ini', 'config_prefix': 'install_', 'default_sqlite_file': './database/install.sqlite', + 'config_override': 'GALAXY_INSTALL_CONFIG_', }, } @@ -118,11 +121,12 @@ def get_config( argv, cwd=None ): config_file = read_config_file_arg( argv, default, old_default ) repo = database_defaults[ 'repo' ] config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX ) + config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' ) default_sqlite_file = database_defaults[ 'default_sqlite_file' ] if cwd: config_file = os.path.join( cwd, config_file ) - properties = load_app_properties( ini_file=config_file ) + properties = load_app_properties( ini_file=config_file, config_prefix=config_override ) if ("%sdatabase_connection" % config_prefix) in properties: db_url = properties[ "%sdatabase_connection" % config_prefix ] diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index f62578c6616..07bf067ce2c 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -18,6 +18,7 @@ from galaxy.webapps.util import build_template_error_formatters from galaxy.webapps.tool_shed.framework.middleware import hg from galaxy import util from galaxy.config import process_is_uwsgi +from galaxy.util.properties import load_app_properties log = logging.getLogger( __name__ ) @@ -51,6 +52,10 @@ def add_ui_controllers( webapp, app ): def app_factory( global_conf, **kwargs ): """Return a wsgi application serving the root object""" # Create the Galaxy tool shed application unless passed in + kwargs = load_app_properties( + kwds=kwargs, + config_prefix='TOOL_SHED_CONFIG_' + ) if 'app' in kwargs: app = kwargs.pop( 'app' ) else: diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 9aa91006195..8f5100935f5 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -18,7 +18,7 @@ RUN apt-get update -y && apt-get install -y software-properties-common curl && \ } | debconf-set-selections && \ apt-get update -y && \ apt-get install -y libpq-dev postgresql postgresql-client postgresql-plpython-9.3 \ - ansible postgresql-server-dev-9.3 wget mysql-server="${MYSQL_MAJOR}"* \ + ansible postgresql-server-dev-9.3 wget mysql-server="${MYSQL_MAJOR}"* \ slurm-llnl libmunge-dev slurm-drmaa-dev \ nodejs && \ npm install -g grunt-contrib-qunit grunt grunt-cli casperjs phantomjs && \ @@ -29,6 +29,13 @@ RUN mkdir -p /tmp/ansible && \ chown -R postgres:postgres /opt/galaxy/db && \ sed -Ei 's/^(bind-address|log)/#&/' /etc/mysql/my.cnf +RUN apt-get update -y && apt-get install -y ant atop axel bioperl cmake curl g++ gcc gfortran git-core htop iftop iotop \ + ipython libffi-dev liblapack-dev libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev \ + libsparsehash-dev make mercurial nginx-extras nmon openssh-server patch postgresql postgresql \ + postgresql-client postgresql-plpython-9.3 python-boto python-dev python-prettytable python-psycopg2 \ + python-virtualenv python-pip postgresql-server-dev-9.3 rsync slurm-drmaa-dev supervisor swig \ + sysstat unzip vim wget zlib1g-dev + ADD start_mysql.sh /opt/galaxy/start_mysql.sh ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml ADD provision.yml /tmp/ansible/provision.yml @@ -36,20 +43,21 @@ ADD provision.yml /tmp/ansible/provision.yml ENV PYTHON_EGG_CACHE=/root/.python-eggs \ GALAXY_ROOT=/galaxy -RUN mkdir /etc/galaxy && \ - cd /tmp/ansible && \ - mkdir roles && \ +RUN mkdir /etc/galaxy && cd /tmp/ansible && mkdir roles && \ mkdir roles/galaxyprojectdotorg.galaxy-os && \ - wget -qO- https://github.com/galaxyproject/ansible-galaxy-os/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-os && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy-os/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-os && \ mkdir roles/galaxyprojectdotorg.cloudman-database && \ - wget -qO- https://github.com/galaxyproject/ansible-cloudman-database/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.cloudman-database && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-cloudman-database/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.cloudman-database && \ mkdir roles/galaxyprojectdotorg.galaxy && \ - wget -qO- https://github.com/galaxyproject/ansible-galaxy/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy && \ mkdir roles/galaxyprojectdotorg.galaxy-extras && \ - wget -qO- https://github.com/galaxyproject/ansible-galaxy-extras/archive/dynamic_uwsgi_config.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-extras && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy-extras/archive/dynamic_uwsgi_config.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-extras && \ + mkdir roles/galaxyprojectdotorg.galaxy-toolshed && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy-toolshed/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-toolshed && \ ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=image -c local -e "@ansible_vars.yml" && \ ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=database -c local -e "@ansible_vars.yml" && \ ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=galaxy -c local -e "@ansible_vars.yml" && \ + ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=toolshed -c local -e "@ansible_vars.yml" && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN cd $GALAXY_ROOT && \ @@ -60,7 +68,16 @@ RUN cd $GALAXY_ROOT && \ echo "Prepopuating sqlite database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" sh create_db.sh && \ sh /opt/galaxy/start_mysql.sh && \ - GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh + echo "Prepopuating mysql database" && \ + GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh && \ + echo "Prepopulating toolshed postgres database" && \ + su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ + GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" sh create_db.sh tool_shed && \ + echo "Prepopuating toolshed sqlite database" && \ + GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" sh create_db.sh tool_shed && \ + sh /opt/galaxy/start_mysql.sh && \ + echo "Prepopuating toolshed mysql database" && \ + GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh tool_shed # bcftools for Galaxy. RUN mkdir -p /tmp/install && \ @@ -73,6 +90,8 @@ RUN mkdir -p /tmp/install && \ cd && rm -rf /tmp/install ADD run_test_wrapper.sh /usr/local/bin/run_test_wrapper.sh + +EXPOSE :9009 EXPOSE :8080 EXPOSE :80 ENTRYPOINT ["/bin/bash", "/usr/local/bin/run_test_wrapper.sh"] diff --git a/test/docker/base/ansible_vars.yml b/test/docker/base/ansible_vars.yml index e39108ac433..5353e96b9b1 100644 --- a/test/docker/base/ansible_vars.yml +++ b/test/docker/base/ansible_vars.yml @@ -22,7 +22,6 @@ galaxy_manage_database: no galaxy_user_name: "root" galaxy_extras_install_packages: true galaxy_job_conf_path: "/etc/galaxy/job_conf.xml" -galxy_config_file: "/galaxy/config/galaxy.ini.sample" galaxy_web_processes: 1 galaxy_handler_processes: 2 @@ -38,3 +37,13 @@ galaxy_extras_config_supervisor: true galaxy_extras_config_galaxy_root: false galaxy_extras_config_galaxy_job_metrics: false galaxy_extras_config_uwsgi: true + +manage_shed_db: yes +galaxy_toolshed_manage_static_setup: yes +galaxy_toolshed_manage_database: no +galaxy_toolshed_server_dir: "/galaxy" +galaxy_toolshed_venv_dir: "{{ galaxy_venv_dir }}" + +galaxy_toolshed_config_file: "{{ galaxy_toolshed_server_dir }}/config/tool_shed.ini" +galaxy_toolshed_config_templates: +- {dest: '{{ galaxy_toolshed_config_file }}', src: tool_shed.ini.j2} diff --git a/test/docker/base/provision.yml b/test/docker/base/provision.yml index 79e4d1a5b03..adbeb69d9d0 100644 --- a/test/docker/base/provision.yml +++ b/test/docker/base/provision.yml @@ -11,5 +11,6 @@ - role: galaxyprojectdotorg.galaxy sudo: yes tags: galaxy - - + - role: galaxyprojectdotorg.galaxy-toolshed + sudo: yes + tags: toolshed \ No newline at end of file diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index a3a4a3b5ceb..538bac763c2 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -6,19 +6,27 @@ if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; then su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres sleep 3 + GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="postgres://root@localhost:5930/galaxy" + TOOL_SHED_TEST_DBURI="postgres://root@localhost:5930/toolshed" elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ]; then sh /opt/galaxy/start_mysql.sh + GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" + TOOL_SHED_TEST_DBURI="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ]; then + GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="sqlite:////opt/galaxy/galaxy.sqlite" + TOOL_SHED_TEST_DBURI="sqlite:////opt/galaxy/toolshed.sqlite" else echo "Unknown database type" exit 1 fi export GALAXY_TEST_DBURI +export TOOL_SHED_TEST_DBURI +export GALAXY_TEST_INSTALL_DB_MERGED cd /galaxy GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI"; @@ -32,6 +40,7 @@ then else GALAXY_CONFIG_MASTER_API_KEY=${GALAXY_CONFIG_MASTER_API_KEY:-"testmasterapikey"} GALAXY_CONFIG_FILE=${GALAXY_CONFIG_FILE:-config/galaxy.ini.sample} + TOOL_SHED_CONFIG_FILE=${GALAXY_CONFIG_FILE:-config/tool_shed.ini.sample} GALAXY_CONFIG_CHECK_MIGRATE_TOOLS=false if [ -z "$GALAXY_MULTI_PROCESS" ]; then @@ -44,6 +53,7 @@ else export GALAXY_CONFIG_MASTER_API_KEY export GALAXY_CONFIG_FILE + export TOOL_SHED_CONFIG_FILE export GALAXY_CONFIG_CHECK_MIGRATE_TOOLS export GALAXY_CONFIG_JOB_CONFIG_FILE export GALAXY_CONFIG_FILE_PATH diff --git a/test/docker/base/start_mysql.sh b/test/docker/base/start_mysql.sh index b02c0e5742d..cd378a056fe 100644 --- a/test/docker/base/start_mysql.sh +++ b/test/docker/base/start_mysql.sh @@ -4,6 +4,7 @@ set -e MYSQL_USER=galaxy MYSQL_PASSWORD=galaxy MYSQL_DATABASE=galaxy +SHED_MYSQL_DATABASE=toolshed mkdir -p /var/lib/mysql chown -R mysql:mysql /var/lib/mysql @@ -22,15 +23,22 @@ cat > "$tempSqlFile" <<-EOSQL EOSQL if [ "$MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" +fi + +if [ "$SHED_MYSQL_DATABASE" ]; then + echo "CREATE DATABASE IF NOT EXISTS \`$SHED_MYSQL_DATABASE\` ;" >> "$tempSqlFile" fi if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then - echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" - if [ "$MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" - fi + if [ "$MYSQL_DATABASE" ]; then + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" + fi + if [ "$SHED_MYSQL_DATABASE" ]; then + echo "GRANT ALL ON \`$SHED_MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" + fi fi echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile" From 096adbf8d0776d4b1085705f2881c2d46ab06611 Mon Sep 17 00:00:00 2001 From: Dave B Date: Mon, 19 Oct 2015 14:03:22 -0400 Subject: [PATCH 2/2] Collect apt installs in one place. --- test/docker/base/Dockerfile | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 8f5100935f5..fa946aacb88 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -19,8 +19,14 @@ RUN apt-get update -y && apt-get install -y software-properties-common curl && \ apt-get update -y && \ apt-get install -y libpq-dev postgresql postgresql-client postgresql-plpython-9.3 \ ansible postgresql-server-dev-9.3 wget mysql-server="${MYSQL_MAJOR}"* \ - slurm-llnl libmunge-dev slurm-drmaa-dev \ - nodejs && \ + slurm-llnl libmunge-dev slurm-drmaa-dev ant atop axel bioperl cmake curl \ + g++ gcc gfortran git-core htop iftop iotop ipython libffi-dev liblapack-dev \ + libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev libsparsehash-dev make \ + mercurial nginx-extras nmon patch postgresql postgresql \ + postgresql-client postgresql-plpython-9.3 python-boto python-dev \ + python-prettytable python-psycopg2 python-virtualenv python-pip \ + postgresql-server-dev-9.3 rsync slurm-drmaa-dev supervisor swig sysstat unzip \ + wget zlib1g-dev nodejs && \ npm install -g grunt-contrib-qunit grunt grunt-cli casperjs phantomjs && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -29,13 +35,6 @@ RUN mkdir -p /tmp/ansible && \ chown -R postgres:postgres /opt/galaxy/db && \ sed -Ei 's/^(bind-address|log)/#&/' /etc/mysql/my.cnf -RUN apt-get update -y && apt-get install -y ant atop axel bioperl cmake curl g++ gcc gfortran git-core htop iftop iotop \ - ipython libffi-dev liblapack-dev libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev \ - libsparsehash-dev make mercurial nginx-extras nmon openssh-server patch postgresql postgresql \ - postgresql-client postgresql-plpython-9.3 python-boto python-dev python-prettytable python-psycopg2 \ - python-virtualenv python-pip postgresql-server-dev-9.3 rsync slurm-drmaa-dev supervisor swig \ - sysstat unzip vim wget zlib1g-dev - ADD start_mysql.sh /opt/galaxy/start_mysql.sh ADD ansible_vars.yml /tmp/ansible/ansible_vars.yml ADD provision.yml /tmp/ansible/provision.yml