diff --git a/lib/galaxy/model/orm/scripts.py b/lib/galaxy/model/orm/scripts.py index 86d5dc02852..d7fce87ecb2 100644 --- a/lib/galaxy/model/orm/scripts.py +++ b/lib/galaxy/model/orm/scripts.py @@ -19,6 +19,7 @@ DATABASE = { 'repo': 'lib/galaxy/model/migrate', 'old_config_file': 'universe_wsgi.ini', 'default_sqlite_file': './database/universe.sqlite', + 'config_override': 'GALAXY_CONFIG_', }, "tool_shed": { @@ -26,6 +27,7 @@ DATABASE = { 'config_file': 'config/tool_shed.ini', 'old_config_file': 'tool_shed_wsgi.ini', 'default_sqlite_file': './database/community.sqlite', + 'config_override': 'TOOL_SHED_CONFIG_', }, "install": { @@ -33,6 +35,7 @@ DATABASE = { 'old_config_file': 'universe_wsgi.ini', 'config_prefix': 'install_', 'default_sqlite_file': './database/install.sqlite', + 'config_override': 'GALAXY_INSTALL_CONFIG_', }, } @@ -92,11 +95,12 @@ def get_config( argv, cwd=None ): config_file = read_config_file_arg( argv, default, old_default ) repo = database_defaults[ 'repo' ] config_prefix = database_defaults.get( 'config_prefix', DEFAULT_CONFIG_PREFIX ) + config_override = database_defaults.get( 'config_override', 'GALAXY_CONFIG_' ) default_sqlite_file = database_defaults[ 'default_sqlite_file' ] if cwd: config_file = os.path.join( cwd, config_file ) - properties = load_app_properties( ini_file=config_file ) + properties = load_app_properties( ini_file=config_file, config_prefix=config_override ) if ("%sdatabase_connection" % config_prefix) in properties: db_url = properties[ "%sdatabase_connection" % config_prefix ] diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 67e5620789a..21da0a06f3d 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -16,6 +16,7 @@ from galaxy.webapps.util import build_template_error_formatters from galaxy.webapps.tool_shed.framework.middleware import hg from galaxy import util from galaxy.config import process_is_uwsgi +from galaxy.util.properties import load_app_properties log = logging.getLogger( __name__ ) @@ -49,6 +50,10 @@ def add_ui_controllers( webapp, app ): def app_factory( global_conf, **kwargs ): """Return a wsgi application serving the root object""" # Create the Galaxy tool shed application unless passed in + kwargs = load_app_properties( + kwds=kwargs, + config_prefix='TOOL_SHED_CONFIG_' + ) if 'app' in kwargs: app = kwargs.pop( 'app' ) else: diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 113a9b7595b..77099f8340f 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -18,9 +18,15 @@ RUN apt-get update -y && apt-get install -y software-properties-common curl && \ } | debconf-set-selections && \ apt-get update -y && \ apt-get install -y libpq-dev postgresql postgresql-client postgresql-plpython-9.3 \ - ansible postgresql-server-dev-9.3 wget mysql-server="${MYSQL_MAJOR}"* \ - slurm-llnl libmunge-dev slurm-drmaa-dev \ - nodejs && \ + ansible postgresql-server-dev-9.3 wget mysql-server="${MYSQL_MAJOR}"* \ + slurm-llnl libmunge-dev slurm-drmaa-dev ant atop axel bioperl cmake curl \ + g++ gcc gfortran git-core htop iftop iotop ipython libffi-dev liblapack-dev \ + libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev libsparsehash-dev make \ + mercurial nginx-extras nmon patch postgresql postgresql \ + postgresql-client postgresql-plpython-9.3 python-boto python-dev \ + python-prettytable python-psycopg2 python-virtualenv python-pip \ + postgresql-server-dev-9.3 rsync slurm-drmaa-dev supervisor swig sysstat unzip \ + wget zlib1g-dev nodejs && \ npm install -g grunt-contrib-qunit grunt grunt-cli casperjs phantomjs && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -35,20 +41,21 @@ ADD provision.yml /tmp/ansible/provision.yml ENV GALAXY_ROOT=/galaxy -RUN mkdir /etc/galaxy && \ - cd /tmp/ansible && \ - mkdir roles && \ +RUN mkdir /etc/galaxy && cd /tmp/ansible && mkdir roles && \ mkdir roles/galaxyprojectdotorg.galaxy-os && \ - wget -qO- https://github.com/galaxyproject/ansible-galaxy-os/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-os && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy-os/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-os && \ mkdir roles/galaxyprojectdotorg.cloudman-database && \ - wget -qO- https://github.com/galaxyproject/ansible-cloudman-database/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.cloudman-database && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-cloudman-database/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.cloudman-database && \ mkdir roles/galaxyprojectdotorg.galaxy && \ - wget -qO- https://github.com/galaxyproject/ansible-galaxy/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy && \ mkdir roles/galaxyprojectdotorg.galaxy-extras && \ - wget -qO- https://github.com/galaxyproject/ansible-galaxy-extras/archive/dynamic_uwsgi_config.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-extras && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy-extras/archive/dynamic_uwsgi_config.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-extras && \ + mkdir roles/galaxyprojectdotorg.galaxy-toolshed && \ + wget --quiet -O- https://github.com/galaxyproject/ansible-galaxy-toolshed/archive/master.tar.gz | tar -xzf- --strip-components=1 -C roles/galaxyprojectdotorg.galaxy-toolshed && \ ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=image -c local -e "@ansible_vars.yml" && \ ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=database -c local -e "@ansible_vars.yml" && \ ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=galaxy -c local -e "@ansible_vars.yml" && \ + ANSIBLE_FORCE_COLOR=1 PYTHONUNBUFFERED=1 ansible-playbook /tmp/ansible/provision.yml --tags=toolshed -c local -e "@ansible_vars.yml" && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* RUN cd $GALAXY_ROOT && \ @@ -59,7 +66,16 @@ RUN cd $GALAXY_ROOT && \ echo "Prepopuating sqlite database" && \ GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/galaxy.sqlite" sh create_db.sh && \ sh /opt/galaxy/start_mysql.sh && \ - GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh + echo "Prepopuating mysql database" && \ + GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh && \ + echo "Prepopulating toolshed postgres database" && \ + su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres && \ + GALAXY_CONFIG_DATABASE_CONNECTION="postgresql://root@localhost:5930/toolshed" sh create_db.sh tool_shed && \ + echo "Prepopuating toolshed sqlite database" && \ + GALAXY_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" sh create_db.sh tool_shed && \ + sh /opt/galaxy/start_mysql.sh && \ + echo "Prepopuating toolshed mysql database" && \ + GALAXY_CONFIG_DATABASE_CONNECTION="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" sh create_db.sh tool_shed # bcftools for Galaxy. RUN mkdir -p /tmp/install && \ @@ -72,6 +88,8 @@ RUN mkdir -p /tmp/install && \ cd && rm -rf /tmp/install ADD run_test_wrapper.sh /usr/local/bin/run_test_wrapper.sh + +EXPOSE :9009 EXPOSE :8080 EXPOSE :80 ENTRYPOINT ["/bin/bash", "/usr/local/bin/run_test_wrapper.sh"] diff --git a/test/docker/base/ansible_vars.yml b/test/docker/base/ansible_vars.yml index 61f275b5e71..1db6a9fc25d 100644 --- a/test/docker/base/ansible_vars.yml +++ b/test/docker/base/ansible_vars.yml @@ -21,7 +21,6 @@ galaxy_manage_database: no galaxy_user_name: "root" galaxy_extras_install_packages: true galaxy_job_conf_path: "/etc/galaxy/job_conf.xml" -galxy_config_file: "/galaxy/config/galaxy.ini.sample" galaxy_web_processes: 1 galaxy_handler_processes: 2 @@ -37,3 +36,13 @@ galaxy_extras_config_supervisor: true galaxy_extras_config_galaxy_root: false galaxy_extras_config_galaxy_job_metrics: false galaxy_extras_config_uwsgi: true + +manage_shed_db: yes +galaxy_toolshed_manage_static_setup: yes +galaxy_toolshed_manage_database: no +galaxy_toolshed_server_dir: "/galaxy" +galaxy_toolshed_venv_dir: "{{ galaxy_venv_dir }}" + +galaxy_toolshed_config_file: "{{ galaxy_toolshed_server_dir }}/config/tool_shed.ini" +galaxy_toolshed_config_templates: +- {dest: '{{ galaxy_toolshed_config_file }}', src: tool_shed.ini.j2} diff --git a/test/docker/base/provision.yml b/test/docker/base/provision.yml index 79e4d1a5b03..adbeb69d9d0 100644 --- a/test/docker/base/provision.yml +++ b/test/docker/base/provision.yml @@ -11,5 +11,6 @@ - role: galaxyprojectdotorg.galaxy sudo: yes tags: galaxy - - + - role: galaxyprojectdotorg.galaxy-toolshed + sudo: yes + tags: toolshed \ No newline at end of file diff --git a/test/docker/base/run_test_wrapper.sh b/test/docker/base/run_test_wrapper.sh index bf317ba2afc..95851ba2b92 100644 --- a/test/docker/base/run_test_wrapper.sh +++ b/test/docker/base/run_test_wrapper.sh @@ -6,19 +6,27 @@ if [ "$GALAXY_TEST_DATABASE_TYPE" = "postgres" ]; then su -c '/usr/lib/postgresql/9.3/bin/pg_ctl -o "-F" start -D /opt/galaxy/db' postgres sleep 3 + GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="postgres://root@localhost:5930/galaxy?client_encoding=utf8" + TOOL_SHED_TEST_DBURI="postgres://root@localhost:5930/toolshed" elif [ "$GALAXY_TEST_DATABASE_TYPE" = "mysql" ]; then sh /opt/galaxy/start_mysql.sh + GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="mysql://galaxy:galaxy@localhost/galaxy?unix_socket=/var/run/mysqld/mysqld.sock" + TOOL_SHED_TEST_DBURI="mysql://galaxy:galaxy@localhost/toolshed?unix_socket=/var/run/mysqld/mysqld.sock" elif [ "$GALAXY_TEST_DATABASE_TYPE" = "sqlite" ]; then + GALAXY_TEST_INSTALL_DB_MERGED="true" GALAXY_TEST_DBURI="sqlite:////opt/galaxy/galaxy.sqlite" + TOOL_SHED_TEST_DBURI="sqlite:////opt/galaxy/toolshed.sqlite" else echo "Unknown database type" exit 1 fi export GALAXY_TEST_DBURI +export TOOL_SHED_TEST_DBURI +export GALAXY_TEST_INSTALL_DB_MERGED cd /galaxy GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION="$GALAXY_TEST_DBURI"; @@ -37,6 +45,7 @@ then else GALAXY_CONFIG_MASTER_API_KEY=${GALAXY_CONFIG_MASTER_API_KEY:-"testmasterapikey"} GALAXY_CONFIG_FILE=${GALAXY_CONFIG_FILE:-config/galaxy.ini.sample} + TOOL_SHED_CONFIG_FILE=${GALAXY_CONFIG_FILE:-config/tool_shed.ini.sample} GALAXY_CONFIG_CHECK_MIGRATE_TOOLS=false if [ -z "$GALAXY_MULTI_PROCESS" ]; then @@ -49,6 +58,7 @@ else export GALAXY_CONFIG_MASTER_API_KEY export GALAXY_CONFIG_FILE + export TOOL_SHED_CONFIG_FILE export GALAXY_CONFIG_CHECK_MIGRATE_TOOLS export GALAXY_CONFIG_JOB_CONFIG_FILE export GALAXY_CONFIG_FILE_PATH diff --git a/test/docker/base/start_mysql.sh b/test/docker/base/start_mysql.sh index b02c0e5742d..cd378a056fe 100644 --- a/test/docker/base/start_mysql.sh +++ b/test/docker/base/start_mysql.sh @@ -4,6 +4,7 @@ set -e MYSQL_USER=galaxy MYSQL_PASSWORD=galaxy MYSQL_DATABASE=galaxy +SHED_MYSQL_DATABASE=toolshed mkdir -p /var/lib/mysql chown -R mysql:mysql /var/lib/mysql @@ -22,15 +23,22 @@ cat > "$tempSqlFile" <<-EOSQL EOSQL if [ "$MYSQL_DATABASE" ]; then - echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" + echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" >> "$tempSqlFile" +fi + +if [ "$SHED_MYSQL_DATABASE" ]; then + echo "CREATE DATABASE IF NOT EXISTS \`$SHED_MYSQL_DATABASE\` ;" >> "$tempSqlFile" fi if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then - echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" + echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" >> "$tempSqlFile" - if [ "$MYSQL_DATABASE" ]; then - echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" - fi + if [ "$MYSQL_DATABASE" ]; then + echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" + fi + if [ "$SHED_MYSQL_DATABASE" ]; then + echo "GRANT ALL ON \`$SHED_MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" >> "$tempSqlFile" + fi fi echo 'FLUSH PRIVILEGES ;' >> "$tempSqlFile"