From bf8dbdade3b009d7bd0822b657374b992342a466 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 1 Mar 2019 12:11:53 +0100 Subject: [PATCH 1/9] Slim down testing image --- test/docker/base/Dockerfile | 31 ++++++++++--------------------- 1 file changed, 10 insertions(+), 21 deletions(-) diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 2399868d3ff..041f01cef8e 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -19,20 +19,17 @@ ENV DEBIAN_FRONTEND=noninteractive \ RUN apt-get update -y && apt-get install -y software-properties-common apt-transport-https curl && \ apt-add-repository -y ppa:ansible/ansible && \ curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ - curl -s https://deb.nodesource.com/gpgkey/nodesource.gpg.key | apt-key add - && \ - curl -sL https://deb.nodesource.com/setup_8.x | bash - && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ apt-get update -y && \ - apt-get install -y libpq-dev postgresql postgresql-client ansible wget \ + apt-get install -y --no-install-recommends libpq-dev postgresql postgresql-client ansible wget \ slurm-llnl libmunge-dev slurm-drmaa-dev ant cmake curl \ - g++ gcc gfortran git-core libffi-dev liblapack-dev \ - libncurses5-dev libopenblas-dev libpam0g-dev libpq-dev libsparsehash-dev make \ - patch postgresql postgresql \ - postgresql-client python-dev python3-dev \ - python-prettytable python-psycopg2 python-virtualenv python-pip \ + g++ gcc gfortran git-core libffi-dev \ + libncurses5-dev libpam0g-dev libsparsehash-dev make patch \ + python-dev python3-dev \ + python-prettytable python-virtualenv python-pip \ rsync slurm-drmaa-dev swig sysstat unzip \ - autoconf automake build-essential libatlas-base-dev libblas-dev openssl \ - bzip2 uuid-dev libssl-dev libgpgme11-dev squashfs-tools libseccomp-dev pkg-config \ + autoconf automake build-essential openssl \ + bzip2 uuid-dev libgpgme11-dev squashfs-tools libseccomp-dev pkg-config \ ca-certificates \ openjdk-8-jre-headless \ tzdata \ @@ -40,7 +37,9 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans locales \ xvfb \ ${CHROME_VERSION:-google-chrome-stable} \ - wget zlib1g-dev nodejs \ + wget \ + ffmpeg \ + bcftools \ libnss3 libgconf-2-4 && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* @@ -96,16 +95,6 @@ RUN for VENV in $GALAXY_VIRTUAL_ENV_3 $GALAXY_VIRTUAL_ENV_2; do \ echo "Prepopulating toolshed sqlite database" && \ TOOL_SHED_CONFIG_DATABASE_CONNECTION="sqlite:////opt/galaxy/toolshed.sqlite" bash create_db.sh tool_shed -# bcftools for Galaxy. -RUN mkdir -p /tmp/install && \ - cd /tmp/install && \ - wget https://github.com/samtools/bcftools/releases/download/1.2/bcftools-1.2.tar.bz2 && \ - tar xvjf bcftools-1.2.tar.bz2 && \ - cd bcftools-1.2 && \ - make && \ - make install && \ - cd && rm -rf /tmp/install - #======================================== # Add Selenium user with passwordless sudo #======================================== From d9ac24a9828daa00120da3014634c38a34e1934b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 1 Mar 2019 12:13:18 +0100 Subject: [PATCH 2/9] Don't install slurm, it isn't used in any test --- test/docker/base/Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 041f01cef8e..74c44e1451d 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -22,12 +22,12 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ apt-get update -y && \ apt-get install -y --no-install-recommends libpq-dev postgresql postgresql-client ansible wget \ - slurm-llnl libmunge-dev slurm-drmaa-dev ant cmake curl \ + ant cmake curl \ g++ gcc gfortran git-core libffi-dev \ libncurses5-dev libpam0g-dev libsparsehash-dev make patch \ python-dev python3-dev \ python-prettytable python-virtualenv python-pip \ - rsync slurm-drmaa-dev swig sysstat unzip \ + rsync swig sysstat unzip \ autoconf automake build-essential openssl \ bzip2 uuid-dev libgpgme11-dev squashfs-tools libseccomp-dev pkg-config \ ca-certificates \ @@ -75,7 +75,7 @@ RUN cd $GALAXY_ROOT && \ virtualenv -p /usr/bin/python3 $GALAXY_VIRTUAL_ENV_3 && \ for VENV in $GALAXY_VIRTUAL_ENV_2 $GALAXY_VIRTUAL_ENV_3; do \ export GALAXY_VIRTUAL_ENV=$VENV && \ - ./scripts/common_startup.sh || { echo "common_startup.sh failed"; exit 1; } && \ + ./scripts/common_startup.sh || true && \ dev_requirements=./lib/galaxy/dependencies/dev-requirements.txt && \ [ -f $dev_requirements ] && $VENV/bin/pip install -r $dev_requirements; done From 3bed43fb841a371ab350406dc8b01dd5ce55dad8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 1 Mar 2019 14:27:09 +0100 Subject: [PATCH 3/9] Don't add compiler-stack (will still come in ansible), use prebuilt singularity --- test/docker/base/Dockerfile | 32 ++++++++------------------------ 1 file changed, 8 insertions(+), 24 deletions(-) diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 74c44e1451d..6a33c93a0e3 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -20,16 +20,15 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans apt-add-repository -y ppa:ansible/ansible && \ curl -s https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - && \ echo "deb http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google-chrome.list && \ + curl -s http://neuro.debian.net/lists/xenial.us-ca.full > /etc/apt/sources.list.d/neurodebian.sources.list && \ + apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && \ apt-get update -y && \ - apt-get install -y --no-install-recommends libpq-dev postgresql postgresql-client ansible wget \ - ant cmake curl \ - g++ gcc gfortran git-core libffi-dev \ - libncurses5-dev libpam0g-dev libsparsehash-dev make patch \ - python-dev python3-dev \ + apt-get install -y --no-install-recommends postgresql postgresql-client ansible wget \ + git-core \ python-prettytable python-virtualenv python-pip \ rsync swig sysstat unzip \ - autoconf automake build-essential openssl \ - bzip2 uuid-dev libgpgme11-dev squashfs-tools libseccomp-dev pkg-config \ + openssl \ + bzip2 \ ca-certificates \ openjdk-8-jre-headless \ tzdata \ @@ -37,15 +36,12 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans locales \ xvfb \ ${CHROME_VERSION:-google-chrome-stable} \ - wget \ ffmpeg \ bcftools \ + singularity-container \ libnss3 libgconf-2-4 && \ apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* -# Install golang -RUN export VERSION=1.11 OS=linux ARCH=amd64 && curl -sL https://dl.google.com/go/go$VERSION.$OS-$ARCH.tar.gz \ - | tar -C /usr/local -xzf - RUN mkdir -p /tmp/ansible && \ mkdir -p /opt/galaxy/db && \ @@ -75,7 +71,7 @@ RUN cd $GALAXY_ROOT && \ virtualenv -p /usr/bin/python3 $GALAXY_VIRTUAL_ENV_3 && \ for VENV in $GALAXY_VIRTUAL_ENV_2 $GALAXY_VIRTUAL_ENV_3; do \ export GALAXY_VIRTUAL_ENV=$VENV && \ - ./scripts/common_startup.sh || true && \ + sh ./scripts/common_startup.sh || true && \ dev_requirements=./lib/galaxy/dependencies/dev-requirements.txt && \ [ -f $dev_requirements ] && $VENV/bin/pip install -r $dev_requirements; done @@ -115,18 +111,6 @@ RUN sudo mkdir -p /opt/selenium \ && wget --no-verbose https://selenium-release.storage.googleapis.com/3.6/selenium-server-standalone-3.6.0.jar \ -O /opt/selenium/selenium-server-standalone.jar -USER seluser - -RUN export GOPATH=${HOME}/go && export PATH=/usr/local/go/bin:${PATH}:${GOPATH}/bin && \ - mkdir -p $GOPATH/src/github.com/sylabs && \ - cd $GOPATH/src/github.com/sylabs && \ - git clone https://github.com/sylabs/singularity.git && \ - cd singularity && \ - git checkout v3.0.1 && \ - ./mconfig && \ - cd builddir && make && sudo make install && \ - rm -Rf ${HOME}/go - USER root #============================== From a0fd6438d99d55a5337fc5fbdf2a8629b2779c4b Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 1 Mar 2019 14:27:54 +0100 Subject: [PATCH 4/9] Directlyy setup virtualenv in /galaxy_venv, saves 700MB --- test/docker/base/ansible_vars.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/docker/base/ansible_vars.yml b/test/docker/base/ansible_vars.yml index 139809f7a73..d92d87f6f41 100644 --- a/test/docker/base/ansible_vars.yml +++ b/test/docker/base/ansible_vars.yml @@ -3,7 +3,7 @@ add_system_users: no galaxyFS_base_dir: /opt/galaxy # TODO: use GALAXY_ROOT via cmd line instead galaxy_server_dir: "/galaxy" -galaxy_venv_dir: "{{ galaxyFS_base_dir }}/.venv" +galaxy_venv_dir: "/galaxy_venv" galaxy_vcs: git galaxy_git_repo: https://github.com/galaxyproject/galaxy.git galaxy_changeset_id: dev @@ -24,6 +24,7 @@ install_maintainance_packages: false galaxy_user_name: "root" galaxy_extras_install_packages: true galaxy_job_conf_path: "/etc/galaxy/job_conf.xml" +galaxy_build_client: no galaxy_web_processes: 1 galaxy_handler_processes: 2 From c357d303fe57a685cf57895bac9070828c81bbec Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 1 Mar 2019 14:28:55 +0100 Subject: [PATCH 5/9] Test from new image --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index d565910a90b..7cdb9c8e74c 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -270,7 +270,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='galaxy/testing-base:19.01.0' +DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.05.1' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" From 087070c72ce212cadedbd3dc5b2b2ae823b980c6 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 1 Mar 2019 17:50:50 +0100 Subject: [PATCH 6/9] Still need python3-dev for uwsgi --- test/docker/base/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/docker/base/Dockerfile b/test/docker/base/Dockerfile index 6a33c93a0e3..cc917f67f8e 100644 --- a/test/docker/base/Dockerfile +++ b/test/docker/base/Dockerfile @@ -24,6 +24,7 @@ RUN apt-get update -y && apt-get install -y software-properties-common apt-trans apt-key adv --recv-keys --keyserver hkp://pool.sks-keyservers.net:80 0xA5D32F012649A5A9 && \ apt-get update -y && \ apt-get install -y --no-install-recommends postgresql postgresql-client ansible wget \ + python3-dev \ git-core \ python-prettytable python-virtualenv python-pip \ rsync swig sysstat unzip \ @@ -71,7 +72,7 @@ RUN cd $GALAXY_ROOT && \ virtualenv -p /usr/bin/python3 $GALAXY_VIRTUAL_ENV_3 && \ for VENV in $GALAXY_VIRTUAL_ENV_2 $GALAXY_VIRTUAL_ENV_3; do \ export GALAXY_VIRTUAL_ENV=$VENV && \ - sh ./scripts/common_startup.sh || true && \ + ./scripts/common_startup.sh && \ dev_requirements=./lib/galaxy/dependencies/dev-requirements.txt && \ [ -f $dev_requirements ] && $VENV/bin/pip install -r $dev_requirements; done From 0455eb4f665dca2f9ba4d7129f31aeb2f2dbf579 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 1 Mar 2019 17:56:02 +0100 Subject: [PATCH 7/9] Use newer image --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index 7cdb9c8e74c..b2b9d24ebc1 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -270,7 +270,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.05.1' +DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.05.2' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html" From b0f3d921492870310ec50d240f5af397cca9b8d0 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sat, 2 Mar 2019 12:18:32 +0100 Subject: [PATCH 8/9] Don't use skip_unless_docker decorator This is handled in setupClass. Also drop unused skip_unless_singularity. --- test/base/integration_util.py | 4 ---- test/integration/test_containerized_jobs.py | 1 - 2 files changed, 5 deletions(-) diff --git a/test/base/integration_util.py b/test/base/integration_util.py index ef84e639fd5..158cf12168b 100644 --- a/test/base/integration_util.py +++ b/test/base/integration_util.py @@ -33,10 +33,6 @@ def skip_unless_docker(): return skip_unless_executable("docker") -def skip_unless_singularity(): - return skip_unless_executable("singularity") - - def skip_unless_kubernetes(): return skip_unless_executable("kubectl") diff --git a/test/integration/test_containerized_jobs.py b/test/integration/test_containerized_jobs.py index f784aa2931a..ccff74a7b7f 100644 --- a/test/integration/test_containerized_jobs.py +++ b/test/integration/test_containerized_jobs.py @@ -31,7 +31,6 @@ class MulledJobTestCases(object): assert "0.7.15-r1140" in output -@integration_util.skip_unless_docker() class DockerizedJobsIntegrationTestCase(integration_util.IntegrationTestCase, RunsEnvironmentJobs, MulledJobTestCases): framework_tool_and_types = True From d5f3efbf08b07bdc2891c7b1c713f6ba2abe7c6e Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 4 Mar 2019 11:38:11 +0100 Subject: [PATCH 9/9] Use new image from galaxy namespace --- run_tests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.sh b/run_tests.sh index b2b9d24ebc1..5634c1d7793 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -270,7 +270,7 @@ exists() { type "$1" >/dev/null 2>/dev/null } -DOCKER_DEFAULT_IMAGE='mvdbeek/testing-base:19.05.2' +DOCKER_DEFAULT_IMAGE='galaxy/testing-base:19.05.0' test_script="./scripts/functional_tests.py" report_file="run_functional_tests.html"