From c4a1ffa2daa56367cd6a99b60a621f03e061cfd8 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 3 Nov 2017 11:17:59 -0400 Subject: [PATCH 1/2] Lighter weight Selenium test setup for PRs. The Dockerfile and image update in #4900 baked chrome, chrome-wrapper, xvfb, and a Selenium server into the default Galaxy testing image. This means we should no longer need to run three docker containers in a compose setup to run Selenium tests. This swaps the script target used by Jenkins to use this newer variant of the tests. In addition to simply consuming less CPU and booting up much faster thanks to pre-installed dependencies and pre-migrated database, this setup is much easier to cleanup and so we don't need to restrict it to one test per host - these tests I think should run just like API and framework tests. This should also be easier for people running tests locally. I've kept the old compose setup around and I'll setup a Jenkins job against dev that continues to run it periodically (just not on PRs). It does testing of proxy prefix things this variant doesn't and serves as a good reference implementation for multi-container Galaxy testing - which we may wish to do for various categories of tests in the future. The multi-container variant makes it much easier to bring in various services - lots of which one can imagine writing useful Galaxy tests for - AMQP, statistics collection, Docker itself, etc.... --- .../{selenium => selenium-compose}/Dockerfile | 0 .../docker-compose.yml | 0 .../{selenium => selenium-compose}/galaxy.ini | 0 .../run_galaxy.bash | 0 .ci/jenkins/selenium-compose/run_tests.sh | 124 +++++++++++++++++ .ci/jenkins/selenium/run_tests.sh | 127 +----------------- run_tests.sh | 6 +- 7 files changed, 132 insertions(+), 125 deletions(-) rename .ci/jenkins/{selenium => selenium-compose}/Dockerfile (100%) rename .ci/jenkins/{selenium => selenium-compose}/docker-compose.yml (100%) rename .ci/jenkins/{selenium => selenium-compose}/galaxy.ini (100%) rename .ci/jenkins/{selenium => selenium-compose}/run_galaxy.bash (100%) create mode 100755 .ci/jenkins/selenium-compose/run_tests.sh mode change 100755 => 100644 .ci/jenkins/selenium/run_tests.sh diff --git a/.ci/jenkins/selenium/Dockerfile b/.ci/jenkins/selenium-compose/Dockerfile similarity index 100% rename from .ci/jenkins/selenium/Dockerfile rename to .ci/jenkins/selenium-compose/Dockerfile diff --git a/.ci/jenkins/selenium/docker-compose.yml b/.ci/jenkins/selenium-compose/docker-compose.yml similarity index 100% rename from .ci/jenkins/selenium/docker-compose.yml rename to .ci/jenkins/selenium-compose/docker-compose.yml diff --git a/.ci/jenkins/selenium/galaxy.ini b/.ci/jenkins/selenium-compose/galaxy.ini similarity index 100% rename from .ci/jenkins/selenium/galaxy.ini rename to .ci/jenkins/selenium-compose/galaxy.ini diff --git a/.ci/jenkins/selenium/run_galaxy.bash b/.ci/jenkins/selenium-compose/run_galaxy.bash similarity index 100% rename from .ci/jenkins/selenium/run_galaxy.bash rename to .ci/jenkins/selenium-compose/run_galaxy.bash diff --git a/.ci/jenkins/selenium-compose/run_tests.sh b/.ci/jenkins/selenium-compose/run_tests.sh new file mode 100755 index 00000000000..0a11bfc8898 --- /dev/null +++ b/.ci/jenkins/selenium-compose/run_tests.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +TEST_DIRECTORY=`dirname $0` + +DEFAULT_COMPOSE_PROJECT_NAME=`basename $TEST_DIRECTORY` +COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-$DEFAULT_COMPOSE_PROJECT_NAME} +# If in Jenkins environment, append ${BUILD_NUMBER} to project so builds don't interfer. +if [ ! -z "$BUILD_NUMBER" ]; +then + COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME}${BUILD_NUMBER}" +fi +export COMPOSE_PROJECT_NAME + +echo "Running Jenkins test from $TEST_DIRECTORY with compose project name $COMPOSE_PROJECT_NAME" + +find lib -iname '*pyc' -exec rm -rf {} \; +find test -iname '*pyc' -exec rm -rf {} \; + +./scripts/common_startup.sh --dev-wheels + +. .venv/bin/activate + +pip install docker-compose + +# TODO: Let docker-compose pick these at random. +export GALAXY_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` +export SELENIUM_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` + +export TARGET_ROOT=`pwd` +export TARGET_PATH=/galaxy +export MY_UID=$(id -u) + +cd $TEST_DIRECTORY + +echo "Cleaning up previous executions if needed." +docker-compose down | true +docker-compose build galaxy +docker-compose up -d + +function tear_down { + docker-compose down +} + +trap tear_down EXIT + +for service_name in postgres galaxy selenium +do + echo "Waiting on service ${service_name}" + while true + do + if docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}" + then + container_id=`docker ps | grep "${COMPOSE_PROJECT_NAME}_${service_name}" | cut -d " " -f 1` + eval "${service_name}_container_id=${container_id}" + echo "Service ${service_name} ready - with container ID ${container_id}" + break + fi + printf "." + sleep 1; + done +done + + +if [ "$1" = "--debug-running-containers" ]; +then + env + exit 0 +fi + +export GALAXY_TEST_EXTERNAL="http://localhost:$GALAXY_PORT/" + +echo "Waiting on docker-compose managed Galaxy server - $GALAXY_TEST_EXTERNAL." +while ! curl -s "$GALAXY_TEST_EXTERNAL"; +do + for service_name in postgres galaxy selenium + do + if ! docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}" + then + echo "Service ${service_name} stopped before Galaxy came up, exiting and halting containers." + for service_name in postgres galaxy selenium + do + container_id_var="${service_name}_container_id" + container_id="${!container_id_var}" + echo "Dumping logs for $service_name container (${container_id})..." + echo "---" + docker logs "${container_id}" + echo "---" + done + exit 1 + fi + done + + printf "." + sleep 4; +done; + +# Access Selenium on localhost via port $SELENIUM_PORT +export GALAXY_TEST_SELENIUM_REMOTE=1 +export GALAXY_TEST_SELENIUM_REMOTE_PORT="${SELENIUM_PORT}" + +# Retry all failed Selenium tests a second time to deal +# with transiently failing tests. Failure information for +# first tests is still populated in database/test_errors +# and available at the top of the Jenkins test report. +export GALAXY_TEST_SELENIUM_RETRIES=1 + +# Access Galaxy on localhost via port $GALAXY_PORT +export GALAXY_TEST_PORT="${GALAXY_PORT}" + +# Have Selenium access Galaxy at this URL +export GALAXY_TEST_EXTERNAL_FROM_SELENIUM="http://galaxy:8080/galaxypf" +export GALAXY_TEST_EXTERNAL="http://localhost:${GALAXY_TEST_PORT}/galaxypf" + +# Point tests at the Master API Key configured in the Dockerfile. +export GALAXY_CONFIG_MASTER_API_KEY=94a548bea347a35e457a804bf75bec53 + +cd ../../.. + +./run_tests.sh --selenium "$@" +exit_code=$? + +cd $TEST_DIRECTORY + +exit $exit_code diff --git a/.ci/jenkins/selenium/run_tests.sh b/.ci/jenkins/selenium/run_tests.sh old mode 100755 new mode 100644 index 0a11bfc8898..24df53232cf --- a/.ci/jenkins/selenium/run_tests.sh +++ b/.ci/jenkins/selenium/run_tests.sh @@ -1,124 +1,11 @@ #!/bin/bash -TEST_DIRECTORY=`dirname $0` +# Enable retries on tests to reduce chances of transient failures. +: ${GALAXY_TEST_SELENIUM_RETRIES:=1} +: ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors} -DEFAULT_COMPOSE_PROJECT_NAME=`basename $TEST_DIRECTORY` -COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-$DEFAULT_COMPOSE_PROJECT_NAME} -# If in Jenkins environment, append ${BUILD_NUMBER} to project so builds don't interfer. -if [ ! -z "$BUILD_NUMBER" ]; -then - COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME}${BUILD_NUMBER}" -fi -export COMPOSE_PROJECT_NAME +# Start Selenium server in the test Docker container. +DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} ${DOCKER_RUN_EXTRA_ARGS}" +export DOCKER_RUN_EXTRA_ARGS -echo "Running Jenkins test from $TEST_DIRECTORY with compose project name $COMPOSE_PROJECT_NAME" - -find lib -iname '*pyc' -exec rm -rf {} \; -find test -iname '*pyc' -exec rm -rf {} \; - -./scripts/common_startup.sh --dev-wheels - -. .venv/bin/activate - -pip install docker-compose - -# TODO: Let docker-compose pick these at random. -export GALAXY_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` -export SELENIUM_PORT=`python -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` - -export TARGET_ROOT=`pwd` -export TARGET_PATH=/galaxy -export MY_UID=$(id -u) - -cd $TEST_DIRECTORY - -echo "Cleaning up previous executions if needed." -docker-compose down | true -docker-compose build galaxy -docker-compose up -d - -function tear_down { - docker-compose down -} - -trap tear_down EXIT - -for service_name in postgres galaxy selenium -do - echo "Waiting on service ${service_name}" - while true - do - if docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}" - then - container_id=`docker ps | grep "${COMPOSE_PROJECT_NAME}_${service_name}" | cut -d " " -f 1` - eval "${service_name}_container_id=${container_id}" - echo "Service ${service_name} ready - with container ID ${container_id}" - break - fi - printf "." - sleep 1; - done -done - - -if [ "$1" = "--debug-running-containers" ]; -then - env - exit 0 -fi - -export GALAXY_TEST_EXTERNAL="http://localhost:$GALAXY_PORT/" - -echo "Waiting on docker-compose managed Galaxy server - $GALAXY_TEST_EXTERNAL." -while ! curl -s "$GALAXY_TEST_EXTERNAL"; -do - for service_name in postgres galaxy selenium - do - if ! docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}" - then - echo "Service ${service_name} stopped before Galaxy came up, exiting and halting containers." - for service_name in postgres galaxy selenium - do - container_id_var="${service_name}_container_id" - container_id="${!container_id_var}" - echo "Dumping logs for $service_name container (${container_id})..." - echo "---" - docker logs "${container_id}" - echo "---" - done - exit 1 - fi - done - - printf "." - sleep 4; -done; - -# Access Selenium on localhost via port $SELENIUM_PORT -export GALAXY_TEST_SELENIUM_REMOTE=1 -export GALAXY_TEST_SELENIUM_REMOTE_PORT="${SELENIUM_PORT}" - -# Retry all failed Selenium tests a second time to deal -# with transiently failing tests. Failure information for -# first tests is still populated in database/test_errors -# and available at the top of the Jenkins test report. -export GALAXY_TEST_SELENIUM_RETRIES=1 - -# Access Galaxy on localhost via port $GALAXY_PORT -export GALAXY_TEST_PORT="${GALAXY_PORT}" - -# Have Selenium access Galaxy at this URL -export GALAXY_TEST_EXTERNAL_FROM_SELENIUM="http://galaxy:8080/galaxypf" -export GALAXY_TEST_EXTERNAL="http://localhost:${GALAXY_TEST_PORT}/galaxypf" - -# Point tests at the Master API Key configured in the Dockerfile. -export GALAXY_CONFIG_MASTER_API_KEY=94a548bea347a35e457a804bf75bec53 - -cd ../../.. - -./run_tests.sh --selenium "$@" -exit_code=$? - -cd $TEST_DIRECTORY - -exit $exit_code +./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc --selenium "$@" diff --git a/run_tests.sh b/run_tests.sh index 496b0b20bd7..589c8e5d19b 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -285,10 +285,6 @@ then else db_type="sqlite" fi - if [ "$1" = "--selenium" ]; then - DOCKER_RUN_EXTRA_ARGS="-e USE_SELENIUM=1 ${DOCKER_RUN_EXTRA_ARGS}" - shift - fi if [ "$1" = "--external_tmp" ]; then # If /tmp is a tmpfs there may be better performance by reusing # the parent's temp file system. Also, it seems to decrease the @@ -301,7 +297,7 @@ then fi MY_UID=$(id -u) DOCKER_RUN_EXTRA_ARGS="-e GALAXY_TEST_UID=${MY_UID} ${DOCKER_RUN_EXTRA_ARGS}" - echo "Launching docker container for testing..." + echo "Launching docker container for testing with extra args ${DOCKER_RUN_EXTRA_ARGS}..." docker $DOCKER_EXTRA_ARGS run $DOCKER_RUN_EXTRA_ARGS -e "BUILD_NUMBER=$BUILD_NUMBER" -e "GALAXY_TEST_DATABASE_TYPE=$db_type" --rm -v `pwd`:/galaxy $DOCKER_IMAGE "$@" exit $? fi From 3a1ca26339b928c362d9f7b9f0db9cecca364c77 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 3 Nov 2017 19:32:15 -0400 Subject: [PATCH 2/2] Fix published histories grid test. --- .../selenium_tests/test_published_histories_grid.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/test/selenium_tests/test_published_histories_grid.py b/test/selenium_tests/test_published_histories_grid.py index 701da7ca04d..627673d1261 100644 --- a/test/selenium_tests/test_published_histories_grid.py +++ b/test/selenium_tests/test_published_histories_grid.py @@ -88,9 +88,16 @@ class HistoryGridTestCase(SharedStateSeleniumTestCase): if tags is None: raise AssertionError('Failed to find history with name [%s]' % self.history1_name) - tag_button_selector = '.tag-area > .tag-button:first-child > .tag-name' - tag_button = tags.find_element_by_css_selector(tag_button_selector) - self.assertEqual(tag_button.text, self.history1_tags[0]) + tag_button_selector = '.tag-area > .tag-button > .tag-name' + tag_buttons = tags.find_elements_by_css_selector(tag_button_selector) + tag_button_text = None + target_tag_button_text = self.history1_tags[0] + for tag_button in tag_buttons: + tag_button_text = tag_button.text + if tag_button_text == target_tag_button_text: + break + + self.assertEqual(tag_button_text, target_tag_button_text) tag_button.click() self.assert_grid_histories_are([self.history1_name, self.history3_name], False)