diff --git a/.ci/jenkins/api-py3 b/.ci/jenkins/api-py3 deleted file mode 120000 index 9e5bfb42d23..00000000000 --- a/.ci/jenkins/api-py3 +++ /dev/null @@ -1 +0,0 @@ -api \ No newline at end of file diff --git a/.ci/jenkins/api/run_tests.sh b/.ci/jenkins/api/run_tests.sh deleted file mode 100755 index ea0f7c1cfb3..00000000000 --- a/.ci/jenkins/api/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --db postgres --clean_pyc --skip_flakey_fails -api "$@" diff --git a/.ci/jenkins/framework-py3 b/.ci/jenkins/framework-py3 deleted file mode 120000 index e3939d229ce..00000000000 --- a/.ci/jenkins/framework-py3 +++ /dev/null @@ -1 +0,0 @@ -framework \ No newline at end of file diff --git a/.ci/jenkins/framework/run_tests.sh b/.ci/jenkins/framework/run_tests.sh deleted file mode 100755 index df1bad416a3..00000000000 --- a/.ci/jenkins/framework/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --db postgres --clean_pyc --framework "$@" diff --git a/.ci/jenkins/integration-py3 b/.ci/jenkins/integration-py3 deleted file mode 120000 index c0bfe0d4f94..00000000000 --- a/.ci/jenkins/integration-py3 +++ /dev/null @@ -1 +0,0 @@ -integration \ No newline at end of file diff --git a/.ci/jenkins/integration/run_tests.sh b/.ci/jenkins/integration/run_tests.sh deleted file mode 100755 index 4119aef60b2..00000000000 --- a/.ci/jenkins/integration/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -DOCKER_RUN_EXTRA_ARGS="--privileged" ./run_tests.sh --dockerize --db postgres --clean_pyc --integration "$@" diff --git a/.ci/jenkins/main-tools-py3 b/.ci/jenkins/main-tools-py3 deleted file mode 120000 index 49589fdd3a8..00000000000 --- a/.ci/jenkins/main-tools-py3 +++ /dev/null @@ -1 +0,0 @@ -main-tools \ No newline at end of file diff --git a/.ci/jenkins/main-tools/run_tests.sh b/.ci/jenkins/main-tools/run_tests.sh deleted file mode 100755 index cb8a0d27a7f..00000000000 --- a/.ci/jenkins/main-tools/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --db postgres --clean_pyc -main "$@" diff --git a/.ci/jenkins/selenium-compose/Dockerfile b/.ci/jenkins/selenium-compose/Dockerfile deleted file mode 100644 index d9f3def9761..00000000000 --- a/.ci/jenkins/selenium-compose/Dockerfile +++ /dev/null @@ -1,24 +0,0 @@ -FROM toolshed/requirements -LABEL maintainer="John Chilton " - -RUN apt-get -qq update && \ - apt-get install --no-install-recommends -y postgresql-client python-pip libffi-dev python-cffi && \ - apt-get autoremove -y && apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -ENV GALAXY_ROOT=/galaxy \ - GALAXY_CONFIG_OVERRIDE_ADMIN_USERS=admin@galaxy.org,test@bx.psu.edu \ - GALAXY_CONFIG_OVERRIDE_ALLOW_USER_DATASET_PURGE=true \ - GALAXY_CONFIG_OVERRIDE_ALLOW_LIBRARY_PATH_PASTE=true \ - GALAXY_CONFIG_OVERRIDE_ENABLE_BETA_WORKFLOW_MODULES=true \ - GALAXY_CONFIG_OVERRIDE_ALLOW_USER_DELETION=true \ - GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=postgresql://postgres@postgres/galaxy \ - GALAXY_CONFIG_OVERRIDE_MASTER_API_KEY=94a548bea347a35e457a804bf75bec53 \ - GALAXY_CONFIG_OVERRIDE_JOB_CONFIG_FILE=/galaxy/config/job_conf.xml.sample_basic \ - GALAXY_CONFIG_OVERRIDE_DATATYPES_CONFIG_FILE=/galaxy/test/functional/tools/sample_datatypes_conf.xml \ - GALAXY_CONFIG_OVERRIDE_TOOL_CONFIG_FILE=/galaxy/test/functional/tools/samples_tool_conf.xml \ - GALAXY_CONFIG_FILE=/galaxy/.ci/jenkins/selenium/galaxy.ini \ - GALAXY_VIRTUAL_ENV=/galaxy_env - -ADD run_galaxy.bash /run_galaxy.bash -EXPOSE 8080 -CMD /run_galaxy.bash diff --git a/.ci/jenkins/selenium-compose/docker-compose.yml b/.ci/jenkins/selenium-compose/docker-compose.yml deleted file mode 100644 index 152e0cf55fc..00000000000 --- a/.ci/jenkins/selenium-compose/docker-compose.yml +++ /dev/null @@ -1,21 +0,0 @@ -version: '2' -services: - postgres: - image: postgres - galaxy: - build: . - image: ubuntu - volumes: - - "${TARGET_ROOT}:${TARGET_PATH}" - links: - - postgres - ports: - - "${GALAXY_PORT}:8080" - environment: - GALAXY_TEST_UID: "${MY_UID}" - selenium: - image: selenium/standalone-chrome:3.5.2 - ports: - - "${SELENIUM_PORT}:4444" - links: - - galaxy diff --git a/.ci/jenkins/selenium-compose/galaxy.ini b/.ci/jenkins/selenium-compose/galaxy.ini deleted file mode 100644 index 49599359d96..00000000000 --- a/.ci/jenkins/selenium-compose/galaxy.ini +++ /dev/null @@ -1,17 +0,0 @@ -[server:main] - -use = egg:Paste#http -port = 8080 -host = 0.0.0.0 -use_threadpool = True -threadpool_kill_thread_limit = 10800 - -[filter:proxy-prefix] -use = egg:PasteDeploy#prefix -prefix = /galaxypf - -[app:main] - -paste.app_factory = galaxy.web.buildapp:app_factory - -filter-with = proxy-prefix \ No newline at end of file diff --git a/.ci/jenkins/selenium-compose/run_galaxy.bash b/.ci/jenkins/selenium-compose/run_galaxy.bash deleted file mode 100755 index c16901324c1..00000000000 --- a/.ci/jenkins/selenium-compose/run_galaxy.bash +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/bash - -set -e - -# Same hacks to setup a Galaxy user as used in test/docker/base/run_test_wrapper.sh -# We want to make sure Galaxy runs as the same user as the external user so the files have the correct permission. -echo "Deleting galaxy user - it may not exist and this is fine." -deluser galaxy | true - -: ${GALAXY_TEST_UID:-"1"} - -echo "Creating galaxy group with gid $GALAXY_TEST_UID - it may already exist and this is fine." -groupadd -r galaxy -g "$GALAXY_TEST_UID" | true -echo "Creating galaxy user with uid $GALAXY_TEST_UID - it may already exist and this is fine." -useradd -u $GALAXY_TEST_UID -r -g galaxy -d /home/galaxy -c "Galaxy User" galaxy -s /bin/bash | true -echo "Setting galaxy user password - the operation may fail." -echo "galaxy:galaxy" | chpasswd | true - -virtualenv "$GALAXY_VIRTUAL_ENV" -chown -R "$GALAXY_TEST_UID:$GALAXY_TEST_UID" "$GALAXY_VIRTUAL_ENV" - -cd /galaxy -sudo -E -H -u "#${GALAXY_TEST_UID}" ./scripts/common_startup.sh || { echo "common_startup.sh failed"; exit 1; } - -echo "Waiting for postgres to become available" -while ! nc -z postgres 5432; -do - sleep 1 - printf "." -done - -echo "Creating postgres database for Galaxy" -createdb -w -U postgres -h postgres galaxy - -echo "Starting and waiting for Galaxy daemon(s)" -sudo -E -H -u "#${GALAXY_TEST_UID}" GALAXY_RUN_ALL=1 bash "$GALAXY_ROOT/run.sh" --daemon --wait - -echo "Galaxy daemon ready, monitoring Galaxy logs" -tail -f "$GALAXY_ROOT/main.log" diff --git a/.ci/jenkins/selenium-compose/run_tests.sh b/.ci/jenkins/selenium-compose/run_tests.sh deleted file mode 100755 index f8bd2e4d832..00000000000 --- a/.ci/jenkins/selenium-compose/run_tests.sh +++ /dev/null @@ -1,124 +0,0 @@ -#!/bin/bash - -TEST_DIRECTORY=`dirname $0` - -DEFAULT_COMPOSE_PROJECT_NAME=`basename $TEST_DIRECTORY` -COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-$DEFAULT_COMPOSE_PROJECT_NAME} -# If in Jenkins environment, append ${BUILD_NUMBER} to project so builds don't interfer. -if [ ! -z "$BUILD_NUMBER" ]; -then - COMPOSE_PROJECT_NAME="${COMPOSE_PROJECT_NAME}${BUILD_NUMBER}" -fi -export COMPOSE_PROJECT_NAME - -echo "Running Jenkins test from $TEST_DIRECTORY with compose project name $COMPOSE_PROJECT_NAME" - -find lib -iname '*pyc' -exec rm -rf {} \; -find test -iname '*pyc' -exec rm -rf {} \; - -./scripts/common_startup.sh --dev-wheels - -. .venv/bin/activate - -pip install docker-compose - -# TODO: Let docker-compose pick these at random. -export GALAXY_PORT=`python -c 'from __future__ import print_function; import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` -export SELENIUM_PORT=`python -c 'from __future__ import print_function; import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'` - -export TARGET_ROOT=`pwd` -export TARGET_PATH=/galaxy -export MY_UID=$(id -u) - -cd $TEST_DIRECTORY - -echo "Cleaning up previous executions if needed." -docker-compose down | true -docker-compose build galaxy -docker-compose up -d - -function tear_down { - docker-compose down -} - -trap tear_down EXIT - -for service_name in postgres galaxy selenium -do - echo "Waiting on service ${service_name}" - while true - do - if docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}" - then - container_id=`docker ps | grep "${COMPOSE_PROJECT_NAME}_${service_name}" | cut -d " " -f 1` - eval "${service_name}_container_id=${container_id}" - echo "Service ${service_name} ready - with container ID ${container_id}" - break - fi - printf "." - sleep 1; - done -done - - -if [ "$1" = "--debug-running-containers" ]; -then - env - exit 0 -fi - -export GALAXY_TEST_EXTERNAL="http://localhost:$GALAXY_PORT/" - -echo "Waiting on docker-compose managed Galaxy server - $GALAXY_TEST_EXTERNAL." -while ! curl -s "$GALAXY_TEST_EXTERNAL"; -do - for service_name in postgres galaxy selenium - do - if ! docker ps | grep -q "${COMPOSE_PROJECT_NAME}_${service_name}" - then - echo "Service ${service_name} stopped before Galaxy came up, exiting and halting containers." - for service_name in postgres galaxy selenium - do - container_id_var="${service_name}_container_id" - container_id="${!container_id_var}" - echo "Dumping logs for $service_name container (${container_id})..." - echo "---" - docker logs "${container_id}" - echo "---" - done - exit 1 - fi - done - - printf "." - sleep 4; -done; - -# Access Selenium on localhost via port $SELENIUM_PORT -export GALAXY_TEST_SELENIUM_REMOTE=1 -export GALAXY_TEST_SELENIUM_REMOTE_PORT="${SELENIUM_PORT}" - -# Retry all failed Selenium tests a second time to deal -# with transiently failing tests. Failure information for -# first tests is still populated in database/test_errors -# and available at the top of the Jenkins test report. -export GALAXY_TEST_SELENIUM_RETRIES=1 - -# Access Galaxy on localhost via port $GALAXY_PORT -export GALAXY_TEST_PORT="${GALAXY_PORT}" - -# Have Selenium access Galaxy at this URL -export GALAXY_TEST_EXTERNAL_FROM_SELENIUM="http://galaxy:8080/galaxypf" -export GALAXY_TEST_EXTERNAL="http://localhost:${GALAXY_TEST_PORT}/galaxypf" - -# Point tests at the Master API Key configured in the Dockerfile. -export GALAXY_CONFIG_MASTER_API_KEY=94a548bea347a35e457a804bf75bec53 - -cd ../../.. - -./run_tests.sh --selenium "$@" -exit_code=$? - -cd $TEST_DIRECTORY - -exit $exit_code diff --git a/.ci/jenkins/selenium-py3 b/.ci/jenkins/selenium-py3 deleted file mode 120000 index 954f0db0bc0..00000000000 --- a/.ci/jenkins/selenium-py3 +++ /dev/null @@ -1 +0,0 @@ -selenium \ No newline at end of file diff --git a/.ci/jenkins/selenium/run_tests.sh b/.ci/jenkins/selenium/run_tests.sh deleted file mode 100755 index 30840006815..00000000000 --- a/.ci/jenkins/selenium/run_tests.sh +++ /dev/null @@ -1,26 +0,0 @@ -#!/bin/bash - -# Enable retries on tests to reduce chances of transient failures. -: ${GALAXY_TEST_SELENIUM_RETRIES:=1} - -# If in Jenkins environment, use it for artifacts. -if [ -n "$BUILD_NUMBER" ]; -then - : ${GALAXY_TEST_ERRORS_DIRECTORY:=${BUILD_NUMBER}-test-errors} - : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=${BUILD_NUMBER}-test-screenshots} -else - : ${GALAXY_TEST_ERRORS_DIRECTORY:=database/test-errors} - : ${GALAXY_TEST_SCREENSHOTS_DIRECTORY:=database/test-screenshots} -fi - -mkdir -p "$GALAXY_TEST_ERRORS_DIRECTORY" -mkdir -p "$GALAXY_TEST_SCREENSHOTS_DIRECTORY" - -mkdir -p ~/.jenkins-yarn-cache -YARN_CACHE_FOLDER=~/.jenkins-yarn-cache - -# Start Selenium server in the test Docker container. -DOCKER_RUN_EXTRA_ARGS="${DOCKER_RUN_EXTRA_ARGS} --shm-size=2g -v $YARN_CACHE_FOLDER:$YARN_CACHE_FOLDER -e YARN_CACHE_FOLDER=$YARN_CACHE_FOLDER -e USE_SELENIUM=1 -e GALAXY_TEST_SELENIUM_RETRIES=${GALAXY_TEST_SELENIUM_RETRIES} -e GALAXY_TEST_ERRORS_DIRECTORY=${GALAXY_TEST_ERRORS_DIRECTORY} -e GALAXY_TEST_SCREENSHOTS_DIRECTORY=${GALAXY_TEST_SCREENSHOTS_DIRECTORY}" -export DOCKER_RUN_EXTRA_ARGS - -./run_tests.sh --dockerize --db postgres --clean_pyc --skip_flakey_fails --selenium "$@" diff --git a/.ci/jenkins/toolshed/run_tests.sh b/.ci/jenkins/toolshed/run_tests.sh deleted file mode 100755 index c250041c862..00000000000 --- a/.ci/jenkins/toolshed/run_tests.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash - -./run_tests.sh --dockerize --db postgres --clean_pyc --toolshed "$@" diff --git a/.ci/minikube-test-setup/deployment.yaml b/.ci/minikube-test-setup/deployment.yaml new file mode 100644 index 00000000000..d7afe110c83 --- /dev/null +++ b/.ci/minikube-test-setup/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: testing + name: testing +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: test + template: + metadata: + labels: + app.kubernetes.io/name: test + spec: + containers: + - image: postgres:12 + name: postgres + ports: + - containerPort: 5432 + env: + - name: POSTGRES_DB + value: postgres + - name: POSTGRES_USER + value: postgres + - name: POSTGRES_PASSWORD + value: postgres + - image: rabbitmq + name: rabbitmq + ports: + - containerPort: 5672 diff --git a/.ci/minikube-test-setup/start_services.sh b/.ci/minikube-test-setup/start_services.sh new file mode 100644 index 00000000000..1e860dadcb7 --- /dev/null +++ b/.ci/minikube-test-setup/start_services.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -ex + +SCRIPTDIR=$(dirname "${BASH_SOURCE[0]}") +kubectl apply -f "$SCRIPTDIR/deployment.yaml" +kubectl expose deployment testing --type=LoadBalancer --name=testing-service + +CLUSTER_IP=$(kubectl get service testing-service -o jsonpath='{.spec.clusterIP}') +GALAXY_TEST_DBURI="postgresql://postgres:postgres@${CLUSTER_IP}:5432/galaxy?client_encoding=utf-8" +GALAXY_TEST_AMQP_URL="amqp://${CLUSTER_IP}:5672)//" +export GALAXY_TEST_DBURI +export GALAXY_TEST_AMQP_URL diff --git a/.circleci/config.yml b/.circleci/config.yml index 9ecb661653b..bbd1a95329e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -83,14 +83,6 @@ jobs: key: v1-repo-{{ .Environment.CIRCLE_SHA1 }} paths: - ~/repo - py36_docstring: - docker: - - image: circleci/python:3.6 - <<: *set_workdir - steps: - - *restore_repo_cache - - *install_tox - - run: tox -e py36-lint_docstring_include_list py36_unit: docker: - image: circleci/python:3.6 @@ -100,17 +92,6 @@ jobs: - *install_tox - *install_ffprobe - run: tox -e py36-unit - py36_first_startup: - docker: - - image: circleci/python:3.6 - <<: *set_workdir - steps: - - *restore_repo_cache - # Use this job to test the latest migrations - - run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0141.sqlite - - run: mv db_gx_rev_0141.sqlite database/universe.sqlite - - *install_tox - - run: tox -e py36-first_startup validate_test_tools: docker: - image: circleci/python:3.6 @@ -159,12 +140,8 @@ workflows: get_code_and_test: jobs: - get_code - - py36_docstring: - <<: *requires_get_code - py36_unit: <<: *requires_get_code - - py36_first_startup: - <<: *requires_get_code - test_galaxy_packages: <<: *requires_get_code - validate_test_tools: diff --git a/.github/workflows/converter_tests.yaml b/.github/workflows/converter_tests.yaml index 8b20d095f3a..a71ef00c274 100644 --- a/.github/workflows/converter_tests.yaml +++ b/.github/workflows/converter_tests.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/db_indexes.yaml b/.github/workflows/db_indexes.yaml index 846f2cc5956..df9b8573260 100644 --- a/.github/workflows/db_indexes.yaml +++ b/.github/workflows/db_indexes.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] db: ['postgresql', 'sqlite'] services: postgres: diff --git a/.github/workflows/first_startup.yaml b/.github/workflows/first_startup.yaml new file mode 100644 index 00000000000..b2bd7762a76 --- /dev/null +++ b/.github/workflows/first_startup.yaml @@ -0,0 +1,35 @@ +name: first startup +on: [push, pull_request] +jobs: + + test: + name: Startup test + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.6'] + defaults: + run: + shell: bash -l {0} + steps: + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip dir + uses: actions/cache@v1 + id: pip-cache + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Install tox + run: pip install tox + # Use this job to test the latest migrations + - run: wget -q https://github.com/jmchilton/galaxy-downloads/raw/master/db_gx_rev_0141.sqlite + - run: mv db_gx_rev_0141.sqlite 'galaxy root'/database/universe.sqlite + - name: run tests + run: tox -e first_startup + working-directory: 'galaxy root' diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 30c3e1f8174..5bce55d5b3e 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -6,14 +6,15 @@ env: jobs: test: name: Test - runs-on: ubuntu-18.04 + runs-on: ubuntu-latest strategy: + fail-fast: false matrix: - python-version: [3.7] + python-version: ['3.7'] subset: ['upload_datatype', 'extended_metadata', 'kubernetes', 'not (upload_datatype or extended_metadata or kubernetes)'] services: postgres: - image: postgres:11 + image: postgres:13 env: POSTGRES_USER: postgres POSTGRES_PASSWORD: postgres @@ -59,8 +60,15 @@ jobs: run: sudo apt-get update && sudo apt-get install ffmpeg -y if: matrix.subset == 'upload_datatype' - name: Run tests + if: matrix.subset != 'kubernetes' run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' working-directory: 'galaxy root' + - name: Run tests + if: matrix.subset == 'kubernetes' + run: | + . .ci/minikube-test-setup/start_services.sh + ./run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}" + working-directory: 'galaxy root' - uses: actions/upload-artifact@v2 if: failure() with: diff --git a/.github/workflows/integration_selenium.yaml b/.github/workflows/integration_selenium.yaml index 01d87275554..0bfe9859ea4 100644 --- a/.github/workflows/integration_selenium.yaml +++ b/.github/workflows/integration_selenium.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] services: postgres: image: postgres:11 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 211ad869473..d29868062e6 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -6,7 +6,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.9] + python-version: ['3.6', '3.9'] steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 @@ -16,10 +16,12 @@ jobs: uses: actions/cache@v2 with: path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }} - name: Install tox run: pip install tox - - name: run tests + - name: Run linting run: tox -e lint - - name: run mypy checks + - name: Run docstring linting + run: tox -e py36-lint_docstring_include_list + - name: Run mypy checks run: tox -e mypy diff --git a/.github/workflows/maintenance_bot.yaml b/.github/workflows/maintenance_bot.yaml index 0083d243545..3debbce18c9 100644 --- a/.github/workflows/maintenance_bot.yaml +++ b/.github/workflows/maintenance_bot.yaml @@ -1,7 +1,7 @@ name: "Maintenance Bot" on: pull_request_target: - types: [opened, reopened] + types: [opened, reopened, edited, ready_for_review, unlabeled] jobs: labeler: @@ -10,8 +10,21 @@ jobs: env: MILESTONE_NUMBER: 19 steps: + - name: Get latest pull request labels + id: get_pr_labels + uses: actions/github-script@v3 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const response = await github.issues.listLabelsOnIssue({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + console.log(response); + return response.data; - name: Add area labels - if: ${{ ! contains(join(github.event.pull_request.labels.*.name, ', '), 'area/') }} + if: ${{ ! contains(join(fromJSON(steps.get_pr_labels.outputs.result).*.name, ', '), 'area/') }} uses: actions/labeler@main with: repo-token: "${{ secrets.GITHUB_TOKEN }}" diff --git a/.github/workflows/mulled.yaml b/.github/workflows/mulled.yaml index 1bdcd1dfde0..69c78be283b 100644 --- a/.github/workflows/mulled.yaml +++ b/.github/workflows/mulled.yaml @@ -7,7 +7,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] steps: - uses: actions/checkout@v2 with: diff --git a/.github/workflows/performance.yaml b/.github/workflows/performance.yaml new file mode 100644 index 00000000000..0c2a97c9a61 --- /dev/null +++ b/.github/workflows/performance.yaml @@ -0,0 +1,44 @@ +name: Performance tests +on: [push, pull_request] +env: + GALAXY_TEST_DBURI: 'postgres://postgres:postgres@localhost:5432/galaxy?client_encoding=utf8' +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ['3.7'] + services: + postgres: + image: postgres:11 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: postgres + ports: + - 5432:5432 + steps: + - uses: actions/checkout@v2 + with: + path: 'galaxy root' + - uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip dir + uses: actions/cache@v2 + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Run tests + run: ./run_tests.sh --ci_test_metrics --structured_data_html --structured_data_report_file "test.json" --skip_flakey_fails -api lib/galaxy_test/performance + working-directory: 'galaxy root' + - uses: actions/upload-artifact@v2 + if: failure() + with: + name: API test results + path: 'galaxy root/run_api_tests.html' + - uses: actions/upload-artifact@v2.0.1 + with: + name: 'Performance Metrics' + path: 'galaxy root/test.html' diff --git a/.github/workflows/toolshed.yaml b/.github/workflows/toolshed.yaml index 305a236bd88..5d260077cc3 100644 --- a/.github/workflows/toolshed.yaml +++ b/.github/workflows/toolshed.yaml @@ -9,7 +9,7 @@ jobs: runs-on: ubuntu-18.04 strategy: matrix: - python-version: [3.7] + python-version: ['3.7'] services: postgres: image: postgres:11 diff --git a/client/package.json b/client/package.json index 7f3ac4610ed..44eaae21e1a 100644 --- a/client/package.json +++ b/client/package.json @@ -74,15 +74,16 @@ "underscore": "^1.10.2", "underscore.string": "^3.3.5", "vue": "^2.6.11", - "vuedraggable": "2.24.1", "vue-infinite-scroll": "^2.0.2", "vue-multiselect": "^2.1.0", "vue-router": "^3.3.1", "vue-rx": "^6.2.0", + "vuedraggable": "2.24.1", "vueisotope": "^3.1.2", "vuex": "^3.4.0", "vuex-cache": "^3.2.0", - "vuex-persist": "^3.1.3" + "vuex-persist": "^3.1.3", + "vue-observe-visibility": "^1.0.0" }, "scripts": { "watch": "gulp && yarn run save-build-hash && yarn run webpack-watch", diff --git a/client/src/components/Common/SlugInput.vue b/client/src/components/Common/SlugInput.vue index 6b102344fec..27c098eaa22 100644 --- a/client/src/components/Common/SlugInput.vue +++ b/client/src/components/Common/SlugInput.vue @@ -18,6 +18,7 @@ export default { props: { slug: { type: String, + required: true, }, }, data() { diff --git a/client/src/components/DataDialog/DataDialog.vue b/client/src/components/DataDialog/DataDialog.vue index f1b498fcde5..a123cda6429 100644 --- a/client/src/components/DataDialog/DataDialog.vue +++ b/client/src/components/DataDialog/DataDialog.vue @@ -49,7 +49,7 @@ import { UrlTracker } from "./utilities"; import { Model } from "./model"; import { Services } from "./services"; import { getAppRoot } from "onload/loadConfig"; -import { mountUploadModal } from "components/Upload"; +import { openUploadModal } from "components/Upload"; Vue.use(BootstrapVue); @@ -135,7 +135,7 @@ export default { callback: this.callback, modalShow: true, }; - mountUploadModal(propsData); + openUploadModal(propsData); this.modalShow = false; }, /** Called when selection is complete, values are formatted and parsed to external callback **/ diff --git a/client/src/components/Dataset/DatasetIndex/DatasetIndex.vue b/client/src/components/Dataset/DatasetIndex/DatasetIndex.vue index 762e014b574..ef995f7bf5a 100644 --- a/client/src/components/Dataset/DatasetIndex/DatasetIndex.vue +++ b/client/src/components/Dataset/DatasetIndex/DatasetIndex.vue @@ -46,10 +46,7 @@ export default { pathDestination.filepath ); } else if (this.path === undefined || this.path === "undefined") { - this.directoryContent = this.removeParentDirectory( - pathDestination.datasetContent, - pathDestination.datasetRootDir - ); + this.directoryContent = pathDestination.datasetContent; } else { this.errorMessage = `is not found!`; } diff --git a/client/src/components/Dataset/compositeDatasetUtils.js b/client/src/components/Dataset/compositeDatasetUtils.js index 67a0bdf5f27..9c83c561314 100644 --- a/client/src/components/Dataset/compositeDatasetUtils.js +++ b/client/src/components/Dataset/compositeDatasetUtils.js @@ -6,15 +6,11 @@ export const getPathDestination = async (history_dataset_id, path) => { const services = new Services({ root: getAppRoot() }); const computePathDestination = (pathDestination) => { - if (pathDestination.datasetContent[0].class === "Directory") - pathDestination.datasetRootDir = datasetContent[0].path; - else return; - if (path === undefined || path === "undefined") { return pathDestination; } - const filepath = `${pathDestination.datasetRootDir}/${path}`; + const filepath = path; const datasetEntry = datasetContent.find((datasetEntry) => { return filepath === datasetEntry.path; diff --git a/client/src/components/DebouncedInput.js b/client/src/components/DebouncedInput.js index dd3e1cba62f..03a4ee9de5f 100644 --- a/client/src/components/DebouncedInput.js +++ b/client/src/components/DebouncedInput.js @@ -2,15 +2,9 @@ * Renderless component, used to debounce various types of form inputs */ -import Vue from "vue"; -import VueRx from "vue-rx"; import { filter, debounceTime, distinctUntilChanged, finalize } from "rxjs/operators"; -import { vueRxShortcuts } from "./plugins"; - -Vue.use(VueRx); export default { - mixins: [vueRxShortcuts], props: { value: { required: true }, delay: { type: Number, required: false, default: 500 }, diff --git a/client/src/components/History/ContentItem/CollectionContentItem.js b/client/src/components/History/ContentItem/CollectionContentItem.js index e88605af117..efa92a9083a 100644 --- a/client/src/components/History/ContentItem/CollectionContentItem.js +++ b/client/src/components/History/ContentItem/CollectionContentItem.js @@ -5,12 +5,6 @@ export default { computed: { contentItemComponent() { - if (this.item === null) { - return "Loading"; - } - if (this.scrolling) { - return "Placeholder"; - } const { history_content_type } = this.item; switch (history_content_type) { case "dataset": diff --git a/client/src/components/History/ContentItem/ContentItem.js b/client/src/components/History/ContentItem/ContentItem.js index afe62bf8e10..0adfea54b42 100644 --- a/client/src/components/History/ContentItem/ContentItem.js +++ b/client/src/components/History/ContentItem/ContentItem.js @@ -11,22 +11,17 @@ import Placeholder from "./Placeholder"; import Dataset from "./Dataset"; import DatasetCollection from "./DatasetCollection"; +// import SubDataset from "./SubDataset"; import Subcollection from "./Subcollection"; export default { - inject: ["listState", "isSelected", "isExpanded", "setSelected", "setExpanded"], - template: ` + + + + diff --git a/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue b/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue index bc96b65eba6..3982752bf4a 100644 --- a/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue +++ b/client/src/components/History/ContentItem/Dataset/DatasetMenu.vue @@ -72,6 +72,14 @@ icon="fas fa-file-download" /> + +
import { mapGetters } from "vuex"; -import { Dataset } from "../../model"; +import { Dataset, STATES } from "../../model"; import { PriorityMenu, PriorityMenuItem } from "components/PriorityMenu"; -import { legacyNavigationMixin } from "components/plugins"; +import { legacyNavigationMixin } from "components/plugins/legacyNavigation"; export default { - inject: ["STATES"], mixins: [legacyNavigationMixin], components: { @@ -156,6 +163,11 @@ export default { }; }, + created() { + // make available to template + this.STATES = STATES; + }, + computed: { ...mapGetters("user", ["currentUser"]), ...mapGetters("config", ["config"]), @@ -168,10 +180,10 @@ export default { if (this.dataset.purged) { return "Cannot display datasets removed from disk"; } - if (this.dataset.state == this.STATES.UPLOAD) { + if (this.dataset.state == STATES.UPLOAD) { return "This dataset must finish uploading before it can be viewed"; } - if (this.dataset.state == this.STATES.NEW) { + if (this.dataset.state == STATES.NEW) { return "This dataset is not yet viewable"; } return "View data"; @@ -184,7 +196,7 @@ export default { if (this.dataset.purged) { return "Cannot edit attributes of datasets removed from disk"; } - const unreadyStates = new Set([this.STATES.UPLOAD, this.STATES.NEW]); + const unreadyStates = new Set([STATES.UPLOAD, STATES.NEW]); if (unreadyStates.has(this.dataset.state)) { return "This dataset is not yet editable"; } @@ -206,7 +218,7 @@ export default { if (!this.dataset.hasData) { return false; } - const okStates = new Set([this.STATES.OK, this.STATES.FAILED_METADATA, this.STATES.ERROR]); + const okStates = new Set([STATES.OK, STATES.FAILED_METADATA, STATES.ERROR]); return okStates.has(this.dataset.state); }, @@ -261,9 +273,8 @@ export default { }, onDeleteClick() { - const eventName = this.dataset.deleted ? "undeleteDataset" : "deleteDataset"; - // console.log("emitting", eventName, this.dataset); - this.$emit(eventName, this.dataset); + const eventName = this.dataset.deleted ? "undelete" : "delete"; + this.$emit(eventName); }, }, }; diff --git a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue index 8314d8c6d92..36c2da3b28e 100644 --- a/client/src/components/History/ContentItem/Dataset/DatasetUI.vue +++ b/client/src/components/History/ContentItem/Dataset/DatasetUI.vue @@ -14,6 +14,7 @@ either through the props, and make updates through the events --> @keydown.arrow-right.self.stop="$emit('update:expanded', true)" @keydown.space.self.stop.prevent="$emit('update:selected', !selected)" > +
@@ -46,25 +46,30 @@ either through the props, and make updates through the events --> {{ dataset.hid }} {{ dataset.title }} - - + + + + +
+ +
+ +
- - - + +
+ +
@@ -102,20 +108,16 @@ either through the props, and make updates through the events --> diff --git a/client/src/components/History/ContentItem/DatasetCollection/DatasetCollection.vue b/client/src/components/History/ContentItem/DatasetCollection/DatasetCollection.vue new file mode 100644 index 00000000000..86e54a534df --- /dev/null +++ b/client/src/components/History/ContentItem/DatasetCollection/DatasetCollection.vue @@ -0,0 +1,57 @@ + + + diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue b/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue index 6a52cc1a6e1..fbe90484749 100644 --- a/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue +++ b/client/src/components/History/ContentItem/DatasetCollection/DscMenu.vue @@ -5,13 +5,13 @@ Delete Collection - Delete Collection Only + Delete Collection Only - + Delete Contained Datasets - + Purge Contained Datasets diff --git a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue index 57a6cd1966b..25393e98d18 100644 --- a/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue +++ b/client/src/components/History/ContentItem/DatasetCollection/DscUI.vue @@ -6,12 +6,9 @@ class="dataset dataset-collection collapsed" :class="{ selected }" :data-state="dsc.state" - @keydown.arrow-right.self.stop="$emit('select-collection', dsc)" + @keydown.arrow-right.self.stop="$emit('viewCollection')" @keydown.space.self.stop.prevent="$emit('update:selected', !selected)" - @click.stop=" - $emit('select-collection', dsc); - $emit('update:expanded', dsc); - " + @click.stop="$emit('viewCollection')" > + +
+ +
+ -
No summary
@@ -71,27 +75,20 @@ import { DatasetCollection } from "../../model/DatasetCollection"; import { StatusIcon, StateBtn } from "../../StatusIcon"; import JobStateProgress from "./JobStateProgress"; import DscMenu from "./DscMenu"; +import { Nametag } from "components/Nametags"; export default { - inject: ["listState", "STATES"], components: { StatusIcon, StateBtn, JobStateProgress, DscMenu, + Nametag, }, props: { dsc: { type: DatasetCollection, required: true }, selected: { type: Boolean, required: false, default: false }, - showHid: { type: Boolean, required: false, default: true }, - }, - computed: { - counter() { - return this.showHid ? this.dsc.hid : ""; - }, - showSelection() { - return this.listState.showSelection; - }, + showSelection: { type: Boolean, required: false, default: false }, }, methods: { onStatusClick() { diff --git a/client/src/components/History/ContentItem/DatasetCollection/index.js b/client/src/components/History/ContentItem/DatasetCollection/index.js new file mode 100644 index 00000000000..4d7f796c22b --- /dev/null +++ b/client/src/components/History/ContentItem/DatasetCollection/index.js @@ -0,0 +1,6 @@ +import DatasetCollection from "./DatasetCollection"; + +export { default as DscUI } from "./DscUI"; +export { default as DscMenu } from "./DscMenu"; +export { default as JobStateProgress } from "./JobStateProgress"; +export default DatasetCollection; diff --git a/client/src/components/History/ContentItem/DatasetCollection/index.vue b/client/src/components/History/ContentItem/DatasetCollection/index.vue deleted file mode 100644 index 49d4758c5d3..00000000000 --- a/client/src/components/History/ContentItem/DatasetCollection/index.vue +++ /dev/null @@ -1,67 +0,0 @@ - - - diff --git a/client/src/components/History/ContentItem/HistoryContentItem.js b/client/src/components/History/ContentItem/HistoryContentItem.js index fe65f664934..c6d8c6632dc 100644 --- a/client/src/components/History/ContentItem/HistoryContentItem.js +++ b/client/src/components/History/ContentItem/HistoryContentItem.js @@ -7,10 +7,6 @@ export default { contentItemComponent() { if (this.item.id === undefined) { return "Placeholder"; - // return "Loading"; - } - if (this.scrolling) { - return "Placeholder"; } const { history_content_type } = this.item; switch (history_content_type) { diff --git a/client/src/components/History/ContentItem/Subcollection.vue b/client/src/components/History/ContentItem/Subcollection.vue index fbef360ddfe..d17a8f6b5a0 100644 --- a/client/src/components/History/ContentItem/Subcollection.vue +++ b/client/src/components/History/ContentItem/Subcollection.vue @@ -7,12 +7,10 @@ v-on="$listeners" class="collapsed" :data-state="state" - @keydown.arrow-right.self.stop.prevent="$emit('select-collection', dsc)" + @keydown.arrow-right.self.stop.prevent="$emit('viewCollection')" + @click.stop="$emit('viewCollection')" > -