From 23185e70391af2fc04f561109094647da6a83765 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 6 Sep 2023 01:28:31 +0100 Subject: [PATCH] Add GitHub CI workflow to test on Python 3.7 the packages needed by Pulsar so we will be able to drop Python 3.7 support for the rest of the code base. Add a new `test_galaxy_packages_for_pulsar` tox environment and modify the `test.sh` script so that when the `--for-pulsar` argument is passed: - only the 4 packages used by Pulsar are tested - mypy checks are skipped - unit tests don't skip the tests marked with `external_dependency_management` (i.e. those conda/container-related). Also: - Add missing pydantic requirement to the galaxy-objectstore package --- .../test_galaxy_packages_for_pulsar.yaml | 44 +++++++++++++++++++ packages/objectstore/setup.cfg | 1 + packages/packages_for_pulsar_by_dep_dag.txt | 4 ++ packages/test.sh | 30 ++++++++++--- tox.ini | 12 ++--- 5 files changed, 76 insertions(+), 15 deletions(-) create mode 100644 .github/workflows/test_galaxy_packages_for_pulsar.yaml create mode 100644 packages/packages_for_pulsar_by_dep_dag.txt diff --git a/.github/workflows/test_galaxy_packages_for_pulsar.yaml b/.github/workflows/test_galaxy_packages_for_pulsar.yaml new file mode 100644 index 00000000000..7f5f5499bc4 --- /dev/null +++ b/.github/workflows/test_galaxy_packages_for_pulsar.yaml @@ -0,0 +1,44 @@ +name: Test Galaxy packages for Pulsar +on: + push: + paths-ignore: + - 'client/**' + - 'doc/**' + - 'lib/galaxy_test/selenium/**' + pull_request: + paths-ignore: + - 'client/**' + - 'doc/**' + - 'lib/galaxy_test/selenium/**' +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true +jobs: + test: + name: Test + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ['3.7'] + steps: + - uses: actions/checkout@v3 + with: + path: 'galaxy root' + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Cache pip dir + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }} + - name: Install Apptainer's singularity + uses: eWaterCycle/setup-apptainer@v2 + - name: Install ffmpeg + run: sudo apt-get update && sudo apt-get -y install ffmpeg + - name: Install tox + run: pip install tox + - name: Run tests + run: tox -e test_galaxy_packages_for_pulsar + working-directory: 'galaxy root' diff --git a/packages/objectstore/setup.cfg b/packages/objectstore/setup.cfg index e283840cb5d..69fc1352966 100644 --- a/packages/objectstore/setup.cfg +++ b/packages/objectstore/setup.cfg @@ -33,6 +33,7 @@ version = 23.1.dev0 include_package_data = True install_requires = galaxy-util + pydantic<2 PyYAML packages = find: python_requires = >=3.7 diff --git a/packages/packages_for_pulsar_by_dep_dag.txt b/packages/packages_for_pulsar_by_dep_dag.txt new file mode 100644 index 00000000000..b698332ef1e --- /dev/null +++ b/packages/packages_for_pulsar_by_dep_dag.txt @@ -0,0 +1,4 @@ +util +job_metrics +objectstore +tool_util diff --git a/packages/test.sh b/packages/test.sh index afec3c4fab8..87497a561e4 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -2,6 +2,16 @@ set -ex +PACKAGE_LIST_FILE=packages_by_dep_dag.txt +FOR_PULSAR=0 + +for arg in "$@"; do + if [ "$arg" = "--for-pulsar" ]; then + PACKAGE_LIST_FILE=packages_for_pulsar_by_dep_dag.txt + FOR_PULSAR=1 + fi +done + # Don't display the pip progress bar when running under CI [ "$CI" = 'true' ] && export PIP_PROGRESS_BAR=off @@ -15,7 +25,9 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" . "${TEST_ENV_DIR}/bin/activate" pip install --upgrade pip setuptools wheel -pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt +if [ $FOR_PULSAR -eq 0 ]; then + pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt +fi # ensure ordered by dependency DAG while read -r package_dir; do @@ -34,12 +46,18 @@ while read -r package_dir; do pip install -r test-requirements.txt + if [ $FOR_PULSAR -eq 0 ]; then + marker_args=(-m 'not external_dependency_management') + else + marker_args=() + fi # Prevent execution of alembic/env.py at test collection stage (alembic.context not set) # Also ignore functional tests (galaxy_test/ and tool_shed/test/). - unit_extra='--doctest-modules --ignore=galaxy/model/migrations/alembic/ --ignore=galaxy_test/ - --ignore=tool_shed/test/ --ignore=tool_shed/webapp/model/migrations/alembic/' + unit_extra=(--doctest-modules --ignore=galaxy/model/migrations/alembic/ --ignore=galaxy_test/ --ignore=tool_shed/test/ --ignore=tool_shed/webapp/model/migrations/alembic/ "${marker_args[@]}") # Ignore exit code 5 (no tests ran) - pytest $unit_extra -m 'not external_dependency_management' . || test $? -eq 5 - make mypy + pytest "${unit_extra[@]}" . || test $? -eq 5 + if [ $FOR_PULSAR -eq 0 ]; then + make mypy + fi cd .. -done < packages_by_dep_dag.txt +done < $PACKAGE_LIST_FILE diff --git a/tox.ini b/tox.ini index 4e2c1e14ca0..bcc2d1094d9 100644 --- a/tox.ini +++ b/tox.ini @@ -2,13 +2,6 @@ # envlist is the list of environments that are tested when `tox` is run without any option # hyphens in an environment name are used to delimit factors envlist = - py{37,38,39,310,311}-first_startup - py{37,38,39,310,311}-lint - py{37,38,39,310,311}-lint_docstring_include_list - py{37,38,39,310,311}-mypy - py{37,38,39,310,311}-reports_startup - py{37,38,39,310,311}-unit - py{37,38,39,310,311}-unit-coverage first_startup lint lint_docstring_include_list @@ -33,8 +26,9 @@ commands = mulled,unit: bash run_tests.sh -u # start with test here but obviously someday all of it... mypy: mypy test lib - test_galaxy_packages: make generate-cwl-conformance-tests + test_galaxy_packages,test_galaxy_packages_for_pulsar: make generate-cwl-conformance-tests test_galaxy_packages: bash packages/test.sh + test_galaxy_packages_for_pulsar: bash packages/test.sh --for-pulsar allowlist_externals = bash make @@ -58,7 +52,7 @@ deps = lint,lint_docstring,lint_docstring_include_list: -rlib/galaxy/dependencies/pinned-lint-requirements.txt mypy: -rlib/galaxy/dependencies/pinned-typecheck-requirements.txt mypy: -rlib/galaxy/dependencies/pinned-requirements.txt - test_galaxy_packages: pyyaml + test_galaxy_packages,test_galaxy_packages_for_pulsar: pyyaml unit: mock-ssh-server [testenv:mako_count]