Merge pull request #16649 from nsoranzo/test_galaxy_packages_for_pulsar

Add GitHub CI workflow to test on Python 3.7 the packages needed by Pulsar

.github/workflows/test_galaxy_packages_for_pulsar.yaml

@@ -0,0 +1,44 @@
+name: Test Galaxy packages for Pulsar
+on:
+  push:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+      - 'lib/galaxy_test/selenium/**'
+  pull_request:
+    paths-ignore:
+      - 'client/**'
+      - 'doc/**'
+      - 'lib/galaxy_test/selenium/**'
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+jobs:
+  test:
+    name: Test
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ['3.7']
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          path: 'galaxy root'
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache pip dir
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('galaxy root/requirements.txt') }}
+      - name: Install Apptainer's singularity
+        uses: eWaterCycle/setup-apptainer@v2
+      - name: Install ffmpeg
+        run: sudo apt-get update && sudo apt-get -y install ffmpeg
+      - name: Install tox
+        run: pip install tox
+      - name: Run tests
+        run: tox -e test_galaxy_packages_for_pulsar
+        working-directory: 'galaxy root'

packages/objectstore/setup.cfg

@@ -33,6 +33,7 @@ version = 23.1.dev0
 include_package_data = True
 install_requires =
     galaxy-util
+    pydantic<2
     PyYAML
 packages = find:
 python_requires = >=3.7

packages/packages_for_pulsar_by_dep_dag.txt

@@ -0,0 +1,4 @@
+util
+job_metrics
+objectstore
+tool_util

packages/test.sh

@@ -2,6 +2,16 @@
 
 set -ex
 
+PACKAGE_LIST_FILE=packages_by_dep_dag.txt
+FOR_PULSAR=0
+
+for arg in "$@"; do
+    if [ "$arg" = "--for-pulsar" ]; then
+        PACKAGE_LIST_FILE=packages_for_pulsar_by_dep_dag.txt
+        FOR_PULSAR=1
+    fi
+done
+
 # Don't display the pip progress bar when running under CI
 [ "$CI" = 'true' ] && export PIP_PROGRESS_BAR=off
 
@@ -15,7 +25,9 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)}
 virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR"
 . "${TEST_ENV_DIR}/bin/activate"
 pip install --upgrade pip setuptools wheel
-pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
+if [ $FOR_PULSAR -eq 0 ]; then
+    pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
+fi
 
 # ensure ordered by dependency DAG
 while read -r package_dir; do
@@ -34,12 +46,18 @@ while read -r package_dir; do
 
     pip install -r test-requirements.txt
 
+    if [ $FOR_PULSAR -eq 0 ]; then
+        marker_args=(-m 'not external_dependency_management')
+    else
+        marker_args=()
+    fi
     # Prevent execution of alembic/env.py at test collection stage (alembic.context not set)
     # Also ignore functional tests (galaxy_test/ and tool_shed/test/).
-    unit_extra='--doctest-modules --ignore=galaxy/model/migrations/alembic/ --ignore=galaxy_test/
-		--ignore=tool_shed/test/ --ignore=tool_shed/webapp/model/migrations/alembic/'
+    unit_extra=(--doctest-modules --ignore=galaxy/model/migrations/alembic/ --ignore=galaxy_test/ --ignore=tool_shed/test/ --ignore=tool_shed/webapp/model/migrations/alembic/ "${marker_args[@]}")
     # Ignore exit code 5 (no tests ran)
-    pytest $unit_extra -m 'not external_dependency_management' . || test $? -eq 5
-    make mypy
+    pytest "${unit_extra[@]}" . || test $? -eq 5
+    if [ $FOR_PULSAR -eq 0 ]; then
+        make mypy
+    fi
     cd ..
-done < packages_by_dep_dag.txt
+done < $PACKAGE_LIST_FILE

tox.ini

@@ -2,13 +2,6 @@
 # envlist is the list of environments that are tested when `tox` is run without any option
 # hyphens in an environment name are used to delimit factors
 envlist =
-    py{37,38,39,310,311}-first_startup
-    py{37,38,39,310,311}-lint
-    py{37,38,39,310,311}-lint_docstring_include_list
-    py{37,38,39,310,311}-mypy
-    py{37,38,39,310,311}-reports_startup
-    py{37,38,39,310,311}-unit
-    py{37,38,39,310,311}-unit-coverage
     first_startup
     lint
     lint_docstring_include_list
@@ -33,8 +26,9 @@ commands =
     mulled,unit: bash run_tests.sh -u
     # start with test here but obviously someday all of it...
     mypy: mypy test lib
-    test_galaxy_packages: make generate-cwl-conformance-tests
+    test_galaxy_packages,test_galaxy_packages_for_pulsar: make generate-cwl-conformance-tests
     test_galaxy_packages: bash packages/test.sh
+    test_galaxy_packages_for_pulsar: bash packages/test.sh --for-pulsar
 allowlist_externals =
     bash
     make
@@ -58,7 +52,7 @@ deps =
     lint,lint_docstring,lint_docstring_include_list: -rlib/galaxy/dependencies/pinned-lint-requirements.txt
     mypy: -rlib/galaxy/dependencies/pinned-typecheck-requirements.txt
     mypy: -rlib/galaxy/dependencies/pinned-requirements.txt
-    test_galaxy_packages: pyyaml
+    test_galaxy_packages,test_galaxy_packages_for_pulsar: pyyaml
     unit: mock-ssh-server
 
 [testenv:mako_count]