Merge pull request #15004 from jdavcs/dev_mypy_workflow

Run mypy on a fully loaded galaxy venv for stricter type checking

.github/workflows/lint.yaml

@@ -22,6 +22,10 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.7', '3.10']
+    env:
+      LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt'
+      TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt'
+      CORE_PATH: 'lib/galaxy/dependencies/pinned-requirements.txt'
     steps:
       - uses: actions/checkout@v3
       - uses: actions/setup-python@v4
@@ -35,12 +39,12 @@ jobs:
         uses: actions/cache@v3
         with:
           path: ~/.cache/pip
-          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }}
+          key: pip-cache-${{ matrix.python-version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }}
       - name: Cache tox env
         uses: actions/cache@v3
         with:
           path: .tox
-          key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }}-lint
+          key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }}-lint
       - name: Install tox
         run: pip install tox
       - name: Run linting

lib/galaxy/dependencies/lint-requirements.txt

@@ -1,15 +1,2 @@
 flake8
 flake8-bugbear
-mypy
-types-bleach
-types-boto
-types-contextvars
-types-dataclasses
-types-docutils
-types-Markdown
-types-paramiko
-types-pkg-resources
-types-python-dateutil
-types-PyYAML
-types-requests
-types-six

lib/galaxy/dependencies/pinned-lint-requirements.txt

@@ -3,25 +3,7 @@ flake8==5.0.4
 flake8-bugbear==22.10.27
 importlib-metadata==4.2.0
 mccabe==0.7.0
-mypy==0.991
-mypy-extensions==0.4.3
 pycodestyle==2.9.1
 pyflakes==2.5.0
-tomli==2.0.1
-typed-ast==1.5.4
-types-bleach==5.0.3.1
-types-boto==2.49.18.3
-types-contextvars==2.4.7
-types-cryptography==3.3.23.2
-types-dataclasses==0.6.6
-types-docutils==0.19.1.1
-types-Markdown==3.4.2.1
-types-paramiko==2.12.0.1
-types-pkg-resources==0.1.3
-types-python-dateutil==2.8.19.4
-types-PyYAML==6.0.12.2
-types-requests==2.28.11.5
-types-six==1.16.21.4
-types-urllib3==1.26.25.4
 typing_extensions==4.4.0
 zipp==3.11.0

lib/galaxy/dependencies/pinned-typecheck-requirements.txt

@@ -0,0 +1,19 @@
+mypy==0.991
+mypy-extensions==0.4.3
+tomli==2.0.1
+typed-ast==1.5.4
+types-bleach==5.0.3.1
+types-boto==2.49.18.3
+types-contextvars==2.4.7
+types-cryptography==3.3.23.2
+types-dataclasses==0.6.6
+types-docutils==0.19.1.1
+types-Markdown==3.4.2.1
+types-paramiko==2.12.0.1
+types-pkg-resources==0.1.3
+types-python-dateutil==2.8.19.4
+types-PyYAML==6.0.12.2
+types-requests==2.28.11.5
+types-six==1.16.21.4
+types-urllib3==1.26.25.4
+typing_extensions==4.4.0

lib/galaxy/dependencies/typecheck-requirements.txt

@@ -0,0 +1,13 @@
+mypy
+types-bleach
+types-boto
+types-contextvars
+types-dataclasses
+types-docutils
+types-Markdown
+types-paramiko
+types-pkg-resources
+types-python-dateutil
+types-PyYAML
+types-requests
+types-six

lib/galaxy/dependencies/update_lint_requirements.sh

@@ -1,13 +1,26 @@
 #!/bin/sh
+
 set -e
 
+# This script updates the pinned requirements for both linting and typechecking.
+# The requirements are split into two parts due to incompatible dependencies:
+# flake8 requires importlib-metadata 4.2; typecheck requirements do not
+# depende on importlib-metadata; however, they must be installed into the same
+# virtual environment as Galaxy's core dependencies, which depend on
+# importlib-metadata 4.13.
+
 THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)"
 
-LINT_VENV=$(mktemp -d "${TMPDIR:-/tmp}/lint_venv.XXXXXXXXXX")
-python3.7 -m venv "${LINT_VENV}"
-. "${LINT_VENV}/bin/activate"
-pip install --upgrade pip setuptools
-pip install -r "${THIS_DIRECTORY}/lint-requirements.txt"
-# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331
-pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-lint-requirements.txt"
-rm -rf "${LINT_VENV}"
+update_pinned_reqs() {
+    VENV=$(mktemp -d "${TMPDIR:-/tmp}/$1_venv.XXXXXXXXXX")
+    python3.7 -m venv "${VENV}"
+    . "${VENV}/bin/activate"
+    pip install --upgrade pip setuptools
+    pip install -r "${THIS_DIRECTORY}/$1-requirements.txt"
+    # The grep below is needed to workaround https://github.com/pypa/pip/issues/8331
+    pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt"
+    rm -rf "${VENV}"
+}
+
+update_pinned_reqs lint
+update_pinned_reqs typecheck

lib/galaxy/selenium/sizzle.py

@@ -3,7 +3,7 @@
 import json
 
 from selenium.common.exceptions import NoSuchElementException
-from selenium.webdriver.support.ui import WebDriverWait
+from selenium.webdriver.support.wait import WebDriverWait
 
 from .has_driver import exception_indicates_stale_element
 

lib/galaxy/selenium/smart_components.py

@@ -1,4 +1,4 @@
-from selenium.webdriver.support.ui import Select
+from selenium.webdriver.support.select import Select
 
 from galaxy.navigation.components import (
     Component,

lib/galaxy/structured_app.py

@@ -24,10 +24,7 @@ from galaxy.model.security import (
     HostAgent,
 )
 from galaxy.model.tags import GalaxyTagHandler
-from galaxy.objectstore import (
-    BaseObjectStore,
-    ObjectStore,
-)
+from galaxy.objectstore import BaseObjectStore
 from galaxy.quota import QuotaAgent
 from galaxy.security.idencoding import IdEncodingHelper
 from galaxy.security.vault import Vault
@@ -76,9 +73,11 @@ class BasicSharedApp(Container):
 
 class MinimalToolApp(Protocol):
     name: str
-    config: Any  # 'galaxy.config.BaseAppConfiguration'
+    # Leave config as Any: in a full Galaxy app this is a GalaxyAppConfiguration object, but this is mostly dynamically
+    # generated, and here we want to also allow other kinds of configuration objects (e.g. a Bunch).
+    config: Any
     datatypes_registry: Registry
-    object_store: ObjectStore
+    object_store: BaseObjectStore
     tool_data_tables: "ToolDataTableManager"
     file_sources: ConfiguredFileSources
     security: IdEncodingHelper

lib/galaxy/tools/remote_tool_eval.py

@@ -19,7 +19,7 @@ from galaxy.metadata.set_metadata import (
 )
 from galaxy.model import store
 from galaxy.model.store import SessionlessContext
-from galaxy.objectstore import ObjectStore
+from galaxy.objectstore import BaseObjectStore
 from galaxy.structured_app import MinimalToolApp
 from galaxy.tools import (
     create_tool_from_representation,
@@ -55,7 +55,7 @@ class ToolApp(MinimalToolApp):
         sa_session: SessionlessContext,
         tool_app_config: ToolAppConfig,
         datatypes_registry: Registry,
-        object_store: ObjectStore,
+        object_store: BaseObjectStore,
         tool_data_table_manager: ToolDataTableManager,
         file_sources: ConfiguredFileSources,
     ):

mypy.ini

@@ -685,6 +685,8 @@ check_untyped_defs = False
 check_untyped_defs = False
 [mypy-*.workflows.test_modules]
 check_untyped_defs = False
+[mypy-*.workflows.test_refactor_models]
+check_untyped_defs = False
 [mypy-*.managers.test_HDAManager]
 check_untyped_defs = False
 [mypy-*.managers.test_DatasetManager]

packages/test.sh

@@ -15,7 +15,7 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)}
 virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR"
 . "${TEST_ENV_DIR}/bin/activate"
 pip install --upgrade pip setuptools wheel
-pip install -r../lib/galaxy/dependencies/pinned-lint-requirements.txt
+pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
 
 # ensure ordered by dependency DAG
 # TODO: add selenium in once type issues are cleared up

test/integration/test_celery_tasks.py

@@ -72,7 +72,7 @@ class TestCeleryTasksIntegration(IntegrationTestCase):
         assert hda_purged()
 
     def test_pdf_download(self):
-        short_term_storage_allocator = self._app[ShortTermStorageAllocator]
+        short_term_storage_allocator = self._app[ShortTermStorageAllocator]  # type: ignore[type-abstract]
         short_term_storage_target = short_term_storage_allocator.new_target("moo.pdf", "application/pdf")
         request_id = short_term_storage_target.request_id
         pdf_download_request = GeneratePdfDownload(

test/unit/app/authnz/test_custos_authnz.py

@@ -222,7 +222,6 @@ class TestCustosAuthnz(TestCase):
                 self.cookies = {}
                 self.cookies_args = {}
                 self.request = Request()
-                self.session = self.sa_session
                 self.sa_session = Session()
                 self.user = None
 

test/unit/webapps/test_webapp_base.py

@@ -10,6 +10,7 @@ from typing import (
 
 import galaxy.config
 from galaxy.app_unittest_utils import galaxy_mock
+from galaxy.structured_app import BasicSharedApp
 from galaxy.webapps.base.webapp import (
     GalaxyWebTransaction,
     WebApplication,
@@ -36,7 +37,7 @@ class CORSParsingMockConfig(galaxy_mock.MockAppConfig):
 
 class TestGalaxyWebTransactionHeaders:
     def _new_trans(self, allowed_origin_hostnames: Optional[str] = None) -> StubGalaxyWebTransaction:
-        app = galaxy_mock.MockApp()
+        app = cast(BasicSharedApp, galaxy_mock.MockApp())
         app.config = CORSParsingMockConfig(allowed_origin_hostnames=allowed_origin_hostnames)
         webapp = cast(WebApplication, galaxy_mock.MockWebapp(app.security))
         environ = galaxy_mock.buildMockEnviron()

tox.ini

@@ -36,7 +36,9 @@ setenv =
     check_indexes: GALAXY_SKIP_CLIENT_BUILD=1
 deps =
     coverage: coverage
-    lint,lint_docstring,lint_docstring_include_list,mypy: -rlib/galaxy/dependencies/pinned-lint-requirements.txt
+    lint,lint_docstring,lint_docstring_include_list: -rlib/galaxy/dependencies/pinned-lint-requirements.txt
+    mypy: -rlib/galaxy/dependencies/pinned-typecheck-requirements.txt
+    mypy: -rlib/galaxy/dependencies/pinned-requirements.txt
     test_galaxy_packages: pyyaml
     unit: mock-ssh-server