diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 11cbabf98d8..0125f24c4d4 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,6 +22,10 @@ jobs: strategy: matrix: python-version: ['3.7', '3.10'] + env: + LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt' + TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt' + CORE_PATH: 'lib/galaxy/dependencies/pinned-requirements.txt' steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -35,12 +39,12 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }} + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }} - name: Cache tox env uses: actions/cache@v3 with: path: .tox - key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }}-lint + key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }}-lint - name: Install tox run: pip install tox - name: Run linting diff --git a/lib/galaxy/dependencies/lint-requirements.txt b/lib/galaxy/dependencies/lint-requirements.txt index a1909d9164d..675b2c5d0d8 100644 --- a/lib/galaxy/dependencies/lint-requirements.txt +++ b/lib/galaxy/dependencies/lint-requirements.txt @@ -1,15 +1,2 @@ flake8 flake8-bugbear -mypy -types-bleach -types-boto -types-contextvars -types-dataclasses -types-docutils -types-Markdown -types-paramiko -types-pkg-resources -types-python-dateutil -types-PyYAML -types-requests -types-six diff --git a/lib/galaxy/dependencies/pinned-lint-requirements.txt b/lib/galaxy/dependencies/pinned-lint-requirements.txt index bc57d184442..0d242b5c6c6 100644 --- a/lib/galaxy/dependencies/pinned-lint-requirements.txt +++ b/lib/galaxy/dependencies/pinned-lint-requirements.txt @@ -3,25 +3,7 @@ flake8==5.0.4 flake8-bugbear==22.10.27 importlib-metadata==4.2.0 mccabe==0.7.0 -mypy==0.991 -mypy-extensions==0.4.3 pycodestyle==2.9.1 pyflakes==2.5.0 -tomli==2.0.1 -typed-ast==1.5.4 -types-bleach==5.0.3.1 -types-boto==2.49.18.3 -types-contextvars==2.4.7 -types-cryptography==3.3.23.2 -types-dataclasses==0.6.6 -types-docutils==0.19.1.1 -types-Markdown==3.4.2.1 -types-paramiko==2.12.0.1 -types-pkg-resources==0.1.3 -types-python-dateutil==2.8.19.4 -types-PyYAML==6.0.12.2 -types-requests==2.28.11.5 -types-six==1.16.21.4 -types-urllib3==1.26.25.4 typing_extensions==4.4.0 zipp==3.11.0 diff --git a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt new file mode 100644 index 00000000000..0f43f72f17a --- /dev/null +++ b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt @@ -0,0 +1,19 @@ +mypy==0.991 +mypy-extensions==0.4.3 +tomli==2.0.1 +typed-ast==1.5.4 +types-bleach==5.0.3.1 +types-boto==2.49.18.3 +types-contextvars==2.4.7 +types-cryptography==3.3.23.2 +types-dataclasses==0.6.6 +types-docutils==0.19.1.1 +types-Markdown==3.4.2.1 +types-paramiko==2.12.0.1 +types-pkg-resources==0.1.3 +types-python-dateutil==2.8.19.4 +types-PyYAML==6.0.12.2 +types-requests==2.28.11.5 +types-six==1.16.21.4 +types-urllib3==1.26.25.4 +typing_extensions==4.4.0 diff --git a/lib/galaxy/dependencies/typecheck-requirements.txt b/lib/galaxy/dependencies/typecheck-requirements.txt new file mode 100644 index 00000000000..007c33348ec --- /dev/null +++ b/lib/galaxy/dependencies/typecheck-requirements.txt @@ -0,0 +1,13 @@ +mypy +types-bleach +types-boto +types-contextvars +types-dataclasses +types-docutils +types-Markdown +types-paramiko +types-pkg-resources +types-python-dateutil +types-PyYAML +types-requests +types-six diff --git a/lib/galaxy/dependencies/update_lint_requirements.sh b/lib/galaxy/dependencies/update_lint_requirements.sh index d477d941ab9..58c503e2e3f 100755 --- a/lib/galaxy/dependencies/update_lint_requirements.sh +++ b/lib/galaxy/dependencies/update_lint_requirements.sh @@ -1,13 +1,26 @@ #!/bin/sh + set -e +# This script updates the pinned requirements for both linting and typechecking. +# The requirements are split into two parts due to incompatible dependencies: +# flake8 requires importlib-metadata 4.2; typecheck requirements do not +# depende on importlib-metadata; however, they must be installed into the same +# virtual environment as Galaxy's core dependencies, which depend on +# importlib-metadata 4.13. + THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" -LINT_VENV=$(mktemp -d "${TMPDIR:-/tmp}/lint_venv.XXXXXXXXXX") -python3.7 -m venv "${LINT_VENV}" -. "${LINT_VENV}/bin/activate" -pip install --upgrade pip setuptools -pip install -r "${THIS_DIRECTORY}/lint-requirements.txt" -# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 -pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-lint-requirements.txt" -rm -rf "${LINT_VENV}" +update_pinned_reqs() { + VENV=$(mktemp -d "${TMPDIR:-/tmp}/$1_venv.XXXXXXXXXX") + python3.7 -m venv "${VENV}" + . "${VENV}/bin/activate" + pip install --upgrade pip setuptools + pip install -r "${THIS_DIRECTORY}/$1-requirements.txt" + # The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 + pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt" + rm -rf "${VENV}" +} + +update_pinned_reqs lint +update_pinned_reqs typecheck diff --git a/lib/galaxy/selenium/sizzle.py b/lib/galaxy/selenium/sizzle.py index 1282f016ee7..8716a2c55b9 100644 --- a/lib/galaxy/selenium/sizzle.py +++ b/lib/galaxy/selenium/sizzle.py @@ -3,7 +3,7 @@ import json from selenium.common.exceptions import NoSuchElementException -from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support.wait import WebDriverWait from .has_driver import exception_indicates_stale_element diff --git a/lib/galaxy/selenium/smart_components.py b/lib/galaxy/selenium/smart_components.py index 3f1d3d17fca..3f25f7d4a11 100644 --- a/lib/galaxy/selenium/smart_components.py +++ b/lib/galaxy/selenium/smart_components.py @@ -1,4 +1,4 @@ -from selenium.webdriver.support.ui import Select +from selenium.webdriver.support.select import Select from galaxy.navigation.components import ( Component, diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index fc55a642de0..eaea543fd12 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -24,10 +24,7 @@ from galaxy.model.security import ( HostAgent, ) from galaxy.model.tags import GalaxyTagHandler -from galaxy.objectstore import ( - BaseObjectStore, - ObjectStore, -) +from galaxy.objectstore import BaseObjectStore from galaxy.quota import QuotaAgent from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.vault import Vault @@ -76,9 +73,11 @@ class BasicSharedApp(Container): class MinimalToolApp(Protocol): name: str - config: Any # 'galaxy.config.BaseAppConfiguration' + # Leave config as Any: in a full Galaxy app this is a GalaxyAppConfiguration object, but this is mostly dynamically + # generated, and here we want to also allow other kinds of configuration objects (e.g. a Bunch). + config: Any datatypes_registry: Registry - object_store: ObjectStore + object_store: BaseObjectStore tool_data_tables: "ToolDataTableManager" file_sources: ConfiguredFileSources security: IdEncodingHelper diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index d7c5a62b4f2..8dd158f822d 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -19,7 +19,7 @@ from galaxy.metadata.set_metadata import ( ) from galaxy.model import store from galaxy.model.store import SessionlessContext -from galaxy.objectstore import ObjectStore +from galaxy.objectstore import BaseObjectStore from galaxy.structured_app import MinimalToolApp from galaxy.tools import ( create_tool_from_representation, @@ -55,7 +55,7 @@ class ToolApp(MinimalToolApp): sa_session: SessionlessContext, tool_app_config: ToolAppConfig, datatypes_registry: Registry, - object_store: ObjectStore, + object_store: BaseObjectStore, tool_data_table_manager: ToolDataTableManager, file_sources: ConfiguredFileSources, ): diff --git a/mypy.ini b/mypy.ini index bbefdbab609..aeac561f64a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -685,6 +685,8 @@ check_untyped_defs = False check_untyped_defs = False [mypy-*.workflows.test_modules] check_untyped_defs = False +[mypy-*.workflows.test_refactor_models] +check_untyped_defs = False [mypy-*.managers.test_HDAManager] check_untyped_defs = False [mypy-*.managers.test_DatasetManager] diff --git a/packages/test.sh b/packages/test.sh index d2334003e41..ee0cef696b5 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -15,7 +15,7 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" . "${TEST_ENV_DIR}/bin/activate" pip install --upgrade pip setuptools wheel -pip install -r../lib/galaxy/dependencies/pinned-lint-requirements.txt +pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt # ensure ordered by dependency DAG # TODO: add selenium in once type issues are cleared up diff --git a/test/integration/test_celery_tasks.py b/test/integration/test_celery_tasks.py index 7bf22660ae8..41e93814239 100644 --- a/test/integration/test_celery_tasks.py +++ b/test/integration/test_celery_tasks.py @@ -72,7 +72,7 @@ class TestCeleryTasksIntegration(IntegrationTestCase): assert hda_purged() def test_pdf_download(self): - short_term_storage_allocator = self._app[ShortTermStorageAllocator] + short_term_storage_allocator = self._app[ShortTermStorageAllocator] # type: ignore[type-abstract] short_term_storage_target = short_term_storage_allocator.new_target("moo.pdf", "application/pdf") request_id = short_term_storage_target.request_id pdf_download_request = GeneratePdfDownload( diff --git a/test/unit/app/authnz/test_custos_authnz.py b/test/unit/app/authnz/test_custos_authnz.py index 9b8d5402f3a..e308bb8f9c1 100644 --- a/test/unit/app/authnz/test_custos_authnz.py +++ b/test/unit/app/authnz/test_custos_authnz.py @@ -222,7 +222,6 @@ class TestCustosAuthnz(TestCase): self.cookies = {} self.cookies_args = {} self.request = Request() - self.session = self.sa_session self.sa_session = Session() self.user = None diff --git a/test/unit/webapps/test_webapp_base.py b/test/unit/webapps/test_webapp_base.py index 9f329b9ea98..d0aa4a42f22 100644 --- a/test/unit/webapps/test_webapp_base.py +++ b/test/unit/webapps/test_webapp_base.py @@ -10,6 +10,7 @@ from typing import ( import galaxy.config from galaxy.app_unittest_utils import galaxy_mock +from galaxy.structured_app import BasicSharedApp from galaxy.webapps.base.webapp import ( GalaxyWebTransaction, WebApplication, @@ -36,7 +37,7 @@ class CORSParsingMockConfig(galaxy_mock.MockAppConfig): class TestGalaxyWebTransactionHeaders: def _new_trans(self, allowed_origin_hostnames: Optional[str] = None) -> StubGalaxyWebTransaction: - app = galaxy_mock.MockApp() + app = cast(BasicSharedApp, galaxy_mock.MockApp()) app.config = CORSParsingMockConfig(allowed_origin_hostnames=allowed_origin_hostnames) webapp = cast(WebApplication, galaxy_mock.MockWebapp(app.security)) environ = galaxy_mock.buildMockEnviron() diff --git a/tox.ini b/tox.ini index 14030c6a210..bb0c5b5d95e 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,9 @@ setenv = check_indexes: GALAXY_SKIP_CLIENT_BUILD=1 deps = coverage: coverage - lint,lint_docstring,lint_docstring_include_list,mypy: -rlib/galaxy/dependencies/pinned-lint-requirements.txt + lint,lint_docstring,lint_docstring_include_list: -rlib/galaxy/dependencies/pinned-lint-requirements.txt + mypy: -rlib/galaxy/dependencies/pinned-typecheck-requirements.txt + mypy: -rlib/galaxy/dependencies/pinned-requirements.txt test_galaxy_packages: pyyaml unit: mock-ssh-server