From ec0cdc718aef1f46e457f1c9194dd9079f8b4ff1 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 16 Nov 2022 19:54:38 -0500 Subject: [PATCH 01/27] Fix implicit reexport error for selenium lib/galaxy/selenium/smart_components.py:1: error: Module "selenium.webdriver.support.ui" does not explicitly export attribute "Select"; implicit reexport disabled [attr-defined] from selenium.webdriver.support.ui import Select ^ lib/galaxy/selenium/sizzle.py:6: error: Module "selenium.webdriver.support.ui" does not explicitly export attribute "WebDriverWait"; implicit reexport disabled [attr-defined] from selenium.webdriver.support.ui import WebDriverWait --- mypy.ini | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/mypy.ini b/mypy.ini index bbefdbab609..4df64f6629d 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,6 +7,10 @@ pretty = True no_implicit_reexport = True no_implicit_optional = True +# overrides for external code +[mypy-selenium.webdriver.support.ui] +no_implicit_reexport = False + # green list - work on growing these please! [mypy-galaxy.util.compression_utils] disallow_any_generics = True From 4c96135b6c11bd4c19a1f1124e91c52feb31bb6d Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 28 Nov 2022 10:35:15 -0500 Subject: [PATCH 02/27] Fix reexport error (selenium) by direct import --- lib/galaxy/selenium/smart_components.py | 2 +- mypy.ini | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/galaxy/selenium/smart_components.py b/lib/galaxy/selenium/smart_components.py index 3f1d3d17fca..3f25f7d4a11 100644 --- a/lib/galaxy/selenium/smart_components.py +++ b/lib/galaxy/selenium/smart_components.py @@ -1,4 +1,4 @@ -from selenium.webdriver.support.ui import Select +from selenium.webdriver.support.select import Select from galaxy.navigation.components import ( Component, diff --git a/mypy.ini b/mypy.ini index 4df64f6629d..bbefdbab609 100644 --- a/mypy.ini +++ b/mypy.ini @@ -7,10 +7,6 @@ pretty = True no_implicit_reexport = True no_implicit_optional = True -# overrides for external code -[mypy-selenium.webdriver.support.ui] -no_implicit_reexport = False - # green list - work on growing these please! [mypy-galaxy.util.compression_utils] disallow_any_generics = True From 9c4bf23a61b3a8bdeb2d7ef3eb1a826764a6b0ef Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 16 Nov 2022 23:44:15 -0500 Subject: [PATCH 03/27] Fix mypy error (incompatible type) lib/galaxy/managers/model_stores.py:59: error: Argument "app" to "DirectoryModelExportStore" has incompatible type "MinimalManagerApp"; expected "Optional[StoreAppProtocol]" [arg-type] with DirectoryModelExportStore(store_directory, app=self._app, export_files="symlink") as export_store: ^~~~~~~~~ lib/galaxy/managers/model_stores.py:59: note: Following member(s) of "MinimalManagerApp" have conflicts: lib/galaxy/managers/model_stores.py:59: note: object_store: expected "ObjectStore", got "BaseObjectStore" lib/galaxy/managers/model_stores.py:186: error: Argument 2 to "source_to_import_store" has incompatible type "MinimalManagerApp"; expected "StoreAppProtocol" [arg-type] self._app, ^~~~~~~~~ lib/galaxy/managers/model_stores.py:186: note: Following member(s) of "MinimalManagerApp" have conflicts: lib/galaxy/managers/model_stores.py:186: note: object_store: expected "ObjectStore", got "BaseObjectStore" lib/galaxy/managers/model_stores.py:218: error: Argument "app" to "source_to_import_store" has incompatible type "MinimalManagerApp"; expected "StoreAppProtocol" [arg-type] app=app, ^~~ lib/galaxy/managers/model_stores.py:218: note: Following member(s) of "MinimalManagerApp" have conflicts: lib/galaxy/managers/model_stores.py:218: note: object_store: expected "ObjectStore", got "BaseObjectStore" --- lib/galaxy/model/store/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index b6f1bcef74e..00faf353109 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -63,7 +63,6 @@ from galaxy.model.tags import GalaxyTagHandler from galaxy.objectstore import ( BaseObjectStore, ObjectStore, -) from galaxy.schema.bco import ( BioComputeObjectCore, DescriptionDomain, From aba2b4c52a4297cc92ae1339ba5de51a85065205 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 17 Nov 2022 00:07:33 -0500 Subject: [PATCH 04/27] Fix mypy error (has-type) Fixes this error: (the value is of type sqlalchemy.orm.Session; I don't know what it's not inferred) test/unit/app/authnz/test_custos_authnz.py:225: error: Cannot determine type of "sa_session" [has-type] self.session = self.sa_session --- test/unit/app/authnz/test_custos_authnz.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/app/authnz/test_custos_authnz.py b/test/unit/app/authnz/test_custos_authnz.py index 9b8d5402f3a..3ae2dc1f9e3 100644 --- a/test/unit/app/authnz/test_custos_authnz.py +++ b/test/unit/app/authnz/test_custos_authnz.py @@ -222,7 +222,7 @@ class TestCustosAuthnz(TestCase): self.cookies = {} self.cookies_args = {} self.request = Request() - self.session = self.sa_session + self.session = self.sa_session # type: ignore[has-type] self.sa_session = Session() self.user = None From 3b351e0e437401ffd8ddc36503606e70a63e84f7 Mon Sep 17 00:00:00 2001 From: John Davis Date: Thu, 17 Nov 2022 22:58:34 -0500 Subject: [PATCH 05/27] Fix mypy errors for workflow refactor actions --- lib/galaxy/workflow/refactor/schema.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/lib/galaxy/workflow/refactor/schema.py b/lib/galaxy/workflow/refactor/schema.py index f6c3e7434f6..c30671b65cc 100644 --- a/lib/galaxy/workflow/refactor/schema.py +++ b/lib/galaxy/workflow/refactor/schema.py @@ -76,6 +76,15 @@ class BaseAction(BaseModel): class Action(BaseAction): action_type: str + default: Any + input: Any + label: str + name: str + output: Any + optional: bool + step: Any + type: str + tool_state: Any @classmethod def __get_validators__(cls): From ac25a305a3110cd9e3484879433c3e6f7d74b61e Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 28 Nov 2022 10:55:46 -0500 Subject: [PATCH 06/27] Revert "Fix mypy errors for workflow refactor actions" This reverts commit 0c1ff39f72fb56afd04121c49e98cf58cfa597b7. --- lib/galaxy/workflow/refactor/schema.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/galaxy/workflow/refactor/schema.py b/lib/galaxy/workflow/refactor/schema.py index c30671b65cc..f6c3e7434f6 100644 --- a/lib/galaxy/workflow/refactor/schema.py +++ b/lib/galaxy/workflow/refactor/schema.py @@ -76,15 +76,6 @@ class BaseAction(BaseModel): class Action(BaseAction): action_type: str - default: Any - input: Any - label: str - name: str - output: Any - optional: bool - step: Any - type: str - tool_state: Any @classmethod def __get_validators__(cls): From 99cc5adee5ed05a408dd6db79bdb087b30df7b27 Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 28 Nov 2022 11:11:16 -0500 Subject: [PATCH 07/27] Do not type-check workflow unit test test_refactor_ models --- mypy.ini | 2 ++ 1 file changed, 2 insertions(+) diff --git a/mypy.ini b/mypy.ini index bbefdbab609..aeac561f64a 100644 --- a/mypy.ini +++ b/mypy.ini @@ -685,6 +685,8 @@ check_untyped_defs = False check_untyped_defs = False [mypy-*.workflows.test_modules] check_untyped_defs = False +[mypy-*.workflows.test_refactor_models] +check_untyped_defs = False [mypy-*.managers.test_HDAManager] check_untyped_defs = False [mypy-*.managers.test_DatasetManager] From ad59ae54bf7bf02543cb2b073c9f74e77cb74817 Mon Sep 17 00:00:00 2001 From: John Davis Date: Mon, 28 Nov 2022 11:14:02 -0500 Subject: [PATCH 08/27] Fix another selenium reexport error --- lib/galaxy/model/store/__init__.py | 1 + lib/galaxy/selenium/sizzle.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/model/store/__init__.py b/lib/galaxy/model/store/__init__.py index 00faf353109..b6f1bcef74e 100644 --- a/lib/galaxy/model/store/__init__.py +++ b/lib/galaxy/model/store/__init__.py @@ -63,6 +63,7 @@ from galaxy.model.tags import GalaxyTagHandler from galaxy.objectstore import ( BaseObjectStore, ObjectStore, +) from galaxy.schema.bco import ( BioComputeObjectCore, DescriptionDomain, diff --git a/lib/galaxy/selenium/sizzle.py b/lib/galaxy/selenium/sizzle.py index 1282f016ee7..8716a2c55b9 100644 --- a/lib/galaxy/selenium/sizzle.py +++ b/lib/galaxy/selenium/sizzle.py @@ -3,7 +3,7 @@ import json from selenium.common.exceptions import NoSuchElementException -from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support.wait import WebDriverWait from .has_driver import exception_indicates_stale_element From cbabba8f4968430815fbe9ab30590dfa45ecaea2 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 13:32:50 -0500 Subject: [PATCH 09/27] Fix typing error in unit test --- test/unit/webapps/test_webapp_base.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/unit/webapps/test_webapp_base.py b/test/unit/webapps/test_webapp_base.py index 9f329b9ea98..7fffb3a98cc 100644 --- a/test/unit/webapps/test_webapp_base.py +++ b/test/unit/webapps/test_webapp_base.py @@ -10,6 +10,7 @@ from typing import ( import galaxy.config from galaxy.app_unittest_utils import galaxy_mock +from galaxy.structured_app import BasicSharedApp from galaxy.webapps.base.webapp import ( GalaxyWebTransaction, WebApplication, @@ -36,7 +37,7 @@ class CORSParsingMockConfig(galaxy_mock.MockAppConfig): class TestGalaxyWebTransactionHeaders: def _new_trans(self, allowed_origin_hostnames: Optional[str] = None) -> StubGalaxyWebTransaction: - app = galaxy_mock.MockApp() + app = galaxy_mock.MockApp()[BasicSharedApp] app.config = CORSParsingMockConfig(allowed_origin_hostnames=allowed_origin_hostnames) webapp = cast(WebApplication, galaxy_mock.MockWebapp(app.security)) environ = galaxy_mock.buildMockEnviron() From 71769182b6167b33e0feaedd66c25e303cbbcbc6 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 14:11:13 -0500 Subject: [PATCH 10/27] Ignore 2 type errors in tests --- test/integration/test_celery_tasks.py | 2 +- test/unit/app/tools/test_evaluation.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/integration/test_celery_tasks.py b/test/integration/test_celery_tasks.py index 7bf22660ae8..41e93814239 100644 --- a/test/integration/test_celery_tasks.py +++ b/test/integration/test_celery_tasks.py @@ -72,7 +72,7 @@ class TestCeleryTasksIntegration(IntegrationTestCase): assert hda_purged() def test_pdf_download(self): - short_term_storage_allocator = self._app[ShortTermStorageAllocator] + short_term_storage_allocator = self._app[ShortTermStorageAllocator] # type: ignore[type-abstract] short_term_storage_target = short_term_storage_allocator.new_target("moo.pdf", "application/pdf") request_id = short_term_storage_target.request_id pdf_download_request = GeneratePdfDownload( diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index 24473b5d7fc..69f50e641d5 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -45,7 +45,7 @@ class TestToolEvaluator(TestCase, UsesApp): self.job.history = History() self.job.history.id = 42 self.job.parameters = [JobParameter(name="thresh", value="4")] - self.evaluator = ToolEvaluator(self.app, self.tool, self.job, self.test_directory) + self.evaluator = ToolEvaluator(self.app, self.tool, self.job, self.test_directory) # type: ignore[arg-type] def tearDown(self): self.tear_down_app() From a673687ff84995568972d0c28f70699987508e0e Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 18:13:44 -0500 Subject: [PATCH 11/27] Split lint requirements into lint and typecheck --- lib/galaxy/dependencies/lint-requirements.txt | 13 ------------- lib/galaxy/dependencies/typecheck-requirements.txt | 13 +++++++++++++ 2 files changed, 13 insertions(+), 13 deletions(-) create mode 100644 lib/galaxy/dependencies/typecheck-requirements.txt diff --git a/lib/galaxy/dependencies/lint-requirements.txt b/lib/galaxy/dependencies/lint-requirements.txt index a1909d9164d..675b2c5d0d8 100644 --- a/lib/galaxy/dependencies/lint-requirements.txt +++ b/lib/galaxy/dependencies/lint-requirements.txt @@ -1,15 +1,2 @@ flake8 flake8-bugbear -mypy -types-bleach -types-boto -types-contextvars -types-dataclasses -types-docutils -types-Markdown -types-paramiko -types-pkg-resources -types-python-dateutil -types-PyYAML -types-requests -types-six diff --git a/lib/galaxy/dependencies/typecheck-requirements.txt b/lib/galaxy/dependencies/typecheck-requirements.txt new file mode 100644 index 00000000000..007c33348ec --- /dev/null +++ b/lib/galaxy/dependencies/typecheck-requirements.txt @@ -0,0 +1,13 @@ +mypy +types-bleach +types-boto +types-contextvars +types-dataclasses +types-docutils +types-Markdown +types-paramiko +types-pkg-resources +types-python-dateutil +types-PyYAML +types-requests +types-six From 15916a007481f606688b10d48aca4ab903edec7a Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 18:14:21 -0500 Subject: [PATCH 12/27] Add script to update pinned typecheck requirments --- .../dependencies/update_typecheck_requirements.sh | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100755 lib/galaxy/dependencies/update_typecheck_requirements.sh diff --git a/lib/galaxy/dependencies/update_typecheck_requirements.sh b/lib/galaxy/dependencies/update_typecheck_requirements.sh new file mode 100755 index 00000000000..2ca07e332aa --- /dev/null +++ b/lib/galaxy/dependencies/update_typecheck_requirements.sh @@ -0,0 +1,13 @@ +#!/bin/sh +set -e + +THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" + +TYPECHECK_VENV=$(mktemp -d "${TMPDIR:-/tmp}/typecheck_venv.XXXXXXXXXX") +python3.7 -m venv "${TYPECHECK_VENV}" +. "${TYPECHECK_VENV}/bin/activate" +pip install --upgrade pip setuptools +pip install -r "${THIS_DIRECTORY}/typecheck-requirements.txt" +# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 +pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-typecheck-requirements.txt" +rm -rf "${TYPECHECK_VENV}" From ca0fb5c2d42ad17c5c85c880621882229587c0a2 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 18:15:02 -0500 Subject: [PATCH 13/27] Update pinned lint and typecheck requirements --- .../dependencies/pinned-lint-requirements.txt | 18 ------------------ .../pinned-typecheck-requirements.txt | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 18 deletions(-) create mode 100644 lib/galaxy/dependencies/pinned-typecheck-requirements.txt diff --git a/lib/galaxy/dependencies/pinned-lint-requirements.txt b/lib/galaxy/dependencies/pinned-lint-requirements.txt index bc57d184442..0d242b5c6c6 100644 --- a/lib/galaxy/dependencies/pinned-lint-requirements.txt +++ b/lib/galaxy/dependencies/pinned-lint-requirements.txt @@ -3,25 +3,7 @@ flake8==5.0.4 flake8-bugbear==22.10.27 importlib-metadata==4.2.0 mccabe==0.7.0 -mypy==0.991 -mypy-extensions==0.4.3 pycodestyle==2.9.1 pyflakes==2.5.0 -tomli==2.0.1 -typed-ast==1.5.4 -types-bleach==5.0.3.1 -types-boto==2.49.18.3 -types-contextvars==2.4.7 -types-cryptography==3.3.23.2 -types-dataclasses==0.6.6 -types-docutils==0.19.1.1 -types-Markdown==3.4.2.1 -types-paramiko==2.12.0.1 -types-pkg-resources==0.1.3 -types-python-dateutil==2.8.19.4 -types-PyYAML==6.0.12.2 -types-requests==2.28.11.5 -types-six==1.16.21.4 -types-urllib3==1.26.25.4 typing_extensions==4.4.0 zipp==3.11.0 diff --git a/lib/galaxy/dependencies/pinned-typecheck-requirements.txt b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt new file mode 100644 index 00000000000..0f43f72f17a --- /dev/null +++ b/lib/galaxy/dependencies/pinned-typecheck-requirements.txt @@ -0,0 +1,19 @@ +mypy==0.991 +mypy-extensions==0.4.3 +tomli==2.0.1 +typed-ast==1.5.4 +types-bleach==5.0.3.1 +types-boto==2.49.18.3 +types-contextvars==2.4.7 +types-cryptography==3.3.23.2 +types-dataclasses==0.6.6 +types-docutils==0.19.1.1 +types-Markdown==3.4.2.1 +types-paramiko==2.12.0.1 +types-pkg-resources==0.1.3 +types-python-dateutil==2.8.19.4 +types-PyYAML==6.0.12.2 +types-requests==2.28.11.5 +types-six==1.16.21.4 +types-urllib3==1.26.25.4 +typing_extensions==4.4.0 From 9f42c7b71a95d0fd9e18ba036f8a14511e501b47 Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 18:16:25 -0500 Subject: [PATCH 14/27] Add new target to Makefile --- Makefile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Makefile b/Makefile index ca88e90de8a..72ef8212692 100644 --- a/Makefile +++ b/Makefile @@ -134,6 +134,9 @@ release-check-blocking-prs: ## Check github for release blocking PRs release-bootstrap-history: ## bootstrap history for a new release $(IN_VENV) python scripts/bootstrap_history.py --release $(RELEASE_CURR) +update-typecheck-requirements: + ./lib/galaxy/dependencies/update_typecheck_requirements.sh + update-lint-requirements: ./lib/galaxy/dependencies/update_lint_requirements.sh From 10f838f94548ba77d9c52e42d50ff9a2176d748d Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 18:25:34 -0500 Subject: [PATCH 15/27] Update tox.ini --- tox.ini | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 14030c6a210..bb0c5b5d95e 100644 --- a/tox.ini +++ b/tox.ini @@ -36,7 +36,9 @@ setenv = check_indexes: GALAXY_SKIP_CLIENT_BUILD=1 deps = coverage: coverage - lint,lint_docstring,lint_docstring_include_list,mypy: -rlib/galaxy/dependencies/pinned-lint-requirements.txt + lint,lint_docstring,lint_docstring_include_list: -rlib/galaxy/dependencies/pinned-lint-requirements.txt + mypy: -rlib/galaxy/dependencies/pinned-typecheck-requirements.txt + mypy: -rlib/galaxy/dependencies/pinned-requirements.txt test_galaxy_packages: pyyaml unit: mock-ssh-server From 724ff9e17b6c368ac6c06b588da81cdf0b7eb76b Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 18:40:18 -0500 Subject: [PATCH 16/27] Install mypy for package tests --- packages/test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/test.sh b/packages/test.sh index d2334003e41..08aca436312 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -16,6 +16,7 @@ virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" . "${TEST_ENV_DIR}/bin/activate" pip install --upgrade pip setuptools wheel pip install -r../lib/galaxy/dependencies/pinned-lint-requirements.txt +pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt # ensure ordered by dependency DAG # TODO: add selenium in once type issues are cleared up From d797c10ae8230b2de28ea476a00397695a5f4f9e Mon Sep 17 00:00:00 2001 From: John Davis Date: Tue, 29 Nov 2022 22:02:32 -0500 Subject: [PATCH 17/27] Update github lint workflow --- .github/workflows/lint.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 11cbabf98d8..5949d69df58 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -37,10 +37,14 @@ jobs: path: ~/.cache/pip key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }} - name: Cache tox env + env: + LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt' + TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt' + CORE_PATH: 'lib/galaxy/dependencies/pinned-requirements.txt' uses: actions/cache@v3 with: path: .tox - key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }}-lint + key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }}-lint-and-typecheck - name: Install tox run: pip install tox - name: Run linting From 1d75a584d658e42087b0534a8bd3850f5f8e255e Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 30 Nov 2022 11:27:41 -0500 Subject: [PATCH 18/27] Keep cache key suffix same as workflow filename Co-authored-by: Nicola Soranzo --- .github/workflows/lint.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 5949d69df58..7bff665a705 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -44,7 +44,7 @@ jobs: uses: actions/cache@v3 with: path: .tox - key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }}-lint-and-typecheck + key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }}-lint - name: Install tox run: pip install tox - name: Run linting From 786a3e43757895e51d52b3c96f935ddf27c6f18a Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 30 Nov 2022 11:31:22 -0500 Subject: [PATCH 19/27] Don't pip-install lint requirements for package tests Co-authored-by: Nicola Soranzo --- packages/test.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/test.sh b/packages/test.sh index 08aca436312..ee0cef696b5 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -15,7 +15,6 @@ TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} virtualenv -p "$TEST_PYTHON" "$TEST_ENV_DIR" . "${TEST_ENV_DIR}/bin/activate" pip install --upgrade pip setuptools wheel -pip install -r../lib/galaxy/dependencies/pinned-lint-requirements.txt pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt # ensure ordered by dependency DAG From 139d14e95280cfd5c0f391ef36b5e875908aa585 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 30 Nov 2022 17:55:04 +0000 Subject: [PATCH 20/27] Fix ``object_store`` type in ``MinimalToolApp`` protocol --- lib/galaxy/structured_app.py | 7 ++----- lib/galaxy/tools/remote_tool_eval.py | 4 ++-- test/unit/app/tools/test_evaluation.py | 2 +- 3 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index fc55a642de0..217e67bde9f 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -24,10 +24,7 @@ from galaxy.model.security import ( HostAgent, ) from galaxy.model.tags import GalaxyTagHandler -from galaxy.objectstore import ( - BaseObjectStore, - ObjectStore, -) +from galaxy.objectstore import BaseObjectStore from galaxy.quota import QuotaAgent from galaxy.security.idencoding import IdEncodingHelper from galaxy.security.vault import Vault @@ -78,7 +75,7 @@ class MinimalToolApp(Protocol): name: str config: Any # 'galaxy.config.BaseAppConfiguration' datatypes_registry: Registry - object_store: ObjectStore + object_store: BaseObjectStore tool_data_tables: "ToolDataTableManager" file_sources: ConfiguredFileSources security: IdEncodingHelper diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index d7c5a62b4f2..8dd158f822d 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -19,7 +19,7 @@ from galaxy.metadata.set_metadata import ( ) from galaxy.model import store from galaxy.model.store import SessionlessContext -from galaxy.objectstore import ObjectStore +from galaxy.objectstore import BaseObjectStore from galaxy.structured_app import MinimalToolApp from galaxy.tools import ( create_tool_from_representation, @@ -55,7 +55,7 @@ class ToolApp(MinimalToolApp): sa_session: SessionlessContext, tool_app_config: ToolAppConfig, datatypes_registry: Registry, - object_store: ObjectStore, + object_store: BaseObjectStore, tool_data_table_manager: ToolDataTableManager, file_sources: ConfiguredFileSources, ): diff --git a/test/unit/app/tools/test_evaluation.py b/test/unit/app/tools/test_evaluation.py index 69f50e641d5..24473b5d7fc 100644 --- a/test/unit/app/tools/test_evaluation.py +++ b/test/unit/app/tools/test_evaluation.py @@ -45,7 +45,7 @@ class TestToolEvaluator(TestCase, UsesApp): self.job.history = History() self.job.history.id = 42 self.job.parameters = [JobParameter(name="thresh", value="4")] - self.evaluator = ToolEvaluator(self.app, self.tool, self.job, self.test_directory) # type: ignore[arg-type] + self.evaluator = ToolEvaluator(self.app, self.tool, self.job, self.test_directory) def tearDown(self): self.tear_down_app() From b1c35bfc6a49deb6dfe41fb9093909e65d4ce920 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 30 Nov 2022 17:57:29 +0000 Subject: [PATCH 21/27] Add type annotation to ``config`` attribute of ``MinimalToolApp`` protocol --- lib/galaxy/structured_app.py | 3 ++- lib/galaxy/tools/remote_tool_eval.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index 217e67bde9f..7d1ca6830ee 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -38,6 +38,7 @@ from galaxy.webhooks import WebhooksRegistry from galaxy.workflow.trs_proxy import TrsProxy if TYPE_CHECKING: + from galaxy.config import GalaxyAppConfiguration from galaxy.config_watchers import ConfigWatchers from galaxy.jobs import JobConfiguration from galaxy.managers.collections import DatasetCollectionManager @@ -73,7 +74,7 @@ class BasicSharedApp(Container): class MinimalToolApp(Protocol): name: str - config: Any # 'galaxy.config.BaseAppConfiguration' + config: "GalaxyAppConfiguration" datatypes_registry: Registry object_store: BaseObjectStore tool_data_tables: "ToolDataTableManager" diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index 8dd158f822d..dc979c03ae4 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -5,9 +5,11 @@ import tempfile import traceback from typing import ( Callable, + cast, NamedTuple, ) +from galaxy.config import GalaxyAppConfiguration from galaxy.datatypes.registry import Registry from galaxy.files import ConfiguredFileSources from galaxy.job_execution.compute_environment import SharedComputeEnvironment @@ -60,7 +62,7 @@ class ToolApp(MinimalToolApp): file_sources: ConfiguredFileSources, ): self.model = Bunch(context=sa_session) - self.config = tool_app_config + self.config = cast(GalaxyAppConfiguration, tool_app_config) self.datatypes_registry = datatypes_registry self.object_store = object_store self.genome_builds = GenomeBuilds(self) From 67c7e7d1acb210d6c1af7c57fd8ad3a4f1f341f6 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 30 Nov 2022 13:33:42 -0500 Subject: [PATCH 22/27] Use a simple cast instead of relying on dependency injection Co-authored-by: Nicola Soranzo --- test/unit/webapps/test_webapp_base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/webapps/test_webapp_base.py b/test/unit/webapps/test_webapp_base.py index 7fffb3a98cc..d0aa4a42f22 100644 --- a/test/unit/webapps/test_webapp_base.py +++ b/test/unit/webapps/test_webapp_base.py @@ -37,7 +37,7 @@ class CORSParsingMockConfig(galaxy_mock.MockAppConfig): class TestGalaxyWebTransactionHeaders: def _new_trans(self, allowed_origin_hostnames: Optional[str] = None) -> StubGalaxyWebTransaction: - app = galaxy_mock.MockApp()[BasicSharedApp] + app = cast(BasicSharedApp, galaxy_mock.MockApp()) app.config = CORSParsingMockConfig(allowed_origin_hostnames=allowed_origin_hostnames) webapp = cast(WebApplication, galaxy_mock.MockWebapp(app.security)) environ = galaxy_mock.buildMockEnviron() From 02ea4f531edddd0f518551eb85d9974c6e60e69a Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 30 Nov 2022 13:39:16 -0500 Subject: [PATCH 23/27] Drop unused attr Co-authored-by: Nicola Soranzo --- test/unit/app/authnz/test_custos_authnz.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/unit/app/authnz/test_custos_authnz.py b/test/unit/app/authnz/test_custos_authnz.py index 3ae2dc1f9e3..e308bb8f9c1 100644 --- a/test/unit/app/authnz/test_custos_authnz.py +++ b/test/unit/app/authnz/test_custos_authnz.py @@ -222,7 +222,6 @@ class TestCustosAuthnz(TestCase): self.cookies = {} self.cookies_args = {} self.request = Request() - self.session = self.sa_session # type: ignore[has-type] self.sa_session = Session() self.user = None From 578fe8ae98a48e0d49393c72394e5afa602d9e2a Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 30 Nov 2022 13:41:16 -0500 Subject: [PATCH 24/27] Revert "Add new target to Makefile" This reverts commit 9f42c7b71a95d0fd9e18ba036f8a14511e501b47. --- Makefile | 3 --- 1 file changed, 3 deletions(-) diff --git a/Makefile b/Makefile index 72ef8212692..ca88e90de8a 100644 --- a/Makefile +++ b/Makefile @@ -134,9 +134,6 @@ release-check-blocking-prs: ## Check github for release blocking PRs release-bootstrap-history: ## bootstrap history for a new release $(IN_VENV) python scripts/bootstrap_history.py --release $(RELEASE_CURR) -update-typecheck-requirements: - ./lib/galaxy/dependencies/update_typecheck_requirements.sh - update-lint-requirements: ./lib/galaxy/dependencies/update_lint_requirements.sh From b8f2aec15515baf8563320f276014c2a3301969d Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 30 Nov 2022 14:44:30 -0500 Subject: [PATCH 25/27] Combine update lint/typecheck reqs scripts --- .../dependencies/update_lint_requirements.sh | 29 ++++++++++++++----- .../update_typecheck_requirements.sh | 13 --------- 2 files changed, 21 insertions(+), 21 deletions(-) delete mode 100755 lib/galaxy/dependencies/update_typecheck_requirements.sh diff --git a/lib/galaxy/dependencies/update_lint_requirements.sh b/lib/galaxy/dependencies/update_lint_requirements.sh index d477d941ab9..58c503e2e3f 100755 --- a/lib/galaxy/dependencies/update_lint_requirements.sh +++ b/lib/galaxy/dependencies/update_lint_requirements.sh @@ -1,13 +1,26 @@ #!/bin/sh + set -e +# This script updates the pinned requirements for both linting and typechecking. +# The requirements are split into two parts due to incompatible dependencies: +# flake8 requires importlib-metadata 4.2; typecheck requirements do not +# depende on importlib-metadata; however, they must be installed into the same +# virtual environment as Galaxy's core dependencies, which depend on +# importlib-metadata 4.13. + THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" -LINT_VENV=$(mktemp -d "${TMPDIR:-/tmp}/lint_venv.XXXXXXXXXX") -python3.7 -m venv "${LINT_VENV}" -. "${LINT_VENV}/bin/activate" -pip install --upgrade pip setuptools -pip install -r "${THIS_DIRECTORY}/lint-requirements.txt" -# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 -pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-lint-requirements.txt" -rm -rf "${LINT_VENV}" +update_pinned_reqs() { + VENV=$(mktemp -d "${TMPDIR:-/tmp}/$1_venv.XXXXXXXXXX") + python3.7 -m venv "${VENV}" + . "${VENV}/bin/activate" + pip install --upgrade pip setuptools + pip install -r "${THIS_DIRECTORY}/$1-requirements.txt" + # The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 + pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt" + rm -rf "${VENV}" +} + +update_pinned_reqs lint +update_pinned_reqs typecheck diff --git a/lib/galaxy/dependencies/update_typecheck_requirements.sh b/lib/galaxy/dependencies/update_typecheck_requirements.sh deleted file mode 100755 index 2ca07e332aa..00000000000 --- a/lib/galaxy/dependencies/update_typecheck_requirements.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/sh -set -e - -THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" - -TYPECHECK_VENV=$(mktemp -d "${TMPDIR:-/tmp}/typecheck_venv.XXXXXXXXXX") -python3.7 -m venv "${TYPECHECK_VENV}" -. "${TYPECHECK_VENV}/bin/activate" -pip install --upgrade pip setuptools -pip install -r "${THIS_DIRECTORY}/typecheck-requirements.txt" -# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 -pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-typecheck-requirements.txt" -rm -rf "${TYPECHECK_VENV}" From 303f1a04af34af4295f94d46c26f1b8d4bf1b223 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 1 Dec 2022 01:45:33 +0000 Subject: [PATCH 26/27] Revert "Add type annotation to ``config`` attribute of ``MinimalToolApp`` protocol" This reverts commit b1c35bfc6a49deb6dfe41fb9093909e65d4ce920. Also, add comment about why `MinimalToolApp.config` should be type annotated as `Any`. --- lib/galaxy/structured_app.py | 5 +++-- lib/galaxy/tools/remote_tool_eval.py | 4 +--- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/structured_app.py b/lib/galaxy/structured_app.py index 7d1ca6830ee..eaea543fd12 100644 --- a/lib/galaxy/structured_app.py +++ b/lib/galaxy/structured_app.py @@ -38,7 +38,6 @@ from galaxy.webhooks import WebhooksRegistry from galaxy.workflow.trs_proxy import TrsProxy if TYPE_CHECKING: - from galaxy.config import GalaxyAppConfiguration from galaxy.config_watchers import ConfigWatchers from galaxy.jobs import JobConfiguration from galaxy.managers.collections import DatasetCollectionManager @@ -74,7 +73,9 @@ class BasicSharedApp(Container): class MinimalToolApp(Protocol): name: str - config: "GalaxyAppConfiguration" + # Leave config as Any: in a full Galaxy app this is a GalaxyAppConfiguration object, but this is mostly dynamically + # generated, and here we want to also allow other kinds of configuration objects (e.g. a Bunch). + config: Any datatypes_registry: Registry object_store: BaseObjectStore tool_data_tables: "ToolDataTableManager" diff --git a/lib/galaxy/tools/remote_tool_eval.py b/lib/galaxy/tools/remote_tool_eval.py index dc979c03ae4..8dd158f822d 100644 --- a/lib/galaxy/tools/remote_tool_eval.py +++ b/lib/galaxy/tools/remote_tool_eval.py @@ -5,11 +5,9 @@ import tempfile import traceback from typing import ( Callable, - cast, NamedTuple, ) -from galaxy.config import GalaxyAppConfiguration from galaxy.datatypes.registry import Registry from galaxy.files import ConfiguredFileSources from galaxy.job_execution.compute_environment import SharedComputeEnvironment @@ -62,7 +60,7 @@ class ToolApp(MinimalToolApp): file_sources: ConfiguredFileSources, ): self.model = Bunch(context=sa_session) - self.config = cast(GalaxyAppConfiguration, tool_app_config) + self.config = tool_app_config self.datatypes_registry = datatypes_registry self.object_store = object_store self.genome_builds = GenomeBuilds(self) From f07207ad6d2abfa46102f63ec29dca77d1802a94 Mon Sep 17 00:00:00 2001 From: John Davis Date: Wed, 30 Nov 2022 21:10:04 -0500 Subject: [PATCH 27/27] Use 3 requirements files in the key for pip cache --- .github/workflows/lint.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml index 7bff665a705..0125f24c4d4 100644 --- a/.github/workflows/lint.yaml +++ b/.github/workflows/lint.yaml @@ -22,6 +22,10 @@ jobs: strategy: matrix: python-version: ['3.7', '3.10'] + env: + LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt' + TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt' + CORE_PATH: 'lib/galaxy/dependencies/pinned-requirements.txt' steps: - uses: actions/checkout@v3 - uses: actions/setup-python@v4 @@ -35,12 +39,8 @@ jobs: uses: actions/cache@v3 with: path: ~/.cache/pip - key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('lib/galaxy/dependencies/pinned-lint-requirements.txt') }} + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles(env.LINT_PATH, env.TYPE_PATH, env.CORE_PATH) }} - name: Cache tox env - env: - LINT_PATH: 'lib/galaxy/dependencies/pinned-lint-requirements.txt' - TYPE_PATH: 'lib/galaxy/dependencies/pinned-typecheck-requirements.txt' - CORE_PATH: 'lib/galaxy/dependencies/pinned-requirements.txt' uses: actions/cache@v3 with: path: .tox