From 112d34f32c0340a18f6dee38099f69877a11898c Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 4 Apr 2022 13:19:10 +0100 Subject: [PATCH 1/4] Update uvicorn from broken 0.17.1 Fix issue found when testing planemo in https://github.com/galaxyproject/planemo/pull/1224#issuecomment-1086201126 : ``` 2022-04-01T14:44:05.3964452Z [2022-04-01 14:41:23 +0000] [6904] [WARNING] Invalid HTTP request received. 2022-04-01T14:44:05.3964583Z Traceback (most recent call last): 2022-04-01T14:44:05.3965087Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/uvicorn/protocols/http/h11_impl.py", line 136, in handle_events 2022-04-01T14:44:05.3965218Z event = self.conn.next_event() 2022-04-01T14:44:05.3965652Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/h11/_connection.py", line 443, in next_event 2022-04-01T14:44:05.3965801Z exc._reraise_as_remote_protocol_error() 2022-04-01T14:44:05.3966277Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/h11/_util.py", line 76, in _reraise_as_remote_protocol_error 2022-04-01T14:44:05.3966383Z raise self 2022-04-01T14:44:05.3966831Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/h11/_connection.py", line 425, in next_event 2022-04-01T14:44:05.3966977Z event = self._extract_next_receive_event() 2022-04-01T14:44:05.3967445Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/h11/_connection.py", line 367, in _extract_next_receive_event 2022-04-01T14:44:05.3967588Z event = self._reader(self._receive_buffer) 2022-04-01T14:44:05.3968049Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/h11/_readers.py", line 73, in maybe_read_from_IDLE_client 2022-04-01T14:44:05.3968225Z request_line_re, lines[0], "illegal request line: {!r}", lines[0] 2022-04-01T14:44:05.3968629Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/h11/_util.py", line 88, in validate 2022-04-01T14:44:05.3968781Z raise LocalProtocolError(msg) 2022-04-01T14:44:05.3969215Z h11._util.RemoteProtocolError: illegal request line: bytearray(b'--a250a2c0546542508998e4e0ac8f1001') 2022-04-01T14:44:05.3969497Z [2022-04-01 14:41:23 +0000] [6904] [WARNING] Invalid HTTP request received. 2022-04-01T14:44:05.3969623Z Traceback (most recent call last): 2022-04-01T14:44:05.3970105Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/uvicorn/protocols/http/h11_impl.py", line 136, in handle_events 2022-04-01T14:44:05.3970231Z event = self.conn.next_event() 2022-04-01T14:44:05.3970655Z File "/tmp/planemo-test-workspace/gx_venv_3/lib/python3.7/site-packages/h11/_connection.py", line 423, in next_event 2022-04-01T14:44:05.3971003Z raise RemoteProtocolError("Can't receive data when peer state is ERROR") 2022-04-01T14:44:05.3971287Z h11._util.RemoteProtocolError: Can't receive data when peer state is ERROR ``` --- lib/galaxy/dependencies/dev-requirements.txt | 2 +- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/dependencies/dev-requirements.txt b/lib/galaxy/dependencies/dev-requirements.txt index d269a4704b3..ae8d4553c24 100644 --- a/lib/galaxy/dependencies/dev-requirements.txt +++ b/lib/galaxy/dependencies/dev-requirements.txt @@ -220,7 +220,7 @@ typing-extensions==4.0.1; python_version >= "3.6" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.2 urllib3==1.26.8; python_version >= "3.7" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.7" -uvicorn==0.17.1; python_version >= "3.7" +uvicorn==0.17.6; python_version >= "3.7" uvloop==0.16.0; python_version >= "3.7" vine==5.0.0; python_version >= "3.7" watchdog==2.1.6; python_version >= "3.6" diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index bae02105d30..04817885bd1 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -158,7 +158,7 @@ typing-extensions==4.0.1; python_version >= "3.6" tzlocal==2.1; python_version >= "2.7" and python_full_version < "3.0.0" or python_full_version >= "3.5.0" and python_version < "4" ubiquerg==0.6.2 urllib3==1.26.8; python_version >= "3.6" and python_full_version < "3.0.0" and python_version < "4" or python_full_version >= "3.6.0" and python_version < "4" and python_version >= "3.6" -uvicorn==0.17.1; python_version >= "3.7" +uvicorn==0.17.6; python_version >= "3.7" uvloop==0.16.0; python_version >= "3.7" vine==5.0.0; python_version >= "3.7" wcwidth==0.2.5; python_full_version >= "3.6.2" and python_version >= "3.7" From b489d9dc7a27e4e6fbdd54d4ac73244ed5f7d8bb Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 4 Mar 2022 11:07:26 -0500 Subject: [PATCH 2/4] Make test_user_library_permissions.py more robust. --- .../test_user_library_permissions.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/test/integration_selenium/test_user_library_permissions.py b/test/integration_selenium/test_user_library_permissions.py index e346418f2de..21347af1484 100644 --- a/test/integration_selenium/test_user_library_permissions.py +++ b/test/integration_selenium/test_user_library_permissions.py @@ -1,5 +1,6 @@ import os +from galaxy_test.selenium.framework import retry_assertion_during_transitions from .framework import ( selenium_test, SeleniumIntegrationTestCase, @@ -90,7 +91,8 @@ class TestUserLibraryImport(SeleniumIntegrationTestCase): # search for created user and add him to permission field self.components.libraries.add_items_permission.wait_for_and_click() self.components.libraries.add_items_permission_input_field.wait_for_and_send_keys(email) - self.components.libraries.add_items_permission_option.wait_for_and_click() + + self.select_add_items_permission_option(email) # assert that the right email has been saved allowed_user_email = self.components.libraries.add_items_permission_field_text.wait_for_text() @@ -102,3 +104,9 @@ class TestUserLibraryImport(SeleniumIntegrationTestCase): self.logout() # login back to the 'regular' user account self.submit_login(email=email) + + @retry_assertion_during_transitions + def select_add_items_permission_option(self, option_text): + el = self.components.libraries.add_items_permission_option.wait_for_visible() + assert option_text == el.text + el.click() From d5e9901f3fedbb80bac0efbbf90f79c9fd225bcc Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Wed, 30 Mar 2022 19:37:29 +0200 Subject: [PATCH 3/4] Forward type annotation of depends --- lib/galaxy/webapps/galaxy/api/__init__.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 49ea15124b5..2c59117a1e9 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -83,8 +83,7 @@ class GalaxyTypeDepends(Depends): self.galaxy_type_depends = dep_type -def depends(dep_type: Type[T]) -> Any: - +def depends(dep_type: Type[T]) -> T: def _do_resolve(request: Request): return get_app().resolve(dep_type) From b5fb285ca753c3e8fd61e3c86c143491b2255041 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 31 Mar 2022 18:46:37 +0100 Subject: [PATCH 4/4] Update mypy to 0.942 . Fix new mypy errors. Remove unused `optional()` decorator function that had an annotation issue. --- lib/galaxy/celery/_serialization.py | 4 +-- lib/galaxy/datatypes/binary.py | 6 +++-- .../datatypes/converters/cram_to_bam.py | 6 ++--- lib/galaxy/dependencies/lint-requirements.txt | 2 +- .../dependencies/pinned-lint-requirements.txt | 4 +-- lib/galaxy/schema/fields.py | 25 +------------------ lib/galaxy/tool_util/cwl/util.py | 8 +++++- lib/galaxy/util/compression_utils.py | 8 +++--- lib/galaxy/util/properties.py | 20 +-------------- lib/galaxy/webapps/galaxy/api/__init__.py | 2 +- lib/galaxy/workflow/modules.py | 4 +-- lib/galaxy_test/driver/driver_util.py | 11 ++++---- lib/tool_shed/tools/tool_validator.py | 9 +++---- test/unit/data/datatypes/test_bam.py | 14 ++++++----- 14 files changed, 45 insertions(+), 78 deletions(-) diff --git a/lib/galaxy/celery/_serialization.py b/lib/galaxy/celery/_serialization.py index a7a6b3a4670..49ca98bb1a3 100644 --- a/lib/galaxy/celery/_serialization.py +++ b/lib/galaxy/celery/_serialization.py @@ -34,8 +34,8 @@ def schema_decoder(obj): assert clazz_str.startswith(SCHEMA_MODELS_PACKAGE_BASE) and ".." not in clazz_str, f"Invalid class str {clazz_str}" module_name, class_name = clazz_str.rsplit('.', 1) module = import_module(module_name) - clazz = getattr(module, class_name, None) - obj = clazz(**obj['__object__']) + clazz = getattr(module, class_name) + obj = clazz(**obj["__object__"]) return obj return obj diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 28d463c9d22..acb218492c1 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -19,8 +19,10 @@ from typing import Optional import h5py import numpy as np import pysam -import pysam.bcftools -from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP +from bx.seq.twobit import ( + TWOBIT_MAGIC_NUMBER, + TWOBIT_MAGIC_NUMBER_SWAP, +) from galaxy import util from galaxy.datatypes import metadata diff --git a/lib/galaxy/datatypes/converters/cram_to_bam.py b/lib/galaxy/datatypes/converters/cram_to_bam.py index f139442835f..08555f603e6 100644 --- a/lib/galaxy/datatypes/converters/cram_to_bam.py +++ b/lib/galaxy/datatypes/converters/cram_to_bam.py @@ -7,7 +7,7 @@ usage: %prog in_file out_file import optparse import os -import pysam +from pysam import sort # type: ignore[attr-defined] def main(): @@ -15,8 +15,8 @@ def main(): parser = optparse.OptionParser() (options, args) = parser.parse_args() input_fname, output_fname = args - slots = os.getenv('GALAXY_SLOTS', 1) - pysam.sort(f"-@{slots}", '-o', output_fname, '-O', 'bam', '-T', '.', input_fname) + slots = os.getenv("GALAXY_SLOTS", 1) + sort(f"-@{slots}", "-o", output_fname, "-O", "bam", "-T", ".", input_fname) if __name__ == "__main__": diff --git a/lib/galaxy/dependencies/lint-requirements.txt b/lib/galaxy/dependencies/lint-requirements.txt index 4360f8e7166..88af327aff4 100644 --- a/lib/galaxy/dependencies/lint-requirements.txt +++ b/lib/galaxy/dependencies/lint-requirements.txt @@ -1,7 +1,7 @@ flake8 flake8-bugbear flake8-import-order -mypy==0.910 +mypy types-bleach types-boto types-contextvars diff --git a/lib/galaxy/dependencies/pinned-lint-requirements.txt b/lib/galaxy/dependencies/pinned-lint-requirements.txt index f6212039a2f..bf404c41afc 100644 --- a/lib/galaxy/dependencies/pinned-lint-requirements.txt +++ b/lib/galaxy/dependencies/pinned-lint-requirements.txt @@ -4,11 +4,11 @@ flake8-bugbear==22.1.11 flake8-import-order==0.18.1 importlib-metadata==4.2.0 mccabe==0.6.1 -mypy==0.910 +mypy==0.942 mypy-extensions==0.4.3 pycodestyle==2.8.0 pyflakes==2.4.0 -toml==0.10.2 +tomli==2.0.1 typed-ast==1.4.3 types-bleach==4.1.4 types-boto==2.49.7 diff --git a/lib/galaxy/schema/fields.py b/lib/galaxy/schema/fields.py index 01f7ba8b868..e1bc68a7869 100644 --- a/lib/galaxy/schema/fields.py +++ b/lib/galaxy/schema/fields.py @@ -1,11 +1,7 @@ -import inspect import re from typing import Optional -from pydantic import ( - BaseModel, - Field, -) +from pydantic import Field from galaxy.security.idencoding import IdEncodingHelper @@ -110,22 +106,3 @@ def OrderParamField(default_order: str) -> Optional[str]: ), example="name-dsc,create_time", ) - - -def optional(*fields): - """Decorator function used to modify a pydantic model's fields to all be optional. - Alternatively, you can also pass the field names that should be made optional as arguments - to the decorator. - Taken from https://github.com/samuelcolvin/pydantic/issues/1223#issuecomment-775363074 - """ - def dec(_cls): - for field in fields: - _cls.__fields__[field].required = False - return _cls - - if fields and inspect.isclass(fields[0]) and issubclass(fields[0], BaseModel): - cls = fields[0] - fields = cls.__fields__ - return dec(cls) - - return dec diff --git a/lib/galaxy/tool_util/cwl/util.py b/lib/galaxy/tool_util/cwl/util.py index b4b5d9ad9b5..dc7cb6f5b8c 100644 --- a/lib/galaxy/tool_util/cwl/util.py +++ b/lib/galaxy/tool_util/cwl/util.py @@ -10,7 +10,12 @@ import tarfile import tempfile import urllib.parse from collections import namedtuple -from typing import Any, List, Optional +from typing import ( + Any, + BinaryIO, + List, + Optional, +) import yaml from typing_extensions import TypedDict @@ -57,6 +62,7 @@ def output_properties( ) -> OutputPropertiesType: checksum = hashlib.sha1() properties: OutputPropertiesType = {"class": "File", "checksum": "", "size": 0} + f: BinaryIO if path is not None: properties["path"] = path f = open(path, "rb") diff --git a/lib/galaxy/util/compression_utils.py b/lib/galaxy/util/compression_utils.py index e102ff91e54..9ffc81787cd 100644 --- a/lib/galaxy/util/compression_utils.py +++ b/lib/galaxy/util/compression_utils.py @@ -113,8 +113,9 @@ def get_fileobj_raw( compressed_format = None if 'gzip' in compressed_formats and is_gzip(filename): fh: Union[gzip.GzipFile, bz2.BZ2File, IO[bytes]] = gzip.GzipFile(filename, mode) - compressed_format = 'gzip' - elif 'bz2' in compressed_formats and is_bz2(filename): + compressed_format = "gzip" + elif "bz2" in compressed_formats and is_bz2(filename): + mode = cast(Literal["a", "ab", "r", "rb", "w", "wb", "x", "xb"], mode) fh = bz2.BZ2File(filename, mode) compressed_format = 'bz2' elif 'zip' in compressed_formats and zipfile.is_zipfile(filename): @@ -123,7 +124,7 @@ def get_fileobj_raw( # since it always opens files in binary mode. # For emulating text mode, we will be returning the binary fh in a # TextIOWrapper. - zf_mode = mode.replace('b', '') + zf_mode = cast(Literal["r", "w"], mode.replace("b", "")) with zipfile.ZipFile(filename, zf_mode) as zh: fh = zh.open(zh.namelist()[0], zf_mode) compressed_format = 'zip' @@ -325,6 +326,7 @@ class CompressedFile: return tarfile.open(filepath, mode, errorlevel=0) def open_zip(self, filepath: str, mode: str) -> zipfile.ZipFile: + mode = cast(Literal["a", "r", "w", "x"], mode) return zipfile.ZipFile(filepath, mode) def zipfile_ok(self, path_to_archive: str) -> bool: diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 64638a46199..3f69718b418 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -138,11 +138,7 @@ class NicerConfigParser(ConfigParser): def __init__(self, filename, *args, **kw): ConfigParser.__init__(self, *args, **kw) self.filename = filename - if hasattr(self, '_interpolation'): - self._interpolation = self.InterpolateWrapper(self._interpolation) - - read_file = getattr(ConfigParser, 'read_file', ConfigParser.readfp) - read_file.__doc__ = "" + self._interpolation = self.InterpolateWrapper(self._interpolation) def defaults(self): """Return the defaults, with their values interpolated (with the @@ -155,21 +151,7 @@ class NicerConfigParser(ConfigParser): defaults[key] = self.get('DEFAULT', key) or val return defaults - def _interpolate(self, section, option, rawval, vars): - # Python < 3.2 - try: - return ConfigParser._interpolate( - self, section, option, rawval, vars) - except Exception: - e = sys.exc_info()[1] - args = list(e.args) - args[0] = f'Error in file {self.filename}: {e}' - e.args = tuple(args) - e.message = args[0] - raise - class InterpolateWrapper: - # Python >= 3.2 def __init__(self, original): self._original = original diff --git a/lib/galaxy/webapps/galaxy/api/__init__.py b/lib/galaxy/webapps/galaxy/api/__init__.py index 2c59117a1e9..af1a974638a 100644 --- a/lib/galaxy/webapps/galaxy/api/__init__.py +++ b/lib/galaxy/webapps/galaxy/api/__init__.py @@ -87,7 +87,7 @@ def depends(dep_type: Type[T]) -> T: def _do_resolve(request: Request): return get_app().resolve(dep_type) - return GalaxyTypeDepends(_do_resolve, dep_type) + return cast(T, GalaxyTypeDepends(_do_resolve, dep_type)) def get_session_manager(app: StructuredApp = DependsOnApp) -> GalaxySessionManager: diff --git a/lib/galaxy/workflow/modules.py b/lib/galaxy/workflow/modules.py index ba3d80cc432..9185e5d08ec 100644 --- a/lib/galaxy/workflow/modules.py +++ b/lib/galaxy/workflow/modules.py @@ -1755,8 +1755,8 @@ class ToolModule(WorkflowModule): # See https://github.com/galaxyproject/galaxy/pull/1693 for context. replacement = dataset_instance temp = iteration_elements[prefixed_name] - if hasattr(temp, 'element_identifier') and temp.element_identifier: - replacement.element_identifier = temp.element_identifier # type: ignore[attr-defined] + if hasattr(temp, "element_identifier") and temp.element_identifier: + replacement.element_identifier = temp.element_identifier # type: ignore[union-attr] else: # If collection - just use element model object. replacement = iteration_elements[prefixed_name] diff --git a/lib/galaxy_test/driver/driver_util.py b/lib/galaxy_test/driver/driver_util.py index 07e8ad2a81b..88bd7eb556f 100644 --- a/lib/galaxy_test/driver/driver_util.py +++ b/lib/galaxy_test/driver/driver_util.py @@ -1022,6 +1022,8 @@ class GalaxyTestDriver(TestDriver): self.testing_shed_tools = getattr(config_object, "testing_shed_tools", False) + default_tool_conf: Optional[str] + datatypes_conf_override: Optional[str] if getattr(config_object, "framework_tool_and_types", False): default_tool_conf = FRAMEWORK_SAMPLE_TOOLS_CONF datatypes_conf_override = FRAMEWORK_DATATYPES_CONF @@ -1065,8 +1067,8 @@ class GalaxyTestDriver(TestDriver): if callable(galaxy_config): galaxy_config = galaxy_config() if galaxy_config is None: - setup_galaxy_config_kwds = dict( - allow_path_paste=getattr(config_object, "allow_path_paste", False), + galaxy_config = setup_galaxy_config( + galaxy_db_path, use_test_file_dir=not self.testing_shed_tools, default_install_db_merged=True, default_tool_conf=self.default_tool_conf, @@ -1077,10 +1079,7 @@ class GalaxyTestDriver(TestDriver): conda_auto_install=getattr(config_object, "conda_auto_install", False), use_shared_connection_for_amqp=getattr(config_object, "use_shared_connection_for_amqp", False), allow_tool_conf_override=self.allow_tool_conf_override, - ) - galaxy_config = setup_galaxy_config( - galaxy_db_path, - **setup_galaxy_config_kwds + allow_path_paste=getattr(config_object, "allow_path_paste", False), ) isolate_galaxy_config = getattr(config_object, "isolate_galaxy_config", False) diff --git a/lib/tool_shed/tools/tool_validator.py b/lib/tool_shed/tools/tool_validator.py index d609a8ffc90..fb9815cbb25 100644 --- a/lib/tool_shed/tools/tool_validator.py +++ b/lib/tool_shed/tools/tool_validator.py @@ -89,12 +89,9 @@ class ToolValidator(GalaxyToolValidator): # 'ncbi_blastp_wrapper.xml' was moved to 'tools/ncbi_blast_plus/ncbi_blastp_wrapper.xml', # so keep looking for the file until we find the new location. continue - fh = tempfile.NamedTemporaryFile('wb', prefix="tmp-toolshed-gltcrfrm") - tmp_filename = fh.name - fh.close() - fh = open(tmp_filename, 'wb') - fh.write(fctx.data()) - fh.close() + with tempfile.NamedTemporaryFile("wb", prefix="tmp-toolshed-gltcrfrm", delete=False) as fh: + tmp_filename = fh.name + fh.write(fctx.data()) return tmp_filename return None diff --git a/test/unit/data/datatypes/test_bam.py b/test/unit/data/datatypes/test_bam.py index 06ccf924bf4..57f14a61b4f 100644 --- a/test/unit/data/datatypes/test_bam.py +++ b/test/unit/data/datatypes/test_bam.py @@ -1,4 +1,7 @@ -import pysam +from pysam import ( # type: ignore[attr-defined] + AlignmentFile, + view, +) from galaxy.datatypes.binary import Bam from .util import ( @@ -11,8 +14,8 @@ from .util import ( def test_merge_bam(): with get_input_files('1.bam', '1.bam') as input_files, get_tmp_path() as outpath: Bam.merge(input_files, outpath) - alignment_count_output = int(pysam.view('-c', outpath).strip()) - alignment_count_input = int(pysam.view('-c', input_files[0]).strip()) * 2 + alignment_count_output = int(view("-c", outpath).strip()) + alignment_count_input = int(view("-c", input_files[0]).strip()) * 2 assert alignment_count_input == alignment_count_output @@ -42,9 +45,8 @@ def test_set_meta_presorted(): b = Bam() with get_dataset('1.bam') as dataset: b.set_meta(dataset=dataset) - assert dataset.metadata.sort_order == 'coordinate' - bam_file = pysam.AlignmentFile(dataset.file_name, mode='rb', - index_filename=dataset.metadata.bam_index.file_name) + assert dataset.metadata.sort_order == "coordinate" + bam_file = AlignmentFile(dataset.file_name, mode="rb", index_filename=dataset.metadata.bam_index.file_name) assert bam_file.has_index() is True