Merge pull request #11000 from jmchilton/more_types_0

Check Python typing for galaxy-containers and galaxy-tool-util during CI.

lib/galaxy/containers/__init__.py

@@ -15,6 +15,7 @@ from abc import (
     abstractproperty
 )
 from collections import namedtuple
+from typing import Any, Dict, Optional, Type
 
 import yaml
 
@@ -166,13 +167,13 @@ class Container(metaclass=ABCMeta):
 
 class ContainerInterface(metaclass=ABCMeta):
 
-    container_type = None
-    container_class = None
-    volume_class = None
-    conf_defaults = {
+    container_type: Optional[str] = None
+    container_class: Optional[Type[Container]] = None
+    volume_class = Optional[Type[ContainerVolume]]
+    conf_defaults: Dict[str, Optional[Any]] = {
         'name_prefix': 'galaxy_',
     }
-    option_map = {}
+    option_map: Dict[str, Dict] = {}
     publish_port_list_required = False
     supports_volumes = True
 

lib/galaxy/containers/docker.py

@@ -8,19 +8,20 @@ import shlex
 from functools import partial
 from itertools import cycle, repeat
 from time import sleep
+from typing import Any, Dict, Optional, Type
 
 try:
     import docker
 except ImportError:
-    docker = None
+    docker = None  # type: ignore
 
 try:
     from requests.exceptions import ConnectionError, ReadTimeout
 except ImportError:
-    ConnectionError = None
-    ReadTimeout = None
+    ConnectionError = None  # type: ignore
+    ReadTimeout = None  # type: ignore
 
-from galaxy.containers import ContainerInterface
+from galaxy.containers import Container, ContainerInterface
 from galaxy.containers.docker_decorators import (
     docker_columns,
     docker_json
@@ -41,9 +42,9 @@ log = logging.getLogger(__name__)
 
 class DockerInterface(ContainerInterface):
 
-    container_class = DockerContainer
+    container_class: Type[Container] = DockerContainer
     volume_class = DockerVolume
-    conf_defaults = {
+    conf_defaults: Dict[str, Optional[Any]] = {
         'host': None,
         'tls': False,
         'force_tlsverify': False,
@@ -107,7 +108,7 @@ class DockerInterface(ContainerInterface):
 class DockerCLIInterface(DockerInterface):
 
     container_type = 'docker_cli'
-    conf_defaults = {
+    conf_defaults: Dict[str, Optional[Any]] = {
         'command_template': '{executable} {global_kwopts} {subcommand} {args}',
         'executable': 'docker',
     }
@@ -226,7 +227,7 @@ class DockerAPIClient:
     _host_iter = None
     _client = None
     _client_args = ()
-    _client_kwargs = {}
+    _client_kwargs: Dict[str, Optional[Any]] = {}
 
     @staticmethod
     def _qualname(f):

lib/galaxy/containers/docker_swarm.py

@@ -6,6 +6,7 @@ import logging
 import os.path
 import subprocess
 from functools import partial
+from typing import Any, Dict, Optional
 
 try:
     import docker.types
@@ -50,7 +51,7 @@ SWARM_MANAGER_PATH = os.path.abspath(
 class DockerSwarmInterface(DockerInterface):
 
     container_class = DockerService
-    conf_defaults = {
+    conf_defaults: Dict[str, Optional[Any]] = {
         'ignore_volumes': False,
         'node_prefix': None,
         'service_create_image_constraint': False,
@@ -370,7 +371,7 @@ class DockerSwarmAPIInterface(DockerSwarmInterface, DockerAPIInterface):
         'service_mode': {'param': 0, 'default': 'replicated'},
         'replicas': {'default': 1},
     }
-    endpoint_spec_option_map = {
+    endpoint_spec_option_map: Dict[str, Dict] = {
         'ports': {},
     }
     resources_option_map = {

lib/galaxy/tool_util/cwl/cwltool_deps.py

@@ -9,10 +9,7 @@ import warnings
 
 warnings.filterwarnings("ignore", message=r"[\n.]DEPRECATION: Python 2", module="cwltool")
 
-try:
-    import requests
-except ImportError:
-    requests = None
+import requests
 
 try:
     from cwltool import (
@@ -23,11 +20,11 @@ try:
         pathmapper,
     )
 except ImportError:
-    main = None
-    workflow = None
-    job = None
-    process = None
-    pathmapper = None
+    main = None  # type: ignore
+    workflow = None  # type: ignore
+    job = None  # type: ignore
+    process = None  # type: ignore
+    pathmapper = None  # type: ignore
 
 try:
     from cwltool.context import (
@@ -38,11 +35,11 @@ try:
     from cwltool.job import relink_initialworkdir
     from cwltool.stdfsaccess import StdFsAccess
 except ImportError:
-    getdefault = None
-    LoadingContext = None
-    relink_initialworkdir = None
-    RuntimeContext = None
-    StdFsAccess = None
+    getdefault = None  # type: ignore
+    LoadingContext = None  # type: ignore
+    relink_initialworkdir = None  # type: ignore
+    RuntimeContext = None  # type: ignore
+    StdFsAccess = None  # type: ignore
 
 try:
     from cwltool import load_tool
@@ -51,31 +48,31 @@ try:
         resolve_and_validate_document,
     )
 except ImportError:
-    default_loader = None
-    load_tool = None
-    resolve_and_validate_document = None
+    default_loader = None  # type: ignore
+    load_tool = None  # type: ignore
+    resolve_and_validate_document = None  # type: ignore
 
 try:
     from cwltool import command_line_tool
     command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE
 except ImportError:
-    command_line_tool = None
+    command_line_tool = None  # type: ignore
 
 try:
     from cwltool.load_tool import resolve_and_validate_document
 except ImportError:
-    resolve_and_validate_document = None
+    resolve_and_validate_document = None  # type: ignore
 
 try:
     from cwltool import command_line_tool
     command_line_tool.ACCEPTLIST_RE = command_line_tool.ACCEPTLIST_EN_RELAXED_RE
 except ImportError:
-    command_line_tool = None
+    command_line_tool = None  # type: ignore
 
 try:
     import shellescape
 except ImportError:
-    shellescape = None
+    shellescape = None  # type: ignore
 
 try:
     import schema_salad
@@ -84,9 +81,9 @@ try:
         sourceline,
     )
 except ImportError:
-    schema_salad = None
-    ref_resolver = None
-    sourceline = None
+    schema_salad = None  # type: ignore
+    ref_resolver = None  # type: ignore
+    sourceline = None  # type: ignore
 
 needs_shell_quoting = re.compile(r"""(^$|[\s|&;()<>\'"$@])""").search
 

lib/galaxy/tool_util/cwl/representation.py

@@ -5,7 +5,7 @@ import collections
 import json
 import logging
 import os
-
+from typing import Any, NamedTuple, Optional
 
 from galaxy.exceptions import RequestParameterInvalidException
 from galaxy.util import safe_makedirs, string_as_bool
@@ -50,7 +50,18 @@ USE_FIELD_TYPES = True
 #     easier to manage in someways.
 USE_STEP_PARAMETERS = USE_FIELD_TYPES
 
-TypeRepresentation = collections.namedtuple("TypeRepresentation", ["name", "galaxy_param_type", "label", "collection_type"])
+
+class TypeRepresentation(NamedTuple):
+    name: str
+    galaxy_param_type: Any
+    label: str
+    collection_type: Optional[str]
+
+    @property
+    def uses_param(self):
+        return self.galaxy_param_type is not NO_GALAXY_INPUT
+
+
 TYPE_REPRESENTATIONS = [
     TypeRepresentation("null", NO_GALAXY_INPUT, "no input", None),
     TypeRepresentation("integer", INPUT_TYPE.INTEGER, "an integer", None),
@@ -67,7 +78,6 @@ TYPE_REPRESENTATIONS = [
     TypeRepresentation("field", INPUT_TYPE.FIELD, "arbitrary JSON structure", None),
 ]
 FIELD_TYPE_REPRESENTATION = TYPE_REPRESENTATIONS[-1]
-TypeRepresentation.uses_param = lambda self: self.galaxy_param_type is not NO_GALAXY_INPUT
 
 if not USE_FIELD_TYPES:
     CWL_TYPE_TO_REPRESENTATIONS = {

lib/galaxy/tool_util/deps/container_resolvers/mulled.py

@@ -1,10 +1,9 @@
 """This module describes the :class:`MulledContainerResolver` ContainerResolver plugin."""
 
-import collections
 import logging
 import os
 import subprocess
-
+from typing import NamedTuple, Optional
 
 from galaxy.util import (
     string_as_bool,
@@ -38,27 +37,40 @@ from ..requirements import (
 log = logging.getLogger(__name__)
 
 
-CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"])
-CachedV1MulledImageMultiTarget = collections.namedtuple("CachedV1MulledImageMultiTarget", ["hash", "build", "image_identifier"])
-CachedV2MulledImageMultiTarget = collections.namedtuple("CachedV2MulledImageMultiTarget", ["image_name", "version_hash", "build", "image_identifier"])
+class CachedMulledImageSingleTarget(NamedTuple):
+    package_name: str
+    version: str
+    build: str
+    image_identifier: str
 
-CachedMulledImageSingleTarget.multi_target = False
-CachedV1MulledImageMultiTarget.multi_target = "v1"
-CachedV2MulledImageMultiTarget.multi_target = "v2"
+    multi_target: bool = False
 
 
-@property
-def _package_hash(target):
-    # Make this work for Singularity file name or fully qualified Docker repository
-    # image names.
-    image_name = target.image_name
-    if "/" not in image_name:
-        return image_name
-    else:
-        return image_name.rsplit("/")[-1]
+class CachedV1MulledImageMultiTarget(NamedTuple):
+    hash: str
+    build: str
+    image_identifier: str
+
+    multi_target: str = "v1"
 
 
-CachedV2MulledImageMultiTarget.package_hash = _package_hash
+class CachedV2MulledImageMultiTarget(NamedTuple):
+    image_name: str
+    version_hash: str
+    build: str
+    image_identifier: str
+
+    multi_target: str = "v2"
+
+    @property
+    def package_hash(target):
+        # Make this work for Singularity file name or fully qualified Docker repository
+        # image names.
+        image_name = target.image_name
+        if "/" not in image_name:
+            return image_name
+        else:
+            return image_name.rsplit("/")[-1]
 
 
 def list_docker_cached_mulled_images(namespace=None, hash_func="v2", resolution_cache=None):
@@ -373,7 +385,7 @@ class MulledDockerContainerResolver(ContainerResolver):
     resolver_type = "mulled"
     container_type = "docker"
     shell = '/bin/bash'
-    protocol = None
+    protocol: Optional[str] = None
 
     def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds):
         super().__init__(app_info)

lib/galaxy/tool_util/deps/mulled/get_tests.py

@@ -17,8 +17,8 @@ try:
     from jinja2 import Template
     from jinja2.exceptions import UndefinedError
 except ImportError:
-    Template = None
-    UndefinedError = Exception
+    Template = None  # type: ignore
+    UndefinedError = Exception  # type: ignore
 
 from .util import (
     get_file_from_recipe_url,

lib/galaxy/tool_util/deps/mulled/mulled_build.py

@@ -20,10 +20,7 @@ import subprocess
 import sys
 from sys import platform as _platform
 
-try:
-    import yaml
-except ImportError:
-    yaml = None
+import yaml
 
 from galaxy.tool_util.deps import installable
 from galaxy.tool_util.deps.conda_util import best_search_result

lib/galaxy/tool_util/deps/mulled/mulled_build_files.py

@@ -26,7 +26,7 @@ from .mulled_build import (
     target_str_to_targets,
 )
 KNOWN_FIELDS = ["targets", "image_build", "name_override", "base_image"]
-FALLBACK_LINE_TUPLE = collections.namedtuple("_Line", " ".join(KNOWN_FIELDS))
+FALLBACK_LINE_TUPLE = collections.namedtuple("_Line", "targets image_build name_override base_image")
 
 
 def main(argv=None):

lib/galaxy/tool_util/deps/mulled/mulled_list.py

@@ -3,12 +3,9 @@
 import argparse
 import logging
 from glob import glob
+from html.parser import HTMLParser
 
 import requests
-try:
-    from html.parser import HTMLParser
-except ImportError:  # python 2
-    from HTMLParser import HTMLParser
 
 QUAY_API_ENDPOINT = 'https://quay.io/api/v1/repository'
 

lib/galaxy/tool_util/deps/mulled/mulled_search.py

@@ -6,17 +6,15 @@ import logging
 import sys
 import tempfile
 
+import requests
+
 from .mulled_list import get_singularity_containers
 from .util import build_target, v2_image_name
 
 try:
     from conda.cli.python_api import run_command
 except ImportError:
-    run_command = None
-try:
-    import requests
-except ImportError:
-    requests = None
+    run_command = None  # type: ignore
 
 try:
     from whoosh.fields import Schema
@@ -25,7 +23,7 @@ try:
     from whoosh.index import create_in
     from whoosh.qparser import QueryParser
 except ImportError:
-    Schema = TEXT = STORED = create_in = QueryParser = None
+    Schema = TEXT = STORED = create_in = QueryParser = None  # type: ignore
 
 QUAY_API_URL = 'https://quay.io/api/v1/repository'
 

lib/galaxy/tool_util/deps/resolvers/__init__.py

@@ -6,6 +6,7 @@ from abc import (
     abstractmethod,
     abstractproperty,
 )
+from typing import Any, Dict
 
 import yaml
 
@@ -27,7 +28,7 @@ class DependencyResolver(Dictifiable, metaclass=ABCMeta):
     disabled = False
     resolves_simple_dependencies = True
     can_uninstall_dependencies = False
-    config_options = {}
+    config_options: Dict[str, Any] = {}
 
     @abstractmethod
     def resolve(self, requirement, **kwds):

lib/galaxy/tool_util/locations/dockstore.py

@@ -1,9 +1,6 @@
 from urllib.parse import quote
 
-try:
-    import requests
-except ImportError:
-    requests = None
+import requests
 import yaml
 
 from ..locations import (

lib/galaxy/tool_util/verify/interactor.py

@@ -12,16 +12,13 @@ from collections import OrderedDict
 from json import dumps
 from logging import getLogger
 
+import requests
 from packaging.version import parse as parse_version, Version
 try:
     from nose.tools import nottest
 except ImportError:
     def nottest(x):
         return x
-try:
-    import requests
-except ImportError:
-    requests = None
 
 from galaxy import util
 from galaxy.tool_util.parser.interface import TestCollectionDef, TestCollectionOutputDef

packages/containers/galaxy/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore

packages/test.sh

@@ -34,7 +34,7 @@ PACKAGE_DIRS=(
 # tool_util not yet working 100%,
 # data has many problems quota, tool shed install database, etc..
 RUN_TESTS=(1 1 1 1 1 1 1 1 1 1 0 0)
-RUN_MYPY=(1 1 1 0 0 0 0 0 0 0 0 0)
+RUN_MYPY=(1 1 1 1 1 0 0 0 0 0 0 0)
 for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do
     package_dir=${PACKAGE_DIRS[$i]}
     run_tests=${RUN_TESTS[$i]}

packages/tool_util/galaxy/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore