mypy typing for galaxy-app

lib/galaxy/app.py

@@ -7,10 +7,11 @@ import galaxy.model
 import galaxy.model.security
 import galaxy.queues
 import galaxy.security
-from galaxy import config, job_metrics, jobs
+from galaxy import config, jobs
 from galaxy.config_watchers import ConfigWatchers
 from galaxy.containers import build_container_interfaces
 from galaxy.files import ConfiguredFileSources
+from galaxy.job_metrics import JobMetrics
 from galaxy.managers.collections import DatasetCollectionManager
 from galaxy.managers.folders import FolderManager
 from galaxy.managers.hdas import HDAManager
@@ -131,7 +132,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin):
 
         # Initialize job metrics manager, needs to be in place before
         # config so per-destination modifications can be made.
-        self.job_metrics = job_metrics.JobMetrics(self.config.job_metrics_config_file, app=self)
+        self.job_metrics = JobMetrics(self.config.job_metrics_config_file, app=self)
 
         # Initialize error report plugins.
         self.error_reports = ErrorReports(self.config.error_report_file, app=self)

lib/galaxy/config/__init__.py

@@ -19,6 +19,7 @@ import tempfile
 import threading
 import time
 from datetime import timedelta
+from typing import Dict, Optional, Set
 
 import yaml
 from beaker.cache import CacheManager
@@ -104,11 +105,11 @@ def find_root(kwargs):
 class BaseAppConfiguration:
     # Override in subclasses (optional): {KEY: config option, VALUE: deprecated directory name}
     # If VALUE == first directory in a user-supplied path that resolves to KEY, it will be stripped from that path
-    renamed_options = None
-    deprecated_dirs = None
-    paths_to_check_against_root = None  # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root
-    add_sample_file_to_defaults = None  # for these options, add sample config files to their defaults
-    listify_options = None  # values for these options are processed as lists of values
+    renamed_options: Optional[Dict[str, str]] = None
+    deprecated_dirs: Dict[str, str] = {}
+    paths_to_check_against_root: Set[str] = set()  # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root
+    add_sample_file_to_defaults: Set[str] = set()  # for these options, add sample config files to their defaults
+    listify_options: Set[str] = set()  # values for these options are processed as lists of values
 
     def __init__(self, **kwargs):
         self._process_renamed_options(kwargs)

lib/galaxy/config/config_manage.py

@@ -5,11 +5,11 @@ import string
 import sys
 import tempfile
 from collections import (
-    namedtuple,
     OrderedDict
 )
 from io import StringIO
 from textwrap import TextWrapper
+from typing import Any, List, NamedTuple
 
 import requests
 import yaml
@@ -52,11 +52,6 @@ YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ",
 RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent="    ", subsequent_indent="    ", break_long_words=False, break_on_hyphens=False)
 UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml"
 
-App = namedtuple(
-    "App",
-    ["config_paths", "default_port", "expected_app_factories", "destination", "schema_path", "uwsgi_module"]
-)
-
 UWSGI_OPTIONS = OrderedDict([
     ('http', {
         'desc': """The address and port on which to listen.  By default, only listen to localhost ($app_name will not be accessible over the network).  Use ':$default_port' to listen on all available network interfaces.""",
@@ -320,23 +315,31 @@ OPTION_ACTIONS = {
 }
 
 
-def _app_name(self):
-    return os.path.splitext(os.path.basename(self.destination))[0]
+class App(NamedTuple):
+    config_paths: List[str]
+    default_port: str
+    expected_app_factories: List[str]
+    destination: str
+    schema_path: str
+    uwsgi_module: str
+
+    @property
+    def app_name(self):
+        return os.path.splitext(os.path.basename(self.destination))[0]
+
+    @property
+    def sample_destination(self):
+        return self.destination + ".sample"
+
+    @property
+    def schema(self):
+        return AppSchema(self.schema_path, self.app_name)
 
 
-def _sample_destination(self):
-    return self.destination + ".sample"
-
-
-def _schema(self):
-    return AppSchema(self.schema_path, self.app_name)
-
-
-App.app_name = property(_app_name)
-App.sample_destination = property(_sample_destination)
-App.schema = property(_schema)
-
-OptionValue = namedtuple("OptionValue", ["name", "value", "option"])
+class OptionValue(NamedTuple):
+    name: str
+    value: Any
+    option: Any
 
 
 GALAXY_APP = App(

lib/galaxy/forms/forms.py

@@ -69,7 +69,7 @@ class FormDefinitionFactory:
 
 
 class FormDefinitionFieldFactory:
-    type = None
+    type: str
 
     def __get_stored_field_type(self, **kwds):
         raise Exception('not implemented')

lib/galaxy/jobs/dynamic_tool_destination.py

@@ -7,6 +7,7 @@ import os
 import re
 import sys
 from functools import reduce
+from typing import Set
 
 import numpy as np
 import yaml
@@ -25,7 +26,7 @@ verbose = True
 list of all valid priorities, inferred from the global
 default_desinations section of the config
 """
-priority_list = set()
+priority_list: Set[str] = set()
 
 """
 Instantiated to a list of all valid destinations in the job configuration file
@@ -33,7 +34,7 @@ if run directly to validate configs. Otherwise, remains None. We often check
 to see if app is None, because if it is then we'll try using the
 destination_list instead.
 -"""
-destination_list = set()
+destination_list: Set[str] = set()
 
 """
 The largest the edit distance can be for a word to be considered

lib/galaxy/jobs/runners/util/cli/factory.py

@@ -5,7 +5,7 @@ try:
     )
     code_dir = 'lib'
 except ImportError:
-    from pulsar.managers.util.cli import (
+    from pulsar.managers.util.cli import (  # type: ignore
         CliInterface,
         split_params
     )

lib/galaxy/jobs/runners/util/job_script/__init__.py

@@ -3,6 +3,7 @@ import os
 import subprocess
 import time
 from string import Template
+from typing import Any, Dict
 
 from pkg_resources import resource_string
 
@@ -42,7 +43,7 @@ DEFAULT_INTEGRITY_CHECK = True
 DEFAULT_INTEGRITY_COUNT = 35
 DEFAULT_INTEGRITY_SLEEP = .25
 REQUIRED_TEMPLATE_PARAMS = ['working_directory', 'command', 'exit_code_path']
-OPTIONAL_TEMPLATE_PARAMS = {
+OPTIONAL_TEMPLATE_PARAMS: Dict[str, Any] = {
     'galaxy_lib': None,
     'galaxy_virtual_env': None,
     'headers': '',

lib/galaxy/managers/base.py

@@ -28,6 +28,7 @@ attribute change to a model object.
 import datetime
 import logging
 import re
+from typing import Type
 
 import routes
 import sqlalchemy
@@ -156,7 +157,7 @@ class ModelManager:
     over the ORM.
     """
     model_class = object
-    foreign_key_name = None
+    foreign_key_name: str
 
     def __init__(self, app):
         self.app = app
@@ -480,7 +481,7 @@ class HasAModelManager:
     """
 
     #: the class used to create this serializer's generically accessible model_manager
-    model_manager_class = None
+    model_manager_class: Type[object]
     # examples where this doesn't really work are ConfigurationSerializer (no manager)
     # and contents (2 managers)
 

lib/galaxy/managers/containers.py

@@ -8,6 +8,7 @@ and other (nested) containers.
 # Libraries should be DatasetCollections.
 import logging
 import operator
+from typing import Optional
 
 import galaxy.exceptions
 import galaxy.util
@@ -33,7 +34,7 @@ class ContainerManagerMixin:
     contained_class = None
     subcontainer_class = None
     #: how any contents lists produced are ordered - (string) attribute name to sort on or tuple of attribute names
-    default_order_by = None
+    default_order_by: Optional[str] = None
 
     # ---- interface
     def contents(self, container):

lib/galaxy/managers/sharable.py

@@ -11,6 +11,7 @@ A sharable Galaxy object:
 """
 import logging
 import re
+from typing import Optional
 
 from sqlalchemy import true
 
@@ -36,7 +37,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur
     user_share_model = None
 
     #: the single character abbreviation used in username_and_slug: e.g. 'h' for histories: u/user/h/slug
-    SINGLE_CHAR_ABBR = None
+    SINGLE_CHAR_ABBR: Optional[str] = None
 
     def __init__(self, app):
         super().__init__(app)
@@ -316,7 +317,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur
 class SharableModelSerializer(base.ModelSerializer,
        taggable.TaggableSerializerMixin, annotatable.AnnotatableSerializerMixin, ratable.RatableSerializerMixin):
     # TODO: stub
-    SINGLE_CHAR_ABBR = None
+    SINGLE_CHAR_ABBR: Optional[str] = None
 
     def __init__(self, app, **kwargs):
         super().__init__(app, **kwargs)

lib/galaxy/tools/__init__.py

@@ -13,6 +13,7 @@ import threading
 from collections import OrderedDict
 from datetime import datetime
 from pathlib import Path
+from typing import List, Type
 from urllib.parse import unquote_plus
 
 import packaging.version
@@ -3287,11 +3288,25 @@ class FilterFromFileTool(DatabaseOperationTool):
 
 # Populate tool_type to ToolClass mappings
 tool_types = {}
-for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, ExpressionTool, InteractiveTool,
-                   DataManagerTool, DataSourceTool, AsyncDataSourceTool,
-                   UnzipCollectionTool, ZipCollectionTool, MergeCollectionTool, RelabelFromFileTool, FilterFromFileTool,
-                   BuildListCollectionTool, ExtractDatasetCollectionTool,
-                   DataDestinationTool]:
+TOOL_CLASSES: List[Type[Tool]] = [
+    Tool,
+    SetMetadataTool,
+    OutputParameterJSONTool,
+    ExpressionTool,
+    InteractiveTool,
+    DataManagerTool,
+    DataSourceTool,
+    AsyncDataSourceTool,
+    UnzipCollectionTool,
+    ZipCollectionTool,
+    MergeCollectionTool,
+    RelabelFromFileTool,
+    FilterFromFileTool,
+    BuildListCollectionTool,
+    ExtractDatasetCollectionTool,
+    DataDestinationTool
+]
+for tool_class in TOOL_CLASSES:
     tool_types[tool_class.tool_type] = tool_class
 
 

lib/galaxy/tools/data/__init__.py

@@ -17,6 +17,7 @@ import time
 from collections import OrderedDict
 from glob import glob
 from tempfile import NamedTemporaryFile
+from typing import List
 
 import refgenconf
 import requests
@@ -240,6 +241,7 @@ class ToolDataTableManager:
 
 
 class ToolDataTable:
+    type_key: str
 
     @classmethod
     def from_elem(cls, table_elem, tool_data_path, from_shed_config, filename, tool_data_path_files, other_config_dict=None):
@@ -743,7 +745,7 @@ class TabularToolDataTable(ToolDataTable, Dictifiable):
 
 class TabularToolDataField(Dictifiable):
 
-    dict_collection_visible_keys = []
+    dict_collection_visible_keys: List[str] = []
 
     def __init__(self, data):
         self.data = data

lib/galaxy/tools/error_reports/plugins/base_git.py

@@ -8,6 +8,7 @@ from abc import (
     ABCMeta,
     abstractmethod
 )
+from typing import Dict
 
 import requests
 if sys.version_info[0] < 3:
@@ -27,12 +28,12 @@ log = logging.getLogger(__name__)
 class BaseGitPlugin(ErrorPlugin, metaclass=ABCMeta):
     """Base definition to send error reports to a Git repository provider
     """
-    issue_cache = {}
-    ts_urls = {}
-    ts_repo_cache = {}
-    git_project_cache = {}
-    label_cache = {}
-    git_username_id_cache = {}
+    issue_cache: Dict[str, Dict] = {}
+    ts_urls: Dict[str, str] = {}
+    ts_repo_cache: Dict[str, Dict] = {}
+    git_project_cache: Dict[str, Dict] = {}
+    label_cache: Dict[str, Dict] = {}
+    git_username_id_cache: Dict[str, str] = {}
 
     # Git variables
     git_default_repo_owner = False

lib/galaxy/tools/toolbox/lineages/interface.py

@@ -1,4 +1,5 @@
 import threading
+from typing import Dict
 
 import packaging.version
 from sortedcontainers import SortedSet
@@ -35,7 +36,7 @@ class ToolLineage:
     """ Simple tool's loaded directly from file system with lineage
     determined solely by PEP 440 versioning scheme.
     """
-    lineages_by_id = {}
+    lineages_by_id: Dict[str, 'ToolLineage'] = {}
     lock = threading.Lock()
 
     def __init__(self, tool_id, **kwds):

lib/galaxy/visualization/data_providers/genome.py

@@ -10,6 +10,7 @@ import re
 import sys
 from contextlib import contextmanager
 from json import loads
+from typing import Dict, Union
 
 import pysam
 from bx.bbi.bigbed_file import BigBedFile
@@ -126,7 +127,7 @@ class GenomeDataProvider(BaseDataProvider):
     format (0-based, half-open coordinates) for both queries and returned data.
     """
 
-    dataset_type = None
+    dataset_type: str
 
     """
     Mapping from column name to payload data; this mapping is used to create
@@ -135,7 +136,7 @@ class GenomeDataProvider(BaseDataProvider):
 
     col_name_data_attr_mapping = {4 : { index: 5, name: 'Score' } }
     """
-    col_name_data_attr_mapping = {}
+    col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {}
 
     def __init__(self, converted_dataset=None, original_dataset=None, dependencies=None,
                  error_max_vals="Only the first %i %s in this region are displayed."):
@@ -318,7 +319,7 @@ class TabixDataProvider(GenomeDataProvider, FilterableMixin):
     Tabix index data provider for the Galaxy track browser.
     """
 
-    col_name_data_attr_mapping = {4: {'index': 4, 'name': 'Score'}}
+    col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {4: {'index': 4, 'name': 'Score'}}
 
     @contextmanager
     def open_data_file(self):
@@ -596,7 +597,7 @@ class VcfDataProvider(GenomeDataProvider):
         8-end: allele counts for each alternative
     """
 
-    col_name_data_attr_mapping = {'Qual': {'index': 6, 'name': 'Qual'}}
+    col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {'Qual': {'index': 6, 'name': 'Qual'}}
 
     dataset_type = 'variant'
 

lib/galaxy/work/context.py

@@ -29,7 +29,7 @@ class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistor
     def get_history(self, create=False):
         return self.__history
 
-    def set_history(self):
+    def set_history(self, history):
         raise NotImplementedError("Cannot change histories from a work request context.")
 
     history = property(get_history, set_history)

lib/galaxy/workflow/refactor/execute.py

@@ -1,4 +1,5 @@
 import logging
+from typing import Any, Dict
 
 from galaxy.exceptions import (
     RequestParameterInvalidException,
@@ -103,7 +104,7 @@ class WorkflowRefactorExecutor:
         input_type = action.type
         module_type = None
 
-        tool_state = {}
+        tool_state: Dict[str, Any] = {}
         if input_type in ["data", "dataset"]:
             module_type = "data_input"
         elif input_type in ["data_collection", "dataset_collection"]:

lib/galaxy/workflow/refactor/schema.py

@@ -54,6 +54,7 @@ class BaseAction(BaseModel):
 
 
 class Action:
+    action_type: str
 
     @classmethod
     def __get_validators__(cls):
@@ -184,7 +185,7 @@ union_action_classes = Union[
 
 
 ACTION_CLASSES_BY_TYPE = {}
-for action_class in union_action_classes.__args__:
+for action_class in union_action_classes.__args__:  # type: ignore
     action_type = action_class.schema()["properties"]["action_type"]["const"]
     ACTION_CLASSES_BY_TYPE[action_type] = action_class
 

packages/app/galaxy/__init__.py

@@ -1 +1 @@
-__path__ = __import__('pkgutil').extend_path(__path__, __name__)
+__path__ = __import__('pkgutil').extend_path(__path__, __name__)  # type: ignore

packages/test.sh

@@ -34,7 +34,7 @@ PACKAGE_DIRS=(
 # tool_util not yet working 100%,
 # data has many problems quota, tool shed install database, etc..
 RUN_TESTS=(1 1 1 1 1 1 1 1 1 1 1 0)
-RUN_MYPY=(1 1 1 1 1 1 1 1 1 1 0 0)
+RUN_MYPY=(1 1 1 1 1 1 1 1 1 1 1 0)
 for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do
     package_dir=${PACKAGE_DIRS[$i]}
     run_tests=${RUN_TESTS[$i]}

scripts/galaxy_main.py

@@ -45,12 +45,13 @@ except ImportError:
 log = logging.getLogger(__name__)
 
 real_file = os.path.realpath(__file__)
-GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir))
-if not os.path.exists(os.path.join(GALAXY_ROOT_DIR, 'run.sh')):
+GALAXY_ROOT_DIR_ = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir))
+if not os.path.exists(os.path.join(GALAXY_ROOT_DIR_, 'run.sh')):
     # Galaxy is installed
     GALAXY_ROOT_DIR = None
 else:
-    GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib")
+    GALAXY_ROOT_DIR = GALAXY_ROOT_DIR_
+    GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR_, "lib")
     try:
         sys.path.insert(1, GALAXY_LIB_DIR)
     except Exception: