From c599177f5f538e424f94a98fa71dc14e0211e19f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 30 Dec 2020 00:29:57 -0500 Subject: [PATCH] mypy typing for galaxy-app --- lib/galaxy/app.py | 5 ++- lib/galaxy/config/__init__.py | 11 ++--- lib/galaxy/config/config_manage.py | 45 ++++++++++--------- lib/galaxy/forms/forms.py | 2 +- lib/galaxy/jobs/dynamic_tool_destination.py | 5 ++- lib/galaxy/jobs/runners/util/cli/factory.py | 2 +- .../jobs/runners/util/job_script/__init__.py | 3 +- lib/galaxy/managers/base.py | 5 ++- lib/galaxy/managers/containers.py | 3 +- lib/galaxy/managers/sharable.py | 5 ++- lib/galaxy/tools/__init__.py | 25 ++++++++--- lib/galaxy/tools/data/__init__.py | 4 +- .../tools/error_reports/plugins/base_git.py | 13 +++--- .../tools/toolbox/lineages/interface.py | 3 +- .../visualization/data_providers/genome.py | 9 ++-- lib/galaxy/work/context.py | 2 +- lib/galaxy/workflow/refactor/execute.py | 3 +- lib/galaxy/workflow/refactor/schema.py | 3 +- packages/app/galaxy/__init__.py | 2 +- packages/test.sh | 2 +- scripts/galaxy_main.py | 7 +-- 21 files changed, 96 insertions(+), 63 deletions(-) diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index 5197464967f..97c6b971a7b 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -7,10 +7,11 @@ import galaxy.model import galaxy.model.security import galaxy.queues import galaxy.security -from galaxy import config, job_metrics, jobs +from galaxy import config, jobs from galaxy.config_watchers import ConfigWatchers from galaxy.containers import build_container_interfaces from galaxy.files import ConfiguredFileSources +from galaxy.job_metrics import JobMetrics from galaxy.managers.collections import DatasetCollectionManager from galaxy.managers.folders import FolderManager from galaxy.managers.hdas import HDAManager @@ -131,7 +132,7 @@ class UniverseApplication(config.ConfiguresGalaxyMixin): # Initialize job metrics manager, needs to be in place before # config so per-destination modifications can be made. - self.job_metrics = job_metrics.JobMetrics(self.config.job_metrics_config_file, app=self) + self.job_metrics = JobMetrics(self.config.job_metrics_config_file, app=self) # Initialize error report plugins. self.error_reports = ErrorReports(self.config.error_report_file, app=self) diff --git a/lib/galaxy/config/__init__.py b/lib/galaxy/config/__init__.py index eee53c0850e..369d64858cf 100644 --- a/lib/galaxy/config/__init__.py +++ b/lib/galaxy/config/__init__.py @@ -19,6 +19,7 @@ import tempfile import threading import time from datetime import timedelta +from typing import Dict, Optional, Set import yaml from beaker.cache import CacheManager @@ -104,11 +105,11 @@ def find_root(kwargs): class BaseAppConfiguration: # Override in subclasses (optional): {KEY: config option, VALUE: deprecated directory name} # If VALUE == first directory in a user-supplied path that resolves to KEY, it will be stripped from that path - renamed_options = None - deprecated_dirs = None - paths_to_check_against_root = None # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root - add_sample_file_to_defaults = None # for these options, add sample config files to their defaults - listify_options = None # values for these options are processed as lists of values + renamed_options: Optional[Dict[str, str]] = None + deprecated_dirs: Dict[str, str] = {} + paths_to_check_against_root: Set[str] = set() # backward compatibility: if resolved path doesn't exist, try resolving w.r.t root + add_sample_file_to_defaults: Set[str] = set() # for these options, add sample config files to their defaults + listify_options: Set[str] = set() # values for these options are processed as lists of values def __init__(self, **kwargs): self._process_renamed_options(kwargs) diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index 83f0d26c2c2..0bbaa64977e 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -5,11 +5,11 @@ import string import sys import tempfile from collections import ( - namedtuple, OrderedDict ) from io import StringIO from textwrap import TextWrapper +from typing import Any, List, NamedTuple import requests import yaml @@ -52,11 +52,6 @@ YAML_COMMENT_WRAPPER = TextWrapper(initial_indent="# ", subsequent_indent="# ", RST_DESCRIPTION_WRAPPER = TextWrapper(initial_indent=" ", subsequent_indent=" ", break_long_words=False, break_on_hyphens=False) UWSGI_SCHEMA_PATH = "lib/galaxy/webapps/uwsgi_schema.yml" -App = namedtuple( - "App", - ["config_paths", "default_port", "expected_app_factories", "destination", "schema_path", "uwsgi_module"] -) - UWSGI_OPTIONS = OrderedDict([ ('http', { 'desc': """The address and port on which to listen. By default, only listen to localhost ($app_name will not be accessible over the network). Use ':$default_port' to listen on all available network interfaces.""", @@ -320,23 +315,31 @@ OPTION_ACTIONS = { } -def _app_name(self): - return os.path.splitext(os.path.basename(self.destination))[0] +class App(NamedTuple): + config_paths: List[str] + default_port: str + expected_app_factories: List[str] + destination: str + schema_path: str + uwsgi_module: str + + @property + def app_name(self): + return os.path.splitext(os.path.basename(self.destination))[0] + + @property + def sample_destination(self): + return self.destination + ".sample" + + @property + def schema(self): + return AppSchema(self.schema_path, self.app_name) -def _sample_destination(self): - return self.destination + ".sample" - - -def _schema(self): - return AppSchema(self.schema_path, self.app_name) - - -App.app_name = property(_app_name) -App.sample_destination = property(_sample_destination) -App.schema = property(_schema) - -OptionValue = namedtuple("OptionValue", ["name", "value", "option"]) +class OptionValue(NamedTuple): + name: str + value: Any + option: Any GALAXY_APP = App( diff --git a/lib/galaxy/forms/forms.py b/lib/galaxy/forms/forms.py index 1b01ed38021..bf4d281ecbe 100644 --- a/lib/galaxy/forms/forms.py +++ b/lib/galaxy/forms/forms.py @@ -69,7 +69,7 @@ class FormDefinitionFactory: class FormDefinitionFieldFactory: - type = None + type: str def __get_stored_field_type(self, **kwds): raise Exception('not implemented') diff --git a/lib/galaxy/jobs/dynamic_tool_destination.py b/lib/galaxy/jobs/dynamic_tool_destination.py index 2e1b907fb31..36832cd5c6a 100755 --- a/lib/galaxy/jobs/dynamic_tool_destination.py +++ b/lib/galaxy/jobs/dynamic_tool_destination.py @@ -7,6 +7,7 @@ import os import re import sys from functools import reduce +from typing import Set import numpy as np import yaml @@ -25,7 +26,7 @@ verbose = True list of all valid priorities, inferred from the global default_desinations section of the config """ -priority_list = set() +priority_list: Set[str] = set() """ Instantiated to a list of all valid destinations in the job configuration file @@ -33,7 +34,7 @@ if run directly to validate configs. Otherwise, remains None. We often check to see if app is None, because if it is then we'll try using the destination_list instead. -""" -destination_list = set() +destination_list: Set[str] = set() """ The largest the edit distance can be for a word to be considered diff --git a/lib/galaxy/jobs/runners/util/cli/factory.py b/lib/galaxy/jobs/runners/util/cli/factory.py index 89554931d4c..26ed37e8151 100644 --- a/lib/galaxy/jobs/runners/util/cli/factory.py +++ b/lib/galaxy/jobs/runners/util/cli/factory.py @@ -5,7 +5,7 @@ try: ) code_dir = 'lib' except ImportError: - from pulsar.managers.util.cli import ( + from pulsar.managers.util.cli import ( # type: ignore CliInterface, split_params ) diff --git a/lib/galaxy/jobs/runners/util/job_script/__init__.py b/lib/galaxy/jobs/runners/util/job_script/__init__.py index 555ecb8d746..f4cf6a11764 100644 --- a/lib/galaxy/jobs/runners/util/job_script/__init__.py +++ b/lib/galaxy/jobs/runners/util/job_script/__init__.py @@ -3,6 +3,7 @@ import os import subprocess import time from string import Template +from typing import Any, Dict from pkg_resources import resource_string @@ -42,7 +43,7 @@ DEFAULT_INTEGRITY_CHECK = True DEFAULT_INTEGRITY_COUNT = 35 DEFAULT_INTEGRITY_SLEEP = .25 REQUIRED_TEMPLATE_PARAMS = ['working_directory', 'command', 'exit_code_path'] -OPTIONAL_TEMPLATE_PARAMS = { +OPTIONAL_TEMPLATE_PARAMS: Dict[str, Any] = { 'galaxy_lib': None, 'galaxy_virtual_env': None, 'headers': '', diff --git a/lib/galaxy/managers/base.py b/lib/galaxy/managers/base.py index de978773283..00a4ebd150e 100644 --- a/lib/galaxy/managers/base.py +++ b/lib/galaxy/managers/base.py @@ -28,6 +28,7 @@ attribute change to a model object. import datetime import logging import re +from typing import Type import routes import sqlalchemy @@ -156,7 +157,7 @@ class ModelManager: over the ORM. """ model_class = object - foreign_key_name = None + foreign_key_name: str def __init__(self, app): self.app = app @@ -480,7 +481,7 @@ class HasAModelManager: """ #: the class used to create this serializer's generically accessible model_manager - model_manager_class = None + model_manager_class: Type[object] # examples where this doesn't really work are ConfigurationSerializer (no manager) # and contents (2 managers) diff --git a/lib/galaxy/managers/containers.py b/lib/galaxy/managers/containers.py index 2ae960b344d..0588b2a304f 100644 --- a/lib/galaxy/managers/containers.py +++ b/lib/galaxy/managers/containers.py @@ -8,6 +8,7 @@ and other (nested) containers. # Libraries should be DatasetCollections. import logging import operator +from typing import Optional import galaxy.exceptions import galaxy.util @@ -33,7 +34,7 @@ class ContainerManagerMixin: contained_class = None subcontainer_class = None #: how any contents lists produced are ordered - (string) attribute name to sort on or tuple of attribute names - default_order_by = None + default_order_by: Optional[str] = None # ---- interface def contents(self, container): diff --git a/lib/galaxy/managers/sharable.py b/lib/galaxy/managers/sharable.py index 6407cb2b7d6..092c735864a 100644 --- a/lib/galaxy/managers/sharable.py +++ b/lib/galaxy/managers/sharable.py @@ -11,6 +11,7 @@ A sharable Galaxy object: """ import logging import re +from typing import Optional from sqlalchemy import true @@ -36,7 +37,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur user_share_model = None #: the single character abbreviation used in username_and_slug: e.g. 'h' for histories: u/user/h/slug - SINGLE_CHAR_ABBR = None + SINGLE_CHAR_ABBR: Optional[str] = None def __init__(self, app): super().__init__(app) @@ -316,7 +317,7 @@ class SharableModelManager(base.ModelManager, secured.OwnableManagerMixin, secur class SharableModelSerializer(base.ModelSerializer, taggable.TaggableSerializerMixin, annotatable.AnnotatableSerializerMixin, ratable.RatableSerializerMixin): # TODO: stub - SINGLE_CHAR_ABBR = None + SINGLE_CHAR_ABBR: Optional[str] = None def __init__(self, app, **kwargs): super().__init__(app, **kwargs) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index c6d59210488..7becb5cd27a 100644 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -13,6 +13,7 @@ import threading from collections import OrderedDict from datetime import datetime from pathlib import Path +from typing import List, Type from urllib.parse import unquote_plus import packaging.version @@ -3287,11 +3288,25 @@ class FilterFromFileTool(DatabaseOperationTool): # Populate tool_type to ToolClass mappings tool_types = {} -for tool_class in [Tool, SetMetadataTool, OutputParameterJSONTool, ExpressionTool, InteractiveTool, - DataManagerTool, DataSourceTool, AsyncDataSourceTool, - UnzipCollectionTool, ZipCollectionTool, MergeCollectionTool, RelabelFromFileTool, FilterFromFileTool, - BuildListCollectionTool, ExtractDatasetCollectionTool, - DataDestinationTool]: +TOOL_CLASSES: List[Type[Tool]] = [ + Tool, + SetMetadataTool, + OutputParameterJSONTool, + ExpressionTool, + InteractiveTool, + DataManagerTool, + DataSourceTool, + AsyncDataSourceTool, + UnzipCollectionTool, + ZipCollectionTool, + MergeCollectionTool, + RelabelFromFileTool, + FilterFromFileTool, + BuildListCollectionTool, + ExtractDatasetCollectionTool, + DataDestinationTool +] +for tool_class in TOOL_CLASSES: tool_types[tool_class.tool_type] = tool_class diff --git a/lib/galaxy/tools/data/__init__.py b/lib/galaxy/tools/data/__init__.py index b1893a15231..20dc67e7353 100644 --- a/lib/galaxy/tools/data/__init__.py +++ b/lib/galaxy/tools/data/__init__.py @@ -17,6 +17,7 @@ import time from collections import OrderedDict from glob import glob from tempfile import NamedTemporaryFile +from typing import List import refgenconf import requests @@ -240,6 +241,7 @@ class ToolDataTableManager: class ToolDataTable: + type_key: str @classmethod def from_elem(cls, table_elem, tool_data_path, from_shed_config, filename, tool_data_path_files, other_config_dict=None): @@ -743,7 +745,7 @@ class TabularToolDataTable(ToolDataTable, Dictifiable): class TabularToolDataField(Dictifiable): - dict_collection_visible_keys = [] + dict_collection_visible_keys: List[str] = [] def __init__(self, data): self.data = data diff --git a/lib/galaxy/tools/error_reports/plugins/base_git.py b/lib/galaxy/tools/error_reports/plugins/base_git.py index 7d317ab5afd..1b179e30cc2 100644 --- a/lib/galaxy/tools/error_reports/plugins/base_git.py +++ b/lib/galaxy/tools/error_reports/plugins/base_git.py @@ -8,6 +8,7 @@ from abc import ( ABCMeta, abstractmethod ) +from typing import Dict import requests if sys.version_info[0] < 3: @@ -27,12 +28,12 @@ log = logging.getLogger(__name__) class BaseGitPlugin(ErrorPlugin, metaclass=ABCMeta): """Base definition to send error reports to a Git repository provider """ - issue_cache = {} - ts_urls = {} - ts_repo_cache = {} - git_project_cache = {} - label_cache = {} - git_username_id_cache = {} + issue_cache: Dict[str, Dict] = {} + ts_urls: Dict[str, str] = {} + ts_repo_cache: Dict[str, Dict] = {} + git_project_cache: Dict[str, Dict] = {} + label_cache: Dict[str, Dict] = {} + git_username_id_cache: Dict[str, str] = {} # Git variables git_default_repo_owner = False diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tools/toolbox/lineages/interface.py index 88a35f14633..eb305a25d76 100644 --- a/lib/galaxy/tools/toolbox/lineages/interface.py +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -1,4 +1,5 @@ import threading +from typing import Dict import packaging.version from sortedcontainers import SortedSet @@ -35,7 +36,7 @@ class ToolLineage: """ Simple tool's loaded directly from file system with lineage determined solely by PEP 440 versioning scheme. """ - lineages_by_id = {} + lineages_by_id: Dict[str, 'ToolLineage'] = {} lock = threading.Lock() def __init__(self, tool_id, **kwds): diff --git a/lib/galaxy/visualization/data_providers/genome.py b/lib/galaxy/visualization/data_providers/genome.py index d13460c0ae8..57a8e9090d8 100644 --- a/lib/galaxy/visualization/data_providers/genome.py +++ b/lib/galaxy/visualization/data_providers/genome.py @@ -10,6 +10,7 @@ import re import sys from contextlib import contextmanager from json import loads +from typing import Dict, Union import pysam from bx.bbi.bigbed_file import BigBedFile @@ -126,7 +127,7 @@ class GenomeDataProvider(BaseDataProvider): format (0-based, half-open coordinates) for both queries and returned data. """ - dataset_type = None + dataset_type: str """ Mapping from column name to payload data; this mapping is used to create @@ -135,7 +136,7 @@ class GenomeDataProvider(BaseDataProvider): col_name_data_attr_mapping = {4 : { index: 5, name: 'Score' } } """ - col_name_data_attr_mapping = {} + col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {} def __init__(self, converted_dataset=None, original_dataset=None, dependencies=None, error_max_vals="Only the first %i %s in this region are displayed."): @@ -318,7 +319,7 @@ class TabixDataProvider(GenomeDataProvider, FilterableMixin): Tabix index data provider for the Galaxy track browser. """ - col_name_data_attr_mapping = {4: {'index': 4, 'name': 'Score'}} + col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {4: {'index': 4, 'name': 'Score'}} @contextmanager def open_data_file(self): @@ -596,7 +597,7 @@ class VcfDataProvider(GenomeDataProvider): 8-end: allele counts for each alternative """ - col_name_data_attr_mapping = {'Qual': {'index': 6, 'name': 'Qual'}} + col_name_data_attr_mapping: Dict[Union[str, int], Dict] = {'Qual': {'index': 6, 'name': 'Qual'}} dataset_type = 'variant' diff --git a/lib/galaxy/work/context.py b/lib/galaxy/work/context.py index 64a4333ab0b..873d714301f 100644 --- a/lib/galaxy/work/context.py +++ b/lib/galaxy/work/context.py @@ -29,7 +29,7 @@ class WorkRequestContext(ProvidesAppContext, ProvidesUserContext, ProvidesHistor def get_history(self, create=False): return self.__history - def set_history(self): + def set_history(self, history): raise NotImplementedError("Cannot change histories from a work request context.") history = property(get_history, set_history) diff --git a/lib/galaxy/workflow/refactor/execute.py b/lib/galaxy/workflow/refactor/execute.py index d40172cdc75..3faa99f5b25 100644 --- a/lib/galaxy/workflow/refactor/execute.py +++ b/lib/galaxy/workflow/refactor/execute.py @@ -1,4 +1,5 @@ import logging +from typing import Any, Dict from galaxy.exceptions import ( RequestParameterInvalidException, @@ -103,7 +104,7 @@ class WorkflowRefactorExecutor: input_type = action.type module_type = None - tool_state = {} + tool_state: Dict[str, Any] = {} if input_type in ["data", "dataset"]: module_type = "data_input" elif input_type in ["data_collection", "dataset_collection"]: diff --git a/lib/galaxy/workflow/refactor/schema.py b/lib/galaxy/workflow/refactor/schema.py index e4aa34da5a5..37e9c524f38 100644 --- a/lib/galaxy/workflow/refactor/schema.py +++ b/lib/galaxy/workflow/refactor/schema.py @@ -54,6 +54,7 @@ class BaseAction(BaseModel): class Action: + action_type: str @classmethod def __get_validators__(cls): @@ -184,7 +185,7 @@ union_action_classes = Union[ ACTION_CLASSES_BY_TYPE = {} -for action_class in union_action_classes.__args__: +for action_class in union_action_classes.__args__: # type: ignore action_type = action_class.schema()["properties"]["action_type"]["const"] ACTION_CLASSES_BY_TYPE[action_type] = action_class diff --git a/packages/app/galaxy/__init__.py b/packages/app/galaxy/__init__.py index 69e3be50dac..0d1f7edf5dc 100644 --- a/packages/app/galaxy/__init__.py +++ b/packages/app/galaxy/__init__.py @@ -1 +1 @@ -__path__ = __import__('pkgutil').extend_path(__path__, __name__) +__path__ = __import__('pkgutil').extend_path(__path__, __name__) # type: ignore diff --git a/packages/test.sh b/packages/test.sh index 8d17b6f2003..974e121c672 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -34,7 +34,7 @@ PACKAGE_DIRS=( # tool_util not yet working 100%, # data has many problems quota, tool shed install database, etc.. RUN_TESTS=(1 1 1 1 1 1 1 1 1 1 1 0) -RUN_MYPY=(1 1 1 1 1 1 1 1 1 1 0 0) +RUN_MYPY=(1 1 1 1 1 1 1 1 1 1 1 0) for ((i=0; i<${#PACKAGE_DIRS[@]}; i++)); do package_dir=${PACKAGE_DIRS[$i]} run_tests=${RUN_TESTS[$i]} diff --git a/scripts/galaxy_main.py b/scripts/galaxy_main.py index b91ed644182..cc4cdea7374 100755 --- a/scripts/galaxy_main.py +++ b/scripts/galaxy_main.py @@ -45,12 +45,13 @@ except ImportError: log = logging.getLogger(__name__) real_file = os.path.realpath(__file__) -GALAXY_ROOT_DIR = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir)) -if not os.path.exists(os.path.join(GALAXY_ROOT_DIR, 'run.sh')): +GALAXY_ROOT_DIR_ = os.path.abspath(os.path.join(os.path.dirname(real_file), os.pardir)) +if not os.path.exists(os.path.join(GALAXY_ROOT_DIR_, 'run.sh')): # Galaxy is installed GALAXY_ROOT_DIR = None else: - GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR, "lib") + GALAXY_ROOT_DIR = GALAXY_ROOT_DIR_ + GALAXY_LIB_DIR = os.path.join(GALAXY_ROOT_DIR_, "lib") try: sys.path.insert(1, GALAXY_LIB_DIR) except Exception: