diff --git a/lib/galaxy/app.py b/lib/galaxy/app.py index b33f8ef335d..28a039971e5 100644 --- a/lib/galaxy/app.py +++ b/lib/galaxy/app.py @@ -672,7 +672,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication): # Tours registry tour_registry = build_tours_registry(self.config.tour_config_dir) self.tour_registry = tour_registry - self[ToursRegistry] = tour_registry # type: ignore[misc] + self[ToursRegistry] = tour_registry # type: ignore[type-abstract] # Webhooks registry self.webhooks_registry = self._register_singleton(WebhooksRegistry, WebhooksRegistry(self.config.webhooks_dir)) # Heartbeat for thread profiling diff --git a/lib/galaxy/app_unittest_utils/galaxy_mock.py b/lib/galaxy/app_unittest_utils/galaxy_mock.py index 8030fb8d4dc..6351b73fb9f 100644 --- a/lib/galaxy/app_unittest_utils/galaxy_mock.py +++ b/lib/galaxy/app_unittest_utils/galaxy_mock.py @@ -104,7 +104,7 @@ class MockApp(di.Container, GalaxyDataTestApp): job_metrics: JobMetrics stop: bool - def __init__(self, config=None, **kwargs): + def __init__(self, config=None, **kwargs) -> None: super().__init__() config = config or MockAppConfig(**kwargs) GalaxyDataTestApp.__init__(self, config=config, **kwargs) @@ -117,8 +117,8 @@ class MockApp(di.Container, GalaxyDataTestApp): self[GalaxyModelMapping] = self.model sts_config = ShortTermStorageConfiguration(short_term_storage_directory=os.path.join(config.data_dir, "sts")) sts_manager = ShortTermStorageManager(sts_config) - self[ShortTermStorageAllocator] = sts_manager # type: ignore[misc] - self[ShortTermStorageMonitor] = sts_manager # type: ignore[misc] + self[ShortTermStorageAllocator] = sts_manager # type: ignore[type-abstract] + self[ShortTermStorageMonitor] = sts_manager # type: ignore[type-abstract] self[galaxy_scoped_session] = self.model.context self.visualizations_registry = MockVisualizationsRegistry() self.tag_handler = tags.GalaxyTagHandler(self.model.context) diff --git a/lib/galaxy/tool_util/data/__init__.py b/lib/galaxy/tool_util/data/__init__.py index 983ebeb0c1e..f36c61ded47 100644 --- a/lib/galaxy/tool_util/data/__init__.py +++ b/lib/galaxy/tool_util/data/__init__.py @@ -18,12 +18,15 @@ import time from glob import glob from tempfile import NamedTemporaryFile from typing import ( + Any, BinaryIO, Dict, List, Optional, Set, + Tuple, Type, + TYPE_CHECKING, Union, ) @@ -31,7 +34,10 @@ import requests from galaxy import util from galaxy.exceptions import MessageException -from galaxy.util import RW_R__R__ +from galaxy.util import ( + Element, + RW_R__R__, +) from galaxy.util.dictifiable import Dictifiable from galaxy.util.filelock import FileLock from galaxy.util.renamed_temporary_file import RenamedTemporaryFile @@ -40,6 +46,9 @@ from ._schema import ( ToolDataEntryList, ) +if TYPE_CHECKING: + from galaxy.config import GalaxyAppConfiguration + log = logging.getLogger(__name__) DEFAULT_TABLE_TYPE = "tabular" @@ -109,20 +118,20 @@ class ToolDataTable(Dictifiable): def __init__( self, - config_element, - tool_data_path, - from_shed_config=False, - filename=None, - tool_data_path_files=None, - other_config_dict=None, - ): + config_element: Element, + tool_data_path: Union[str, os.PathLike], + from_shed_config: bool = False, + filename: Optional[Union[str, os.PathLike]] = None, + tool_data_path_files: Optional[ToolDataPathFiles] = None, + other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None, + ) -> None: self.name = config_element.get("name") self.comment_char = config_element.get("comment_char") self.empty_field_value = config_element.get("empty_field_value", "") - self.empty_field_values = {} + self.empty_field_values: Dict[str, str] = {} self.allow_duplicate_entries = util.asbool(config_element.get("allow_duplicate_entries", True)) - self.here = filename and os.path.dirname(filename) - self.filenames = {} + self.here = os.path.dirname(filename) if filename else None + self.filenames: Dict[str, Dict[str, Any]] = {} self.tool_data_path = tool_data_path self.tool_data_path_files = tool_data_path_files self.other_config_dict = other_config_dict or {} @@ -139,9 +148,9 @@ class ToolDataTable(Dictifiable): "filename": filename, }, ) - self._merged_load_info = [] + self._merged_load_info: List[Tuple[Type[ToolDataTable], Tuple[List[Any], Dict[str, Any]]]] = [] - def _update_version(self, version=None): + def _update_version(self, version: Optional[int] = None) -> int: if version is not None: self._loaded_content_version = version else: @@ -181,10 +190,10 @@ class ToolDataTable(Dictifiable): def merge_tool_data_table(self, other_table, allow_duplicates=True, persist=False, entry_source=None, **kwd): raise NotImplementedError("Abstract method") - def reload_from_files(self): + def reload_from_files(self) -> int: new_version = self._update_version() merged_info = self._merged_load_info - self.__init__(*self._load_info[0], **self._load_info[1]) + self.__init__(*self._load_info[0], **self._load_info[1]) # type: ignore[misc] self._update_version(version=new_version) for (tool_data_table_class, load_info) in merged_info: self.merge_tool_data_table(tool_data_table_class(*load_info[0], **load_info[1]), allow_duplicates=False) @@ -214,13 +223,13 @@ class TabularToolDataTable(ToolDataTable): def __init__( self, - config_element, - tool_data_path, - from_shed_config=False, - filename=None, - tool_data_path_files=None, - other_config_dict=None, - ): + config_element: Element, + tool_data_path: Union[str, os.PathLike], + from_shed_config: bool = False, + filename: Optional[Union[str, os.PathLike]] = None, + tool_data_path_files: Optional[ToolDataPathFiles] = None, + other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None, + ) -> None: super().__init__( config_element, tool_data_path, @@ -233,7 +242,13 @@ class TabularToolDataTable(ToolDataTable): self.data = [] self.configure_and_load(config_element, tool_data_path, from_shed_config) - def configure_and_load(self, config_element, tool_data_path, from_shed_config=False, url_timeout=10): + def configure_and_load( + self, + config_element: Element, + tool_data_path: Union[str, os.PathLike], + from_shed_config: bool = False, + url_timeout: int = 10, + ) -> None: """ Configure and load table from an XML element. """ @@ -298,7 +313,7 @@ class TabularToolDataTable(ToolDataTable): # directory which is hard-coded into the tool_data_table_conf.xml entries. filename = os.path.split(file_path)[1] filename = os.path.join(tool_data_path, filename) - if self.tool_data_path_files.exists(filename): + if self.tool_data_path_files and self.tool_data_path_files.exists(filename): found = True elif not os.path.isabs(filename): # Since the path attribute can include a hard-coded path to a specific directory @@ -308,10 +323,14 @@ class TabularToolDataTable(ToolDataTable): file_path, file_name = os.path.split(filename) if file_path != self.tool_data_path: corrected_filename = os.path.join(self.tool_data_path, file_name) - if self.tool_data_path_files.exists(corrected_filename): + if self.tool_data_path_files and self.tool_data_path_files.exists(corrected_filename): filename = corrected_filename found = True - elif not from_shed_config and self.tool_data_path_files.exists(f"{corrected_filename}.sample"): + elif ( + not from_shed_config + and self.tool_data_path_files + and self.tool_data_path_files.exists(f"{corrected_filename}.sample") + ): log.info(f"Could not find tool data {corrected_filename}, reading sample") filename = f"{corrected_filename}.sample" found = True @@ -403,7 +422,7 @@ class TabularToolDataTable(ToolDataTable): def get_version_fields(self): return (self._loaded_content_version, self.get_fields()) - def parse_column_spec(self, config_element): + def parse_column_spec(self, config_element: Element) -> None: """ Parse column definitions, which can either be a set of 'column' elements with a name and index (as in dynamic options config), or a shorthand @@ -437,13 +456,15 @@ class TabularToolDataTable(ToolDataTable): if "name" not in self.columns: self.columns["name"] = self.columns["value"] - def extend_data_with(self, filename, errors=None): + def extend_data_with(self, filename: str, errors: Optional[List[Any]] = None) -> None: here = os.path.dirname(os.path.abspath(filename)) self.data.extend(self.parse_file_fields(filename, errors=errors, here=here)) if not self.allow_duplicate_entries: self._deduplicate_data() - def parse_file_fields(self, filename, errors: Optional[List[str]] = None, here="__HERE__"): + def parse_file_fields( + self, filename: str, errors: Optional[List[str]] = None, here: str = "__HERE__" + ) -> List[List[str]]: """ Parse separated lines from file and return a list of tuples. @@ -542,7 +563,9 @@ class TabularToolDataTable(ToolDataTable): break return filename - def _add_entry(self, entry, allow_duplicates=True, persist=False, entry_source=None, **kwd): + def _add_entry( + self, entry: Union[List[str], Dict[str, str]], allow_duplicates=True, persist=False, entry_source=None, **kwd + ) -> None: # accepts dict or list of columns if isinstance(entry, dict): fields = [] @@ -737,7 +760,7 @@ class TabularToolDataField(Dictifiable): return rval -def _expand_here_template(content, here=None): +def _expand_here_template(content: str, here: Optional[str]) -> str: if here and content: content = string.Template(content).safe_substitute({"__HERE__": here}) return content @@ -750,7 +773,7 @@ tool_data_table_types_list: List[Type[ToolDataTable]] = [TabularToolDataTable] class ToolDataTableManager(Dictifiable): """Manages a collection of tool data tables""" - data_tables: Dict[str, "ToolDataTable"] + data_tables: Dict[str, ToolDataTable] tool_data_table_types = {cls.type_key: cls for cls in tool_data_table_types_list} def __init__( @@ -758,8 +781,8 @@ class ToolDataTableManager(Dictifiable): tool_data_path: str, config_filename: Optional[ConfigFilesT] = None, tool_data_table_config_path_set=None, - other_config_dict=None, - ): + other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None, + ) -> None: self.tool_data_path = tool_data_path # This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file # at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries @@ -776,7 +799,7 @@ class ToolDataTableManager(Dictifiable): data_tables = [ToolDataEntry(**table.to_dict()) for table in self.data_tables.values()] return ToolDataEntryList.construct(__root__=data_tables) - def __getitem__(self, key: str): + def __getitem__(self, key: str) -> ToolDataTable: return self.data_tables.__getitem__(key) def __setitem__(self, key: str, value): @@ -851,8 +874,14 @@ class ToolDataTableManager(Dictifiable): return table_elems def from_elem( - self, table_elem, tool_data_path, from_shed_config, filename, tool_data_path_files, other_config_dict=None - ): + self, + table_elem: Element, + tool_data_path: Union[str, os.PathLike], + from_shed_config: bool, + filename: Union[str, os.PathLike], + tool_data_path_files: ToolDataPathFiles, + other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None, + ) -> ToolDataTable: table_type = table_elem.get("type", "tabular") assert table_type in self.tool_data_table_types, f"Unknown data table type '{table_type}'" return self.tool_data_table_types[table_type]( @@ -950,7 +979,9 @@ class ToolDataTableManager(Dictifiable): if out_path_is_new: self.tool_data_path_files.update_files() - def reload_tables(self, table_names=None, path=None): + def reload_tables( + self, table_names: Optional[Union[List[str], str]] = None, path: Optional[str] = None + ) -> List[str]: """ Reload tool data tables. If neither table_names nor path is given, reloads all tool data tables. """ @@ -967,7 +998,7 @@ class ToolDataTableManager(Dictifiable): log.debug("Reloaded tool data table '%s' from files.", table_name) return table_names - def get_table_names_by_path(self, path): + def get_table_names_by_path(self, path: str) -> List[str]: """Returns a list of table names given a path""" table_names = set() for name, data_table in self.data_tables.items(): diff --git a/lib/galaxy/tool_util/parser/output_collection_def.py b/lib/galaxy/tool_util/parser/output_collection_def.py index 8e5481feef2..b2c76f51fcd 100644 --- a/lib/galaxy/tool_util/parser/output_collection_def.py +++ b/lib/galaxy/tool_util/parser/output_collection_def.py @@ -118,7 +118,7 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription): self.recurse = asbool(kwargs.get("recurse", False)) self.match_relative_path = asbool(kwargs.get("match_relative_path", False)) if pattern in NAMED_PATTERNS: - pattern = NAMED_PATTERNS.get(pattern) + pattern = NAMED_PATTERNS[pattern] self.pattern = pattern self.sort_by = sort_by = kwargs.get("sort_by", DEFAULT_SORT_BY) if sort_by.startswith("reverse_"): @@ -149,7 +149,7 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription): return as_dict @property - def discover_patterns(self): + def discover_patterns(self) -> List[str]: return [self.pattern] diff --git a/lib/galaxy/tool_util/parser/output_objects.py b/lib/galaxy/tool_util/parser/output_objects.py index 48314fb5090..9cf29a42077 100644 --- a/lib/galaxy/tool_util/parser/output_objects.py +++ b/lib/galaxy/tool_util/parser/output_objects.py @@ -1,5 +1,11 @@ -from typing import List +from typing import ( + Any, + Dict, + List, + Optional, +) +from galaxy.util import Element from galaxy.util.dictifiable import Dictifiable from .output_actions import ToolOutputActionGroup from .output_collection_def import ( @@ -9,7 +15,14 @@ from .output_collection_def import ( class ToolOutputBase(Dictifiable): - def __init__(self, name, label=None, filters=None, hidden=False, from_expression=None): + def __init__( + self, + name: str, + label: Optional[str] = None, + filters: Optional[List[Element]] = None, + hidden: bool = False, + from_expression: Optional[str] = None, + ) -> None: super().__init__() self.name = name self.label = label @@ -50,18 +63,18 @@ class ToolOutput(ToolOutputBase): def __init__( self, - name, - format=None, - format_source=None, - metadata_source=None, - parent=None, - label=None, - filters=None, - actions=None, - hidden=False, - implicit=False, - from_expression=None, - ): + name: str, + format: Optional[str] = None, + format_source: Optional[str] = None, + metadata_source: Optional[str] = None, + parent: Optional[str] = None, + label: Optional[str] = None, + filters: Optional[List[Element]] = None, + actions: Optional[ToolOutputActionGroup] = None, + hidden: bool = False, + implicit: bool = False, + from_expression: Optional[str] = None, + ) -> None: super().__init__(name, label=label, filters=filters, hidden=hidden, from_expression=from_expression) self.output_type = "data" self.format = format @@ -71,14 +84,17 @@ class ToolOutput(ToolOutputBase): self.actions = actions # Initialize default values - self.change_format = [] + self.change_format: List[Element] = [] self.implicit = implicit - self.from_work_dir = None + self.from_work_dir: Optional[str] = None self.dataset_collector_descriptions: List[DatasetCollectionDescription] = [] + self.default_identifier_source: Optional[str] = None + self.count: Optional[int] = None + self.tool: Optional[Any] # Tuple emulation - def __len__(self): + def __len__(self) -> int: return 3 def __getitem__(self, index): @@ -106,7 +122,7 @@ class ToolOutput(ToolOutputBase): return as_dict @staticmethod - def from_dict(name, output_dict, tool=None): + def from_dict(name: str, output_dict: Dict[str, Any], tool: Optional[object] = None) -> "ToolOutput": output = ToolOutput(name) output.format = output_dict.get("format", "data") output.change_format = [] @@ -119,7 +135,7 @@ class ToolOutput(ToolOutputBase): output.filters = [] output.tool = tool output.from_work_dir = output_dict.get("from_work_dir", None) - output.hidden = output_dict.get("hidden", "") + output.hidden = output_dict.get("hidden", False) # TODO: implement tool output action group fixes output.actions = ToolOutputActionGroup(output, None) output.dataset_collector_descriptions = dataset_collector_descriptions_from_output_dict(output_dict) @@ -183,30 +199,30 @@ class ToolOutputCollection(ToolOutputBase): def __init__( self, - name, - structure, - label=None, - filters=None, - hidden=False, - default_format="data", - default_format_source=None, - default_metadata_source=None, - inherit_format=False, - inherit_metadata=False, - ): + name: str, + structure: "ToolOutputCollectionStructure", + label: Optional[str] = None, + filters: Optional[List[Element]] = None, + hidden: bool = False, + default_format: str = "data", + default_format_source: Optional[str] = None, + default_metadata_source: Optional[str] = None, + inherit_format: bool = False, + inherit_metadata: bool = False, + ) -> None: super().__init__(name, label=label, filters=filters, hidden=hidden) self.output_type = "collection" self.collection = True self.default_format = default_format self.structure = structure - self.outputs = {} + self.outputs: Dict[str, str] = {} self.inherit_format = inherit_format self.inherit_metadata = inherit_metadata self.metadata_source = default_metadata_source self.format_source = default_format_source - self.change_format = [] # TODO + self.change_format: List = [] # TODO: not implemented def known_outputs(self, inputs, type_registry): if self.dynamic_structure: @@ -275,13 +291,13 @@ class ToolOutputCollection(ToolOutputBase): return as_dict @staticmethod - def from_dict(name, output_dict, tool=None): + def from_dict(name, output_dict, tool=None) -> "ToolOutputCollection": structure = ToolOutputCollectionStructure.from_dict(output_dict["structure"]) rval = ToolOutputCollection( name, structure=structure, label=output_dict.get("label", None), - filters=None, + filters=[], hidden=output_dict.get("hidden", False), default_format=output_dict.get("default_format", "data"), default_format_source=output_dict.get("default_format_source", None), @@ -299,12 +315,12 @@ class ToolOutputCollection(ToolOutputBase): class ToolOutputCollectionStructure: def __init__( self, - collection_type, - collection_type_source=None, - collection_type_from_rules=None, - structured_like=None, - dataset_collector_descriptions=None, - ): + collection_type: Optional[str], + collection_type_source: Optional[str] = None, + collection_type_from_rules: Optional[str] = None, + structured_like: Optional[str] = None, + dataset_collector_descriptions: Optional[List[DatasetCollectionDescription]] = None, + ) -> None: self.collection_type = collection_type self.collection_type_source = collection_type_source self.collection_type_from_rules = collection_type_from_rules @@ -349,7 +365,7 @@ class ToolOutputCollectionStructure: } @staticmethod - def from_dict(as_dict): + def from_dict(as_dict) -> "ToolOutputCollectionStructure": structure = ToolOutputCollectionStructure( collection_type=as_dict["collection_type"], collection_type_source=as_dict["collection_type_source"], diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py index e8852fa3832..619ed8f87fe 100644 --- a/lib/galaxy/tool_util/verify/script.py +++ b/lib/galaxy/tool_util/verify/script.py @@ -1,18 +1,25 @@ #!/usr/bin/env python import argparse +import concurrent.futures.thread import datetime as dt import json import logging import os import sys import tempfile -from collections import namedtuple from concurrent.futures import ( thread, ThreadPoolExecutor, ) -from typing import List +from typing import ( + Any, + Callable, + Dict, + List, + NamedTuple, + Optional, +) import yaml @@ -30,18 +37,27 @@ ALL_VERSION = "*" LATEST_VERSION = None -TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"]) -TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"]) +class TestReference(NamedTuple): + tool_id: str + tool_version: Optional[str] + test_index: int + + +class TestException(NamedTuple): + tool_id: str + exception: Exception + was_recorded: bool class Results: - test_exceptions: List[Exception] + test_exceptions: List[TestException] - def __init__(self, default_suitename, test_json, append=False, galaxy_url=None): + def __init__( + self, default_suitename: str, test_json: str, append: bool = False, galaxy_url: Optional[str] = None + ) -> None: self.test_json = test_json or "-" self.galaxy_url = galaxy_url test_results = [] - test_exceptions: List[Exception] = [] suitename = default_suitename if append: assert test_json != "-" @@ -51,16 +67,16 @@ class Results: if "suitename" in previous_results: suitename = previous_results["suitename"] self.test_results = test_results - self.test_exceptions = test_exceptions + self.test_exceptions = [] self.suitename = suitename - def register_result(self, result): + def register_result(self, result: Dict[str, Any]) -> None: self.test_results.append(result) - def register_exception(self, test_exception): + def register_exception(self, test_exception: TestException) -> None: self.test_exceptions.append(test_exception) - def already_successful(self, test_reference): + def already_successful(self, test_reference: TestReference) -> bool: test_data = self._previous_test_data(test_reference) if test_data: if "status" in test_data and test_data["status"] == "success": @@ -68,7 +84,7 @@ class Results: return False - def already_executed(self, test_reference): + def already_executed(self, test_reference: TestReference) -> bool: test_data = self._previous_test_data(test_reference) if test_data: if "status" in test_data and test_data["status"] != "skipped": @@ -76,7 +92,7 @@ class Results: return False - def _previous_test_data(self, test_reference): + def _previous_test_data(self, test_reference: TestReference) -> Optional[Dict[str, Any]]: test_id = _test_id_for_reference(test_reference) for test_result in self.test_results: if test_result.get("id") != test_id: @@ -89,7 +105,7 @@ class Results: return None - def write(self): + def write(self) -> None: tests = sorted(self.test_results, key=lambda el: el["id"]) n_passed, n_failures, n_skips = 0, 0, 0 n_errors = len([e for e in self.test_exceptions if not e.was_recorded]) @@ -127,56 +143,37 @@ class Results: with open(self.test_json, "w") as f: json.dump(report_obj, f) - def info_message(self): + def info_message(self) -> str: messages = [] passed_tests = self._tests_with_status("success") messages.append("Passed tool tests ({}): {}".format(len(passed_tests), [t["id"] for t in passed_tests])) failed_tests = self._tests_with_status("failure") messages.append("Failed tool tests ({}): {}".format(len(failed_tests), [t["id"] for t in failed_tests])) - skiped_tests = self._tests_with_status("skip") - messages.append("Skipped tool tests ({}): {}".format(len(skiped_tests), [t["id"] for t in skiped_tests])) + skipped_tests = self._tests_with_status("skip") + messages.append("Skipped tool tests ({}): {}".format(len(skipped_tests), [t["id"] for t in skipped_tests])) errored_tests = self._tests_with_status("error") messages.append("Errored tool tests ({}): {}".format(len(errored_tests), [t["id"] for t in errored_tests])) return "\n".join(messages) - @property - def success_count(self): - self._tests_with_status("success") - - @property - def skip_count(self): - self._tests_with_status("skip") - - @property - def error_count(self): - return self._tests_with_status("error") + len(self.test_exceptions) - - @property - def failure_count(self): - return self._tests_with_status("failure") - - def _tests_with_status(self, status): + def _tests_with_status(self, status: str) -> List[Dict[str, Any]]: return [t for t in self.test_results if t.get("data", {}).get("status") == status] def test_tools( - galaxy_interactor, - test_references, - results, - log=None, - parallel_tests=1, - history_per_test_case=False, - history_name=None, - no_history_reuse=False, - no_history_cleanup=False, - publish_history=False, - retries=0, - verify_kwds=None, -): - """Run through tool tests and write report. - - Refactor this into Galaxy in 21.01. - """ + galaxy_interactor: GalaxyInteractorApi, + test_references: List[TestReference], + results: Results, + log: Optional[logging.Logger] = None, + parallel_tests: int = 1, + history_per_test_case: bool = False, + history_name: Optional[str] = None, + no_history_reuse: bool = False, + no_history_cleanup: bool = False, + publish_history: bool = False, + retries: int = 0, + verify_kwds: Optional[Dict[str, Any]] = None, +) -> None: + """Run through tool tests and write report.""" verify_kwds = (verify_kwds or {}).copy() tool_test_start = dt.datetime.now() history_created = False @@ -221,8 +218,8 @@ def test_tools( try: executor.shutdown(wait=True) except KeyboardInterrupt: - executor._threads.clear() - thread._threads_queues.clear() + executor._threads.clear() # type: ignore[attr-defined] + thread._threads_queues.clear() # type: ignore[attr-defined] results.write() if log: if results.test_json == "-": @@ -236,7 +233,7 @@ def test_tools( galaxy_interactor.delete_history(test_history) -def _test_id_for_reference(test_reference): +def _test_id_for_reference(test_reference: "TestReference") -> str: tool_id = test_reference.tool_id tool_version = test_reference.tool_version test_index = test_reference.test_index @@ -253,15 +250,15 @@ def _test_id_for_reference(test_reference): def _test_tool( - executor, - test_reference, - results, - galaxy_interactor, - log, - retries, - publish_history, - verify_kwds, -): + executor: concurrent.futures.thread.ThreadPoolExecutor, + test_reference: "TestReference", + results: Results, + galaxy_interactor: GalaxyInteractorApi, + log: Optional[logging.Logger], + retries: int, + publish_history: bool, + verify_kwds: Dict[str, Any], +) -> None: tool_id = test_reference.tool_id tool_version = test_reference.tool_version test_index = test_reference.test_index @@ -272,7 +269,7 @@ def _test_tool( test_id = _test_id_for_reference(test_reference) - def run_test(): + def run_test() -> None: run_retries = retries job_data = None job_exception = None @@ -323,16 +320,16 @@ def _test_tool( def build_case_references( - galaxy_interactor, - tool_id=ALL_TOOLS, - tool_version=LATEST_VERSION, - test_index=ALL_TESTS, - page_size=0, - page_number=0, - test_filters=None, - log=None, -): - test_references = [] + galaxy_interactor: GalaxyInteractorApi, + tool_id: str = ALL_TOOLS, + tool_version: Optional[str] = LATEST_VERSION, + test_index: int = ALL_TESTS, + page_size: int = 0, + page_number: int = 0, + test_filters: Optional[List[Callable[[TestReference], bool]]] = None, + log: Optional[logging.Logger] = None, +) -> List[TestReference]: + test_references: List[TestReference] = [] if tool_id == ALL_TOOLS: tests_summary = galaxy_interactor.get_tests_summary() for tool_id, tool_versions_dict in tests_summary.items(): @@ -341,8 +338,7 @@ def build_case_references( test_reference = TestReference(tool_id, tool_version, test_index) test_references.append(test_reference) else: - assert tool_id - tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {} + tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) for i, tool_test_dict in enumerate(tool_test_dicts): this_tool_version = tool_test_dict.get("tool_version", tool_version) this_test_index = i @@ -351,7 +347,7 @@ def build_case_references( test_references.append(test_reference) if test_filters is not None and len(test_filters) > 0: - filtered_test_references = [] + filtered_test_references: List[TestReference] = [] for test_reference in test_references: skip_test = False for test_filter in test_filters: @@ -361,7 +357,10 @@ def build_case_references( skip_test = True if not skip_test: filtered_test_references.append(test_reference) - log.info(f"Skipping {len(test_references)-len(filtered_test_references)} out of {len(test_references)} tests.") + if log is not None: + log.info( + f"Skipping {len(test_references)-len(filtered_test_references)} out of {len(test_references)} tests." + ) test_references = filtered_test_references if page_size > 0: @@ -372,7 +371,7 @@ def build_case_references( return test_references -def main(argv=None): +def main(argv=None) -> None: if argv is None: argv = sys.argv[1:] @@ -384,7 +383,11 @@ def main(argv=None): sys.exit(1) -def run_tests(args, test_filters=None, log=None): +def run_tests( + args: argparse.Namespace, + test_filters: Optional[List[Callable[[TestReference], bool]]] = None, + log: Optional[logging.Logger] = None, +) -> None: # Split out argument parsing so we can quickly build other scripts - such as a script # to run all tool tests for a workflow by just passing in a custom test_filters. test_filters = test_filters or [] @@ -464,12 +467,10 @@ def run_tests(args, test_filters=None, log=None): exceptions = results.test_exceptions if exceptions: exception = exceptions[0] - if hasattr(exception, "exception"): - exception = exception.exception - raise exception + raise exception.exception -def setup_global_logger(name, log_file=None, verbose=False): +def setup_global_logger(name: str, log_file: Optional[str] = None, verbose: bool = False) -> logging.Logger: formatter = logging.Formatter("%(asctime)s %(levelname)-5s - %(message)s") console = logging.StreamHandler() console.setFormatter(formatter) @@ -490,7 +491,7 @@ def setup_global_logger(name, log_file=None, verbose=False): return logger -def arg_parser(): +def arg_parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(description=DESCRIPTION) parser.add_argument("-u", "--galaxy-url", default="http://localhost:8080", help="Galaxy URL") parser.add_argument("-k", "--key", default=None, help="Galaxy User API Key") diff --git a/lib/galaxy/tools/evaluation.py b/lib/galaxy/tools/evaluation.py index ae211ecf0e2..2ae9cd70ca2 100644 --- a/lib/galaxy/tools/evaluation.py +++ b/lib/galaxy/tools/evaluation.py @@ -27,6 +27,7 @@ from galaxy.structured_app import ( BasicSharedApp, MinimalToolApp, ) +from galaxy.tool_util.data import TabularToolDataTable from galaxy.tools.parameters import ( visit_input_values, wrapped_json, @@ -463,7 +464,10 @@ class ToolEvaluator: Queries and returns an entry in a data table. """ if table_name in self.app.tool_data_tables: - return self.app.tool_data_tables[table_name].get_entry(query_attr, query_val, return_attr) + table = self.app.tool_data_tables[table_name] + if not isinstance(table, TabularToolDataTable): + raise Exception(f"Expected a TabularToolDataTable but got a {type(table)}: {table}.") + return table.get_entry(query_attr, query_val, return_attr) param_dict["__tool_directory__"] = self.compute_environment.tool_directory() param_dict["__get_data_table_entry__"] = get_data_table_entry diff --git a/mypy.ini b/mypy.ini index 6340a19a970..bbefdbab609 100644 --- a/mypy.ini +++ b/mypy.ini @@ -210,8 +210,6 @@ check_untyped_defs = False check_untyped_defs = False [mypy-galaxy.tools.bundled.extract.extract_genomic_dna] check_untyped_defs = False -[mypy-galaxy.tool_util.parser.output_objects] -check_untyped_defs = False [mypy-galaxy.tool_util.deps.resolvers] check_untyped_defs = False [mypy-galaxy.tool_util.deps.mulled.mulled_update_singularity_containers] @@ -248,8 +246,6 @@ check_untyped_defs = False check_untyped_defs = False [mypy-galaxy.tools.expressions.evaluation] check_untyped_defs = False -[mypy-galaxy.tool_util.data] -check_untyped_defs = False [mypy-galaxy.tool_util.verify] check_untyped_defs = False [mypy-galaxy.tool_util.toolbox.watcher] diff --git a/test/integration/test_job_recovery.py b/test/integration/test_job_recovery.py index d6a10ed73b0..22baa127989 100644 --- a/test/integration/test_job_recovery.py +++ b/test/integration/test_job_recovery.py @@ -24,7 +24,7 @@ class TestJobRecoveryBeforeHandledIntegration(integration_util.IntegrationTestCa config["job_config_file"] = SIMPLE_JOB_CONFIG_FILE config["server_name"] = "moo" - def handle_reconfigure_galaxy_config_kwds(self, config) -> None: + def handle_reconfigure_galaxy_config_kwds(self, config): config["server_name"] = "main" def test_recovery(self) -> None: diff --git a/test/unit/app/tools/test_actions.py b/test/unit/app/tools/test_actions.py index 8b4b8898eeb..dff02b07ed6 100644 --- a/test/unit/app/tools/test_actions.py +++ b/test/unit/app/tools/test_actions.py @@ -235,7 +235,7 @@ def quick_output(format, format_source=None, change_format_xml=None): if change_format_xml: test_output.change_format = XML(change_format_xml) else: - test_output.change_format = None + test_output.change_format = [] return test_output diff --git a/test/unit/tool_util/test_verify_script.py b/test/unit/tool_util/test_verify_script.py index 6d6a9484bd0..4c8f42c26a7 100644 --- a/test/unit/tool_util/test_verify_script.py +++ b/test/unit/tool_util/test_verify_script.py @@ -1,6 +1,10 @@ import json import os from tempfile import NamedTemporaryFile +from typing import ( + cast, + NoReturn, +) from unittest import mock from galaxy.tool_util.unittest_utils.interactor import ( @@ -10,6 +14,7 @@ from galaxy.tool_util.unittest_utils.interactor import ( MockGalaxyInteractor, NEW_HISTORY_ID, ) +from galaxy.tool_util.verify.interactor import GalaxyInteractorApi from galaxy.tool_util.verify.script import ( arg_parser, build_case_references, @@ -21,7 +26,7 @@ from galaxy.tool_util.verify.script import ( VT_PATH = "galaxy.tool_util.verify.script.verify_tool" -def test_arg_parse(): +def test_arg_parse() -> None: parser = arg_parser() # defaults @@ -61,7 +66,7 @@ def test_arg_parse(): assert args.skip == "executed" -def test_test_tools(): +def test_test_tools() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results("my suite", f.name) @@ -73,7 +78,7 @@ def test_test_tools(): with mock.patch(VT_PATH) as mock_verify: assert_results_not_written(results) run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, ) @@ -85,7 +90,7 @@ def test_test_tools(): assert interactor.history_deleted -def test_test_tools_no_history_cleanup(): +def test_test_tools_no_history_cleanup() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results("my suite", f.name) @@ -95,7 +100,7 @@ def test_test_tools_no_history_cleanup(): with mock.patch(VT_PATH) as mock_verify: assert_results_not_written(results) run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, no_history_cleanup=True, @@ -108,7 +113,7 @@ def test_test_tools_no_history_cleanup(): assert not interactor.history_deleted -def test_test_tools_history_reuse(): +def test_test_tools_history_reuse() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results(EXISTING_SUITE_NAME, f.name) @@ -118,7 +123,7 @@ def test_test_tools_history_reuse(): with mock.patch(VT_PATH) as mock_verify: assert_results_not_written(results) run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, no_history_reuse=False, @@ -133,7 +138,7 @@ def test_test_tools_history_reuse(): assert not interactor.history_deleted -def test_test_tools_no_history_reuse(): +def test_test_tools_no_history_reuse() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results("existing suite", f.name) @@ -143,7 +148,7 @@ def test_test_tools_no_history_reuse(): with mock.patch(VT_PATH) as mock_verify: assert_results_not_written(results) run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, no_history_reuse=True, @@ -158,7 +163,7 @@ def test_test_tools_no_history_reuse(): assert interactor.history_deleted -def test_test_tools_history_name(): +def test_test_tools_history_name() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results("my suite", f.name) @@ -168,7 +173,7 @@ def test_test_tools_history_name(): with mock.patch(VT_PATH) as mock_verify: assert_results_not_written(results) run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, history_name="testfoo", @@ -183,7 +188,7 @@ def test_test_tools_history_name(): assert interactor.history_deleted -def test_test_tool_per_test_history(): +def test_test_tool_per_test_history() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results("my suite", f.name) @@ -194,7 +199,7 @@ def test_test_tool_per_test_history(): with mock.patch(VT_PATH) as mock_verify: assert_results_not_written(results) run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, history_per_test_case=True, @@ -207,7 +212,7 @@ def test_test_tool_per_test_history(): assert not interactor.history_deleted -def test_test_tools_records_exception(): +def test_test_tools_records_exception() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results("my suite", f.name) @@ -217,12 +222,12 @@ def test_test_tools_records_exception(): with mock.patch(VT_PATH) as mock_verify: assert_results_not_written(results) - def side_effect(*args, **kwd): + def side_effect(*args, **kwd) -> NoReturn: raise Exception("Cow") mock_verify.side_effect = side_effect run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, ) @@ -232,7 +237,7 @@ def test_test_tools_records_exception(): assert_results_written(results) -def test_test_tools_records_retry_exception(): +def test_test_tools_records_retry_exception() -> None: interactor = MockGalaxyInteractor() f = NamedTemporaryFile() results = Results("my suite", f.name) @@ -253,7 +258,7 @@ def test_test_tools_records_retry_exception(): mock_verify.side_effect = side_effect run( - interactor, + cast(GalaxyInteractorApi, interactor), test_references, results, retries=1, @@ -294,8 +299,8 @@ def test_results(): assert "Skipped tool tests (1)" in message -def test_build_references(): - interactor = MockGalaxyInteractor() +def test_build_references() -> None: + interactor = cast(GalaxyInteractorApi, MockGalaxyInteractor()) test_references = build_case_references(interactor) assert len(test_references) == 6 @@ -332,11 +337,11 @@ def test_build_references(): assert test_reference.test_index == 2 -def assert_results_not_written(results): +def assert_results_not_written(results: Results) -> None: assert os.stat(results.test_json).st_size == 0 -def assert_results_written(results): +def assert_results_written(results: Results) -> None: assert os.stat(results.test_json).st_size > 0 with open(results.test_json) as f: json.load(f)