opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

mypy 0.991 + bonus extra typing

Browse Source
This commit is contained in:
Michael R. Crusoe
2022-11-28 04:50:00 +00:00
committed by Nicola Soranzo
parent ce1e5811ef
commit 2e69f64496
11 changed files with 253 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/app.py
+1 -1
View File
@@ -672,7 +672,7 @@ class UniverseApplication(StructuredApp, GalaxyManagerApplication):
# Tours registry
tour_registry = build_tours_registry(self.config.tour_config_dir)
self.tour_registry = tour_registry
self[ToursRegistry] = tour_registry # type: ignore[misc]
self[ToursRegistry] = tour_registry # type: ignore[type-abstract]
# Webhooks registry
self.webhooks_registry = self._register_singleton(WebhooksRegistry, WebhooksRegistry(self.config.webhooks_dir))
# Heartbeat for thread profiling
lib/galaxy/app_unittest_utils/galaxy_mock.py
+3 -3
View File
@@ -104,7 +104,7 @@ class MockApp(di.Container, GalaxyDataTestApp):
job_metrics: JobMetrics
stop: bool
def __init__(self, config=None, **kwargs):
def __init__(self, config=None, **kwargs) -> None:
super().__init__()
config = config or MockAppConfig(**kwargs)
GalaxyDataTestApp.__init__(self, config=config, **kwargs)
@@ -117,8 +117,8 @@ class MockApp(di.Container, GalaxyDataTestApp):
self[GalaxyModelMapping] = self.model
sts_config = ShortTermStorageConfiguration(short_term_storage_directory=os.path.join(config.data_dir, "sts"))
sts_manager = ShortTermStorageManager(sts_config)
self[ShortTermStorageAllocator] = sts_manager # type: ignore[misc]
self[ShortTermStorageMonitor] = sts_manager # type: ignore[misc]
self[ShortTermStorageAllocator] = sts_manager # type: ignore[type-abstract]
self[ShortTermStorageMonitor] = sts_manager # type: ignore[type-abstract]
self[galaxy_scoped_session] = self.model.context
self.visualizations_registry = MockVisualizationsRegistry()
self.tag_handler = tags.GalaxyTagHandler(self.model.context)
lib/galaxy/tool_util/data/__init__.py
+70 -39
View File
@@ -18,12 +18,15 @@ import time
from glob import glob
from tempfile import NamedTemporaryFile
from typing import (
Any,
BinaryIO,
Dict,
List,
Optional,
Set,
Tuple,
Type,
TYPE_CHECKING,
Union,
)
@@ -31,7 +34,10 @@ import requests
from galaxy import util
from galaxy.exceptions import MessageException
from galaxy.util import RW_R__R__
from galaxy.util import (
Element,
RW_R__R__,
)
from galaxy.util.dictifiable import Dictifiable
from galaxy.util.filelock import FileLock
from galaxy.util.renamed_temporary_file import RenamedTemporaryFile
@@ -40,6 +46,9 @@ from ._schema import (
ToolDataEntryList,
)
if TYPE_CHECKING:
from galaxy.config import GalaxyAppConfiguration
log = logging.getLogger(__name__)
DEFAULT_TABLE_TYPE = "tabular"
@@ -109,20 +118,20 @@ class ToolDataTable(Dictifiable):
def __init__(
self,
config_element,
tool_data_path,
from_shed_config=False,
filename=None,
tool_data_path_files=None,
other_config_dict=None,
):
config_element: Element,
tool_data_path: Union[str, os.PathLike],
from_shed_config: bool = False,
filename: Optional[Union[str, os.PathLike]] = None,
tool_data_path_files: Optional[ToolDataPathFiles] = None,
other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None,
) -> None:
self.name = config_element.get("name")
self.comment_char = config_element.get("comment_char")
self.empty_field_value = config_element.get("empty_field_value", "")
self.empty_field_values = {}
self.empty_field_values: Dict[str, str] = {}
self.allow_duplicate_entries = util.asbool(config_element.get("allow_duplicate_entries", True))
self.here = filename and os.path.dirname(filename)
self.filenames = {}
self.here = os.path.dirname(filename) if filename else None
self.filenames: Dict[str, Dict[str, Any]] = {}
self.tool_data_path = tool_data_path
self.tool_data_path_files = tool_data_path_files
self.other_config_dict = other_config_dict or {}
@@ -139,9 +148,9 @@ class ToolDataTable(Dictifiable):
"filename": filename,
},
)
self._merged_load_info = []
self._merged_load_info: List[Tuple[Type[ToolDataTable], Tuple[List[Any], Dict[str, Any]]]] = []
def _update_version(self, version=None):
def _update_version(self, version: Optional[int] = None) -> int:
if version is not None:
self._loaded_content_version = version
else:
@@ -181,10 +190,10 @@ class ToolDataTable(Dictifiable):
def merge_tool_data_table(self, other_table, allow_duplicates=True, persist=False, entry_source=None, **kwd):
raise NotImplementedError("Abstract method")
def reload_from_files(self):
def reload_from_files(self) -> int:
new_version = self._update_version()
merged_info = self._merged_load_info
self.__init__(*self._load_info[0], **self._load_info[1])
self.__init__(*self._load_info[0], **self._load_info[1]) # type: ignore[misc]
self._update_version(version=new_version)
for (tool_data_table_class, load_info) in merged_info:
self.merge_tool_data_table(tool_data_table_class(*load_info[0], **load_info[1]), allow_duplicates=False)
@@ -214,13 +223,13 @@ class TabularToolDataTable(ToolDataTable):
def __init__(
self,
config_element,
tool_data_path,
from_shed_config=False,
filename=None,
tool_data_path_files=None,
other_config_dict=None,
):
config_element: Element,
tool_data_path: Union[str, os.PathLike],
from_shed_config: bool = False,
filename: Optional[Union[str, os.PathLike]] = None,
tool_data_path_files: Optional[ToolDataPathFiles] = None,
other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None,
) -> None:
super().__init__(
config_element,
tool_data_path,
@@ -233,7 +242,13 @@ class TabularToolDataTable(ToolDataTable):
self.data = []
self.configure_and_load(config_element, tool_data_path, from_shed_config)
def configure_and_load(self, config_element, tool_data_path, from_shed_config=False, url_timeout=10):
def configure_and_load(
self,
config_element: Element,
tool_data_path: Union[str, os.PathLike],
from_shed_config: bool = False,
url_timeout: int = 10,
) -> None:
"""
Configure and load table from an XML element.
"""
@@ -298,7 +313,7 @@ class TabularToolDataTable(ToolDataTable):
# directory which is hard-coded into the tool_data_table_conf.xml entries.
filename = os.path.split(file_path)[1]
filename = os.path.join(tool_data_path, filename)
if self.tool_data_path_files.exists(filename):
if self.tool_data_path_files and self.tool_data_path_files.exists(filename):
found = True
elif not os.path.isabs(filename):
# Since the path attribute can include a hard-coded path to a specific directory
@@ -308,10 +323,14 @@ class TabularToolDataTable(ToolDataTable):
file_path, file_name = os.path.split(filename)
if file_path != self.tool_data_path:
corrected_filename = os.path.join(self.tool_data_path, file_name)
if self.tool_data_path_files.exists(corrected_filename):
if self.tool_data_path_files and self.tool_data_path_files.exists(corrected_filename):
filename = corrected_filename
found = True
elif not from_shed_config and self.tool_data_path_files.exists(f"{corrected_filename}.sample"):
elif (
not from_shed_config
and self.tool_data_path_files
and self.tool_data_path_files.exists(f"{corrected_filename}.sample")
):
log.info(f"Could not find tool data {corrected_filename}, reading sample")
filename = f"{corrected_filename}.sample"
found = True
@@ -403,7 +422,7 @@ class TabularToolDataTable(ToolDataTable):
def get_version_fields(self):
return (self._loaded_content_version, self.get_fields())
def parse_column_spec(self, config_element):
def parse_column_spec(self, config_element: Element) -> None:
"""
Parse column definitions, which can either be a set of 'column' elements
with a name and index (as in dynamic options config), or a shorthand
@@ -437,13 +456,15 @@ class TabularToolDataTable(ToolDataTable):
if "name" not in self.columns:
self.columns["name"] = self.columns["value"]
def extend_data_with(self, filename, errors=None):
def extend_data_with(self, filename: str, errors: Optional[List[Any]] = None) -> None:
here = os.path.dirname(os.path.abspath(filename))
self.data.extend(self.parse_file_fields(filename, errors=errors, here=here))
if not self.allow_duplicate_entries:
self._deduplicate_data()
def parse_file_fields(self, filename, errors: Optional[List[str]] = None, here="__HERE__"):
def parse_file_fields(
self, filename: str, errors: Optional[List[str]] = None, here: str = "__HERE__"
) -> List[List[str]]:
"""
Parse separated lines from file and return a list of tuples.
@@ -542,7 +563,9 @@ class TabularToolDataTable(ToolDataTable):
break
return filename
def _add_entry(self, entry, allow_duplicates=True, persist=False, entry_source=None, **kwd):
def _add_entry(
self, entry: Union[List[str], Dict[str, str]], allow_duplicates=True, persist=False, entry_source=None, **kwd
) -> None:
# accepts dict or list of columns
if isinstance(entry, dict):
fields = []
@@ -737,7 +760,7 @@ class TabularToolDataField(Dictifiable):
return rval
def _expand_here_template(content, here=None):
def _expand_here_template(content: str, here: Optional[str]) -> str:
if here and content:
content = string.Template(content).safe_substitute({"__HERE__": here})
return content
@@ -750,7 +773,7 @@ tool_data_table_types_list: List[Type[ToolDataTable]] = [TabularToolDataTable]
class ToolDataTableManager(Dictifiable):
"""Manages a collection of tool data tables"""
data_tables: Dict[str, "ToolDataTable"]
data_tables: Dict[str, ToolDataTable]
tool_data_table_types = {cls.type_key: cls for cls in tool_data_table_types_list}
def __init__(
@@ -758,8 +781,8 @@ class ToolDataTableManager(Dictifiable):
tool_data_path: str,
config_filename: Optional[ConfigFilesT] = None,
tool_data_table_config_path_set=None,
other_config_dict=None,
):
other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None,
) -> None:
self.tool_data_path = tool_data_path
# This stores all defined data table entries from both the tool_data_table_conf.xml file and the shed_tool_data_table_conf.xml file
# at server startup. If tool shed repositories are installed that contain a valid file named tool_data_table_conf.xml.sample, entries
@@ -776,7 +799,7 @@ class ToolDataTableManager(Dictifiable):
data_tables = [ToolDataEntry(**table.to_dict()) for table in self.data_tables.values()]
return ToolDataEntryList.construct(__root__=data_tables)
def __getitem__(self, key: str):
def __getitem__(self, key: str) -> ToolDataTable:
return self.data_tables.__getitem__(key)
def __setitem__(self, key: str, value):
@@ -851,8 +874,14 @@ class ToolDataTableManager(Dictifiable):
return table_elems
def from_elem(
self, table_elem, tool_data_path, from_shed_config, filename, tool_data_path_files, other_config_dict=None
):
self,
table_elem: Element,
tool_data_path: Union[str, os.PathLike],
from_shed_config: bool,
filename: Union[str, os.PathLike],
tool_data_path_files: ToolDataPathFiles,
other_config_dict: Optional[Union["GalaxyAppConfiguration", Dict[str, Any]]] = None,
) -> ToolDataTable:
table_type = table_elem.get("type", "tabular")
assert table_type in self.tool_data_table_types, f"Unknown data table type '{table_type}'"
return self.tool_data_table_types[table_type](
@@ -950,7 +979,9 @@ class ToolDataTableManager(Dictifiable):
if out_path_is_new:
self.tool_data_path_files.update_files()
def reload_tables(self, table_names=None, path=None):
def reload_tables(
self, table_names: Optional[Union[List[str], str]] = None, path: Optional[str] = None
) -> List[str]:
"""
Reload tool data tables. If neither table_names nor path is given, reloads all tool data tables.
"""
@@ -967,7 +998,7 @@ class ToolDataTableManager(Dictifiable):
log.debug("Reloaded tool data table '%s' from files.", table_name)
return table_names
def get_table_names_by_path(self, path):
def get_table_names_by_path(self, path: str) -> List[str]:
"""Returns a list of table names given a path"""
table_names = set()
for name, data_table in self.data_tables.items():
lib/galaxy/tool_util/parser/output_collection_def.py
+2 -2
View File
@@ -118,7 +118,7 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription):
self.recurse = asbool(kwargs.get("recurse", False))
self.match_relative_path = asbool(kwargs.get("match_relative_path", False))
if pattern in NAMED_PATTERNS:
pattern = NAMED_PATTERNS.get(pattern)
pattern = NAMED_PATTERNS[pattern]
self.pattern = pattern
self.sort_by = sort_by = kwargs.get("sort_by", DEFAULT_SORT_BY)
if sort_by.startswith("reverse_"):
@@ -149,7 +149,7 @@ class FilePatternDatasetCollectionDescription(DatasetCollectionDescription):
return as_dict
@property
def discover_patterns(self):
def discover_patterns(self) -> List[str]:
return [self.pattern]
lib/galaxy/tool_util/parser/output_objects.py
+57 -41
View File
@@ -1,5 +1,11 @@
from typing import List
from typing import (
Any,
Dict,
List,
Optional,
)
from galaxy.util import Element
from galaxy.util.dictifiable import Dictifiable
from .output_actions import ToolOutputActionGroup
from .output_collection_def import (
@@ -9,7 +15,14 @@ from .output_collection_def import (
class ToolOutputBase(Dictifiable):
def __init__(self, name, label=None, filters=None, hidden=False, from_expression=None):
def __init__(
self,
name: str,
label: Optional[str] = None,
filters: Optional[List[Element]] = None,
hidden: bool = False,
from_expression: Optional[str] = None,
) -> None:
super().__init__()
self.name = name
self.label = label
@@ -50,18 +63,18 @@ class ToolOutput(ToolOutputBase):
def __init__(
self,
name,
format=None,
format_source=None,
metadata_source=None,
parent=None,
label=None,
filters=None,
actions=None,
hidden=False,
implicit=False,
from_expression=None,
):
name: str,
format: Optional[str] = None,
format_source: Optional[str] = None,
metadata_source: Optional[str] = None,
parent: Optional[str] = None,
label: Optional[str] = None,
filters: Optional[List[Element]] = None,
actions: Optional[ToolOutputActionGroup] = None,
hidden: bool = False,
implicit: bool = False,
from_expression: Optional[str] = None,
) -> None:
super().__init__(name, label=label, filters=filters, hidden=hidden, from_expression=from_expression)
self.output_type = "data"
self.format = format
@@ -71,14 +84,17 @@ class ToolOutput(ToolOutputBase):
self.actions = actions
# Initialize default values
self.change_format = []
self.change_format: List[Element] = []
self.implicit = implicit
self.from_work_dir = None
self.from_work_dir: Optional[str] = None
self.dataset_collector_descriptions: List[DatasetCollectionDescription] = []
self.default_identifier_source: Optional[str] = None
self.count: Optional[int] = None
self.tool: Optional[Any]
# Tuple emulation
def __len__(self):
def __len__(self) -> int:
return 3
def __getitem__(self, index):
@@ -106,7 +122,7 @@ class ToolOutput(ToolOutputBase):
return as_dict
@staticmethod
def from_dict(name, output_dict, tool=None):
def from_dict(name: str, output_dict: Dict[str, Any], tool: Optional[object] = None) -> "ToolOutput":
output = ToolOutput(name)
output.format = output_dict.get("format", "data")
output.change_format = []
@@ -119,7 +135,7 @@ class ToolOutput(ToolOutputBase):
output.filters = []
output.tool = tool
output.from_work_dir = output_dict.get("from_work_dir", None)
output.hidden = output_dict.get("hidden", "")
output.hidden = output_dict.get("hidden", False)
# TODO: implement tool output action group fixes
output.actions = ToolOutputActionGroup(output, None)
output.dataset_collector_descriptions = dataset_collector_descriptions_from_output_dict(output_dict)
@@ -183,30 +199,30 @@ class ToolOutputCollection(ToolOutputBase):
def __init__(
self,
name,
structure,
label=None,
filters=None,
hidden=False,
default_format="data",
default_format_source=None,
default_metadata_source=None,
inherit_format=False,
inherit_metadata=False,
):
name: str,
structure: "ToolOutputCollectionStructure",
label: Optional[str] = None,
filters: Optional[List[Element]] = None,
hidden: bool = False,
default_format: str = "data",
default_format_source: Optional[str] = None,
default_metadata_source: Optional[str] = None,
inherit_format: bool = False,
inherit_metadata: bool = False,
) -> None:
super().__init__(name, label=label, filters=filters, hidden=hidden)
self.output_type = "collection"
self.collection = True
self.default_format = default_format
self.structure = structure
self.outputs = {}
self.outputs: Dict[str, str] = {}
self.inherit_format = inherit_format
self.inherit_metadata = inherit_metadata
self.metadata_source = default_metadata_source
self.format_source = default_format_source
self.change_format = [] # TODO
self.change_format: List = [] # TODO: not implemented
def known_outputs(self, inputs, type_registry):
if self.dynamic_structure:
@@ -275,13 +291,13 @@ class ToolOutputCollection(ToolOutputBase):
return as_dict
@staticmethod
def from_dict(name, output_dict, tool=None):
def from_dict(name, output_dict, tool=None) -> "ToolOutputCollection":
structure = ToolOutputCollectionStructure.from_dict(output_dict["structure"])
rval = ToolOutputCollection(
name,
structure=structure,
label=output_dict.get("label", None),
filters=None,
filters=[],
hidden=output_dict.get("hidden", False),
default_format=output_dict.get("default_format", "data"),
default_format_source=output_dict.get("default_format_source", None),
@@ -299,12 +315,12 @@ class ToolOutputCollection(ToolOutputBase):
class ToolOutputCollectionStructure:
def __init__(
self,
collection_type,
collection_type_source=None,
collection_type_from_rules=None,
structured_like=None,
dataset_collector_descriptions=None,
):
collection_type: Optional[str],
collection_type_source: Optional[str] = None,
collection_type_from_rules: Optional[str] = None,
structured_like: Optional[str] = None,
dataset_collector_descriptions: Optional[List[DatasetCollectionDescription]] = None,
) -> None:
self.collection_type = collection_type
self.collection_type_source = collection_type_source
self.collection_type_from_rules = collection_type_from_rules
@@ -349,7 +365,7 @@ class ToolOutputCollectionStructure:
}
@staticmethod
def from_dict(as_dict):
def from_dict(as_dict) -> "ToolOutputCollectionStructure":
structure = ToolOutputCollectionStructure(
collection_type=as_dict["collection_type"],
collection_type_source=as_dict["collection_type_source"],
lib/galaxy/tool_util/verify/script.py
+86 -85
View File
@@ -1,18 +1,25 @@
#!/usr/bin/env python
import argparse
import concurrent.futures.thread
import datetime as dt
import json
import logging
import os
import sys
import tempfile
from collections import namedtuple
from concurrent.futures import (
thread,
ThreadPoolExecutor,
)
from typing import List
from typing import (
Any,
Callable,
Dict,
List,
NamedTuple,
Optional,
)
import yaml
@@ -30,18 +37,27 @@ ALL_VERSION = "*"
LATEST_VERSION = None
TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"])
TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"])
class TestReference(NamedTuple):
tool_id: str
tool_version: Optional[str]
test_index: int
class TestException(NamedTuple):
tool_id: str
exception: Exception
was_recorded: bool
class Results:
test_exceptions: List[Exception]
test_exceptions: List[TestException]
def __init__(self, default_suitename, test_json, append=False, galaxy_url=None):
def __init__(
self, default_suitename: str, test_json: str, append: bool = False, galaxy_url: Optional[str] = None
) -> None:
self.test_json = test_json or "-"
self.galaxy_url = galaxy_url
test_results = []
test_exceptions: List[Exception] = []
suitename = default_suitename
if append:
assert test_json != "-"
@@ -51,16 +67,16 @@ class Results:
if "suitename" in previous_results:
suitename = previous_results["suitename"]
self.test_results = test_results
self.test_exceptions = test_exceptions
self.test_exceptions = []
self.suitename = suitename
def register_result(self, result):
def register_result(self, result: Dict[str, Any]) -> None:
self.test_results.append(result)
def register_exception(self, test_exception):
def register_exception(self, test_exception: TestException) -> None:
self.test_exceptions.append(test_exception)
def already_successful(self, test_reference):
def already_successful(self, test_reference: TestReference) -> bool:
test_data = self._previous_test_data(test_reference)
if test_data:
if "status" in test_data and test_data["status"] == "success":
@@ -68,7 +84,7 @@ class Results:
return False
def already_executed(self, test_reference):
def already_executed(self, test_reference: TestReference) -> bool:
test_data = self._previous_test_data(test_reference)
if test_data:
if "status" in test_data and test_data["status"] != "skipped":
@@ -76,7 +92,7 @@ class Results:
return False
def _previous_test_data(self, test_reference):
def _previous_test_data(self, test_reference: TestReference) -> Optional[Dict[str, Any]]:
test_id = _test_id_for_reference(test_reference)
for test_result in self.test_results:
if test_result.get("id") != test_id:
@@ -89,7 +105,7 @@ class Results:
return None
def write(self):
def write(self) -> None:
tests = sorted(self.test_results, key=lambda el: el["id"])
n_passed, n_failures, n_skips = 0, 0, 0
n_errors = len([e for e in self.test_exceptions if not e.was_recorded])
@@ -127,56 +143,37 @@ class Results:
with open(self.test_json, "w") as f:
json.dump(report_obj, f)
def info_message(self):
def info_message(self) -> str:
messages = []
passed_tests = self._tests_with_status("success")
messages.append("Passed tool tests ({}): {}".format(len(passed_tests), [t["id"] for t in passed_tests]))
failed_tests = self._tests_with_status("failure")
messages.append("Failed tool tests ({}): {}".format(len(failed_tests), [t["id"] for t in failed_tests]))
skiped_tests = self._tests_with_status("skip")
messages.append("Skipped tool tests ({}): {}".format(len(skiped_tests), [t["id"] for t in skiped_tests]))
skipped_tests = self._tests_with_status("skip")
messages.append("Skipped tool tests ({}): {}".format(len(skipped_tests), [t["id"] for t in skipped_tests]))
errored_tests = self._tests_with_status("error")
messages.append("Errored tool tests ({}): {}".format(len(errored_tests), [t["id"] for t in errored_tests]))
return "\n".join(messages)
@property
def success_count(self):
self._tests_with_status("success")
@property
def skip_count(self):
self._tests_with_status("skip")
@property
def error_count(self):
return self._tests_with_status("error") + len(self.test_exceptions)
@property
def failure_count(self):
return self._tests_with_status("failure")
def _tests_with_status(self, status):
def _tests_with_status(self, status: str) -> List[Dict[str, Any]]:
return [t for t in self.test_results if t.get("data", {}).get("status") == status]
def test_tools(
galaxy_interactor,
test_references,
results,
log=None,
parallel_tests=1,
history_per_test_case=False,
history_name=None,
no_history_reuse=False,
no_history_cleanup=False,
publish_history=False,
retries=0,
verify_kwds=None,
):
"""Run through tool tests and write report.
Refactor this into Galaxy in 21.01.
"""
galaxy_interactor: GalaxyInteractorApi,
test_references: List[TestReference],
results: Results,
log: Optional[logging.Logger] = None,
parallel_tests: int = 1,
history_per_test_case: bool = False,
history_name: Optional[str] = None,
no_history_reuse: bool = False,
no_history_cleanup: bool = False,
publish_history: bool = False,
retries: int = 0,
verify_kwds: Optional[Dict[str, Any]] = None,
) -> None:
"""Run through tool tests and write report."""
verify_kwds = (verify_kwds or {}).copy()
tool_test_start = dt.datetime.now()
history_created = False
@@ -221,8 +218,8 @@ def test_tools(
try:
executor.shutdown(wait=True)
except KeyboardInterrupt:
executor._threads.clear()
thread._threads_queues.clear()
executor._threads.clear() # type: ignore[attr-defined]
thread._threads_queues.clear() # type: ignore[attr-defined]
results.write()
if log:
if results.test_json == "-":
@@ -236,7 +233,7 @@ def test_tools(
galaxy_interactor.delete_history(test_history)
def _test_id_for_reference(test_reference):
def _test_id_for_reference(test_reference: "TestReference") -> str:
tool_id = test_reference.tool_id
tool_version = test_reference.tool_version
test_index = test_reference.test_index
@@ -253,15 +250,15 @@ def _test_id_for_reference(test_reference):
def _test_tool(
executor,
test_reference,
results,
galaxy_interactor,
log,
retries,
publish_history,
verify_kwds,
):
executor: concurrent.futures.thread.ThreadPoolExecutor,
test_reference: "TestReference",
results: Results,
galaxy_interactor: GalaxyInteractorApi,
log: Optional[logging.Logger],
retries: int,
publish_history: bool,
verify_kwds: Dict[str, Any],
) -> None:
tool_id = test_reference.tool_id
tool_version = test_reference.tool_version
test_index = test_reference.test_index
@@ -272,7 +269,7 @@ def _test_tool(
test_id = _test_id_for_reference(test_reference)
def run_test():
def run_test() -> None:
run_retries = retries
job_data = None
job_exception = None
@@ -323,16 +320,16 @@ def _test_tool(
def build_case_references(
galaxy_interactor,
tool_id=ALL_TOOLS,
tool_version=LATEST_VERSION,
test_index=ALL_TESTS,
page_size=0,
page_number=0,
test_filters=None,
log=None,
):
test_references = []
galaxy_interactor: GalaxyInteractorApi,
tool_id: str = ALL_TOOLS,
tool_version: Optional[str] = LATEST_VERSION,
test_index: int = ALL_TESTS,
page_size: int = 0,
page_number: int = 0,
test_filters: Optional[List[Callable[[TestReference], bool]]] = None,
log: Optional[logging.Logger] = None,
) -> List[TestReference]:
test_references: List[TestReference] = []
if tool_id == ALL_TOOLS:
tests_summary = galaxy_interactor.get_tests_summary()
for tool_id, tool_versions_dict in tests_summary.items():
@@ -341,8 +338,7 @@ def build_case_references(
test_reference = TestReference(tool_id, tool_version, test_index)
test_references.append(test_reference)
else:
assert tool_id
tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {}
tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
for i, tool_test_dict in enumerate(tool_test_dicts):
this_tool_version = tool_test_dict.get("tool_version", tool_version)
this_test_index = i
@@ -351,7 +347,7 @@ def build_case_references(
test_references.append(test_reference)
if test_filters is not None and len(test_filters) > 0:
filtered_test_references = []
filtered_test_references: List[TestReference] = []
for test_reference in test_references:
skip_test = False
for test_filter in test_filters:
@@ -361,7 +357,10 @@ def build_case_references(
skip_test = True
if not skip_test:
filtered_test_references.append(test_reference)
log.info(f"Skipping {len(test_references)-len(filtered_test_references)} out of {len(test_references)} tests.")
if log is not None:
log.info(
f"Skipping {len(test_references)-len(filtered_test_references)} out of {len(test_references)} tests."
)
test_references = filtered_test_references
if page_size > 0:
@@ -372,7 +371,7 @@ def build_case_references(
return test_references
def main(argv=None):
def main(argv=None) -> None:
if argv is None:
argv = sys.argv[1:]
@@ -384,7 +383,11 @@ def main(argv=None):
sys.exit(1)
def run_tests(args, test_filters=None, log=None):
def run_tests(
args: argparse.Namespace,
test_filters: Optional[List[Callable[[TestReference], bool]]] = None,
log: Optional[logging.Logger] = None,
) -> None:
# Split out argument parsing so we can quickly build other scripts - such as a script
# to run all tool tests for a workflow by just passing in a custom test_filters.
test_filters = test_filters or []
@@ -464,12 +467,10 @@ def run_tests(args, test_filters=None, log=None):
exceptions = results.test_exceptions
if exceptions:
exception = exceptions[0]
if hasattr(exception, "exception"):
exception = exception.exception
raise exception
raise exception.exception
def setup_global_logger(name, log_file=None, verbose=False):
def setup_global_logger(name: str, log_file: Optional[str] = None, verbose: bool = False) -> logging.Logger:
formatter = logging.Formatter("%(asctime)s %(levelname)-5s - %(message)s")
console = logging.StreamHandler()
console.setFormatter(formatter)
@@ -490,7 +491,7 @@ def setup_global_logger(name, log_file=None, verbose=False):
return logger
def arg_parser():
def arg_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description=DESCRIPTION)
parser.add_argument("-u", "--galaxy-url", default="http://localhost:8080", help="Galaxy URL")
parser.add_argument("-k", "--key", default=None, help="Galaxy User API Key")
lib/galaxy/tools/evaluation.py
+5 -1
View File
@@ -27,6 +27,7 @@ from galaxy.structured_app import (
BasicSharedApp,
MinimalToolApp,
)
from galaxy.tool_util.data import TabularToolDataTable
from galaxy.tools.parameters import (
visit_input_values,
wrapped_json,
@@ -463,7 +464,10 @@ class ToolEvaluator:
Queries and returns an entry in a data table.
"""
if table_name in self.app.tool_data_tables:
return self.app.tool_data_tables[table_name].get_entry(query_attr, query_val, return_attr)
table = self.app.tool_data_tables[table_name]
if not isinstance(table, TabularToolDataTable):
raise Exception(f"Expected a TabularToolDataTable but got a {type(table)}: {table}.")
return table.get_entry(query_attr, query_val, return_attr)
param_dict["__tool_directory__"] = self.compute_environment.tool_directory()
param_dict["__get_data_table_entry__"] = get_data_table_entry
mypy.ini
-4
View File
@@ -210,8 +210,6 @@ check_untyped_defs = False
check_untyped_defs = False
[mypy-galaxy.tools.bundled.extract.extract_genomic_dna]
check_untyped_defs = False
[mypy-galaxy.tool_util.parser.output_objects]
check_untyped_defs = False
[mypy-galaxy.tool_util.deps.resolvers]
check_untyped_defs = False
[mypy-galaxy.tool_util.deps.mulled.mulled_update_singularity_containers]
@@ -248,8 +246,6 @@ check_untyped_defs = False
check_untyped_defs = False
[mypy-galaxy.tools.expressions.evaluation]
check_untyped_defs = False
[mypy-galaxy.tool_util.data]
check_untyped_defs = False
[mypy-galaxy.tool_util.verify]
check_untyped_defs = False
[mypy-galaxy.tool_util.toolbox.watcher]
test/integration/test_job_recovery.py
+1 -1
View File
@@ -24,7 +24,7 @@ class TestJobRecoveryBeforeHandledIntegration(integration_util.IntegrationTestCa
config["job_config_file"] = SIMPLE_JOB_CONFIG_FILE
config["server_name"] = "moo"
def handle_reconfigure_galaxy_config_kwds(self, config) -> None:
def handle_reconfigure_galaxy_config_kwds(self, config):
config["server_name"] = "main"
def test_recovery(self) -> None:
test/unit/app/tools/test_actions.py
+1 -1
View File
@@ -235,7 +235,7 @@ def quick_output(format, format_source=None, change_format_xml=None):
if change_format_xml:
test_output.change_format = XML(change_format_xml)
else:
test_output.change_format = None
test_output.change_format = []
return test_output
test/unit/tool_util/test_verify_script.py
+27 -22
View File
@@ -1,6 +1,10 @@
import json
import os
from tempfile import NamedTemporaryFile
from typing import (
cast,
NoReturn,
)
from unittest import mock
from galaxy.tool_util.unittest_utils.interactor import (
@@ -10,6 +14,7 @@ from galaxy.tool_util.unittest_utils.interactor import (
MockGalaxyInteractor,
NEW_HISTORY_ID,
)
from galaxy.tool_util.verify.interactor import GalaxyInteractorApi
from galaxy.tool_util.verify.script import (
arg_parser,
build_case_references,
@@ -21,7 +26,7 @@ from galaxy.tool_util.verify.script import (
VT_PATH = "galaxy.tool_util.verify.script.verify_tool"
def test_arg_parse():
def test_arg_parse() -> None:
parser = arg_parser()
# defaults
@@ -61,7 +66,7 @@ def test_arg_parse():
assert args.skip == "executed"
def test_test_tools():
def test_test_tools() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results("my suite", f.name)
@@ -73,7 +78,7 @@ def test_test_tools():
with mock.patch(VT_PATH) as mock_verify:
assert_results_not_written(results)
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
)
@@ -85,7 +90,7 @@ def test_test_tools():
assert interactor.history_deleted
def test_test_tools_no_history_cleanup():
def test_test_tools_no_history_cleanup() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results("my suite", f.name)
@@ -95,7 +100,7 @@ def test_test_tools_no_history_cleanup():
with mock.patch(VT_PATH) as mock_verify:
assert_results_not_written(results)
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
no_history_cleanup=True,
@@ -108,7 +113,7 @@ def test_test_tools_no_history_cleanup():
assert not interactor.history_deleted
def test_test_tools_history_reuse():
def test_test_tools_history_reuse() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results(EXISTING_SUITE_NAME, f.name)
@@ -118,7 +123,7 @@ def test_test_tools_history_reuse():
with mock.patch(VT_PATH) as mock_verify:
assert_results_not_written(results)
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
no_history_reuse=False,
@@ -133,7 +138,7 @@ def test_test_tools_history_reuse():
assert not interactor.history_deleted
def test_test_tools_no_history_reuse():
def test_test_tools_no_history_reuse() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results("existing suite", f.name)
@@ -143,7 +148,7 @@ def test_test_tools_no_history_reuse():
with mock.patch(VT_PATH) as mock_verify:
assert_results_not_written(results)
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
no_history_reuse=True,
@@ -158,7 +163,7 @@ def test_test_tools_no_history_reuse():
assert interactor.history_deleted
def test_test_tools_history_name():
def test_test_tools_history_name() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results("my suite", f.name)
@@ -168,7 +173,7 @@ def test_test_tools_history_name():
with mock.patch(VT_PATH) as mock_verify:
assert_results_not_written(results)
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
history_name="testfoo",
@@ -183,7 +188,7 @@ def test_test_tools_history_name():
assert interactor.history_deleted
def test_test_tool_per_test_history():
def test_test_tool_per_test_history() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results("my suite", f.name)
@@ -194,7 +199,7 @@ def test_test_tool_per_test_history():
with mock.patch(VT_PATH) as mock_verify:
assert_results_not_written(results)
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
history_per_test_case=True,
@@ -207,7 +212,7 @@ def test_test_tool_per_test_history():
assert not interactor.history_deleted
def test_test_tools_records_exception():
def test_test_tools_records_exception() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results("my suite", f.name)
@@ -217,12 +222,12 @@ def test_test_tools_records_exception():
with mock.patch(VT_PATH) as mock_verify:
assert_results_not_written(results)
def side_effect(*args, **kwd):
def side_effect(*args, **kwd) -> NoReturn:
raise Exception("Cow")
mock_verify.side_effect = side_effect
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
)
@@ -232,7 +237,7 @@ def test_test_tools_records_exception():
assert_results_written(results)
def test_test_tools_records_retry_exception():
def test_test_tools_records_retry_exception() -> None:
interactor = MockGalaxyInteractor()
f = NamedTemporaryFile()
results = Results("my suite", f.name)
@@ -253,7 +258,7 @@ def test_test_tools_records_retry_exception():
mock_verify.side_effect = side_effect
run(
interactor,
cast(GalaxyInteractorApi, interactor),
test_references,
results,
retries=1,
@@ -294,8 +299,8 @@ def test_results():
assert "Skipped tool tests (1)" in message
def test_build_references():
interactor = MockGalaxyInteractor()
def test_build_references() -> None:
interactor = cast(GalaxyInteractorApi, MockGalaxyInteractor())
test_references = build_case_references(interactor)
assert len(test_references) == 6
@@ -332,11 +337,11 @@ def test_build_references():
assert test_reference.test_index == 2
def assert_results_not_written(results):
def assert_results_not_written(results: Results) -> None:
assert os.stat(results.test_json).st_size == 0
def assert_results_written(results):
def assert_results_written(results: Results) -> None:
assert os.stat(results.test_json).st_size > 0
with open(results.test_json) as f:
json.load(f)