diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index f22c81842b4..2259e88ed58 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -8,6 +8,7 @@ from abc import ( abstractmethod, ) from typing import ( + Any, Callable, Container as TypingContainer, Dict, @@ -734,7 +735,7 @@ class BuildMulledDockerContainerResolver(CliContainerResolver): self.namespace = namespace self.hash_func = hash_func self.auto_install = string_as_bool(auto_install) - self._mulled_kwds = { + self._mulled_kwds: Dict[str, Any] = { "namespace": namespace, "hash_func": self.hash_func, "command": "build-and-test", diff --git a/lib/galaxy/tool_util/deps/mulled/invfile.lua b/lib/galaxy/tool_util/deps/mulled/invfile.lua index f7bef84d18b..a63664d4d45 100644 --- a/lib/galaxy/tool_util/deps/mulled/invfile.lua +++ b/lib/galaxy/tool_util/deps/mulled/invfile.lua @@ -88,7 +88,7 @@ inv.task('build') .using(conda_image) .withHostConfig({binds = bind_args}) .run('/bin/sh', '-c', preinstall - .. conda_bin .. ' install ' + .. conda_bin .. ' create ' .. channel_args .. ' ' .. target_args .. ' --strict-channel-priority -p /usr/local --copy --yes ' diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 0d00de2e8ff..47ad7709883 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -20,12 +20,17 @@ import sys from sys import platform as _platform from typing import ( Any, + Callable, Dict, Iterable, List, + NoReturn, + Optional, + TYPE_CHECKING, ) import yaml +from typing_extensions import Literal from galaxy.tool_util.deps import installable from galaxy.tool_util.deps.conda_util import ( @@ -56,6 +61,9 @@ from .util import ( ) from ..conda_compat import MetaData +if TYPE_CHECKING: + from galaxy.util.path import StrPath + log = logging.getLogger(__name__) INVFILE = os.environ.get("INVFILE", os.path.join(os.path.dirname(__file__), "invfile.lua")) @@ -199,30 +207,30 @@ class BuildExistsException(Exception): def mull_targets( targets: List[CondaTarget], - involucro_context=None, - command="build", - channels=DEFAULT_CHANNELS, - namespace="biocontainers", - test="true", - test_files=None, - image_build=None, - name_override=None, - repository_template=DEFAULT_REPOSITORY_TEMPLATE, - dry_run=False, - conda_version=None, - mamba_version=None, - use_mamba=False, - verbose=False, - binds=DEFAULT_BINDS, - rebuild=True, - oauth_token=None, - hash_func="v2", - singularity=False, - singularity_image_dir="singularity_import", - base_image=None, - determine_base_image=True, - invfile=INVFILE, -): + involucro_context: Optional["InvolucroContext"] = None, + command: str = "build", + channels: List[str] = DEFAULT_CHANNELS, + namespace: str = "biocontainers", + test: str = "true", + test_files: Optional[List[str]] = None, + image_build: Optional[str] = None, + name_override: Optional[str] = None, + repository_template: str = DEFAULT_REPOSITORY_TEMPLATE, + dry_run: bool = False, + conda_version: Optional[str] = None, + mamba_version: Optional[str] = None, + use_mamba: bool = False, + verbose: bool = False, + binds: List[str] = DEFAULT_BINDS, + rebuild: bool = True, + oauth_token: Optional[str] = None, + hash_func: Literal["v1", "v2"] = "v2", + singularity: bool = False, + singularity_image_dir: "StrPath" = "singularity_import", + base_image: Optional[str] = None, + determine_base_image: bool = True, + invfile: str = INVFILE, +) -> int: if involucro_context is None: involucro_context = InvolucroContext() @@ -300,26 +308,22 @@ def mull_targets( if test: involucro_args.extend(["-set", f"TEST={test}"]) - verbose = "--verbose" if verbose else "--quiet" + verbose_opt = "--verbose" if verbose else "--quiet" + specs: List[str] = [] + if conda_version is not None: + specs.append(f"conda={conda_version}") conda_bin = "conda" if use_mamba: conda_bin = "mamba" - if mamba_version is None: - mamba_version = "" - involucro_args.extend(["-set", f"CONDA_BIN={conda_bin}"]) - if conda_version is not None or mamba_version is not None: - mamba_test = "true" - specs = [] - if conda_version is not None: - specs.append(f"conda={conda_version}") if mamba_version is not None: - if mamba_version == "" and not specs: - # If nothing but mamba without a specific version is requested, - # then only run conda install if mamba is not already installed. - mamba_test = "[ '[]' = \"$( conda list --json --full-name mamba )\" ]" specs.append(f"mamba={mamba_version}") - conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}""" - involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"]) + else: + # For https://github.com/mamba-org/mamba/pull/3919 + specs.append("mamba>=2.2.0") + involucro_args.extend(["-set", f"CONDA_BIN={conda_bin}"]) + if specs: + conda_install = f"""conda install {verbose_opt} --yes {" ".join(f"'{spec}'" for spec in specs)}""" + involucro_args.extend(["-set", f"PREINSTALL={conda_install}"]) involucro_args.append(command) if test_files: @@ -365,7 +369,12 @@ def context_from_args(args): class InvolucroContext(installable.InstallableContext): installable_description = "Involucro" - def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"): + def __init__( + self, + involucro_bin: Optional[str] = None, + shell_exec: Optional[Callable[[List[str]], int]] = None, + verbose: str = "3", + ) -> None: if involucro_bin is None: if os.path.exists("./involucro"): self.involucro_bin = "./involucro" @@ -376,10 +385,10 @@ class InvolucroContext(installable.InstallableContext): self.shell_exec = shell_exec or commands.shell self.verbose = verbose - def build_command(self, involucro_args): + def build_command(self, involucro_args: List[str]) -> List[str]: return [self.involucro_bin, f"-v={self.verbose}"] + involucro_args - def exec_command(self, involucro_args): + def exec_command(self, involucro_args: List[str]) -> int: cmd = self.build_command(involucro_args) # Create ./build dir manually, otherwise Docker will do it as root created_build_dir = False @@ -563,7 +572,7 @@ def args_to_mull_targets_kwds(args): return kwds -def main(argv=None): +def main(argv=None) -> NoReturn: """Main entry-point for the CLI tool.""" parser = arg_parser(argv, globals()) add_build_arguments(parser) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py index 8dd6b570222..4656a6fe031 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_channel.py @@ -63,7 +63,7 @@ def _new_versions(quay, conda): return sconda - squay # sconda.symmetric_difference(squay) -def run_channel(args, build_last_n_versions=1): +def run_channel(args, build_last_n_versions: int = 1) -> None: """Build list of involucro commands (as shell snippet) to run.""" session = requests.session() for pkg_name, pkg_tests in get_affected_packages(args): diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py index 9cbfff1c6df..9d1fa871e32 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py @@ -12,11 +12,19 @@ Build all recipes discovered in tsv files in a single directory. """ -import collections import glob import os import sys +from dataclasses import dataclass +from typing import ( + Any, + Iterator, + List, + Optional, + Sequence, +) +from galaxy.tool_util.deps.conda_util import CondaTarget from ._cli import arg_parser from .mulled_build import ( add_build_arguments, @@ -27,7 +35,15 @@ from .mulled_build import ( ) KNOWN_FIELDS = ["targets", "image_build", "name_override", "base_image"] -FALLBACK_LINE_TUPLE = collections.namedtuple("FALLBACK_LINE_TUPLE", "targets image_build name_override base_image") +FALLBACK_FIELD_ORDER = ("targets", "image_build", "name_override", "base_image") + + +@dataclass +class Target: + targets: List[CondaTarget] + image_build: Optional[str] + name_override: Optional[str] + base_image: Optional[str] def main(argv=None): @@ -58,7 +74,7 @@ def main(argv=None): sys.exit(ret) -def generate_targets(target_source): +def generate_targets(target_source) -> Iterator[Target]: """Generate all targets from TSV files in specified file or directory.""" target_source = os.path.abspath(target_source) if os.path.isdir(target_source): @@ -69,19 +85,19 @@ def generate_targets(target_source): for target_source_file in target_source_files: # If no headers are defined we use the 4 default fields in the order # that has been used in galaxy-tool-util / galaxy-lib < 20.01 - line_tuple = FALLBACK_LINE_TUPLE + field_order: Sequence[str] = FALLBACK_FIELD_ORDER with open(target_source_file) as f: for line in f.readlines(): if line: line = line.strip() if line.startswith("#"): # headers can define a different column order - line_tuple = tuple_from_header(line) + field_order = field_order_from_header(line) else: - yield line_to_targets(line, line_tuple) + yield line_to_targets(line, field_order) -def tuple_from_header(header): +def field_order_from_header(header: str) -> List[str]: fields = header[1:].split("\t") for field in fields: assert field in KNOWN_FIELDS, f"'{field}' is not one of {KNOWN_FIELDS}" @@ -89,20 +105,20 @@ def tuple_from_header(header): for field in KNOWN_FIELDS: if field not in fields: fields.append(field) - return collections.namedtuple("_Line", f"{' '.join(fields)}") + return fields -def line_to_targets(line_str, line_tuple): +def line_to_targets(line_str: str, field_order: Sequence[str]) -> Target: """Parse a line so that some columns can remain unspecified.""" - line_parts = line_str.split("\t") - n_fields = len(line_tuple._fields) - targets_column = line_tuple._fields.index("targets") + line_parts: List[Any] = line_str.split("\t") + n_fields = len(field_order) + targets_column = field_order.index("targets") assert ( len(line_parts) <= n_fields ), f"Too many fields in line [{line_str}], expect at most {n_fields} - targets, image build number, and name override." line_parts += [None] * (n_fields - len(line_parts)) line_parts[targets_column] = target_str_to_targets(line_parts[targets_column]) - return line_tuple(*line_parts) + return Target(**dict(zip(field_order, line_parts))) __all__ = ("main",) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py index 6be1a2de4a5..cfb4dd376f9 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py @@ -25,9 +25,10 @@ from .mulled_build import ( if TYPE_CHECKING: from galaxy.tool_util.deps.conda_util import CondaTarget + from galaxy.util.path import StrPath -def _mulled_build_tool(tool, args): +def _mulled_build_tool(tool: "StrPath", args): tool_source = get_tool_source(tool) requirements, *_ = tool_source.parse_requirements_and_containers() targets = requirements_to_mulled_targets(requirements) diff --git a/lib/galaxy/tool_util/verify/asserts/image.py b/lib/galaxy/tool_util/verify/asserts/image.py index d9972994d3f..daf4ab8e70c 100644 --- a/lib/galaxy/tool_util/verify/asserts/image.py +++ b/lib/galaxy/tool_util/verify/asserts/image.py @@ -320,19 +320,19 @@ CenterOfMassEps = Annotated[ ), ] Labels = Annotated[ - Optional[Union[str, List[int]]], + Optional[Union[str, List[Union[float, int]]]], AssertionParameter( "List of labels, separated by a comma. Labels *not* on this list will be excluded from consideration. Cannot be used in combination with ``exclude_labels``.", xml_type="xs:string", - json_type="typing.Optional[typing.List[int]]", + json_type=f"typing.Optional[typing.List[{JSON_STRICT_NUMBER}]]", ), ] ExcludeLabels = Annotated[ - Optional[Union[str, List[int]]], + Optional[Union[str, List[Union[float, int]]]], AssertionParameter( "List of labels to be excluded from consideration, separated by a comma. The primary usage of this attribute is to exclude the background of a label image. Cannot be used in combination with ``labels``.", xml_type="xs:string", - json_type="typing.Optional[typing.List[int]]", + json_type=f"typing.Optional[typing.List[{JSON_STRICT_NUMBER}]]", ), ] MeanObjectSize = Annotated[ @@ -734,8 +734,8 @@ def _get_image_labels( channel: Optional[Union[int, str]] = None, slice: Optional[Union[int, str]] = None, frame: Optional[Union[int, str]] = None, - labels: Optional[Union[str, List[int]]] = None, - exclude_labels: Optional[Union[str, List[int]]] = None, + labels: Labels = None, + exclude_labels: ExcludeLabels = None, ) -> Tuple["numpy.typing.NDArray", List[Any]]: """ Determines the unique labels in the output image or a specific channel. @@ -743,7 +743,7 @@ def _get_image_labels( assert labels is None or exclude_labels is None im_arr = _get_image(output_bytes, channel, slice, frame) - def cast_label(label): + def cast_label(label: str) -> Union[float, int]: label = label.strip() if numpy.issubdtype(im_arr.dtype, numpy.integer): return int(label) @@ -752,12 +752,12 @@ def _get_image_labels( raise AssertionError(f'Unsupported image label type: "{im_arr.dtype}"') # Determine labels present in the image. - present_labels = numpy.unique(im_arr) + present_labels: List[Any] = numpy.unique(im_arr).tolist() # Apply filtering due to `labels` (keep only those). if labels is None: labels = [] - if isinstance(labels, str): + elif isinstance(labels, str): labels = [cast_label(label) for label in labels.split(",") if len(label) > 0] if len(labels) > 0: present_labels = [label for label in present_labels if label in labels] @@ -765,7 +765,7 @@ def _get_image_labels( # Apply filtering due to `exclude_labels`. if exclude_labels is None: exclude_labels = [] - if isinstance(exclude_labels, str): + elif isinstance(exclude_labels, str): exclude_labels = [cast_label(label) for label in exclude_labels.split(",") if len(label) > 0] present_labels = [label for label in present_labels if label not in exclude_labels] diff --git a/test/unit/tool_util/mulled/test_mulled_build_files.py b/test/unit/tool_util/mulled/test_mulled_build_files.py index dc3f7ffd13a..c3fb4d84ca0 100644 --- a/test/unit/tool_util/mulled/test_mulled_build_files.py +++ b/test/unit/tool_util/mulled/test_mulled_build_files.py @@ -5,8 +5,8 @@ import yaml from galaxy.tool_util.deps.mulled.mulled_build import target_str_to_targets from galaxy.tool_util.deps.mulled.mulled_build_files import ( - FALLBACK_LINE_TUPLE, generate_targets, + Target, ) TESTCASES = yaml.safe_load( @@ -63,7 +63,7 @@ TEST_IDS = [next(iter(k.keys())) for k in TESTCASES] ) def test_generate_targets(content, equals): equals["targets"] = target_str_to_targets(equals["targets"]) - equals = FALLBACK_LINE_TUPLE(**equals) + equals = Target(**equals) with tempfile.NamedTemporaryFile(mode="w") as tmpfile: tmpfile.write(content) tmpfile.flush()