opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-19 10:51:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #16181 from nsoranzo/mulled_fixes

Browse Source 
Merge ``Target`` class with ``CondaTarget``
This commit is contained in:
Nicola Soranzo
2023-06-05 16:46:02 +01:00
committed by GitHub
parent bddd8909e4 d65721c78d
commit 6829d5b645
19 changed files with 414 additions and 291 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/tool_util/deps/conda_util.py
+31 -12
View File
@@ -409,7 +409,7 @@ def installed_conda_targets(conda_context: CondaContext) -> Iterator["CondaTarge
for name in dir_contents:
versioned_match = VERSIONED_ENV_DIR_NAME.match(name)
if versioned_match:
yield CondaTarget(versioned_match.group(1), versioned_match.group(2))
yield CondaTarget(versioned_match.group(1), version=versioned_match.group(2))
unversioned_match = UNVERSIONED_ENV_DIR_NAME.match(name)
if unversioned_match:
@@ -417,13 +417,18 @@ def installed_conda_targets(conda_context: CondaContext) -> Iterator["CondaTarge
class CondaTarget:
def __init__(self, package: str, version: Optional[str] = None, channel: Optional[str] = None) -> None:
def __init__(
self, package: str, version: Optional[str] = None, build: Optional[str] = None, channel: Optional[str] = None
) -> None:
if SHELL_UNSAFE_PATTERN.search(package) is not None:
raise ValueError(f"Invalid package [{package}] encountered.")
self.package = package
self.package = package.lower()
if version and SHELL_UNSAFE_PATTERN.search(version) is not None:
raise ValueError(f"Invalid version [{version}] encountered.")
self.version = version
if build is not None and SHELL_UNSAFE_PATTERN.search(build) is not None:
raise ValueError(f"Invalid build [{build}] encountered.")
self.build = build
if channel and SHELL_UNSAFE_PATTERN.search(channel) is not None:
raise ValueError(f"Invalid version [{channel}] encountered.")
self.channel = channel
@@ -432,8 +437,8 @@ class CondaTarget:
attributes = f"package={self.package}"
if self.version is not None:
attributes += f",version={self.version}"
else:
attributes += ",unversioned"
if self.build is not None:
attributes += f",build={self.build}"
if self.channel:
attributes += f",channel={self.channel}"
@@ -446,9 +451,12 @@ class CondaTarget:
def package_specifier(self) -> str:
"""Return a package specifier as consumed by conda install/create."""
if self.version:
return f"{self.package}={self.version}"
spec = f"{self.package}={self.version}"
else:
return self.package
spec = f"{self.package}=*"
if self.build:
spec += f"={self.build}"
return spec
@property
def install_environment(self) -> str:
@@ -462,11 +470,16 @@ class CondaTarget:
return f"__{self.package}@_uv_"
def __hash__(self) -> int:
return hash((self.package, self.version, self.channel))
return hash((self.package, self.version, self.build, self.channel))
def __eq__(self, other: Any) -> bool:
if isinstance(other, self.__class__):
return (self.package, self.version, self.channel) == (other.package, other.version, other.channel)
return (self.package, self.version, self.build, self.channel) == (
other.package,
other.version,
other.build,
other.channel,
)
return False
@@ -557,7 +570,7 @@ def best_search_result(
) -> Union[Tuple[None, None], Tuple[Dict[str, Any], bool]]:
"""Find best "conda search" result for specified target.
Return ``None`` if no results match.
Return (``None``, ``None``) if no results match.
"""
# Cannot specify the version here (i.e. conda_target.package_specifier)
# because if the version is not found, the exec_search() call would fail.
@@ -589,10 +602,15 @@ def best_search_result(
def is_search_hit_exact(conda_target: CondaTarget, search_hit: Dict[str, Any]) -> bool:
target_version = conda_target.version
# It'd be nice to make request verson of 1.0 match available
# version of 1.0.3 or something like that.
return bool(not target_version or search_hit["version"] == target_version)
target_version = conda_target.version
if target_version and search_hit["version"] != target_version:
return False
target_build = conda_target.build
if target_build and search_hit["build"] != target_build:
return False
return True
def is_conda_target_installed(conda_target: CondaTarget, conda_context: CondaContext) -> bool:
@@ -671,6 +689,7 @@ def build_isolated_environment(
def requirement_to_conda_targets(requirement: "ToolRequirement") -> Optional[CondaTarget]:
conda_target = None
if requirement.type == "package":
assert requirement.name
conda_target = CondaTarget(requirement.name, version=requirement.version)
return conda_target
lib/galaxy/tool_util/deps/container_resolvers/__init__.py
+16 -6
View File
@@ -6,6 +6,8 @@ from abc import (
)
from typing import (
Any,
Container,
List,
Optional,
TYPE_CHECKING,
)
@@ -16,7 +18,11 @@ from galaxy.util.dictifiable import Dictifiable
if TYPE_CHECKING:
from beaker.cache import Cache
from ..dependencies import AppInfo
from ..dependencies import (
AppInfo,
ToolInfo,
)
from ..requirements import ContainerDescription
class ResolutionCache(Bunch):
@@ -53,20 +59,24 @@ class ContainerResolver(Dictifiable, metaclass=ABCMeta):
return default
@abstractmethod
def resolve(self, enabled_container_types, tool_info, resolution_cache=None, **kwds):
def resolve(
self, enabled_container_types: List[str], tool_info: "ToolInfo", **kwds
) -> Optional["ContainerDescription"]:
"""Find a container matching all supplied requirements for tool.
The supplied argument is a :class:`galaxy.tool_util.deps.containers.ToolInfo` description
The supplied argument is a :class:`galaxy.tool_util.deps.dependencies.ToolInfo` description
of the tool and its requirements.
"""
@abstractproperty
def resolver_type(self):
def resolver_type(self) -> str:
"""Short label for the type of container resolution."""
def _container_type_enabled(self, container_description, enabled_container_types):
def _container_type_enabled(
self, container_description: "ContainerDescription", enabled_container_types: Container[str]
) -> bool:
"""Return a boolean indicating if the specified container type is enabled."""
return container_description.type in enabled_container_types
def __str__(self):
def __str__(self) -> str:
return f"{self.__class__.__name__}[]"
lib/galaxy/tool_util/deps/container_resolvers/explicit.py
+16 -9
View File
@@ -2,7 +2,11 @@
import copy
import logging
import os
from typing import cast
from typing import (
cast,
Optional,
TYPE_CHECKING,
)
from galaxy.util.commands import shell
from . import ContainerResolver
@@ -10,6 +14,9 @@ from .mulled import CliContainerResolver
from ..container_classes import SingularityContainer
from ..requirements import ContainerDescription
if TYPE_CHECKING:
from ..dependencies import AppInfo
log = logging.getLogger(__name__)
DEFAULT_SHELL = "/bin/bash"
@@ -62,8 +69,8 @@ class CachedExplicitSingularityContainerResolver(CliContainerResolver):
container_type = "singularity"
cli = "singularity"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def __init__(self, app_info: Optional["AppInfo"] = None, **kwargs) -> None:
super().__init__(app_info=app_info, **kwargs)
self.cache_directory_path = kwargs.get(
"cache_directory", os.path.join(kwargs["app_info"].container_image_cache_path, "singularity", "explicit")
)
@@ -116,8 +123,8 @@ class CachedExplicitSingularityContainerResolver(CliContainerResolver):
class BaseAdminConfiguredContainerResolver(ContainerResolver):
def __init__(self, app_info=None, shell=DEFAULT_SHELL, **kwds):
super().__init__(app_info, **kwds)
def __init__(self, app_info: Optional["AppInfo"] = None, shell=DEFAULT_SHELL, **kwds) -> None:
super().__init__(app_info=app_info, **kwds)
self.shell = shell
def _container_description(self, identifier, container_type):
@@ -135,8 +142,8 @@ class FallbackContainerResolver(BaseAdminConfiguredContainerResolver):
resolver_type = "fallback"
container_type = "docker"
def __init__(self, app_info=None, identifier="", **kwds):
super().__init__(app_info, **kwds)
def __init__(self, app_info: Optional["AppInfo"] = None, identifier="", **kwds) -> None:
super().__init__(app_info=app_info, **kwds)
assert identifier, "fallback container resolver must be specified with non-empty identifier"
self.identifier = identifier
@@ -187,8 +194,8 @@ class RequiresGalaxyEnvironmentSingularityContainerResolver(RequiresGalaxyEnviro
class MappingContainerResolver(BaseAdminConfiguredContainerResolver):
resolver_type = "mapping"
def __init__(self, app_info=None, **kwds):
super().__init__(app_info, **kwds)
def __init__(self, app_info: Optional["AppInfo"] = None, **kwds) -> None:
super().__init__(app_info=app_info, **kwds)
mappings = self.resolver_kwds["mappings"]
assert isinstance(mappings, list), "mapping container resolver must be specified with mapping list"
self.mappings = mappings
lib/galaxy/tool_util/deps/container_resolvers/mulled.py
+177 -136
View File
@@ -8,12 +8,18 @@ from abc import (
abstractmethod,
)
from typing import (
Callable,
Dict,
List,
NamedTuple,
Optional,
Type,
TYPE_CHECKING,
Union,
)
from requests import Session
from galaxy.util import (
safe_makedirs,
string_as_bool,
@@ -25,7 +31,13 @@ from . import (
ContainerResolver,
ResolutionCache,
)
from ..container_classes import CONTAINER_CLASSES
from ..conda_util import CondaTarget
from ..container_classes import (
Container,
CONTAINER_CLASSES,
DockerContainer,
SingularityContainer,
)
from ..docker_util import build_docker_images_command
from ..mulled.mulled_build import (
DEFAULT_CHANNELS,
@@ -38,7 +50,6 @@ from ..mulled.util import (
default_mulled_conda_channels_from_env,
mulled_tags_for,
split_tag,
Target,
v1_image_name,
v2_image_name,
version_sorted,
@@ -49,64 +60,64 @@ from ..requirements import (
)
if TYPE_CHECKING:
from ..dependencies import AppInfo
from ..dependencies import (
AppInfo,
ToolInfo,
)
log = logging.getLogger(__name__)
class CachedMulledImageSingleTarget(NamedTuple):
package_name: str
version: str
build: str
version: Optional[str]
build: Optional[str]
image_identifier: str
multi_target: bool = False
class CachedV1MulledImageMultiTarget(NamedTuple):
hash: str
build: str
build: Optional[str]
image_identifier: str
multi_target: str = "v1"
class CachedV2MulledImageMultiTarget(NamedTuple):
image_name: str
version_hash: str
build: str
version_hash: Optional[str]
build: Optional[str]
image_identifier: str
multi_target: str = "v2"
@property
def package_hash(target):
def package_hash(self) -> str:
# Make this work for Singularity file name or fully qualified Docker repository
# image names.
image_name = target.image_name
image_name = self.image_name
if "/" not in image_name:
return image_name
else:
return image_name.rsplit("/")[-1]
CachedTarget = Union[CachedMulledImageSingleTarget, CachedV1MulledImageMultiTarget, CachedV2MulledImageMultiTarget]
class CacheDirectory(metaclass=ABCMeta):
def __init__(self, path, hash_func="v2"):
def __init__(self, path: str, hash_func: str = "v2") -> None:
self.path = path
self.hash_func = hash_func
def _list_cached_mulled_images_from_path(self):
def _list_cached_mulled_images_from_path(self) -> List[CachedTarget]:
contents = os.listdir(self.path)
sorted_images = version_sorted(contents)
raw_images = map(lambda name: identifier_to_cached_target(name, self.hash_func), sorted_images)
return list(i for i in raw_images if i is not None)
@abstractmethod
def list_cached_mulled_images_from_path(self):
def list_cached_mulled_images_from_path(self) -> List[CachedTarget]:
"""Generate a list of cached, mulled images in the cache."""
@abstractmethod
def invalidate_cache(self):
def invalidate_cache(self) -> None:
"""Invalidate the cache."""
@@ -127,10 +138,10 @@ class DirMtimeCacheDirectory(CacheDirectory):
super().__init__(path, **kwargs)
self.invalidate_cache()
def __get_mtime(self):
def __get_mtime(self) -> float:
return os.stat(self.path).st_mtime
def __cache(self):
def __cache(self) -> None:
self.__contents = self._list_cached_mulled_images_from_path()
self.__mtime = self.__get_mtime()
log.debug(f"Cached images in path {self.path} at directory mtime {self.__mtime}")
@@ -151,9 +162,9 @@ class DirMtimeCacheDirectory(CacheDirectory):
self.__contents = []
def get_cache_directory_cacher(cacher_type):
def get_cache_directory_cacher(cacher_type: Optional[str]) -> Type[CacheDirectory]:
# these can become a separate module and use plugin_config if we need more
cachers = {
cachers: Dict[str, Type[CacheDirectory]] = {
UncachedCacheDirectory.cacher_type: UncachedCacheDirectory,
DirMtimeCacheDirectory.cacher_type: DirMtimeCacheDirectory,
}
@@ -161,7 +172,9 @@ def get_cache_directory_cacher(cacher_type):
return cachers[cacher_type]
def list_docker_cached_mulled_images(namespace=None, hash_func="v2", resolution_cache=None):
def list_docker_cached_mulled_images(
namespace: Optional[str] = None, hash_func: str = "v2", resolution_cache: Optional[ResolutionCache] = None
) -> List[CachedTarget]:
cache_key = "galaxy.tool_util.deps.container_resolvers.mulled:cached_images"
if resolution_cache is not None and cache_key in resolution_cache:
images_and_versions = resolution_cache.get(cache_key)
@@ -176,7 +189,7 @@ def list_docker_cached_mulled_images(namespace=None, hash_func="v2", resolution_
if resolution_cache is not None:
resolution_cache[cache_key] = images_and_versions
def output_line_to_image(line):
def output_line_to_image(line: str) -> Optional[CachedTarget]:
image = identifier_to_cached_target(line, hash_func, namespace=namespace)
return image
@@ -186,7 +199,9 @@ def list_docker_cached_mulled_images(namespace=None, hash_func="v2", resolution_
return [i for i in raw_images if i is not None]
def identifier_to_cached_target(identifier, hash_func, namespace=None):
def identifier_to_cached_target(
identifier: str, hash_func: str, namespace: Optional[str] = None
) -> Optional[CachedTarget]:
if ":" in identifier:
image_name, version = identifier.rsplit(":", 1)
else:
@@ -196,7 +211,7 @@ def identifier_to_cached_target(identifier, hash_func, namespace=None):
if not version or version == "latest":
version = None
image = None
image: Optional[CachedTarget] = None
prefix = ""
if namespace is not None:
prefix = f"quay.io/{namespace}/"
@@ -215,13 +230,13 @@ def identifier_to_cached_target(identifier, hash_func, namespace=None):
version_hash = None
build = None
if version and "-" in version:
version_hash, build = version.rsplit("-", 1)
elif version.isdigit():
version_hash, build = None, version
elif version:
log.debug(f"Unparsable mulled image tag encountered [{version}]")
if version:
if "-" in version:
version_hash, build = version.rsplit("-", 1)
elif version.isdigit():
version_hash, build = None, version
else:
log.debug(f"Unparsable mulled image tag encountered [{version}]")
image = CachedV2MulledImageMultiTarget(image_name, version_hash, build, identifier)
else:
@@ -234,22 +249,25 @@ def identifier_to_cached_target(identifier, hash_func, namespace=None):
return image
def get_filter(namespace):
def get_filter(namespace: Optional[str]) -> Callable[[str], bool]:
prefix = "quay.io/" if namespace is None else f"quay.io/{namespace}"
return lambda name: name.startswith(prefix) and name.count("/") == 2
def find_best_matching_cached_image(targets, cached_images, hash_func):
def find_best_matching_cached_image(
targets: List[CondaTarget], cached_images: List[CachedTarget], hash_func: str
) -> Optional[CachedTarget]:
if len(targets) == 0:
return None
image = None
image: Optional[CachedTarget] = None
cached_image: CachedTarget
if len(targets) == 1:
target = targets[0]
for cached_image in cached_images:
if cached_image.multi_target:
if not isinstance(cached_image, CachedMulledImageSingleTarget):
continue
if not cached_image.package_name == target.package_name:
if not cached_image.package_name == target.package:
continue
if not target.version or target.version == cached_image.version:
image = cached_image
@@ -262,7 +280,7 @@ def find_best_matching_cached_image(targets, cached_images, hash_func):
package_hash, version_hash = name, None
for cached_image in cached_images:
if cached_image.multi_target != "v2":
if not isinstance(cached_image, CachedV2MulledImageMultiTarget):
continue
if version_hash is None:
@@ -279,7 +297,7 @@ def find_best_matching_cached_image(targets, cached_images, hash_func):
elif hash_func == "v1":
name = v1_image_name(targets)
for cached_image in cached_images:
if cached_image.multi_target != "v1":
if not isinstance(cached_image, CachedV1MulledImageMultiTarget):
continue
if name == cached_image.hash:
@@ -289,12 +307,12 @@ def find_best_matching_cached_image(targets, cached_images, hash_func):
def docker_cached_container_description(
targets: List[Target],
targets: List[CondaTarget],
namespace: str,
hash_func: str = "v2",
shell: str = DEFAULT_CONTAINER_SHELL,
resolution_cache: Optional[ResolutionCache] = None,
):
) -> Optional[ContainerDescription]:
if len(targets) == 0:
return None
@@ -312,7 +330,12 @@ def docker_cached_container_description(
return container
def singularity_cached_container_description(targets, cache_directory, hash_func="v2", shell=DEFAULT_CONTAINER_SHELL):
def singularity_cached_container_description(
targets: List[CondaTarget],
cache_directory: CacheDirectory,
hash_func: str = "v2",
shell: str = DEFAULT_CONTAINER_SHELL,
) -> Optional[ContainerDescription]:
if len(targets) == 0:
return None
@@ -334,8 +357,12 @@ def singularity_cached_container_description(targets, cache_directory, hash_func
def targets_to_mulled_name(
targets, hash_func, namespace, resolution_cache: Optional[ResolutionCache] = None, session=None
):
targets: List[CondaTarget],
hash_func: str,
namespace: str,
resolution_cache: Optional[ResolutionCache] = None,
session: Optional[Session] = None,
) -> Optional[str]:
unresolved_cache_key = "galaxy.tool_util.deps.container_resolvers.mulled:unresolved"
if resolution_cache is not None:
if unresolved_cache_key not in resolution_cache:
@@ -350,10 +377,10 @@ def targets_to_mulled_name(
name = None
def cached_name(cache_key):
def cached_name(cache_key: str) -> Optional[str]:
if mulled_resolution_cache:
try:
return resolution_cache.get(cache_key)
return resolution_cache.get(cache_key) # type: ignore[union-attr] # mulled_resolution_cache not None implies resolution_cache not None
except KeyError:
return None
return None
@@ -361,14 +388,14 @@ def targets_to_mulled_name(
if len(targets) == 1:
target = targets[0]
target_version = target.version
cache_key = f"ns[{namespace}]__single__{target.package_name}__@__{target_version}"
cache_key = f"ns[{namespace}]__single__{target.package}__@__{target_version}"
if cache_key in unresolved_cache:
return None
name = cached_name(cache_key)
if name:
return name
tags = mulled_tags_for(namespace, target.package_name, resolution_cache=resolution_cache, session=session)
tags = mulled_tags_for(namespace, target.package, resolution_cache=resolution_cache, session=session)
if tags:
for tag in tags:
@@ -377,7 +404,7 @@ def targets_to_mulled_name(
else:
version = tag
if target_version and version == target_version:
name = f"{target.package_name}:{tag}"
name = f"{target.package}:{tag}"
break
else:
@@ -432,16 +459,16 @@ class CliContainerResolver(ContainerResolver):
container_type = "docker"
cli = "docker"
def __init__(self, *args, **kwargs):
def __init__(self, app_info: Optional["AppInfo"] = None, **kwargs) -> None:
super().__init__(app_info=app_info, **kwargs)
self._cli_available = bool(which(self.cli))
super().__init__(*args, **kwargs)
@property
def cli_available(self):
def cli_available(self) -> bool:
return self._cli_available
@cli_available.setter
def cli_available(self, value):
def cli_available(self, value: bool) -> None:
if not value:
log.info(
f"{self.cli} CLI not available, cannot list or pull images in Galaxy process. Does not impact kubernetes."
@@ -453,18 +480,16 @@ class SingularityCliContainerResolver(CliContainerResolver):
container_type = "singularity"
cli = "singularity"
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.cache_directory_path = kwargs.get(
"cache_directory", os.path.join(kwargs["app_info"].container_image_cache_path, "singularity", "mulled")
)
self.cache_directory_cacher_type = kwargs.get("cache_directory_cacher_type", None)
self.cache_directory = None
self.hash_func = None
def _init_cache_directory(self):
def __init__(self, app_info: Optional["AppInfo"] = None, hash_func: str = "v2", **kwargs) -> None:
super().__init__(app_info=app_info, **kwargs)
self.hash_func = hash_func
self.cache_directory_cacher_type = kwargs.get("cache_directory_cacher_type")
cacher_class = get_cache_directory_cacher(self.cache_directory_cacher_type)
self.cache_directory = cacher_class(self.cache_directory_path, hash_func=self.hash_func)
cache_directory_path = kwargs.get("cache_directory")
if not cache_directory_path:
assert self.app_info and self.app_info.container_image_cache_path
cache_directory_path = os.path.join(self.app_info.container_image_cache_path, "singularity", "mulled")
self.cache_directory = cacher_class(cache_directory_path, hash_func=self.hash_func)
safe_makedirs(self.cache_directory.path)
@@ -472,12 +497,12 @@ class CachedMulledDockerContainerResolver(CliContainerResolver):
resolver_type = "cached_mulled"
shell = "/bin/bash"
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", **kwds):
def __init__(self, app_info=None, namespace: str = "biocontainers", hash_func: str = "v2", **kwds):
super().__init__(app_info=app_info, **kwds)
self.namespace = namespace
self.hash_func = hash_func
def resolve(self, enabled_container_types, tool_info, **kwds):
def resolve(self, enabled_container_types, tool_info, **kwds) -> Optional[ContainerDescription]:
if (
not self.cli_available
or tool_info.requires_galaxy_python_environment
@@ -500,12 +525,7 @@ class CachedMulledSingularityContainerResolver(SingularityCliContainerResolver):
resolver_type = "cached_mulled_singularity"
shell = "/bin/bash"
def __init__(self, app_info=None, hash_func="v2", **kwds):
super().__init__(app_info=app_info, **kwds)
self.hash_func = hash_func
self._init_cache_directory()
def resolve(self, enabled_container_types, tool_info, **kwds):
def resolve(self, enabled_container_types, tool_info, **kwds) -> Optional[ContainerDescription]:
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
@@ -515,7 +535,7 @@ class CachedMulledSingularityContainerResolver(SingularityCliContainerResolver):
targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell
)
def __str__(self):
def __str__(self) -> str:
return f"CachedMulledSingularityContainerResolver[cache_directory={self.cache_directory.path}]"
@@ -526,13 +546,22 @@ class MulledDockerContainerResolver(CliContainerResolver):
shell = "/bin/bash"
protocol: Optional[str] = None
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds):
def __init__(
self,
app_info: Optional["AppInfo"] = None,
namespace: str = "biocontainers",
hash_func: str = "v2",
auto_install: bool = True,
**kwds,
) -> None:
super().__init__(app_info=app_info, **kwds)
self.namespace = namespace
self.hash_func = hash_func
self.auto_install = string_as_bool(auto_install)
def cached_container_description(self, targets, namespace, hash_func, resolution_cache):
def cached_container_description(
self, targets: List[CondaTarget], namespace: str, hash_func: str, resolution_cache: Optional[ResolutionCache]
) -> Optional[ContainerDescription]:
try:
return docker_cached_container_description(
targets, namespace, hash_func=hash_func, resolution_cache=resolution_cache
@@ -543,16 +572,19 @@ class MulledDockerContainerResolver(CliContainerResolver):
log.exception("An error occured while listing cached docker image. Docker daemon may need to be restarted.")
return None
def pull(self, container):
def pull(self, container: Container) -> None:
if self.cli_available:
assert isinstance(container, DockerContainer)
command = container.build_pull_command()
shell(command)
@property
def can_list_containers(self):
def can_list_containers(self) -> bool:
return self.cli_available
def resolve(self, enabled_container_types, tool_info, install=False, session=None, **kwds):
def resolve(
self, enabled_container_types, tool_info, install: bool = False, session: Optional[Session] = None, **kwds
) -> Optional[ContainerDescription]:
resolution_cache = kwds.get("resolution_cache")
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
@@ -569,48 +601,49 @@ class MulledDockerContainerResolver(CliContainerResolver):
resolution_cache=resolution_cache,
session=session,
)
if name:
container_id = f"quay.io/{self.namespace}/{name}"
if self.protocol:
container_id = f"{self.protocol}{container_id}"
container_description = ContainerDescription(
container_id,
type=self.container_type,
shell=self.shell,
)
if self.can_list_containers:
if install and not self.cached_container_description(
targets,
namespace=self.namespace,
hash_func=self.hash_func,
resolution_cache=resolution_cache,
):
destination_info = {}
destination_for_container_type = kwds.get("destination_for_container_type")
if destination_for_container_type:
destination_info = destination_for_container_type(self.container_type)
container = CONTAINER_CLASSES[self.container_type](
container_description.identifier,
self.app_info,
tool_info,
destination_info,
{},
container_description,
if not name:
return None
container_id = f"quay.io/{self.namespace}/{name}"
if self.protocol:
container_id = f"{self.protocol}{container_id}"
container_description = ContainerDescription(
container_id,
type=self.container_type,
shell=self.shell,
)
if self.can_list_containers:
if install and not self.cached_container_description(
targets,
namespace=self.namespace,
hash_func=self.hash_func,
resolution_cache=resolution_cache,
):
destination_info = {}
destination_for_container_type = kwds.get("destination_for_container_type")
if destination_for_container_type:
destination_info = destination_for_container_type(self.container_type)
container = CONTAINER_CLASSES[self.container_type](
container_description.identifier,
self.app_info,
tool_info,
destination_info,
{},
container_description,
)
self.pull(container)
if not self.auto_install:
container_description = (
self.cached_container_description(
targets,
namespace=self.namespace,
hash_func=self.hash_func,
resolution_cache=resolution_cache,
)
self.pull(container)
if not self.auto_install:
container_description = (
self.cached_container_description(
targets,
namespace=self.namespace,
hash_func=self.hash_func,
resolution_cache=resolution_cache,
)
or container_description
)
return container_description
or container_description
)
return container_description
def __str__(self):
def __str__(self) -> str:
return f"MulledDockerContainerResolver[namespace={self.namespace}]"
@@ -618,11 +651,16 @@ class MulledSingularityContainerResolver(SingularityCliContainerResolver, Mulled
resolver_type = "mulled_singularity"
protocol = "docker://"
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2", auto_install=True, **kwds):
super().__init__(app_info=app_info, **kwds)
def __init__(
self,
app_info: Optional["AppInfo"] = None,
hash_func: str = "v2",
namespace: str = "biocontainers",
auto_install: bool = True,
**kwds,
) -> None:
super().__init__(app_info=app_info, hash_func=hash_func, **kwds)
self.namespace = namespace
self.hash_func = hash_func
self._init_cache_directory()
self.auto_install = string_as_bool(auto_install)
def cached_container_description(self, targets, namespace, hash_func, resolution_cache):
@@ -635,15 +673,16 @@ class MulledSingularityContainerResolver(SingularityCliContainerResolver, Mulled
# Only needs access to path, doesn't require CLI
return True
def pull(self, container):
def pull(self, container: Container) -> None:
if self.cli_available:
assert isinstance(container, SingularityContainer)
cmds = container.build_mulled_singularity_pull_command(
cache_directory=self.cache_directory.path, namespace=self.namespace
)
shell(cmds=cmds)
self.cache_directory.invalidate_cache()
def __str__(self):
def __str__(self) -> str:
return f"MulledSingularityContainerResolver[namespace={self.namespace}]"
@@ -680,7 +719,9 @@ class BuildMulledDockerContainerResolver(CliContainerResolver):
auto_init = self._get_config_option("involucro_auto_init", True)
self.enabled = ensure_installed(self.involucro_context, auto_init)
def resolve(self, enabled_container_types, tool_info, install=False, **kwds):
def resolve(
self, enabled_container_types, tool_info, install: bool = False, **kwds
) -> Optional[ContainerDescription]:
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
@@ -692,7 +733,7 @@ class BuildMulledDockerContainerResolver(CliContainerResolver):
mull_targets(targets, involucro_context=self.involucro_context, **self._mulled_kwds)
return docker_cached_container_description(targets, self.namespace, hash_func=self.hash_func, shell=self.shell)
def __str__(self):
def __str__(self) -> str:
return f"BuildDockerContainerResolver[namespace={self.namespace}]"
@@ -706,10 +747,8 @@ class BuildMulledSingularityContainerResolver(SingularityCliContainerResolver):
def __init__(
self, app_info: Optional["AppInfo"] = None, hash_func: str = "v2", auto_install: bool = True, **kwds
) -> None:
super().__init__(app_info=app_info, **kwds)
super().__init__(app_info=app_info, hash_func=hash_func, **kwds)
self._involucro_context_kwds = {"involucro_bin": self._get_config_option("involucro_path", None)}
self.hash_func = hash_func
self._init_cache_directory()
self.auto_install = string_as_bool(auto_install)
self._mulled_kwds = {
"channels": self._get_config_option("mulled_channels", DEFAULT_CHANNELS),
@@ -723,7 +762,9 @@ class BuildMulledSingularityContainerResolver(SingularityCliContainerResolver):
auto_init = self._get_config_option("involucro_auto_init", True)
self.enabled = ensure_installed(self.involucro_context, auto_init)
def resolve(self, enabled_container_types, tool_info, install=False, **kwds):
def resolve(
self, enabled_container_types, tool_info, install: bool = False, **kwds
) -> Optional[ContainerDescription]:
if tool_info.requires_galaxy_python_environment or self.container_type not in enabled_container_types:
return None
@@ -738,15 +779,15 @@ class BuildMulledSingularityContainerResolver(SingularityCliContainerResolver):
targets, self.cache_directory, hash_func=self.hash_func, shell=self.shell
)
def __str__(self):
def __str__(self) -> str:
return f"BuildSingularityContainerResolver[cache_directory={self.cache_directory.path}]"
def mulled_targets(tool_info):
def mulled_targets(tool_info: "ToolInfo") -> List[CondaTarget]:
return requirements_to_mulled_targets(tool_info.requirements)
def image_name(targets, hash_func):
def image_name(targets: List[CondaTarget], hash_func: str) -> str:
if len(targets) == 0:
return "no targets"
elif hash_func == "v2":
lib/galaxy/tool_util/deps/mulled/mulled_build.py
+5 -8
View File
@@ -21,8 +21,8 @@ from sys import platform as _platform
from typing import (
Any,
Dict,
Iterable,
List,
TYPE_CHECKING,
)
import yaml
@@ -31,6 +31,7 @@ from galaxy.tool_util.deps import installable
from galaxy.tool_util.deps.conda_util import (
best_search_result,
CondaContext,
CondaTarget,
)
from galaxy.tool_util.deps.docker_util import command_list as docker_command_list
from galaxy.util import (
@@ -54,9 +55,6 @@ from .util import (
)
from ..conda_compat import MetaData
if TYPE_CHECKING:
from .util import Target
log = logging.getLogger(__name__)
DIRNAME = os.path.dirname(__file__)
@@ -159,12 +157,12 @@ def conda_versions(pkg_name, file_name):
return ret
def get_conda_hits_for_targets(targets, conda_context: CondaContext) -> List[Dict[str, Any]]:
def get_conda_hits_for_targets(targets: Iterable[CondaTarget], conda_context: CondaContext) -> List[Dict[str, Any]]:
search_results = (best_search_result(t, conda_context, platform="linux-64")[0] for t in targets)
return [r for r in search_results if r]
def base_image_for_targets(targets: List["Target"], conda_context: CondaContext) -> str:
def base_image_for_targets(targets: Iterable[CondaTarget], conda_context: CondaContext) -> str:
"""
determine base image (DEFAULT_BASE_IMAGE/DEFAULT_EXTENDED_BASE_IMAGE) for a
list of targets by inspecting the conda package (i.e. if the use of an
@@ -201,7 +199,7 @@ class BuildExistsException(Exception):
def mull_targets(
targets,
targets: List[CondaTarget],
involucro_context=None,
command="build",
channels=DEFAULT_CHANNELS,
@@ -225,7 +223,6 @@ def mull_targets(
base_image=None,
determine_base_image=True,
):
targets = list(targets)
if involucro_context is None:
involucro_context = InvolucroContext()
lib/galaxy/tool_util/deps/mulled/mulled_build_tool.py
+10 -3
View File
@@ -8,6 +8,10 @@ Build mulled images for requirements defined in a tool:
mulled-build-tool build path/to/tool_file.xml
"""
from typing import (
List,
TYPE_CHECKING,
)
from galaxy.tool_util.parser import get_tool_source
from ._cli import arg_parser
@@ -19,8 +23,11 @@ from .mulled_build import (
)
from .util import build_target
if TYPE_CHECKING:
from galaxy.tool_util.deps.conda_util import CondaTarget
def main(argv=None):
def main(argv=None) -> None:
"""Main entry-point for the CLI tool."""
parser = arg_parser(argv, globals())
add_build_arguments(parser)
@@ -35,8 +42,8 @@ def main(argv=None):
mull_targets(targets, **kwds)
def requirements_to_mulled_targets(requirements):
"""Convert Galaxy's representation of requirements into mulled Target objects.
def requirements_to_mulled_targets(requirements) -> List["CondaTarget"]:
"""Convert Galaxy's representation of requirements into a list of CondaTarget objects.
Only package requirements are retained.
"""
lib/galaxy/tool_util/deps/mulled/util.py
+59 -41
View File
@@ -8,19 +8,27 @@ import re
import sys
import threading
from typing import (
Any,
Dict,
Iterable,
List,
NamedTuple,
Optional,
TYPE_CHECKING,
Union,
)
import requests
from conda_package_streaming.package_streaming import stream_conda_info
from conda_package_streaming.url import stream_conda_info as stream_conda_info_from_url
from packaging.version import Version
from requests import Session
from galaxy.tool_util.version import parse_version
from galaxy.tool_util.deps.conda_util import CondaTarget
from galaxy.tool_util.version import (
LegacyVersion,
parse_version,
)
if TYPE_CHECKING:
from galaxy.tool_util.deps.container_resolvers import ResolutionCache
@@ -29,21 +37,27 @@ log = logging.getLogger(__name__)
QUAY_REPOSITORY_API_ENDPOINT = "https://quay.io/api/v1/repository"
BUILD_NUMBER_REGEX = re.compile(r"\d+$")
PARSED_TAG = collections.namedtuple("PARSED_TAG", "tag version build_string build_number")
MULLED_SOCKET_TIMEOUT = 12
QUAY_VERSIONS_CACHE_EXPIRY = 300
NAMESPACE_HAS_REPO_NAME_KEY = "galaxy.tool_util.deps.container_resolvers.mulled.util:namespace_repo_names"
TAG_CACHE_KEY = "galaxy.tool_util.deps.container_resolvers.mulled.util:tag_cache"
def default_mulled_conda_channels_from_env():
class PARSED_TAG(NamedTuple):
tag: str
version: Union[LegacyVersion, Version]
build_string: Union[LegacyVersion, Version]
build_number: int
def default_mulled_conda_channels_from_env() -> Optional[List[str]]:
if "DEFAULT_MULLED_CONDA_CHANNELS" in os.environ:
return os.environ["DEFAULT_MULLED_CONDA_CHANNELS"].split(",")
else:
return None
def create_repository(namespace, repo_name, oauth_token):
def create_repository(namespace: str, repo_name: str, oauth_token: str) -> None:
assert oauth_token
headers = {"Authorization": f"Bearer {oauth_token}"}
data = {
@@ -55,7 +69,7 @@ def create_repository(namespace, repo_name, oauth_token):
requests.post("https://quay.io/api/v1/repository", json=data, headers=headers, timeout=MULLED_SOCKET_TIMEOUT)
def quay_versions(namespace, pkg_name, session=None):
def quay_versions(namespace: str, pkg_name: str, session: Optional[Session] = None) -> List[str]:
"""Get all version tags for a Docker image stored on quay.io for supplied package name."""
data = quay_repository(namespace, pkg_name, session=session)
@@ -68,7 +82,7 @@ def quay_versions(namespace, pkg_name, session=None):
return [tag for tag in data["tags"].keys() if tag != "latest"]
def quay_repository(namespace, pkg_name, session=None):
def quay_repository(namespace: str, pkg_name: str, session: Optional[Session] = None) -> Dict[str, Any]:
assert namespace is not None
assert pkg_name is not None
url = f"https://quay.io/api/v1/repository/{namespace}/{pkg_name}"
@@ -121,8 +135,13 @@ def _namespace_has_repo_name(namespace: str, repo_name: str, resolution_cache: "
def mulled_tags_for(
namespace, image, tag_prefix=None, resolution_cache=None, session=None, expire=QUAY_VERSIONS_CACHE_EXPIRY
):
namespace: str,
image: str,
tag_prefix: Optional[str] = None,
resolution_cache: Optional["ResolutionCache"] = None,
session: Optional[Session] = None,
expire: float = QUAY_VERSIONS_CACHE_EXPIRY,
) -> List[str]:
"""Fetch remote tags available for supplied image name.
The result will be sorted so newest tags are first.
@@ -141,6 +160,7 @@ def mulled_tags_for(
resolution_cache = resolution_cache.mulled_resolution_cache._get_cache(
"mulled_tag_cache", {"expire": expire}
)
assert resolution_cache is not None
if cache_key not in resolution_cache:
resolution_cache[cache_key] = collections.defaultdict(dict)
tag_cache = resolution_cache.get(cache_key)
@@ -164,12 +184,12 @@ def mulled_tags_for(
return tags
def split_tag(tag):
def split_tag(tag: str) -> List[str]:
"""Split mulled image tag into conda version and conda build."""
return tag.rsplit("--", 1)
def parse_tag(tag):
def parse_tag(tag: str) -> PARSED_TAG:
"""Decompose tag of mulled images into version, build string and build number."""
version = tag.rsplit(":")[-1]
build_string = "-1"
@@ -195,35 +215,30 @@ def parse_tag(tag):
)
def version_sorted(elements):
def version_sorted(elements: Iterable[str]) -> List[str]:
"""Sort iterable based on loose description of "version" from newest to oldest."""
elements = (parse_tag(tag) for tag in elements)
elements = sorted(elements, key=lambda tag: tag.build_string, reverse=True)
elements = sorted(elements, key=lambda tag: tag.build_number, reverse=True)
elements = sorted(elements, key=lambda tag: tag.version, reverse=True)
return [e.tag for e in elements]
parsed_tags_iter = (parse_tag(tag) for tag in elements)
sorted_tags = sorted(parsed_tags_iter, key=lambda tag: tag.build_string, reverse=True)
sorted_tags = sorted(sorted_tags, key=lambda tag: tag.build_number, reverse=True)
sorted_tags = sorted(sorted_tags, key=lambda tag: tag.version, reverse=True)
return [e.tag for e in sorted_tags]
class Target(NamedTuple):
package_name: str
version: Optional[str]
build: Optional[str]
package: Optional[str]
def build_target(package_name, version=None, build=None, tag=None):
"""Use supplied arguments to build a :class:`Target` object."""
def build_target(
package_name: str, version: Optional[str] = None, build: Optional[str] = None, tag: Optional[str] = None
) -> CondaTarget:
"""Use supplied arguments to build a :class:`CondaTarget` object."""
if tag is not None:
assert version is None
assert build is None
version, build = split_tag(tag)
# conda package and quay image names are lowercase
return Target(package_name.lower(), version, build, package_name)
return CondaTarget(package_name, version=version, build=build)
def conda_build_target_str(target):
rval = target.package_name
def conda_build_target_str(target: CondaTarget) -> str:
rval = target.package
if target.version:
rval += f"={target.version}"
@@ -233,7 +248,7 @@ def conda_build_target_str(target):
return rval
def _simple_image_name(targets, image_build=None):
def _simple_image_name(targets: List[CondaTarget], image_build: Optional[str] = None) -> str:
target = targets[0]
suffix = ""
if target.version is not None:
@@ -245,10 +260,12 @@ def _simple_image_name(targets, image_build=None):
suffix += f":{target.version}"
if build is not None:
suffix += f"--{build}"
return f"{target.package_name}{suffix}"
return f"{target.package}{suffix}"
def v1_image_name(targets, image_build=None, name_override=None):
def v1_image_name(
targets: Iterable[CondaTarget], image_build: Optional[str] = None, name_override: Optional[str] = None
) -> str:
"""Generate mulled hash version 1 container identifier for supplied arguments.
If a single target is specified, simply use the supplied name and version as
@@ -279,7 +296,7 @@ def v1_image_name(targets, image_build=None, name_override=None):
if len(targets) == 1:
return _simple_image_name(targets, image_build=image_build)
else:
targets_order = sorted(targets, key=lambda t: t.package_name)
targets_order = sorted(targets, key=lambda t: t.package)
requirements_buffer = "\n".join(map(conda_build_target_str, targets_order))
m = hashlib.sha1()
m.update(requirements_buffer.encode())
@@ -287,7 +304,9 @@ def v1_image_name(targets, image_build=None, name_override=None):
return f"mulled-v1-{m.hexdigest()}{suffix}"
def v2_image_name(targets, image_build=None, name_override=None):
def v2_image_name(
targets: Iterable[CondaTarget], image_build: Optional[str] = None, name_override: Optional[str] = None
) -> str:
"""Generate mulled hash version 2 container identifier for supplied arguments.
If a single target is specified, simply use the supplied name and version as
@@ -327,8 +346,8 @@ def v2_image_name(targets, image_build=None, name_override=None):
if len(targets) == 1:
return _simple_image_name(targets, image_build=image_build)
else:
targets_order = sorted(targets, key=lambda t: t.package_name)
package_name_buffer = "\n".join(map(lambda t: t.package_name, targets_order))
targets_order = sorted(targets, key=lambda t: t.package)
package_name_buffer = "\n".join(map(lambda t: t.package, targets_order))
package_hash = hashlib.sha1()
package_hash.update(package_name_buffer.encode())
@@ -383,7 +402,7 @@ def get_files_from_conda_package(url: str, filepaths: Iterable[str]) -> Dict[str
return ret
def split_container_name(name):
def split_container_name(name: str) -> List[str]:
"""
Takes a container name (e.g. samtools:1.7--1) and returns a list (e.g. ['samtools', '1.7', '1'])
>>> split_container_name('samtools:1.7--1')
@@ -393,22 +412,22 @@ def split_container_name(name):
class PrintProgress:
def __init__(self):
def __init__(self) -> None:
self.thread = threading.Thread(target=self.progress)
self.stop = threading.Event()
def progress(self):
def progress(self) -> None:
while not self.stop.is_set():
print(".", end="")
sys.stdout.flush()
self.stop.wait(60)
print("")
def __enter__(self):
def __enter__(self) -> "PrintProgress":
self.thread.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
def __exit__(self, exc_type, exc_val, exc_tb) -> None:
self.stop.set()
self.thread.join()
@@ -424,7 +443,6 @@ __all__ = (
"quay_versions",
"split_container_name",
"split_tag",
"Target",
"v1_image_name",
"v2_image_name",
"version_sorted",
lib/galaxy/tool_util/deps/requirements.py
+48 -45
View File
@@ -5,6 +5,8 @@ from typing import (
Callable,
cast,
Dict,
Iterable,
Iterator,
List,
Optional,
Tuple,
@@ -33,7 +35,13 @@ class ToolRequirement:
optionally assert a specific version.
"""
def __init__(self, name=None, type=None, version=None, specs=None):
def __init__(
self,
name: Optional[str] = None,
type: Optional[str] = None,
version: Optional[str] = None,
specs: Optional[Iterable["RequirementSpecification"]] = None,
) -> None:
if specs is None:
specs = []
self.name = name
@@ -41,11 +49,11 @@ class ToolRequirement:
self.version = version
self.specs = specs
def to_dict(self):
def to_dict(self) -> Dict[str, Any]:
specs = [s.to_dict() for s in self.specs]
return dict(name=self.name, type=self.type, version=self.version, specs=specs)
def copy(self):
def copy(self) -> "ToolRequirement":
return copy.deepcopy(self)
@staticmethod
@@ -56,7 +64,7 @@ class ToolRequirement:
specs = [RequirementSpecification.from_dict(s) for s in d.get("specs", [])]
return ToolRequirement(name=name, type=type, version=version, specs=specs)
def __eq__(self, other):
def __eq__(self, other: Any) -> bool:
return (
self.name == other.name
and self.type == other.type
@@ -64,13 +72,10 @@ class ToolRequirement:
and self.specs == other.specs
)
def __ne__(self, other):
return not self.__eq__(other)
def __hash__(self):
def __hash__(self) -> int:
return hash((self.name, self.type, self.version, frozenset(self.specs)))
def __str__(self):
def __str__(self) -> str:
return f"ToolRequirement[{self.name},version={self.version},type={self.type},specs={self.specs}]"
__repr__ = __str__
@@ -79,34 +84,31 @@ class ToolRequirement:
class RequirementSpecification:
"""Refine a requirement using a URI."""
def __init__(self, uri, version=None):
def __init__(self, uri: str, version: Optional[str] = None) -> None:
self.uri = uri
self.version = version
@property
def specifies_version(self):
def specifies_version(self) -> bool:
return self.version is not None
@property
def short_name(self):
def short_name(self) -> str:
return self.uri.split("/")[-1]
def to_dict(self):
def to_dict(self) -> Dict[str, Any]:
return dict(uri=self.uri, version=self.version)
@staticmethod
def from_dict(dict):
def from_dict(dict) -> "RequirementSpecification":
uri = dict.get("uri")
version = dict.get("version", None)
return RequirementSpecification(uri=uri, version=version)
def __eq__(self, other):
def __eq__(self, other: Any) -> bool:
return self.uri == other.uri and self.version == other.version
def __ne__(self, other):
return not self.__eq__(other)
def __hash__(self):
def __hash__(self) -> int:
return hash((self.uri, self.version))
@@ -115,59 +117,56 @@ class ToolRequirements:
Represents all requirements (packages, env vars) needed to run a tool.
"""
def __init__(self, tool_requirements=None):
def __init__(self, tool_requirements: Optional[List[Union[ToolRequirement, Dict[str, Any]]]] = None) -> None:
if tool_requirements:
if not isinstance(tool_requirements, list):
raise ToolRequirementsException("ToolRequirements Constructor expects a list")
self.tool_requirements = OrderedSet(
[r if isinstance(r, ToolRequirement) else ToolRequirement.from_dict(r) for r in tool_requirements]
r if isinstance(r, ToolRequirement) else ToolRequirement.from_dict(r) for r in tool_requirements
)
else:
self.tool_requirements = OrderedSet()
@staticmethod
def from_list(requirements: Union[List[ToolRequirement], Dict[str, Any]]) -> "ToolRequirements":
def from_list(requirements: List[Union[ToolRequirement, Dict[str, Any]]]) -> "ToolRequirements":
return ToolRequirements(requirements)
@property
def resolvable(self):
def resolvable(self) -> "ToolRequirements":
return ToolRequirements([r for r in self.tool_requirements if r.type in {"package", "set_environment"}])
@property
def packages(self):
def packages(self) -> "ToolRequirements":
return ToolRequirements([r for r in self.tool_requirements if r.type == "package"])
def to_list(self):
return [r.to_dict() for r in self.tool_requirements]
def append(self, requirement):
def append(self, requirement: Union[ToolRequirement, Dict[str, Any]]) -> None:
if not isinstance(requirement, ToolRequirement):
requirement = ToolRequirement.from_dict(requirement)
self.tool_requirements.add(requirement)
def __eq__(self, other):
def __eq__(self, other: Any) -> bool:
return (
len(self.tool_requirements & other.tool_requirements)
== len(self.tool_requirements)
== len(other.tool_requirements)
)
def __ne__(self, other):
return not self.__eq__(other)
def __iter__(self):
def __iter__(self) -> Iterator[ToolRequirement]:
yield from self.tool_requirements
def __getitem__(self, ii):
def __getitem__(self, ii) -> ToolRequirement:
return list(self.tool_requirements)[ii]
def __len__(self):
def __len__(self) -> int:
return len(self.tool_requirements)
def __hash__(self):
def __hash__(self) -> int:
return sum(r.__hash__() for r in self.tool_requirements)
def to_dict(self):
def to_dict(self) -> List[Dict[str, Any]]:
return [r.to_dict() for r in self.tool_requirements]
@@ -201,7 +200,7 @@ class ContainerDescription:
self.shell = shell
self.explicit = False
def to_dict(self, *args, **kwds):
def to_dict(self, *args, **kwds) -> Dict[str, Any]:
return dict(
identifier=self.identifier,
type=self.type,
@@ -210,7 +209,7 @@ class ContainerDescription:
)
@staticmethod
def from_dict(dict):
def from_dict(dict: Dict[str, Any]) -> "ContainerDescription":
identifier = dict["identifier"]
type = dict.get("type", DEFAULT_CONTAINER_TYPE)
resolve_dependencies = dict.get("resolve_dependencies", DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES)
@@ -222,7 +221,7 @@ class ContainerDescription:
shell=shell,
)
def __str__(self):
def __str__(self) -> str:
return f"ContainerDescription[identifier={self.identifier},type={self.type}]"
@@ -243,7 +242,7 @@ VALID_RESOURCE_TYPES = get_args(ResourceType)
class ResourceRequirement:
def __init__(self, value_or_expression: Union[int, float, str], resource_type: ResourceType):
def __init__(self, value_or_expression: Union[int, float, str], resource_type: ResourceType) -> None:
self.value_or_expression = value_or_expression
if not resource_type:
raise ValueError("Missing resource requirement type")
@@ -256,10 +255,10 @@ class ResourceRequirement:
except ValueError:
self.runtime_required = True
def to_dict(self) -> Dict:
def to_dict(self) -> Dict[str, Any]:
return {"resource_type": self.resource_type, "value_or_expression": self.value_or_expression}
def get_value(self, runtime: Optional[Dict] = None, js_evaluator: Optional[Callable] = None):
def get_value(self, runtime: Optional[Dict] = None, js_evaluator: Optional[Callable] = None) -> float:
if self.runtime_required:
# TODO: hook up evaluator
# return js_evaluator(self.value_or_expression, runtime)
@@ -269,7 +268,7 @@ class ResourceRequirement:
return float(self.value_or_expression)
def resource_requirements_from_list(requirements) -> List[ResourceRequirement]:
def resource_requirements_from_list(requirements: Iterable[Dict[str, Any]]) -> List[ResourceRequirement]:
cwl_to_galaxy = {
"coresMin": "cores_min",
"coresMax": "cores_max",
@@ -298,7 +297,11 @@ def resource_requirements_from_list(requirements) -> List[ResourceRequirement]:
return rr
def parse_requirements_from_lists(software_requirements, containers, resource_requirements) -> Tuple:
def parse_requirements_from_lists(
software_requirements: List[Union[ToolRequirement, Dict[str, Any]]],
containers: Iterable[Dict[str, Any]],
resource_requirements: Iterable[Dict[str, Any]],
) -> Tuple[ToolRequirements, List[ContainerDescription], List[ResourceRequirement]]:
return (
ToolRequirements.from_list(software_requirements),
[ContainerDescription.from_dict(c) for c in containers],
@@ -306,7 +309,7 @@ def parse_requirements_from_lists(software_requirements, containers, resource_re
)
def parse_requirements_from_xml(xml_root, parse_resources=False):
def parse_requirements_from_xml(xml_root, parse_resources: bool = False):
"""
Parses requirements, containers and optionally resource requirements from Xml tree.
@@ -357,13 +360,13 @@ def parse_requirements_from_xml(xml_root, parse_resources=False):
return requirements, containers
def resource_from_element(resource_elem):
def resource_from_element(resource_elem) -> ResourceRequirement:
value_or_expression = xml_text(resource_elem)
resource_type = resource_elem.get("type")
return ResourceRequirement(value_or_expression=value_or_expression, resource_type=resource_type)
def container_from_element(container_elem):
def container_from_element(container_elem) -> ContainerDescription:
identifier = xml_text(container_elem)
type = container_elem.get("type", DEFAULT_CONTAINER_TYPE)
resolve_dependencies = asbool(container_elem.get("resolve_dependencies", DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES))
lib/galaxy/tool_util/deps/resolvers/conda.py
+2 -2
View File
@@ -487,9 +487,9 @@ class CondaDependency(Dependency):
self.cache_path = cache_path
self.environment_path = cache_path
def build_environment(self):
def build_environment(self) -> None:
env_path, exit_code = build_isolated_environment(
CondaTarget(self.name, self.version),
CondaTarget(self.name, version=self.version),
conda_context=self.conda_context,
path=self.environment_path,
copy=self.conda_context.copy_dependencies,
lib/galaxy/tool_util/parser/interface.py
+11 -1
View File
@@ -12,6 +12,7 @@ from typing import (
List,
Optional,
Tuple,
TYPE_CHECKING,
Union,
)
@@ -21,6 +22,13 @@ from typing_extensions import TypedDict
from galaxy.util.path import safe_walk
from .util import _parse_name
if TYPE_CHECKING:
from galaxy.tool_util.deps.requirements import (
ContainerDescription,
ResourceRequirement,
ToolRequirements,
)
NOT_IMPLEMENTED_MESSAGE = "Galaxy tool format does not yet support this tool feature."
@@ -217,7 +225,9 @@ class ToolSource(metaclass=ABCMeta):
return None
@abstractmethod
def parse_requirements_and_containers(self) -> Tuple:
def parse_requirements_and_containers(
self,
) -> Tuple["ToolRequirements", List["ContainerDescription"], List["ResourceRequirement"]]:
"""Return triple of ToolRequirement, ContainerDescription and ResourceRequirement lists."""
@abstractmethod
lib/galaxy/tool_util/version.py
+1 -1
View File
@@ -42,7 +42,7 @@ __all__ = ["parse_version", "LegacyVersion"]
LegacyCmpKey = Tuple[int, Tuple[str, ...]]
def parse_version(version: str) -> Union["LegacyVersion", "Version"]:
def parse_version(version: str) -> Union["LegacyVersion", Version]:
"""
Parse the given version string and return either a :class:`Version` object
or a :class:`LegacyVersion` object depending on if the given version is
lib/galaxy/util/unittest_utils/__init__.py
+13
View File
@@ -5,8 +5,11 @@ from typing import (
)
from unittest import SkipTest
import pytest
import requests
from galaxy.util.commands import which
def is_site_up(url: str) -> bool:
try:
@@ -30,3 +33,13 @@ def skip_if_site_down(url: str) -> Callable:
skip_if_github_down = skip_if_site_down("https://github.com/")
def _identity(func: Callable) -> Callable:
return func
def skip_unless_executable(executable):
if which(executable):
return _identity
return pytest.mark.skip(f"PATH doesn't contain executable {executable}")
lib/galaxy_test/driver/integration_util.py
+4 -11
View File
@@ -24,8 +24,11 @@ import pytest
from galaxy.app import UniverseApplication
from galaxy.tool_util.verify.test_data import TestDataResolver
from galaxy.util import safe_makedirs
from galaxy.util.commands import which
from galaxy.util.unittest import TestCase
from galaxy.util.unittest_utils import (
_identity,
skip_unless_executable,
)
from galaxy_test.base.api import (
UsesApiTestCaseMixin,
UsesCeleryTasks,
@@ -43,10 +46,6 @@ SCRIPT_DIRECTORY = os.path.abspath(os.path.dirname(__file__))
VAULT_CONF = os.path.join(SCRIPT_DIRECTORY, "vault_conf.yml")
def _identity(func):
return func
def skip_if_jenkins(cls):
if os.environ.get("BUILD_NUMBER", ""):
return skip
@@ -66,12 +65,6 @@ def skip_unless_postgres():
return pytest.mark.skip("GALAXY_TEST_DBURI does not point to postgres database, required for this test.")
def skip_unless_executable(executable):
if which(executable):
return _identity
return pytest.mark.skip(f"PATH doesn't contain executable {executable}")
def skip_unless_docker():
return skip_unless_executable("docker")
test/unit/tool_util/mulled/test_mulled_build.py
+5 -4
View File
@@ -32,11 +32,11 @@ def test_base_image_for_targets(target, version, base_image):
@pytest.mark.parametrize("use_mamba", [False, True])
@external_dependency_management
def test_mulled_build_files_cli(use_mamba, tmpdir):
def test_mulled_build_files_cli(use_mamba: bool, tmpdir) -> None:
singularity_image_dir = tmpdir.mkdir("singularity image dir")
target = build_target("zlib")
target = build_target("zlib", version="1.2.13", build="h166bdaf_4")
involucro_context = InvolucroContext(involucro_bin=os.path.join(tmpdir, "involucro"))
mull_targets(
exit_code = mull_targets(
[target],
involucro_context=involucro_context,
command="build-and-test",
@@ -44,7 +44,8 @@ def test_mulled_build_files_cli(use_mamba, tmpdir):
use_mamba=use_mamba,
singularity_image_dir=singularity_image_dir,
)
assert singularity_image_dir.join("zlib").exists()
assert exit_code == 0
assert singularity_image_dir.join("zlib:1.2.13--h166bdaf_4").exists()
def test_target_str_to_targets():
test/unit/tool_util/mulled/test_mulled_search.py
+2 -2
View File
@@ -1,13 +1,13 @@
import pytest
from galaxy.tool_util.deps.mulled.mulled_search import (
conda_path,
CondaSearch,
get_package_hash,
GitHubSearch,
QuaySearch,
singularity_search,
)
from galaxy.util.unittest_utils import skip_unless_executable
from ..util import external_dependency_management
@@ -22,7 +22,7 @@ def test_quay_search():
@external_dependency_management
@pytest.mark.skipif(not conda_path, reason="requires conda on path")
@skip_unless_executable("conda")
def test_conda_search():
t = CondaSearch("bioconda")
search1 = t.get_json("asdfasdf")
test/unit/tool_util/mulled/test_mulled_update_singularity_containers.py
+3 -5
View File
@@ -1,14 +1,12 @@
import os
import pytest
from galaxy.tool_util.deps.mulled.get_tests import main_test_search
from galaxy.tool_util.deps.mulled.mulled_update_singularity_containers import (
docker_to_singularity,
get_list_from_file,
singularity_container_test,
)
from galaxy.util import which
from galaxy.util.unittest_utils import skip_unless_executable
from ..util import external_dependency_management
@@ -26,14 +24,14 @@ def test_get_list_from_file(tmp_path) -> None:
@external_dependency_management
@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")
@skip_unless_executable("singularity")
def test_docker_to_singularity(tmp_path) -> None:
docker_to_singularity("abundancebin:1.0.1--0", "singularity", tmp_path, no_sudo=True)
assert tmp_path.joinpath("abundancebin:1.0.1--0").exists()
@external_dependency_management
@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")
@skip_unless_executable("singularity")
def test_singularity_container_test(tmp_path) -> None:
containers = [
"pybigwig:0.3.22--py36h54a71a5_0", # test Python imports
test/unit/tool_util/test_conda_resolution.py
+2 -1
View File
@@ -81,10 +81,11 @@ def test_best_search_result(tmp_path) -> None:
assert hit is not None
assert hit["name"] == "samtools"
assert exact is True
(hit, exact) = best_search_result(CondaTarget("samtools", version="1.3.1"), conda_context)
(hit, exact) = best_search_result(CondaTarget("samtools", version="1.3.1", build="h0cf4675_11"), conda_context)
assert hit is not None
assert hit["name"] == "samtools"
assert hit["version"] == "1.3.1"
assert hit["build"] == "h0cf4675_11"
assert exact is True
# Search non-existent version
(hit, exact) = best_search_result(CondaTarget("samtools", version="1.16"), conda_context)
test/unit/tool_util/test_container_resolution.py
+7 -2
View File
@@ -56,11 +56,11 @@ def test_docker_container_resolver_detects_docker_cli(mocker):
assert resolver.cli_available
def test_cached_docker_container_docker_cli_absent_resolve(mocker):
def test_cached_docker_container_docker_cli_absent_resolve(mocker) -> None:
mocker.patch("galaxy.tool_util.deps.container_resolvers.mulled.which", return_value=None)
resolver = CachedMulledDockerContainerResolver()
assert resolver.cli_available is False
assert resolver.resolve(enabled_container_types=[], tool_info={}) is None
assert resolver.resolve(enabled_container_types=[], tool_info=ToolInfo()) is None
def test_docker_container_docker_cli_absent_resolve(mocker):
@@ -74,6 +74,7 @@ def test_docker_container_docker_cli_absent_resolve(mocker):
return_value="samtools:1.10--h2e538c0_3",
)
container_description = resolver.resolve(enabled_container_types=["docker"], tool_info=tool_info)
assert container_description
assert container_description.type == "docker"
assert container_description.identifier == "quay.io/biocontainers/samtools:1.10--h2e538c0_3"
@@ -94,6 +95,7 @@ def test_docker_container_docker_cli_exception_resolve(mocker):
)
container_description = resolver.resolve(enabled_container_types=["docker"], tool_info=tool_info, install=True)
assert resolver.cli_available is True
assert container_description
assert container_description.type == "docker"
assert container_description.identifier == "quay.io/biocontainers/samtools:1.10--h2e538c0_3"
@@ -106,6 +108,7 @@ def test_cached_singularity_container_resolver_uncached(mocker):
requirement = ToolRequirement(name="foo", version="1.0", type="package")
tool_info = ToolInfo(requirements=[requirement])
container_description = resolver.resolve(enabled_container_types=["singularity"], tool_info=tool_info)
assert container_description
assert container_description.type == "singularity"
assert container_description.identifier == "/singularity/mulled/foo:1.0--bar"
@@ -121,11 +124,13 @@ def test_cached_singularity_container_resolver_dir_mtime_cached(mocker):
requirement = ToolRequirement(name="baz", version="2.22", type="package")
tool_info = ToolInfo(requirements=[requirement])
container_description = resolver.resolve(enabled_container_types=["singularity"], tool_info=tool_info)
assert container_description
assert container_description.type == "singularity"
assert container_description.identifier == "/singularity/mulled/baz:2.22"
requirement = ToolRequirement(name="foo", version="1.0", type="package")
tool_info.requirements.append(requirement)
container_description = resolver.resolve(enabled_container_types=["singularity"], tool_info=tool_info)
assert container_description
assert container_description.type == "singularity"
assert (
container_description.identifier
test/unit/tool_util/test_tool_deps.py
+2 -2
View File
@@ -153,8 +153,8 @@ REQUIREMENT_B["version"] = "4.7"
def test_tool_requirement_equality():
a = ToolRequirement.from_dict(REQUIREMENT_A)
assert a == ToolRequirement(**REQUIREMENT_A)
b = ToolRequirement(**REQUIREMENT_B)
assert a == ToolRequirement(**REQUIREMENT_A) # type: ignore[arg-type] # https://github.com/python/mypy/issues/10008
b = ToolRequirement(**REQUIREMENT_B) # type: ignore[arg-type] # https://github.com/python/mypy/issues/10008
assert a != b