From 5a72449246be6f0f2fe3955bebf6f42ae89966bc Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 29 Sep 2016 14:28:56 -0400 Subject: [PATCH] Implement beta support for dynamically discovering mulled containers for dependencies. Contains all mulled fixes and enhancements through 16.10.7 including several bug fixes from @bgruening. --- config/galaxy.ini.sample | 31 ++ lib/galaxy/config.py | 20 +- lib/galaxy/tools/deps/__init__.py | 4 +- lib/galaxy/tools/deps/conda_compat.py | 112 +++++++ lib/galaxy/tools/deps/conda_util.py | 84 +++++- .../deps/container_resolvers/__init__.py | 50 +++ .../deps/container_resolvers/explicit.py | 26 ++ .../tools/deps/container_resolvers/mulled.py | 207 +++++++++++++ lib/galaxy/tools/deps/containers.py | 106 +++++-- lib/galaxy/tools/deps/installable.py | 77 +++++ lib/galaxy/tools/deps/mulled/__init__.py | 0 lib/galaxy/tools/deps/mulled/_cli.py | 19 ++ lib/galaxy/tools/deps/mulled/invfile.lua | 67 +++++ lib/galaxy/tools/deps/mulled/mulled_build.py | 284 ++++++++++++++++++ .../tools/deps/mulled/mulled_build_channel.py | 109 +++++++ .../tools/deps/mulled/mulled_build_files.py | 80 +++++ .../tools/deps/mulled/mulled_build_tool.py | 52 ++++ lib/galaxy/tools/deps/mulled/mulled_search.py | 131 ++++++++ lib/galaxy/tools/deps/mulled/util.py | 116 +++++++ lib/galaxy/tools/deps/resolvers/__init__.py | 19 +- lib/galaxy/tools/deps/resolvers/conda.py | 43 +-- .../tools/mulled_example_multi_1.xml | 18 ++ test/functional/tools/samples_tool_conf.xml | 2 + 23 files changed, 1569 insertions(+), 88 deletions(-) create mode 100644 lib/galaxy/tools/deps/conda_compat.py create mode 100644 lib/galaxy/tools/deps/container_resolvers/__init__.py create mode 100644 lib/galaxy/tools/deps/container_resolvers/explicit.py create mode 100644 lib/galaxy/tools/deps/container_resolvers/mulled.py create mode 100644 lib/galaxy/tools/deps/installable.py create mode 100644 lib/galaxy/tools/deps/mulled/__init__.py create mode 100644 lib/galaxy/tools/deps/mulled/_cli.py create mode 100644 lib/galaxy/tools/deps/mulled/invfile.lua create mode 100644 lib/galaxy/tools/deps/mulled/mulled_build.py create mode 100644 lib/galaxy/tools/deps/mulled/mulled_build_channel.py create mode 100644 lib/galaxy/tools/deps/mulled/mulled_build_files.py create mode 100644 lib/galaxy/tools/deps/mulled/mulled_build_tool.py create mode 100644 lib/galaxy/tools/deps/mulled/mulled_search.py create mode 100644 lib/galaxy/tools/deps/mulled/util.py create mode 100644 test/functional/tools/mulled_example_multi_1.xml diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 72876dfc65c..f60ce30d917 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -228,6 +228,37 @@ paste.app_factory = galaxy.web.buildapp:app_factory # than the watchdog default. #watch_tools = False +# Enable Galaxy to fetch Docker containers registered with quay.io generated +# from tool requirements resolved through conda. These containers (when +# available) have been generated using mulled - https://github.com/mulled. +# These containers are highly beta and availablity will vary by tool. +# This option will additionally only be used for job destinations with +# Docker enabled. +#enable_beta_mulled_containers = False + +# Container resolvers configuration (beta). Setup a file describing container +# resolvers to use when discovering containers for Galaxy. If this is set to +# None, the default containers loaded is determined by +# enable_beta_mulled_containers. +#containers_resolvers_config_file = None + +# Container resolvers configuration (beta). Setup a file describing container +# resolvers to use when discovering containers for Galaxy. If this is set to +# None, the default containers loaded is determined by +# enable_beta_mulled_containers. +#containers_resolvers_config_file = None + +# involucro is a tool used to build Docker containers for tools from Conda +# dependencies referenced in tools as `requirement`s. The following path is +# the location of involucro on the Galaxy host. This is ignored if the relevant +# container resolver isn't enabled, and will install on demand unless +# involucro_auto_init is set to False. +#involucro_path = database/dependencies/involucro + +# Install involucro as needed to build Docker containers for tools. Ignored if +# relevant container resolver is not used. +#involucro_auto_init = True + # Enable automatic polling of relative tool sheds to see if any updates # are available for installed repositories. Ideally only one Galaxy # server process should be able to check for repository updates. The diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 9c590c4c747..86b9cc82785 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -226,6 +226,7 @@ class Configuration( object ): self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2)) self.tool_submission_burst_threads = int( kwargs.get( 'tool_submission_burst_threads', '1' ) ) self.tool_submission_burst_at = int( kwargs.get( 'tool_submission_burst_at', '10' ) ) + # Enable new interface for API installations from TS. # Admin menu will list both if enabled. self.enable_beta_ts_api_install = string_as_bool( kwargs.get( 'enable_beta_ts_api_install', 'False' ) ) @@ -323,6 +324,19 @@ class Configuration( object ): else: self.tool_dependency_dir = None self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file) + + self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) ) + containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None ) + if containers_resolvers_config_file: + containers_resolvers_config_file = resolve_path(containers_resolvers_config_file, self.root) + self.containers_resolvers_config_file = containers_resolvers_config_file + + involucro_path = kwargs.get('involucro_path', None) + if involucro_path is None: + involucro_path = os.path.join(tool_dependency_dir, "involucro") + self.involucro_path = resolve_path(involucro_path, self.root) + self.involucro_auto_init = string_as_bool(kwargs.get( 'involucro_auto_init', True)) + # Configuration options for taking advantage of nginx features self.upstream_gzip = string_as_bool( kwargs.get( 'upstream_gzip', False ) ) self.apache_xsendfile = string_as_bool( kwargs.get( 'apache_xsendfile', False ) ) @@ -846,7 +860,11 @@ class ConfiguresGalaxyMixin: default_file_path=file_path, outputs_to_working_directory=self.config.outputs_to_working_directory, container_image_cache_path=self.config.container_image_cache_path, - library_import_dir=self.config.library_import_dir + library_import_dir=self.config.library_import_dir, + enable_beta_mulled_containers=self.config.enable_beta_mulled_containers, + containers_resolvers_config_file=self.config.containers_resolvers_config_file, + involucro_path=self.config.involucro_path, + involucro_auto_init=self.config.involucro_auto_init, ) self.container_finder = containers.ContainerFinder(app_info) diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py index 763732e4dc7..ccf5bb25753 100644 --- a/lib/galaxy/tools/deps/__init__.py +++ b/lib/galaxy/tools/deps/__init__.py @@ -8,7 +8,7 @@ import os.path from galaxy.util import plugin_config from .resolvers import NullDependency -from .resolvers.conda import CondaDependencyResolver +from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver @@ -21,7 +21,7 @@ EXTRA_CONFIG_KWDS = { 'conda_prefix': None, 'conda_exec': None, 'conda_debug': None, - 'conda_ensure_channels': 'r,bioconda,iuc', + 'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS, 'conda_auto_install': False, 'conda_auto_init': False, 'conda_copy_dependencies': False, diff --git a/lib/galaxy/tools/deps/conda_compat.py b/lib/galaxy/tools/deps/conda_compat.py new file mode 100644 index 00000000000..1443378f9f5 --- /dev/null +++ b/lib/galaxy/tools/deps/conda_compat.py @@ -0,0 +1,112 @@ +"""Compat. layer with conda_build/verify if Galaxy/galaxy-lib not installed through conda. + +In general there are utilities available for Conda building and parsing that are high-quality +and should be utilized when available but that are only available in conda channels and not in +PyPI. This module serves as a PyPI capable interface to these utilities. +""" +import collections +import os +import yaml + + +try: + from conda_build.metadata import MetaData +except ImportError: + MetaData = None + +try: + from anaconda_verify.recipe import render_jinja2, parse +except ImportError: + render_jinja2 = None + parse = None + + +class _Memoized(object): + + def __init__(self, func): + self.func = func + self.cache = {} + + def __call__(self, *args): + if not isinstance(args, collections.Hashable): + # uncacheable. a list, for instance. + # better to not cache than blow up. + return self.func(*args) + if args in self.cache: + return self.cache[args] + else: + value = self.func(*args) + self.cache[args] = value + return value + + +def _parse(data, cfg): + """Parse metadata YAML.""" + assert cfg is None, "Conda utilities for evaluating cfg are not available." + return dict(yamlize(data)) + + +def _render_jinja2(recipe_dir): + """Evaluate Conda recipe as a jinja template.""" + try: + import jinja2 + except ImportError: + raise Exception("Failed to import jinja2 for evaluating Conda recipe templates.") + + loaders = [jinja2.FileSystemLoader(recipe_dir)] + env = jinja2.Environment(loader=jinja2.ChoiceLoader(loaders)) + template = env.get_or_select_template('meta.yaml') + return template.render(environment=env) + + +@_Memoized +def yamlize(data): + res = yaml.load(data) + # ensure the result is a dict + if res is None: + res = {} + return res + + +if render_jinja2 is None: + render_jinja2 = _render_jinja2 + +if parse is None: + parse = _parse + + +def raw_metadata(recipe_dir): + """Evaluate Conda template if needed and return raw metadata for supplied recipe directory.""" + meta_path = os.path.join(recipe_dir, 'meta.yaml') + with open(meta_path, 'rb') as fi: + data = fi.read() + if b'{{' in data: + data = render_jinja2(recipe_dir) + meta = parse(data, None) + return meta + + +class _MetaData(object): + + def __init__(self, input_dir): + self.meta = raw_metadata(input_dir) + + def get_value(self, field, default=None): + """Get nested field value or supplied default is not present.""" + section, key = field.split('/') + submeta = self.meta.get(section) + if submeta is None: + submeta = {} + res = submeta.get(key) + if res is None: + res = default + return res + + +if MetaData is None: + MetaData = _MetaData + +__all__ = [ + "MetaData", + "raw_metadata", +] diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py index dc91f9f967a..117acca4389 100644 --- a/lib/galaxy/tools/deps/conda_util.py +++ b/lib/galaxy/tools/deps/conda_util.py @@ -7,12 +7,14 @@ import re import shutil import tempfile +from distutils.version import LooseVersion from sys import platform as _platform import six import yaml from ..deps import commands +from ..deps import installable log = logging.getLogger(__name__) @@ -47,7 +49,8 @@ def find_conda_prefix(conda_prefix=None): return conda_prefix -class CondaContext(object): +class CondaContext(installable.InstallableContext): + installable_description = "Conda" def __init__(self, conda_prefix=None, conda_exec=None, shell_exec=None, debug=False, ensure_channels='', @@ -230,6 +233,16 @@ class CondaContext(object): def activate(self): return self._bin("activate") + def is_installed(self): + return self.is_conda_installed() + + def can_install(self): + return self.can_install_conda() + + @property + def parent_path(self): + return os.path.dirname(os.path.abspath(self.conda_prefix)) + def _bin(self, name): return os.path.join(self.conda_prefix, "bin", name) @@ -273,6 +286,8 @@ class CondaTarget(object): return "CondaTarget[%s]" % attributes + __repr__ = __str__ + @property def package_specifier(self): """ Return a package specifier as consumed by conda install/create. @@ -293,6 +308,17 @@ class CondaTarget(object): else: return "__%s@_uv_" % (self.package) + def __hash__(self): + return hash((self.package, self.version, self.channel)) + + def __eq__(self, other): + if isinstance(other, self.__class__): + return (self.package, self.version, self.channel) == (other.package, other.version, other.channel) + return False + + def __ne__(self, other): + return not(self == other) + def hash_conda_packages(conda_packages, conda_target=None): """ Produce a unique hash on supplied packages. @@ -339,23 +365,55 @@ def cleanup_failed_install(conda_target, conda_context=None): conda_context.exec_remove([conda_target.install_environment]) -def is_target_available(conda_target, conda_context=None): - """ Checks if a specified target is available for installation. - If the package name exists return "True". If in addition the version matches exactly return "exact". - Otherwise return False. +def best_search_result(conda_target, conda_context=None, channels_override=None): + """Find best "conda search" result for specified target. + + Return ``None`` if no results match. """ conda_context = _ensure_conda_context(conda_context) - conda_context.ensure_channels_configured() - search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json", conda_target.package] + if not channels_override: + conda_context.ensure_channels_configured() + + search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"] + if channels_override: + search_cmd.append("--override-channels") + for channel in channels_override: + search_cmd.extend(["--channel", channel]) + search_cmd.append(conda_target.package) res = commands.execute(search_cmd) hits = json.loads(res).get(conda_target.package, []) + hits = sorted(hits, key=lambda hit: LooseVersion(hit['version']), reverse=True) - if len(hits) > 0: - if conda_target.version: - for hit in hits: - if hit['version'] == conda_target.version: - return 'exact' - return True + if len(hits) == 0: + return (None, None) + + best_result = (hits[0], False) + + for hit in hits: + if is_search_hit_exact(conda_target, hit): + best_result = (hit, True) + break + + return best_result + + +def is_search_hit_exact(conda_target, search_hit): + target_version = conda_target.version + # It'd be nice to make request verson of 1.0 match available + # version of 1.0.3 or something like that. + return not target_version or search_hit['version'] == target_version + + +def is_target_available(conda_target, conda_context=None, channels_override=None): + """Check if a specified target is available for installation. + + If the package name exists return ``True`` (the ``bool``). If in addition + the version matches exactly return "exact" (a string). Otherwise return + ``False``. + """ + (best_hit, exact) = best_search_result(conda_target, conda_context, channels_override) + if best_hit: + return 'exact' if exact else True else: return False diff --git a/lib/galaxy/tools/deps/container_resolvers/__init__.py b/lib/galaxy/tools/deps/container_resolvers/__init__.py new file mode 100644 index 00000000000..c478bcce3a2 --- /dev/null +++ b/lib/galaxy/tools/deps/container_resolvers/__init__.py @@ -0,0 +1,50 @@ +"""The module defines the abstract interface for resolving container images for tool execution.""" +from abc import ( + ABCMeta, + abstractmethod, + abstractproperty, +) + +from galaxy.util.dictifiable import Dictifiable + + +class ContainerResolver(Dictifiable, object): + """Description of a technique for resolving container images for tool execution.""" + + # Keys for dictification. + dict_collection_visible_keys = ['resolver_type'] + + __metaclass__ = ABCMeta + + def __init__(self, app_info=None, **kwds): + """Default initializer for ``ContainerResolver`` subclasses.""" + self.app_info = app_info + self.resolver_kwds = kwds + + def _get_config_option(self, key, default=None, config_prefix=None, **kwds): + """Look in resolver-specific settings for option and then fallback to + global settings. + """ + global_key = "%s_%s" % (config_prefix, key) + if key in kwds: + return kwds.get(key) + elif self.app_info and hasattr(self.app_info, global_key): + return getattr(self.app_info, global_key) + else: + return default + + @abstractmethod + def resolve(self, tool_info): + """Find a container matching all supplied requirements for tool. + + The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description + of the tool and its requirements. + """ + + @abstractproperty + def resolver_type(self): + """Short label for the type of container resolution.""" + + def _container_type_enabled(self, container_description, enabled_container_types): + """Return a boolean indicating if the specified container type is enabled.""" + return container_description.type in enabled_container_types diff --git a/lib/galaxy/tools/deps/container_resolvers/explicit.py b/lib/galaxy/tools/deps/container_resolvers/explicit.py new file mode 100644 index 00000000000..851f0c1f99c --- /dev/null +++ b/lib/galaxy/tools/deps/container_resolvers/explicit.py @@ -0,0 +1,26 @@ +"""This module describes the :class:`ExplicitContainerResolver` ContainerResolver plugin.""" +import logging + +from ..container_resolvers import ( + ContainerResolver, +) + +log = logging.getLogger(__name__) + + +class ExplicitContainerResolver(ContainerResolver): + """Find explicit containers referenced in the tool description (e.g. tool XML file) if present.""" + + resolver_type = "explicit" + + def resolve(self, enabled_container_types, tool_info): + """Find a container explicitly mentioned in tool description. + + This ignores the tool requirements and assumes the tool author crafted + a correct container. + """ + for container_description in tool_info.container_descriptions: + if self._container_type_enabled(container_description): + return True + + return False diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py new file mode 100644 index 00000000000..2b2701270ef --- /dev/null +++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py @@ -0,0 +1,207 @@ +"""This module describes the :class:`MulledContainerResolver` ContainerResolver plugin.""" + +import collections +import logging + +from ..container_resolvers import ( + ContainerResolver, +) +from ..docker_util import build_docker_images_command +from ..mulled.mulled_build import ( + check_output, + DEFAULT_CHANNELS, + ensure_installed, + InvolucroContext, + mull_targets, +) +from ..mulled.mulled_build_tool import requirements_to_mulled_targets +from ..mulled.util import ( + image_name, + mulled_tags_for, + split_tag, +) +from ..requirements import ContainerDescription + +log = logging.getLogger(__name__) + + +CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"]) +CachedMulledImageMultiTarget = collections.namedtuple("CachedMulledImageMultiTarget", ["hash", "image_identifier"]) + +CachedMulledImageSingleTarget.multi_target = False +CachedMulledImageMultiTarget.multi_target = True + + +def list_cached_mulled_images(namespace=None): + command = build_docker_images_command(truncate=True, sudo_docker=False) + command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command + images_and_versions = check_output(command) + name_filter = get_filter(namespace) + + def output_line_to_image(line): + image_name, version = line.split(" ", 1) + identifier = "%s:%s" % (image_name, version) + url, namespace, package_description = image_name.split("/") + + if package_description.startswith("mulled-v1-"): + hash = package_description + image = CachedMulledImageMultiTarget(hash, identifier) + else: + build = None + if not version or version == "latest": + version = None + + if version and "--" in version: + version, build = split_tag(version) + + image = CachedMulledImageSingleTarget(image_name, version, build, identifier) + + return image + + return map(output_line_to_image, filter(name_filter, images_and_versions.splitlines())) + + +def get_filter(namespace): + prefix = "quay.io/" if namespace is None else "quay.io/%s" % namespace + return lambda name: name.startswith(prefix) and name.count("/") == 2 + + +def cached_container_description(targets, namespace): + if len(targets) == 0: + return None + + cached_images = list_cached_mulled_images(namespace) + image = None + if len(targets) == 1: + target = targets[0] + for cached_image in cached_images: + if cached_image.multi_target: + continue + if not cached_image.package_name == target.package_name: + continue + if not target.version or target.version == cached_image.version: + image = cached_image + break + else: + name = image_name(targets) + for cached_image in cached_images: + if not cached_image.multi_target: + continue + + if name == cached_image.hash: + image = cached_image + break + + container = None + if image: + container = ContainerDescription( + image.image_identifier, + type="docker", + ) + + return container + + +class CachedMulledContainerResolver(ContainerResolver): + + resolver_type = "cached_mulled" + + def __init__(self, app_info=None, namespace=None): + super(CachedMulledContainerResolver, self).__init__(app_info) + self.namespace = namespace + + def resolve(self, enabled_container_types, tool_info): + targets = mulled_targets(tool_info) + return cached_container_description(targets, self.namespace) + + +class MulledContainerResolver(ContainerResolver): + """Look for mulled images matching tool dependencies.""" + + resolver_type = "mulled" + + def __init__(self, app_info=None, namespace="mulled"): + super(MulledContainerResolver, self).__init__(app_info) + self.namespace = namespace + + def resolve(self, enabled_container_types, tool_info): + targets = mulled_targets(tool_info) + if len(targets) == 0: + return None + + name = None + + if len(targets) == 1: + target = targets[0] + target_version = target.version + tags = mulled_tags_for(self.namespace, target.package_name) + + if not tags: + return None + + if target_version: + for tag in tags: + version, build = split_tag(tag) + if version == target_version: + name = "%s:%s--%s" % (target.package_name, version, build) + break + else: + version, build = split_tag(tags[0]) + name = "%s:%s--%s" % (target.package_name, version, build) + else: + base_image_name = image_name(targets) + tags = mulled_tags_for(self.namespace, base_image_name) + if tags: + name = "%s:%s" % (base_image_name, tags[0]) + + if name: + return ContainerDescription( + "quay.io/%s/%s" % (self.namespace, name), + type="docker", + ) + + +class BuildMulledContainerResolver(ContainerResolver): + """Look for mulled images matching tool dependencies.""" + + resolver_type = "build_mulled" + + def __init__(self, app_info=None, namespace="local", **kwds): + super(BuildMulledContainerResolver, self).__init__(app_info) + self._involucro_context_kwds = { + 'involucro_bin': self._get_config_option("involucro_path", None) + } + self.namespace = namespace + self._mulled_kwds = { + 'namespace': namespace, + 'channels': self._get_config_option("channels", DEFAULT_CHANNELS, prefix="mulled"), + } + self.auto_init = self._get_config_option("auto_init", DEFAULT_CHANNELS, prefix="involucro") + + def resolve(self, enabled_container_types, tool_info): + targets = mulled_targets(tool_info) + if len(targets) == 0: + return None + + mull_targets( + targets, + involucro_context=self._get_involucro_context(), + **self._mulled_kwds + ) + return cached_container_description(targets, self.namespace) + + def _get_involucro_context(self): + involucro_context = InvolucroContext(**self._involucro_context_kwds) + self.enabled = ensure_installed(involucro_context, self.auto_init) + return involucro_context + + +def mulled_targets(tool_info): + return requirements_to_mulled_targets(tool_info.requirements) + + +__all__ = [ + "CachedMulledContainerResolver", + "MulledContainerResolver", + "BuildMulledContainerResolver", +] diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py index aadf73ef482..fff1989986b 100644 --- a/lib/galaxy/tools/deps/containers.py +++ b/lib/galaxy/tools/deps/containers.py @@ -10,14 +10,23 @@ from abc import ( import six from galaxy.util import asbool +from galaxy.util import plugin_config +from .container_resolvers.explicit import ExplicitContainerResolver +from .container_resolvers.mulled import ( + BuildMulledContainerResolver, + CachedMulledContainerResolver, + MulledContainerResolver, +) from .requirements import ContainerDescription from .requirements import DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, DEFAULT_CONTAINER_SHELL from ..deps import docker_util log = logging.getLogger(__name__) -DEFAULT_CONTAINER_TYPE = "docker" +DOCKER_CONTAINER_TYPE = "docker" +DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE +ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE] LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = ''' python << EOF @@ -47,9 +56,18 @@ class ContainerFinder(object): def __init__(self, app_info): self.app_info = app_info - self.container_registry = ContainerRegistry() + self.container_registry = ContainerRegistry(app_info) + + def __enabled_container_types(self, destination_info): + return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)] def find_container(self, tool_info, destination_info, job_info): + enabled_container_types = self.__enabled_container_types(destination_info) + + # Short-cut everything else and just skip checks if no container type is enabled. + if not enabled_container_types: + return NULL_CONTAINER + def __destination_container(container_description=None, container_id=None, container_type=None): if container_description: container_id = container_description.identifier @@ -82,23 +100,10 @@ class ContainerFinder(object): return container # Otherwise lets see if we can find container for the tool. - - # Exact matches first from explicitly listed containers in tools... - for container_description in tool_info.container_descriptions: - container = __destination_container(container_description) - if container: - return container - - # Implement vague concept of looping through all containers - # matching requirements. Exact details need to be worked through - # but hopefully the idea that it sits below find_container somewhere - # external components to this module don't need to worry about it - # is good enough. - container_descriptions = self.container_registry.container_descriptions_for_requirements(tool_info.requirements) - for container_description in container_descriptions: - container = __destination_container(container_description) - if container: - return container + container_description = self.container_registry.find_best_container_description(enabled_container_types, tool_info) + container = __destination_container(container_description) + if container: + return container # If we still don't have a container, check to see if any container # types define a default container id and use that. @@ -175,14 +180,55 @@ class NullContainerFinder(object): return [] -class ContainerRegistry(): +class ContainerRegistry(object): + """Loop through enabled ContainerResolver plugins and find first match.""" - def __init__(self): - pass + def __init__(self, app_info): + self.resolver_classes = self.__resolvers_dict() + self.enable_beta_mulled_containers = app_info.enable_beta_mulled_containers + self.app_info = app_info + self.container_resolvers = self.__build_container_resolvers(app_info) - def container_descriptions_for_requirements(self, requirements): - # Return lists of containers that would match requirements... - return [] + def __build_container_resolvers( self, app_info ): + conf_file = getattr(app_info, 'containers_resolvers_config_file', None) + if not conf_file: + return self.__default_containers_resolvers() + if not os.path.exists( conf_file ): + log.debug( "Unable to find config file '%s'", conf_file) + return self.__default_containers_resolvers() + plugin_source = plugin_config.plugin_source_from_path( conf_file ) + return self.__parse_resolver_conf_xml( plugin_source ) + + def __parse_resolver_conf_xml(self, plugin_source): + extra_kwds = {} + return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds) + + def __default_containers_resolvers(self): + default_resolvers = [ + ExplicitContainerResolver(self.app_info), + ] + if self.enable_beta_mulled_containers: + default_resolvers.extend([ + CachedMulledContainerResolver(self.app_info), + MulledContainerResolver(self.app_info, namespace="mulled"), + BuildMulledContainerResolver(self.app_info), + ]) + return default_resolvers + + def __resolvers_dict( self ): + import galaxy.tools.deps.container_resolvers + return plugin_config.plugins_dict( galaxy.tools.deps.container_resolvers, 'resolver_type' ) + + def find_best_container_description(self, enabled_container_types, tool_info): + """Yield best container description of supplied types matching tool info.""" + for container_resolver in self.container_resolvers: + container_description = container_resolver.resolve(enabled_container_types, tool_info) + log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description)) + if container_description: + assert container_description.type in enabled_container_types + return container_description + + return None class AppInfo(object): @@ -193,7 +239,11 @@ class AppInfo(object): default_file_path=None, outputs_to_working_directory=False, container_image_cache_path=None, - library_import_dir=None + library_import_dir=None, + enable_beta_mulled_containers=False, + containers_resolvers_config_file=None, + involucro_path=None, + involucro_auto_init=True, ): self.galaxy_root_dir = galaxy_root_dir self.default_file_path = default_file_path @@ -201,6 +251,10 @@ class AppInfo(object): self.outputs_to_working_directory = outputs_to_working_directory self.container_image_cache_path = container_image_cache_path self.library_import_dir = library_import_dir + self.enable_beta_mulled_containers = enable_beta_mulled_containers + self.containers_resolvers_config_file = containers_resolvers_config_file + self.involucro_path = involucro_path + self.involucro_auto_init = involucro_auto_init class ToolInfo(object): diff --git a/lib/galaxy/tools/deps/installable.py b/lib/galaxy/tools/deps/installable.py new file mode 100644 index 00000000000..cacc069fe65 --- /dev/null +++ b/lib/galaxy/tools/deps/installable.py @@ -0,0 +1,77 @@ +"""Abstractions for installing local software managed and required by Galaxy/galaxy-lib.""" + +import logging +import os + +from abc import ( + ABCMeta, + abstractmethod, + abstractproperty, +) + +from galaxy.util.filelock import ( + FileLock, + FileLockException +) + +log = logging.getLogger(__name__) + + +class InstallableContext(object): + """Represent a directory/configuration of something that can be installed.""" + + __metaclass__ = ABCMeta + + @abstractmethod + def is_installed(self): + """Return bool indicating if the configured software is installed.""" + + @abstractmethod + def can_install(self): + """Check preconditions for installation.""" + + @abstractproperty + def installable_description(self): + """Short description of thing being installed for log statements.""" + + @abstractproperty + def parent_path(self): + """Return parent path of the location the installable will be created within.""" + + +def ensure_installed(installable_context, install_func, auto_init): + """Make sure target is installed - handle multiple processes potentially attempting installation.""" + parent_path = installable_context.parent_path + desc = installable_context.installable_description + + def _check(): + if not installable_context.is_installed(): + if auto_init: + if installable_context.can_install(): + if install_func(installable_context): + installed = False + log.warning("%s installation requested and failed." % desc) + else: + installed = installable_context.is_installed() + if not installed: + log.warning("%s installation requested, seemed to succeed, but not found." % desc) + else: + installed = False + else: + installed = False + log.warning("%s not installed and auto-installation disabled.", desc) + else: + installed = True + return installed + + if not os.path.exists(parent_path): + os.mkdir(parent_path) + + try: + if auto_init and os.access(parent_path, os.W_OK): + with FileLock(os.path.join(parent_path, desc.lower())): + return _check() + else: + return _check() + except FileLockException: + return ensure_installed(installable_context, auto_init) diff --git a/lib/galaxy/tools/deps/mulled/__init__.py b/lib/galaxy/tools/deps/mulled/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/tools/deps/mulled/_cli.py b/lib/galaxy/tools/deps/mulled/_cli.py new file mode 100644 index 00000000000..3cf32bad5de --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/_cli.py @@ -0,0 +1,19 @@ +"""CLI helpers for mulled command-line tools.""" + +import argparse + + +def arg_parser(argv, globals): + """Build an argparser for this CLI tool.""" + doc = globals["__doc__"] + description, epilog = doc.split("\n", 1) + parser = argparse.ArgumentParser( + description=description, + epilog=epilog, + formatter_class=argparse.RawTextHelpFormatter, + ) + return parser + +__all__ = [ + "arg_parser" +] diff --git a/lib/galaxy/tools/deps/mulled/invfile.lua b/lib/galaxy/tools/deps/mulled/invfile.lua new file mode 100644 index 00000000000..629c15f7b67 --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/invfile.lua @@ -0,0 +1,67 @@ +-- http://stackoverflow.com/questions/19262761/lua-need-to-split-at-comma/19263313#19263313 +function string:split( inSplitPattern, outResults ) + if not outResults then + outResults = { } + end + local theStart = 1 + local theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart ) + while theSplitStart do + table.insert( outResults, string.sub( self, theStart, theSplitStart-1 ) ) + theStart = theSplitEnd + 1 + theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart ) + end + table.insert( outResults, string.sub( self, theStart ) ) + return outResults +end + +local repo = VAR.REPO + +local channel_args = '' +local channels = VAR.CHANNELS:split(",") +for i = 1, #channels do + channel_args = channel_args .. " -c " .. channels[i] +end + +local target_args = '' +local targets = VAR.TARGETS:split(",") +for i = 1, #targets do + target_args = target_args .. " " .. targets[i] +end + +local bind_args = {} +local binds_table = VAR.BINDS:split(",") +for i = 1, #binds_table do + table.insert(bind_args, binds_table[i]) +end + +inv.task('build') + .using('continuumio/miniconda:latest') + .withHostConfig({binds = {"build:/data"}}) + .run('rm', '-rf', '/data/dist') + .using('continuumio/miniconda:latest') + .withHostConfig({binds = bind_args}) + .run('/bin/sh', '-c', 'conda install ' + .. channel_args .. ' ' + .. target_args + .. ' -p /usr/local --copy --yes') + .wrap('build/dist') + .at('/usr/local') + .inImage('bgruening/busybox-bash:0.1') + .as(repo) + +inv.task('test') + .using(repo) + .withConfig({entrypoint = {'/bin/sh', '-c'}}) + .run(VAR.TEST) + +inv.task('push') + .push(repo) + +inv.task('build-and-test') + .runTask('build') + .runTask('test') + +inv.task('all') + .runTask('build') + .runTask('test') + .runTask('push') diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py new file mode 100644 index 00000000000..b359c540cbf --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/mulled_build.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +"""Build a mulled image for specified conda targets. + +Examples + +Build a mulled image with: + + mulled-build build 'samtools=1.3.1,bedtools=2.22' + +""" +from __future__ import print_function + +import json +import os +import string +import subprocess + +from sys import platform as _platform + +from galaxy.tools.deps import commands +from galaxy.tools.deps import installable + +try: + import yaml +except ImportError: + yaml = None + +from ._cli import arg_parser +from .util import build_target, conda_build_target_str, image_name +from ..conda_compat import MetaData + +DIRNAME = os.path.dirname(__file__) +DEFAULT_CHANNEL = "bioconda" +DEFAULT_EXTRA_CHANNELS = ["conda-forge", "r"] +DEFAULT_CHANNELS = [DEFAULT_CHANNEL] + DEFAULT_EXTRA_CHANNELS +DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}" +DEFAULT_BINDS = ["build/dist:/usr/local/"] +IS_OS_X = _platform == "darwin" +INVOLUCRO_VERSION = "1.1.2" + + +def involucro_link(): + if IS_OS_X: + url = "https://github.com/involucro/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION + else: + url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION + return url + + +def get_tests(args, pkg_path): + """Extract test cases given a recipe's meta.yaml file.""" + recipes_dir = args.recipes_dir + + tests = "" + input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path)) + recipe_meta = MetaData(input_dir) + + tests_commands = recipe_meta.get_value('test/commands') + tests_imports = recipe_meta.get_value('test/imports') + requirements = recipe_meta.get_value('requirements/run') + + if tests_imports or tests_commands: + if tests_commands: + tests = ' && '.join(tests_commands) + elif tests_imports and 'python' in requirements: + tests = ' && '.join('python -c "import %s"' % imp for imp in tests_imports) + elif tests_imports and ('perl' in requirements or 'perl-threaded' in requirements): + tests = ' && '.join('''perl -e "use %s;"''' % imp for imp in tests_imports) + tests = tests.replace('$R ', 'Rscript ') + else: + pass + return tests + + +def get_pkg_name(args, pkg_path): + """Extract the package name from a given meta.yaml file.""" + recipes_dir = args.recipes_dir + + input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path)) + recipe_meta = MetaData(input_dir) + return recipe_meta.get_value('package/name') + + +def get_affected_packages(args): + """Return a list of all meta.yaml file that where modified/created recently. + + Length of time to check for indicated by the ``hours`` parameter. + """ + recipes_dir = args.recipes_dir + hours = args.diff_hours + cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours) + pkg_list = check_output(cmd, shell=True) + ret = list() + for pkg in pkg_list.strip().split('\n'): + if pkg and os.path.exists(os.path.join( recipes_dir, pkg )): + ret.append( (get_pkg_name(args, pkg), get_tests(args, pkg)) ) + return ret + + +def check_output(cmd, shell=True): + return subprocess.check_output(cmd, shell=shell) + + +def conda_versions(pkg_name, file_name): + """Return all conda version strings for a specified package name.""" + j = json.load(open(file_name)) + ret = list() + for pkg in j['packages'].values(): + if pkg['name'] == pkg_name: + ret.append('%s--%s' % (pkg['version'], pkg['build'])) + return ret + + +def mull_targets( + targets, involucro_context=None, + command="build", channels=DEFAULT_CHANNELS, namespace="mulled", + test='true', image_build=None, name_override=None, + repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False, + binds=DEFAULT_BINDS +): + if involucro_context is None: + involucro_context = InvolucroContext() + + repo_template_kwds = { + "namespace": namespace, + "image": image_name(targets, image_build=image_build, name_override=name_override) + } + repo = string.Template(repository_template).safe_substitute(repo_template_kwds) + + for channel in channels: + if channel.startswith('file://'): + bind_path = channel.lstrip('file://') + binds.append('/%s:/%s' % (bind_path, bind_path)) + + channels = ",".join(channels) + target_str = ",".join(map(conda_build_target_str, targets)) + bind_str = ",".join(binds) + involucro_args = [ + '-f', '%s/invfile.lua' % DIRNAME, + '-set', "CHANNELS='%s'" % channels, + '-set', "TEST='%s'" % test, + '-set', "TARGETS='%s'" % target_str, + '-set', "REPO='%s'" % repo, + '-set', "BINDS='%s'" % bind_str, + command, + ] + print(" ".join(involucro_context.build_command(involucro_args))) + if not dry_run: + ensure_installed(involucro_context, True) + involucro_context.exec_command(involucro_args) + + +def context_from_args(args): + return InvolucroContext(involucro_bin=args.involucro_path) + + +class InvolucroContext(installable.InstallableContext): + + installable_description = "Involucro" + + def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"): + if involucro_bin is None: + if os.path.exists("./involucro"): + self.involucro_bin = "./involucro" + else: + self.involucro_bin = "involucro" + else: + self.involucro_bin = involucro_bin + self.shell_exec = shell_exec or commands.shell + self.verbose = verbose + + def build_command(self, involucro_args): + return [self.involucro_bin, "-v=%s" % self.verbose] + involucro_args + + def exec_command(self, involucro_args): + cmd = self.build_command(involucro_args) + return self.shell_exec(" ".join(cmd)) + + def is_installed(self): + return os.path.exists(self.involucro_bin) + + def can_install(self): + return True + + @property + def parent_path(self): + return os.path.dirname(os.path.abspath(self.involucro_bin)) + + +def ensure_installed(involucro_context, auto_init): + return installable.ensure_installed(involucro_context, install_involucro, auto_init) + + +def install_involucro(involucro_context=None, to_path=None): + to_path = involucro_context.involucro_bin + download_cmd = " ".join(commands.download_command(involucro_link(), to=to_path, quote_url=True)) + full_cmd = "%s && chmod +x %s" % (download_cmd, to_path) + return involucro_context.shell_exec(full_cmd) + + +def add_build_arguments(parser): + """Base arguments describing how to 'mull'.""" + parser.add_argument('--involucro-path', dest="involucro_path", default=None, + help="Path to involucro (if not set will look in working directory and on PATH).") + parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true", + help="Rebuild package even if already published.") + parser.add_argument('--dry-run', dest='dry_run', action="store_true", + help='Just print commands instead of executing them.') + parser.add_argument('-n', '--namespace', dest='namespace', default="mulled", + help='quay.io namespace.') + parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE, + help='Docker repository target for publication (only quay.io or compat. API is currently supported).') + parser.add_argument('-c', '--channel', dest='channel', default=DEFAULT_CHANNEL, + help='Target conda channel') + parser.add_argument('--extra-channels', dest='extra_channels', default=",".join(DEFAULT_EXTRA_CHANNELS), + help='Dependent conda channels.') + + +def add_single_image_arguments(parser): + parser.add_argument("--name-override", dest="name_override", default=None, + help="Override mulled image name - this is not recommended since metadata will not be detectable from the name of resulting images") + parser.add_argument("--image-build", dest="image_build", default=None, + help="Build a versioned variant of this image.") + + +def target_str_to_targets(targets_raw): + def parse_target(target_str): + if "=" in target_str: + package_name, version = target_str.split("=", 1) + target = build_target(package_name, version) + else: + target = build_target(target_str) + return target + + targets = map(parse_target, targets_raw.split(",")) + return targets + + +def args_to_mull_targets_kwds(args): + kwds = {} + if hasattr(args, "image_build"): + kwds["image_build"] = args.image_build + if hasattr(args, "name_override"): + kwds["name_override"] = args.name_override + if hasattr(args, "namespace"): + kwds["namespace"] = args.namespace + if hasattr(args, "dry_run"): + kwds["dry_run"] = args.dry_run + if hasattr(args, "test"): + kwds["test"] = args.test + if hasattr(args, "channel"): + channels = [args.channel] + if hasattr(args, "extra_channels"): + channels += args.extra_channels.split(",") + kwds["channels"] = channels + if hasattr(args, "command"): + kwds["command"] = args.command + if hasattr(args, "repository_template"): + kwds["repository_template"] = args.repository_template + + kwds["involucro_context"] = context_from_args(args) + + return kwds + + +def main(argv=None): + """Main entry-point for the CLI tool.""" + parser = arg_parser(argv, globals()) + add_build_arguments(parser) + add_single_image_arguments(parser) + parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)') + parser.add_argument('targets', metavar="TARGETS", default=None, help="Build a single container with specific package(s).") + parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of mulled container (leave blank to auto-generate based on packages - recommended).") + parser.add_argument('--test', help='Provide a test command for the container.') + args = parser.parse_args() + targets = target_str_to_targets(args.targets) + mull_targets(targets, **args_to_mull_targets_kwds(args)) + + +__all__ = ["main"] + + +if __name__ == '__main__': + main() diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py new file mode 100644 index 00000000000..b5082224649 --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +"""Build a mulled images for all recent conda recipe updates that don't have existing images. + +Examples: + +Build mulled images for recent bioconda changes with: + + mulled-build-channel build + +Build, test, and publish images with the follow command: + + mulled-build-channel all + +See recent changes that would be built with: + + mulled-build-channel list + +""" + + +import os +import time + +from ._cli import arg_parser +from .mulled_build import ( + add_build_arguments, + args_to_mull_targets_kwds, + build_target, + check_output, + conda_versions, + get_affected_packages, + mull_targets, +) +from .util import quay_versions, version_sorted + + +def _fetch_repo_data(args): + repo_data = args.repo_data + channel = args.channel + if repo_data is None: + repo_data = "%s-repodata.json" % channel + if not os.path.exists(repo_data): + check_output("wget --quiet https://conda.anaconda.org/%s/linux-64/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, repo_data, repo_data)) + return repo_data + + +def _new_versions(quay, conda): + """Calculate the versions that are in conda but not on quay.io.""" + sconda = set(conda) + squay = set(quay) if quay else set() + return sconda - squay # sconda.symmetric_difference(squay) + + +def run_channel(args, build_last_n_versions=1): + """Build list of involucro commands (as shell snippet) to run.""" + pkgs = get_affected_packages(args) + for pkg_name, pkg_tests in pkgs: + repo_data = _fetch_repo_data(args) + c = conda_versions(pkg_name, repo_data) + # only package the most recent N versions + c = version_sorted(c)[:build_last_n_versions] + + if not args.force_rebuild: + time.sleep(1) + q = quay_versions(args.namespace, pkg_name) + versions = _new_versions(q, c) + else: + versions = c + + for tag in versions: + target = build_target(pkg_name, tag=tag) + targets = [target] + mull_targets(targets, test=pkg_tests, **args_to_mull_targets_kwds(args)) + + +def get_pkg_names(args): + """Print package names that would be affected.""" + print('\n'.join([pkg_name for pkg_name, pkg_tests in get_affected_packages(args)])) + + +def add_channel_arguments(parser): + """Add arguments only used if running mulled over a whole conda channel.""" + parser.add_argument('--repo-data', dest='repo_data', default=None, + help='Published repository data (will be fetched from --channel if not available and written). Defaults to [channel_name]-repodata.json.') + parser.add_argument('--diff-hours', dest='diff_hours', default="25", + help='If finding all recently changed recipes, use this number of hours.') + parser.add_argument('--recipes-dir', dest="recipes_dir", default="./bioconda-recipes") + + +def main(argv=None): + """Main entry-point for the CLI tool.""" + parser = arg_parser(argv, globals()) + add_channel_arguments(parser) + add_build_arguments(parser) + parser.add_argument('command', metavar='COMMAND', help='Command (list, build-and-test, build, all)') + parser.add_argument('--targets', dest="targets", default=None, help="Build a single container with specific package(s).") + parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of a single container (leave blank to auto-generate based on packages).") + args = parser.parse_args() + if args.command == "list": + get_pkg_names(args) + else: + run_channel(args) + + +__all__ = ["main"] + + +if __name__ == '__main__': + main() diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_files.py b/lib/galaxy/tools/deps/mulled/mulled_build_files.py new file mode 100644 index 00000000000..0fd065c504d --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/mulled_build_files.py @@ -0,0 +1,80 @@ +"""Build all composite mulled recipes discovered in TSV files. + +Use mulled-build-channel to build images for single recipes for a whole conda +channel. This script instead builds images for combinations of recipes. This +script can be given a single TSV file or a directory of TSV files to process. + +Examples: + +Build all recipes discovered in tsv files in a single directory. + + mulled-build-files build + +""" + +import collections +import glob +import os + +from ._cli import arg_parser + +from .mulled_build import ( + add_build_arguments, + args_to_mull_targets_kwds, + mull_targets, + target_str_to_targets, +) + + +def main(argv=None): + """Main entry-point for the CLI tool.""" + parser = arg_parser(argv, globals()) + add_build_arguments(parser) + parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)') + parser.add_argument('files', metavar="FILES", default=".", + help="Path to directory (or single file) of TSV files describing composite recipes.") + args = parser.parse_args() + for targets in generate_targets(args.files): + mull_targets(targets, **args_to_mull_targets_kwds(args)) + + +def generate_targets(target_source): + """Generate all targets from TSV files in specified file or directory.""" + target_source = os.path.abspath(target_source) + if os.path.isdir(target_source): + target_source_files = glob.glob(target_source + "/*.tsv") + else: + target_source_files = [target_source] + + for target_source_file in target_source_files: + with open(target_source_file, "r") as f: + for line in f.readlines(): + if line: + line = line.strip() + + if not line or line.startswith("#"): + continue + + yield line_to_targets(line) + + +def line_to_targets(line_str): + line = _parse_line(line_str) + return target_str_to_targets(line) + + +_Line = collections.namedtuple("_Line", ["targets", "image_build", "name_override"]) + + +def _parse_line(line_str): + line_parts = line_str.split(" ") + assert len(line_parts) < 3, "Too many fields in line [%s], expect at most 3 - targets, image build number, and name override." % line_str + line_parts += [None] * (3 - len(line_parts)) + return _Line(*line_parts) + + +__all__ = ["main"] + + +if __name__ == '__main__': + main() diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_tool.py b/lib/galaxy/tools/deps/mulled/mulled_build_tool.py new file mode 100644 index 00000000000..7281c7940ae --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/mulled_build_tool.py @@ -0,0 +1,52 @@ +#!/usr/bin/env python +"""Build a mulled images for a tool source (Galaxy or CWL tool). + +Examples: + +Build mulled images for requirements defined in a tool: + + mulled-build-tool build path/to/tool_file.xml + +""" + +from galaxy.tools.parser import get_tool_source + +from ._cli import arg_parser +from .mulled_build import ( + add_build_arguments, + add_single_image_arguments, + args_to_mull_targets_kwds, + build_target, + mull_targets, +) + + +def main(argv=None): + """Main entry-point for the CLI tool.""" + parser = arg_parser(argv, globals()) + add_build_arguments(parser) + add_single_image_arguments(parser) + parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)') + parser.add_argument('tool', metavar="TOOL", default=None, help="Path to tool to build mulled image for.") + args = parser.parse_args() + tool_source = get_tool_source(args.tool) + requirements, _ = tool_source.parse_requirements_and_containers() + targets = requirements_to_mulled_targets(requirements) + mull_targets(targets, **args_to_mull_targets_kwds(args)) + + +def requirements_to_mulled_targets(requirements): + """Convert Galaxy's representation of requirements into mulled Target objects. + + Only package requirements are retained. + """ + package_requirements = filter(lambda r: r.type == "package", requirements) + targets = map(lambda r: build_target(r.name, r.version), package_requirements) + return targets + + +__all__ = ["main", "requirements_to_mulled_targets"] + + +if __name__ == '__main__': + main() diff --git a/lib/galaxy/tools/deps/mulled/mulled_search.py b/lib/galaxy/tools/deps/mulled/mulled_search.py new file mode 100644 index 00000000000..724956a1335 --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/mulled_search.py @@ -0,0 +1,131 @@ +#!/usr/bin/env python + +import argparse +import json +import sys +import tempfile + +try: + import requests +except ImportError: + requests = None + +try: + from whoosh.fields import Schema + from whoosh.fields import TEXT + from whoosh.fields import STORED + from whoosh.index import create_in + from whoosh.qparser import QueryParser +except ImportError: + Schema = TEXT = STORED = create_in = QueryParser = None + +QUAY_API_URL = 'https://quay.io/api/v1/repository' + + +class QuaySearch(): + """ + Tool to search within a quay organization for a given software name. + """ + def __init__(self, organization): + self.index = None + self.organization = organization + + def build_index(self): + """ + Create an index to quickly examine the repositories of a given quay.io organization. + """ + # download all information about the repositories from the + # given organization in self.organization + + parameters = {'public': 'true', 'namespace': self.organization} + r = requests.get(QUAY_API_URL, headers={'Accept-encoding': 'gzip'}, params=parameters, + timeout=12) + + tmp_dir = tempfile.mkdtemp() + schema = Schema(title=TEXT(stored=True), content=STORED) + self.index = create_in(tmp_dir, schema) + + json_decoder = json.JSONDecoder() + decoded_request = json_decoder.decode(r.text) + writer = self.index.writer() + for repository in decoded_request['repositories']: + writer.add_document(title=repository['name'], content=repository['description']) + writer.commit() + + def search_repository(self, search_string, non_strict): + """ + Search Docker containers on quay.io. + Results are displayed with all available versions, + including the complete image name. + """ + # with statement closes searcher after usage. + with self.index.searcher() as searcher: + search_string = "*%s*" % search_string + query = QueryParser("title", self.index.schema).parse(search_string) + results = searcher.search(query) + if non_strict: + # look for spelling errors and use suggestions as a search term too + corrector = searcher.corrector("title") + suggestions = corrector.suggest(search_string, limit=2) + + # get all repositories with suggested keywords + for suggestion in suggestions: + search_string = "*%s*" % suggestion + query = QueryParser("title", self.index.schema).parse(search_string) + results_tmp = searcher.search(query) + results.extend(results_tmp) + + sys.stdout.write("The query \033[1m %s \033[0m resulted in %s result(s).\n" % (search_string, len(results))) + + if non_strict: + sys.stdout.write('The search was relaxed and the following search terms were searched: ') + sys.stdout.write('\033[1m %s \033[0m\n' % ', '.join(suggestions)) + + out = list() + for result in results: + title = result['title'] + for version in self.get_additional_repository_information(title): + row = [title] + row.append(version) + out.append(row) + if out: + col_width = max(len(word) for row in out for word in row) + 2 # padding + for row in out: + name = row[0] + version = row[1] + sys.stdout.write("".join(word.ljust(col_width) for word in row) + "docker pull quay.io/%s/%s:%s\n" % (self.organization, name, version)) + else: + sys.stdout.write("No results found for %s in quay.io/%s.\n" % (search_string, self.organization)) + + def get_additional_repository_information(self, repository_string): + """ + Function downloads additional information from quay.io to + get the tag-field which includes the version number. + """ + url = "%s/%s/%s" % (QUAY_API_URL, self.organization, repository_string) + r = requests.get(url, headers={'Accept-encoding': 'gzip'}, timeout=12) + + json_decoder = json.JSONDecoder() + decoded_request = json_decoder.decode(r.text) + return decoded_request['tags'] + + +def main(argv=None): + parser = argparse.ArgumentParser(description='Searches in a given quay organization for a repository') + parser.add_argument('-o', '--organization', dest='organization_string', default="mulled", + help='Change organization. Default is mulled.') + parser.add_argument('--non-strict', dest='non_strict', action="store_true", + help='Autocorrection of typos activated. Lists more results but can be confusing.\ + For too many queries quay.io blocks the request and the results can be incomplete.') + parser.add_argument('-s', '--search', required=True, + help='The name of the tool you want to search for.') + args = parser.parse_args() + + quay = QuaySearch(args.organization_string) + quay.build_index() + + quay.search_repository(args.search, args.non_strict) + + +if __name__ == "__main__": + main() diff --git a/lib/galaxy/tools/deps/mulled/util.py b/lib/galaxy/tools/deps/mulled/util.py new file mode 100644 index 00000000000..4fc5312afbe --- /dev/null +++ b/lib/galaxy/tools/deps/mulled/util.py @@ -0,0 +1,116 @@ +"""Utilities for working with mulled abstractions outside the mulled package.""" +from __future__ import print_function + +import collections +import hashlib + +from distutils.version import LooseVersion + +try: + import requests +except ImportError: + requests = None + + +def quay_versions(namespace, pkg_name): + """Get all version tags for a Docker image stored on quay.io for supplied package name.""" + if requests is None: + raise Exception("requets library is unavailable, functionality not available.") + + assert namespace is not None + assert pkg_name is not None + url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name) + response = requests.get(url, timeout=None) + data = response.json() + if 'error_type' in data and data['error_type'] == "invalid_token": + return [] + + if 'tags' not in data: + raise Exception("Unexpected response from quay.io - not tags description found [%s]" % data) + + return [tag for tag in data['tags'] if tag != 'latest'] + + +def mulled_tags_for(namespace, image): + """Fetch remote tags available for supplied image name. + + The result will be sorted so newest tags are first. + """ + tags = quay_versions(namespace, image) + tags = version_sorted(tags) + return tags + + +def split_tag(tag): + """Split mulled image name into conda version and conda build.""" + version = tag.split('--', 1)[0] + build = tag.split('--', 1)[1] + return version, build + + +def version_sorted(elements): + """Sort iterable based on loose description of "version" from newest to oldest.""" + return sorted(elements, key=LooseVersion, reverse=True) + + +Target = collections.namedtuple("Target", ["package_name", "version", "build"]) + + +def build_target(package_name, version=None, build=None, tag=None): + """Use supplied arguments to build a :class:`Target` object.""" + if tag is not None: + assert version is None + assert build is None + version, build = split_tag(tag) + + return Target(package_name, version, build) + + +def conda_build_target_str(target): + rval = target.package_name + if target.version: + rval += "=%s" % target.version + + if target.build: + rval += "=%s" % target.build + + return rval + + +def image_name(targets, image_build=None, name_override=None): + if name_override is not None: + print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.") + return name_override + + if len(targets) == 1: + target = targets[0] + suffix = "" + if target.version is not None: + if image_build is not None: + print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.") + suffix = image_build + else: + suffix += ":%s" % target.version + build = target.build + if build is not None: + suffix += "--%s" % build + return "%s%s" % (target.package_name, suffix) + else: + targets_order = sorted(targets, key=lambda t: t.package_name) + requirements_buffer = "\n".join(map(conda_build_target_str, targets_order)) + m = hashlib.sha1() + m.update(requirements_buffer) + suffix = "" if not image_build else ":%s" % image_build + return "mulled-v1-%s%s" % (m.hexdigest(), suffix) + + +__all__ = [ + "build_target", + "conda_build_target_str", + "image_name", + "mulled_tags_for", + "quay_versions", + "split_tag", + "Target", + "version_sorted", +] diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py index 2ea5abeaa4f..6165fdb6af8 100644 --- a/lib/galaxy/tools/deps/resolvers/__init__.py +++ b/lib/galaxy/tools/deps/resolvers/__init__.py @@ -1,3 +1,4 @@ +"""The module defines the abstract interface for dealing tool dependency resolution plugins.""" from abc import ( ABCMeta, abstractmethod, @@ -10,6 +11,8 @@ from ..requirements import ToolRequirement class DependencyResolver(Dictifiable, object): + """Abstract description of a technique for resolving container images for tool execution.""" + # Keys for dictification. dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies'] # A "simple" dependency is one that does not depend on the the tool @@ -23,13 +26,15 @@ class DependencyResolver(Dictifiable, object): @abstractmethod def resolve( self, name, version, type, **kwds ): - """ - Given inputs describing dependency in the abstract, yield tuple of - (script, bin, version). Here script is the env.sh file to source - before running a job, if that is not found the bin directory will be - appended to the path (if it is not None). Finally, version is the - resolved tool dependency version (which may differ from requested - version for instance if the request version is 'default'.) + """Given inputs describing dependency in the abstract yield a Dependency object. + + The Dependency object describes various attributes (script, bin, + version) used to build scripts with the dependency availble. Here + script is the env.sh file to source before running a job, if that is + not found the bin directory will be appended to the path (if it is + not ``None``). Finally, version is the resolved tool dependency + version (which may differ from requested version for instance if the + request version is 'default'.) """ def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds): diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py index 3b3d85d16f9..f16734a268e 100644 --- a/lib/galaxy/tools/deps/resolvers/conda.py +++ b/lib/galaxy/tools/deps/resolvers/conda.py @@ -6,10 +6,8 @@ incompatible changes coming. import logging import os -from galaxy.util.filelock import ( - FileLock, - FileLockException -) +import galaxy.tools.deps.installable + from ..conda_util import ( build_isolated_environment, cleanup_failed_install, @@ -32,7 +30,7 @@ from ..resolvers import ( DEFAULT_BASE_PATH_DIRECTORY = "_conda" DEFAULT_CONDARC_OVERRIDE = "_condarc" -DEFAULT_ENSURE_CHANNELS = "r,bioconda,iuc" +DEFAULT_ENSURE_CHANNELS = "conda-forge,r,bioconda,iuc" log = logging.getLogger(__name__) @@ -95,44 +93,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In copy_dependencies = _string_as_bool(get_option("copy_dependencies")) self.auto_init = _string_as_bool(get_option("auto_init")) self.conda_context = conda_context - self.ensure_conda_installed() + self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init) self.auto_install = auto_install self.copy_dependencies = copy_dependencies self.verbose_install_check = verbose_install_check - def ensure_conda_installed(self): - """ - Make sure that conda is installed, and if conda can't be installed, mark resolver as disabled. - We acquire a lock, so that multiple handlers do not attempt to install conda simultaneously. - """ - target_path = self.conda_prefix_parent - - def _check(): - if not self.conda_context.is_conda_installed(): - if self.auto_init: - if self.conda_context.can_install_conda(): - if install_conda(self.conda_context): - self.disabled = True - log.warning("Conda installation requested and failed.") - else: - self.disabled = True - else: - self.disabled = True - log.warning("Conda not installed and auto-installation disabled.") - else: - self.disabled = False - - if not os.path.exists(target_path): - os.mkdir(target_path) - try: - if self.auto_init and os.access(target_path, os.W_OK): - with FileLock(os.path.join(target_path, 'conda')): - _check() - else: - _check() - except FileLockException: - self.ensure_conda_installed() - def resolve(self, name, version, type, **kwds): # Check for conda just not being there, this way we can enable # conda by default and just do nothing in not configured. diff --git a/test/functional/tools/mulled_example_multi_1.xml b/test/functional/tools/mulled_example_multi_1.xml new file mode 100644 index 00000000000..1f025598c04 --- /dev/null +++ b/test/functional/tools/mulled_example_multi_1.xml @@ -0,0 +1,18 @@ + + $out_file1 ; + echo "Moo" >> $out_file1 ; + samtools >> $out_file1 2>&1 ; + echo "Cow" >> $out_file1 ; + ]]> + + samtools + bedtools + + + + + + + + diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index d57356e25d9..90d64f86c37 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -111,6 +111,8 @@ + +