Implement beta support for dynamically discovering mulled containers for dependencies.

Contains all mulled fixes and enhancements through 16.10.7 including several bug fixes from @bgruening.

config/galaxy.ini.sample

@@ -228,6 +228,37 @@ paste.app_factory = galaxy.web.buildapp:app_factory
 # than the watchdog default.
 #watch_tools = False
 
+# Enable Galaxy to fetch Docker containers registered with quay.io generated
+# from tool requirements resolved through conda. These containers (when
+# available) have been generated using mulled - https://github.com/mulled.
+# These containers are highly beta and availablity will vary by tool.
+# This option will additionally only be used for job destinations with
+# Docker enabled.
+#enable_beta_mulled_containers = False
+
+# Container resolvers configuration (beta). Setup a file describing container
+# resolvers to use when discovering containers for Galaxy. If this is set to
+# None, the default containers loaded is determined by
+# enable_beta_mulled_containers.
+#containers_resolvers_config_file = None
+
+# Container resolvers configuration (beta). Setup a file describing container
+# resolvers to use when discovering containers for Galaxy. If this is set to
+# None, the default containers loaded is determined by
+# enable_beta_mulled_containers.
+#containers_resolvers_config_file = None
+
+# involucro is a tool used to build Docker containers for tools from Conda
+# dependencies referenced in tools as `requirement`s. The following path is
+# the location of involucro on the Galaxy host. This is ignored if the relevant
+# container resolver isn't enabled, and will install on demand unless
+# involucro_auto_init is set to False.
+#involucro_path = database/dependencies/involucro
+
+# Install involucro as needed to build Docker containers for tools. Ignored if
+# relevant container resolver is not used.
+#involucro_auto_init = True
+
 # Enable automatic polling of relative tool sheds to see if any updates
 # are available for installed repositories.  Ideally only one Galaxy
 # server process should be able to check for repository updates.  The

lib/galaxy/config.py

@@ -226,6 +226,7 @@ class Configuration( object ):
         self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2))
         self.tool_submission_burst_threads = int( kwargs.get( 'tool_submission_burst_threads', '1' ) )
         self.tool_submission_burst_at = int( kwargs.get( 'tool_submission_burst_at', '10' ) )
+
         # Enable new interface for API installations from TS.
         # Admin menu will list both if enabled.
         self.enable_beta_ts_api_install = string_as_bool( kwargs.get( 'enable_beta_ts_api_install', 'False' ) )
@@ -323,6 +324,19 @@ class Configuration( object ):
         else:
             self.tool_dependency_dir = None
             self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file)
+
+        self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) )
+        containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None )
+        if containers_resolvers_config_file:
+            containers_resolvers_config_file = resolve_path(containers_resolvers_config_file, self.root)
+        self.containers_resolvers_config_file = containers_resolvers_config_file
+
+        involucro_path = kwargs.get('involucro_path', None)
+        if involucro_path is None:
+            involucro_path = os.path.join(tool_dependency_dir, "involucro")
+        self.involucro_path = resolve_path(involucro_path, self.root)
+        self.involucro_auto_init = string_as_bool(kwargs.get( 'involucro_auto_init', True))
+
         # Configuration options for taking advantage of nginx features
         self.upstream_gzip = string_as_bool( kwargs.get( 'upstream_gzip', False ) )
         self.apache_xsendfile = string_as_bool( kwargs.get( 'apache_xsendfile', False ) )
@@ -846,7 +860,11 @@ class ConfiguresGalaxyMixin:
             default_file_path=file_path,
             outputs_to_working_directory=self.config.outputs_to_working_directory,
             container_image_cache_path=self.config.container_image_cache_path,
-            library_import_dir=self.config.library_import_dir
+            library_import_dir=self.config.library_import_dir,
+            enable_beta_mulled_containers=self.config.enable_beta_mulled_containers,
+            containers_resolvers_config_file=self.config.containers_resolvers_config_file,
+            involucro_path=self.config.involucro_path,
+            involucro_auto_init=self.config.involucro_auto_init,
         )
         self.container_finder = containers.ContainerFinder(app_info)
 

lib/galaxy/tools/deps/__init__.py

@@ -8,7 +8,7 @@ import os.path
 from galaxy.util import plugin_config
 
 from .resolvers import NullDependency
-from .resolvers.conda import CondaDependencyResolver
+from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
 from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
 from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
 
@@ -21,7 +21,7 @@ EXTRA_CONFIG_KWDS = {
     'conda_prefix': None,
     'conda_exec': None,
     'conda_debug': None,
-    'conda_ensure_channels': 'r,bioconda,iuc',
+    'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS,
     'conda_auto_install': False,
     'conda_auto_init': False,
     'conda_copy_dependencies': False,

lib/galaxy/tools/deps/conda_compat.py

@@ -0,0 +1,112 @@
+"""Compat. layer with conda_build/verify if Galaxy/galaxy-lib not installed through conda.
+
+In general there are utilities available for Conda building and parsing that are high-quality
+and should be utilized when available but that are only available in conda channels and not in
+PyPI. This module serves as a PyPI capable interface to these utilities.
+"""
+import collections
+import os
+import yaml
+
+
+try:
+    from conda_build.metadata import MetaData
+except ImportError:
+    MetaData = None
+
+try:
+    from anaconda_verify.recipe import render_jinja2, parse
+except ImportError:
+    render_jinja2 = None
+    parse = None
+
+
+class _Memoized(object):
+
+    def __init__(self, func):
+        self.func = func
+        self.cache = {}
+
+    def __call__(self, *args):
+        if not isinstance(args, collections.Hashable):
+            # uncacheable. a list, for instance.
+            # better to not cache than blow up.
+            return self.func(*args)
+        if args in self.cache:
+            return self.cache[args]
+        else:
+            value = self.func(*args)
+            self.cache[args] = value
+            return value
+
+
+def _parse(data, cfg):
+    """Parse metadata YAML."""
+    assert cfg is None, "Conda utilities for evaluating cfg are not available."
+    return dict(yamlize(data))
+
+
+def _render_jinja2(recipe_dir):
+    """Evaluate Conda recipe as a jinja template."""
+    try:
+        import jinja2
+    except ImportError:
+        raise Exception("Failed to import jinja2 for evaluating Conda recipe templates.")
+
+    loaders = [jinja2.FileSystemLoader(recipe_dir)]
+    env = jinja2.Environment(loader=jinja2.ChoiceLoader(loaders))
+    template = env.get_or_select_template('meta.yaml')
+    return template.render(environment=env)
+
+
+@_Memoized
+def yamlize(data):
+    res = yaml.load(data)
+    # ensure the result is a dict
+    if res is None:
+        res = {}
+    return res
+
+
+if render_jinja2 is None:
+    render_jinja2 = _render_jinja2
+
+if parse is None:
+    parse = _parse
+
+
+def raw_metadata(recipe_dir):
+    """Evaluate Conda template if needed and return raw metadata for supplied recipe directory."""
+    meta_path = os.path.join(recipe_dir, 'meta.yaml')
+    with open(meta_path, 'rb') as fi:
+        data = fi.read()
+        if b'{{' in data:
+            data = render_jinja2(recipe_dir)
+    meta = parse(data, None)
+    return meta
+
+
+class _MetaData(object):
+
+    def __init__(self, input_dir):
+        self.meta = raw_metadata(input_dir)
+
+    def get_value(self, field, default=None):
+        """Get nested field value or supplied default is not present."""
+        section, key = field.split('/')
+        submeta = self.meta.get(section)
+        if submeta is None:
+            submeta = {}
+        res = submeta.get(key)
+        if res is None:
+            res = default
+        return res
+
+
+if MetaData is None:
+    MetaData = _MetaData
+
+__all__ = [
+    "MetaData",
+    "raw_metadata",
+]

lib/galaxy/tools/deps/conda_util.py

@@ -7,12 +7,14 @@ import re
 import shutil
 import tempfile
 
+from distutils.version import LooseVersion
 from sys import platform as _platform
 
 import six
 import yaml
 
 from ..deps import commands
+from ..deps import installable
 
 log = logging.getLogger(__name__)
 
@@ -47,7 +49,8 @@ def find_conda_prefix(conda_prefix=None):
     return conda_prefix
 
 
-class CondaContext(object):
+class CondaContext(installable.InstallableContext):
+    installable_description = "Conda"
 
     def __init__(self, conda_prefix=None, conda_exec=None,
                  shell_exec=None, debug=False, ensure_channels='',
@@ -230,6 +233,16 @@ class CondaContext(object):
     def activate(self):
         return self._bin("activate")
 
+    def is_installed(self):
+        return self.is_conda_installed()
+
+    def can_install(self):
+        return self.can_install_conda()
+
+    @property
+    def parent_path(self):
+        return os.path.dirname(os.path.abspath(self.conda_prefix))
+
     def _bin(self, name):
         return os.path.join(self.conda_prefix, "bin", name)
 
@@ -273,6 +286,8 @@ class CondaTarget(object):
 
         return "CondaTarget[%s]" % attributes
 
+    __repr__ = __str__
+
     @property
     def package_specifier(self):
         """ Return a package specifier as consumed by conda install/create.
@@ -293,6 +308,17 @@ class CondaTarget(object):
         else:
             return "__%s@_uv_" % (self.package)
 
+    def __hash__(self):
+        return hash((self.package, self.version, self.channel))
+
+    def __eq__(self, other):
+        if isinstance(other, self.__class__):
+            return (self.package, self.version, self.channel) == (other.package, other.version, other.channel)
+        return False
+
+    def __ne__(self, other):
+        return not(self == other)
+
 
 def hash_conda_packages(conda_packages, conda_target=None):
     """ Produce a unique hash on supplied packages.
@@ -339,23 +365,55 @@ def cleanup_failed_install(conda_target, conda_context=None):
         conda_context.exec_remove([conda_target.install_environment])
 
 
-def is_target_available(conda_target, conda_context=None):
-    """ Checks if a specified target is available for installation.
-        If the package name exists return "True". If in addition the version matches exactly return "exact".
-        Otherwise return False.
+def best_search_result(conda_target, conda_context=None, channels_override=None):
+    """Find best "conda search" result for specified target.
+
+    Return ``None`` if no results match.
     """
     conda_context = _ensure_conda_context(conda_context)
-    conda_context.ensure_channels_configured()
-    search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json", conda_target.package]
+    if not channels_override:
+        conda_context.ensure_channels_configured()
+
+    search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"]
+    if channels_override:
+        search_cmd.append("--override-channels")
+        for channel in channels_override:
+            search_cmd.extend(["--channel", channel])
+    search_cmd.append(conda_target.package)
     res = commands.execute(search_cmd)
     hits = json.loads(res).get(conda_target.package, [])
+    hits = sorted(hits, key=lambda hit: LooseVersion(hit['version']), reverse=True)
 
-    if len(hits) > 0:
-        if conda_target.version:
-            for hit in hits:
-                if hit['version'] == conda_target.version:
-                    return 'exact'
-        return True
+    if len(hits) == 0:
+        return (None, None)
+
+    best_result = (hits[0], False)
+
+    for hit in hits:
+        if is_search_hit_exact(conda_target, hit):
+            best_result = (hit, True)
+            break
+
+    return best_result
+
+
+def is_search_hit_exact(conda_target, search_hit):
+    target_version = conda_target.version
+    # It'd be nice to make request verson of 1.0 match available
+    # version of 1.0.3 or something like that.
+    return not target_version or search_hit['version'] == target_version
+
+
+def is_target_available(conda_target, conda_context=None, channels_override=None):
+    """Check if a specified target is available for installation.
+
+    If the package name exists return ``True`` (the ``bool``). If in addition
+    the version matches exactly return "exact" (a string). Otherwise return
+    ``False``.
+    """
+    (best_hit, exact) = best_search_result(conda_target, conda_context, channels_override)
+    if best_hit:
+        return 'exact' if exact else True
     else:
         return False
 

lib/galaxy/tools/deps/container_resolvers/__init__.py

@@ -0,0 +1,50 @@
+"""The module defines the abstract interface for resolving container images for tool execution."""
+from abc import (
+    ABCMeta,
+    abstractmethod,
+    abstractproperty,
+)
+
+from galaxy.util.dictifiable import Dictifiable
+
+
+class ContainerResolver(Dictifiable, object):
+    """Description of a technique for resolving container images for tool execution."""
+
+    # Keys for dictification.
+    dict_collection_visible_keys = ['resolver_type']
+
+    __metaclass__ = ABCMeta
+
+    def __init__(self, app_info=None, **kwds):
+        """Default initializer for ``ContainerResolver`` subclasses."""
+        self.app_info = app_info
+        self.resolver_kwds = kwds
+
+    def _get_config_option(self, key, default=None, config_prefix=None, **kwds):
+        """Look in resolver-specific settings for option and then fallback to
+        global settings.
+        """
+        global_key = "%s_%s" % (config_prefix, key)
+        if key in kwds:
+            return kwds.get(key)
+        elif self.app_info and hasattr(self.app_info, global_key):
+            return getattr(self.app_info, global_key)
+        else:
+            return default
+
+    @abstractmethod
+    def resolve(self, tool_info):
+        """Find a container matching all supplied requirements for tool.
+
+        The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description
+        of the tool and its requirements.
+        """
+
+    @abstractproperty
+    def resolver_type(self):
+        """Short label for the type of container resolution."""
+
+    def _container_type_enabled(self, container_description, enabled_container_types):
+        """Return a boolean indicating if the specified container type is enabled."""
+        return container_description.type in enabled_container_types

lib/galaxy/tools/deps/container_resolvers/explicit.py

@@ -0,0 +1,26 @@
+"""This module describes the :class:`ExplicitContainerResolver` ContainerResolver plugin."""
+import logging
+
+from ..container_resolvers import (
+    ContainerResolver,
+)
+
+log = logging.getLogger(__name__)
+
+
+class ExplicitContainerResolver(ContainerResolver):
+    """Find explicit containers referenced in the tool description (e.g. tool XML file) if present."""
+
+    resolver_type = "explicit"
+
+    def resolve(self, enabled_container_types, tool_info):
+        """Find a container explicitly mentioned in tool description.
+
+        This ignores the tool requirements and assumes the tool author crafted
+        a correct container.
+        """
+        for container_description in tool_info.container_descriptions:
+            if self._container_type_enabled(container_description):
+                return True
+
+        return False

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -0,0 +1,207 @@
+"""This module describes the :class:`MulledContainerResolver` ContainerResolver plugin."""
+
+import collections
+import logging
+
+from ..container_resolvers import (
+    ContainerResolver,
+)
+from ..docker_util import build_docker_images_command
+from ..mulled.mulled_build import (
+    check_output,
+    DEFAULT_CHANNELS,
+    ensure_installed,
+    InvolucroContext,
+    mull_targets,
+)
+from ..mulled.mulled_build_tool import requirements_to_mulled_targets
+from ..mulled.util import (
+    image_name,
+    mulled_tags_for,
+    split_tag,
+)
+from ..requirements import ContainerDescription
+
+log = logging.getLogger(__name__)
+
+
+CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"])
+CachedMulledImageMultiTarget = collections.namedtuple("CachedMulledImageMultiTarget", ["hash", "image_identifier"])
+
+CachedMulledImageSingleTarget.multi_target = False
+CachedMulledImageMultiTarget.multi_target = True
+
+
+def list_cached_mulled_images(namespace=None):
+    command = build_docker_images_command(truncate=True, sudo_docker=False)
+    command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
+    images_and_versions = check_output(command)
+    name_filter = get_filter(namespace)
+
+    def output_line_to_image(line):
+        image_name, version = line.split(" ", 1)
+        identifier = "%s:%s" % (image_name, version)
+        url, namespace, package_description = image_name.split("/")
+
+        if package_description.startswith("mulled-v1-"):
+            hash = package_description
+            image = CachedMulledImageMultiTarget(hash, identifier)
+        else:
+            build = None
+            if not version or version == "latest":
+                version = None
+
+            if version and "--" in version:
+                version, build = split_tag(version)
+
+            image = CachedMulledImageSingleTarget(image_name, version, build, identifier)
+
+        return image
+
+    return map(output_line_to_image, filter(name_filter, images_and_versions.splitlines()))
+
+
+def get_filter(namespace):
+    prefix = "quay.io/" if namespace is None else "quay.io/%s" % namespace
+    return lambda name: name.startswith(prefix) and name.count("/") == 2
+
+
+def cached_container_description(targets, namespace):
+    if len(targets) == 0:
+        return None
+
+    cached_images = list_cached_mulled_images(namespace)
+    image = None
+    if len(targets) == 1:
+        target = targets[0]
+        for cached_image in cached_images:
+            if cached_image.multi_target:
+                continue
+            if not cached_image.package_name == target.package_name:
+                continue
+            if not target.version or target.version == cached_image.version:
+                image = cached_image
+                break
+    else:
+        name = image_name(targets)
+        for cached_image in cached_images:
+            if not cached_image.multi_target:
+                continue
+
+            if name == cached_image.hash:
+                image = cached_image
+                break
+
+    container = None
+    if image:
+        container = ContainerDescription(
+            image.image_identifier,
+            type="docker",
+        )
+
+    return container
+
+
+class CachedMulledContainerResolver(ContainerResolver):
+
+    resolver_type = "cached_mulled"
+
+    def __init__(self, app_info=None, namespace=None):
+        super(CachedMulledContainerResolver, self).__init__(app_info)
+        self.namespace = namespace
+
+    def resolve(self, enabled_container_types, tool_info):
+        targets = mulled_targets(tool_info)
+        return cached_container_description(targets, self.namespace)
+
+
+class MulledContainerResolver(ContainerResolver):
+    """Look for mulled images matching tool dependencies."""
+
+    resolver_type = "mulled"
+
+    def __init__(self, app_info=None, namespace="mulled"):
+        super(MulledContainerResolver, self).__init__(app_info)
+        self.namespace = namespace
+
+    def resolve(self, enabled_container_types, tool_info):
+        targets = mulled_targets(tool_info)
+        if len(targets) == 0:
+            return None
+
+        name = None
+
+        if len(targets) == 1:
+            target = targets[0]
+            target_version = target.version
+            tags = mulled_tags_for(self.namespace, target.package_name)
+
+            if not tags:
+                return None
+
+            if target_version:
+                for tag in tags:
+                    version, build = split_tag(tag)
+                    if version == target_version:
+                        name = "%s:%s--%s" % (target.package_name, version, build)
+                        break
+            else:
+                version, build = split_tag(tags[0])
+                name = "%s:%s--%s" % (target.package_name, version, build)
+        else:
+            base_image_name = image_name(targets)
+            tags = mulled_tags_for(self.namespace, base_image_name)
+            if tags:
+                name = "%s:%s" % (base_image_name, tags[0])
+
+        if name:
+            return ContainerDescription(
+                "quay.io/%s/%s" % (self.namespace, name),
+                type="docker",
+            )
+
+
+class BuildMulledContainerResolver(ContainerResolver):
+    """Look for mulled images matching tool dependencies."""
+
+    resolver_type = "build_mulled"
+
+    def __init__(self, app_info=None, namespace="local", **kwds):
+        super(BuildMulledContainerResolver, self).__init__(app_info)
+        self._involucro_context_kwds = {
+            'involucro_bin': self._get_config_option("involucro_path", None)
+        }
+        self.namespace = namespace
+        self._mulled_kwds = {
+            'namespace': namespace,
+            'channels': self._get_config_option("channels", DEFAULT_CHANNELS, prefix="mulled"),
+        }
+        self.auto_init = self._get_config_option("auto_init", DEFAULT_CHANNELS, prefix="involucro")
+
+    def resolve(self, enabled_container_types, tool_info):
+        targets = mulled_targets(tool_info)
+        if len(targets) == 0:
+            return None
+
+        mull_targets(
+            targets,
+            involucro_context=self._get_involucro_context(),
+            **self._mulled_kwds
+        )
+        return cached_container_description(targets, self.namespace)
+
+    def _get_involucro_context(self):
+        involucro_context = InvolucroContext(**self._involucro_context_kwds)
+        self.enabled = ensure_installed(involucro_context, self.auto_init)
+        return involucro_context
+
+
+def mulled_targets(tool_info):
+    return requirements_to_mulled_targets(tool_info.requirements)
+
+
+__all__ = [
+    "CachedMulledContainerResolver",
+    "MulledContainerResolver",
+    "BuildMulledContainerResolver",
+]

lib/galaxy/tools/deps/containers.py

@@ -10,14 +10,23 @@ from abc import (
 import six
 
 from galaxy.util import asbool
+from galaxy.util import plugin_config
 
+from .container_resolvers.explicit import ExplicitContainerResolver
+from .container_resolvers.mulled import (
+    BuildMulledContainerResolver,
+    CachedMulledContainerResolver,
+    MulledContainerResolver,
+)
 from .requirements import ContainerDescription
 from .requirements import DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, DEFAULT_CONTAINER_SHELL
 from ..deps import docker_util
 
 log = logging.getLogger(__name__)
 
-DEFAULT_CONTAINER_TYPE = "docker"
+DOCKER_CONTAINER_TYPE = "docker"
+DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE
+ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE]
 
 LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = '''
 python << EOF
@@ -47,9 +56,18 @@ class ContainerFinder(object):
 
     def __init__(self, app_info):
         self.app_info = app_info
-        self.container_registry = ContainerRegistry()
+        self.container_registry = ContainerRegistry(app_info)
+
+    def __enabled_container_types(self, destination_info):
+        return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)]
 
     def find_container(self, tool_info, destination_info, job_info):
+        enabled_container_types = self.__enabled_container_types(destination_info)
+
+        # Short-cut everything else and just skip checks if no container type is enabled.
+        if not enabled_container_types:
+            return NULL_CONTAINER
+
         def __destination_container(container_description=None, container_id=None, container_type=None):
             if container_description:
                 container_id = container_description.identifier
@@ -82,23 +100,10 @@ class ContainerFinder(object):
                     return container
 
         # Otherwise lets see if we can find container for the tool.
-
-        # Exact matches first from explicitly listed containers in tools...
-        for container_description in tool_info.container_descriptions:
-            container = __destination_container(container_description)
-            if container:
-                return container
-
-        # Implement vague concept of looping through all containers
-        # matching requirements. Exact details need to be worked through
-        # but hopefully the idea that it sits below find_container somewhere
-        # external components to this module don't need to worry about it
-        # is good enough.
-        container_descriptions = self.container_registry.container_descriptions_for_requirements(tool_info.requirements)
-        for container_description in container_descriptions:
-            container = __destination_container(container_description)
-            if container:
-                return container
+        container_description = self.container_registry.find_best_container_description(enabled_container_types, tool_info)
+        container = __destination_container(container_description)
+        if container:
+            return container
 
         # If we still don't have a container, check to see if any container
         # types define a default container id and use that.
@@ -175,14 +180,55 @@ class NullContainerFinder(object):
         return []
 
 
-class ContainerRegistry():
+class ContainerRegistry(object):
+    """Loop through enabled ContainerResolver plugins and find first match."""
 
-    def __init__(self):
-        pass
+    def __init__(self, app_info):
+        self.resolver_classes = self.__resolvers_dict()
+        self.enable_beta_mulled_containers = app_info.enable_beta_mulled_containers
+        self.app_info = app_info
+        self.container_resolvers = self.__build_container_resolvers(app_info)
 
-    def container_descriptions_for_requirements(self, requirements):
-        # Return lists of containers that would match requirements...
-        return []
+    def __build_container_resolvers( self, app_info ):
+        conf_file = getattr(app_info, 'containers_resolvers_config_file', None)
+        if not conf_file:
+            return self.__default_containers_resolvers()
+        if not os.path.exists( conf_file ):
+            log.debug( "Unable to find config file '%s'", conf_file)
+            return self.__default_containers_resolvers()
+        plugin_source = plugin_config.plugin_source_from_path( conf_file )
+        return self.__parse_resolver_conf_xml( plugin_source )
+
+    def __parse_resolver_conf_xml(self, plugin_source):
+        extra_kwds = {}
+        return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds)
+
+    def __default_containers_resolvers(self):
+        default_resolvers = [
+            ExplicitContainerResolver(self.app_info),
+        ]
+        if self.enable_beta_mulled_containers:
+            default_resolvers.extend([
+                CachedMulledContainerResolver(self.app_info),
+                MulledContainerResolver(self.app_info, namespace="mulled"),
+                BuildMulledContainerResolver(self.app_info),
+            ])
+        return default_resolvers
+
+    def __resolvers_dict( self ):
+        import galaxy.tools.deps.container_resolvers
+        return plugin_config.plugins_dict( galaxy.tools.deps.container_resolvers, 'resolver_type' )
+
+    def find_best_container_description(self, enabled_container_types, tool_info):
+        """Yield best container description of supplied types matching tool info."""
+        for container_resolver in self.container_resolvers:
+            container_description = container_resolver.resolve(enabled_container_types, tool_info)
+            log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description))
+            if container_description:
+                assert container_description.type in enabled_container_types
+                return container_description
+
+        return None
 
 
 class AppInfo(object):
@@ -193,7 +239,11 @@ class AppInfo(object):
         default_file_path=None,
         outputs_to_working_directory=False,
         container_image_cache_path=None,
-        library_import_dir=None
+        library_import_dir=None,
+        enable_beta_mulled_containers=False,
+        containers_resolvers_config_file=None,
+        involucro_path=None,
+        involucro_auto_init=True,
     ):
         self.galaxy_root_dir = galaxy_root_dir
         self.default_file_path = default_file_path
@@ -201,6 +251,10 @@ class AppInfo(object):
         self.outputs_to_working_directory = outputs_to_working_directory
         self.container_image_cache_path = container_image_cache_path
         self.library_import_dir = library_import_dir
+        self.enable_beta_mulled_containers = enable_beta_mulled_containers
+        self.containers_resolvers_config_file = containers_resolvers_config_file
+        self.involucro_path = involucro_path
+        self.involucro_auto_init = involucro_auto_init
 
 
 class ToolInfo(object):

lib/galaxy/tools/deps/installable.py

@@ -0,0 +1,77 @@
+"""Abstractions for installing local software managed and required by Galaxy/galaxy-lib."""
+
+import logging
+import os
+
+from abc import (
+    ABCMeta,
+    abstractmethod,
+    abstractproperty,
+)
+
+from galaxy.util.filelock import (
+    FileLock,
+    FileLockException
+)
+
+log = logging.getLogger(__name__)
+
+
+class InstallableContext(object):
+    """Represent a directory/configuration of something that can be installed."""
+
+    __metaclass__ = ABCMeta
+
+    @abstractmethod
+    def is_installed(self):
+        """Return bool indicating if the configured software is installed."""
+
+    @abstractmethod
+    def can_install(self):
+        """Check preconditions for installation."""
+
+    @abstractproperty
+    def installable_description(self):
+        """Short description of thing being installed for log statements."""
+
+    @abstractproperty
+    def parent_path(self):
+        """Return parent path of the location the installable will be created within."""
+
+
+def ensure_installed(installable_context, install_func, auto_init):
+    """Make sure target is installed - handle multiple processes potentially attempting installation."""
+    parent_path = installable_context.parent_path
+    desc = installable_context.installable_description
+
+    def _check():
+        if not installable_context.is_installed():
+            if auto_init:
+                if installable_context.can_install():
+                    if install_func(installable_context):
+                        installed = False
+                        log.warning("%s installation requested and failed." % desc)
+                    else:
+                        installed = installable_context.is_installed()
+                        if not installed:
+                            log.warning("%s installation requested, seemed to succeed, but not found." % desc)
+                else:
+                    installed = False
+            else:
+                installed = False
+                log.warning("%s not installed and auto-installation disabled.", desc)
+        else:
+            installed = True
+        return installed
+
+    if not os.path.exists(parent_path):
+        os.mkdir(parent_path)
+
+    try:
+        if auto_init and os.access(parent_path, os.W_OK):
+            with FileLock(os.path.join(parent_path, desc.lower())):
+                return _check()
+        else:
+            return _check()
+    except FileLockException:
+        return ensure_installed(installable_context, auto_init)

lib/galaxy/tools/deps/mulled/_cli.py

@@ -0,0 +1,19 @@
+"""CLI helpers for mulled command-line tools."""
+
+import argparse
+
+
+def arg_parser(argv, globals):
+    """Build an argparser for this CLI tool."""
+    doc = globals["__doc__"]
+    description, epilog = doc.split("\n", 1)
+    parser = argparse.ArgumentParser(
+        description=description,
+        epilog=epilog,
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    return parser
+
+__all__ = [
+    "arg_parser"
+]

lib/galaxy/tools/deps/mulled/invfile.lua

@@ -0,0 +1,67 @@
+-- http://stackoverflow.com/questions/19262761/lua-need-to-split-at-comma/19263313#19263313
+function string:split( inSplitPattern, outResults )
+  if not outResults then
+    outResults = { }
+  end
+  local theStart = 1
+  local theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart )
+  while theSplitStart do
+    table.insert( outResults, string.sub( self, theStart, theSplitStart-1 ) )
+    theStart = theSplitEnd + 1
+    theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart )
+  end
+  table.insert( outResults, string.sub( self, theStart ) )
+  return outResults
+end
+
+local repo = VAR.REPO
+
+local channel_args = ''
+local channels = VAR.CHANNELS:split(",")
+for i = 1, #channels do
+    channel_args = channel_args .. " -c " .. channels[i]
+end
+
+local target_args = ''
+local targets = VAR.TARGETS:split(",")
+for i = 1, #targets do
+    target_args = target_args .. " " .. targets[i]
+end
+
+local bind_args = {}
+local binds_table = VAR.BINDS:split(",")
+for i = 1, #binds_table do
+    table.insert(bind_args, binds_table[i])
+end
+
+inv.task('build')
+    .using('continuumio/miniconda:latest')
+        .withHostConfig({binds = {"build:/data"}})
+        .run('rm', '-rf', '/data/dist')
+    .using('continuumio/miniconda:latest')
+        .withHostConfig({binds = bind_args})
+        .run('/bin/sh', '-c', 'conda install '
+            .. channel_args .. ' '
+            .. target_args
+            .. ' -p /usr/local --copy --yes')
+    .wrap('build/dist')
+        .at('/usr/local')
+        .inImage('bgruening/busybox-bash:0.1')
+        .as(repo)
+
+inv.task('test')
+    .using(repo)
+    .withConfig({entrypoint = {'/bin/sh', '-c'}})
+    .run(VAR.TEST)
+
+inv.task('push')
+    .push(repo)
+
+inv.task('build-and-test')
+    .runTask('build')
+    .runTask('test')
+
+inv.task('all')
+    .runTask('build')
+    .runTask('test')
+    .runTask('push')

lib/galaxy/tools/deps/mulled/mulled_build.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python
+"""Build a mulled image for specified conda targets.
+
+Examples
+
+Build a mulled image with:
+
+    mulled-build build 'samtools=1.3.1,bedtools=2.22'
+
+"""
+from __future__ import print_function
+
+import json
+import os
+import string
+import subprocess
+
+from sys import platform as _platform
+
+from galaxy.tools.deps import commands
+from galaxy.tools.deps import installable
+
+try:
+    import yaml
+except ImportError:
+    yaml = None
+
+from ._cli import arg_parser
+from .util import build_target, conda_build_target_str, image_name
+from ..conda_compat import MetaData
+
+DIRNAME = os.path.dirname(__file__)
+DEFAULT_CHANNEL = "bioconda"
+DEFAULT_EXTRA_CHANNELS = ["conda-forge", "r"]
+DEFAULT_CHANNELS = [DEFAULT_CHANNEL] + DEFAULT_EXTRA_CHANNELS
+DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}"
+DEFAULT_BINDS = ["build/dist:/usr/local/"]
+IS_OS_X = _platform == "darwin"
+INVOLUCRO_VERSION = "1.1.2"
+
+
+def involucro_link():
+    if IS_OS_X:
+        url = "https://github.com/involucro/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION
+    else:
+        url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION
+    return url
+
+
+def get_tests(args, pkg_path):
+    """Extract test cases given a recipe's meta.yaml file."""
+    recipes_dir = args.recipes_dir
+
+    tests = ""
+    input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
+    recipe_meta = MetaData(input_dir)
+
+    tests_commands = recipe_meta.get_value('test/commands')
+    tests_imports = recipe_meta.get_value('test/imports')
+    requirements = recipe_meta.get_value('requirements/run')
+
+    if tests_imports or tests_commands:
+        if tests_commands:
+            tests = ' && '.join(tests_commands)
+        elif tests_imports and 'python' in requirements:
+            tests = ' && '.join('python -c "import %s"' % imp for imp in tests_imports)
+        elif tests_imports and ('perl' in requirements or 'perl-threaded' in requirements):
+            tests = ' && '.join('''perl -e "use %s;"''' % imp for imp in tests_imports)
+        tests = tests.replace('$R ', 'Rscript ')
+    else:
+        pass
+    return tests
+
+
+def get_pkg_name(args, pkg_path):
+    """Extract the package name from a given meta.yaml file."""
+    recipes_dir = args.recipes_dir
+
+    input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
+    recipe_meta = MetaData(input_dir)
+    return recipe_meta.get_value('package/name')
+
+
+def get_affected_packages(args):
+    """Return a list of all meta.yaml file that where modified/created recently.
+
+    Length of time to check for indicated by the ``hours`` parameter.
+    """
+    recipes_dir = args.recipes_dir
+    hours = args.diff_hours
+    cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours)
+    pkg_list = check_output(cmd, shell=True)
+    ret = list()
+    for pkg in pkg_list.strip().split('\n'):
+        if pkg and os.path.exists(os.path.join( recipes_dir, pkg )):
+            ret.append( (get_pkg_name(args, pkg), get_tests(args, pkg)) )
+    return ret
+
+
+def check_output(cmd, shell=True):
+    return subprocess.check_output(cmd, shell=shell)
+
+
+def conda_versions(pkg_name, file_name):
+    """Return all conda version strings for a specified package name."""
+    j = json.load(open(file_name))
+    ret = list()
+    for pkg in j['packages'].values():
+        if pkg['name'] == pkg_name:
+            ret.append('%s--%s' % (pkg['version'], pkg['build']))
+    return ret
+
+
+def mull_targets(
+    targets, involucro_context=None,
+    command="build", channels=DEFAULT_CHANNELS, namespace="mulled",
+    test='true', image_build=None, name_override=None,
+    repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
+    binds=DEFAULT_BINDS
+):
+    if involucro_context is None:
+        involucro_context = InvolucroContext()
+
+    repo_template_kwds = {
+        "namespace": namespace,
+        "image": image_name(targets, image_build=image_build, name_override=name_override)
+    }
+    repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
+
+    for channel in channels:
+        if channel.startswith('file://'):
+            bind_path = channel.lstrip('file://')
+            binds.append('/%s:/%s' % (bind_path, bind_path))
+
+    channels = ",".join(channels)
+    target_str = ",".join(map(conda_build_target_str, targets))
+    bind_str = ",".join(binds)
+    involucro_args = [
+        '-f', '%s/invfile.lua' % DIRNAME,
+        '-set', "CHANNELS='%s'" % channels,
+        '-set', "TEST='%s'" % test,
+        '-set', "TARGETS='%s'" % target_str,
+        '-set', "REPO='%s'" % repo,
+        '-set', "BINDS='%s'" % bind_str,
+        command,
+    ]
+    print(" ".join(involucro_context.build_command(involucro_args)))
+    if not dry_run:
+        ensure_installed(involucro_context, True)
+        involucro_context.exec_command(involucro_args)
+
+
+def context_from_args(args):
+    return InvolucroContext(involucro_bin=args.involucro_path)
+
+
+class InvolucroContext(installable.InstallableContext):
+
+    installable_description = "Involucro"
+
+    def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"):
+        if involucro_bin is None:
+            if os.path.exists("./involucro"):
+                self.involucro_bin = "./involucro"
+            else:
+                self.involucro_bin = "involucro"
+        else:
+            self.involucro_bin = involucro_bin
+        self.shell_exec = shell_exec or commands.shell
+        self.verbose = verbose
+
+    def build_command(self, involucro_args):
+        return [self.involucro_bin, "-v=%s" % self.verbose] + involucro_args
+
+    def exec_command(self, involucro_args):
+        cmd = self.build_command(involucro_args)
+        return self.shell_exec(" ".join(cmd))
+
+    def is_installed(self):
+        return os.path.exists(self.involucro_bin)
+
+    def can_install(self):
+        return True
+
+    @property
+    def parent_path(self):
+        return os.path.dirname(os.path.abspath(self.involucro_bin))
+
+
+def ensure_installed(involucro_context, auto_init):
+    return installable.ensure_installed(involucro_context, install_involucro, auto_init)
+
+
+def install_involucro(involucro_context=None, to_path=None):
+    to_path = involucro_context.involucro_bin
+    download_cmd = " ".join(commands.download_command(involucro_link(), to=to_path, quote_url=True))
+    full_cmd = "%s && chmod +x %s" % (download_cmd, to_path)
+    return involucro_context.shell_exec(full_cmd)
+
+
+def add_build_arguments(parser):
+    """Base arguments describing how to 'mull'."""
+    parser.add_argument('--involucro-path', dest="involucro_path", default=None,
+                        help="Path to involucro (if not set will look in working directory and on PATH).")
+    parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true",
+                        help="Rebuild package even if already published.")
+    parser.add_argument('--dry-run', dest='dry_run', action="store_true",
+                        help='Just print commands instead of executing them.')
+    parser.add_argument('-n', '--namespace', dest='namespace', default="mulled",
+                        help='quay.io namespace.')
+    parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE,
+                        help='Docker repository target for publication (only quay.io or compat. API is currently supported).')
+    parser.add_argument('-c', '--channel', dest='channel', default=DEFAULT_CHANNEL,
+                        help='Target conda channel')
+    parser.add_argument('--extra-channels', dest='extra_channels', default=",".join(DEFAULT_EXTRA_CHANNELS),
+                        help='Dependent conda channels.')
+
+
+def add_single_image_arguments(parser):
+    parser.add_argument("--name-override", dest="name_override", default=None,
+                        help="Override mulled image name - this is not recommended since metadata will not be detectable from the name of resulting images")
+    parser.add_argument("--image-build", dest="image_build", default=None,
+                        help="Build a versioned variant of this image.")
+
+
+def target_str_to_targets(targets_raw):
+    def parse_target(target_str):
+        if "=" in target_str:
+            package_name, version = target_str.split("=", 1)
+            target = build_target(package_name, version)
+        else:
+            target = build_target(target_str)
+        return target
+
+    targets = map(parse_target, targets_raw.split(","))
+    return targets
+
+
+def args_to_mull_targets_kwds(args):
+    kwds = {}
+    if hasattr(args, "image_build"):
+        kwds["image_build"] = args.image_build
+    if hasattr(args, "name_override"):
+        kwds["name_override"] = args.name_override
+    if hasattr(args, "namespace"):
+        kwds["namespace"] = args.namespace
+    if hasattr(args, "dry_run"):
+        kwds["dry_run"] = args.dry_run
+    if hasattr(args, "test"):
+        kwds["test"] = args.test
+    if hasattr(args, "channel"):
+        channels = [args.channel]
+        if hasattr(args, "extra_channels"):
+            channels += args.extra_channels.split(",")
+        kwds["channels"] = channels
+    if hasattr(args, "command"):
+        kwds["command"] = args.command
+    if hasattr(args, "repository_template"):
+        kwds["repository_template"] = args.repository_template
+
+    kwds["involucro_context"] = context_from_args(args)
+
+    return kwds
+
+
+def main(argv=None):
+    """Main entry-point for the CLI tool."""
+    parser = arg_parser(argv, globals())
+    add_build_arguments(parser)
+    add_single_image_arguments(parser)
+    parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
+    parser.add_argument('targets', metavar="TARGETS", default=None, help="Build a single container with specific package(s).")
+    parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of mulled container (leave blank to auto-generate based on packages - recommended).")
+    parser.add_argument('--test', help='Provide a test command for the container.')
+    args = parser.parse_args()
+    targets = target_str_to_targets(args.targets)
+    mull_targets(targets, **args_to_mull_targets_kwds(args))
+
+
+__all__ = ["main"]
+
+
+if __name__ == '__main__':
+    main()

lib/galaxy/tools/deps/mulled/mulled_build_channel.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+"""Build a mulled images for all recent conda recipe updates that don't have existing images.
+
+Examples:
+
+Build mulled images for recent bioconda changes with:
+
+    mulled-build-channel build
+
+Build, test, and publish images with the follow command:
+
+    mulled-build-channel all
+
+See recent changes that would be built with:
+
+    mulled-build-channel list
+
+"""
+
+
+import os
+import time
+
+from ._cli import arg_parser
+from .mulled_build import (
+    add_build_arguments,
+    args_to_mull_targets_kwds,
+    build_target,
+    check_output,
+    conda_versions,
+    get_affected_packages,
+    mull_targets,
+)
+from .util import quay_versions, version_sorted
+
+
+def _fetch_repo_data(args):
+    repo_data = args.repo_data
+    channel = args.channel
+    if repo_data is None:
+        repo_data = "%s-repodata.json" % channel
+    if not os.path.exists(repo_data):
+        check_output("wget --quiet https://conda.anaconda.org/%s/linux-64/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, repo_data, repo_data))
+    return repo_data
+
+
+def _new_versions(quay, conda):
+    """Calculate the versions that are in conda but not on quay.io."""
+    sconda = set(conda)
+    squay = set(quay) if quay else set()
+    return sconda - squay  # sconda.symmetric_difference(squay)
+
+
+def run_channel(args, build_last_n_versions=1):
+    """Build list of involucro commands (as shell snippet) to run."""
+    pkgs = get_affected_packages(args)
+    for pkg_name, pkg_tests in pkgs:
+        repo_data = _fetch_repo_data(args)
+        c = conda_versions(pkg_name, repo_data)
+        # only package the most recent N versions
+        c = version_sorted(c)[:build_last_n_versions]
+
+        if not args.force_rebuild:
+            time.sleep(1)
+            q = quay_versions(args.namespace, pkg_name)
+            versions = _new_versions(q, c)
+        else:
+            versions = c
+
+        for tag in versions:
+            target = build_target(pkg_name, tag=tag)
+            targets = [target]
+            mull_targets(targets, test=pkg_tests, **args_to_mull_targets_kwds(args))
+
+
+def get_pkg_names(args):
+    """Print package names that would be affected."""
+    print('\n'.join([pkg_name for pkg_name, pkg_tests in get_affected_packages(args)]))
+
+
+def add_channel_arguments(parser):
+    """Add arguments only used if running mulled over a whole conda channel."""
+    parser.add_argument('--repo-data', dest='repo_data', default=None,
+                        help='Published repository data (will be fetched from --channel if not available and written). Defaults to [channel_name]-repodata.json.')
+    parser.add_argument('--diff-hours', dest='diff_hours', default="25",
+                        help='If finding all recently changed recipes, use this number of hours.')
+    parser.add_argument('--recipes-dir', dest="recipes_dir", default="./bioconda-recipes")
+
+
+def main(argv=None):
+    """Main entry-point for the CLI tool."""
+    parser = arg_parser(argv, globals())
+    add_channel_arguments(parser)
+    add_build_arguments(parser)
+    parser.add_argument('command', metavar='COMMAND', help='Command (list, build-and-test, build, all)')
+    parser.add_argument('--targets', dest="targets", default=None, help="Build a single container with specific package(s).")
+    parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of a single container (leave blank to auto-generate based on packages).")
+    args = parser.parse_args()
+    if args.command == "list":
+        get_pkg_names(args)
+    else:
+        run_channel(args)
+
+
+__all__ = ["main"]
+
+
+if __name__ == '__main__':
+    main()

lib/galaxy/tools/deps/mulled/mulled_build_files.py

@@ -0,0 +1,80 @@
+"""Build all composite mulled recipes discovered in TSV files.
+
+Use mulled-build-channel to build images for single recipes for a whole conda
+channel. This script instead builds images for combinations of recipes. This
+script can be given a single TSV file or a directory of TSV files to process.
+
+Examples:
+
+Build all recipes discovered in tsv files in a single directory.
+
+    mulled-build-files build
+
+"""
+
+import collections
+import glob
+import os
+
+from ._cli import arg_parser
+
+from .mulled_build import (
+    add_build_arguments,
+    args_to_mull_targets_kwds,
+    mull_targets,
+    target_str_to_targets,
+)
+
+
+def main(argv=None):
+    """Main entry-point for the CLI tool."""
+    parser = arg_parser(argv, globals())
+    add_build_arguments(parser)
+    parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
+    parser.add_argument('files', metavar="FILES", default=".",
+                        help="Path to directory (or single file) of TSV files describing composite recipes.")
+    args = parser.parse_args()
+    for targets in generate_targets(args.files):
+        mull_targets(targets, **args_to_mull_targets_kwds(args))
+
+
+def generate_targets(target_source):
+    """Generate all targets from TSV files in specified file or directory."""
+    target_source = os.path.abspath(target_source)
+    if os.path.isdir(target_source):
+        target_source_files = glob.glob(target_source + "/*.tsv")
+    else:
+        target_source_files = [target_source]
+
+    for target_source_file in target_source_files:
+        with open(target_source_file, "r") as f:
+            for line in f.readlines():
+                if line:
+                    line = line.strip()
+
+                if not line or line.startswith("#"):
+                    continue
+
+                yield line_to_targets(line)
+
+
+def line_to_targets(line_str):
+    line = _parse_line(line_str)
+    return target_str_to_targets(line)
+
+
+_Line = collections.namedtuple("_Line", ["targets", "image_build", "name_override"])
+
+
+def _parse_line(line_str):
+    line_parts = line_str.split(" ")
+    assert len(line_parts) < 3, "Too many fields in line [%s], expect at most 3 - targets, image build number, and name override." % line_str
+    line_parts += [None] * (3 - len(line_parts))
+    return _Line(*line_parts)
+
+
+__all__ = ["main"]
+
+
+if __name__ == '__main__':
+    main()

lib/galaxy/tools/deps/mulled/mulled_build_tool.py

@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+"""Build a mulled images for a tool source (Galaxy or CWL tool).
+
+Examples:
+
+Build mulled images for requirements defined in a tool:
+
+    mulled-build-tool build path/to/tool_file.xml
+
+"""
+
+from galaxy.tools.parser import get_tool_source
+
+from ._cli import arg_parser
+from .mulled_build import (
+    add_build_arguments,
+    add_single_image_arguments,
+    args_to_mull_targets_kwds,
+    build_target,
+    mull_targets,
+)
+
+
+def main(argv=None):
+    """Main entry-point for the CLI tool."""
+    parser = arg_parser(argv, globals())
+    add_build_arguments(parser)
+    add_single_image_arguments(parser)
+    parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
+    parser.add_argument('tool', metavar="TOOL", default=None, help="Path to tool to build mulled image for.")
+    args = parser.parse_args()
+    tool_source = get_tool_source(args.tool)
+    requirements, _ = tool_source.parse_requirements_and_containers()
+    targets = requirements_to_mulled_targets(requirements)
+    mull_targets(targets, **args_to_mull_targets_kwds(args))
+
+
+def requirements_to_mulled_targets(requirements):
+    """Convert Galaxy's representation of requirements into mulled Target objects.
+
+    Only package requirements are retained.
+    """
+    package_requirements = filter(lambda r: r.type == "package", requirements)
+    targets = map(lambda r: build_target(r.name, r.version), package_requirements)
+    return targets
+
+
+__all__ = ["main", "requirements_to_mulled_targets"]
+
+
+if __name__ == '__main__':
+    main()

lib/galaxy/tools/deps/mulled/mulled_search.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import sys
+import tempfile
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+try:
+    from whoosh.fields import Schema
+    from whoosh.fields import TEXT
+    from whoosh.fields import STORED
+    from whoosh.index import create_in
+    from whoosh.qparser import QueryParser
+except ImportError:
+    Schema = TEXT = STORED = create_in = QueryParser = None
+
+QUAY_API_URL = 'https://quay.io/api/v1/repository'
+
+
+class QuaySearch():
+    """
+    Tool to search within a quay organization for a given software name.
+    """
+    def __init__(self, organization):
+        self.index = None
+        self.organization = organization
+
+    def build_index(self):
+        """
+        Create an index to quickly examine the repositories of a given quay.io organization.
+        """
+        # download all information about the repositories from the
+        # given organization in self.organization
+
+        parameters = {'public': 'true', 'namespace': self.organization}
+        r = requests.get(QUAY_API_URL, headers={'Accept-encoding': 'gzip'}, params=parameters,
+                         timeout=12)
+
+        tmp_dir = tempfile.mkdtemp()
+        schema = Schema(title=TEXT(stored=True), content=STORED)
+        self.index = create_in(tmp_dir, schema)
+
+        json_decoder = json.JSONDecoder()
+        decoded_request = json_decoder.decode(r.text)
+        writer = self.index.writer()
+        for repository in decoded_request['repositories']:
+            writer.add_document(title=repository['name'], content=repository['description'])
+        writer.commit()
+
+    def search_repository(self, search_string, non_strict):
+        """
+        Search Docker containers on quay.io.
+        Results are displayed with all available versions,
+        including the complete image name.
+        """
+        # with statement closes searcher after usage.
+        with self.index.searcher() as searcher:
+            search_string = "*%s*" % search_string
+            query = QueryParser("title", self.index.schema).parse(search_string)
+            results = searcher.search(query)
+            if non_strict:
+                # look for spelling errors and use suggestions as a search term too
+                corrector = searcher.corrector("title")
+                suggestions = corrector.suggest(search_string, limit=2)
+
+                # get all repositories with suggested keywords
+                for suggestion in suggestions:
+                    search_string = "*%s*" % suggestion
+                    query = QueryParser("title", self.index.schema).parse(search_string)
+                    results_tmp = searcher.search(query)
+                    results.extend(results_tmp)
+
+            sys.stdout.write("The query \033[1m %s \033[0m resulted in %s result(s).\n" % (search_string, len(results)))
+
+            if non_strict:
+                sys.stdout.write('The search was relaxed and the following search terms were searched: ')
+                sys.stdout.write('\033[1m %s \033[0m\n' % ', '.join(suggestions))
+
+            out = list()
+            for result in results:
+                title = result['title']
+                for version in self.get_additional_repository_information(title):
+                    row = [title]
+                    row.append(version)
+                    out.append(row)
+            if out:
+                col_width = max(len(word) for row in out for word in row) + 2  # padding
+                for row in out:
+                    name = row[0]
+                    version = row[1]
+                    sys.stdout.write("".join(word.ljust(col_width) for word in row) + "docker pull quay.io/%s/%s:%s\n" % (self.organization, name, version))
+            else:
+                sys.stdout.write("No results found for %s in quay.io/%s.\n" % (search_string, self.organization))
+
+    def get_additional_repository_information(self, repository_string):
+        """
+        Function downloads additional information from quay.io to
+        get the tag-field which includes the version number.
+        """
+        url = "%s/%s/%s" % (QUAY_API_URL, self.organization, repository_string)
+        r = requests.get(url, headers={'Accept-encoding': 'gzip'}, timeout=12)
+
+        json_decoder = json.JSONDecoder()
+        decoded_request = json_decoder.decode(r.text)
+        return decoded_request['tags']
+
+
+def main(argv=None):
+    parser = argparse.ArgumentParser(description='Searches in a given quay organization for a repository')
+    parser.add_argument('-o', '--organization', dest='organization_string', default="mulled",
+                        help='Change organization. Default is mulled.')
+    parser.add_argument('--non-strict', dest='non_strict', action="store_true",
+                        help='Autocorrection of typos activated. Lists more results but can be confusing.\
+                        For too many queries quay.io blocks the request and the results can be incomplete.')
+    parser.add_argument('-s', '--search', required=True,
+                        help='The name of the tool you want to search for.')
+    args = parser.parse_args()
+
+    quay = QuaySearch(args.organization_string)
+    quay.build_index()
+
+    quay.search_repository(args.search, args.non_strict)
+
+
+if __name__ == "__main__":
+    main()

lib/galaxy/tools/deps/mulled/util.py

@@ -0,0 +1,116 @@
+"""Utilities for working with mulled abstractions outside the mulled package."""
+from __future__ import print_function
+
+import collections
+import hashlib
+
+from distutils.version import LooseVersion
+
+try:
+    import requests
+except ImportError:
+    requests = None
+
+
+def quay_versions(namespace, pkg_name):
+    """Get all version tags for a Docker image stored on quay.io for supplied package name."""
+    if requests is None:
+        raise Exception("requets library is unavailable, functionality not available.")
+
+    assert namespace is not None
+    assert pkg_name is not None
+    url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name)
+    response = requests.get(url, timeout=None)
+    data = response.json()
+    if 'error_type' in data and data['error_type'] == "invalid_token":
+        return []
+
+    if 'tags' not in data:
+        raise Exception("Unexpected response from quay.io - not tags description found [%s]" % data)
+
+    return [tag for tag in data['tags'] if tag != 'latest']
+
+
+def mulled_tags_for(namespace, image):
+    """Fetch remote tags available for supplied image name.
+
+    The result will be sorted so newest tags are first.
+    """
+    tags = quay_versions(namespace, image)
+    tags = version_sorted(tags)
+    return tags
+
+
+def split_tag(tag):
+    """Split mulled image name into conda version and conda build."""
+    version = tag.split('--', 1)[0]
+    build = tag.split('--', 1)[1]
+    return version, build
+
+
+def version_sorted(elements):
+    """Sort iterable based on loose description of "version" from newest to oldest."""
+    return sorted(elements, key=LooseVersion, reverse=True)
+
+
+Target = collections.namedtuple("Target", ["package_name", "version", "build"])
+
+
+def build_target(package_name, version=None, build=None, tag=None):
+    """Use supplied arguments to build a :class:`Target` object."""
+    if tag is not None:
+        assert version is None
+        assert build is None
+        version, build = split_tag(tag)
+
+    return Target(package_name, version, build)
+
+
+def conda_build_target_str(target):
+    rval = target.package_name
+    if target.version:
+        rval += "=%s" % target.version
+
+        if target.build:
+            rval += "=%s" % target.build
+
+    return rval
+
+
+def image_name(targets, image_build=None, name_override=None):
+    if name_override is not None:
+        print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.")
+        return name_override
+
+    if len(targets) == 1:
+        target = targets[0]
+        suffix = ""
+        if target.version is not None:
+            if image_build is not None:
+                print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.")
+                suffix = image_build
+            else:
+                suffix += ":%s" % target.version
+                build = target.build
+                if build is not None:
+                    suffix += "--%s" % build
+        return "%s%s" % (target.package_name, suffix)
+    else:
+        targets_order = sorted(targets, key=lambda t: t.package_name)
+        requirements_buffer = "\n".join(map(conda_build_target_str, targets_order))
+        m = hashlib.sha1()
+        m.update(requirements_buffer)
+        suffix = "" if not image_build else ":%s" % image_build
+        return "mulled-v1-%s%s" % (m.hexdigest(), suffix)
+
+
+__all__ = [
+    "build_target",
+    "conda_build_target_str",
+    "image_name",
+    "mulled_tags_for",
+    "quay_versions",
+    "split_tag",
+    "Target",
+    "version_sorted",
+]

lib/galaxy/tools/deps/resolvers/__init__.py

@@ -1,3 +1,4 @@
+"""The module defines the abstract interface for dealing tool dependency resolution plugins."""
 from abc import (
     ABCMeta,
     abstractmethod,
@@ -10,6 +11,8 @@ from ..requirements import ToolRequirement
 
 
 class DependencyResolver(Dictifiable, object):
+    """Abstract description of a technique for resolving container images for tool execution."""
+
     # Keys for dictification.
     dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies']
     # A "simple" dependency is one that does not depend on the the tool
@@ -23,13 +26,15 @@ class DependencyResolver(Dictifiable, object):
 
     @abstractmethod
     def resolve( self, name, version, type, **kwds ):
-        """
-        Given inputs describing dependency in the abstract, yield tuple of
-        (script, bin, version). Here script is the env.sh file to source
-        before running a job, if that is not found the bin directory will be
-        appended to the path (if it is not None). Finally, version is the
-        resolved tool dependency version (which may differ from requested
-        version for instance if the request version is 'default'.)
+        """Given inputs describing dependency in the abstract yield a Dependency object.
+
+        The Dependency object describes various attributes (script, bin,
+        version) used to build scripts with the dependency availble. Here
+        script is the env.sh file to source before running a job, if that is
+        not found the bin directory will be appended to the path (if it is
+        not ``None``). Finally, version is the resolved tool dependency
+        version (which may differ from requested version for instance if the
+        request version is 'default'.)
         """
 
     def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds):

lib/galaxy/tools/deps/resolvers/conda.py

@@ -6,10 +6,8 @@ incompatible changes coming.
 import logging
 import os
 
-from galaxy.util.filelock import (
-    FileLock,
-    FileLockException
-)
+import galaxy.tools.deps.installable
+
 from ..conda_util import (
     build_isolated_environment,
     cleanup_failed_install,
@@ -32,7 +30,7 @@ from ..resolvers import (
 
 DEFAULT_BASE_PATH_DIRECTORY = "_conda"
 DEFAULT_CONDARC_OVERRIDE = "_condarc"
-DEFAULT_ENSURE_CHANNELS = "r,bioconda,iuc"
+DEFAULT_ENSURE_CHANNELS = "conda-forge,r,bioconda,iuc"
 
 log = logging.getLogger(__name__)
 
@@ -95,44 +93,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
         copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
         self.auto_init = _string_as_bool(get_option("auto_init"))
         self.conda_context = conda_context
-        self.ensure_conda_installed()
+        self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
         self.auto_install = auto_install
         self.copy_dependencies = copy_dependencies
         self.verbose_install_check = verbose_install_check
 
-    def ensure_conda_installed(self):
-        """
-        Make sure that conda is installed, and if conda can't be installed, mark resolver as disabled.
-        We acquire a lock, so that multiple handlers do not attempt to install conda simultaneously.
-        """
-        target_path = self.conda_prefix_parent
-
-        def _check():
-            if not self.conda_context.is_conda_installed():
-                if self.auto_init:
-                    if self.conda_context.can_install_conda():
-                        if install_conda(self.conda_context):
-                            self.disabled = True
-                            log.warning("Conda installation requested and failed.")
-                    else:
-                        self.disabled = True
-                else:
-                    self.disabled = True
-                    log.warning("Conda not installed and auto-installation disabled.")
-            else:
-                self.disabled = False
-
-        if not os.path.exists(target_path):
-            os.mkdir(target_path)
-        try:
-            if self.auto_init and os.access(target_path, os.W_OK):
-                with FileLock(os.path.join(target_path, 'conda')):
-                    _check()
-            else:
-                _check()
-        except FileLockException:
-            self.ensure_conda_installed()
-
     def resolve(self, name, version, type, **kwds):
         # Check for conda just not being there, this way we can enable
         # conda by default and just do nothing in not configured.

test/functional/tools/mulled_example_multi_1.xml

@@ -0,0 +1,18 @@
+<tool id="mulled_example_multi_1" name="mulled_example_multi_1" version="0.1.0">
+    <command><![CDATA[
+        bedtools --version > $out_file1 ;
+        echo "Moo" >> $out_file1 ;
+        samtools >> $out_file1 2>&1 ;
+        echo "Cow" >> $out_file1 ;
+    ]]></command>
+    <requirements>
+        <requirement type="package" version="1.3.1">samtools</requirement>
+        <requirement type="package" version="2.26.0">bedtools</requirement>
+    </requirements>
+    <inputs>
+        <param name="input1" type="data" optional="true" />
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -111,6 +111,8 @@
   <tool file="for_workflows/split.xml" />
   <tool file="for_workflows/create_input_collection.xml" />
 
+  <tool file="mulled_example_multi_1.xml" />
+
   <tool file="simple_constructs.yml" />
 
   <!-- Load collection operation tools - I consider these part of the