Merge pull request #4173 from jmchilton/galaxy-lib-mulled-update

Improved Mulled Containers

lib/galaxy/tools/deps/conda_util.py

@@ -488,7 +488,7 @@ def cleanup_failed_install(conda_target, conda_context=None):
     cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
 
 
-def best_search_result(conda_target, conda_context=None, channels_override=None):
+def best_search_result(conda_target, conda_context=None, channels_override=None, offline=False):
     """Find best "conda search" result for specified target.
 
     Return ``None`` if no results match.
@@ -498,6 +498,8 @@ def best_search_result(conda_target, conda_context=None, channels_override=None)
         conda_context.ensure_channels_configured()
 
     search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"]
+    if offline:
+        search_cmd.append("--offline")
     if channels_override:
         search_cmd.append("--override-channels")
         for channel in channels_override:

lib/galaxy/tools/deps/container_resolvers/mulled.py

@@ -18,9 +18,10 @@ from ..mulled.mulled_build import (
 )
 from ..mulled.mulled_build_tool import requirements_to_mulled_targets
 from ..mulled.util import (
-    image_name,
     mulled_tags_for,
     split_tag,
+    v1_image_name,
+    v2_image_name,
 )
 from ..requirements import ContainerDescription
 
@@ -28,13 +29,15 @@ log = logging.getLogger(__name__)
 
 
 CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"])
-CachedMulledImageMultiTarget = collections.namedtuple("CachedMulledImageMultiTarget", ["hash", "image_identifier"])
+CachedV1MulledImageMultiTarget = collections.namedtuple("CachedV1MulledImageMultiTarget", ["hash", "build", "image_identifier"])
+CachedV2MulledImageMultiTarget = collections.namedtuple("CachedV2MulledImageMultiTarget", ["package_hash", "version_hash", "build", "image_identifier"])
 
 CachedMulledImageSingleTarget.multi_target = False
-CachedMulledImageMultiTarget.multi_target = True
+CachedV1MulledImageMultiTarget.multi_target = "v1"
+CachedV2MulledImageMultiTarget.multi_target = "v2"
 
 
-def list_cached_mulled_images(namespace=None):
+def list_cached_mulled_images(namespace=None, hash_func="v2"):
     command = build_docker_images_command(truncate=True, sudo=False)
     command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
     images_and_versions = check_output(command)
@@ -44,15 +47,36 @@ def list_cached_mulled_images(namespace=None):
         image_name, version = line.split(" ", 1)
         identifier = "%s:%s" % (image_name, version)
         url, namespace, package_description = image_name.split("/")
+        if not version or version == "latest":
+            version = None
 
+        image = None
         if package_description.startswith("mulled-v1-"):
+            if hash_func == "v2":
+                return None
+
             hash = package_description
-            image = CachedMulledImageMultiTarget(hash, identifier)
+            build = None
+            if version and version.isdigit():
+                build = version
+            image = CachedV1MulledImageMultiTarget(hash, build, identifier)
+        elif package_description.startswith("mulled-v2-"):
+            if hash_func == "v1":
+                return None
+
+            version_hash = None
+            build = None
+
+            if version and "-" in version:
+                version_hash, build = version.rsplit("-", 1)
+            elif version.isdigit():
+                version_hash, build = None, version
+            elif version:
+                log.debug("Unparsable mulled image tag encountered [%s]" % version)
+
+            image = CachedV2MulledImageMultiTarget(package_description, version_hash, build, identifier)
         else:
             build = None
-            if not version or version == "latest":
-                version = None
-
             if version and "--" in version:
                 version, build = split_tag(version)
 
@@ -60,7 +84,9 @@ def list_cached_mulled_images(namespace=None):
 
         return image
 
-    return [output_line_to_image(_) for _ in filter(name_filter, images_and_versions.splitlines())]
+    # TODO: Sort on build ...
+    raw_images = [output_line_to_image(_) for _ in filter(name_filter, images_and_versions.splitlines())]
+    return [i for i in raw_images if i is not None]
 
 
 def get_filter(namespace):
@@ -68,11 +94,11 @@ def get_filter(namespace):
     return lambda name: name.startswith(prefix) and name.count("/") == 2
 
 
-def cached_container_description(targets, namespace):
+def cached_container_description(targets, namespace, hash_func="v2"):
     if len(targets) == 0:
         return None
 
-    cached_images = list_cached_mulled_images(namespace)
+    cached_images = list_cached_mulled_images(namespace, hash_func=hash_func)
     image = None
     if len(targets) == 1:
         target = targets[0]
@@ -84,10 +110,32 @@ def cached_container_description(targets, namespace):
             if not target.version or target.version == cached_image.version:
                 image = cached_image
                 break
-    else:
-        name = image_name(targets)
+    elif hash_func == "v2":
+        name = v2_image_name(targets)
+        if ":" in name:
+            package_hash, version_hash = name.split(":", 2)
+        else:
+            package_hash, version_hash = name, None
+
         for cached_image in cached_images:
-            if not cached_image.multi_target:
+            if cached_image.multi_target != "v2":
+                continue
+
+            if version_hash is None:
+                # Just match on package hash...
+                if package_hash == cached_image.package_hash:
+                    image = cached_image
+                    break
+            else:
+                # Match on package and version hash...
+                if package_hash == cached_image.package_hash and version_hash == cached_image.version_hash:
+                    image = cached_image
+                    break
+
+    elif hash_func == "v1":
+        name = v1_image_name(targets)
+        for cached_image in cached_images:
+            if cached_image.multi_target != "v1":
                 continue
 
             if name == cached_image.hash:
@@ -109,16 +157,17 @@ class CachedMulledContainerResolver(ContainerResolver):
 
     resolver_type = "cached_mulled"
 
-    def __init__(self, app_info=None, namespace=None):
+    def __init__(self, app_info=None, namespace=None, hash_func="v2"):
         super(CachedMulledContainerResolver, self).__init__(app_info)
         self.namespace = namespace
+        self.hash_func = hash_func
 
     def resolve(self, enabled_container_types, tool_info):
         if tool_info.requires_galaxy_python_environment:
             return None
 
         targets = mulled_targets(tool_info)
-        return cached_container_description(targets, self.namespace)
+        return cached_container_description(targets, self.namespace, hash_func=self.hash_func)
 
     def __str__(self):
         return "CachedMulledContainerResolver[namespace=%s]" % self.namespace
@@ -130,9 +179,10 @@ class MulledContainerResolver(ContainerResolver):
 
     resolver_type = "mulled"
 
-    def __init__(self, app_info=None, namespace="biocontainers"):
+    def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"):
         super(MulledContainerResolver, self).__init__(app_info)
         self.namespace = namespace
+        self.hash_func = hash_func
 
     def resolve(self, enabled_container_types, tool_info):
         if tool_info.requires_galaxy_python_environment:
@@ -162,10 +212,25 @@ class MulledContainerResolver(ContainerResolver):
                 version, build = split_tag(tags[0])
                 name = "%s:%s--%s" % (target.package_name, version, build)
         else:
-            base_image_name = image_name(targets)
-            tags = mulled_tags_for(self.namespace, base_image_name)
-            if tags:
-                name = "%s:%s" % (base_image_name, tags[0])
+            def tags_if_available(image_name):
+                if ":" in image_name:
+                    repo_name, tag_prefix = image_name.split(":", 2)
+                else:
+                    repo_name = image_name
+                    tag_prefix = None
+                tags = mulled_tags_for(self.namespace, repo_name, tag_prefix=tag_prefix)
+                return tags
+
+            if self.hash_func == "v2":
+                base_image_name = v2_image_name(targets)
+                tags = tags_if_available(base_image_name)
+                if tags:
+                    name = "%s:%s" % (base_image_name, tags[0])
+            elif self.hash_func == "v1":
+                base_image_name = v1_image_name(targets)
+                tags = tags_if_available(base_image_name)
+                if tags:
+                    name = "%s:%s" % (base_image_name, tags[0])
 
         if name:
             return ContainerDescription(
@@ -183,12 +248,13 @@ class BuildMulledContainerResolver(ContainerResolver):
 
     resolver_type = "build_mulled"
 
-    def __init__(self, app_info=None, namespace="local", **kwds):
+    def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds):
         super(BuildMulledContainerResolver, self).__init__(app_info)
         self._involucro_context_kwds = {
             'involucro_bin': self._get_config_option("involucro_path", None)
         }
         self.namespace = namespace
+        self.hash_func = hash_func
         self._mulled_kwds = {
             'namespace': namespace,
             'channels': self._get_config_option("channels", DEFAULT_CHANNELS, prefix="mulled"),
@@ -206,9 +272,10 @@ class BuildMulledContainerResolver(ContainerResolver):
         mull_targets(
             targets,
             involucro_context=self._get_involucro_context(),
+            hash_func=self.hash_func,
             **self._mulled_kwds
         )
-        return cached_container_description(targets, self.namespace)
+        return cached_container_description(targets, self.namespace, hash_func=self.hash_func)
 
     def _get_involucro_context(self):
         involucro_context = InvolucroContext(**self._involucro_context_kwds)

lib/galaxy/tools/deps/mulled/invfile.lua

@@ -45,6 +45,13 @@ if conda_image == '' then
     conda_image = 'continuumio/miniconda:latest'
 end
 
+
+local singularity_image = VAR.SINGULARITY_IMAGE
+if singularity_image == '' then
+    singularity_image = 'quay.io/biocontainers/singularity:2.3--0'
+end
+
+
 local destination_base_image = VAR.DEST_BASE_IMAGE
 if destination_base_image == '' then
     destination_base_image = 'bgruening/busybox-bash:0.1'
@@ -85,6 +92,25 @@ inv.task('build')
         .inImage(destination_base_image)
         .as(repo)
 
+if VAR.SINGULARITY ~= '' then
+    inv.task('singularity')
+        .using(singularity_image)
+        .withHostConfig({binds = {"build:/data","singularity_import:/import"}, privileged = true})
+        .withConfig({entrypoint = {'/bin/sh', '-c'}})
+        -- this will create a container that is the size of our conda dependencies + 20MiB
+        -- The 20 MiB can be improved at some point, but this seems to work for now.
+        .run("du -sc  /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)+20}'")
+        .run("singularity create --size `du -sc  /data/dist/ | tail -n 1 | cut -f 1 | awk '{print int($1/1024)+20}'` /import/" .. VAR.SINGULARITY_IMAGE_NAME)
+        .run('mkdir -p /usr/local/var/singularity/mnt/container && singularity bootstrap /import/' .. VAR.SINGULARITY_IMAGE_NAME .. ' /import/Singularity')
+        .run('chown ' .. VAR.USER_ID .. ' /import/' .. VAR.SINGULARITY_IMAGE_NAME)
+end
+
+inv.task('cleanup')
+    .using(conda_image)
+    .withHostConfig({binds = {"build:/data"}})
+    .run('rm', '-rf', '/data/dist')
+
+
 if VAR.TEST_BINDS == '' then
     inv.task('test')
         .using(repo)
@@ -103,9 +129,14 @@ inv.task('push')
 
 inv.task('build-and-test')
     .runTask('build')
+    .runTask('singularity')
+    .runTask('cleanup')
     .runTask('test')
 
+
 inv.task('all')
     .runTask('build')
+    .runTask('singularity')
+    .runTask('cleanup')
     .runTask('test')
     .runTask('push')

lib/galaxy/tools/deps/mulled/mulled_build.py

@@ -12,6 +12,7 @@ from __future__ import print_function
 
 import json
 import os
+import shutil
 import string
 import subprocess
 import sys
@@ -25,7 +26,14 @@ except ImportError:
 from galaxy.tools.deps import commands, installable
 
 from ._cli import arg_parser
-from .util import build_target, conda_build_target_str, image_name
+from .util import (
+    build_target,
+    conda_build_target_str,
+    create_repository,
+    quay_repository,
+    v1_image_name,
+    v2_image_name,
+)
 from ..conda_compat import MetaData
 
 DIRNAME = os.path.dirname(__file__)
@@ -37,6 +45,24 @@ DEFAULT_BINDS = ["build/dist:/usr/local/"]
 DEFAULT_WORKING_DIR = '/source/'
 IS_OS_X = _platform == "darwin"
 INVOLUCRO_VERSION = "1.1.2"
+DEST_BASE_IMAGE = os.environ.get('DEST_BASE_IMAGE', None)
+
+SINGULARITY_TEMPLATE = """Bootstrap: docker
+From: bgruening/busybox-bash:0.1
+
+%%setup
+
+    echo "Copying conda environment"
+    mkdir -p /tmp/conda
+    cp -r /data/dist/* /tmp/conda/
+
+%%post
+    mkdir -p /usr/local
+    cp -R /tmp/conda/* /usr/local/
+
+%%test
+    %(container_test)s
+"""
 
 
 def involucro_link():
@@ -111,23 +137,56 @@ def conda_versions(pkg_name, file_name):
     return ret
 
 
+class BuildExistsException(Exception):
+    """Exception indicating mull_targets is skipping an existing build.
+
+    If mull_targets is called with rebuild=False and the target built is already published
+    an instance of this exception is thrown.
+    """
+
+
 def mull_targets(
     targets, involucro_context=None,
-    command="build", channels=DEFAULT_CHANNELS, namespace="mulled",
+    command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers",
     test='true', test_files=None, image_build=None, name_override=None,
     repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
-    conda_version=None, verbose=False, binds=DEFAULT_BINDS
+    conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True,
+    oauth_token=None, hash_func="v2", singularity=False,
 ):
     targets = list(targets)
     if involucro_context is None:
         involucro_context = InvolucroContext()
 
+    image_function = v1_image_name if hash_func == "v1" else v2_image_name
+    if len(targets) > 2 and image_build is None:
+        # Force an image build in this case - this seems hacky probably
+        # shouldn't work this way but single case broken else wise.
+        image_build = "0"
+
     repo_template_kwds = {
         "namespace": namespace,
-        "image": image_name(targets, image_build=image_build, name_override=name_override)
+        "image": image_function(targets, image_build=image_build, name_override=name_override)
     }
     repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
 
+    if not rebuild or "push" in command:
+        repo_name = repo_template_kwds["image"].split(":", 1)[0]
+        repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
+        if not rebuild:
+            tags = repo_data.get("tags", [])
+
+            target_tag = None
+            if ":" in repo_template_kwds["image"]:
+                image_name_parts = repo_template_kwds["image"].split(":")
+                assert len(image_name_parts) == 2, ": not allowed in image name [%s]" % repo_template_kwds["image"]
+                target_tag = image_name_parts[1]
+
+            if tags and (target_tag is None or target_tag in tags):
+                raise BuildExistsException()
+        if "push" in command and "error_type" in repo_data and oauth_token:
+            # Explicitly create the repository so it can be built as public.
+            create_repository(repo_template_kwds["namespace"], repo_name, oauth_token)
+
     for channel in channels:
         if channel.startswith('file://'):
             bind_path = channel.lstrip('file://')
@@ -144,8 +203,16 @@ def mull_targets(
         '-set', "REPO='%s'" % repo,
         '-set', "BINDS='%s'" % bind_str,
     ]
+
+    if DEST_BASE_IMAGE:
+        involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE])
     if verbose:
         involucro_args.extend(["-set", "VERBOSE='1'"])
+    if singularity:
+        singularity_image_name = repo_template_kwds['image']
+        involucro_args.extend(["-set", "SINGULARITY='1'"])
+        involucro_args.extend(["-set", "SINGULARITY_IMAGE_NAME='%s'" % singularity_image_name])
+        involucro_args.extend(["-set", "USER_ID='%s:%s'" % (os.getuid(), os.getgid() )])
     if conda_version is not None:
         verbose = "--verbose" if verbose else "--quiet"
         involucro_args.extend(["-set", "PREINSTALL='conda install %s --yes conda=%s'" % (verbose, conda_version)])
@@ -165,7 +232,18 @@ def mull_targets(
     print(" ".join(involucro_context.build_command(involucro_args)))
     if not dry_run:
         ensure_installed(involucro_context, True)
-        return involucro_context.exec_command(involucro_args)
+        if singularity:
+            if not os.path.exists('./singularity_import'):
+                os.mkdir('./singularity_import')
+            with open('./singularity_import/Singularity', 'w+') as sin_def:
+                fill_template = SINGULARITY_TEMPLATE % {'container_test': test}
+                sin_def.write(fill_template)
+        ret = involucro_context.exec_command(involucro_args)
+        if singularity:
+            # we can not remove this folder as it contains the image wich is owned by root
+            pass
+            # shutil.rmtree('./singularity_import')
+        return ret
     return 0
 
 
@@ -194,7 +272,14 @@ class InvolucroContext(installable.InstallableContext):
 
     def exec_command(self, involucro_args):
         cmd = self.build_command(involucro_args)
-        return self.shell_exec(" ".join(cmd))
+        # Create ./build dir manually, otherwise Docker will do it as root
+        os.mkdir('./build')
+        try:
+            res = self.shell_exec(" ".join(cmd))
+        finally:
+            # delete build directory in any case
+            shutil.rmtree('./build')
+        return res
 
     def is_installed(self):
         return os.path.exists(self.involucro_bin)
@@ -212,9 +297,10 @@ def ensure_installed(involucro_context, auto_init):
 
 
 def install_involucro(involucro_context=None, to_path=None):
-    to_path = involucro_context.involucro_bin
-    download_cmd = " ".join(commands.download_command(involucro_link(), to=to_path, quote_url=True))
-    full_cmd = "%s && chmod +x %s" % (download_cmd, to_path)
+    install_path = os.path.abspath(involucro_context.involucro_bin)
+    involucro_context.involucro_bin = install_path
+    download_cmd = " ".join(commands.download_command(involucro_link(), to=install_path, quote_url=True))
+    full_cmd = "%s && chmod +x %s" % (download_cmd, install_path)
     return involucro_context.shell_exec(full_cmd)
 
 
@@ -222,13 +308,13 @@ def add_build_arguments(parser):
     """Base arguments describing how to 'mull'."""
     parser.add_argument('--involucro-path', dest="involucro_path", default=None,
                         help="Path to involucro (if not set will look in working directory and on PATH).")
-    parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true",
-                        help="Rebuild package even if already published.")
     parser.add_argument('--dry-run', dest='dry_run', action="store_true",
                         help='Just print commands instead of executing them.')
     parser.add_argument('--verbose', dest='verbose', action="store_true",
                         help='Cause process to be verbose.')
-    parser.add_argument('-n', '--namespace', dest='namespace', default="mulled",
+    parser.add_argument('--singularity', action="store_true",
+                        help='Additionally build a singularity image.')
+    parser.add_argument('-n', '--namespace', dest='namespace', default="biocontainers",
                         help='quay.io namespace.')
     parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE,
                         help='Docker repository target for publication (only quay.io or compat. API is currently supported).')
@@ -238,6 +324,10 @@ def add_build_arguments(parser):
                         help='Dependent conda channels.')
     parser.add_argument('--conda-version', dest="conda_version", default=None,
                         help="Change to specified version of Conda before installing packages.")
+    parser.add_argument('--oauth-token', dest="oauth_token", default=None,
+                        help="If set, use this token when communicating with quay.io API.")
+    parser.add_argument('--check-published', dest="rebuild", action='store_false')
+    parser.add_argument('--hash', dest="hash", choices=["v1", "v2"], default="v2")
 
 
 def add_single_image_arguments(parser):
@@ -273,6 +363,8 @@ def args_to_mull_targets_kwds(args):
         kwds["namespace"] = args.namespace
     if hasattr(args, "dry_run"):
         kwds["dry_run"] = args.dry_run
+    if hasattr(args, "singularity"):
+        kwds["singularity"] = args.singularity
     if hasattr(args, "test"):
         kwds["test"] = args.test
     if hasattr(args, "test_files"):
@@ -289,6 +381,12 @@ def args_to_mull_targets_kwds(args):
         kwds["repository_template"] = args.repository_template
     if hasattr(args, "conda_version"):
         kwds["conda_version"] = args.conda_version
+    if hasattr(args, "oauth_token"):
+        kwds["oauth_token"] = args.oauth_token
+    if hasattr(args, "rebuild"):
+        kwds["rebuild"] = args.rebuild
+    if hasattr(args, "hash"):
+        kwds["hash_func"] = args.hash
 
     kwds["involucro_context"] = context_from_args(args)
 

lib/galaxy/tools/deps/mulled/mulled_build_channel.py

@@ -87,6 +87,8 @@ def add_channel_arguments(parser):
     parser.add_argument('--diff-hours', dest='diff_hours', default="25",
                         help='If finding all recently changed recipes, use this number of hours.')
     parser.add_argument('--recipes-dir', dest="recipes_dir", default="./bioconda-recipes")
+    parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true",
+                        help="Rebuild package even if already published.")
 
 
 def main(argv=None):

lib/galaxy/tools/deps/mulled/mulled_build_files.py

@@ -20,6 +20,7 @@ from ._cli import arg_parser
 from .mulled_build import (
     add_build_arguments,
     args_to_mull_targets_kwds,
+    BuildExistsException,
     mull_targets,
     target_str_to_targets,
 )
@@ -33,8 +34,19 @@ def main(argv=None):
     parser.add_argument('files', metavar="FILES", default=".",
                         help="Path to directory (or single file) of TSV files describing composite recipes.")
     args = parser.parse_args()
-    for targets in generate_targets(args.files):
-        mull_targets(targets, **args_to_mull_targets_kwds(args))
+    for (targets, image_build, name_override) in generate_targets(args.files):
+        if not image_build and len(targets) > 1:
+            # Specify an explict tag in this case.
+            image_build = "0"
+        try:
+            mull_targets(
+                targets,
+                image_build=image_build,
+                name_override=name_override,
+                **args_to_mull_targets_kwds(args)
+            )
+        except BuildExistsException:
+            continue
 
 
 def generate_targets(target_source):
@@ -59,14 +71,14 @@ def generate_targets(target_source):
 
 def line_to_targets(line_str):
     line = _parse_line(line_str)
-    return target_str_to_targets(line)
+    return (target_str_to_targets(line.targets), line.image_build, line.name_override)
 
 
 _Line = collections.namedtuple("_Line", ["targets", "image_build", "name_override"])
 
 
 def _parse_line(line_str):
-    line_parts = line_str.split(" ")
+    line_parts = line_str.split("\t")
     assert len(line_parts) < 3, "Too many fields in line [%s], expect at most 3 - targets, image build number, and name override." % line_str
     line_parts += [None] * (3 - len(line_parts))
     return _Line(*line_parts)

lib/galaxy/tools/deps/mulled/mulled_search.py

@@ -112,8 +112,8 @@ class QuaySearch():
 
 def main(argv=None):
     parser = argparse.ArgumentParser(description='Searches in a given quay organization for a repository')
-    parser.add_argument('-o', '--organization', dest='organization_string', default="mulled",
-                        help='Change organization. Default is mulled.')
+    parser.add_argument('-o', '--organization', dest='organization_string', default="biocontainers",
+                        help='Change organization. Default is biocontainers.')
     parser.add_argument('--non-strict', dest='non_strict', action="store_true",
                         help='Autocorrection of typos activated. Lists more results but can be confusing.\
                         For too many queries quay.io blocks the request and the results can be incomplete.')

lib/galaxy/tools/deps/mulled/util.py

@@ -12,16 +12,22 @@ except ImportError:
     requests = None
 
 
+def create_repository(namespace, repo_name, oauth_token):
+    assert oauth_token
+    headers = {'Authorization': 'Bearer %s' % oauth_token}
+    data = {
+        "repository": repo_name,
+        "namespace": namespace,
+        "description": "",
+        "visibility": "public",
+    }
+    requests.post("https://quay.io/api/v1/repository", json=data, headers=headers)
+
+
 def quay_versions(namespace, pkg_name):
     """Get all version tags for a Docker image stored on quay.io for supplied package name."""
-    if requests is None:
-        raise Exception("requets library is unavailable, functionality not available.")
+    data = quay_repository(namespace, pkg_name)
 
-    assert namespace is not None
-    assert pkg_name is not None
-    url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name)
-    response = requests.get(url, timeout=None)
-    data = response.json()
     if 'error_type' in data and data['error_type'] == "invalid_token":
         return []
 
@@ -31,12 +37,26 @@ def quay_versions(namespace, pkg_name):
     return [tag for tag in data['tags'] if tag != 'latest']
 
 
-def mulled_tags_for(namespace, image):
+def quay_repository(namespace, pkg_name):
+    if requests is None:
+        raise Exception("requets library is unavailable, functionality not available.")
+
+    assert namespace is not None
+    assert pkg_name is not None
+    url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name)
+    response = requests.get(url, timeout=None)
+    data = response.json()
+    return data
+
+
+def mulled_tags_for(namespace, image, tag_prefix=None):
     """Fetch remote tags available for supplied image name.
 
     The result will be sorted so newest tags are first.
     """
     tags = quay_versions(namespace, image)
+    if tag_prefix is not None:
+        tags = [t for t in tags if t.startswith(tag_prefix)]
     tags = version_sorted(tags)
     return tags
 
@@ -77,25 +97,49 @@ def conda_build_target_str(target):
     return rval
 
 
-def image_name(targets, image_build=None, name_override=None):
+def _simple_image_name(targets, image_build=None):
+    target = targets[0]
+    suffix = ""
+    if target.version is not None:
+        if image_build is not None:
+            print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.")
+            suffix = image_build
+        else:
+            suffix += ":%s" % target.version
+            build = target.build
+            if build is not None:
+                suffix += "--%s" % build
+    return "%s%s" % (target.package_name, suffix)
+
+
+def v1_image_name(targets, image_build=None, name_override=None):
+    """Generate mulled hash version 1 container identifier for supplied arguments.
+
+    If a single target is specified, simply use the supplied name and version as
+    the repository name and tag respectively. If multiple targets are supplied,
+    hash the package names and versions together as the repository name. For mulled
+    version 1 containers the image build is the repository tag (if supplied).
+
+    >>> single_targets = [build_target("samtools", version="1.3.1")]
+    >>> v1_image_name(single_targets)
+    'samtools:1.3.1'
+    >>> multi_targets = [build_target("samtools", version="1.3.1"), build_target("bwa", version="0.7.13")]
+    >>> v1_image_name(multi_targets)
+    'mulled-v1-b06ecbd9141f0dbbc0c287375fc0813adfcbdfbd'
+    >>> multi_targets_on_versionless = [build_target("samtools", version="1.3.1"), build_target("bwa")]
+    >>> v1_image_name(multi_targets_on_versionless)
+    'mulled-v1-bda945976caa5734347fbf7f35066d9f58519e0c'
+    >>> multi_targets_versionless = [build_target("samtools"), build_target("bwa")]
+    >>> v1_image_name(multi_targets_versionless)
+    'mulled-v1-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40'
+    """
     if name_override is not None:
         print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.")
         return name_override
 
     targets = list(targets)
     if len(targets) == 1:
-        target = targets[0]
-        suffix = ""
-        if target.version is not None:
-            if image_build is not None:
-                print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.")
-                suffix = image_build
-            else:
-                suffix += ":%s" % target.version
-                build = target.build
-                if build is not None:
-                    suffix += "--%s" % build
-        return "%s%s" % (target.package_name, suffix)
+        return _simple_image_name(targets, image_build=image_build)
     else:
         targets_order = sorted(targets, key=lambda t: t.package_name)
         requirements_buffer = "\n".join(map(conda_build_target_str, targets_order))
@@ -105,6 +149,66 @@ def image_name(targets, image_build=None, name_override=None):
         return "mulled-v1-%s%s" % (m.hexdigest(), suffix)
 
 
+def v2_image_name(targets, image_build=None, name_override=None):
+    """Generate mulled hash version 2 container identifier for supplied arguments.
+
+    If a single target is specified, simply use the supplied name and version as
+    the repository name and tag respectively. If multiple targets are supplied,
+    hash the package names as the repository name and hash the package versions (if set)
+    as the tag.
+
+    >>> single_targets = [build_target("samtools", version="1.3.1")]
+    >>> v2_image_name(single_targets)
+    'samtools:1.3.1'
+    >>> multi_targets = [build_target("samtools", version="1.3.1"), build_target("bwa", version="0.7.13")]
+    >>> v2_image_name(multi_targets)
+    'mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:4d0535c94ef45be8459f429561f0894c3fe0ebcf'
+    >>> multi_targets_on_versionless = [build_target("samtools", version="1.3.1"), build_target("bwa")]
+    >>> v2_image_name(multi_targets_on_versionless)
+    'mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:b0c847e4fb89c343b04036e33b2daa19c4152cf5'
+    >>> multi_targets_versionless = [build_target("samtools"), build_target("bwa")]
+    >>> v2_image_name(multi_targets_versionless)
+    'mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40'
+    """
+    if name_override is not None:
+        print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.")
+        return name_override
+
+    targets = list(targets)
+    if len(targets) == 1:
+        return _simple_image_name(targets, image_build=image_build)
+    else:
+        targets_order = sorted(targets, key=lambda t: t.package_name)
+        package_name_buffer = "\n".join(map(lambda t: t.package_name, targets_order))
+        package_hash = hashlib.sha1()
+        package_hash.update(package_name_buffer.encode())
+
+        versions = map(lambda t: t.version, targets_order)
+        if any(versions):
+            # Only hash versions if at least one package has versions...
+            version_name_buffer = "\n".join(map(lambda t: t.version or "null", targets_order))
+            version_hash = hashlib.sha1()
+            version_hash.update(version_name_buffer.encode())
+            version_hash_str = version_hash.hexdigest()
+        else:
+            version_hash_str = ""
+
+        if not image_build:
+            build_suffix = ""
+        elif version_hash_str:
+            # tagged verson is <version_hash>-<build>
+            build_suffix = "-%s" % image_build
+        else:
+            # tagged version is simply the build
+            build_suffix = image_build
+        suffix = ""
+        if version_hash_str or build_suffix:
+            suffix = ":%s%s" % (version_hash_str, build_suffix)
+        return "mulled-v2-%s%s" % (package_hash.hexdigest(), suffix)
+
+
+image_name = v1_image_name  # deprecated
+
 __all__ = (
     "build_target",
     "conda_build_target_str",
@@ -113,5 +217,7 @@ __all__ = (
     "quay_versions",
     "split_tag",
     "Target",
+    "v1_image_name",
+    "v2_image_name",
     "version_sorted",
 )

test/functional/tools/mulled_example_multi_versionless.xml

@@ -0,0 +1,18 @@
+<tool id="mulled_example_multi_versionless" name="mulled_example_multi_versionless" version="0.1.0">
+    <command><![CDATA[
+        bedtools --version > $out_file1 ;
+        echo "Moo" >> $out_file1 ;
+        samtools >> $out_file1 2>&1 ;
+        echo "Cow" >> $out_file1 ;
+    ]]></command>
+    <requirements>
+        <requirement type="package">samtools</requirement>
+        <requirement type="package">bedtools</requirement>
+    </requirements>
+    <inputs>
+        <param name="input1" type="data" optional="true" />
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="txt" />
+    </outputs>
+</tool>

test/functional/tools/samples_tool_conf.xml

@@ -135,6 +135,7 @@
   <!-- Tools for dependency and container testing. -->
   <tool file="legacy_r.xml" />
   <tool file="mulled_example_multi_1.xml" />
+  <tool file="mulled_example_multi_versionless.xml" />
   <tool file="mulled_example_multi_incomplete.xml" />
   <tool file="mulled_example_conflict.xml" />