mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
Bring in all recent mulled container changes from galaxy-lib.
- Update mulled container resolution to use a version 2 (v2) hashing function. https://github.com/galaxyproject/galaxy-lib/pull/63 - When building images with mulled, always cleanup build directory. https://github.com/galaxyproject/galaxy-lib/pull/55/files and https://github.com/galaxyproject/galaxy-lib/pull/61 - Expose an offline search option in library Conda search utility. https://github.com/galaxyproject/galaxy-lib/pull/60 - Fix first use of mulled-build-* for auto-install of Involucro. https://github.com/galaxyproject/galaxy-lib/pull/59/commits/c5e7778f51d735a8bb6a4a955a0768e65bfb0548 - Fix up and improve logic related to rebuilding mulled containers. https://github.com/galaxyproject/galaxy-lib/pull/59/commits/faf528997c36f8a9a00566549ab2bcb07e93a9d7 - Fix mulled-build-files to properly handle tab separated files. https://github.com/galaxyproject/galaxy-lib/pull/59/commits/10b1cf3addf03e1e442c2d1617c7468de67b2cac - Various other mulled-build-file enhancements. https://github.com/galaxyproject/galaxy-lib/pull/59/commits/70314e3e65921a544f2188eb2a16f8e7394ff1b7 https://github.com/galaxyproject/galaxy-lib/pull/59/commits/21b4bc58013dc53509f02d027afa73396a15aabd https://github.com/galaxyproject/galaxy-lib/pull/59/commits/74abe8a107044e106e14c244d03b030e0ed5d43d https://github.com/galaxyproject/galaxy-lib/pull/58 - Correct mulled-search default namespace from mulled to biocontainers. https://github.com/galaxyproject/galaxy-lib/pull/57 - Add ability to specify a new base image when building mulled images. https://github.com/galaxyproject/galaxy-lib/pull/56 - Update the default mulled target namespace from the older mulled to the new biocontainers.
This commit is contained in:
@@ -488,7 +488,7 @@ def cleanup_failed_install(conda_target, conda_context=None):
|
||||
cleanup_failed_install_of_environment(conda_target.install_environment, conda_context=conda_context)
|
||||
|
||||
|
||||
def best_search_result(conda_target, conda_context=None, channels_override=None):
|
||||
def best_search_result(conda_target, conda_context=None, channels_override=None, offline=False):
|
||||
"""Find best "conda search" result for specified target.
|
||||
|
||||
Return ``None`` if no results match.
|
||||
@@ -498,6 +498,8 @@ def best_search_result(conda_target, conda_context=None, channels_override=None)
|
||||
conda_context.ensure_channels_configured()
|
||||
|
||||
search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"]
|
||||
if offline:
|
||||
search_cmd.append("--offline")
|
||||
if channels_override:
|
||||
search_cmd.append("--override-channels")
|
||||
for channel in channels_override:
|
||||
|
||||
@@ -18,9 +18,10 @@ from ..mulled.mulled_build import (
|
||||
)
|
||||
from ..mulled.mulled_build_tool import requirements_to_mulled_targets
|
||||
from ..mulled.util import (
|
||||
image_name,
|
||||
mulled_tags_for,
|
||||
split_tag,
|
||||
v1_image_name,
|
||||
v2_image_name,
|
||||
)
|
||||
from ..requirements import ContainerDescription
|
||||
|
||||
@@ -28,13 +29,15 @@ log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"])
|
||||
CachedMulledImageMultiTarget = collections.namedtuple("CachedMulledImageMultiTarget", ["hash", "image_identifier"])
|
||||
CachedV1MulledImageMultiTarget = collections.namedtuple("CachedV1MulledImageMultiTarget", ["hash", "build", "image_identifier"])
|
||||
CachedV2MulledImageMultiTarget = collections.namedtuple("CachedV2MulledImageMultiTarget", ["package_hash", "version_hash", "build", "image_identifier"])
|
||||
|
||||
CachedMulledImageSingleTarget.multi_target = False
|
||||
CachedMulledImageMultiTarget.multi_target = True
|
||||
CachedV1MulledImageMultiTarget.multi_target = "v1"
|
||||
CachedV2MulledImageMultiTarget.multi_target = "v2"
|
||||
|
||||
|
||||
def list_cached_mulled_images(namespace=None):
|
||||
def list_cached_mulled_images(namespace=None, hash_func="v2"):
|
||||
command = build_docker_images_command(truncate=True, sudo=False)
|
||||
command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
|
||||
images_and_versions = check_output(command)
|
||||
@@ -44,15 +47,36 @@ def list_cached_mulled_images(namespace=None):
|
||||
image_name, version = line.split(" ", 1)
|
||||
identifier = "%s:%s" % (image_name, version)
|
||||
url, namespace, package_description = image_name.split("/")
|
||||
if not version or version == "latest":
|
||||
version = None
|
||||
|
||||
image = None
|
||||
if package_description.startswith("mulled-v1-"):
|
||||
if hash_func == "v2":
|
||||
return None
|
||||
|
||||
hash = package_description
|
||||
image = CachedMulledImageMultiTarget(hash, identifier)
|
||||
build = None
|
||||
if version and version.isdigit():
|
||||
build = version
|
||||
image = CachedV1MulledImageMultiTarget(hash, build, identifier)
|
||||
elif package_description.startswith("mulled-v2-"):
|
||||
if hash_func == "v1":
|
||||
return None
|
||||
|
||||
version_hash = None
|
||||
build = None
|
||||
|
||||
if version and "-" in version:
|
||||
version_hash, build = version.rsplit("-", 1)
|
||||
elif version.isdigit():
|
||||
version_hash, build = None, version
|
||||
elif version:
|
||||
log.debug("Unparsable mulled image tag encountered [%s]" % version)
|
||||
|
||||
image = CachedV2MulledImageMultiTarget(package_description, version_hash, build, identifier)
|
||||
else:
|
||||
build = None
|
||||
if not version or version == "latest":
|
||||
version = None
|
||||
|
||||
if version and "--" in version:
|
||||
version, build = split_tag(version)
|
||||
|
||||
@@ -60,7 +84,9 @@ def list_cached_mulled_images(namespace=None):
|
||||
|
||||
return image
|
||||
|
||||
return [output_line_to_image(_) for _ in filter(name_filter, images_and_versions.splitlines())]
|
||||
# TODO: Sort on build ...
|
||||
raw_images = [output_line_to_image(_) for _ in filter(name_filter, images_and_versions.splitlines())]
|
||||
return [i for i in raw_images if i is not None]
|
||||
|
||||
|
||||
def get_filter(namespace):
|
||||
@@ -68,11 +94,11 @@ def get_filter(namespace):
|
||||
return lambda name: name.startswith(prefix) and name.count("/") == 2
|
||||
|
||||
|
||||
def cached_container_description(targets, namespace):
|
||||
def cached_container_description(targets, namespace, hash_func="v2"):
|
||||
if len(targets) == 0:
|
||||
return None
|
||||
|
||||
cached_images = list_cached_mulled_images(namespace)
|
||||
cached_images = list_cached_mulled_images(namespace, hash_func=hash_func)
|
||||
image = None
|
||||
if len(targets) == 1:
|
||||
target = targets[0]
|
||||
@@ -84,10 +110,32 @@ def cached_container_description(targets, namespace):
|
||||
if not target.version or target.version == cached_image.version:
|
||||
image = cached_image
|
||||
break
|
||||
else:
|
||||
name = image_name(targets)
|
||||
elif hash_func == "v2":
|
||||
name = v2_image_name(targets)
|
||||
if ":" in name:
|
||||
package_hash, version_hash = name.split(":", 2)
|
||||
else:
|
||||
package_hash, version_hash = name, None
|
||||
|
||||
for cached_image in cached_images:
|
||||
if not cached_image.multi_target:
|
||||
if cached_image.multi_target != "v2":
|
||||
continue
|
||||
|
||||
if version_hash is None:
|
||||
# Just match on package hash...
|
||||
if package_hash == cached_image.package_hash:
|
||||
image = cached_image
|
||||
break
|
||||
else:
|
||||
# Match on package and version hash...
|
||||
if package_hash == cached_image.package_hash and version_hash == cached_image.version_hash:
|
||||
image = cached_image
|
||||
break
|
||||
|
||||
elif hash_func == "v1":
|
||||
name = v1_image_name(targets)
|
||||
for cached_image in cached_images:
|
||||
if cached_image.multi_target != "v1":
|
||||
continue
|
||||
|
||||
if name == cached_image.hash:
|
||||
@@ -109,16 +157,17 @@ class CachedMulledContainerResolver(ContainerResolver):
|
||||
|
||||
resolver_type = "cached_mulled"
|
||||
|
||||
def __init__(self, app_info=None, namespace=None):
|
||||
def __init__(self, app_info=None, namespace=None, hash_func="v2"):
|
||||
super(CachedMulledContainerResolver, self).__init__(app_info)
|
||||
self.namespace = namespace
|
||||
self.hash_func = hash_func
|
||||
|
||||
def resolve(self, enabled_container_types, tool_info):
|
||||
if tool_info.requires_galaxy_python_environment:
|
||||
return None
|
||||
|
||||
targets = mulled_targets(tool_info)
|
||||
return cached_container_description(targets, self.namespace)
|
||||
return cached_container_description(targets, self.namespace, hash_func=self.hash_func)
|
||||
|
||||
def __str__(self):
|
||||
return "CachedMulledContainerResolver[namespace=%s]" % self.namespace
|
||||
@@ -130,9 +179,10 @@ class MulledContainerResolver(ContainerResolver):
|
||||
|
||||
resolver_type = "mulled"
|
||||
|
||||
def __init__(self, app_info=None, namespace="biocontainers"):
|
||||
def __init__(self, app_info=None, namespace="biocontainers", hash_func="v2"):
|
||||
super(MulledContainerResolver, self).__init__(app_info)
|
||||
self.namespace = namespace
|
||||
self.hash_func = hash_func
|
||||
|
||||
def resolve(self, enabled_container_types, tool_info):
|
||||
if tool_info.requires_galaxy_python_environment:
|
||||
@@ -162,10 +212,25 @@ class MulledContainerResolver(ContainerResolver):
|
||||
version, build = split_tag(tags[0])
|
||||
name = "%s:%s--%s" % (target.package_name, version, build)
|
||||
else:
|
||||
base_image_name = image_name(targets)
|
||||
tags = mulled_tags_for(self.namespace, base_image_name)
|
||||
if tags:
|
||||
name = "%s:%s" % (base_image_name, tags[0])
|
||||
def tags_if_available(image_name):
|
||||
if ":" in image_name:
|
||||
repo_name, tag_prefix = image_name.split(":", 2)
|
||||
else:
|
||||
repo_name = image_name
|
||||
tag_prefix = None
|
||||
tags = mulled_tags_for(self.namespace, repo_name, tag_prefix=tag_prefix)
|
||||
return tags
|
||||
|
||||
if self.hash_func == "v2":
|
||||
base_image_name = v2_image_name(targets)
|
||||
tags = tags_if_available(base_image_name)
|
||||
if tags:
|
||||
name = "%s:%s" % (base_image_name, tags[0])
|
||||
elif self.hash_func == "v1":
|
||||
base_image_name = v1_image_name(targets)
|
||||
tags = tags_if_available(base_image_name)
|
||||
if tags:
|
||||
name = "%s:%s" % (base_image_name, tags[0])
|
||||
|
||||
if name:
|
||||
return ContainerDescription(
|
||||
@@ -183,12 +248,13 @@ class BuildMulledContainerResolver(ContainerResolver):
|
||||
|
||||
resolver_type = "build_mulled"
|
||||
|
||||
def __init__(self, app_info=None, namespace="local", **kwds):
|
||||
def __init__(self, app_info=None, namespace="local", hash_func="v2", **kwds):
|
||||
super(BuildMulledContainerResolver, self).__init__(app_info)
|
||||
self._involucro_context_kwds = {
|
||||
'involucro_bin': self._get_config_option("involucro_path", None)
|
||||
}
|
||||
self.namespace = namespace
|
||||
self.hash_func = hash_func
|
||||
self._mulled_kwds = {
|
||||
'namespace': namespace,
|
||||
'channels': self._get_config_option("channels", DEFAULT_CHANNELS, prefix="mulled"),
|
||||
@@ -206,9 +272,10 @@ class BuildMulledContainerResolver(ContainerResolver):
|
||||
mull_targets(
|
||||
targets,
|
||||
involucro_context=self._get_involucro_context(),
|
||||
hash_func=self.hash_func,
|
||||
**self._mulled_kwds
|
||||
)
|
||||
return cached_container_description(targets, self.namespace)
|
||||
return cached_container_description(targets, self.namespace, hash_func=self.hash_func)
|
||||
|
||||
def _get_involucro_context(self):
|
||||
involucro_context = InvolucroContext(**self._involucro_context_kwds)
|
||||
|
||||
@@ -84,6 +84,9 @@ inv.task('build')
|
||||
.at('/usr/local')
|
||||
.inImage(destination_base_image)
|
||||
.as(repo)
|
||||
.using(conda_image)
|
||||
.withHostConfig({binds = {"build:/data"}})
|
||||
.run('rm', '-rf', '/data/dist')
|
||||
|
||||
if VAR.TEST_BINDS == '' then
|
||||
inv.task('test')
|
||||
|
||||
@@ -12,6 +12,7 @@ from __future__ import print_function
|
||||
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -25,7 +26,14 @@ except ImportError:
|
||||
from galaxy.tools.deps import commands, installable
|
||||
|
||||
from ._cli import arg_parser
|
||||
from .util import build_target, conda_build_target_str, image_name
|
||||
from .util import (
|
||||
build_target,
|
||||
conda_build_target_str,
|
||||
create_repository,
|
||||
quay_repository,
|
||||
v1_image_name,
|
||||
v2_image_name,
|
||||
)
|
||||
from ..conda_compat import MetaData
|
||||
|
||||
DIRNAME = os.path.dirname(__file__)
|
||||
@@ -37,6 +45,7 @@ DEFAULT_BINDS = ["build/dist:/usr/local/"]
|
||||
DEFAULT_WORKING_DIR = '/source/'
|
||||
IS_OS_X = _platform == "darwin"
|
||||
INVOLUCRO_VERSION = "1.1.2"
|
||||
DEST_BASE_IMAGE = os.environ.get('DEST_BASE_IMAGE', None)
|
||||
|
||||
|
||||
def involucro_link():
|
||||
@@ -111,23 +120,52 @@ def conda_versions(pkg_name, file_name):
|
||||
return ret
|
||||
|
||||
|
||||
class BuildExistsException(Exception):
|
||||
"""Exception indicating mull_targets is skipping an existing build.
|
||||
|
||||
If mull_targets is called with rebuild=False and the target built is already published
|
||||
an instance of this exception is thrown.
|
||||
"""
|
||||
|
||||
|
||||
def mull_targets(
|
||||
targets, involucro_context=None,
|
||||
command="build", channels=DEFAULT_CHANNELS, namespace="mulled",
|
||||
command="build", channels=DEFAULT_CHANNELS, namespace="biocontainers",
|
||||
test='true', test_files=None, image_build=None, name_override=None,
|
||||
repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
|
||||
conda_version=None, verbose=False, binds=DEFAULT_BINDS
|
||||
conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True,
|
||||
oauth_token=None, hash_func="v2",
|
||||
):
|
||||
targets = list(targets)
|
||||
if involucro_context is None:
|
||||
involucro_context = InvolucroContext()
|
||||
|
||||
image_function = v1_image_name if hash_func == "v1" else v2_image_name
|
||||
|
||||
repo_template_kwds = {
|
||||
"namespace": namespace,
|
||||
"image": image_name(targets, image_build=image_build, name_override=name_override)
|
||||
"image": image_function(targets, image_build=image_build or '0', name_override=name_override)
|
||||
}
|
||||
repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
|
||||
|
||||
if not rebuild or "push" in command:
|
||||
repo_name = repo_template_kwds["image"].split(":", 1)[0]
|
||||
repo_data = quay_repository(repo_template_kwds["namespace"], repo_name)
|
||||
if not rebuild:
|
||||
tags = repo_data.get("tags", [])
|
||||
|
||||
target_tag = None
|
||||
if ":" in repo_template_kwds["image"]:
|
||||
image_name_parts = repo_template_kwds["image"].split(":")
|
||||
assert len(image_name_parts) == 2, ": not allowed in image name [%s]" % repo_template_kwds["image"]
|
||||
target_tag = image_name_parts[1]
|
||||
|
||||
if tags and (target_tag is None or target_tag in tags):
|
||||
raise BuildExistsException()
|
||||
if "push" in command and "error_type" in repo_data and oauth_token:
|
||||
# Explicitly create the repository so it can be built as public.
|
||||
create_repository(repo_template_kwds["namespace"], repo_name, oauth_token)
|
||||
|
||||
for channel in channels:
|
||||
if channel.startswith('file://'):
|
||||
bind_path = channel.lstrip('file://')
|
||||
@@ -144,6 +182,9 @@ def mull_targets(
|
||||
'-set', "REPO='%s'" % repo,
|
||||
'-set', "BINDS='%s'" % bind_str,
|
||||
]
|
||||
|
||||
if DEST_BASE_IMAGE:
|
||||
involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % DEST_BASE_IMAGE])
|
||||
if verbose:
|
||||
involucro_args.extend(["-set", "VERBOSE='1'"])
|
||||
if conda_version is not None:
|
||||
@@ -194,7 +235,14 @@ class InvolucroContext(installable.InstallableContext):
|
||||
|
||||
def exec_command(self, involucro_args):
|
||||
cmd = self.build_command(involucro_args)
|
||||
return self.shell_exec(" ".join(cmd))
|
||||
# Create ./build dir manually, otherwise Docker will do it as root
|
||||
os.mkdir('./build')
|
||||
try:
|
||||
res = self.shell_exec(" ".join(cmd))
|
||||
finally:
|
||||
# delete build directory in any case
|
||||
shutil.rmtree('./build')
|
||||
return res
|
||||
|
||||
def is_installed(self):
|
||||
return os.path.exists(self.involucro_bin)
|
||||
@@ -212,9 +260,10 @@ def ensure_installed(involucro_context, auto_init):
|
||||
|
||||
|
||||
def install_involucro(involucro_context=None, to_path=None):
|
||||
to_path = involucro_context.involucro_bin
|
||||
download_cmd = " ".join(commands.download_command(involucro_link(), to=to_path, quote_url=True))
|
||||
full_cmd = "%s && chmod +x %s" % (download_cmd, to_path)
|
||||
install_path = os.path.abspath(involucro_context.involucro_bin)
|
||||
involucro_context.involucro_bin = install_path
|
||||
download_cmd = " ".join(commands.download_command(involucro_link(), to=install_path, quote_url=True))
|
||||
full_cmd = "%s && chmod +x %s" % (download_cmd, install_path)
|
||||
return involucro_context.shell_exec(full_cmd)
|
||||
|
||||
|
||||
@@ -222,13 +271,11 @@ def add_build_arguments(parser):
|
||||
"""Base arguments describing how to 'mull'."""
|
||||
parser.add_argument('--involucro-path', dest="involucro_path", default=None,
|
||||
help="Path to involucro (if not set will look in working directory and on PATH).")
|
||||
parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true",
|
||||
help="Rebuild package even if already published.")
|
||||
parser.add_argument('--dry-run', dest='dry_run', action="store_true",
|
||||
help='Just print commands instead of executing them.')
|
||||
parser.add_argument('--verbose', dest='verbose', action="store_true",
|
||||
help='Cause process to be verbose.')
|
||||
parser.add_argument('-n', '--namespace', dest='namespace', default="mulled",
|
||||
parser.add_argument('-n', '--namespace', dest='namespace', default="biocontainers",
|
||||
help='quay.io namespace.')
|
||||
parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE,
|
||||
help='Docker repository target for publication (only quay.io or compat. API is currently supported).')
|
||||
@@ -238,6 +285,10 @@ def add_build_arguments(parser):
|
||||
help='Dependent conda channels.')
|
||||
parser.add_argument('--conda-version', dest="conda_version", default=None,
|
||||
help="Change to specified version of Conda before installing packages.")
|
||||
parser.add_argument('--oauth-token', dest="oauth_token", default=None,
|
||||
help="If set, use this token when communicating with quay.io API.")
|
||||
parser.add_argument('--check-published', dest="rebuild", action='store_false')
|
||||
parser.add_argument('--hash', dest="hash", choices=["v1", "v2"], default="v2")
|
||||
|
||||
|
||||
def add_single_image_arguments(parser):
|
||||
@@ -289,6 +340,12 @@ def args_to_mull_targets_kwds(args):
|
||||
kwds["repository_template"] = args.repository_template
|
||||
if hasattr(args, "conda_version"):
|
||||
kwds["conda_version"] = args.conda_version
|
||||
if hasattr(args, "oauth_token"):
|
||||
kwds["oauth_token"] = args.oauth_token
|
||||
if hasattr(args, "rebuild"):
|
||||
kwds["rebuild"] = args.rebuild
|
||||
if hasattr(args, "hash"):
|
||||
kwds["hash_func"] = args.hash
|
||||
|
||||
kwds["involucro_context"] = context_from_args(args)
|
||||
|
||||
|
||||
@@ -87,6 +87,8 @@ def add_channel_arguments(parser):
|
||||
parser.add_argument('--diff-hours', dest='diff_hours', default="25",
|
||||
help='If finding all recently changed recipes, use this number of hours.')
|
||||
parser.add_argument('--recipes-dir', dest="recipes_dir", default="./bioconda-recipes")
|
||||
parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true",
|
||||
help="Rebuild package even if already published.")
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
|
||||
@@ -20,6 +20,7 @@ from ._cli import arg_parser
|
||||
from .mulled_build import (
|
||||
add_build_arguments,
|
||||
args_to_mull_targets_kwds,
|
||||
BuildExistsException,
|
||||
mull_targets,
|
||||
target_str_to_targets,
|
||||
)
|
||||
@@ -33,8 +34,19 @@ def main(argv=None):
|
||||
parser.add_argument('files', metavar="FILES", default=".",
|
||||
help="Path to directory (or single file) of TSV files describing composite recipes.")
|
||||
args = parser.parse_args()
|
||||
for targets in generate_targets(args.files):
|
||||
mull_targets(targets, **args_to_mull_targets_kwds(args))
|
||||
for (targets, image_build, name_override) in generate_targets(args.files):
|
||||
if not image_build and len(targets) > 1:
|
||||
# Specify an explict tag in this case.
|
||||
image_build = "0"
|
||||
try:
|
||||
mull_targets(
|
||||
targets,
|
||||
image_build=image_build,
|
||||
name_override=name_override,
|
||||
**args_to_mull_targets_kwds(args)
|
||||
)
|
||||
except BuildExistsException:
|
||||
continue
|
||||
|
||||
|
||||
def generate_targets(target_source):
|
||||
@@ -59,14 +71,14 @@ def generate_targets(target_source):
|
||||
|
||||
def line_to_targets(line_str):
|
||||
line = _parse_line(line_str)
|
||||
return target_str_to_targets(line)
|
||||
return (target_str_to_targets(line.targets), line.image_build, line.name_override)
|
||||
|
||||
|
||||
_Line = collections.namedtuple("_Line", ["targets", "image_build", "name_override"])
|
||||
|
||||
|
||||
def _parse_line(line_str):
|
||||
line_parts = line_str.split(" ")
|
||||
line_parts = line_str.split("\t")
|
||||
assert len(line_parts) < 3, "Too many fields in line [%s], expect at most 3 - targets, image build number, and name override." % line_str
|
||||
line_parts += [None] * (3 - len(line_parts))
|
||||
return _Line(*line_parts)
|
||||
|
||||
Regular → Executable
+2
-2
@@ -112,8 +112,8 @@ class QuaySearch():
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description='Searches in a given quay organization for a repository')
|
||||
parser.add_argument('-o', '--organization', dest='organization_string', default="mulled",
|
||||
help='Change organization. Default is mulled.')
|
||||
parser.add_argument('-o', '--organization', dest='organization_string', default="biocontainers",
|
||||
help='Change organization. Default is biocontainers.')
|
||||
parser.add_argument('--non-strict', dest='non_strict', action="store_true",
|
||||
help='Autocorrection of typos activated. Lists more results but can be confusing.\
|
||||
For too many queries quay.io blocks the request and the results can be incomplete.')
|
||||
|
||||
@@ -12,16 +12,22 @@ except ImportError:
|
||||
requests = None
|
||||
|
||||
|
||||
def create_repository(namespace, repo_name, oauth_token):
|
||||
assert oauth_token
|
||||
headers = {'Authorization': 'Bearer %s' % oauth_token}
|
||||
data = {
|
||||
"repository": repo_name,
|
||||
"namespace": namespace,
|
||||
"description": "",
|
||||
"visibility": "public",
|
||||
}
|
||||
requests.post("https://quay.io/api/v1/repository", json=data, headers=headers)
|
||||
|
||||
|
||||
def quay_versions(namespace, pkg_name):
|
||||
"""Get all version tags for a Docker image stored on quay.io for supplied package name."""
|
||||
if requests is None:
|
||||
raise Exception("requets library is unavailable, functionality not available.")
|
||||
data = quay_repository(namespace, pkg_name)
|
||||
|
||||
assert namespace is not None
|
||||
assert pkg_name is not None
|
||||
url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name)
|
||||
response = requests.get(url, timeout=None)
|
||||
data = response.json()
|
||||
if 'error_type' in data and data['error_type'] == "invalid_token":
|
||||
return []
|
||||
|
||||
@@ -31,12 +37,26 @@ def quay_versions(namespace, pkg_name):
|
||||
return [tag for tag in data['tags'] if tag != 'latest']
|
||||
|
||||
|
||||
def mulled_tags_for(namespace, image):
|
||||
def quay_repository(namespace, pkg_name):
|
||||
if requests is None:
|
||||
raise Exception("requets library is unavailable, functionality not available.")
|
||||
|
||||
assert namespace is not None
|
||||
assert pkg_name is not None
|
||||
url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name)
|
||||
response = requests.get(url, timeout=None)
|
||||
data = response.json()
|
||||
return data
|
||||
|
||||
|
||||
def mulled_tags_for(namespace, image, tag_prefix=None):
|
||||
"""Fetch remote tags available for supplied image name.
|
||||
|
||||
The result will be sorted so newest tags are first.
|
||||
"""
|
||||
tags = quay_versions(namespace, image)
|
||||
if tag_prefix is not None:
|
||||
tags = [t for t in tags if t.startswith(tag_prefix)]
|
||||
tags = version_sorted(tags)
|
||||
return tags
|
||||
|
||||
@@ -77,25 +97,49 @@ def conda_build_target_str(target):
|
||||
return rval
|
||||
|
||||
|
||||
def image_name(targets, image_build=None, name_override=None):
|
||||
def _simple_image_name(targets, image_build=None):
|
||||
target = targets[0]
|
||||
suffix = ""
|
||||
if target.version is not None:
|
||||
if image_build is not None:
|
||||
print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.")
|
||||
suffix = image_build
|
||||
else:
|
||||
suffix += ":%s" % target.version
|
||||
build = target.build
|
||||
if build is not None:
|
||||
suffix += "--%s" % build
|
||||
return "%s%s" % (target.package_name, suffix)
|
||||
|
||||
|
||||
def v1_image_name(targets, image_build=None, name_override=None):
|
||||
"""Generate mulled hash version 1 container identifier for supplied arguments.
|
||||
|
||||
If a single target is specified, simply use the supplied name and version as
|
||||
the repository name and tag respectively. If multiple targets are supplied,
|
||||
hash the package names and versions together as the repository name. For mulled
|
||||
version 1 containers the image build is the repository tag (if supplied).
|
||||
|
||||
>>> single_targets = [build_target("samtools", version="1.3.1")]
|
||||
>>> v1_image_name(single_targets)
|
||||
'samtools:1.3.1'
|
||||
>>> multi_targets = [build_target("samtools", version="1.3.1"), build_target("bwa", version="0.7.13")]
|
||||
>>> v1_image_name(multi_targets)
|
||||
'mulled-v1-b06ecbd9141f0dbbc0c287375fc0813adfcbdfbd'
|
||||
>>> multi_targets_on_versionless = [build_target("samtools", version="1.3.1"), build_target("bwa")]
|
||||
>>> v1_image_name(multi_targets_on_versionless)
|
||||
'mulled-v1-bda945976caa5734347fbf7f35066d9f58519e0c'
|
||||
>>> multi_targets_versionless = [build_target("samtools"), build_target("bwa")]
|
||||
>>> v1_image_name(multi_targets_versionless)
|
||||
'mulled-v1-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40'
|
||||
"""
|
||||
if name_override is not None:
|
||||
print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.")
|
||||
return name_override
|
||||
|
||||
targets = list(targets)
|
||||
if len(targets) == 1:
|
||||
target = targets[0]
|
||||
suffix = ""
|
||||
if target.version is not None:
|
||||
if image_build is not None:
|
||||
print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.")
|
||||
suffix = image_build
|
||||
else:
|
||||
suffix += ":%s" % target.version
|
||||
build = target.build
|
||||
if build is not None:
|
||||
suffix += "--%s" % build
|
||||
return "%s%s" % (target.package_name, suffix)
|
||||
return _simple_image_name(targets, image_build=image_build)
|
||||
else:
|
||||
targets_order = sorted(targets, key=lambda t: t.package_name)
|
||||
requirements_buffer = "\n".join(map(conda_build_target_str, targets_order))
|
||||
@@ -105,6 +149,66 @@ def image_name(targets, image_build=None, name_override=None):
|
||||
return "mulled-v1-%s%s" % (m.hexdigest(), suffix)
|
||||
|
||||
|
||||
def v2_image_name(targets, image_build=None, name_override=None):
|
||||
"""Generate mulled hash version 2 container identifier for supplied arguments.
|
||||
|
||||
If a single target is specified, simply use the supplied name and version as
|
||||
the repository name and tag respectively. If multiple targets are supplied,
|
||||
hash the package names as the repository name and hash the package versions (if set)
|
||||
as the tag.
|
||||
|
||||
>>> single_targets = [build_target("samtools", version="1.3.1")]
|
||||
>>> v2_image_name(single_targets)
|
||||
'samtools:1.3.1'
|
||||
>>> multi_targets = [build_target("samtools", version="1.3.1"), build_target("bwa", version="0.7.13")]
|
||||
>>> v2_image_name(multi_targets)
|
||||
'mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:4d0535c94ef45be8459f429561f0894c3fe0ebcf'
|
||||
>>> multi_targets_on_versionless = [build_target("samtools", version="1.3.1"), build_target("bwa")]
|
||||
>>> v2_image_name(multi_targets_on_versionless)
|
||||
'mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40:b0c847e4fb89c343b04036e33b2daa19c4152cf5'
|
||||
>>> multi_targets_versionless = [build_target("samtools"), build_target("bwa")]
|
||||
>>> v2_image_name(multi_targets_versionless)
|
||||
'mulled-v2-fe8faa35dbf6dc65a0f7f5d4ea12e31a79f73e40'
|
||||
"""
|
||||
if name_override is not None:
|
||||
print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.")
|
||||
return name_override
|
||||
|
||||
targets = list(targets)
|
||||
if len(targets) == 1:
|
||||
return _simple_image_name(targets, image_build=image_build)
|
||||
else:
|
||||
targets_order = sorted(targets, key=lambda t: t.package_name)
|
||||
package_name_buffer = "\n".join(map(lambda t: t.package_name, targets_order))
|
||||
package_hash = hashlib.sha1()
|
||||
package_hash.update(package_name_buffer.encode())
|
||||
|
||||
versions = map(lambda t: t.version, targets_order)
|
||||
if any(versions):
|
||||
# Only hash versions if at least one package has versions...
|
||||
version_name_buffer = "\n".join(map(lambda t: t.version or "null", targets_order))
|
||||
version_hash = hashlib.sha1()
|
||||
version_hash.update(version_name_buffer.encode())
|
||||
version_hash_str = version_hash.hexdigest()
|
||||
else:
|
||||
version_hash_str = ""
|
||||
|
||||
if not image_build:
|
||||
build_suffix = ""
|
||||
elif version_hash_str:
|
||||
# tagged verson is <version_hash>-<build>
|
||||
build_suffix = "-%s" % image_build
|
||||
else:
|
||||
# tagged version is simply the build
|
||||
build_suffix = image_build
|
||||
suffix = ""
|
||||
if version_hash_str or build_suffix:
|
||||
suffix = ":%s%s" % (version_hash_str, build_suffix)
|
||||
return "mulled-v2-%s%s" % (package_hash.hexdigest(), suffix)
|
||||
|
||||
|
||||
image_name = v1_image_name # deprecated
|
||||
|
||||
__all__ = (
|
||||
"build_target",
|
||||
"conda_build_target_str",
|
||||
@@ -113,5 +217,7 @@ __all__ = (
|
||||
"quay_versions",
|
||||
"split_tag",
|
||||
"Target",
|
||||
"v1_image_name",
|
||||
"v2_image_name",
|
||||
"version_sorted",
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user