diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample
index 72876dfc65c..f60ce30d917 100644
--- a/config/galaxy.ini.sample
+++ b/config/galaxy.ini.sample
@@ -228,6 +228,37 @@ paste.app_factory = galaxy.web.buildapp:app_factory
# than the watchdog default.
#watch_tools = False
+# Enable Galaxy to fetch Docker containers registered with quay.io generated
+# from tool requirements resolved through conda. These containers (when
+# available) have been generated using mulled - https://github.com/mulled.
+# These containers are highly beta and availablity will vary by tool.
+# This option will additionally only be used for job destinations with
+# Docker enabled.
+#enable_beta_mulled_containers = False
+
+# Container resolvers configuration (beta). Setup a file describing container
+# resolvers to use when discovering containers for Galaxy. If this is set to
+# None, the default containers loaded is determined by
+# enable_beta_mulled_containers.
+#containers_resolvers_config_file = None
+
+# Container resolvers configuration (beta). Setup a file describing container
+# resolvers to use when discovering containers for Galaxy. If this is set to
+# None, the default containers loaded is determined by
+# enable_beta_mulled_containers.
+#containers_resolvers_config_file = None
+
+# involucro is a tool used to build Docker containers for tools from Conda
+# dependencies referenced in tools as `requirement`s. The following path is
+# the location of involucro on the Galaxy host. This is ignored if the relevant
+# container resolver isn't enabled, and will install on demand unless
+# involucro_auto_init is set to False.
+#involucro_path = database/dependencies/involucro
+
+# Install involucro as needed to build Docker containers for tools. Ignored if
+# relevant container resolver is not used.
+#involucro_auto_init = True
+
# Enable automatic polling of relative tool sheds to see if any updates
# are available for installed repositories. Ideally only one Galaxy
# server process should be able to check for repository updates. The
diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py
index 9c590c4c747..86b9cc82785 100644
--- a/lib/galaxy/config.py
+++ b/lib/galaxy/config.py
@@ -226,6 +226,7 @@ class Configuration( object ):
self.local_task_queue_workers = int(kwargs.get("local_task_queue_workers", 2))
self.tool_submission_burst_threads = int( kwargs.get( 'tool_submission_burst_threads', '1' ) )
self.tool_submission_burst_at = int( kwargs.get( 'tool_submission_burst_at', '10' ) )
+
# Enable new interface for API installations from TS.
# Admin menu will list both if enabled.
self.enable_beta_ts_api_install = string_as_bool( kwargs.get( 'enable_beta_ts_api_install', 'False' ) )
@@ -323,6 +324,19 @@ class Configuration( object ):
else:
self.tool_dependency_dir = None
self.use_tool_dependencies = os.path.exists(self.dependency_resolvers_config_file)
+
+ self.enable_beta_mulled_containers = string_as_bool( kwargs.get( 'enable_beta_mulled_containers', 'False' ) )
+ containers_resolvers_config_file = kwargs.get( 'containers_resolvers_config_file', None )
+ if containers_resolvers_config_file:
+ containers_resolvers_config_file = resolve_path(containers_resolvers_config_file, self.root)
+ self.containers_resolvers_config_file = containers_resolvers_config_file
+
+ involucro_path = kwargs.get('involucro_path', None)
+ if involucro_path is None:
+ involucro_path = os.path.join(tool_dependency_dir, "involucro")
+ self.involucro_path = resolve_path(involucro_path, self.root)
+ self.involucro_auto_init = string_as_bool(kwargs.get( 'involucro_auto_init', True))
+
# Configuration options for taking advantage of nginx features
self.upstream_gzip = string_as_bool( kwargs.get( 'upstream_gzip', False ) )
self.apache_xsendfile = string_as_bool( kwargs.get( 'apache_xsendfile', False ) )
@@ -846,7 +860,11 @@ class ConfiguresGalaxyMixin:
default_file_path=file_path,
outputs_to_working_directory=self.config.outputs_to_working_directory,
container_image_cache_path=self.config.container_image_cache_path,
- library_import_dir=self.config.library_import_dir
+ library_import_dir=self.config.library_import_dir,
+ enable_beta_mulled_containers=self.config.enable_beta_mulled_containers,
+ containers_resolvers_config_file=self.config.containers_resolvers_config_file,
+ involucro_path=self.config.involucro_path,
+ involucro_auto_init=self.config.involucro_auto_init,
)
self.container_finder = containers.ContainerFinder(app_info)
diff --git a/lib/galaxy/tools/deps/__init__.py b/lib/galaxy/tools/deps/__init__.py
index 763732e4dc7..ccf5bb25753 100644
--- a/lib/galaxy/tools/deps/__init__.py
+++ b/lib/galaxy/tools/deps/__init__.py
@@ -8,7 +8,7 @@ import os.path
from galaxy.util import plugin_config
from .resolvers import NullDependency
-from .resolvers.conda import CondaDependencyResolver
+from .resolvers.conda import CondaDependencyResolver, DEFAULT_ENSURE_CHANNELS
from .resolvers.galaxy_packages import GalaxyPackageDependencyResolver
from .resolvers.tool_shed_packages import ToolShedPackageDependencyResolver
@@ -21,7 +21,7 @@ EXTRA_CONFIG_KWDS = {
'conda_prefix': None,
'conda_exec': None,
'conda_debug': None,
- 'conda_ensure_channels': 'r,bioconda,iuc',
+ 'conda_ensure_channels': DEFAULT_ENSURE_CHANNELS,
'conda_auto_install': False,
'conda_auto_init': False,
'conda_copy_dependencies': False,
diff --git a/lib/galaxy/tools/deps/conda_compat.py b/lib/galaxy/tools/deps/conda_compat.py
new file mode 100644
index 00000000000..1443378f9f5
--- /dev/null
+++ b/lib/galaxy/tools/deps/conda_compat.py
@@ -0,0 +1,112 @@
+"""Compat. layer with conda_build/verify if Galaxy/galaxy-lib not installed through conda.
+
+In general there are utilities available for Conda building and parsing that are high-quality
+and should be utilized when available but that are only available in conda channels and not in
+PyPI. This module serves as a PyPI capable interface to these utilities.
+"""
+import collections
+import os
+import yaml
+
+
+try:
+ from conda_build.metadata import MetaData
+except ImportError:
+ MetaData = None
+
+try:
+ from anaconda_verify.recipe import render_jinja2, parse
+except ImportError:
+ render_jinja2 = None
+ parse = None
+
+
+class _Memoized(object):
+
+ def __init__(self, func):
+ self.func = func
+ self.cache = {}
+
+ def __call__(self, *args):
+ if not isinstance(args, collections.Hashable):
+ # uncacheable. a list, for instance.
+ # better to not cache than blow up.
+ return self.func(*args)
+ if args in self.cache:
+ return self.cache[args]
+ else:
+ value = self.func(*args)
+ self.cache[args] = value
+ return value
+
+
+def _parse(data, cfg):
+ """Parse metadata YAML."""
+ assert cfg is None, "Conda utilities for evaluating cfg are not available."
+ return dict(yamlize(data))
+
+
+def _render_jinja2(recipe_dir):
+ """Evaluate Conda recipe as a jinja template."""
+ try:
+ import jinja2
+ except ImportError:
+ raise Exception("Failed to import jinja2 for evaluating Conda recipe templates.")
+
+ loaders = [jinja2.FileSystemLoader(recipe_dir)]
+ env = jinja2.Environment(loader=jinja2.ChoiceLoader(loaders))
+ template = env.get_or_select_template('meta.yaml')
+ return template.render(environment=env)
+
+
+@_Memoized
+def yamlize(data):
+ res = yaml.load(data)
+ # ensure the result is a dict
+ if res is None:
+ res = {}
+ return res
+
+
+if render_jinja2 is None:
+ render_jinja2 = _render_jinja2
+
+if parse is None:
+ parse = _parse
+
+
+def raw_metadata(recipe_dir):
+ """Evaluate Conda template if needed and return raw metadata for supplied recipe directory."""
+ meta_path = os.path.join(recipe_dir, 'meta.yaml')
+ with open(meta_path, 'rb') as fi:
+ data = fi.read()
+ if b'{{' in data:
+ data = render_jinja2(recipe_dir)
+ meta = parse(data, None)
+ return meta
+
+
+class _MetaData(object):
+
+ def __init__(self, input_dir):
+ self.meta = raw_metadata(input_dir)
+
+ def get_value(self, field, default=None):
+ """Get nested field value or supplied default is not present."""
+ section, key = field.split('/')
+ submeta = self.meta.get(section)
+ if submeta is None:
+ submeta = {}
+ res = submeta.get(key)
+ if res is None:
+ res = default
+ return res
+
+
+if MetaData is None:
+ MetaData = _MetaData
+
+__all__ = [
+ "MetaData",
+ "raw_metadata",
+]
diff --git a/lib/galaxy/tools/deps/conda_util.py b/lib/galaxy/tools/deps/conda_util.py
index dc91f9f967a..117acca4389 100644
--- a/lib/galaxy/tools/deps/conda_util.py
+++ b/lib/galaxy/tools/deps/conda_util.py
@@ -7,12 +7,14 @@ import re
import shutil
import tempfile
+from distutils.version import LooseVersion
from sys import platform as _platform
import six
import yaml
from ..deps import commands
+from ..deps import installable
log = logging.getLogger(__name__)
@@ -47,7 +49,8 @@ def find_conda_prefix(conda_prefix=None):
return conda_prefix
-class CondaContext(object):
+class CondaContext(installable.InstallableContext):
+ installable_description = "Conda"
def __init__(self, conda_prefix=None, conda_exec=None,
shell_exec=None, debug=False, ensure_channels='',
@@ -230,6 +233,16 @@ class CondaContext(object):
def activate(self):
return self._bin("activate")
+ def is_installed(self):
+ return self.is_conda_installed()
+
+ def can_install(self):
+ return self.can_install_conda()
+
+ @property
+ def parent_path(self):
+ return os.path.dirname(os.path.abspath(self.conda_prefix))
+
def _bin(self, name):
return os.path.join(self.conda_prefix, "bin", name)
@@ -273,6 +286,8 @@ class CondaTarget(object):
return "CondaTarget[%s]" % attributes
+ __repr__ = __str__
+
@property
def package_specifier(self):
""" Return a package specifier as consumed by conda install/create.
@@ -293,6 +308,17 @@ class CondaTarget(object):
else:
return "__%s@_uv_" % (self.package)
+ def __hash__(self):
+ return hash((self.package, self.version, self.channel))
+
+ def __eq__(self, other):
+ if isinstance(other, self.__class__):
+ return (self.package, self.version, self.channel) == (other.package, other.version, other.channel)
+ return False
+
+ def __ne__(self, other):
+ return not(self == other)
+
def hash_conda_packages(conda_packages, conda_target=None):
""" Produce a unique hash on supplied packages.
@@ -339,23 +365,55 @@ def cleanup_failed_install(conda_target, conda_context=None):
conda_context.exec_remove([conda_target.install_environment])
-def is_target_available(conda_target, conda_context=None):
- """ Checks if a specified target is available for installation.
- If the package name exists return "True". If in addition the version matches exactly return "exact".
- Otherwise return False.
+def best_search_result(conda_target, conda_context=None, channels_override=None):
+ """Find best "conda search" result for specified target.
+
+ Return ``None`` if no results match.
"""
conda_context = _ensure_conda_context(conda_context)
- conda_context.ensure_channels_configured()
- search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json", conda_target.package]
+ if not channels_override:
+ conda_context.ensure_channels_configured()
+
+ search_cmd = [conda_context.conda_exec, "search", "--full-name", "--json"]
+ if channels_override:
+ search_cmd.append("--override-channels")
+ for channel in channels_override:
+ search_cmd.extend(["--channel", channel])
+ search_cmd.append(conda_target.package)
res = commands.execute(search_cmd)
hits = json.loads(res).get(conda_target.package, [])
+ hits = sorted(hits, key=lambda hit: LooseVersion(hit['version']), reverse=True)
- if len(hits) > 0:
- if conda_target.version:
- for hit in hits:
- if hit['version'] == conda_target.version:
- return 'exact'
- return True
+ if len(hits) == 0:
+ return (None, None)
+
+ best_result = (hits[0], False)
+
+ for hit in hits:
+ if is_search_hit_exact(conda_target, hit):
+ best_result = (hit, True)
+ break
+
+ return best_result
+
+
+def is_search_hit_exact(conda_target, search_hit):
+ target_version = conda_target.version
+ # It'd be nice to make request verson of 1.0 match available
+ # version of 1.0.3 or something like that.
+ return not target_version or search_hit['version'] == target_version
+
+
+def is_target_available(conda_target, conda_context=None, channels_override=None):
+ """Check if a specified target is available for installation.
+
+ If the package name exists return ``True`` (the ``bool``). If in addition
+ the version matches exactly return "exact" (a string). Otherwise return
+ ``False``.
+ """
+ (best_hit, exact) = best_search_result(conda_target, conda_context, channels_override)
+ if best_hit:
+ return 'exact' if exact else True
else:
return False
diff --git a/lib/galaxy/tools/deps/container_resolvers/__init__.py b/lib/galaxy/tools/deps/container_resolvers/__init__.py
new file mode 100644
index 00000000000..c478bcce3a2
--- /dev/null
+++ b/lib/galaxy/tools/deps/container_resolvers/__init__.py
@@ -0,0 +1,50 @@
+"""The module defines the abstract interface for resolving container images for tool execution."""
+from abc import (
+ ABCMeta,
+ abstractmethod,
+ abstractproperty,
+)
+
+from galaxy.util.dictifiable import Dictifiable
+
+
+class ContainerResolver(Dictifiable, object):
+ """Description of a technique for resolving container images for tool execution."""
+
+ # Keys for dictification.
+ dict_collection_visible_keys = ['resolver_type']
+
+ __metaclass__ = ABCMeta
+
+ def __init__(self, app_info=None, **kwds):
+ """Default initializer for ``ContainerResolver`` subclasses."""
+ self.app_info = app_info
+ self.resolver_kwds = kwds
+
+ def _get_config_option(self, key, default=None, config_prefix=None, **kwds):
+ """Look in resolver-specific settings for option and then fallback to
+ global settings.
+ """
+ global_key = "%s_%s" % (config_prefix, key)
+ if key in kwds:
+ return kwds.get(key)
+ elif self.app_info and hasattr(self.app_info, global_key):
+ return getattr(self.app_info, global_key)
+ else:
+ return default
+
+ @abstractmethod
+ def resolve(self, tool_info):
+ """Find a container matching all supplied requirements for tool.
+
+ The supplied argument is a :class:`galaxy.tools.deps.containers.ToolInfo` description
+ of the tool and its requirements.
+ """
+
+ @abstractproperty
+ def resolver_type(self):
+ """Short label for the type of container resolution."""
+
+ def _container_type_enabled(self, container_description, enabled_container_types):
+ """Return a boolean indicating if the specified container type is enabled."""
+ return container_description.type in enabled_container_types
diff --git a/lib/galaxy/tools/deps/container_resolvers/explicit.py b/lib/galaxy/tools/deps/container_resolvers/explicit.py
new file mode 100644
index 00000000000..851f0c1f99c
--- /dev/null
+++ b/lib/galaxy/tools/deps/container_resolvers/explicit.py
@@ -0,0 +1,26 @@
+"""This module describes the :class:`ExplicitContainerResolver` ContainerResolver plugin."""
+import logging
+
+from ..container_resolvers import (
+ ContainerResolver,
+)
+
+log = logging.getLogger(__name__)
+
+
+class ExplicitContainerResolver(ContainerResolver):
+ """Find explicit containers referenced in the tool description (e.g. tool XML file) if present."""
+
+ resolver_type = "explicit"
+
+ def resolve(self, enabled_container_types, tool_info):
+ """Find a container explicitly mentioned in tool description.
+
+ This ignores the tool requirements and assumes the tool author crafted
+ a correct container.
+ """
+ for container_description in tool_info.container_descriptions:
+ if self._container_type_enabled(container_description):
+ return True
+
+ return False
diff --git a/lib/galaxy/tools/deps/container_resolvers/mulled.py b/lib/galaxy/tools/deps/container_resolvers/mulled.py
new file mode 100644
index 00000000000..2b2701270ef
--- /dev/null
+++ b/lib/galaxy/tools/deps/container_resolvers/mulled.py
@@ -0,0 +1,207 @@
+"""This module describes the :class:`MulledContainerResolver` ContainerResolver plugin."""
+
+import collections
+import logging
+
+from ..container_resolvers import (
+ ContainerResolver,
+)
+from ..docker_util import build_docker_images_command
+from ..mulled.mulled_build import (
+ check_output,
+ DEFAULT_CHANNELS,
+ ensure_installed,
+ InvolucroContext,
+ mull_targets,
+)
+from ..mulled.mulled_build_tool import requirements_to_mulled_targets
+from ..mulled.util import (
+ image_name,
+ mulled_tags_for,
+ split_tag,
+)
+from ..requirements import ContainerDescription
+
+log = logging.getLogger(__name__)
+
+
+CachedMulledImageSingleTarget = collections.namedtuple("CachedMulledImageSingleTarget", ["package_name", "version", "build", "image_identifier"])
+CachedMulledImageMultiTarget = collections.namedtuple("CachedMulledImageMultiTarget", ["hash", "image_identifier"])
+
+CachedMulledImageSingleTarget.multi_target = False
+CachedMulledImageMultiTarget.multi_target = True
+
+
+def list_cached_mulled_images(namespace=None):
+ command = build_docker_images_command(truncate=True, sudo_docker=False)
+ command = "%s | tail -n +2 | tr -s ' ' | cut -d' ' -f1,2" % command
+ images_and_versions = check_output(command)
+ name_filter = get_filter(namespace)
+
+ def output_line_to_image(line):
+ image_name, version = line.split(" ", 1)
+ identifier = "%s:%s" % (image_name, version)
+ url, namespace, package_description = image_name.split("/")
+
+ if package_description.startswith("mulled-v1-"):
+ hash = package_description
+ image = CachedMulledImageMultiTarget(hash, identifier)
+ else:
+ build = None
+ if not version or version == "latest":
+ version = None
+
+ if version and "--" in version:
+ version, build = split_tag(version)
+
+ image = CachedMulledImageSingleTarget(image_name, version, build, identifier)
+
+ return image
+
+ return map(output_line_to_image, filter(name_filter, images_and_versions.splitlines()))
+
+
+def get_filter(namespace):
+ prefix = "quay.io/" if namespace is None else "quay.io/%s" % namespace
+ return lambda name: name.startswith(prefix) and name.count("/") == 2
+
+
+def cached_container_description(targets, namespace):
+ if len(targets) == 0:
+ return None
+
+ cached_images = list_cached_mulled_images(namespace)
+ image = None
+ if len(targets) == 1:
+ target = targets[0]
+ for cached_image in cached_images:
+ if cached_image.multi_target:
+ continue
+ if not cached_image.package_name == target.package_name:
+ continue
+ if not target.version or target.version == cached_image.version:
+ image = cached_image
+ break
+ else:
+ name = image_name(targets)
+ for cached_image in cached_images:
+ if not cached_image.multi_target:
+ continue
+
+ if name == cached_image.hash:
+ image = cached_image
+ break
+
+ container = None
+ if image:
+ container = ContainerDescription(
+ image.image_identifier,
+ type="docker",
+ )
+
+ return container
+
+
+class CachedMulledContainerResolver(ContainerResolver):
+
+ resolver_type = "cached_mulled"
+
+ def __init__(self, app_info=None, namespace=None):
+ super(CachedMulledContainerResolver, self).__init__(app_info)
+ self.namespace = namespace
+
+ def resolve(self, enabled_container_types, tool_info):
+ targets = mulled_targets(tool_info)
+ return cached_container_description(targets, self.namespace)
+
+
+class MulledContainerResolver(ContainerResolver):
+ """Look for mulled images matching tool dependencies."""
+
+ resolver_type = "mulled"
+
+ def __init__(self, app_info=None, namespace="mulled"):
+ super(MulledContainerResolver, self).__init__(app_info)
+ self.namespace = namespace
+
+ def resolve(self, enabled_container_types, tool_info):
+ targets = mulled_targets(tool_info)
+ if len(targets) == 0:
+ return None
+
+ name = None
+
+ if len(targets) == 1:
+ target = targets[0]
+ target_version = target.version
+ tags = mulled_tags_for(self.namespace, target.package_name)
+
+ if not tags:
+ return None
+
+ if target_version:
+ for tag in tags:
+ version, build = split_tag(tag)
+ if version == target_version:
+ name = "%s:%s--%s" % (target.package_name, version, build)
+ break
+ else:
+ version, build = split_tag(tags[0])
+ name = "%s:%s--%s" % (target.package_name, version, build)
+ else:
+ base_image_name = image_name(targets)
+ tags = mulled_tags_for(self.namespace, base_image_name)
+ if tags:
+ name = "%s:%s" % (base_image_name, tags[0])
+
+ if name:
+ return ContainerDescription(
+ "quay.io/%s/%s" % (self.namespace, name),
+ type="docker",
+ )
+
+
+class BuildMulledContainerResolver(ContainerResolver):
+ """Look for mulled images matching tool dependencies."""
+
+ resolver_type = "build_mulled"
+
+ def __init__(self, app_info=None, namespace="local", **kwds):
+ super(BuildMulledContainerResolver, self).__init__(app_info)
+ self._involucro_context_kwds = {
+ 'involucro_bin': self._get_config_option("involucro_path", None)
+ }
+ self.namespace = namespace
+ self._mulled_kwds = {
+ 'namespace': namespace,
+ 'channels': self._get_config_option("channels", DEFAULT_CHANNELS, prefix="mulled"),
+ }
+ self.auto_init = self._get_config_option("auto_init", DEFAULT_CHANNELS, prefix="involucro")
+
+ def resolve(self, enabled_container_types, tool_info):
+ targets = mulled_targets(tool_info)
+ if len(targets) == 0:
+ return None
+
+ mull_targets(
+ targets,
+ involucro_context=self._get_involucro_context(),
+ **self._mulled_kwds
+ )
+ return cached_container_description(targets, self.namespace)
+
+ def _get_involucro_context(self):
+ involucro_context = InvolucroContext(**self._involucro_context_kwds)
+ self.enabled = ensure_installed(involucro_context, self.auto_init)
+ return involucro_context
+
+
+def mulled_targets(tool_info):
+ return requirements_to_mulled_targets(tool_info.requirements)
+
+
+__all__ = [
+ "CachedMulledContainerResolver",
+ "MulledContainerResolver",
+ "BuildMulledContainerResolver",
+]
diff --git a/lib/galaxy/tools/deps/containers.py b/lib/galaxy/tools/deps/containers.py
index aadf73ef482..fff1989986b 100644
--- a/lib/galaxy/tools/deps/containers.py
+++ b/lib/galaxy/tools/deps/containers.py
@@ -10,14 +10,23 @@ from abc import (
import six
from galaxy.util import asbool
+from galaxy.util import plugin_config
+from .container_resolvers.explicit import ExplicitContainerResolver
+from .container_resolvers.mulled import (
+ BuildMulledContainerResolver,
+ CachedMulledContainerResolver,
+ MulledContainerResolver,
+)
from .requirements import ContainerDescription
from .requirements import DEFAULT_CONTAINER_RESOLVE_DEPENDENCIES, DEFAULT_CONTAINER_SHELL
from ..deps import docker_util
log = logging.getLogger(__name__)
-DEFAULT_CONTAINER_TYPE = "docker"
+DOCKER_CONTAINER_TYPE = "docker"
+DEFAULT_CONTAINER_TYPE = DOCKER_CONTAINER_TYPE
+ALL_CONTAINER_TYPES = [DOCKER_CONTAINER_TYPE]
LOAD_CACHED_IMAGE_COMMAND_TEMPLATE = '''
python << EOF
@@ -47,9 +56,18 @@ class ContainerFinder(object):
def __init__(self, app_info):
self.app_info = app_info
- self.container_registry = ContainerRegistry()
+ self.container_registry = ContainerRegistry(app_info)
+
+ def __enabled_container_types(self, destination_info):
+ return [t for t in ALL_CONTAINER_TYPES if self.__container_type_enabled(t, destination_info)]
def find_container(self, tool_info, destination_info, job_info):
+ enabled_container_types = self.__enabled_container_types(destination_info)
+
+ # Short-cut everything else and just skip checks if no container type is enabled.
+ if not enabled_container_types:
+ return NULL_CONTAINER
+
def __destination_container(container_description=None, container_id=None, container_type=None):
if container_description:
container_id = container_description.identifier
@@ -82,23 +100,10 @@ class ContainerFinder(object):
return container
# Otherwise lets see if we can find container for the tool.
-
- # Exact matches first from explicitly listed containers in tools...
- for container_description in tool_info.container_descriptions:
- container = __destination_container(container_description)
- if container:
- return container
-
- # Implement vague concept of looping through all containers
- # matching requirements. Exact details need to be worked through
- # but hopefully the idea that it sits below find_container somewhere
- # external components to this module don't need to worry about it
- # is good enough.
- container_descriptions = self.container_registry.container_descriptions_for_requirements(tool_info.requirements)
- for container_description in container_descriptions:
- container = __destination_container(container_description)
- if container:
- return container
+ container_description = self.container_registry.find_best_container_description(enabled_container_types, tool_info)
+ container = __destination_container(container_description)
+ if container:
+ return container
# If we still don't have a container, check to see if any container
# types define a default container id and use that.
@@ -175,14 +180,55 @@ class NullContainerFinder(object):
return []
-class ContainerRegistry():
+class ContainerRegistry(object):
+ """Loop through enabled ContainerResolver plugins and find first match."""
- def __init__(self):
- pass
+ def __init__(self, app_info):
+ self.resolver_classes = self.__resolvers_dict()
+ self.enable_beta_mulled_containers = app_info.enable_beta_mulled_containers
+ self.app_info = app_info
+ self.container_resolvers = self.__build_container_resolvers(app_info)
- def container_descriptions_for_requirements(self, requirements):
- # Return lists of containers that would match requirements...
- return []
+ def __build_container_resolvers( self, app_info ):
+ conf_file = getattr(app_info, 'containers_resolvers_config_file', None)
+ if not conf_file:
+ return self.__default_containers_resolvers()
+ if not os.path.exists( conf_file ):
+ log.debug( "Unable to find config file '%s'", conf_file)
+ return self.__default_containers_resolvers()
+ plugin_source = plugin_config.plugin_source_from_path( conf_file )
+ return self.__parse_resolver_conf_xml( plugin_source )
+
+ def __parse_resolver_conf_xml(self, plugin_source):
+ extra_kwds = {}
+ return plugin_config.load_plugins(self.resolver_classes, plugin_source, extra_kwds)
+
+ def __default_containers_resolvers(self):
+ default_resolvers = [
+ ExplicitContainerResolver(self.app_info),
+ ]
+ if self.enable_beta_mulled_containers:
+ default_resolvers.extend([
+ CachedMulledContainerResolver(self.app_info),
+ MulledContainerResolver(self.app_info, namespace="mulled"),
+ BuildMulledContainerResolver(self.app_info),
+ ])
+ return default_resolvers
+
+ def __resolvers_dict( self ):
+ import galaxy.tools.deps.container_resolvers
+ return plugin_config.plugins_dict( galaxy.tools.deps.container_resolvers, 'resolver_type' )
+
+ def find_best_container_description(self, enabled_container_types, tool_info):
+ """Yield best container description of supplied types matching tool info."""
+ for container_resolver in self.container_resolvers:
+ container_description = container_resolver.resolve(enabled_container_types, tool_info)
+ log.info("Checking with container resolver [%s] found description [%s]" % (container_resolver, container_description))
+ if container_description:
+ assert container_description.type in enabled_container_types
+ return container_description
+
+ return None
class AppInfo(object):
@@ -193,7 +239,11 @@ class AppInfo(object):
default_file_path=None,
outputs_to_working_directory=False,
container_image_cache_path=None,
- library_import_dir=None
+ library_import_dir=None,
+ enable_beta_mulled_containers=False,
+ containers_resolvers_config_file=None,
+ involucro_path=None,
+ involucro_auto_init=True,
):
self.galaxy_root_dir = galaxy_root_dir
self.default_file_path = default_file_path
@@ -201,6 +251,10 @@ class AppInfo(object):
self.outputs_to_working_directory = outputs_to_working_directory
self.container_image_cache_path = container_image_cache_path
self.library_import_dir = library_import_dir
+ self.enable_beta_mulled_containers = enable_beta_mulled_containers
+ self.containers_resolvers_config_file = containers_resolvers_config_file
+ self.involucro_path = involucro_path
+ self.involucro_auto_init = involucro_auto_init
class ToolInfo(object):
diff --git a/lib/galaxy/tools/deps/installable.py b/lib/galaxy/tools/deps/installable.py
new file mode 100644
index 00000000000..cacc069fe65
--- /dev/null
+++ b/lib/galaxy/tools/deps/installable.py
@@ -0,0 +1,77 @@
+"""Abstractions for installing local software managed and required by Galaxy/galaxy-lib."""
+
+import logging
+import os
+
+from abc import (
+ ABCMeta,
+ abstractmethod,
+ abstractproperty,
+)
+
+from galaxy.util.filelock import (
+ FileLock,
+ FileLockException
+)
+
+log = logging.getLogger(__name__)
+
+
+class InstallableContext(object):
+ """Represent a directory/configuration of something that can be installed."""
+
+ __metaclass__ = ABCMeta
+
+ @abstractmethod
+ def is_installed(self):
+ """Return bool indicating if the configured software is installed."""
+
+ @abstractmethod
+ def can_install(self):
+ """Check preconditions for installation."""
+
+ @abstractproperty
+ def installable_description(self):
+ """Short description of thing being installed for log statements."""
+
+ @abstractproperty
+ def parent_path(self):
+ """Return parent path of the location the installable will be created within."""
+
+
+def ensure_installed(installable_context, install_func, auto_init):
+ """Make sure target is installed - handle multiple processes potentially attempting installation."""
+ parent_path = installable_context.parent_path
+ desc = installable_context.installable_description
+
+ def _check():
+ if not installable_context.is_installed():
+ if auto_init:
+ if installable_context.can_install():
+ if install_func(installable_context):
+ installed = False
+ log.warning("%s installation requested and failed." % desc)
+ else:
+ installed = installable_context.is_installed()
+ if not installed:
+ log.warning("%s installation requested, seemed to succeed, but not found." % desc)
+ else:
+ installed = False
+ else:
+ installed = False
+ log.warning("%s not installed and auto-installation disabled.", desc)
+ else:
+ installed = True
+ return installed
+
+ if not os.path.exists(parent_path):
+ os.mkdir(parent_path)
+
+ try:
+ if auto_init and os.access(parent_path, os.W_OK):
+ with FileLock(os.path.join(parent_path, desc.lower())):
+ return _check()
+ else:
+ return _check()
+ except FileLockException:
+ return ensure_installed(installable_context, auto_init)
diff --git a/lib/galaxy/tools/deps/mulled/__init__.py b/lib/galaxy/tools/deps/mulled/__init__.py
new file mode 100644
index 00000000000..e69de29bb2d
diff --git a/lib/galaxy/tools/deps/mulled/_cli.py b/lib/galaxy/tools/deps/mulled/_cli.py
new file mode 100644
index 00000000000..3cf32bad5de
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/_cli.py
@@ -0,0 +1,19 @@
+"""CLI helpers for mulled command-line tools."""
+
+import argparse
+
+
+def arg_parser(argv, globals):
+ """Build an argparser for this CLI tool."""
+ doc = globals["__doc__"]
+ description, epilog = doc.split("\n", 1)
+ parser = argparse.ArgumentParser(
+ description=description,
+ epilog=epilog,
+ formatter_class=argparse.RawTextHelpFormatter,
+ )
+ return parser
+
+__all__ = [
+ "arg_parser"
+]
diff --git a/lib/galaxy/tools/deps/mulled/invfile.lua b/lib/galaxy/tools/deps/mulled/invfile.lua
new file mode 100644
index 00000000000..629c15f7b67
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/invfile.lua
@@ -0,0 +1,67 @@
+-- http://stackoverflow.com/questions/19262761/lua-need-to-split-at-comma/19263313#19263313
+function string:split( inSplitPattern, outResults )
+ if not outResults then
+ outResults = { }
+ end
+ local theStart = 1
+ local theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart )
+ while theSplitStart do
+ table.insert( outResults, string.sub( self, theStart, theSplitStart-1 ) )
+ theStart = theSplitEnd + 1
+ theSplitStart, theSplitEnd = string.find( self, inSplitPattern, theStart )
+ end
+ table.insert( outResults, string.sub( self, theStart ) )
+ return outResults
+end
+
+local repo = VAR.REPO
+
+local channel_args = ''
+local channels = VAR.CHANNELS:split(",")
+for i = 1, #channels do
+ channel_args = channel_args .. " -c " .. channels[i]
+end
+
+local target_args = ''
+local targets = VAR.TARGETS:split(",")
+for i = 1, #targets do
+ target_args = target_args .. " " .. targets[i]
+end
+
+local bind_args = {}
+local binds_table = VAR.BINDS:split(",")
+for i = 1, #binds_table do
+ table.insert(bind_args, binds_table[i])
+end
+
+inv.task('build')
+ .using('continuumio/miniconda:latest')
+ .withHostConfig({binds = {"build:/data"}})
+ .run('rm', '-rf', '/data/dist')
+ .using('continuumio/miniconda:latest')
+ .withHostConfig({binds = bind_args})
+ .run('/bin/sh', '-c', 'conda install '
+ .. channel_args .. ' '
+ .. target_args
+ .. ' -p /usr/local --copy --yes')
+ .wrap('build/dist')
+ .at('/usr/local')
+ .inImage('bgruening/busybox-bash:0.1')
+ .as(repo)
+
+inv.task('test')
+ .using(repo)
+ .withConfig({entrypoint = {'/bin/sh', '-c'}})
+ .run(VAR.TEST)
+
+inv.task('push')
+ .push(repo)
+
+inv.task('build-and-test')
+ .runTask('build')
+ .runTask('test')
+
+inv.task('all')
+ .runTask('build')
+ .runTask('test')
+ .runTask('push')
diff --git a/lib/galaxy/tools/deps/mulled/mulled_build.py b/lib/galaxy/tools/deps/mulled/mulled_build.py
new file mode 100644
index 00000000000..b359c540cbf
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/mulled_build.py
@@ -0,0 +1,284 @@
+#!/usr/bin/env python
+"""Build a mulled image for specified conda targets.
+
+Examples
+
+Build a mulled image with:
+
+ mulled-build build 'samtools=1.3.1,bedtools=2.22'
+
+"""
+from __future__ import print_function
+
+import json
+import os
+import string
+import subprocess
+
+from sys import platform as _platform
+
+from galaxy.tools.deps import commands
+from galaxy.tools.deps import installable
+
+try:
+ import yaml
+except ImportError:
+ yaml = None
+
+from ._cli import arg_parser
+from .util import build_target, conda_build_target_str, image_name
+from ..conda_compat import MetaData
+
+DIRNAME = os.path.dirname(__file__)
+DEFAULT_CHANNEL = "bioconda"
+DEFAULT_EXTRA_CHANNELS = ["conda-forge", "r"]
+DEFAULT_CHANNELS = [DEFAULT_CHANNEL] + DEFAULT_EXTRA_CHANNELS
+DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}"
+DEFAULT_BINDS = ["build/dist:/usr/local/"]
+IS_OS_X = _platform == "darwin"
+INVOLUCRO_VERSION = "1.1.2"
+
+
+def involucro_link():
+ if IS_OS_X:
+ url = "https://github.com/involucro/involucro/releases/download/v%s/involucro.darwin" % INVOLUCRO_VERSION
+ else:
+ url = "https://github.com/involucro/involucro/releases/download/v%s/involucro" % INVOLUCRO_VERSION
+ return url
+
+
+def get_tests(args, pkg_path):
+ """Extract test cases given a recipe's meta.yaml file."""
+ recipes_dir = args.recipes_dir
+
+ tests = ""
+ input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
+ recipe_meta = MetaData(input_dir)
+
+ tests_commands = recipe_meta.get_value('test/commands')
+ tests_imports = recipe_meta.get_value('test/imports')
+ requirements = recipe_meta.get_value('requirements/run')
+
+ if tests_imports or tests_commands:
+ if tests_commands:
+ tests = ' && '.join(tests_commands)
+ elif tests_imports and 'python' in requirements:
+ tests = ' && '.join('python -c "import %s"' % imp for imp in tests_imports)
+ elif tests_imports and ('perl' in requirements or 'perl-threaded' in requirements):
+ tests = ' && '.join('''perl -e "use %s;"''' % imp for imp in tests_imports)
+ tests = tests.replace('$R ', 'Rscript ')
+ else:
+ pass
+ return tests
+
+
+def get_pkg_name(args, pkg_path):
+ """Extract the package name from a given meta.yaml file."""
+ recipes_dir = args.recipes_dir
+
+ input_dir = os.path.dirname(os.path.join(recipes_dir, pkg_path))
+ recipe_meta = MetaData(input_dir)
+ return recipe_meta.get_value('package/name')
+
+
+def get_affected_packages(args):
+ """Return a list of all meta.yaml file that where modified/created recently.
+
+ Length of time to check for indicated by the ``hours`` parameter.
+ """
+ recipes_dir = args.recipes_dir
+ hours = args.diff_hours
+ cmd = """cd '%s' && git log --diff-filter=ACMRTUXB --name-only --pretty="" --since="%s hours ago" | grep -E '^recipes/.*/meta.yaml' | sort | uniq""" % (recipes_dir, hours)
+ pkg_list = check_output(cmd, shell=True)
+ ret = list()
+ for pkg in pkg_list.strip().split('\n'):
+ if pkg and os.path.exists(os.path.join( recipes_dir, pkg )):
+ ret.append( (get_pkg_name(args, pkg), get_tests(args, pkg)) )
+ return ret
+
+
+def check_output(cmd, shell=True):
+ return subprocess.check_output(cmd, shell=shell)
+
+
+def conda_versions(pkg_name, file_name):
+ """Return all conda version strings for a specified package name."""
+ j = json.load(open(file_name))
+ ret = list()
+ for pkg in j['packages'].values():
+ if pkg['name'] == pkg_name:
+ ret.append('%s--%s' % (pkg['version'], pkg['build']))
+ return ret
+
+
+def mull_targets(
+ targets, involucro_context=None,
+ command="build", channels=DEFAULT_CHANNELS, namespace="mulled",
+ test='true', image_build=None, name_override=None,
+ repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False,
+ binds=DEFAULT_BINDS
+):
+ if involucro_context is None:
+ involucro_context = InvolucroContext()
+
+ repo_template_kwds = {
+ "namespace": namespace,
+ "image": image_name(targets, image_build=image_build, name_override=name_override)
+ }
+ repo = string.Template(repository_template).safe_substitute(repo_template_kwds)
+
+ for channel in channels:
+ if channel.startswith('file://'):
+ bind_path = channel.lstrip('file://')
+ binds.append('/%s:/%s' % (bind_path, bind_path))
+
+ channels = ",".join(channels)
+ target_str = ",".join(map(conda_build_target_str, targets))
+ bind_str = ",".join(binds)
+ involucro_args = [
+ '-f', '%s/invfile.lua' % DIRNAME,
+ '-set', "CHANNELS='%s'" % channels,
+ '-set', "TEST='%s'" % test,
+ '-set', "TARGETS='%s'" % target_str,
+ '-set', "REPO='%s'" % repo,
+ '-set', "BINDS='%s'" % bind_str,
+ command,
+ ]
+ print(" ".join(involucro_context.build_command(involucro_args)))
+ if not dry_run:
+ ensure_installed(involucro_context, True)
+ involucro_context.exec_command(involucro_args)
+
+
+def context_from_args(args):
+ return InvolucroContext(involucro_bin=args.involucro_path)
+
+
+class InvolucroContext(installable.InstallableContext):
+
+ installable_description = "Involucro"
+
+ def __init__(self, involucro_bin=None, shell_exec=None, verbose="3"):
+ if involucro_bin is None:
+ if os.path.exists("./involucro"):
+ self.involucro_bin = "./involucro"
+ else:
+ self.involucro_bin = "involucro"
+ else:
+ self.involucro_bin = involucro_bin
+ self.shell_exec = shell_exec or commands.shell
+ self.verbose = verbose
+
+ def build_command(self, involucro_args):
+ return [self.involucro_bin, "-v=%s" % self.verbose] + involucro_args
+
+ def exec_command(self, involucro_args):
+ cmd = self.build_command(involucro_args)
+ return self.shell_exec(" ".join(cmd))
+
+ def is_installed(self):
+ return os.path.exists(self.involucro_bin)
+
+ def can_install(self):
+ return True
+
+ @property
+ def parent_path(self):
+ return os.path.dirname(os.path.abspath(self.involucro_bin))
+
+
+def ensure_installed(involucro_context, auto_init):
+ return installable.ensure_installed(involucro_context, install_involucro, auto_init)
+
+
+def install_involucro(involucro_context=None, to_path=None):
+ to_path = involucro_context.involucro_bin
+ download_cmd = " ".join(commands.download_command(involucro_link(), to=to_path, quote_url=True))
+ full_cmd = "%s && chmod +x %s" % (download_cmd, to_path)
+ return involucro_context.shell_exec(full_cmd)
+
+
+def add_build_arguments(parser):
+ """Base arguments describing how to 'mull'."""
+ parser.add_argument('--involucro-path', dest="involucro_path", default=None,
+ help="Path to involucro (if not set will look in working directory and on PATH).")
+ parser.add_argument('--force-rebuild', dest="force_rebuild", action="store_true",
+ help="Rebuild package even if already published.")
+ parser.add_argument('--dry-run', dest='dry_run', action="store_true",
+ help='Just print commands instead of executing them.')
+ parser.add_argument('-n', '--namespace', dest='namespace', default="mulled",
+ help='quay.io namespace.')
+ parser.add_argument('-r', '--repository_template', dest='repository_template', default=DEFAULT_REPOSITORY_TEMPLATE,
+ help='Docker repository target for publication (only quay.io or compat. API is currently supported).')
+ parser.add_argument('-c', '--channel', dest='channel', default=DEFAULT_CHANNEL,
+ help='Target conda channel')
+ parser.add_argument('--extra-channels', dest='extra_channels', default=",".join(DEFAULT_EXTRA_CHANNELS),
+ help='Dependent conda channels.')
+
+
+def add_single_image_arguments(parser):
+ parser.add_argument("--name-override", dest="name_override", default=None,
+ help="Override mulled image name - this is not recommended since metadata will not be detectable from the name of resulting images")
+ parser.add_argument("--image-build", dest="image_build", default=None,
+ help="Build a versioned variant of this image.")
+
+
+def target_str_to_targets(targets_raw):
+ def parse_target(target_str):
+ if "=" in target_str:
+ package_name, version = target_str.split("=", 1)
+ target = build_target(package_name, version)
+ else:
+ target = build_target(target_str)
+ return target
+
+ targets = map(parse_target, targets_raw.split(","))
+ return targets
+
+
+def args_to_mull_targets_kwds(args):
+ kwds = {}
+ if hasattr(args, "image_build"):
+ kwds["image_build"] = args.image_build
+ if hasattr(args, "name_override"):
+ kwds["name_override"] = args.name_override
+ if hasattr(args, "namespace"):
+ kwds["namespace"] = args.namespace
+ if hasattr(args, "dry_run"):
+ kwds["dry_run"] = args.dry_run
+ if hasattr(args, "test"):
+ kwds["test"] = args.test
+ if hasattr(args, "channel"):
+ channels = [args.channel]
+ if hasattr(args, "extra_channels"):
+ channels += args.extra_channels.split(",")
+ kwds["channels"] = channels
+ if hasattr(args, "command"):
+ kwds["command"] = args.command
+ if hasattr(args, "repository_template"):
+ kwds["repository_template"] = args.repository_template
+
+ kwds["involucro_context"] = context_from_args(args)
+
+ return kwds
+
+
+def main(argv=None):
+ """Main entry-point for the CLI tool."""
+ parser = arg_parser(argv, globals())
+ add_build_arguments(parser)
+ add_single_image_arguments(parser)
+ parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
+ parser.add_argument('targets', metavar="TARGETS", default=None, help="Build a single container with specific package(s).")
+ parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of mulled container (leave blank to auto-generate based on packages - recommended).")
+ parser.add_argument('--test', help='Provide a test command for the container.')
+ args = parser.parse_args()
+ targets = target_str_to_targets(args.targets)
+ mull_targets(targets, **args_to_mull_targets_kwds(args))
+
+
+__all__ = ["main"]
+
+
+if __name__ == '__main__':
+ main()
diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_channel.py b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py
new file mode 100644
index 00000000000..b5082224649
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/mulled_build_channel.py
@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+"""Build a mulled images for all recent conda recipe updates that don't have existing images.
+
+Examples:
+
+Build mulled images for recent bioconda changes with:
+
+ mulled-build-channel build
+
+Build, test, and publish images with the follow command:
+
+ mulled-build-channel all
+
+See recent changes that would be built with:
+
+ mulled-build-channel list
+
+"""
+
+
+import os
+import time
+
+from ._cli import arg_parser
+from .mulled_build import (
+ add_build_arguments,
+ args_to_mull_targets_kwds,
+ build_target,
+ check_output,
+ conda_versions,
+ get_affected_packages,
+ mull_targets,
+)
+from .util import quay_versions, version_sorted
+
+
+def _fetch_repo_data(args):
+ repo_data = args.repo_data
+ channel = args.channel
+ if repo_data is None:
+ repo_data = "%s-repodata.json" % channel
+ if not os.path.exists(repo_data):
+ check_output("wget --quiet https://conda.anaconda.org/%s/linux-64/repodata.json.bz2 -O '%s.bz2' && bzip2 -d '%s.bz2'" % (channel, repo_data, repo_data))
+ return repo_data
+
+
+def _new_versions(quay, conda):
+ """Calculate the versions that are in conda but not on quay.io."""
+ sconda = set(conda)
+ squay = set(quay) if quay else set()
+ return sconda - squay # sconda.symmetric_difference(squay)
+
+
+def run_channel(args, build_last_n_versions=1):
+ """Build list of involucro commands (as shell snippet) to run."""
+ pkgs = get_affected_packages(args)
+ for pkg_name, pkg_tests in pkgs:
+ repo_data = _fetch_repo_data(args)
+ c = conda_versions(pkg_name, repo_data)
+ # only package the most recent N versions
+ c = version_sorted(c)[:build_last_n_versions]
+
+ if not args.force_rebuild:
+ time.sleep(1)
+ q = quay_versions(args.namespace, pkg_name)
+ versions = _new_versions(q, c)
+ else:
+ versions = c
+
+ for tag in versions:
+ target = build_target(pkg_name, tag=tag)
+ targets = [target]
+ mull_targets(targets, test=pkg_tests, **args_to_mull_targets_kwds(args))
+
+
+def get_pkg_names(args):
+ """Print package names that would be affected."""
+ print('\n'.join([pkg_name for pkg_name, pkg_tests in get_affected_packages(args)]))
+
+
+def add_channel_arguments(parser):
+ """Add arguments only used if running mulled over a whole conda channel."""
+ parser.add_argument('--repo-data', dest='repo_data', default=None,
+ help='Published repository data (will be fetched from --channel if not available and written). Defaults to [channel_name]-repodata.json.')
+ parser.add_argument('--diff-hours', dest='diff_hours', default="25",
+ help='If finding all recently changed recipes, use this number of hours.')
+ parser.add_argument('--recipes-dir', dest="recipes_dir", default="./bioconda-recipes")
+
+
+def main(argv=None):
+ """Main entry-point for the CLI tool."""
+ parser = arg_parser(argv, globals())
+ add_channel_arguments(parser)
+ add_build_arguments(parser)
+ parser.add_argument('command', metavar='COMMAND', help='Command (list, build-and-test, build, all)')
+ parser.add_argument('--targets', dest="targets", default=None, help="Build a single container with specific package(s).")
+ parser.add_argument('--repository-name', dest="repository_name", default=None, help="Name of a single container (leave blank to auto-generate based on packages).")
+ args = parser.parse_args()
+ if args.command == "list":
+ get_pkg_names(args)
+ else:
+ run_channel(args)
+
+
+__all__ = ["main"]
+
+
+if __name__ == '__main__':
+ main()
diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_files.py b/lib/galaxy/tools/deps/mulled/mulled_build_files.py
new file mode 100644
index 00000000000..0fd065c504d
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/mulled_build_files.py
@@ -0,0 +1,80 @@
+"""Build all composite mulled recipes discovered in TSV files.
+
+Use mulled-build-channel to build images for single recipes for a whole conda
+channel. This script instead builds images for combinations of recipes. This
+script can be given a single TSV file or a directory of TSV files to process.
+
+Examples:
+
+Build all recipes discovered in tsv files in a single directory.
+
+ mulled-build-files build
+
+"""
+
+import collections
+import glob
+import os
+
+from ._cli import arg_parser
+
+from .mulled_build import (
+ add_build_arguments,
+ args_to_mull_targets_kwds,
+ mull_targets,
+ target_str_to_targets,
+)
+
+
+def main(argv=None):
+ """Main entry-point for the CLI tool."""
+ parser = arg_parser(argv, globals())
+ add_build_arguments(parser)
+ parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
+ parser.add_argument('files', metavar="FILES", default=".",
+ help="Path to directory (or single file) of TSV files describing composite recipes.")
+ args = parser.parse_args()
+ for targets in generate_targets(args.files):
+ mull_targets(targets, **args_to_mull_targets_kwds(args))
+
+
+def generate_targets(target_source):
+ """Generate all targets from TSV files in specified file or directory."""
+ target_source = os.path.abspath(target_source)
+ if os.path.isdir(target_source):
+ target_source_files = glob.glob(target_source + "/*.tsv")
+ else:
+ target_source_files = [target_source]
+
+ for target_source_file in target_source_files:
+ with open(target_source_file, "r") as f:
+ for line in f.readlines():
+ if line:
+ line = line.strip()
+
+ if not line or line.startswith("#"):
+ continue
+
+ yield line_to_targets(line)
+
+
+def line_to_targets(line_str):
+ line = _parse_line(line_str)
+ return target_str_to_targets(line)
+
+
+_Line = collections.namedtuple("_Line", ["targets", "image_build", "name_override"])
+
+
+def _parse_line(line_str):
+ line_parts = line_str.split(" ")
+ assert len(line_parts) < 3, "Too many fields in line [%s], expect at most 3 - targets, image build number, and name override." % line_str
+ line_parts += [None] * (3 - len(line_parts))
+ return _Line(*line_parts)
+
+
+__all__ = ["main"]
+
+
+if __name__ == '__main__':
+ main()
diff --git a/lib/galaxy/tools/deps/mulled/mulled_build_tool.py b/lib/galaxy/tools/deps/mulled/mulled_build_tool.py
new file mode 100644
index 00000000000..7281c7940ae
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/mulled_build_tool.py
@@ -0,0 +1,52 @@
+#!/usr/bin/env python
+"""Build a mulled images for a tool source (Galaxy or CWL tool).
+
+Examples:
+
+Build mulled images for requirements defined in a tool:
+
+ mulled-build-tool build path/to/tool_file.xml
+
+"""
+
+from galaxy.tools.parser import get_tool_source
+
+from ._cli import arg_parser
+from .mulled_build import (
+ add_build_arguments,
+ add_single_image_arguments,
+ args_to_mull_targets_kwds,
+ build_target,
+ mull_targets,
+)
+
+
+def main(argv=None):
+ """Main entry-point for the CLI tool."""
+ parser = arg_parser(argv, globals())
+ add_build_arguments(parser)
+ add_single_image_arguments(parser)
+ parser.add_argument('command', metavar='COMMAND', help='Command (build-and-test, build, all)')
+ parser.add_argument('tool', metavar="TOOL", default=None, help="Path to tool to build mulled image for.")
+ args = parser.parse_args()
+ tool_source = get_tool_source(args.tool)
+ requirements, _ = tool_source.parse_requirements_and_containers()
+ targets = requirements_to_mulled_targets(requirements)
+ mull_targets(targets, **args_to_mull_targets_kwds(args))
+
+
+def requirements_to_mulled_targets(requirements):
+ """Convert Galaxy's representation of requirements into mulled Target objects.
+
+ Only package requirements are retained.
+ """
+ package_requirements = filter(lambda r: r.type == "package", requirements)
+ targets = map(lambda r: build_target(r.name, r.version), package_requirements)
+ return targets
+
+
+__all__ = ["main", "requirements_to_mulled_targets"]
+
+
+if __name__ == '__main__':
+ main()
diff --git a/lib/galaxy/tools/deps/mulled/mulled_search.py b/lib/galaxy/tools/deps/mulled/mulled_search.py
new file mode 100644
index 00000000000..724956a1335
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/mulled_search.py
@@ -0,0 +1,131 @@
+#!/usr/bin/env python
+
+import argparse
+import json
+import sys
+import tempfile
+
+try:
+ import requests
+except ImportError:
+ requests = None
+
+try:
+ from whoosh.fields import Schema
+ from whoosh.fields import TEXT
+ from whoosh.fields import STORED
+ from whoosh.index import create_in
+ from whoosh.qparser import QueryParser
+except ImportError:
+ Schema = TEXT = STORED = create_in = QueryParser = None
+
+QUAY_API_URL = 'https://quay.io/api/v1/repository'
+
+
+class QuaySearch():
+ """
+ Tool to search within a quay organization for a given software name.
+ """
+ def __init__(self, organization):
+ self.index = None
+ self.organization = organization
+
+ def build_index(self):
+ """
+ Create an index to quickly examine the repositories of a given quay.io organization.
+ """
+ # download all information about the repositories from the
+ # given organization in self.organization
+
+ parameters = {'public': 'true', 'namespace': self.organization}
+ r = requests.get(QUAY_API_URL, headers={'Accept-encoding': 'gzip'}, params=parameters,
+ timeout=12)
+
+ tmp_dir = tempfile.mkdtemp()
+ schema = Schema(title=TEXT(stored=True), content=STORED)
+ self.index = create_in(tmp_dir, schema)
+
+ json_decoder = json.JSONDecoder()
+ decoded_request = json_decoder.decode(r.text)
+ writer = self.index.writer()
+ for repository in decoded_request['repositories']:
+ writer.add_document(title=repository['name'], content=repository['description'])
+ writer.commit()
+
+ def search_repository(self, search_string, non_strict):
+ """
+ Search Docker containers on quay.io.
+ Results are displayed with all available versions,
+ including the complete image name.
+ """
+ # with statement closes searcher after usage.
+ with self.index.searcher() as searcher:
+ search_string = "*%s*" % search_string
+ query = QueryParser("title", self.index.schema).parse(search_string)
+ results = searcher.search(query)
+ if non_strict:
+ # look for spelling errors and use suggestions as a search term too
+ corrector = searcher.corrector("title")
+ suggestions = corrector.suggest(search_string, limit=2)
+
+ # get all repositories with suggested keywords
+ for suggestion in suggestions:
+ search_string = "*%s*" % suggestion
+ query = QueryParser("title", self.index.schema).parse(search_string)
+ results_tmp = searcher.search(query)
+ results.extend(results_tmp)
+
+ sys.stdout.write("The query \033[1m %s \033[0m resulted in %s result(s).\n" % (search_string, len(results)))
+
+ if non_strict:
+ sys.stdout.write('The search was relaxed and the following search terms were searched: ')
+ sys.stdout.write('\033[1m %s \033[0m\n' % ', '.join(suggestions))
+
+ out = list()
+ for result in results:
+ title = result['title']
+ for version in self.get_additional_repository_information(title):
+ row = [title]
+ row.append(version)
+ out.append(row)
+ if out:
+ col_width = max(len(word) for row in out for word in row) + 2 # padding
+ for row in out:
+ name = row[0]
+ version = row[1]
+ sys.stdout.write("".join(word.ljust(col_width) for word in row) + "docker pull quay.io/%s/%s:%s\n" % (self.organization, name, version))
+ else:
+ sys.stdout.write("No results found for %s in quay.io/%s.\n" % (search_string, self.organization))
+
+ def get_additional_repository_information(self, repository_string):
+ """
+ Function downloads additional information from quay.io to
+ get the tag-field which includes the version number.
+ """
+ url = "%s/%s/%s" % (QUAY_API_URL, self.organization, repository_string)
+ r = requests.get(url, headers={'Accept-encoding': 'gzip'}, timeout=12)
+
+ json_decoder = json.JSONDecoder()
+ decoded_request = json_decoder.decode(r.text)
+ return decoded_request['tags']
+
+
+def main(argv=None):
+ parser = argparse.ArgumentParser(description='Searches in a given quay organization for a repository')
+ parser.add_argument('-o', '--organization', dest='organization_string', default="mulled",
+ help='Change organization. Default is mulled.')
+ parser.add_argument('--non-strict', dest='non_strict', action="store_true",
+ help='Autocorrection of typos activated. Lists more results but can be confusing.\
+ For too many queries quay.io blocks the request and the results can be incomplete.')
+ parser.add_argument('-s', '--search', required=True,
+ help='The name of the tool you want to search for.')
+ args = parser.parse_args()
+
+ quay = QuaySearch(args.organization_string)
+ quay.build_index()
+
+ quay.search_repository(args.search, args.non_strict)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/lib/galaxy/tools/deps/mulled/util.py b/lib/galaxy/tools/deps/mulled/util.py
new file mode 100644
index 00000000000..4fc5312afbe
--- /dev/null
+++ b/lib/galaxy/tools/deps/mulled/util.py
@@ -0,0 +1,116 @@
+"""Utilities for working with mulled abstractions outside the mulled package."""
+from __future__ import print_function
+
+import collections
+import hashlib
+
+from distutils.version import LooseVersion
+
+try:
+ import requests
+except ImportError:
+ requests = None
+
+
+def quay_versions(namespace, pkg_name):
+ """Get all version tags for a Docker image stored on quay.io for supplied package name."""
+ if requests is None:
+ raise Exception("requets library is unavailable, functionality not available.")
+
+ assert namespace is not None
+ assert pkg_name is not None
+ url = 'https://quay.io/api/v1/repository/%s/%s' % (namespace, pkg_name)
+ response = requests.get(url, timeout=None)
+ data = response.json()
+ if 'error_type' in data and data['error_type'] == "invalid_token":
+ return []
+
+ if 'tags' not in data:
+ raise Exception("Unexpected response from quay.io - not tags description found [%s]" % data)
+
+ return [tag for tag in data['tags'] if tag != 'latest']
+
+
+def mulled_tags_for(namespace, image):
+ """Fetch remote tags available for supplied image name.
+
+ The result will be sorted so newest tags are first.
+ """
+ tags = quay_versions(namespace, image)
+ tags = version_sorted(tags)
+ return tags
+
+
+def split_tag(tag):
+ """Split mulled image name into conda version and conda build."""
+ version = tag.split('--', 1)[0]
+ build = tag.split('--', 1)[1]
+ return version, build
+
+
+def version_sorted(elements):
+ """Sort iterable based on loose description of "version" from newest to oldest."""
+ return sorted(elements, key=LooseVersion, reverse=True)
+
+
+Target = collections.namedtuple("Target", ["package_name", "version", "build"])
+
+
+def build_target(package_name, version=None, build=None, tag=None):
+ """Use supplied arguments to build a :class:`Target` object."""
+ if tag is not None:
+ assert version is None
+ assert build is None
+ version, build = split_tag(tag)
+
+ return Target(package_name, version, build)
+
+
+def conda_build_target_str(target):
+ rval = target.package_name
+ if target.version:
+ rval += "=%s" % target.version
+
+ if target.build:
+ rval += "=%s" % target.build
+
+ return rval
+
+
+def image_name(targets, image_build=None, name_override=None):
+ if name_override is not None:
+ print("WARNING: Overriding mulled image name, auto-detection of 'mulled' package attributes will fail to detect result.")
+ return name_override
+
+ if len(targets) == 1:
+ target = targets[0]
+ suffix = ""
+ if target.version is not None:
+ if image_build is not None:
+ print("WARNING: Hard-coding image build instead of using Conda build - this is not recommended.")
+ suffix = image_build
+ else:
+ suffix += ":%s" % target.version
+ build = target.build
+ if build is not None:
+ suffix += "--%s" % build
+ return "%s%s" % (target.package_name, suffix)
+ else:
+ targets_order = sorted(targets, key=lambda t: t.package_name)
+ requirements_buffer = "\n".join(map(conda_build_target_str, targets_order))
+ m = hashlib.sha1()
+ m.update(requirements_buffer)
+ suffix = "" if not image_build else ":%s" % image_build
+ return "mulled-v1-%s%s" % (m.hexdigest(), suffix)
+
+
+__all__ = [
+ "build_target",
+ "conda_build_target_str",
+ "image_name",
+ "mulled_tags_for",
+ "quay_versions",
+ "split_tag",
+ "Target",
+ "version_sorted",
+]
diff --git a/lib/galaxy/tools/deps/resolvers/__init__.py b/lib/galaxy/tools/deps/resolvers/__init__.py
index 2ea5abeaa4f..6165fdb6af8 100644
--- a/lib/galaxy/tools/deps/resolvers/__init__.py
+++ b/lib/galaxy/tools/deps/resolvers/__init__.py
@@ -1,3 +1,4 @@
+"""The module defines the abstract interface for dealing tool dependency resolution plugins."""
from abc import (
ABCMeta,
abstractmethod,
@@ -10,6 +11,8 @@ from ..requirements import ToolRequirement
class DependencyResolver(Dictifiable, object):
+ """Abstract description of a technique for resolving container images for tool execution."""
+
# Keys for dictification.
dict_collection_visible_keys = ['resolver_type', 'resolves_simple_dependencies']
# A "simple" dependency is one that does not depend on the the tool
@@ -23,13 +26,15 @@ class DependencyResolver(Dictifiable, object):
@abstractmethod
def resolve( self, name, version, type, **kwds ):
- """
- Given inputs describing dependency in the abstract, yield tuple of
- (script, bin, version). Here script is the env.sh file to source
- before running a job, if that is not found the bin directory will be
- appended to the path (if it is not None). Finally, version is the
- resolved tool dependency version (which may differ from requested
- version for instance if the request version is 'default'.)
+ """Given inputs describing dependency in the abstract yield a Dependency object.
+
+ The Dependency object describes various attributes (script, bin,
+ version) used to build scripts with the dependency availble. Here
+ script is the env.sh file to source before running a job, if that is
+ not found the bin directory will be appended to the path (if it is
+ not ``None``). Finally, version is the resolved tool dependency
+ version (which may differ from requested version for instance if the
+ request version is 'default'.)
"""
def _get_config_option(self, key, dependency_resolver, default=None, config_prefix=None, **kwds):
diff --git a/lib/galaxy/tools/deps/resolvers/conda.py b/lib/galaxy/tools/deps/resolvers/conda.py
index 3b3d85d16f9..f16734a268e 100644
--- a/lib/galaxy/tools/deps/resolvers/conda.py
+++ b/lib/galaxy/tools/deps/resolvers/conda.py
@@ -6,10 +6,8 @@ incompatible changes coming.
import logging
import os
-from galaxy.util.filelock import (
- FileLock,
- FileLockException
-)
+import galaxy.tools.deps.installable
+
from ..conda_util import (
build_isolated_environment,
cleanup_failed_install,
@@ -32,7 +30,7 @@ from ..resolvers import (
DEFAULT_BASE_PATH_DIRECTORY = "_conda"
DEFAULT_CONDARC_OVERRIDE = "_condarc"
-DEFAULT_ENSURE_CHANNELS = "r,bioconda,iuc"
+DEFAULT_ENSURE_CHANNELS = "conda-forge,r,bioconda,iuc"
log = logging.getLogger(__name__)
@@ -95,44 +93,11 @@ class CondaDependencyResolver(DependencyResolver, ListableDependencyResolver, In
copy_dependencies = _string_as_bool(get_option("copy_dependencies"))
self.auto_init = _string_as_bool(get_option("auto_init"))
self.conda_context = conda_context
- self.ensure_conda_installed()
+ self.disabled = not galaxy.tools.deps.installable.ensure_installed(conda_context, install_conda, self.auto_init)
self.auto_install = auto_install
self.copy_dependencies = copy_dependencies
self.verbose_install_check = verbose_install_check
- def ensure_conda_installed(self):
- """
- Make sure that conda is installed, and if conda can't be installed, mark resolver as disabled.
- We acquire a lock, so that multiple handlers do not attempt to install conda simultaneously.
- """
- target_path = self.conda_prefix_parent
-
- def _check():
- if not self.conda_context.is_conda_installed():
- if self.auto_init:
- if self.conda_context.can_install_conda():
- if install_conda(self.conda_context):
- self.disabled = True
- log.warning("Conda installation requested and failed.")
- else:
- self.disabled = True
- else:
- self.disabled = True
- log.warning("Conda not installed and auto-installation disabled.")
- else:
- self.disabled = False
-
- if not os.path.exists(target_path):
- os.mkdir(target_path)
- try:
- if self.auto_init and os.access(target_path, os.W_OK):
- with FileLock(os.path.join(target_path, 'conda')):
- _check()
- else:
- _check()
- except FileLockException:
- self.ensure_conda_installed()
-
def resolve(self, name, version, type, **kwds):
# Check for conda just not being there, this way we can enable
# conda by default and just do nothing in not configured.
diff --git a/test/functional/tools/mulled_example_multi_1.xml b/test/functional/tools/mulled_example_multi_1.xml
new file mode 100644
index 00000000000..1f025598c04
--- /dev/null
+++ b/test/functional/tools/mulled_example_multi_1.xml
@@ -0,0 +1,18 @@
+
+ $out_file1 ;
+ echo "Moo" >> $out_file1 ;
+ samtools >> $out_file1 2>&1 ;
+ echo "Cow" >> $out_file1 ;
+ ]]>
+
+ samtools
+ bedtools
+
+
+
+
+
+
+
+
diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml
index d57356e25d9..90d64f86c37 100644
--- a/test/functional/tools/samples_tool_conf.xml
+++ b/test/functional/tools/samples_tool_conf.xml
@@ -111,6 +111,8 @@
+
+