From 4960e0567a1e3eead7e182b2512d88dbe1a54432 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 19 Apr 2022 15:37:19 +0200 Subject: [PATCH 1/3] make mulled docker container building respect conda channels DEFAULT_MULLED_CONDA_CHANNELS is now respected when building mulled docker containers. In addition the change to CondaInDockerContext allows to use local packages. Ideally planemo users would be able to set the channels via a command line option, but I could not find a way. --- .../tool_util/deps/container_resolvers/mulled.py | 5 ++++- lib/galaxy/tool_util/deps/mulled/mulled_build.py | 13 ++++++++----- lib/galaxy/tool_util/deps/mulled/util.py | 8 ++++++++ 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py index ea932da9000..33e9085bc7a 100644 --- a/lib/galaxy/tool_util/deps/container_resolvers/mulled.py +++ b/lib/galaxy/tool_util/deps/container_resolvers/mulled.py @@ -30,6 +30,7 @@ from ..mulled.mulled_build import ( ) from ..mulled.mulled_build_tool import requirements_to_mulled_targets from ..mulled.util import ( + default_mulled_conda_channels_from_env, mulled_tags_for, split_tag, v1_image_name, @@ -647,10 +648,12 @@ class BuildMulledDockerContainerResolver(CliContainerResolver): self.auto_install = string_as_bool(auto_install) self._mulled_kwds = { "namespace": namespace, - "channels": self._get_config_option("mulled_channels", DEFAULT_CHANNELS), "hash_func": self.hash_func, "command": "build-and-test", } + self._mulled_kwds["channels"] = default_mulled_conda_channels_from_env() or self._get_config_option( + "mulled_channels", DEFAULT_CHANNELS + ) self.auto_init = self._get_config_option("involucro_auto_init", True) def resolve(self, enabled_container_types, tool_info, install=False, **kwds): diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 4625175cb8d..cfab06ffd66 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -38,6 +38,7 @@ from .util import ( build_target, conda_build_target_str, create_repository, + default_mulled_conda_channels_from_env, get_file_from_recipe_url, PrintProgress, quay_repository, @@ -53,10 +54,7 @@ DEFAULT_BASE_IMAGE = os.environ.get("DEFAULT_BASE_IMAGE", "quay.io/bioconda/base DEFAULT_EXTENDED_BASE_IMAGE = os.environ.get( "DEFAULT_EXTENDED_BASE_IMAGE", "quay.io/bioconda/base-glibc-debian-bash:latest" ) -if "DEFAULT_MULLED_CONDA_CHANNELS" in os.environ: - DEFAULT_CHANNELS = os.environ["DEFAULT_MULLED_CONDA_CHANNELS"].split(",") -else: - DEFAULT_CHANNELS = ["conda-forge", "bioconda"] +DEFAULT_CHANNELS = default_mulled_conda_channels_from_env() or ["conda-forge", "bioconda"] DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}" DEFAULT_BINDS = ["build/dist:/usr/local/"] DEFAULT_WORKING_DIR = "/source/" @@ -341,7 +339,12 @@ class CondaInDockerContext(CondaContext): ): if not conda_exec: conda_image = CONDA_IMAGE or "continuumio/miniconda3:latest" - conda_exec = docker_command_list("run", [conda_image, "conda"]) + binds = [] + for channel in ensure_channels: + if channel.startswith("file://"): + bind_path = channel[7:] + binds.extend(["-v", f"/{bind_path}:/{bind_path}"]) + conda_exec = docker_command_list("run", binds + [conda_image, "conda"]) super().__init__( conda_prefix=conda_prefix, conda_exec=conda_exec, diff --git a/lib/galaxy/tool_util/deps/mulled/util.py b/lib/galaxy/tool_util/deps/mulled/util.py index cb832455cc0..83579d6a087 100644 --- a/lib/galaxy/tool_util/deps/mulled/util.py +++ b/lib/galaxy/tool_util/deps/mulled/util.py @@ -3,6 +3,7 @@ import collections import hashlib import logging +import os import re import sys import tarfile @@ -20,6 +21,13 @@ PARSED_TAG = collections.namedtuple("PARSED_TAG", "tag version build_string buil MULLED_SOCKET_TIMEOUT = 12 +def default_mulled_conda_channels_from_env(): + if "DEFAULT_MULLED_CONDA_CHANNELS" in os.environ: + return os.environ["DEFAULT_MULLED_CONDA_CHANNELS"].split(",") + else: + return None + + def create_repository(namespace, repo_name, oauth_token): assert oauth_token headers = {"Authorization": f"Bearer {oauth_token}"} From 6f7a609665d53d75ae01c33722b5061f963f8460 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 20 Apr 2022 17:06:09 +0200 Subject: [PATCH 2/3] explicitly set conda channels for CondaInDockerContext --- lib/galaxy/tool_util/deps/mulled/mulled_build.py | 7 ++++--- test/unit/tool_util/mulled/test_mulled_build.py | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index cfab06ffd66..6eeeabdef4d 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -154,8 +154,8 @@ def get_conda_hits_for_targets(targets, conda_context): return [r for r in search_results if r] -def base_image_for_targets(targets, conda_context=None): - hits = get_conda_hits_for_targets(targets, conda_context or CondaInDockerContext()) +def base_image_for_targets(targets, conda_context): + hits = get_conda_hits_for_targets(targets, conda_context) for hit in hits: try: tarball = get_file_from_recipe_url(hit["url"]) @@ -267,7 +267,8 @@ def mull_targets( elif DEST_BASE_IMAGE: dest_base_image = DEST_BASE_IMAGE elif determine_base_image: - dest_base_image = base_image_for_targets(targets) + conda_context = CondaInDockerContext(ensure_channels=channels) + dest_base_image = base_image_for_targets(targets, conda_context) if dest_base_image: involucro_args.extend(["-set", f"DEST_BASE_IMAGE={dest_base_image}"]) diff --git a/test/unit/tool_util/mulled/test_mulled_build.py b/test/unit/tool_util/mulled/test_mulled_build.py index 64f4e2bfa59..ce48a4b9fca 100644 --- a/test/unit/tool_util/mulled/test_mulled_build.py +++ b/test/unit/tool_util/mulled/test_mulled_build.py @@ -3,6 +3,7 @@ import pytest from galaxy.tool_util.deps.mulled.mulled_build import ( base_image_for_targets, build_target, + CondaInDockerContext, DEFAULT_BASE_IMAGE, DEFAULT_EXTENDED_BASE_IMAGE, mull_targets, @@ -22,7 +23,8 @@ from ..util import external_dependency_management @external_dependency_management def test_base_image_for_targets(target, version, base_image): target = build_target(target, version=version) - assert base_image_for_targets([target]) == base_image + conda_context = CondaInDockerContext() + assert base_image_for_targets([target], conda_context) == base_image @external_dependency_management From 17947484965a75258450f7fb2d45e3b25fc6cbb6 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 20 Apr 2022 17:07:43 +0200 Subject: [PATCH 3/3] remove superficial slash in bind path --- lib/galaxy/tool_util/deps/mulled/mulled_build.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 6eeeabdef4d..659a4f1a5d2 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -244,7 +244,7 @@ def mull_targets( for channel in channels: if channel.startswith("file://"): bind_path = channel[7:] - binds.append(f"/{bind_path}:/{bind_path}") + binds.append(f"{bind_path}:{bind_path}") channels = ",".join(channels) target_str = ",".join(map(conda_build_target_str, targets)) @@ -344,7 +344,7 @@ class CondaInDockerContext(CondaContext): for channel in ensure_channels: if channel.startswith("file://"): bind_path = channel[7:] - binds.extend(["-v", f"/{bind_path}:/{bind_path}"]) + binds.extend(["-v", f"{bind_path}:{bind_path}"]) conda_exec = docker_command_list("run", binds + [conda_image, "conda"]) super().__init__( conda_prefix=conda_prefix,