Merge pull request #13755 from bernt-matthias/fix_mulled_conda

Respect conda channels in mulled docker container building

lib/galaxy/tool_util/deps/container_resolvers/mulled.py

@@ -30,6 +30,7 @@ from ..mulled.mulled_build import (
 )
 from ..mulled.mulled_build_tool import requirements_to_mulled_targets
 from ..mulled.util import (
+    default_mulled_conda_channels_from_env,
     mulled_tags_for,
     split_tag,
     v1_image_name,
@@ -647,10 +648,12 @@ class BuildMulledDockerContainerResolver(CliContainerResolver):
         self.auto_install = string_as_bool(auto_install)
         self._mulled_kwds = {
             "namespace": namespace,
-            "channels": self._get_config_option("mulled_channels", DEFAULT_CHANNELS),
             "hash_func": self.hash_func,
             "command": "build-and-test",
         }
+        self._mulled_kwds["channels"] = default_mulled_conda_channels_from_env() or self._get_config_option(
+            "mulled_channels", DEFAULT_CHANNELS
+        )
         self.auto_init = self._get_config_option("involucro_auto_init", True)
 
     def resolve(self, enabled_container_types, tool_info, install=False, **kwds):

lib/galaxy/tool_util/deps/mulled/mulled_build.py

@@ -38,6 +38,7 @@ from .util import (
     build_target,
     conda_build_target_str,
     create_repository,
+    default_mulled_conda_channels_from_env,
     get_file_from_recipe_url,
     PrintProgress,
     quay_repository,
@@ -53,10 +54,7 @@ DEFAULT_BASE_IMAGE = os.environ.get("DEFAULT_BASE_IMAGE", "quay.io/bioconda/base
 DEFAULT_EXTENDED_BASE_IMAGE = os.environ.get(
     "DEFAULT_EXTENDED_BASE_IMAGE", "quay.io/bioconda/base-glibc-debian-bash:latest"
 )
-if "DEFAULT_MULLED_CONDA_CHANNELS" in os.environ:
-    DEFAULT_CHANNELS = os.environ["DEFAULT_MULLED_CONDA_CHANNELS"].split(",")
-else:
-    DEFAULT_CHANNELS = ["conda-forge", "bioconda"]
+DEFAULT_CHANNELS = default_mulled_conda_channels_from_env() or ["conda-forge", "bioconda"]
 DEFAULT_REPOSITORY_TEMPLATE = "quay.io/${namespace}/${image}"
 DEFAULT_BINDS = ["build/dist:/usr/local/"]
 DEFAULT_WORKING_DIR = "/source/"
@@ -156,8 +154,8 @@ def get_conda_hits_for_targets(targets, conda_context):
     return [r for r in search_results if r]
 
 
-def base_image_for_targets(targets, conda_context=None):
-    hits = get_conda_hits_for_targets(targets, conda_context or CondaInDockerContext())
+def base_image_for_targets(targets, conda_context):
+    hits = get_conda_hits_for_targets(targets, conda_context)
     for hit in hits:
         try:
             tarball = get_file_from_recipe_url(hit["url"])
@@ -246,7 +244,7 @@ def mull_targets(
     for channel in channels:
         if channel.startswith("file://"):
             bind_path = channel[7:]
-            binds.append(f"/{bind_path}:/{bind_path}")
+            binds.append(f"{bind_path}:{bind_path}")
 
     channels = ",".join(channels)
     target_str = ",".join(map(conda_build_target_str, targets))
@@ -269,7 +267,8 @@ def mull_targets(
     elif DEST_BASE_IMAGE:
         dest_base_image = DEST_BASE_IMAGE
     elif determine_base_image:
-        dest_base_image = base_image_for_targets(targets)
+        conda_context = CondaInDockerContext(ensure_channels=channels)
+        dest_base_image = base_image_for_targets(targets, conda_context)
 
     if dest_base_image:
         involucro_args.extend(["-set", f"DEST_BASE_IMAGE={dest_base_image}"])
@@ -341,7 +340,12 @@ class CondaInDockerContext(CondaContext):
     ):
         if not conda_exec:
             conda_image = CONDA_IMAGE or "continuumio/miniconda3:latest"
-            conda_exec = docker_command_list("run", [conda_image, "conda"])
+            binds = []
+            for channel in ensure_channels:
+                if channel.startswith("file://"):
+                    bind_path = channel[7:]
+                    binds.extend(["-v", f"{bind_path}:{bind_path}"])
+            conda_exec = docker_command_list("run", binds + [conda_image, "conda"])
         super().__init__(
             conda_prefix=conda_prefix,
             conda_exec=conda_exec,

lib/galaxy/tool_util/deps/mulled/util.py

@@ -3,6 +3,7 @@
 import collections
 import hashlib
 import logging
+import os
 import re
 import sys
 import tarfile
@@ -20,6 +21,13 @@ PARSED_TAG = collections.namedtuple("PARSED_TAG", "tag version build_string buil
 MULLED_SOCKET_TIMEOUT = 12
 
 
+def default_mulled_conda_channels_from_env():
+    if "DEFAULT_MULLED_CONDA_CHANNELS" in os.environ:
+        return os.environ["DEFAULT_MULLED_CONDA_CHANNELS"].split(",")
+    else:
+        return None
+
+
 def create_repository(namespace, repo_name, oauth_token):
     assert oauth_token
     headers = {"Authorization": f"Bearer {oauth_token}"}

test/unit/tool_util/mulled/test_mulled_build.py

@@ -3,6 +3,7 @@ import pytest
 from galaxy.tool_util.deps.mulled.mulled_build import (
     base_image_for_targets,
     build_target,
+    CondaInDockerContext,
     DEFAULT_BASE_IMAGE,
     DEFAULT_EXTENDED_BASE_IMAGE,
     mull_targets,
@@ -22,7 +23,8 @@ from ..util import external_dependency_management
 @external_dependency_management
 def test_base_image_for_targets(target, version, base_image):
     target = build_target(target, version=version)
-    assert base_image_for_targets([target]) == base_image
+    conda_context = CondaInDockerContext()
+    assert base_image_for_targets([target], conda_context) == base_image
 
 
 @external_dependency_management