Merge pull request #9230 from jmchilton/dev_merge_2

Merge release_20.01 into dev.

.github/workflows/integration.yaml

@@ -40,9 +40,9 @@ jobs:
         python-version: ${{ matrix.python-version }}
     - name: Cache venv dir
       uses: actions/cache@v1
-      id: cache-venv-galaxy
+      id: pip-cache
       with:
-        path: .venv
-        key: cache-venv-galaxy-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+        path: ~/.cache/pip
+        key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
     - name: Run tests
       run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"'

lib/galaxy/jobs/runners/kubernetes.py

@@ -8,6 +8,8 @@ import os
 import re
 from time import sleep
 
+import yaml
+
 from galaxy import model
 from galaxy.jobs.runners import (
     AsynchronousJobRunner,
@@ -46,6 +48,8 @@ class KubernetesJobRunner(AsynchronousJobRunner):
             k8s_persistent_volume_claims=dict(map=str),
             k8s_namespace=dict(map=str, default="default"),
             k8s_pod_priority_class=dict(map=str, default=None),
+            k8s_affinity=dict(map=str, default=None),
+            k8s_tolerations=dict(map=str, default=None),
             k8s_galaxy_instance_id=dict(map=str),
             k8s_timeout_seconds_job_deletion=dict(map=int, valid=lambda x: int > 0, default=30),
             k8s_job_api_version=dict(map=str, default=DEFAULT_JOB_API_VERSION),
@@ -241,7 +245,9 @@ class KubernetesJobRunner(AsynchronousJobRunner):
                 "volumes": self.runner_params['k8s_mountable_volumes'],
                 "restartPolicy": self.__get_k8s_restart_policy(ajs.job_wrapper),
                 "containers": self.__get_k8s_containers(ajs),
-                "priorityClassName": self.runner_params['k8s_pod_priority_class']
+                "priorityClassName": self.runner_params['k8s_pod_priority_class'],
+                "tolerations": yaml.safe_load(self.runner_params['k8s_tolerations'] or "[]"),
+                "affinity": yaml.safe_load(self.runner_params['k8s_affinity'] or "{}")
             }
         }
         # TODO include other relevant elements that people might want to use from

lib/galaxy/tool_util/deps/mulled/mulled_build.py

@@ -69,6 +69,7 @@ From: %(base_image)s
     cp -r /data/dist/* /tmp/conda/
 
 %%post
+    rm -R /usr/local || true
     mkdir -p /usr/local
     cp -R /tmp/conda/* /usr/local/
 
@@ -180,14 +181,8 @@ def mull_targets(
     conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True,
     oauth_token=None, hash_func="v2", singularity=False,
     singularity_image_dir="singularity_import", base_image=None,
+    determine_base_image=True,
 ):
-    if base_image:
-        dest_base_image = base_image
-    elif DEST_BASE_IMAGE:
-        dest_base_image = DEST_BASE_IMAGE
-    else:
-        dest_base_image = base_image_for_targets(targets)
-
     targets = list(targets)
     if involucro_context is None:
         involucro_context = InvolucroContext()
@@ -237,6 +232,13 @@ def mull_targets(
         '-set', "REPO='%s'" % repo,
         '-set', "BINDS='%s'" % bind_str,
     ]
+    dest_base_image = None
+    if base_image:
+        dest_base_image = base_image
+    elif DEST_BASE_IMAGE:
+        dest_base_image = DEST_BASE_IMAGE
+    elif determine_base_image:
+        dest_base_image = base_image_for_targets(targets)
 
     if dest_base_image:
         involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % dest_base_image])

lib/galaxy/tool_util/deps/mulled/mulled_build_files.py

@@ -44,6 +44,7 @@ def main(argv=None):
                 image_build=target.image_build,
                 name_override=target.name_override,
                 base_image=target.base_image,
+                determine_base_image=False,
                 **args_to_mull_targets_kwds(args)
             )
         except BuildExistsException:
@@ -79,6 +80,10 @@ def tuple_from_header(header):
     fields = header[1:].split('\t')
     for field in fields:
         assert field in KNOWN_FIELDS, "'%s' is not one of %s" % (field, KNOWN_FIELDS)
+    # Make sure tuple contains all fields
+    for field in KNOWN_FIELDS:
+        if field not in fields:
+            fields.append(field)
     return collections.namedtuple("_Line", "%s" % " ".join(fields))
 
 

packages/tool_util/galaxy/project_galaxy_tool_util.py

@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
 
-__version__ = '20.1.0.dev3'
+__version__ = '20.1.0.dev4'
 
 PROJECT_NAME = "galaxy-tool-util"
 PROJECT_OWNER = PROJECT_USERAME = "galaxyproject"

test/unit/tool_util/mulled/test_mulled_build_files.py

@@ -39,6 +39,13 @@ TESTCASES = yaml.safe_load(r"""
       image_build: '10'
       name_override: image_name
       targets: samtools
+- test_files_package_image_build_base_image_with_header:
+    content: "#targets\timage_build\tbase_image\nsamtools\t10\textended_image"
+    equals:
+      base_image: extended_image
+      image_build: '10'
+      name_override: null
+      targets: samtools
 - test_files_package_image_build_name_override_base_image_with_header_reordered:
     content: "#base_image\ttargets\timage_build\tname_override\nextended_image\tsamtools\t10\timage_name"
     equals: