diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 3a3e5437c63..4e24129f482 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -40,9 +40,9 @@ jobs: python-version: ${{ matrix.python-version }} - name: Cache venv dir uses: actions/cache@v1 - id: cache-venv-galaxy + id: pip-cache with: - path: .venv - key: cache-venv-galaxy-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} + path: ~/.cache/pip + key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }} - name: Run tests run: './run_tests.sh -integration test/integration -- -k "${{ matrix.subset }}"' diff --git a/lib/galaxy/jobs/runners/kubernetes.py b/lib/galaxy/jobs/runners/kubernetes.py index 4201a7e2b57..0ef82aacd06 100644 --- a/lib/galaxy/jobs/runners/kubernetes.py +++ b/lib/galaxy/jobs/runners/kubernetes.py @@ -8,6 +8,8 @@ import os import re from time import sleep +import yaml + from galaxy import model from galaxy.jobs.runners import ( AsynchronousJobRunner, @@ -46,6 +48,8 @@ class KubernetesJobRunner(AsynchronousJobRunner): k8s_persistent_volume_claims=dict(map=str), k8s_namespace=dict(map=str, default="default"), k8s_pod_priority_class=dict(map=str, default=None), + k8s_affinity=dict(map=str, default=None), + k8s_tolerations=dict(map=str, default=None), k8s_galaxy_instance_id=dict(map=str), k8s_timeout_seconds_job_deletion=dict(map=int, valid=lambda x: int > 0, default=30), k8s_job_api_version=dict(map=str, default=DEFAULT_JOB_API_VERSION), @@ -241,7 +245,9 @@ class KubernetesJobRunner(AsynchronousJobRunner): "volumes": self.runner_params['k8s_mountable_volumes'], "restartPolicy": self.__get_k8s_restart_policy(ajs.job_wrapper), "containers": self.__get_k8s_containers(ajs), - "priorityClassName": self.runner_params['k8s_pod_priority_class'] + "priorityClassName": self.runner_params['k8s_pod_priority_class'], + "tolerations": yaml.safe_load(self.runner_params['k8s_tolerations'] or "[]"), + "affinity": yaml.safe_load(self.runner_params['k8s_affinity'] or "{}") } } # TODO include other relevant elements that people might want to use from diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 9b28850914c..94e0b5f7162 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -69,6 +69,7 @@ From: %(base_image)s cp -r /data/dist/* /tmp/conda/ %%post + rm -R /usr/local || true mkdir -p /usr/local cp -R /tmp/conda/* /usr/local/ @@ -180,14 +181,8 @@ def mull_targets( conda_version=None, verbose=False, binds=DEFAULT_BINDS, rebuild=True, oauth_token=None, hash_func="v2", singularity=False, singularity_image_dir="singularity_import", base_image=None, + determine_base_image=True, ): - if base_image: - dest_base_image = base_image - elif DEST_BASE_IMAGE: - dest_base_image = DEST_BASE_IMAGE - else: - dest_base_image = base_image_for_targets(targets) - targets = list(targets) if involucro_context is None: involucro_context = InvolucroContext() @@ -237,6 +232,13 @@ def mull_targets( '-set', "REPO='%s'" % repo, '-set', "BINDS='%s'" % bind_str, ] + dest_base_image = None + if base_image: + dest_base_image = base_image + elif DEST_BASE_IMAGE: + dest_base_image = DEST_BASE_IMAGE + elif determine_base_image: + dest_base_image = base_image_for_targets(targets) if dest_base_image: involucro_args.extend(["-set", "DEST_BASE_IMAGE='%s'" % dest_base_image]) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py b/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py index 477d48eb2d2..24b554e4695 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build_files.py @@ -44,6 +44,7 @@ def main(argv=None): image_build=target.image_build, name_override=target.name_override, base_image=target.base_image, + determine_base_image=False, **args_to_mull_targets_kwds(args) ) except BuildExistsException: @@ -79,6 +80,10 @@ def tuple_from_header(header): fields = header[1:].split('\t') for field in fields: assert field in KNOWN_FIELDS, "'%s' is not one of %s" % (field, KNOWN_FIELDS) + # Make sure tuple contains all fields + for field in KNOWN_FIELDS: + if field not in fields: + fields.append(field) return collections.namedtuple("_Line", "%s" % " ".join(fields)) diff --git a/packages/tool_util/galaxy/project_galaxy_tool_util.py b/packages/tool_util/galaxy/project_galaxy_tool_util.py index 24e778668d6..bea1fcc0870 100644 --- a/packages/tool_util/galaxy/project_galaxy_tool_util.py +++ b/packages/tool_util/galaxy/project_galaxy_tool_util.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- -__version__ = '20.1.0.dev3' +__version__ = '20.1.0.dev4' PROJECT_NAME = "galaxy-tool-util" PROJECT_OWNER = PROJECT_USERAME = "galaxyproject" diff --git a/test/unit/tool_util/mulled/test_mulled_build_files.py b/test/unit/tool_util/mulled/test_mulled_build_files.py index dc7b07455bf..ed8b22f4e56 100644 --- a/test/unit/tool_util/mulled/test_mulled_build_files.py +++ b/test/unit/tool_util/mulled/test_mulled_build_files.py @@ -39,6 +39,13 @@ TESTCASES = yaml.safe_load(r""" image_build: '10' name_override: image_name targets: samtools +- test_files_package_image_build_base_image_with_header: + content: "#targets\timage_build\tbase_image\nsamtools\t10\textended_image" + equals: + base_image: extended_image + image_build: '10' + name_override: null + targets: samtools - test_files_package_image_build_name_override_base_image_with_header_reordered: content: "#base_image\ttargets\timage_build\tname_override\nextended_image\tsamtools\t10\timage_name" equals: