From 5053dd3510478b74ea19ee17d854d53a4830988b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Sun, 9 Oct 2022 13:06:51 +0200 Subject: [PATCH 1/8] feat: mamba support for mulled-build --- lib/galaxy/tool_util/deps/mulled/invfile.lua | 3 +- .../tool_util/deps/mulled/mulled_build.py | 33 +++++++++++++++++-- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/invfile.lua b/lib/galaxy/tool_util/deps/mulled/invfile.lua index 1f3e0389862..467e6db0fd2 100644 --- a/lib/galaxy/tool_util/deps/mulled/invfile.lua +++ b/lib/galaxy/tool_util/deps/mulled/invfile.lua @@ -45,6 +45,7 @@ if conda_image == '' then conda_image = 'continuumio/miniconda3:latest' end +local conda_bin = VAR.CONDA_BIN local singularity_image = VAR.SINGULARITY_IMAGE if singularity_image == '' then @@ -87,7 +88,7 @@ inv.task('build') .using(conda_image) .withHostConfig({binds = bind_args}) .run('/bin/sh', '-c', preinstall - .. 'conda install ' + .. conda_bin .. ' install ' .. channel_args .. ' ' .. target_args .. ' --strict-channel-priority -p /usr/local --copy --yes ' diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index beb8b693f1f..b97160cad2b 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -197,6 +197,8 @@ def mull_targets( repository_template=DEFAULT_REPOSITORY_TEMPLATE, dry_run=False, conda_version=None, + mamba_version=None, + use_mamba=False, verbose=False, binds=DEFAULT_BINDS, rebuild=True, @@ -284,9 +286,18 @@ def mull_targets( involucro_args.extend(["-set", f"USER_ID={os.getuid()}:{os.getgid()}"]) if test: involucro_args.extend(["-set", f"TEST={test}"]) - if conda_version is not None: - verbose = "--verbose" if verbose else "--quiet" - involucro_args.extend(["-set", f"PREINSTALL=conda install {verbose} --yes conda={conda_version}"]) + + verbose = "--verbose" if verbose else "--quiet" + conda_bin = "conda" + if use_mamba: + constraint = "" if mamba_version is None else f"={mamba_version}" + involucro_args.extend(["-set", f"PREINSTALL=conda install {verbose} --yes mamba{constraint}"]) + conda_bin = "mamba" + else: + if conda_version is not None: + involucro_args.extend(["-set", f"PREINSTALL=conda install {verbose} --yes conda={conda_version}"]) + involucro_args.extend(["-set", "CONDA_BIN=%s" % conda_bin]) + involucro_args.append(command) if test_files: test_bind = [] @@ -456,6 +467,18 @@ def add_build_arguments(parser): default=None, help="Change to specified version of Conda before installing packages.", ) + parser.add_argument( + "--mamba-version", + dest="mamba_version", + default=None, + help="Change to specified version of Mamba before installing packages.", + ) + parser.add_argument( + "--use-mamba", + dest="use_mamba", + action="store_true", + help="Use Mamba instead of Conda for package installation.", + ) parser.add_argument( "--oauth-token", dest="oauth_token", @@ -521,6 +544,10 @@ def args_to_mull_targets_kwds(args): kwds["repository_template"] = args.repository_template if hasattr(args, "conda_version"): kwds["conda_version"] = args.conda_version + if hasattr(args, "mamba_version"): + kwds["mamba_version"] = args.mamba_version + if hasattr(args, "use_mamba"): + kwds["use_mamba"] = args.use_mamba if hasattr(args, "oauth_token"): kwds["oauth_token"] = args.oauth_token if hasattr(args, "rebuild"): From e2723be5a1524d6d2e276973ca6eb15e9a654d1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20K=C3=B6ster?= Date: Sun, 9 Oct 2022 13:20:28 +0200 Subject: [PATCH 2/8] extend test case --- test/unit/tool_util/mulled/test_mulled_build.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/test/unit/tool_util/mulled/test_mulled_build.py b/test/unit/tool_util/mulled/test_mulled_build.py index ce48a4b9fca..d161545e7ae 100644 --- a/test/unit/tool_util/mulled/test_mulled_build.py +++ b/test/unit/tool_util/mulled/test_mulled_build.py @@ -27,11 +27,18 @@ def test_base_image_for_targets(target, version, base_image): assert base_image_for_targets([target], conda_context) == base_image +@pytest.mark.parametrize("use_mamba", [False, True]) @external_dependency_management -def test_mulled_build_files_cli(tmpdir): +def test_mulled_build_files_cli(target, tmpdir): singularity_image_dir = tmpdir.mkdir("singularity image dir") target = build_target("zlib") - mull_targets([target], command="build-and-test", singularity=True, singularity_image_dir=singularity_image_dir) + mull_targets( + [target], + command="build-and-test", + singularity=True, + use_mamba=use_mamba, + singularity_image_dir=singularity_image_dir, + ) assert singularity_image_dir.join("zlib").exists() From 10ed75a177ba5be43abb56e2ef5a4f651ab8c72b Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Thu, 13 Oct 2022 10:29:27 +0200 Subject: [PATCH 3/8] Tweak use_mamba logic, skip install in mamba is installed Decouple use_mamba and the version requirements (e.g., one could use mamba, but still require a specific conda version). Also, if both conda_version and mamba_version are given, still have only one conda install invocation. Even if mamba_version is None, we can set append an empty constraint = since conda install handles mamba= and mamba equivalently. Skip conda install mamba if mamba is already installed. This is, e.g., the case in the quay.io/bioconda/create-env container image we use in bioconda-utils. Co-authored-by: Marcel Bargull --- .../tool_util/deps/mulled/mulled_build.py | 20 ++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index b97160cad2b..84dfe1e6605 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -290,13 +290,23 @@ def mull_targets( verbose = "--verbose" if verbose else "--quiet" conda_bin = "conda" if use_mamba: - constraint = "" if mamba_version is None else f"={mamba_version}" - involucro_args.extend(["-set", f"PREINSTALL=conda install {verbose} --yes mamba{constraint}"]) conda_bin = "mamba" - else: - if conda_version is not None: - involucro_args.extend(["-set", f"PREINSTALL=conda install {verbose} --yes conda={conda_version}"]) + if mamba_version is None: + mamba_version = "" involucro_args.extend(["-set", "CONDA_BIN=%s" % conda_bin]) + if conda_version is not None or mamba_version is not None: + mamba_test = "true" + specs = [] + if conda_version is not None: + specs.append(f"conda={conda_version}") + if mamba_version is not None: + specs.append(f"mamba={mamba_version}") + if mamba_version == "" and not specs: + # If nothing but mamba without a specific version is requested, + # then only run conda install if mamba is not already installed. + mamba_test = f"[ '[]' = \"$( conda list --json --full-name mamba )\" ]" + conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}""" + involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"]} involucro_args.append(command) if test_files: From 32c330809e7a9b58e49a3a4b6b170f0a7aaad745 Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Thu, 13 Oct 2022 10:31:54 +0200 Subject: [PATCH 4/8] Fix parametrized test Co-authored-by: Marcel Bargull --- test/unit/tool_util/mulled/test_mulled_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/unit/tool_util/mulled/test_mulled_build.py b/test/unit/tool_util/mulled/test_mulled_build.py index d161545e7ae..fec67ff115b 100644 --- a/test/unit/tool_util/mulled/test_mulled_build.py +++ b/test/unit/tool_util/mulled/test_mulled_build.py @@ -29,7 +29,7 @@ def test_base_image_for_targets(target, version, base_image): @pytest.mark.parametrize("use_mamba", [False, True]) @external_dependency_management -def test_mulled_build_files_cli(target, tmpdir): +def test_mulled_build_files_cli(use_mamba, tmpdir): singularity_image_dir = tmpdir.mkdir("singularity image dir") target = build_target("zlib") mull_targets( From 803cc3d270910c98e2910bad0b818a958a2342ac Mon Sep 17 00:00:00 2001 From: Marius van den Beek Date: Thu, 13 Oct 2022 10:32:23 +0200 Subject: [PATCH 5/8] Drop unused f-string marker --- lib/galaxy/tool_util/deps/mulled/mulled_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 84dfe1e6605..f7264363a57 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -304,7 +304,7 @@ def mull_targets( if mamba_version == "" and not specs: # If nothing but mamba without a specific version is requested, # then only run conda install if mamba is not already installed. - mamba_test = f"[ '[]' = \"$( conda list --json --full-name mamba )\" ]" + mamba_test = "[ '[]' = \"$( conda list --json --full-name mamba )\" ]" conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}""" involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"]} From bd6b881f8d89030aa7edc61980c34888f7ebc0d8 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Oct 2022 10:36:42 +0200 Subject: [PATCH 6/8] Fix syntax error --- lib/galaxy/tool_util/deps/mulled/mulled_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index f7264363a57..7f99a829b3f 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -306,7 +306,7 @@ def mull_targets( # then only run conda install if mamba is not already installed. mamba_test = "[ '[]' = \"$( conda list --json --full-name mamba )\" ]" conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}""" - involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"]} + involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"]) involucro_args.append(command) if test_files: From f92d4b3f6133f5dcb53f7f156e8496d7bb45b2ad Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 13 Oct 2022 10:55:33 +0200 Subject: [PATCH 7/8] Add conda-forge channel when installing mamba --- lib/galaxy/tool_util/deps/mulled/mulled_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 7f99a829b3f..766c4d3c6ee 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -305,7 +305,7 @@ def mull_targets( # If nothing but mamba without a specific version is requested, # then only run conda install if mamba is not already installed. mamba_test = "[ '[]' = \"$( conda list --json --full-name mamba )\" ]" - conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}""" + conda_install = f"""conda install {verbose} -c conda-forge --yes {" ".join(f"'{spec}'" for spec in specs)}""" involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"]) involucro_args.append(command) From 163ad7ed82700e96e8981efb72e1a5393484b744 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Mon, 17 Oct 2022 12:18:37 +0200 Subject: [PATCH 8/8] Switch conda install image, drop extra -c conda-forge --- lib/galaxy/tool_util/deps/mulled/invfile.lua | 2 +- lib/galaxy/tool_util/deps/mulled/mulled_build.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/deps/mulled/invfile.lua b/lib/galaxy/tool_util/deps/mulled/invfile.lua index 467e6db0fd2..98dea273bd9 100644 --- a/lib/galaxy/tool_util/deps/mulled/invfile.lua +++ b/lib/galaxy/tool_util/deps/mulled/invfile.lua @@ -42,7 +42,7 @@ end local conda_image = VAR.CONDA_IMAGE if conda_image == '' then - conda_image = 'continuumio/miniconda3:latest' + conda_image = 'quay.io/condaforge/mambaforge:latest' end local conda_bin = VAR.CONDA_BIN diff --git a/lib/galaxy/tool_util/deps/mulled/mulled_build.py b/lib/galaxy/tool_util/deps/mulled/mulled_build.py index 766c4d3c6ee..7f99a829b3f 100644 --- a/lib/galaxy/tool_util/deps/mulled/mulled_build.py +++ b/lib/galaxy/tool_util/deps/mulled/mulled_build.py @@ -305,7 +305,7 @@ def mull_targets( # If nothing but mamba without a specific version is requested, # then only run conda install if mamba is not already installed. mamba_test = "[ '[]' = \"$( conda list --json --full-name mamba )\" ]" - conda_install = f"""conda install {verbose} -c conda-forge --yes {" ".join(f"'{spec}'" for spec in specs)}""" + conda_install = f"""conda install {verbose} --yes {" ".join(f"'{spec}'" for spec in specs)}""" involucro_args.extend(["-set", f"PREINSTALL=if {mamba_test} ; then {conda_install} ; fi"]) involucro_args.append(command)