Merge pull request #16125 from nsoranzo/release_22.05_fix_get_file_from_conda_package

[22.05] Fix ``get_test_from_anaconda()`` and ``base_image_for_targets()`` functions
This commit is contained in:
Nicola Soranzo
2023-05-25 11:58:59 +01:00
committed by GitHub
11 changed files with 170 additions and 145 deletions
+2
View File
@@ -42,6 +42,8 @@ jobs:
with:
path: .tox
key: tox-cache-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('galaxy root/requirements.txt') }}-mulled
- name: Install Apptainer's singularity
uses: eWaterCycle/setup-apptainer@v2
- name: Install tox
run: pip install tox
- name: run tests
+1 -1
View File
@@ -17,7 +17,7 @@ concurrency:
jobs:
test:
name: Test
runs-on: ubuntu-18.04
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ['3.7']
@@ -153,8 +153,6 @@ Multiple containers can be installed simultaneously by giving ``--containers`` m
$ mulled-update-singularity-containers --containers samtools:1.6--0 bamtools:2.4.1--0 --filepath /tmp/sing/ --installation /usr/local/bin/singularity
.. code-block:: bash
For a large number of containers, it may be more convenient to employ the ``--container-list`` option:
.. code-block:: bash
@@ -163,15 +161,19 @@ For a large number of containers, it may be more convenient to employ the ``--co
Here ``list.txt`` should contain a list of containers, each on a new line.
In order to generate the list file the ``mulled-list`` command may be useful. The following command returns a list of all Docker containers available on the quay.io biocontainers organization, excluding those already available as Singularity containers via https://depot.galaxyproject.org/singularity/.:: bash
In order to generate the list file the ``mulled-list`` command may be useful. The following command returns a list of all Docker containers available on the quay.io biocontainers organization, excluding those already available as Singularity containers on https://depot.galaxyproject.org/singularity/ .
.. code-block:: bash
$ mulled-list --source docker --not-singularity --blacklist blacklist.txt --file output.txt
The list of containers will be saved as ``output.txt``. The (optional) ``--blacklist`` option may be used to exclude containers which should not included in the output; ``blacklist.txt`` should contain a list of the 'blacklisted' containers, each on a new line.
Containers, once generated, should be tested. This can be achieved by affixing ``--testing test-output.log`` to the command, or alternatively, by use of the dedicated ``mulled-singularity-testing`` tool.:: bash
The generated containers should also be tested. This can be achieved by affixing ``--testing test-output.log`` to the ``mulled-update-singularity-containers`` command:
$ mulled-singularity-testing --container-list list.txt --filepath /tmp/sing/ --installation /usr/local/bin/singularity --logfile test-output.txt
.. code-block:: bash
$ mulled-update-singularity-containers --container-list list.txt --filepath /tmp/sing/ --installation /usr/local/bin/singularity --testing test-output.log
.. _IUC: https://galaxyproject.org/iuc/
.. _container annotation: https://github.com/galaxyproject/galaxy/blob/dev/test/functional/tools/catDocker.xml#L4
+2
View File
@@ -517,6 +517,8 @@ def best_search_result(
Return ``None`` if no results match.
"""
# Cannot specify the version here (i.e. conda_target.package_specifier)
# because if the version is not found, the exec_search() call would fail.
search_args = [conda_target.package]
try:
res = conda_context.exec_search(search_args, json=True, offline=offline, platform=platform)
+32 -18
View File
@@ -12,6 +12,7 @@ from glob import glob
from typing import (
Any,
Dict,
List,
Optional,
)
@@ -26,8 +27,9 @@ except ImportError:
UndefinedError = Exception # type: ignore[assignment,misc]
from galaxy.util import unicodify
from galaxy.util.commands import argv_to_str
from .util import (
get_file_from_conda_package,
get_files_from_conda_package,
MULLED_SOCKET_TIMEOUT,
split_container_name,
)
@@ -66,6 +68,7 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]:
return None
# need to know what scripting languages are needed to run the container
package_tests["import_lang"] = "python -c" # python by default
try:
requirements = list(meta_yaml["requirements"]["run"])
except (KeyError, TypeError):
@@ -77,8 +80,6 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]:
break
# elif ... :
# other languages if necessary ... hopefully python and perl should suffice though
else: # python by default
package_tests["import_lang"] = "python -c"
return package_tests
@@ -110,15 +111,16 @@ def get_test_from_anaconda(url: str) -> Optional[Dict[str, Any]]:
"""
Given the URL of an anaconda tarball, return tests
"""
name, content = get_file_from_conda_package(
content_dict = get_files_from_conda_package(
url, ["info/recipe/meta.yaml", "info/recipe/meta.yaml.template", "info/recipe/run_test.sh"]
)
if name and content and name.startswith("info/recipe/meta.yaml"):
content = content_dict.get("info/recipe/meta.yaml", content_dict.get("info/recipe/meta.yaml.template"))
if content:
package_tests = get_commands_from_yaml(content)
if package_tests:
return package_tests
if name and content and name == "info/recipe/run_test.sh":
return get_run_test(unicodify(content))
if "info/recipe/run_test.sh" in content_dict:
return get_run_test(unicodify(content_dict["info/recipe/run_test.sh"]))
return None
@@ -128,7 +130,7 @@ def find_anaconda_versions(name, anaconda_channel="bioconda"):
"""
r = requests.get(f"https://anaconda.org/{anaconda_channel}/{name}/files", timeout=MULLED_SOCKET_TIMEOUT)
urls = []
for line in r.text.split("\n"):
for line in r.text.splitlines():
if "download/linux" in line:
urls.append(line.split('"')[1])
return urls
@@ -259,23 +261,34 @@ def main_test_search(
return {"container": container}
def import_test_to_command_list(import_lang: str, import_: str) -> List[str]:
if import_lang == "python -c":
return ["python", "-c", f"import {import_}"]
elif import_lang == "perl -e":
return ["perl", "-e", f"use {import_}"]
else:
raise ValueError(f"Unsupported import_lang '{import_lang}'")
def hashed_test_search(
container, recipes_path=None, deep=False, anaconda_channel="bioconda", github_repo="bioconda/bioconda-recipes"
):
container: str, recipes_path=None, deep=False, anaconda_channel="bioconda", github_repo="bioconda/bioconda-recipes"
) -> Dict[str, Any]:
"""
Get test for hashed containers
"""
package_tests = {"commands": [], "imports": [], "container": container, "import_lang": "python -c"}
package_tests: Dict[str, Any] = {"commands": [], "imports": [], "container": container, "import_lang": "python -c"}
githubpage = requests.get(
response = requests.get(
f"https://raw.githubusercontent.com/BioContainers/multi-package-containers/master/combinations/{container}.tsv",
timeout=MULLED_SOCKET_TIMEOUT,
)
if githubpage.status_code == 200:
packages = githubpage.text.split(",") # get names of packages from github
packages = [package.split("=") for package in packages]
else:
packages = []
response.raise_for_status()
for line in response.text.splitlines():
if not line.startswith("#"):
break
concatenated_targets = line.split("\t")[0]
targets = concatenated_targets.split(",")
packages = [target.split("=") for target in targets]
containers = []
for package in packages:
@@ -293,7 +306,8 @@ def hashed_test_search(
for container in containers:
tests = main_test_search(container, recipes_path, deep, anaconda_channel, github_repo)
package_tests["commands"] += tests.get("commands", []) # not a very nice solution but probably the simplest
# Given that this could be a mix of Python and Perl packages, translate imports to commands
for imp in tests.get("imports", []):
package_tests["imports"].append(f"{tests['import_lang']} 'import {imp}'")
package_tests["commands"].append(argv_to_str(import_test_to_command_list(tests["import_lang"], imp)))
return package_tests
@@ -43,7 +43,7 @@ from .util import (
conda_build_target_str,
create_repository,
default_mulled_conda_channels_from_env,
get_file_from_conda_package,
get_files_from_conda_package,
PrintProgress,
quay_repository,
v1_image_name,
@@ -170,14 +170,16 @@ def base_image_for_targets(targets: List["Target"], conda_context: CondaContext)
hits = get_conda_hits_for_targets(targets, conda_context)
for hit in hits:
try:
name, content = get_file_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])
strcontent = unicodify(content)
if name == "info/about.json" and json.loads(strcontent).get("extra", {}).get("container", {}).get(
"extended-base", False
):
content_dict = get_files_from_conda_package(hit["url"], ["info/about.json", "info/recipe/meta.yaml"])
if "info/about.json" in content_dict and json.loads(unicodify(content_dict["info/about.json"])).get(
"extra", {}
).get("container", {}).get("extended-base", False):
return DEFAULT_EXTENDED_BASE_IMAGE
elif name == "info/recipe/meta.yaml" and (
yaml.safe_load(strcontent).get("extra", {}).get("container", {}).get("extended-base", False)
elif "info/recipe/meta.yaml" in content_dict and (
yaml.safe_load(unicodify(content_dict["info/recipe/meta.yaml"]))
.get("extra", {})
.get("container", {})
.get("extended-base", False)
):
return DEFAULT_EXTENDED_BASE_IMAGE
except Exception:
@@ -1,14 +1,23 @@
#!/usr/bin/env python
import argparse
import os
import os.path
import subprocess
import tempfile
from glob import glob
from subprocess import check_output
from typing import (
Any,
Dict,
List,
Union,
)
from galaxy.util import unicodify
from .get_tests import (
hashed_test_search,
import_test_to_command_list,
main_test_search,
)
@@ -25,7 +34,7 @@ def docker_to_singularity(container, installation, filepath, no_sudo=False):
"""
Convert docker to singularity container.
"""
cmd = [installation, "build", "/".join((filepath, container)), f"docker://quay.io/biocontainers/{container}"]
cmd = [installation, "build", os.path.join(filepath, container), f"docker://quay.io/biocontainers/{container}"]
try:
if no_sudo:
check_output(cmd, stderr=subprocess.STDOUT)
@@ -36,47 +45,47 @@ def docker_to_singularity(container, installation, filepath, no_sudo=False):
raise Exception(f"Docker to Singularity conversion failed.\nOutput was:\n{unicodify(e.output)}")
def singularity_container_test(tests, installation, filepath):
def singularity_container_test(
tests: Dict[str, Dict[str, Any]], installation: str, filepath: Union[str, os.PathLike]
) -> Dict[str, List]:
"""
Run tests, record if they pass or fail
"""
test_results = {"passed": [], "failed": [], "notest": []}
test_results: Dict[str, List] = {"passed": [], "failed": [], "notest": []}
# create a 'sanitised home' directory in which the containers may be mounted - see http://singularity.lbl.gov/faq#solution-1-specify-the-home-to-mount
with tempfile.TemporaryDirectory() as tmpdirname:
for container, test in tests.items():
if "commands" not in test and "imports" not in test:
test_results["notest"].append(container)
else:
exec_command = [installation, "exec", "-H", tmpdirname, "/".join((filepath, container))]
exec_command = [installation, "exec", "-H", tmpdirname, os.path.join(filepath, container)]
test_passed = True
errors = []
if test.get("commands", False):
for test_command in test["commands"]:
test_command = test_command.replace("$PREFIX", "/usr/local/")
test_command = test_command.replace("${PREFIX}", "/usr/local/")
test_command = test_command.replace("$R ", "Rscript ")
for test_command in test.get("commands", []):
test_command = test_command.replace("$PREFIX", "/usr/local/")
test_command = test_command.replace("${PREFIX}", "/usr/local/")
test_command = test_command.replace("$R ", "Rscript ")
try:
check_output(exec_command + ["bash", "-c", test_command], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
try:
check_output(exec_command.extend(["bash", "-c", test_command]), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
try:
check_output(exec_command.append(test_command), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
errors.append({"command": test_command, "output": unicodify(e.output)})
test_passed = False
if test.get("imports", False):
for imp in test["imports"]:
try:
check_output(
exec_command.extend([test["import_lang"], f"import {imp}"]), stderr=subprocess.STDOUT
)
check_output(exec_command + [test_command], stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
errors.append({"import": imp, "output": unicodify(e.output)})
errors.append({"command": test_command, "output": unicodify(e.output)})
test_passed = False
for imp in test.get("imports", []):
try:
check_output(
exec_command + import_test_to_command_list(test["import_lang"], imp),
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as e:
errors.append({"import": imp, "output": unicodify(e.output)})
test_passed = False
if test_passed:
test_results["passed"].append(container)
else:
+17 -29
View File
@@ -8,12 +8,10 @@ import re
import sys
import threading
from typing import (
Dict,
Iterable,
List,
Optional,
Tuple,
TYPE_CHECKING,
Union,
)
import packaging.version
@@ -350,42 +348,31 @@ def v2_image_name(targets, image_build=None, name_override=None):
return f"mulled-v2-{package_hash.hexdigest()}{suffix}"
def get_file_from_conda_package(
url: str, checklist: Union[str, Iterable[str]]
) -> Tuple[Optional[str], Optional[bytes]]:
def get_files_from_conda_package(url: str, filepaths: Iterable[str]) -> Dict[str, bytes]:
"""
Get file content for an element in a conda package.
Get content of specified files in a conda package.
The url can be a path to a local file or an url.
The checklist can be the name of the element to etract
or a Iterable of potentially desired elements the content
of the first that is contained in the conda package
is returned.
Return the name and content (bytes) of the first found element
and None, None otherwise
The filepaths is an iterable of paths to extract from the conda package, if
found in it.
Return a dictionary mapping each found filepath to the corresponding content
(as bytes).
>>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2", "info/recipe/meta.yaml")
>>> assert name == "info/recipe/meta.yaml"
>>> assert isinstance(content, bytes)
>>> name, content = get_file_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda", ["info/about.json", "info/recipe/meta.yaml"])
>>> assert name == "info/recipe/meta.yaml"
>>> assert isinstance(content, bytes)
>>> content_dict = get_files_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.6/download/noarch/chopin2-1.0.6-pyhd8ed1ab_0.tar.bz2", ["info/recipe/meta.yaml"])
>>> assert "info/recipe/meta.yaml" in content_dict, content_dict
>>> assert isinstance(content_dict["info/recipe/meta.yaml"], bytes)
>>> content_dict = get_files_from_conda_package("https://anaconda.org/conda-forge/chopin2/1.0.7/download/noarch/chopin2-1.0.7-pyhd8ed1ab_1.conda", ["info/about.json", "info/recipe/meta.yaml", "foo/bar"])
>>> assert sorted(content_dict.keys()) == ["info/about.json", "info/recipe/meta.yaml"], content_dict
"""
if isinstance(checklist, str):
checklist = set([checklist])
else:
checklist = set(checklist)
# print(checklist)
try:
stream = stream_conda_info(url)
except FileNotFoundError:
stream = stream_conda_info_from_url(url)
ret = {}
for tar, member in stream:
# print(member.name)
if member.name in checklist:
return member.name, tar.extractfile(member).read()
# print("None")
return None, None
if member.name in filepaths:
ret[member.name] = tar.extractfile(member).read()
return ret
def split_container_name(name):
@@ -423,6 +410,7 @@ image_name = v1_image_name # deprecated
__all__ = (
"build_target",
"conda_build_target_str",
"get_files_from_conda_package",
"image_name",
"mulled_tags_for",
"quay_versions",
+6 -5
View File
@@ -42,7 +42,6 @@ def test_get_run_test():
assert commands["commands"] == [' #!/bin/bash && pslScore 2> /dev/null || [[ "$?" == 255 ]]']
@external_dependency_management
def test_get_anaconda_url():
url = get_anaconda_url("samtools:1.7--1")
assert url == "https://anaconda.org/bioconda/samtools/1.7/download/linux-64/samtools-1.7-1.tar.bz2"
@@ -72,9 +71,11 @@ def test_get_test_from_anaconda():
# test for package defining tests in info/recipe/run_test.sh
tests = get_test_from_anaconda(
"https://anaconda.org/bioconda/ucsc-pslmap/366/download/linux-64/ucsc-pslmap-366-hdd26221_0.tar.bz2"
"https://anaconda.org/bioconda/mugsy/1.2.3/download/noarch/mugsy-1.2.3-hdfd78af_4.tar.bz2"
)
assert tests and tests["commands"] == ['#!/bin/bash && pslMap 2> /dev/null || [[ "$?" == 255 ]] && ']
assert tests and tests["commands"] == [
"#!/bin/bash && && export MUGSY_INSTALL=${PREFIX}/bin && mugsy -h | grep mugsy > /dev/null && mugsyWGA --version && synchain-mugsy 2>&1 | grep mugsy > /dev/null && "
] # This script is clearly broken, but the whole get_tests module is currently far from usable
@external_dependency_management
@@ -91,8 +92,8 @@ def test_open_recipe_file():
@external_dependency_management
def test_get_alternative_versions():
versions = get_alternative_versions("recipes/bamtools", "meta.yaml")
assert versions == ["recipes/bamtools/2.3.0/meta.yaml"]
versions = get_alternative_versions("recipes/bioblend", "meta.yaml")
assert versions == ["recipes/bioblend/0.7.0/meta.yaml"]
@external_dependency_management
@@ -1,9 +1,8 @@
import os
import shutil
import tempfile
import pytest
from galaxy.tool_util.deps.mulled.get_tests import main_test_search
from galaxy.tool_util.deps.mulled.mulled_update_singularity_containers import (
docker_to_singularity,
get_list_from_file,
@@ -13,59 +12,42 @@ from galaxy.util import which
from ..util import external_dependency_management
def test_get_list_from_file():
test_dir = tempfile.mkdtemp()
try:
list_file = os.path.join(test_dir, "list_file.txt")
with open(list_file, "w") as f:
f.write("bbmap:36.84--0\nbiobambam:2.0.42--0\nconnor:0.5.1--py35_0\ndiamond:0.8.26--0\nedd:1.1.18--py27_0")
assert get_list_from_file(list_file) == [
"bbmap:36.84--0",
"biobambam:2.0.42--0",
"connor:0.5.1--py35_0",
"diamond:0.8.26--0",
"edd:1.1.18--py27_0",
]
finally:
shutil.rmtree(test_dir)
def test_get_list_from_file(tmp_path) -> None:
list_file = os.path.join(tmp_path, "list_file.txt")
with open(list_file, "w") as f:
f.write("bbmap:36.84--0\nbiobambam:2.0.42--0\nconnor:0.5.1--py35_0\ndiamond:0.8.26--0\nedd:1.1.18--py27_0")
assert get_list_from_file(list_file) == [
"bbmap:36.84--0",
"biobambam:2.0.42--0",
"connor:0.5.1--py35_0",
"diamond:0.8.26--0",
"edd:1.1.18--py27_0",
]
@external_dependency_management
@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")
def test_docker_to_singularity(tmp_path):
tmp_dir = str(tmp_path)
docker_to_singularity("abundancebin:1.0.1--0", "singularity", tmp_dir, no_sudo=True)
def test_docker_to_singularity(tmp_path) -> None:
docker_to_singularity("abundancebin:1.0.1--0", "singularity", tmp_path, no_sudo=True)
assert tmp_path.joinpath("abundancebin:1.0.1--0").exists()
@external_dependency_management
@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")
def test_singularity_container_test(tmp_path):
test_dir = tempfile.mkdtemp()
try:
for n in ["pybigwig:0.1.11--py36_0", "samtools:1.0--1", "yasm:1.3.0--0"]:
docker_to_singularity(n, "singularity", test_dir, no_sudo=True)
results = singularity_container_test(
{
"pybigwig:0.1.11--py36_0": {
"imports": ["pyBigWig"],
"commands": [
'python -c "import pyBigWig; assert(pyBigWig.numpy == 1); assert(pyBigWig.remote == 1)"'
],
"import_lang": "python -c",
},
"samtools:1.0--1": {
"commands": ["samtools --help"],
"import_lang": "python -c",
"container": "samtools:1.0--1",
},
"yasm:1.3.0--0": {},
},
"singularity",
test_dir,
)
assert "samtools:1.0--1" in results["passed"]
assert results["failed"][0]["imports"] == ["pyBigWig"]
assert "yasm:1.3.0--0" in results["notest"]
finally:
shutil.rmtree(test_dir)
def test_singularity_container_test(tmp_path) -> None:
containers = [
"pybigwig:0.3.22--py36h54a71a5_0", # test Python imports
"samtools:1.0--1",
"yasm:1.3.0--0", # test missing tests
]
tests = {container: main_test_search(container) for container in containers}
for n in tests.keys():
docker_to_singularity(n, "singularity", tmp_path, no_sudo=True)
results = singularity_container_test(
tests,
"singularity",
tmp_path,
)
assert "samtools:1.0--1" in results["passed"]
assert "pybigwig:0.3.22--py36h54a71a5_0" in results["passed"]
assert "yasm:1.3.0--0" in results["notest"]
+28 -5
View File
@@ -2,9 +2,13 @@ import os
import shutil
from tempfile import mkdtemp
from galaxy.tool_util.deps import (
conda_util,
DependencyManager,
from galaxy.tool_util.deps import DependencyManager
from galaxy.tool_util.deps.conda_util import (
best_search_result,
CondaContext,
CondaTarget,
install_conda,
installed_conda_targets,
)
from galaxy.tool_util.deps.requirements import ToolRequirement
from galaxy.tool_util.deps.resolvers.conda import CondaDependencyResolver
@@ -51,11 +55,30 @@ def test_against_conda_prefix_regression():
use_path_exec=False, # For the test ensure this is always a clean install
)
conda_context = resolver.conda_context
assert len(list(conda_util.installed_conda_targets(conda_context))) == 0
assert len(list(installed_conda_targets(conda_context))) == 0
req = ToolRequirement(name="samtools", version="0.1.16", type="package")
dependency = resolver.resolve(req, job_directory=job_dir)
assert dependency.shell_commands() is not None # install should not fail anymore
installed_targets = list(conda_util.installed_conda_targets(conda_context))
installed_targets = list(installed_conda_targets(conda_context))
assert len(installed_targets) > 0
finally:
shutil.rmtree(base_path)
@external_dependency_management
def test_best_search_result(tmp_path) -> None:
conda_context = CondaContext(
conda_prefix=os.path.join(tmp_path, "_conda"), condarc_override=os.path.join(tmp_path, "_condarc")
)
install_conda(conda_context)
(hit, exact) = best_search_result(CondaTarget("samtools"), conda_context)
assert hit["name"] == "samtools"
assert exact is True
(hit, exact) = best_search_result(CondaTarget("samtools", version="1.3.1"), conda_context)
assert hit["name"] == "samtools"
assert hit["version"] == "1.3.1"
assert exact is True
# Search non-existent version
(hit, exact) = best_search_result(CondaTarget("samtools", version="1.16"), conda_context)
assert hit["name"] == "samtools"
assert exact is False