Fix testing importing modules in containers

This commit is contained in:
Nicola Soranzo
2023-05-25 08:32:21 +01:00
parent 1fedac75ab
commit a0d6e87fcc
3 changed files with 27 additions and 22 deletions
+14 -3
View File
@@ -12,6 +12,7 @@ from glob import glob
from typing import (
Any,
Dict,
List,
Optional,
)
@@ -26,6 +27,7 @@ except ImportError:
UndefinedError = Exception # type: ignore[assignment,misc]
from galaxy.util import unicodify
from galaxy.util.commands import argv_to_str
from .util import (
get_files_from_conda_package,
MULLED_SOCKET_TIMEOUT,
@@ -66,6 +68,7 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]:
return None
# need to know what scripting languages are needed to run the container
package_tests["import_lang"] = "python -c" # python by default
try:
requirements = list(meta_yaml["requirements"]["run"])
except (KeyError, TypeError):
@@ -77,8 +80,6 @@ def get_commands_from_yaml(yaml_content: bytes) -> Optional[Dict[str, Any]]:
break
# elif ... :
# other languages if necessary ... hopefully python and perl should suffice though
else: # python by default
package_tests["import_lang"] = "python -c"
return package_tests
@@ -260,6 +261,15 @@ def main_test_search(
return {"container": container}
def import_test_to_command_list(import_lang: str, import_: str) -> List[str]:
if import_lang == "python -c":
return ["python", "-c", f"import {import_}"]
elif import_lang == "perl -e":
return ["perl", "-e", f"use {import_}"]
else:
raise ValueError(f"Unsupported import_lang '{import_lang}'")
def hashed_test_search(
container, recipes_path=None, deep=False, anaconda_channel="bioconda", github_repo="bioconda/bioconda-recipes"
):
@@ -294,7 +304,8 @@ def hashed_test_search(
for container in containers:
tests = main_test_search(container, recipes_path, deep, anaconda_channel, github_repo)
package_tests["commands"] += tests.get("commands", []) # not a very nice solution but probably the simplest
# Given that this could be a mix of Python and Perl packages, translate imports to commands
for imp in tests.get("imports", []):
package_tests["imports"].append(f"{tests['import_lang']} 'import {imp}'")
package_tests["commands"].append(argv_to_str(import_test_to_command_list(tests["import_lang"], imp)))
return package_tests
@@ -17,6 +17,7 @@ from typing import (
from galaxy.util import unicodify
from .get_tests import (
hashed_test_search,
import_test_to_command_list,
main_test_search,
)
@@ -77,7 +78,10 @@ def singularity_container_test(
for imp in test.get("imports", []):
try:
check_output(exec_command + [test["import_lang"], f"import {imp}"], stderr=subprocess.STDOUT)
check_output(
exec_command + import_test_to_command_list(test["import_lang"], imp),
stderr=subprocess.STDOUT,
)
except subprocess.CalledProcessError as e:
errors.append({"import": imp, "output": unicodify(e.output)})
test_passed = False
@@ -1,11 +1,8 @@
import os
from typing import (
Any,
Dict,
)
import pytest
from galaxy.tool_util.deps.mulled.get_tests import main_test_search
from galaxy.tool_util.deps.mulled.mulled_update_singularity_containers import (
docker_to_singularity,
get_list_from_file,
@@ -38,19 +35,12 @@ def test_docker_to_singularity(tmp_path) -> None:
@external_dependency_management
@pytest.mark.skipif(not which("singularity"), reason="requires singularity but singularity not on PATH")
def test_singularity_container_test(tmp_path) -> None:
tests: Dict[str, Dict[str, Any]] = {
"pybigwig:0.1.11--py36_0": {
"imports": ["pyBigWig"],
"commands": ['python -c "import pyBigWig; assert(pyBigWig.numpy == 1); assert(pyBigWig.remote == 1)"'],
"import_lang": "python -c",
},
"samtools:1.0--1": {
"commands": ["samtools --help"],
"import_lang": "python -c",
"container": "samtools:1.0--1",
},
"yasm:1.3.0--0": {},
}
containers = [
"pybigwig:0.3.22--py36h54a71a5_0", # test Python imports
"samtools:1.0--1",
"yasm:1.3.0--0", # test missing tests
]
tests = {container: main_test_search(container) for container in containers}
for n in tests.keys():
docker_to_singularity(n, "singularity", tmp_path, no_sudo=True)
results = singularity_container_test(
@@ -59,5 +49,5 @@ def test_singularity_container_test(tmp_path) -> None:
tmp_path,
)
assert "samtools:1.0--1" in results["passed"]
assert results["failed"][0]["imports"] == ["pyBigWig"]
assert "pybigwig:0.3.22--py36h54a71a5_0" in results["passed"]
assert "yasm:1.3.0--0" in results["notest"]