mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-19 10:51:34 +08:00
370 lines
12 KiB
Python
370 lines
12 KiB
Python
import json
|
|
import os
|
|
from tempfile import NamedTemporaryFile
|
|
from typing import (
|
|
cast,
|
|
NoReturn,
|
|
)
|
|
from unittest import mock
|
|
|
|
from galaxy.tool_util.unittest_utils.interactor import (
|
|
EXISTING_HISTORY,
|
|
EXISTING_HISTORY_NAME,
|
|
EXISTING_SUITE_NAME,
|
|
MockGalaxyInteractor,
|
|
NEW_HISTORY_ID,
|
|
)
|
|
from galaxy.tool_util.verify.interactor import GalaxyInteractorApi
|
|
from galaxy.tool_util.verify.script import (
|
|
arg_parser,
|
|
build_case_references,
|
|
Results,
|
|
test_tools as run,
|
|
TestReference,
|
|
)
|
|
|
|
VT_PATH = "galaxy.tool_util.verify.script.verify_tool"
|
|
|
|
|
|
def test_arg_parse() -> None:
|
|
parser = arg_parser()
|
|
|
|
# defaults
|
|
args = parser.parse_args([])
|
|
assert not args.with_reference_data
|
|
assert not args.history_per_test_case
|
|
assert args.page_size == 0
|
|
assert args.page_number == 0
|
|
assert args.parallel_tests == 1
|
|
|
|
# skip flags
|
|
args = parser.parse_args(["--skip-with-reference-data", "--history-per-test-case"])
|
|
assert not args.with_reference_data
|
|
assert args.history_per_test_case
|
|
|
|
# enable flags
|
|
args = parser.parse_args(["--with-reference-data", "--history-per-suite"])
|
|
assert args.with_reference_data
|
|
assert not args.history_per_test_case
|
|
|
|
# pagination
|
|
args = parser.parse_args(["--page-size", "5", "--page-number", "40", "--parallel-tests", "3"])
|
|
assert args.page_size == 5
|
|
assert args.page_number == 40
|
|
assert args.parallel_tests == 3
|
|
assert args.test_data is None
|
|
|
|
args = parser.parse_args(["--test-data", "/foo/bar", "--test-data", "cow"])
|
|
assert len(args.test_data) == 2
|
|
assert args.test_data[0] == "/foo/bar"
|
|
assert args.test_data[1] == "cow"
|
|
|
|
args = parser.parse_args(["--skip-previously-successful"])
|
|
assert args.skip == "successful"
|
|
|
|
args = parser.parse_args(["--skip-previously-executed"])
|
|
assert args.skip == "executed"
|
|
|
|
|
|
def test_test_tools() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results("my suite", f.name)
|
|
test_references = [
|
|
TestReference("cat", "0.1.0", 0),
|
|
TestReference("cat", "0.1.0", 1),
|
|
TestReference("cat", "0.2.0", 0),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 3
|
|
assert len(results.test_exceptions) == 0
|
|
assert_results_written(results)
|
|
assert interactor.history_created
|
|
assert interactor.history_deleted
|
|
|
|
|
|
def test_test_tools_no_history_cleanup() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results("my suite", f.name)
|
|
test_references = [
|
|
TestReference("cat", "0.1.0", 0),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
no_history_cleanup=True,
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 1
|
|
assert len(results.test_exceptions) == 0
|
|
assert_results_written(results)
|
|
assert interactor.history_created
|
|
assert not interactor.history_deleted
|
|
|
|
|
|
def test_test_tools_history_reuse() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results(EXISTING_SUITE_NAME, f.name)
|
|
test_references = [
|
|
TestReference("cat", "0.1.0", 0),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
no_history_reuse=False,
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 1
|
|
assert len(results.test_exceptions) == 0
|
|
assert_results_written(results)
|
|
assert interactor.history_id == EXISTING_HISTORY["id"]
|
|
assert interactor.history_name == EXISTING_HISTORY_NAME
|
|
assert not interactor.history_created
|
|
assert not interactor.history_deleted
|
|
|
|
|
|
def test_test_tools_no_history_reuse() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results("existing suite", f.name)
|
|
test_references = [
|
|
TestReference("cat", "0.1.0", 0),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
no_history_reuse=True,
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 1
|
|
assert len(results.test_exceptions) == 0
|
|
assert_results_written(results)
|
|
assert interactor.history_id == NEW_HISTORY_ID
|
|
assert interactor.history_name == EXISTING_HISTORY_NAME
|
|
assert interactor.history_created
|
|
assert interactor.history_deleted
|
|
|
|
|
|
def test_test_tools_history_name() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results("my suite", f.name)
|
|
test_references = [
|
|
TestReference("cat", "0.1.0", 0),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
history_name="testfoo",
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 1
|
|
assert len(results.test_exceptions) == 0
|
|
assert_results_written(results)
|
|
assert interactor.history_id == NEW_HISTORY_ID
|
|
assert interactor.history_name == "testfoo"
|
|
assert interactor.history_created
|
|
assert interactor.history_deleted
|
|
|
|
|
|
def test_test_tool_per_test_history() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results("my suite", f.name)
|
|
test_references = [
|
|
TestReference("cat", "0.1.0", 0),
|
|
TestReference("cat", "0.1.0", 1),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
history_per_test_case=True,
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 2
|
|
assert len(results.test_exceptions) == 0
|
|
assert_results_written(results)
|
|
assert not interactor.history_created
|
|
assert not interactor.history_deleted
|
|
|
|
|
|
def test_test_tools_records_exception() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results("my suite", f.name)
|
|
test_references = [
|
|
TestReference("bad", "0.1.0", 0),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
|
|
def side_effect(*args, **kwd) -> NoReturn:
|
|
raise Exception("Cow")
|
|
|
|
mock_verify.side_effect = side_effect
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 1
|
|
assert len(results.test_exceptions) == 1
|
|
assert_results_written(results)
|
|
|
|
|
|
def test_test_tools_records_retry_exception() -> None:
|
|
interactor = MockGalaxyInteractor()
|
|
f = NamedTemporaryFile()
|
|
results = Results("my suite", f.name)
|
|
test_references = [
|
|
TestReference("bad", "0.1.0", 0),
|
|
]
|
|
with mock.patch(VT_PATH) as mock_verify:
|
|
assert_results_not_written(results)
|
|
|
|
count = 0
|
|
|
|
def side_effect(*args, **kwd):
|
|
nonlocal count
|
|
raise_exception = count == 0
|
|
count += 1
|
|
if raise_exception:
|
|
raise Exception("Cow")
|
|
|
|
mock_verify.side_effect = side_effect
|
|
run(
|
|
cast(GalaxyInteractorApi, interactor),
|
|
test_references,
|
|
results,
|
|
retries=1,
|
|
)
|
|
calls = mock_verify.call_args_list
|
|
assert len(calls) == 2
|
|
assert len(results.test_exceptions) == 0
|
|
assert_results_written(results)
|
|
|
|
|
|
def test_append_results() -> None:
|
|
tf = NamedTemporaryFile()
|
|
tf.close()
|
|
assert not os.path.exists(tf.name)
|
|
# Now try to append to the non-existent file.
|
|
results = Results("my suite", tf.name, True)
|
|
results.register_result({"id": "foo", "has_data": True, "data": {"status": "success"}})
|
|
try:
|
|
results.write()
|
|
assert os.path.exists(tf.name)
|
|
with open(tf.name) as f:
|
|
report_obj = json.load(f)
|
|
assert "tests" in report_obj
|
|
assert len(report_obj["tests"]) == 1
|
|
assert report_obj["results"]["total"] == 1, report_obj["results"]
|
|
assert report_obj["results"]["errors"] == 0
|
|
assert report_obj["results"]["skips"] == 0
|
|
finally:
|
|
os.remove(tf.name)
|
|
assert not os.path.exists(tf.name)
|
|
|
|
|
|
def test_results():
|
|
tf = NamedTemporaryFile()
|
|
results = Results("my suite", tf.name)
|
|
results.register_result({"id": "foo", "has_data": True, "data": {"status": "success"}})
|
|
results.write()
|
|
message = results.info_message()
|
|
|
|
with open(tf.name) as f:
|
|
report_obj = json.load(f)
|
|
assert "tests" in report_obj
|
|
assert len(report_obj["tests"]) == 1
|
|
assert report_obj["results"]["total"] == 1, report_obj["results"]
|
|
assert report_obj["results"]["errors"] == 0
|
|
assert report_obj["results"]["skips"] == 0
|
|
|
|
assert "Passed tool tests (1)" in message
|
|
assert "Skipped tool tests (0)" in message
|
|
|
|
results.register_result({"id": "bar", "has_data": True, "data": {"status": "skip"}})
|
|
results.write()
|
|
message = results.info_message()
|
|
|
|
with open(tf.name) as f:
|
|
report_obj = json.load(f)
|
|
assert len(report_obj["tests"]) == 2
|
|
assert report_obj["results"]["skips"] == 1
|
|
assert "Passed tool tests (1)" in message
|
|
assert "Skipped tool tests (1)" in message
|
|
|
|
|
|
def test_build_references() -> None:
|
|
interactor = cast(GalaxyInteractorApi, MockGalaxyInteractor())
|
|
test_references = build_case_references(interactor)
|
|
assert len(test_references) == 6
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version="*")
|
|
assert len(test_references) == 6
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version="*", page_size=3, page_number=1)
|
|
assert len(test_references) == 3
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version="*", page_size=3, page_number=2)
|
|
assert len(test_references) == 0
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version="*", page_size=3, page_number=3)
|
|
assert len(test_references) == 0
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version=None)
|
|
assert len(test_references) == 4
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version="0.2.0")
|
|
assert len(test_references) == 4
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version="0.1.0")
|
|
assert len(test_references) == 2
|
|
|
|
test_references = build_case_references(interactor, "cat1", tool_version="0.1.0", test_index=1)
|
|
assert len(test_references) == 1
|
|
|
|
# Specifying an index but not a version, grabs latest version and fills it in.
|
|
test_references = build_case_references(interactor, "cat1", test_index=2)
|
|
assert len(test_references) == 1
|
|
test_reference = test_references[0]
|
|
assert test_reference.tool_id == "cat1"
|
|
assert test_reference.tool_version == "0.2.0"
|
|
assert test_reference.test_index == 2
|
|
|
|
|
|
def assert_results_not_written(results: Results) -> None:
|
|
assert os.stat(results.test_json).st_size == 0
|
|
|
|
|
|
def assert_results_written(results: Results) -> None:
|
|
assert os.stat(results.test_json).st_size > 0
|
|
with open(results.test_json) as f:
|
|
json.load(f)
|