diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py
index 452b7593d09..64a00980d3c 100644
--- a/lib/galaxy/tool_util/verify/interactor.py
+++ b/lib/galaxy/tool_util/verify/interactor.py
@@ -466,7 +466,7 @@ class GalaxyInteractorApi:
return output_data['id']
def delete_history(self, history):
- return None
+ self._delete(f"histories/{history}")
def __job_ready(self, job_id, history_id=None):
if job_id is None:
@@ -876,10 +876,12 @@ def verify_tool(tool_id,
tool_version=None,
quiet=False,
test_history=None,
+ no_history_cleanup=False,
force_path_paste=False,
maxseconds=DEFAULT_TOOL_TEST_WAIT,
tool_test_dicts=None,
client_test_config=None,
+ skip_with_reference_data=False,
skip_on_dynamic_param_errors=False):
if resource_parameters is None:
resource_parameters = {}
@@ -900,20 +902,34 @@ def verify_tool(tool_id,
"test_index": test_index,
}
client_config = client_test_config.get_test_config(job_data)
+ skip_message = None
if client_config is not None:
job_data.update(client_config)
skip_message = job_data.get("skip")
- if skip_message:
- job_data["status"] = "skip"
- register_job_data(job_data)
- return
+
+ if not skip_message and skip_with_reference_data:
+ required_data_tables = tool_test_dict.get("required_data_tables")
+ required_loc_files = tool_test_dict.get("required_loc_files")
+ # TODO: actually hit the API and see if these tables are available.
+ if required_data_tables:
+ skip_message = f"Skipping test because of required data tables ({required_data_tables})"
+ if required_loc_files:
+ skip_message = f"Skipping test because of required loc files ({required_loc_files})"
+
+ if skip_message:
+ job_data["status"] = "skip"
+ register_job_data(job_data)
+ return
tool_test_dict.setdefault('maxseconds', maxseconds)
testdef = ToolTestDescription(tool_test_dict)
_handle_def_errors(testdef)
+ created_history = False
if test_history is None:
- test_history = galaxy_interactor.new_history()
+ created_history = True
+ history_name = f"Tool Test History for {tool_id}/{tool_version}-{test_index}"
+ test_history = galaxy_interactor.new_history(history_name=history_name)
# Upload data to test_history, run the tool and check the outputs - record
# API input, job info, tool run exception, as well as exceptions related to
@@ -979,7 +995,8 @@ def verify_tool(tool_id,
job_data["status"] = status
register_job_data(job_data)
- galaxy_interactor.delete_history(test_history)
+ if created_history and not no_history_cleanup:
+ galaxy_interactor.delete_history(test_history)
def _handle_def_errors(testdef):
diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py
index fc7b644137f..688608c990c 100644
--- a/lib/galaxy/tool_util/verify/script.py
+++ b/lib/galaxy/tool_util/verify/script.py
@@ -1,8 +1,12 @@
#!/usr/bin/env python
import argparse
+import datetime as dt
import json
+import os
import sys
+from collections import namedtuple
+from concurrent.futures import thread, ThreadPoolExecutor
import yaml
@@ -13,7 +17,256 @@ from galaxy.tool_util.verify.interactor import (
)
DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance."""
-ALL_TESTS = "*all_tests*"
+DEFAULT_SUITE_NAME = "Galaxy Tool Tests"
+ALL_TESTS = -1
+ALL_TOOLS = "*"
+ALL_VERSION = "*"
+LATEST_VERSION = None
+
+
+TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"])
+TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"])
+
+
+class Results:
+
+ def __init__(self, default_suitename, test_json, append=False):
+ self.test_json = test_json or "-"
+ test_results = []
+ test_exceptions = []
+ suitename = default_suitename
+ if append:
+ assert test_json != "-"
+ with open(test_json) as f:
+ previous_results = json.load(f)
+ test_results = previous_results["tests"]
+ if "suitename" in previous_results:
+ suitename = previous_results["suitename"]
+ self.test_results = test_results
+ self.test_exceptions = test_exceptions
+ self.suitename = suitename
+
+ def register_result(self, result):
+ self.test_results.append(result)
+
+ def register_exception(self, test_exception):
+ self.test_exceptions.append(test_exception)
+
+ def write(self):
+ tests = sorted(self.test_results, key=lambda el: el['id'])
+ n_passed, n_failures, n_skips = 0, 0, 0
+ n_errors = len([e for e in self.test_exceptions if not e.was_recorded])
+ for test in tests:
+ status = test['status']
+ if status == "success":
+ n_passed += 1
+ elif status == "error":
+ n_errors += 1
+ elif status == "skip":
+ n_skips += 1
+ elif status == "failure":
+ n_failures += 1
+ report_obj = {
+ 'version': '0.1',
+ 'suitename': self.suitename,
+ 'results': {
+ 'total': n_passed + n_failures + n_skips + n_errors,
+ 'errors': n_errors,
+ 'failures': n_failures,
+ 'skips': n_skips,
+ },
+ 'tests': tests,
+ }
+ if self.test_json == "-":
+ print(json.dumps(report_obj))
+ else:
+ with open(self.test_json, "w") as f:
+ json.dump(report_obj, f)
+
+ def info_message(self):
+ messages = []
+ passed_tests = self._tests_with_status('success')
+ messages.append("Passed tool tests ({0}): {1}".format(
+ len(passed_tests),
+ [t["id"] for t in passed_tests]
+ ))
+ failed_tests = self._tests_with_status('failure')
+ messages.append("Failed tool tests ({0}): {1}".format(
+ len(failed_tests),
+ [t["id"] for t in failed_tests]
+ ))
+ skiped_tests = self._tests_with_status('skip')
+ messages.append("Skipped tool tests ({0}): {1}".format(
+ len(skiped_tests),
+ [t["id"] for t in skiped_tests]
+ ))
+ errored_tests = self._tests_with_status('error')
+ messages.append("Errored tool tests ({0}): {1}".format(
+ len(errored_tests),
+ [t["id"] for t in errored_tests]
+ ))
+ return "\n".join(messages)
+
+ @property
+ def success_count(self):
+ self._tests_with_status('success')
+
+ @property
+ def skip_count(self):
+ self._tests_with_status('skip')
+
+ @property
+ def error_count(self):
+ return self._tests_with_status('error') + len(self.test_exceptions)
+
+ @property
+ def failure_count(self):
+ return self._tests_with_status('failure')
+
+ def _tests_with_status(self, status):
+ return [t for t in self.test_results if t.get("status") == status]
+
+
+def test_tools(
+ galaxy_interactor,
+ test_references,
+ results,
+ log=None,
+ parallel_tests=1,
+ history_per_test_case=False,
+ no_history_cleanup=False,
+ retries=0,
+ verify_kwds=None,
+):
+ """Run through tool tests and write report.
+
+ Refactor this into Galaxy in 21.01.
+ """
+ verify_kwds = (verify_kwds or {}).copy()
+ tool_test_start = dt.datetime.now()
+ history_created = False
+ if history_per_test_case:
+ test_history = None
+ else:
+ history_created = True
+ test_history = galaxy_interactor.new_history(history_name=f"History for {results.suitename}")
+ verify_kwds.update({
+ "no_history_cleanup": no_history_cleanup,
+ "test_history": test_history,
+ })
+ with ThreadPoolExecutor(max_workers=parallel_tests) as executor:
+ try:
+ for test_reference in test_references:
+ _test_tool(
+ executor=executor,
+ test_reference=test_reference,
+ results=results,
+ galaxy_interactor=galaxy_interactor,
+ log=log,
+ retries=retries,
+ verify_kwds=verify_kwds,
+ )
+ finally:
+ # Always write report, even if test was cancelled.
+ try:
+ executor.shutdown(wait=True)
+ except KeyboardInterrupt:
+ executor._threads.clear()
+ thread._threads_queues.clear()
+ results.write()
+ if log:
+ log.info("Report written to '%s'", os.path.abspath(results.test_json))
+ log.info(results.info_message())
+ log.info("Total tool test time: {0}".format(dt.datetime.now() - tool_test_start))
+ if history_created and not no_history_cleanup:
+ galaxy_interactor.delete_history(test_history)
+
+
+def _test_tool(
+ executor,
+ test_reference,
+ results,
+ galaxy_interactor,
+ log,
+ retries,
+ verify_kwds,
+):
+ tool_id = test_reference.tool_id
+ tool_version = test_reference.tool_version
+ test_index = test_reference.test_index
+ # If given a tool_id with a version suffix, strip it off so we can treat tool_version
+ # correctly at least in client_test_config.
+ if tool_version and tool_id.endswith("/" + tool_version):
+ tool_id = tool_id[:-len("/" + tool_version)]
+
+ label_base = tool_id
+ if tool_version:
+ label_base += "/" + str(tool_version)
+
+ test_id = label_base + "-" + str(test_index)
+
+ def run_test():
+ run_retries = retries
+ job_data = None
+ job_exception = None
+
+ def register(job_data_):
+ nonlocal job_data
+ job_data = job_data_
+
+ try:
+ while run_retries >= 0:
+ job_exception = None
+ try:
+ if log:
+ log.info("Executing test '%s'", test_id)
+ verify_tool(
+ tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
+ register_job_data=register, **verify_kwds
+ )
+ if log:
+ log.info("Test '%s' passed", test_id)
+ break
+ except Exception as e:
+ if log:
+ log.warning("Test '%s' failed", test_id, exc_info=True)
+
+ job_exception = e
+ run_retries -= 1
+ finally:
+ if job_data is not None:
+ results.register_result({
+ "id": test_id,
+ "has_data": True,
+ "data": job_data,
+ })
+ if job_exception is not None:
+ was_recorded = job_data is not None
+ test_exception = TestException(tool_id, job_exception, was_recorded)
+ results.register_exception(test_exception)
+
+ executor.submit(run_test)
+
+
+def build_case_references(galaxy_interactor, tool_id=ALL_TOOLS, tool_version=LATEST_VERSION, test_index=ALL_TESTS):
+ test_references = []
+ if tool_id == ALL_TOOLS:
+ tests_summary = galaxy_interactor.get_tests_summary()
+ for tool_id, tool_versions_dict in tests_summary.items():
+ for tool_version, summary in tool_versions_dict.items():
+ for test_index in range(summary["count"]):
+ test_reference = TestReference(tool_id, tool_version, test_index)
+ test_references.append(test_reference)
+ else:
+ assert tool_id
+ tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {}
+ for i, tool_test_dict in enumerate(tool_test_dicts):
+ this_tool_version = tool_test_dict.get("tool_version", tool_version)
+ this_test_index = i
+ if test_index == ALL_TESTS or i == test_index:
+ test_reference = TestReference(tool_id, this_tool_version, this_test_index)
+ test_references.append(test_reference)
+ return test_references
def main(argv=None):
@@ -46,66 +299,30 @@ def main(argv=None):
tool_id = args.tool_id
tool_version = args.tool_version
tools_client_test_config = DictClientTestConfig(client_test_config.get("tools"))
+ verbose = args.verbose
galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
- raw_test_index = args.test_index
- if raw_test_index == ALL_TESTS:
- tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
- test_indices = list(range(len(tool_test_dicts)))
- else:
- test_indices = [int(raw_test_index)]
-
- test_results = []
-
- if args.append:
- assert output_json_path != "-"
- with open(output_json_path) as f:
- previous_results = json.load(f)
- test_results = previous_results["tests"]
-
- exceptions = []
- verbose = args.verbose
- for test_index in test_indices:
- if tool_version:
- tool_id_and_version = f"{tool_id}/{tool_version}"
- else:
- tool_id_and_version = tool_id
-
- test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index)
-
- def register(job_data):
- test_results.append({
- 'id': tool_id + "-" + str(test_index),
- 'has_data': True,
- 'data': job_data,
- })
-
- try:
- verify_tool(
- tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
- register_job_data=register, quiet=not verbose, force_path_paste=args.force_path_paste,
- client_test_config=tools_client_test_config,
- )
-
- if verbose:
- print("%s passed" % test_identifier)
-
- except Exception as e:
- if verbose:
- print(f"{test_identifier} failed, {e}")
- exceptions.append(e)
-
- report_obj = {
- 'version': '0.1',
- 'tests': test_results,
- }
- if output_json_path:
- if output_json_path == "-":
- print(json.dumps(report_obj))
- else:
- with open(output_json_path, "w") as f:
- json.dump(report_obj, f)
-
+ test_references = build_case_references(
+ galaxy_interactor,
+ tool_id=tool_id,
+ tool_version=tool_version,
+ test_index=args.test_index,
+ )
+ results = Results(args.suite_name, output_json_path, append=args.append)
+ verify_kwds = dict(
+ client_test_config=tools_client_test_config,
+ force_path_paste=args.force_path_paste,
+ skip_with_reference_data=not args.with_reference_data,
+ quiet=not verbose,
+ )
+ test_tools(
+ galaxy_interactor,
+ test_references,
+ results,
+ log=None,
+ verify_kwds=verify_kwds,
+ )
+ exceptions = results.test_exceptions
if exceptions:
raise exceptions[0]
@@ -116,14 +333,21 @@ def _arg_parser():
parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key')
parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.')
- parser.add_argument('-t', '--tool-id', default=None, help='Tool ID')
- parser.add_argument('--tool-version', default=None, help='Tool Version')
- parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.')
+ parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID')
+ parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions')
+ parser.add_argument('-i', '--test-index', default=ALL_TESTS, type=int, help='Tool Test Index (starting at 0) - by default all tests will run.')
parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.")
parser.add_argument('-j', '--output-json', default=None, help='output metadata json')
parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.")
parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing")
+ parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output")
+ parser.add_argument('--with-reference-data', dest="with_reference_data", default=False, action="store_true")
+ parser.add_argument('--skip-with-reference-data', dest="with_reference_data", action="store_false", help="Skip tests the Galaxy server believes use data tables or loc files.")
+ parser.add_argument('--history-per-suite', dest="history_per_test_case", default=False, action="store_false", help="Create new history per test suite (all tests in same history).")
+ parser.add_argument('--history-per-test-case', dest="history_per_test_case", action="store_true", help="Create new history per test case.")
+ parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.")
+ parser.add_argument('--retries', default=0, type=int, help="Retry failed tests.")
return parser
diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py
index 8f6f6b0ad88..2d80b7cdfd2 100644
--- a/lib/galaxy/tools/toolbox/base.py
+++ b/lib/galaxy/tools/toolbox/base.py
@@ -522,6 +522,11 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
for tool in self._tools_by_id.values():
if tool.old_id == tool_id:
rval.append(tool)
+ if get_all_versions and tool_id in self._tool_versions_by_id:
+ for tool in self._tool_versions_by_id[tool_id].values():
+ if tool not in rval:
+ rval.append(tool)
+
# if we don't have a lineage_map for this tool we need to sort by version,
# so that the last tool in rval is the newest tool.
rval.sort(key=lambda t: t.version)
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index ee13fd4b858..4b6adb72495 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -193,13 +193,25 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
internals/Pythonisms in a rough way). If this endpoint is being used from outside
of scripts shipped with Galaxy let us know and please be prepared for the response
from this API to change its format in some ways.
+
+ If tool version is not passed, it is assumed to be latest. Tool version can be
+ set as '*' to get tests for all configured versions.
"""
- # TODO: eliminate copy and paste with above code.
if 'payload' in kwd:
kwd = kwd.get('payload')
tool_version = kwd.get('tool_version', None)
- tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
- return [t.to_dict() for t in tool.tests]
+ if tool_version == "*":
+ tools = self.app.toolbox.get_tool(id, get_all_versions=True)
+ for tool in tools:
+ if not tool.allow_user_access(trans.user):
+ raise exceptions.AuthenticationFailed("Access denied, please login for tool with id '%s'." % id)
+ else:
+ tools = [self._get_tool(id, tool_version=tool_version, user=trans.user)]
+
+ test_defs = []
+ for tool in tools:
+ test_defs.extend([t.to_dict() for t in tool.tests])
+ return test_defs
@web.require_admin
@expose_api
diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py
index 6247bb7e446..1ba097a5117 100644
--- a/lib/galaxy_test/api/test_tools.py
+++ b/lib/galaxy_test/api/test_tools.py
@@ -699,6 +699,20 @@ class ToolsTestCase(ApiTestCase, TestsTools):
output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
self.assertEqual(output1_content.strip(), "Version " + version)
+ @skip_without_tool("multiple_versions")
+ @uses_test_history(require_new=False)
+ def test_test_by_versions(self, history_id):
+ test_data_response = self._get("tools/%s/test_data" % "multiple_versions")
+ test_data_response.raise_for_status()
+ test_data_dicts = test_data_response.json()
+ assert len(test_data_dicts) == 1
+ assert test_data_dicts[0]["tool_version"] == "0.2"
+
+ test_data_response = self._get("tools/%s/test_data?tool_version=*" % "multiple_versions")
+ test_data_response.raise_for_status()
+ test_data_dicts = test_data_response.json()
+ assert len(test_data_dicts) == 2
+
@skip_without_tool("multiple_versions")
@uses_test_history(require_new=False)
def test_show_with_wrong_tool_version_in_tool_id(self, history_id):
diff --git a/test/functional/tools/multiple_versions_v01.xml b/test/functional/tools/multiple_versions_v01.xml
index 8b05454529f..f7f4580049b 100644
--- a/test/functional/tools/multiple_versions_v01.xml
+++ b/test/functional/tools/multiple_versions_v01.xml
@@ -8,4 +8,14 @@
+
+
+
+
+
+
diff --git a/test/functional/tools/multiple_versions_v02.xml b/test/functional/tools/multiple_versions_v02.xml
index 7a7a3a14937..5493c7f8140 100644
--- a/test/functional/tools/multiple_versions_v02.xml
+++ b/test/functional/tools/multiple_versions_v02.xml
@@ -8,4 +8,14 @@
+
+
+
+
+
+
diff --git a/test/unit/tool_util/test_verify_script.py b/test/unit/tool_util/test_verify_script.py
new file mode 100644
index 00000000000..357964a9ff1
--- /dev/null
+++ b/test/unit/tool_util/test_verify_script.py
@@ -0,0 +1,277 @@
+import json
+import os
+from tempfile import NamedTemporaryFile
+from unittest import mock
+
+from galaxy.tool_util.verify.script import (
+ _arg_parser,
+ build_case_references,
+ Results,
+ test_tools as run,
+ TestReference,
+)
+
+VT_PATH = 'galaxy.tool_util.verify.script.verify_tool'
+NEW_HISTORY = object()
+
+
+def test_arg_parse():
+ parser = _arg_parser()
+
+ # defaults
+ args = parser.parse_args([])
+ assert not args.with_reference_data
+ assert not args.history_per_test_case
+
+ # skip flags
+ args = parser.parse_args(["--skip-with-reference-data", "--history-per-test-case"])
+ assert not args.with_reference_data
+ assert args.history_per_test_case
+
+ # enable flags
+ args = parser.parse_args(["--with-reference-data", "--history-per-suite"])
+ assert args.with_reference_data
+ assert not args.history_per_test_case
+
+
+def test_test_tools():
+ interactor = MockGalaxyInteractor()
+ f = NamedTemporaryFile()
+ results = Results("my suite", f.name)
+ test_references = [
+ TestReference("cat", "0.1.0", 0),
+ TestReference("cat", "0.1.0", 1),
+ TestReference("cat", "0.2.0", 0),
+ ]
+ with mock.patch(VT_PATH) as mock_verify:
+ assert_results_not_written(results)
+ run(
+ interactor,
+ test_references,
+ results,
+ )
+ calls = mock_verify.call_args_list
+ assert len(calls) == 3
+ assert len(results.test_exceptions) == 0
+ assert_results_written(results)
+ assert interactor.history_created
+ assert interactor.history_deleted
+
+
+def test_test_tools_no_history_cleanup():
+ interactor = MockGalaxyInteractor()
+ f = NamedTemporaryFile()
+ results = Results("my suite", f.name)
+ test_references = [
+ TestReference("cat", "0.1.0", 0),
+ ]
+ with mock.patch(VT_PATH) as mock_verify:
+ assert_results_not_written(results)
+ run(
+ interactor,
+ test_references,
+ results,
+ no_history_cleanup=True,
+ )
+ calls = mock_verify.call_args_list
+ assert len(calls) == 1
+ assert len(results.test_exceptions) == 0
+ assert_results_written(results)
+ assert interactor.history_created
+ assert not interactor.history_deleted
+
+
+def test_test_tool_per_test_history():
+ interactor = MockGalaxyInteractor()
+ f = NamedTemporaryFile()
+ results = Results("my suite", f.name)
+ test_references = [
+ TestReference("cat", "0.1.0", 0),
+ TestReference("cat", "0.1.0", 1),
+ ]
+ with mock.patch(VT_PATH) as mock_verify:
+ assert_results_not_written(results)
+ run(
+ interactor,
+ test_references,
+ results,
+ history_per_test_case=True,
+ )
+ calls = mock_verify.call_args_list
+ assert len(calls) == 2
+ assert len(results.test_exceptions) == 0
+ assert_results_written(results)
+ assert not interactor.history_created
+ assert not interactor.history_deleted
+
+
+def test_test_tools_records_exception():
+ interactor = MockGalaxyInteractor()
+ f = NamedTemporaryFile()
+ results = Results("my suite", f.name)
+ test_references = [
+ TestReference("bad", "0.1.0", 0),
+ ]
+ with mock.patch(VT_PATH) as mock_verify:
+ assert_results_not_written(results)
+
+ def side_effect(*args, **kwd):
+ raise Exception("Cow")
+
+ mock_verify.side_effect = side_effect
+ run(
+ interactor,
+ test_references,
+ results,
+ )
+ calls = mock_verify.call_args_list
+ assert len(calls) == 1
+ assert len(results.test_exceptions) == 1
+ assert_results_written(results)
+
+
+def test_test_tools_records_retry_exception():
+ interactor = MockGalaxyInteractor()
+ f = NamedTemporaryFile()
+ results = Results("my suite", f.name)
+ test_references = [
+ TestReference("bad", "0.1.0", 0),
+ ]
+ with mock.patch(VT_PATH) as mock_verify:
+ assert_results_not_written(results)
+
+ count = 0
+
+ def side_effect(*args, **kwd):
+ nonlocal count
+ raise_exception = count == 0
+ count += 1
+ if raise_exception:
+ raise Exception("Cow")
+
+ mock_verify.side_effect = side_effect
+ run(
+ interactor,
+ test_references,
+ results,
+ retries=1,
+ )
+ calls = mock_verify.call_args_list
+ assert len(calls) == 2
+ assert len(results.test_exceptions) == 0
+ assert_results_written(results)
+
+
+def test_results():
+ f = NamedTemporaryFile()
+ results = Results("my suite", f.name)
+ results.register_result({"id": "foo", "status": "success"})
+ results.write()
+ message = results.info_message()
+
+ with open(f.name, "r") as f:
+ report_obj = json.load(f)
+ assert "tests" in report_obj
+ assert len(report_obj["tests"]) == 1
+ assert report_obj["results"]["total"] == 1
+ assert report_obj["results"]["errors"] == 0
+ assert report_obj["results"]["skips"] == 0
+
+ assert "Passed tool tests (1)" in message
+ assert "Skipped tool tests (0)" in message
+
+ results.register_result({"id": "bar", "status": "skip"})
+ results.write()
+ message = results.info_message()
+
+ with open(f.name, "r") as f:
+ report_obj = json.load(f)
+ assert len(report_obj["tests"]) == 2
+ assert report_obj["results"]["skips"] == 1
+ assert "Passed tool tests (1)" in message
+ assert "Skipped tool tests (1)" in message
+
+
+def test_build_references():
+ interactor = MockGalaxyInteractor()
+ test_references = build_case_references(interactor)
+ assert len(test_references) == 6
+
+ test_references = build_case_references(interactor, 'cat1', tool_version="*")
+ assert len(test_references) == 6
+
+ test_references = build_case_references(interactor, 'cat1', tool_version=None)
+ assert len(test_references) == 4
+
+ test_references = build_case_references(interactor, 'cat1', tool_version="0.2.0")
+ assert len(test_references) == 4
+
+ test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0")
+ assert len(test_references) == 2
+
+ test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0", test_index=1)
+ assert len(test_references) == 1
+
+ # Specifying an index but not a version, grabs latest version and fills it in.
+ test_references = build_case_references(interactor, 'cat1', test_index=2)
+ assert len(test_references) == 1
+ test_reference = test_references[0]
+ assert test_reference.tool_id == "cat1"
+ assert test_reference.tool_version == "0.2.0"
+ assert test_reference.test_index == 2
+
+
+def assert_results_not_written(results):
+ assert os.stat(results.test_json).st_size == 0
+
+
+def assert_results_written(results):
+ assert os.stat(results.test_json).st_size > 0
+ with open(results.test_json, "r") as f:
+ json.load(f)
+
+
+class MockGalaxyInteractor:
+
+ def __init__(self):
+ self.history_deleted = False
+ self.history_created = False
+
+ def new_history(self, history_name=""):
+ self.history_created = True
+ return NEW_HISTORY
+
+ def delete_history(self, history):
+ self.history_deleted = True
+
+ def get_tests_summary(self):
+ return {
+ 'cat1': {
+ '0.2.0': {
+ 'count': 4,
+ },
+ '0.1.0': {
+ 'count': 2,
+ },
+ },
+ }
+
+ def get_tool_tests(self, tool_id, tool_version=None):
+ tool_dict = self.get_tests_summary().get(tool_id)
+ test_defs = []
+ for this_tool_version, version_defs in tool_dict.items():
+ if tool_version is not None and tool_version != "*" and this_tool_version != tool_version:
+ continue
+
+ count = version_defs['count']
+ for i in range(count):
+ test_def = {
+ 'tool_id': tool_id,
+ 'tool_version': this_tool_version or '0.1.1-default',
+ }
+ test_defs.append(test_def)
+
+ if tool_version is None or tool_version != "*":
+ break
+
+ return test_defs