Merge pull request #10620 from jmchilton/refine_client_test_config

Enhancements for galaxy-tool-test script.

lib/galaxy/tool_util/verify/interactor.py

@@ -466,7 +466,7 @@ class GalaxyInteractorApi:
         return output_data['id']
 
     def delete_history(self, history):
-        return None
+        self._delete(f"histories/{history}")
 
     def __job_ready(self, job_id, history_id=None):
         if job_id is None:
@@ -876,10 +876,12 @@ def verify_tool(tool_id,
                 tool_version=None,
                 quiet=False,
                 test_history=None,
+                no_history_cleanup=False,
                 force_path_paste=False,
                 maxseconds=DEFAULT_TOOL_TEST_WAIT,
                 tool_test_dicts=None,
                 client_test_config=None,
+                skip_with_reference_data=False,
                 skip_on_dynamic_param_errors=False):
     if resource_parameters is None:
         resource_parameters = {}
@@ -900,20 +902,34 @@ def verify_tool(tool_id,
         "test_index": test_index,
     }
     client_config = client_test_config.get_test_config(job_data)
+    skip_message = None
     if client_config is not None:
         job_data.update(client_config)
         skip_message = job_data.get("skip")
-        if skip_message:
-            job_data["status"] = "skip"
-            register_job_data(job_data)
-            return
+
+    if not skip_message and skip_with_reference_data:
+        required_data_tables = tool_test_dict.get("required_data_tables")
+        required_loc_files = tool_test_dict.get("required_loc_files")
+        # TODO: actually hit the API and see if these tables are available.
+        if required_data_tables:
+            skip_message = f"Skipping test because of required data tables ({required_data_tables})"
+        if required_loc_files:
+            skip_message = f"Skipping test because of required loc files ({required_loc_files})"
+
+    if skip_message:
+        job_data["status"] = "skip"
+        register_job_data(job_data)
+        return
 
     tool_test_dict.setdefault('maxseconds', maxseconds)
     testdef = ToolTestDescription(tool_test_dict)
     _handle_def_errors(testdef)
 
+    created_history = False
     if test_history is None:
-        test_history = galaxy_interactor.new_history()
+        created_history = True
+        history_name = f"Tool Test History for {tool_id}/{tool_version}-{test_index}"
+        test_history = galaxy_interactor.new_history(history_name=history_name)
 
     # Upload data to test_history, run the tool and check the outputs - record
     # API input, job info, tool run exception, as well as exceptions related to
@@ -979,7 +995,8 @@ def verify_tool(tool_id,
             job_data["status"] = status
             register_job_data(job_data)
 
-    galaxy_interactor.delete_history(test_history)
+    if created_history and not no_history_cleanup:
+        galaxy_interactor.delete_history(test_history)
 
 
 def _handle_def_errors(testdef):

lib/galaxy/tool_util/verify/script.py

@@ -1,8 +1,12 @@
 #!/usr/bin/env python
 
 import argparse
+import datetime as dt
 import json
+import os
 import sys
+from collections import namedtuple
+from concurrent.futures import thread, ThreadPoolExecutor
 
 import yaml
 
@@ -13,7 +17,256 @@ from galaxy.tool_util.verify.interactor import (
 )
 
 DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance."""
-ALL_TESTS = "*all_tests*"
+DEFAULT_SUITE_NAME = "Galaxy Tool Tests"
+ALL_TESTS = -1
+ALL_TOOLS = "*"
+ALL_VERSION = "*"
+LATEST_VERSION = None
+
+
+TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"])
+TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"])
+
+
+class Results:
+
+    def __init__(self, default_suitename, test_json, append=False):
+        self.test_json = test_json or "-"
+        test_results = []
+        test_exceptions = []
+        suitename = default_suitename
+        if append:
+            assert test_json != "-"
+            with open(test_json) as f:
+                previous_results = json.load(f)
+                test_results = previous_results["tests"]
+                if "suitename" in previous_results:
+                    suitename = previous_results["suitename"]
+        self.test_results = test_results
+        self.test_exceptions = test_exceptions
+        self.suitename = suitename
+
+    def register_result(self, result):
+        self.test_results.append(result)
+
+    def register_exception(self, test_exception):
+        self.test_exceptions.append(test_exception)
+
+    def write(self):
+        tests = sorted(self.test_results, key=lambda el: el['id'])
+        n_passed, n_failures, n_skips = 0, 0, 0
+        n_errors = len([e for e in self.test_exceptions if not e.was_recorded])
+        for test in tests:
+            status = test['status']
+            if status == "success":
+                n_passed += 1
+            elif status == "error":
+                n_errors += 1
+            elif status == "skip":
+                n_skips += 1
+            elif status == "failure":
+                n_failures += 1
+        report_obj = {
+            'version': '0.1',
+            'suitename': self.suitename,
+            'results': {
+                'total': n_passed + n_failures + n_skips + n_errors,
+                'errors': n_errors,
+                'failures': n_failures,
+                'skips': n_skips,
+            },
+            'tests': tests,
+        }
+        if self.test_json == "-":
+            print(json.dumps(report_obj))
+        else:
+            with open(self.test_json, "w") as f:
+                json.dump(report_obj, f)
+
+    def info_message(self):
+        messages = []
+        passed_tests = self._tests_with_status('success')
+        messages.append("Passed tool tests ({0}): {1}".format(
+            len(passed_tests),
+            [t["id"] for t in passed_tests]
+        ))
+        failed_tests = self._tests_with_status('failure')
+        messages.append("Failed tool tests ({0}): {1}".format(
+            len(failed_tests),
+            [t["id"] for t in failed_tests]
+        ))
+        skiped_tests = self._tests_with_status('skip')
+        messages.append("Skipped tool tests ({0}): {1}".format(
+            len(skiped_tests),
+            [t["id"] for t in skiped_tests]
+        ))
+        errored_tests = self._tests_with_status('error')
+        messages.append("Errored tool tests ({0}): {1}".format(
+            len(errored_tests),
+            [t["id"] for t in errored_tests]
+        ))
+        return "\n".join(messages)
+
+    @property
+    def success_count(self):
+        self._tests_with_status('success')
+
+    @property
+    def skip_count(self):
+        self._tests_with_status('skip')
+
+    @property
+    def error_count(self):
+        return self._tests_with_status('error') + len(self.test_exceptions)
+
+    @property
+    def failure_count(self):
+        return self._tests_with_status('failure')
+
+    def _tests_with_status(self, status):
+        return [t for t in self.test_results if t.get("status") == status]
+
+
+def test_tools(
+    galaxy_interactor,
+    test_references,
+    results,
+    log=None,
+    parallel_tests=1,
+    history_per_test_case=False,
+    no_history_cleanup=False,
+    retries=0,
+    verify_kwds=None,
+):
+    """Run through tool tests and write report.
+
+    Refactor this into Galaxy in 21.01.
+    """
+    verify_kwds = (verify_kwds or {}).copy()
+    tool_test_start = dt.datetime.now()
+    history_created = False
+    if history_per_test_case:
+        test_history = None
+    else:
+        history_created = True
+        test_history = galaxy_interactor.new_history(history_name=f"History for {results.suitename}")
+    verify_kwds.update({
+        "no_history_cleanup": no_history_cleanup,
+        "test_history": test_history,
+    })
+    with ThreadPoolExecutor(max_workers=parallel_tests) as executor:
+        try:
+            for test_reference in test_references:
+                _test_tool(
+                    executor=executor,
+                    test_reference=test_reference,
+                    results=results,
+                    galaxy_interactor=galaxy_interactor,
+                    log=log,
+                    retries=retries,
+                    verify_kwds=verify_kwds,
+                )
+        finally:
+            # Always write report, even if test was cancelled.
+            try:
+                executor.shutdown(wait=True)
+            except KeyboardInterrupt:
+                executor._threads.clear()
+                thread._threads_queues.clear()
+            results.write()
+            if log:
+                log.info("Report written to '%s'", os.path.abspath(results.test_json))
+                log.info(results.info_message())
+                log.info("Total tool test time: {0}".format(dt.datetime.now() - tool_test_start))
+            if history_created and not no_history_cleanup:
+                galaxy_interactor.delete_history(test_history)
+
+
+def _test_tool(
+    executor,
+    test_reference,
+    results,
+    galaxy_interactor,
+    log,
+    retries,
+    verify_kwds,
+):
+    tool_id = test_reference.tool_id
+    tool_version = test_reference.tool_version
+    test_index = test_reference.test_index
+    # If given a tool_id with a version suffix, strip it off so we can treat tool_version
+    # correctly at least in client_test_config.
+    if tool_version and tool_id.endswith("/" + tool_version):
+        tool_id = tool_id[:-len("/" + tool_version)]
+
+    label_base = tool_id
+    if tool_version:
+        label_base += "/" + str(tool_version)
+
+    test_id = label_base + "-" + str(test_index)
+
+    def run_test():
+        run_retries = retries
+        job_data = None
+        job_exception = None
+
+        def register(job_data_):
+            nonlocal job_data
+            job_data = job_data_
+
+        try:
+            while run_retries >= 0:
+                job_exception = None
+                try:
+                    if log:
+                        log.info("Executing test '%s'", test_id)
+                    verify_tool(
+                        tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
+                        register_job_data=register, **verify_kwds
+                    )
+                    if log:
+                        log.info("Test '%s' passed", test_id)
+                    break
+                except Exception as e:
+                    if log:
+                        log.warning("Test '%s' failed", test_id, exc_info=True)
+
+                    job_exception = e
+                    run_retries -= 1
+        finally:
+            if job_data is not None:
+                results.register_result({
+                    "id": test_id,
+                    "has_data": True,
+                    "data": job_data,
+                })
+            if job_exception is not None:
+                was_recorded = job_data is not None
+                test_exception = TestException(tool_id, job_exception, was_recorded)
+                results.register_exception(test_exception)
+
+    executor.submit(run_test)
+
+
+def build_case_references(galaxy_interactor, tool_id=ALL_TOOLS, tool_version=LATEST_VERSION, test_index=ALL_TESTS):
+    test_references = []
+    if tool_id == ALL_TOOLS:
+        tests_summary = galaxy_interactor.get_tests_summary()
+        for tool_id, tool_versions_dict in tests_summary.items():
+            for tool_version, summary in tool_versions_dict.items():
+                for test_index in range(summary["count"]):
+                    test_reference = TestReference(tool_id, tool_version, test_index)
+                    test_references.append(test_reference)
+    else:
+        assert tool_id
+        tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {}
+        for i, tool_test_dict in enumerate(tool_test_dicts):
+            this_tool_version = tool_test_dict.get("tool_version", tool_version)
+            this_test_index = i
+            if test_index == ALL_TESTS or i == test_index:
+                test_reference = TestReference(tool_id, this_tool_version, this_test_index)
+                test_references.append(test_reference)
+    return test_references
 
 
 def main(argv=None):
@@ -46,66 +299,30 @@ def main(argv=None):
     tool_id = args.tool_id
     tool_version = args.tool_version
     tools_client_test_config = DictClientTestConfig(client_test_config.get("tools"))
+    verbose = args.verbose
 
     galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
-    raw_test_index = args.test_index
-    if raw_test_index == ALL_TESTS:
-        tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
-        test_indices = list(range(len(tool_test_dicts)))
-    else:
-        test_indices = [int(raw_test_index)]
-
-    test_results = []
-
-    if args.append:
-        assert output_json_path != "-"
-        with open(output_json_path) as f:
-            previous_results = json.load(f)
-            test_results = previous_results["tests"]
-
-    exceptions = []
-    verbose = args.verbose
-    for test_index in test_indices:
-        if tool_version:
-            tool_id_and_version = f"{tool_id}/{tool_version}"
-        else:
-            tool_id_and_version = tool_id
-
-        test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index)
-
-        def register(job_data):
-            test_results.append({
-                'id': tool_id + "-" + str(test_index),
-                'has_data': True,
-                'data': job_data,
-            })
-
-        try:
-            verify_tool(
-                tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
-                register_job_data=register, quiet=not verbose, force_path_paste=args.force_path_paste,
-                client_test_config=tools_client_test_config,
-            )
-
-            if verbose:
-                print("%s passed" % test_identifier)
-
-        except Exception as e:
-            if verbose:
-                print(f"{test_identifier} failed, {e}")
-            exceptions.append(e)
-
-    report_obj = {
-        'version': '0.1',
-        'tests': test_results,
-    }
-    if output_json_path:
-        if output_json_path == "-":
-            print(json.dumps(report_obj))
-        else:
-            with open(output_json_path, "w") as f:
-                json.dump(report_obj, f)
-
+    test_references = build_case_references(
+        galaxy_interactor,
+        tool_id=tool_id,
+        tool_version=tool_version,
+        test_index=args.test_index,
+    )
+    results = Results(args.suite_name, output_json_path, append=args.append)
+    verify_kwds = dict(
+        client_test_config=tools_client_test_config,
+        force_path_paste=args.force_path_paste,
+        skip_with_reference_data=not args.with_reference_data,
+        quiet=not verbose,
+    )
+    test_tools(
+        galaxy_interactor,
+        test_references,
+        results,
+        log=None,
+        verify_kwds=verify_kwds,
+    )
+    exceptions = results.test_exceptions
     if exceptions:
         raise exceptions[0]
 
@@ -116,14 +333,21 @@ def _arg_parser():
     parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key')
     parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
     parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.')
-    parser.add_argument('-t', '--tool-id', default=None, help='Tool ID')
-    parser.add_argument('--tool-version', default=None, help='Tool Version')
-    parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.')
+    parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID')
+    parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions')
+    parser.add_argument('-i', '--test-index', default=ALL_TESTS, type=int, help='Tool Test Index (starting at 0) - by default all tests will run.')
     parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
     parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.")
     parser.add_argument('-j', '--output-json', default=None, help='output metadata json')
     parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.")
     parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing")
+    parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output")
+    parser.add_argument('--with-reference-data', dest="with_reference_data", default=False, action="store_true")
+    parser.add_argument('--skip-with-reference-data', dest="with_reference_data", action="store_false", help="Skip tests the Galaxy server believes use data tables or loc files.")
+    parser.add_argument('--history-per-suite', dest="history_per_test_case", default=False, action="store_false", help="Create new history per test suite (all tests in same history).")
+    parser.add_argument('--history-per-test-case', dest="history_per_test_case", action="store_true", help="Create new history per test case.")
+    parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.")
+    parser.add_argument('--retries', default=0, type=int, help="Retry failed tests.")
     return parser
 
 

lib/galaxy/tools/toolbox/base.py

@@ -522,6 +522,11 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin):
                 for tool in self._tools_by_id.values():
                     if tool.old_id == tool_id:
                         rval.append(tool)
+                if get_all_versions and tool_id in self._tool_versions_by_id:
+                    for tool in self._tool_versions_by_id[tool_id].values():
+                        if tool not in rval:
+                            rval.append(tool)
+
                 # if we don't have a lineage_map for this tool we need to sort by version,
                 # so that the last tool in rval is the newest tool.
                 rval.sort(key=lambda t: t.version)

lib/galaxy/webapps/galaxy/api/tools.py

@@ -193,13 +193,25 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
         internals/Pythonisms in a rough way). If this endpoint is being used from outside
         of scripts shipped with Galaxy let us know and please be prepared for the response
         from this API to change its format in some ways.
+
+        If tool version is not passed, it is assumed to be latest. Tool version can be
+        set as '*' to get tests for all configured versions.
         """
-        # TODO: eliminate copy and paste with above code.
         if 'payload' in kwd:
             kwd = kwd.get('payload')
         tool_version = kwd.get('tool_version', None)
-        tool = self._get_tool(id, tool_version=tool_version, user=trans.user)
-        return [t.to_dict() for t in tool.tests]
+        if tool_version == "*":
+            tools = self.app.toolbox.get_tool(id, get_all_versions=True)
+            for tool in tools:
+                if not tool.allow_user_access(trans.user):
+                    raise exceptions.AuthenticationFailed("Access denied, please login for tool with id '%s'." % id)
+        else:
+            tools = [self._get_tool(id, tool_version=tool_version, user=trans.user)]
+
+        test_defs = []
+        for tool in tools:
+            test_defs.extend([t.to_dict() for t in tool.tests])
+        return test_defs
 
     @web.require_admin
     @expose_api

lib/galaxy_test/api/test_tools.py

@@ -699,6 +699,20 @@ class ToolsTestCase(ApiTestCase, TestsTools):
             output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1)
             self.assertEqual(output1_content.strip(), "Version " + version)
 
+    @skip_without_tool("multiple_versions")
+    @uses_test_history(require_new=False)
+    def test_test_by_versions(self, history_id):
+        test_data_response = self._get("tools/%s/test_data" % "multiple_versions")
+        test_data_response.raise_for_status()
+        test_data_dicts = test_data_response.json()
+        assert len(test_data_dicts) == 1
+        assert test_data_dicts[0]["tool_version"] == "0.2"
+
+        test_data_response = self._get("tools/%s/test_data?tool_version=*" % "multiple_versions")
+        test_data_response.raise_for_status()
+        test_data_dicts = test_data_response.json()
+        assert len(test_data_dicts) == 2
+
     @skip_without_tool("multiple_versions")
     @uses_test_history(require_new=False)
     def test_show_with_wrong_tool_version_in_tool_id(self, history_id):

test/functional/tools/multiple_versions_v01.xml

@@ -8,4 +8,14 @@
     <outputs>
         <data name="out_file1" format="txt" />
     </outputs>
+    <tests>
+        <test>
+            <param name="intest" value="1" />
+            <output name="out_file1">
+                <assert_contents>
+                    <has_line line="Version 0.1" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
 </tool>

test/functional/tools/multiple_versions_v02.xml

@@ -8,4 +8,14 @@
     <outputs>
         <data name="out_file1" format="txt" />
     </outputs>
+    <tests>
+        <test>
+            <param name="intest" value="1" />
+            <output name="out_file1">
+                <assert_contents>
+                    <has_line line="Version 0.2" />
+                </assert_contents>
+            </output>
+        </test>
+    </tests>
 </tool>

test/unit/tool_util/test_verify_script.py

@@ -0,0 +1,277 @@
+import json
+import os
+from tempfile import NamedTemporaryFile
+from unittest import mock
+
+from galaxy.tool_util.verify.script import (
+    _arg_parser,
+    build_case_references,
+    Results,
+    test_tools as run,
+    TestReference,
+)
+
+VT_PATH = 'galaxy.tool_util.verify.script.verify_tool'
+NEW_HISTORY = object()
+
+
+def test_arg_parse():
+    parser = _arg_parser()
+
+    # defaults
+    args = parser.parse_args([])
+    assert not args.with_reference_data
+    assert not args.history_per_test_case
+
+    # skip flags
+    args = parser.parse_args(["--skip-with-reference-data", "--history-per-test-case"])
+    assert not args.with_reference_data
+    assert args.history_per_test_case
+
+    # enable flags
+    args = parser.parse_args(["--with-reference-data", "--history-per-suite"])
+    assert args.with_reference_data
+    assert not args.history_per_test_case
+
+
+def test_test_tools():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("cat", "0.1.0", 0),
+        TestReference("cat", "0.1.0", 1),
+        TestReference("cat", "0.2.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+        run(
+            interactor,
+            test_references,
+            results,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 3
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+        assert interactor.history_created
+        assert interactor.history_deleted
+
+
+def test_test_tools_no_history_cleanup():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("cat", "0.1.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+        run(
+            interactor,
+            test_references,
+            results,
+            no_history_cleanup=True,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 1
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+        assert interactor.history_created
+        assert not interactor.history_deleted
+
+
+def test_test_tool_per_test_history():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("cat", "0.1.0", 0),
+        TestReference("cat", "0.1.0", 1),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+        run(
+            interactor,
+            test_references,
+            results,
+            history_per_test_case=True,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 2
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+        assert not interactor.history_created
+        assert not interactor.history_deleted
+
+
+def test_test_tools_records_exception():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("bad", "0.1.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+
+        def side_effect(*args, **kwd):
+            raise Exception("Cow")
+
+        mock_verify.side_effect = side_effect
+        run(
+            interactor,
+            test_references,
+            results,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 1
+        assert len(results.test_exceptions) == 1
+        assert_results_written(results)
+
+
+def test_test_tools_records_retry_exception():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("bad", "0.1.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+
+        count = 0
+
+        def side_effect(*args, **kwd):
+            nonlocal count
+            raise_exception = count == 0
+            count += 1
+            if raise_exception:
+                raise Exception("Cow")
+
+        mock_verify.side_effect = side_effect
+        run(
+            interactor,
+            test_references,
+            results,
+            retries=1,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 2
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+
+
+def test_results():
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    results.register_result({"id": "foo", "status": "success"})
+    results.write()
+    message = results.info_message()
+
+    with open(f.name, "r") as f:
+        report_obj = json.load(f)
+    assert "tests" in report_obj
+    assert len(report_obj["tests"]) == 1
+    assert report_obj["results"]["total"] == 1
+    assert report_obj["results"]["errors"] == 0
+    assert report_obj["results"]["skips"] == 0
+
+    assert "Passed tool tests (1)" in message
+    assert "Skipped tool tests (0)" in message
+
+    results.register_result({"id": "bar", "status": "skip"})
+    results.write()
+    message = results.info_message()
+
+    with open(f.name, "r") as f:
+        report_obj = json.load(f)
+    assert len(report_obj["tests"]) == 2
+    assert report_obj["results"]["skips"] == 1
+    assert "Passed tool tests (1)" in message
+    assert "Skipped tool tests (1)" in message
+
+
+def test_build_references():
+    interactor = MockGalaxyInteractor()
+    test_references = build_case_references(interactor)
+    assert len(test_references) == 6
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="*")
+    assert len(test_references) == 6
+
+    test_references = build_case_references(interactor, 'cat1', tool_version=None)
+    assert len(test_references) == 4
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="0.2.0")
+    assert len(test_references) == 4
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0")
+    assert len(test_references) == 2
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0", test_index=1)
+    assert len(test_references) == 1
+
+    # Specifying an index but not a version, grabs latest version and fills it in.
+    test_references = build_case_references(interactor, 'cat1', test_index=2)
+    assert len(test_references) == 1
+    test_reference = test_references[0]
+    assert test_reference.tool_id == "cat1"
+    assert test_reference.tool_version == "0.2.0"
+    assert test_reference.test_index == 2
+
+
+def assert_results_not_written(results):
+    assert os.stat(results.test_json).st_size == 0
+
+
+def assert_results_written(results):
+    assert os.stat(results.test_json).st_size > 0
+    with open(results.test_json, "r") as f:
+        json.load(f)
+
+
+class MockGalaxyInteractor:
+
+    def __init__(self):
+        self.history_deleted = False
+        self.history_created = False
+
+    def new_history(self, history_name=""):
+        self.history_created = True
+        return NEW_HISTORY
+
+    def delete_history(self, history):
+        self.history_deleted = True
+
+    def get_tests_summary(self):
+        return {
+            'cat1': {
+                '0.2.0': {
+                    'count': 4,
+                },
+                '0.1.0': {
+                    'count': 2,
+                },
+            },
+        }
+
+    def get_tool_tests(self, tool_id, tool_version=None):
+        tool_dict = self.get_tests_summary().get(tool_id)
+        test_defs = []
+        for this_tool_version, version_defs in tool_dict.items():
+            if tool_version is not None and tool_version != "*" and this_tool_version != tool_version:
+                continue
+
+            count = version_defs['count']
+            for i in range(count):
+                test_def = {
+                    'tool_id': tool_id,
+                    'tool_version': this_tool_version or '0.1.1-default',
+                }
+                test_defs.append(test_def)
+
+            if tool_version is None or tool_version != "*":
+                break
+
+        return test_defs