Enhancements for galaxy-tool-test script.

- Option to skip if reference data is annotated. - Try to build more abstractions to allow reuse with ephemeris. - Enhance to allow parallelization and enhnaced reporting like ephemeris. - Allow testing a whole tool panel. - Fix bugs in imported ephemeris logic (skipped failed tests and skipped tests for in counts/reporting for instance). - Unit tests for abstractions.
2026-10-16 15:45:04 +08:00 · 2020-11-11 22:21:27 -05:00
parent 4d33251943
commit be2f25c8b8
4 changed files with 562 additions and 69 deletions
@@ -466,7 +466,7 @@ class GalaxyInteractorApi:
        return output_data['id']

    def delete_history(self, history):
-        return None
+        self._delete(f"histories/{history}")

    def __job_ready(self, job_id, history_id=None):
        if job_id is None:
@@ -876,10 +876,12 @@ def verify_tool(tool_id,
                tool_version=None,
                quiet=False,
                test_history=None,
+                no_history_cleanup=False,
                force_path_paste=False,
                maxseconds=DEFAULT_TOOL_TEST_WAIT,
                tool_test_dicts=None,
                client_test_config=None,
+                skip_with_reference_data=False,
                skip_on_dynamic_param_errors=False):
    if resource_parameters is None:
        resource_parameters = {}
@@ -900,20 +902,34 @@ def verify_tool(tool_id,
        "test_index": test_index,
    }
    client_config = client_test_config.get_test_config(job_data)
+    skip_message = None
    if client_config is not None:
        job_data.update(client_config)
        skip_message = job_data.get("skip")
-        if skip_message:
-            job_data["status"] = "skip"
-            register_job_data(job_data)
-            return
+
+    if not skip_message and skip_with_reference_data:
+        required_data_tables = tool_test_dict.get("required_data_tables")
+        required_loc_files = tool_test_dict.get("required_loc_files")
+        # TODO: actually hit the API and see if these tables are available.
+        if required_data_tables:
+            skip_message = f"Skipping test because of required data tables ({required_data_tables})"
+        if required_loc_files:
+            skip_message = f"Skipping test because of required loc files ({required_loc_files})"
+
+    if skip_message:
+        job_data["status"] = "skip"
+        register_job_data(job_data)
+        return

    tool_test_dict.setdefault('maxseconds', maxseconds)
    testdef = ToolTestDescription(tool_test_dict)
    _handle_def_errors(testdef)

+    created_history = False
    if test_history is None:
-        test_history = galaxy_interactor.new_history()
+        created_history = True
+        history_name = f"Tool Test History for {tool_id}/{tool_version}-{test_index}"
+        test_history = galaxy_interactor.new_history(history_name=history_name)

    # Upload data to test_history, run the tool and check the outputs - record
    # API input, job info, tool run exception, as well as exceptions related to
@@ -979,7 +995,8 @@ def verify_tool(tool_id,
            job_data["status"] = status
            register_job_data(job_data)

-    galaxy_interactor.delete_history(test_history)
+    if created_history and not no_history_cleanup:
+        galaxy_interactor.delete_history(test_history)


 def _handle_def_errors(testdef):
@@ -1,8 +1,12 @@
 #!/usr/bin/env python

 import argparse
+import datetime as dt
 import json
+import os
 import sys
+from collections import namedtuple
+from concurrent.futures import thread, ThreadPoolExecutor

 import yaml

@@ -13,7 +17,252 @@ from galaxy.tool_util.verify.interactor import (
 )

 DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance."""
-ALL_TESTS = "*all_tests*"
+DEFAULT_SUITE_NAME = "Galaxy Tool Tests"
+ALL_TESTS = "*"
+ALL_TOOLS = "*"
+ALL_VERSION = "*"
+LATEST_VERSION = None
+
+
+TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"])
+TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"])
+
+
+class Results:
+
+    def __init__(self, default_suitename, test_json, append=False):
+        self.test_json = test_json or "-"
+        test_results = []
+        test_exceptions = []
+        suitename = default_suitename
+        if append:
+            assert test_json != "-"
+            with open(test_json) as f:
+                previous_results = json.load(f)
+                test_results = previous_results["tests"]
+                if "suitename" in previous_results:
+                    suitename = previous_results["suitename"]
+        self.test_results = test_results
+        self.test_exceptions = test_exceptions
+        self.suitename = suitename
+
+    def register_result(self, result):
+        self.test_results.append(result)
+
+    def register_exception(self, test_exception):
+        self.test_exceptions.append(test_exception)
+
+    def write(self):
+        tests = sorted(self.test_results, key=lambda el: el['id'])
+        n_passed, n_failures, n_skips = 0, 0, 0
+        n_errors = len([e for e in self.test_exceptions if not e.was_recorded])
+        for test in tests:
+            status = test['status']
+            if status == "success":
+                n_passed += 1
+            elif status == "error":
+                n_errors += 1
+            elif status == "skip":
+                n_skips += 1
+            elif status == "failure":
+                n_failures += 1
+        report_obj = {
+            'version': '0.1',
+            'suitename': self.suitename,
+            'results': {
+                'total': n_passed + n_failures + n_skips + n_errors,
+                'errors': n_errors,
+                'failures': n_failures,
+                'skips': n_skips,
+            },
+            'tests': tests,
+        }
+        if self.test_json == "-":
+            print(json.dumps(report_obj))
+        else:
+            with open(self.test_json, "w") as f:
+                json.dump(report_obj, f)
+
+    def info_message(self):
+        messages = []
+        passed_tests = self._tests_with_status('success')
+        messages.append("Passed tool tests ({0}): {1}".format(
+            len(passed_tests),
+            [t["id"] for t in passed_tests]
+        ))
+        failed_tests = self._tests_with_status('failure')
+        messages.append("Failed tool tests ({0}): {1}".format(
+            len(failed_tests),
+            [t["id"] for t in failed_tests]
+        ))
+        skiped_tests = self._tests_with_status('skip')
+        messages.append("Skipped tool tests ({0}): {1}".format(
+            len(skiped_tests),
+            [t["id"] for t in skiped_tests]
+        ))
+        errored_tests = self._tests_with_status('error')
+        messages.append("Errored tool tests ({0}): {1}".format(
+            len(errored_tests),
+            [t["id"] for t in errored_tests]
+        ))
+        return "\n".join(messages)
+
+    @property
+    def success_count(self):
+        self._tests_with_status('success')
+
+    @property
+    def skip_count(self):
+        self._tests_with_status('skip')
+
+    @property
+    def error_count(self):
+        return self._tests_with_status('error') + len(self.test_exceptions)
+
+    @property
+    def failure_count(self):
+        return self._tests_with_status('failure')
+
+    def _tests_with_status(self, status):
+        return [t for t in self.test_results if t.get("status") == status]
+
+
+def test_tools(
+    galaxy_interactor,
+    test_references,
+    results,
+    log=None,
+    parallel_tests=1,
+    history_per_test_case=False,
+    no_history_cleanup=False,
+    retries=0,
+    verify_kwds=None,
+):
+    """Run through tool tests and write report.
+
+    Refactor this into Galaxy in 21.01.
+    """
+    verify_kwds = (verify_kwds or {}).copy()
+    tool_test_start = dt.datetime.now()
+    history_created = False
+    if history_per_test_case:
+        test_history = None
+    else:
+        history_created = True
+        test_history = galaxy_interactor.new_history(history_name=f"History for {results.suitename}")
+    verify_kwds.update({
+        "no_history_cleanup": no_history_cleanup,
+        "test_history": test_history,
+    })
+    with ThreadPoolExecutor(max_workers=parallel_tests) as executor:
+        try:
+            for test_reference in test_references:
+                _test_tool(
+                    executor=executor,
+                    test_reference=test_reference,
+                    results=results,
+                    galaxy_interactor=galaxy_interactor,
+                    log=log,
+                    retries=retries,
+                    verify_kwds=verify_kwds,
+                )
+        finally:
+            # Always write report, even if test was cancelled.
+            try:
+                executor.shutdown(wait=True)
+            except KeyboardInterrupt:
+                executor._threads.clear()
+                thread._threads_queues.clear()
+            results.write()
+            if log:
+                log.info("Report written to '%s'", os.path.abspath(results.test_json))
+                log.info(results.info_message())
+                log.info("Total tool test time: {0}".format(dt.datetime.now() - tool_test_start))
+            if history_created and not no_history_cleanup:
+                galaxy_interactor.delete_history(test_history)
+
+
+def _test_tool(
+    executor,
+    test_reference,
+    results,
+    galaxy_interactor,
+    log,
+    retries,
+    verify_kwds,
+):
+    tool_id = test_reference.tool_id
+    tool_version = test_reference.tool_version
+    test_index = test_reference.test_index
+    # If given a tool_id with a version suffix, strip it off so we can treat tool_version
+    # correctly at least in client_test_config.
+    if tool_version and tool_id.endswith("/" + tool_version):
+        tool_id = tool_id[:-len("/" + tool_version)]
+
+    label_base = tool_id
+    if tool_version:
+        label_base += "/" + str(tool_version)
+
+    test_id = label_base + "-" + str(test_index)
+
+    def run_test():
+        run_retries = retries
+        job_data = None
+        job_exception = None
+
+        def register(job_data_):
+            nonlocal job_data
+            job_data = job_data_
+
+        try:
+            while run_retries >= 0:
+                job_exception = None
+                try:
+                    if log:
+                        log.info("Executing test '%s'", test_id)
+                    verify_tool(
+                        tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
+                        register_job_data=register, **verify_kwds
+                    )
+                    if log:
+                        log.info("Test '%s' passed", test_id)
+                    break
+                except Exception as e:
+                    if log:
+                        log.warning("Test '%s' failed", test_id, exc_info=True)
+
+                    job_exception = e
+                    run_retries -= 1
+        finally:
+            if job_data is not None:
+                results.register_result(job_data)
+            if job_exception is not None:
+                was_recorded = job_data is not None
+                test_exception = TestException(tool_id, job_exception, was_recorded)
+                results.register_exception(test_exception)
+
+    executor.submit(run_test)
+
+
+def build_case_references(galaxy_interactor, tool_id=ALL_TOOLS, tool_version=LATEST_VERSION, test_index=ALL_TESTS):
+    test_references = []
+    if tool_id == ALL_TOOLS:
+        tests_summary = galaxy_interactor.get_tests_summary()
+        for tool_id, tool_versions_dict in tests_summary.items():
+            for tool_version, summary in tool_versions_dict.items():
+                for test_index in range(summary["count"]):
+                    test_reference = TestReference(tool_id, tool_version, test_index)
+                    test_references.append(test_reference)
+    else:
+        assert tool_id
+        tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {}
+        for i, tool_test_dict in enumerate(tool_test_dicts):
+            this_tool_version = tool_test_dict.get("tool_version", tool_version)
+            this_test_index = i
+            if test_index == ALL_TESTS or i == test_index:
+                test_reference = TestReference(tool_id, this_tool_version, this_test_index)
+                test_references.append(test_reference)
+    return test_references


 def main(argv=None):
@@ -46,66 +295,31 @@ def main(argv=None):
    tool_id = args.tool_id
    tool_version = args.tool_version
    tools_client_test_config = DictClientTestConfig(client_test_config.get("tools"))
+    verbose = args.verbose

    galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds)
-    raw_test_index = args.test_index
-    if raw_test_index == ALL_TESTS:
-        tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version)
-        test_indices = list(range(len(tool_test_dicts)))
-    else:
-        test_indices = [int(raw_test_index)]
-
-    test_results = []
-
-    if args.append:
-        assert output_json_path != "-"
-        with open(output_json_path) as f:
-            previous_results = json.load(f)
-            test_results = previous_results["tests"]
-
-    exceptions = []
-    verbose = args.verbose
-    for test_index in test_indices:
-        if tool_version:
-            tool_id_and_version = f"{tool_id}/{tool_version}"
-        else:
-            tool_id_and_version = tool_id
-
-        test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index)
-
-        def register(job_data):
-            test_results.append({
-                'id': tool_id + "-" + str(test_index),
-                'has_data': True,
-                'data': job_data,
-            })
-
-        try:
-            verify_tool(
-                tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version,
-                register_job_data=register, quiet=not verbose, force_path_paste=args.force_path_paste,
-                client_test_config=tools_client_test_config,
-            )
-
-            if verbose:
-                print("%s passed" % test_identifier)
-
-        except Exception as e:
-            if verbose:
-                print(f"{test_identifier} failed, {e}")
-            exceptions.append(e)
-
-    report_obj = {
-        'version': '0.1',
-        'tests': test_results,
-    }
-    if output_json_path:
-        if output_json_path == "-":
-            print(json.dumps(report_obj))
-        else:
-            with open(output_json_path, "w") as f:
-                json.dump(report_obj, f)
-
+    test_references = build_case_references(
+        galaxy_interactor,
+        tool_id=tool_id,
+        tool_version=tool_version,
+        test_index=args.test_index,
+    )
+    results = Results(args.suite_name, output_json_path, append=args.append)
+    verify_kwds = dict(
+        client_test_config=tools_client_test_config,
+        force_path_paste=args.force_path_paste,
+        skip_with_reference_data=args.skip_with_reference_data,
+        quiet=not verbose,
+    )
+    test_tools(
+        galaxy_interactor,
+        test_references,
+        results,
+        verify_quiet=not verbose,
+        log=None,
+        verify_kwds=verify_kwds,
+    )
+    exceptions = results.test_exceptions
    if exceptions:
        raise exceptions[0]

@@ -116,14 +330,19 @@ def _arg_parser():
    parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key')
    parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key')
    parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.')
-    parser.add_argument('-t', '--tool-id', default=None, help='Tool ID')
-    parser.add_argument('--tool-version', default=None, help='Tool Version')
+    parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID')
+    parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions')
    parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.')
    parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to')
    parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.")
    parser.add_argument('-j', '--output-json', default=None, help='output metadata json')
    parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.")
    parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing")
+    parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output")
+    parser.add_argument('--skip-with-reference-data', default=False, action="store_true", help="Skip tests the Galaxy server believes use data tables or loc files.")
+    parser.add_argument('--history-per-test-case', default=False, action="store_true", help="Create new history per test case.")
+    parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.")
+    parser.add_argument('--retries', default=0, help="Retry failed tests.")
    return parser


@@ -203,7 +203,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
        if tool_version == "*":
            tools = self.app.toolbox.get_tool(id, get_all_versions=True)
            for tool in tools:
-                if not tool.allow_user_access(user):
+                if not tool.allow_user_access(trans.user):
                    raise exceptions.AuthenticationFailed("Access denied, please login for tool with id '%s'." % id)
        else:
            tools = [self._get_tool(id, tool_version=tool_version, user=trans.user)]
@@ -0,0 +1,257 @@
+import json
+import os
+from tempfile import NamedTemporaryFile
+from unittest import mock
+
+from galaxy.tool_util.verify.script import (
+    build_case_references,
+    Results,
+    test_tools as run,
+    TestReference,
+)
+
+VT_PATH = 'galaxy.tool_util.verify.script.verify_tool'
+NEW_HISTORY = object()
+
+
+def test_test_tools():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("cat", "0.1.0", 0),
+        TestReference("cat", "0.1.0", 1),
+        TestReference("cat", "0.2.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+        run(
+            interactor,
+            test_references,
+            results,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 3
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+        assert interactor.history_created
+        assert interactor.history_deleted
+
+
+def test_test_tools_no_history_cleanup():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("cat", "0.1.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+        run(
+            interactor,
+            test_references,
+            results,
+            no_history_cleanup=True,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 1
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+        assert interactor.history_created
+        assert not interactor.history_deleted
+
+
+def test_test_tool_per_test_history():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("cat", "0.1.0", 0),
+        TestReference("cat", "0.1.0", 1),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+        run(
+            interactor,
+            test_references,
+            results,
+            history_per_test_case=True,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 2
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+        assert not interactor.history_created
+        assert not interactor.history_deleted
+
+
+def test_test_tools_records_exception():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("bad", "0.1.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+
+        def side_effect(*args, **kwd):
+            raise Exception("Cow")
+
+        mock_verify.side_effect = side_effect
+        run(
+            interactor,
+            test_references,
+            results,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 1
+        assert len(results.test_exceptions) == 1
+        assert_results_written(results)
+
+
+def test_test_tools_records_retry_exception():
+    interactor = MockGalaxyInteractor()
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    test_references = [
+        TestReference("bad", "0.1.0", 0),
+    ]
+    with mock.patch(VT_PATH) as mock_verify:
+        assert_results_not_written(results)
+
+        count = 0
+
+        def side_effect(*args, **kwd):
+            nonlocal count
+            raise_exception = count == 0
+            count += 1
+            if raise_exception:
+                raise Exception("Cow")
+
+        mock_verify.side_effect = side_effect
+        run(
+            interactor,
+            test_references,
+            results,
+            retries=1,
+        )
+        calls = mock_verify.call_args_list
+        assert len(calls) == 2
+        assert len(results.test_exceptions) == 0
+        assert_results_written(results)
+
+
+def test_results():
+    f = NamedTemporaryFile()
+    results = Results("my suite", f.name)
+    results.register_result({"id": "foo", "status": "success"})
+    results.write()
+    message = results.info_message()
+
+    with open(f.name, "r") as f:
+        report_obj = json.load(f)
+    assert "tests" in report_obj
+    assert len(report_obj["tests"]) == 1
+    assert report_obj["results"]["total"] == 1
+    assert report_obj["results"]["errors"] == 0
+    assert report_obj["results"]["skips"] == 0
+
+    assert "Passed tool tests (1)" in message
+    assert "Skipped tool tests (0)" in message
+
+    results.register_result({"id": "bar", "status": "skip"})
+    results.write()
+    message = results.info_message()
+
+    with open(f.name, "r") as f:
+        report_obj = json.load(f)
+    assert len(report_obj["tests"]) == 2
+    assert report_obj["results"]["skips"] == 1
+    assert "Passed tool tests (1)" in message
+    assert "Skipped tool tests (1)" in message
+
+
+def test_build_references():
+    interactor = MockGalaxyInteractor()
+    test_references = build_case_references(interactor)
+    assert len(test_references) == 6
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="*")
+    assert len(test_references) == 6
+
+    test_references = build_case_references(interactor, 'cat1', tool_version=None)
+    assert len(test_references) == 4
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="0.2.0")
+    assert len(test_references) == 4
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0")
+    assert len(test_references) == 2
+
+    test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0", test_index=1)
+    assert len(test_references) == 1
+
+    # Specifying an index but not a version, grabs latest version and fills it in.
+    test_references = build_case_references(interactor, 'cat1', test_index=2)
+    assert len(test_references) == 1
+    test_reference = test_references[0]
+    assert test_reference.tool_id == "cat1"
+    assert test_reference.tool_version == "0.2.0"
+    assert test_reference.test_index == 2
+
+
+def assert_results_not_written(results):
+    assert os.stat(results.test_json).st_size == 0
+
+
+def assert_results_written(results):
+    assert os.stat(results.test_json).st_size > 0
+    with open(results.test_json, "r") as f:
+        json.load(f)
+
+
+class MockGalaxyInteractor:
+
+    def __init__(self):
+        self.history_deleted = False
+        self.history_created = False
+
+    def new_history(self, history_name=""):
+        self.history_created = True
+        return NEW_HISTORY
+
+    def delete_history(self, history):
+        self.history_deleted = True
+
+    def get_tests_summary(self):
+        return {
+            'cat1': {
+                '0.2.0': {
+                    'count': 4,
+                },
+                '0.1.0': {
+                    'count': 2,
+                },
+            },
+        }
+
+    def get_tool_tests(self, tool_id, tool_version=None):
+        tool_dict = self.get_tests_summary().get(tool_id)
+        test_defs = []
+        for this_tool_version, version_defs in tool_dict.items():
+            if tool_version is not None and tool_version != "*" and this_tool_version != tool_version:
+                continue
+
+            count = version_defs['count']
+            for i in range(count):
+                test_def = {
+                    'tool_id': tool_id,
+                    'tool_version': this_tool_version or '0.1.1-default',
+                }
+                test_defs.append(test_def)
+
+            if tool_version is None or tool_version != "*":
+                break
+
+        return test_defs