diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index 452b7593d09..64a00980d3c 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -466,7 +466,7 @@ class GalaxyInteractorApi: return output_data['id'] def delete_history(self, history): - return None + self._delete(f"histories/{history}") def __job_ready(self, job_id, history_id=None): if job_id is None: @@ -876,10 +876,12 @@ def verify_tool(tool_id, tool_version=None, quiet=False, test_history=None, + no_history_cleanup=False, force_path_paste=False, maxseconds=DEFAULT_TOOL_TEST_WAIT, tool_test_dicts=None, client_test_config=None, + skip_with_reference_data=False, skip_on_dynamic_param_errors=False): if resource_parameters is None: resource_parameters = {} @@ -900,20 +902,34 @@ def verify_tool(tool_id, "test_index": test_index, } client_config = client_test_config.get_test_config(job_data) + skip_message = None if client_config is not None: job_data.update(client_config) skip_message = job_data.get("skip") - if skip_message: - job_data["status"] = "skip" - register_job_data(job_data) - return + + if not skip_message and skip_with_reference_data: + required_data_tables = tool_test_dict.get("required_data_tables") + required_loc_files = tool_test_dict.get("required_loc_files") + # TODO: actually hit the API and see if these tables are available. + if required_data_tables: + skip_message = f"Skipping test because of required data tables ({required_data_tables})" + if required_loc_files: + skip_message = f"Skipping test because of required loc files ({required_loc_files})" + + if skip_message: + job_data["status"] = "skip" + register_job_data(job_data) + return tool_test_dict.setdefault('maxseconds', maxseconds) testdef = ToolTestDescription(tool_test_dict) _handle_def_errors(testdef) + created_history = False if test_history is None: - test_history = galaxy_interactor.new_history() + created_history = True + history_name = f"Tool Test History for {tool_id}/{tool_version}-{test_index}" + test_history = galaxy_interactor.new_history(history_name=history_name) # Upload data to test_history, run the tool and check the outputs - record # API input, job info, tool run exception, as well as exceptions related to @@ -979,7 +995,8 @@ def verify_tool(tool_id, job_data["status"] = status register_job_data(job_data) - galaxy_interactor.delete_history(test_history) + if created_history and not no_history_cleanup: + galaxy_interactor.delete_history(test_history) def _handle_def_errors(testdef): diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py index fc7b644137f..0ff0c437fcf 100644 --- a/lib/galaxy/tool_util/verify/script.py +++ b/lib/galaxy/tool_util/verify/script.py @@ -1,8 +1,12 @@ #!/usr/bin/env python import argparse +import datetime as dt import json +import os import sys +from collections import namedtuple +from concurrent.futures import thread, ThreadPoolExecutor import yaml @@ -13,7 +17,252 @@ from galaxy.tool_util.verify.interactor import ( ) DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance.""" -ALL_TESTS = "*all_tests*" +DEFAULT_SUITE_NAME = "Galaxy Tool Tests" +ALL_TESTS = "*" +ALL_TOOLS = "*" +ALL_VERSION = "*" +LATEST_VERSION = None + + +TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"]) +TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"]) + + +class Results: + + def __init__(self, default_suitename, test_json, append=False): + self.test_json = test_json or "-" + test_results = [] + test_exceptions = [] + suitename = default_suitename + if append: + assert test_json != "-" + with open(test_json) as f: + previous_results = json.load(f) + test_results = previous_results["tests"] + if "suitename" in previous_results: + suitename = previous_results["suitename"] + self.test_results = test_results + self.test_exceptions = test_exceptions + self.suitename = suitename + + def register_result(self, result): + self.test_results.append(result) + + def register_exception(self, test_exception): + self.test_exceptions.append(test_exception) + + def write(self): + tests = sorted(self.test_results, key=lambda el: el['id']) + n_passed, n_failures, n_skips = 0, 0, 0 + n_errors = len([e for e in self.test_exceptions if not e.was_recorded]) + for test in tests: + status = test['status'] + if status == "success": + n_passed += 1 + elif status == "error": + n_errors += 1 + elif status == "skip": + n_skips += 1 + elif status == "failure": + n_failures += 1 + report_obj = { + 'version': '0.1', + 'suitename': self.suitename, + 'results': { + 'total': n_passed + n_failures + n_skips + n_errors, + 'errors': n_errors, + 'failures': n_failures, + 'skips': n_skips, + }, + 'tests': tests, + } + if self.test_json == "-": + print(json.dumps(report_obj)) + else: + with open(self.test_json, "w") as f: + json.dump(report_obj, f) + + def info_message(self): + messages = [] + passed_tests = self._tests_with_status('success') + messages.append("Passed tool tests ({0}): {1}".format( + len(passed_tests), + [t["id"] for t in passed_tests] + )) + failed_tests = self._tests_with_status('failure') + messages.append("Failed tool tests ({0}): {1}".format( + len(failed_tests), + [t["id"] for t in failed_tests] + )) + skiped_tests = self._tests_with_status('skip') + messages.append("Skipped tool tests ({0}): {1}".format( + len(skiped_tests), + [t["id"] for t in skiped_tests] + )) + errored_tests = self._tests_with_status('error') + messages.append("Errored tool tests ({0}): {1}".format( + len(errored_tests), + [t["id"] for t in errored_tests] + )) + return "\n".join(messages) + + @property + def success_count(self): + self._tests_with_status('success') + + @property + def skip_count(self): + self._tests_with_status('skip') + + @property + def error_count(self): + return self._tests_with_status('error') + len(self.test_exceptions) + + @property + def failure_count(self): + return self._tests_with_status('failure') + + def _tests_with_status(self, status): + return [t for t in self.test_results if t.get("status") == status] + + +def test_tools( + galaxy_interactor, + test_references, + results, + log=None, + parallel_tests=1, + history_per_test_case=False, + no_history_cleanup=False, + retries=0, + verify_kwds=None, +): + """Run through tool tests and write report. + + Refactor this into Galaxy in 21.01. + """ + verify_kwds = (verify_kwds or {}).copy() + tool_test_start = dt.datetime.now() + history_created = False + if history_per_test_case: + test_history = None + else: + history_created = True + test_history = galaxy_interactor.new_history(history_name=f"History for {results.suitename}") + verify_kwds.update({ + "no_history_cleanup": no_history_cleanup, + "test_history": test_history, + }) + with ThreadPoolExecutor(max_workers=parallel_tests) as executor: + try: + for test_reference in test_references: + _test_tool( + executor=executor, + test_reference=test_reference, + results=results, + galaxy_interactor=galaxy_interactor, + log=log, + retries=retries, + verify_kwds=verify_kwds, + ) + finally: + # Always write report, even if test was cancelled. + try: + executor.shutdown(wait=True) + except KeyboardInterrupt: + executor._threads.clear() + thread._threads_queues.clear() + results.write() + if log: + log.info("Report written to '%s'", os.path.abspath(results.test_json)) + log.info(results.info_message()) + log.info("Total tool test time: {0}".format(dt.datetime.now() - tool_test_start)) + if history_created and not no_history_cleanup: + galaxy_interactor.delete_history(test_history) + + +def _test_tool( + executor, + test_reference, + results, + galaxy_interactor, + log, + retries, + verify_kwds, +): + tool_id = test_reference.tool_id + tool_version = test_reference.tool_version + test_index = test_reference.test_index + # If given a tool_id with a version suffix, strip it off so we can treat tool_version + # correctly at least in client_test_config. + if tool_version and tool_id.endswith("/" + tool_version): + tool_id = tool_id[:-len("/" + tool_version)] + + label_base = tool_id + if tool_version: + label_base += "/" + str(tool_version) + + test_id = label_base + "-" + str(test_index) + + def run_test(): + run_retries = retries + job_data = None + job_exception = None + + def register(job_data_): + nonlocal job_data + job_data = job_data_ + + try: + while run_retries >= 0: + job_exception = None + try: + if log: + log.info("Executing test '%s'", test_id) + verify_tool( + tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version, + register_job_data=register, **verify_kwds + ) + if log: + log.info("Test '%s' passed", test_id) + break + except Exception as e: + if log: + log.warning("Test '%s' failed", test_id, exc_info=True) + + job_exception = e + run_retries -= 1 + finally: + if job_data is not None: + results.register_result(job_data) + if job_exception is not None: + was_recorded = job_data is not None + test_exception = TestException(tool_id, job_exception, was_recorded) + results.register_exception(test_exception) + + executor.submit(run_test) + + +def build_case_references(galaxy_interactor, tool_id=ALL_TOOLS, tool_version=LATEST_VERSION, test_index=ALL_TESTS): + test_references = [] + if tool_id == ALL_TOOLS: + tests_summary = galaxy_interactor.get_tests_summary() + for tool_id, tool_versions_dict in tests_summary.items(): + for tool_version, summary in tool_versions_dict.items(): + for test_index in range(summary["count"]): + test_reference = TestReference(tool_id, tool_version, test_index) + test_references.append(test_reference) + else: + assert tool_id + tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {} + for i, tool_test_dict in enumerate(tool_test_dicts): + this_tool_version = tool_test_dict.get("tool_version", tool_version) + this_test_index = i + if test_index == ALL_TESTS or i == test_index: + test_reference = TestReference(tool_id, this_tool_version, this_test_index) + test_references.append(test_reference) + return test_references def main(argv=None): @@ -46,66 +295,31 @@ def main(argv=None): tool_id = args.tool_id tool_version = args.tool_version tools_client_test_config = DictClientTestConfig(client_test_config.get("tools")) + verbose = args.verbose galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds) - raw_test_index = args.test_index - if raw_test_index == ALL_TESTS: - tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) - test_indices = list(range(len(tool_test_dicts))) - else: - test_indices = [int(raw_test_index)] - - test_results = [] - - if args.append: - assert output_json_path != "-" - with open(output_json_path) as f: - previous_results = json.load(f) - test_results = previous_results["tests"] - - exceptions = [] - verbose = args.verbose - for test_index in test_indices: - if tool_version: - tool_id_and_version = f"{tool_id}/{tool_version}" - else: - tool_id_and_version = tool_id - - test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index) - - def register(job_data): - test_results.append({ - 'id': tool_id + "-" + str(test_index), - 'has_data': True, - 'data': job_data, - }) - - try: - verify_tool( - tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version, - register_job_data=register, quiet=not verbose, force_path_paste=args.force_path_paste, - client_test_config=tools_client_test_config, - ) - - if verbose: - print("%s passed" % test_identifier) - - except Exception as e: - if verbose: - print(f"{test_identifier} failed, {e}") - exceptions.append(e) - - report_obj = { - 'version': '0.1', - 'tests': test_results, - } - if output_json_path: - if output_json_path == "-": - print(json.dumps(report_obj)) - else: - with open(output_json_path, "w") as f: - json.dump(report_obj, f) - + test_references = build_case_references( + galaxy_interactor, + tool_id=tool_id, + tool_version=tool_version, + test_index=args.test_index, + ) + results = Results(args.suite_name, output_json_path, append=args.append) + verify_kwds = dict( + client_test_config=tools_client_test_config, + force_path_paste=args.force_path_paste, + skip_with_reference_data=args.skip_with_reference_data, + quiet=not verbose, + ) + test_tools( + galaxy_interactor, + test_references, + results, + verify_quiet=not verbose, + log=None, + verify_kwds=verify_kwds, + ) + exceptions = results.test_exceptions if exceptions: raise exceptions[0] @@ -116,14 +330,19 @@ def _arg_parser(): parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key') parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key') parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.') - parser.add_argument('-t', '--tool-id', default=None, help='Tool ID') - parser.add_argument('--tool-version', default=None, help='Tool Version') + parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID') + parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions') parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.') parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to') parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.") parser.add_argument('-j', '--output-json', default=None, help='output metadata json') parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.") parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing") + parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output") + parser.add_argument('--skip-with-reference-data', default=False, action="store_true", help="Skip tests the Galaxy server believes use data tables or loc files.") + parser.add_argument('--history-per-test-case', default=False, action="store_true", help="Create new history per test case.") + parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.") + parser.add_argument('--retries', default=0, help="Retry failed tests.") return parser diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 7bd35cfb3bd..4b6adb72495 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -203,7 +203,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): if tool_version == "*": tools = self.app.toolbox.get_tool(id, get_all_versions=True) for tool in tools: - if not tool.allow_user_access(user): + if not tool.allow_user_access(trans.user): raise exceptions.AuthenticationFailed("Access denied, please login for tool with id '%s'." % id) else: tools = [self._get_tool(id, tool_version=tool_version, user=trans.user)] diff --git a/test/unit/tool_util/test_verify_script.py b/test/unit/tool_util/test_verify_script.py new file mode 100644 index 00000000000..58f52ade7b7 --- /dev/null +++ b/test/unit/tool_util/test_verify_script.py @@ -0,0 +1,257 @@ +import json +import os +from tempfile import NamedTemporaryFile +from unittest import mock + +from galaxy.tool_util.verify.script import ( + build_case_references, + Results, + test_tools as run, + TestReference, +) + +VT_PATH = 'galaxy.tool_util.verify.script.verify_tool' +NEW_HISTORY = object() + + +def test_test_tools(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("cat", "0.1.0", 0), + TestReference("cat", "0.1.0", 1), + TestReference("cat", "0.2.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + run( + interactor, + test_references, + results, + ) + calls = mock_verify.call_args_list + assert len(calls) == 3 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + assert interactor.history_created + assert interactor.history_deleted + + +def test_test_tools_no_history_cleanup(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("cat", "0.1.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + run( + interactor, + test_references, + results, + no_history_cleanup=True, + ) + calls = mock_verify.call_args_list + assert len(calls) == 1 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + assert interactor.history_created + assert not interactor.history_deleted + + +def test_test_tool_per_test_history(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("cat", "0.1.0", 0), + TestReference("cat", "0.1.0", 1), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + run( + interactor, + test_references, + results, + history_per_test_case=True, + ) + calls = mock_verify.call_args_list + assert len(calls) == 2 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + assert not interactor.history_created + assert not interactor.history_deleted + + +def test_test_tools_records_exception(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("bad", "0.1.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + + def side_effect(*args, **kwd): + raise Exception("Cow") + + mock_verify.side_effect = side_effect + run( + interactor, + test_references, + results, + ) + calls = mock_verify.call_args_list + assert len(calls) == 1 + assert len(results.test_exceptions) == 1 + assert_results_written(results) + + +def test_test_tools_records_retry_exception(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("bad", "0.1.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + + count = 0 + + def side_effect(*args, **kwd): + nonlocal count + raise_exception = count == 0 + count += 1 + if raise_exception: + raise Exception("Cow") + + mock_verify.side_effect = side_effect + run( + interactor, + test_references, + results, + retries=1, + ) + calls = mock_verify.call_args_list + assert len(calls) == 2 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + + +def test_results(): + f = NamedTemporaryFile() + results = Results("my suite", f.name) + results.register_result({"id": "foo", "status": "success"}) + results.write() + message = results.info_message() + + with open(f.name, "r") as f: + report_obj = json.load(f) + assert "tests" in report_obj + assert len(report_obj["tests"]) == 1 + assert report_obj["results"]["total"] == 1 + assert report_obj["results"]["errors"] == 0 + assert report_obj["results"]["skips"] == 0 + + assert "Passed tool tests (1)" in message + assert "Skipped tool tests (0)" in message + + results.register_result({"id": "bar", "status": "skip"}) + results.write() + message = results.info_message() + + with open(f.name, "r") as f: + report_obj = json.load(f) + assert len(report_obj["tests"]) == 2 + assert report_obj["results"]["skips"] == 1 + assert "Passed tool tests (1)" in message + assert "Skipped tool tests (1)" in message + + +def test_build_references(): + interactor = MockGalaxyInteractor() + test_references = build_case_references(interactor) + assert len(test_references) == 6 + + test_references = build_case_references(interactor, 'cat1', tool_version="*") + assert len(test_references) == 6 + + test_references = build_case_references(interactor, 'cat1', tool_version=None) + assert len(test_references) == 4 + + test_references = build_case_references(interactor, 'cat1', tool_version="0.2.0") + assert len(test_references) == 4 + + test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0") + assert len(test_references) == 2 + + test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0", test_index=1) + assert len(test_references) == 1 + + # Specifying an index but not a version, grabs latest version and fills it in. + test_references = build_case_references(interactor, 'cat1', test_index=2) + assert len(test_references) == 1 + test_reference = test_references[0] + assert test_reference.tool_id == "cat1" + assert test_reference.tool_version == "0.2.0" + assert test_reference.test_index == 2 + + +def assert_results_not_written(results): + assert os.stat(results.test_json).st_size == 0 + + +def assert_results_written(results): + assert os.stat(results.test_json).st_size > 0 + with open(results.test_json, "r") as f: + json.load(f) + + +class MockGalaxyInteractor: + + def __init__(self): + self.history_deleted = False + self.history_created = False + + def new_history(self, history_name=""): + self.history_created = True + return NEW_HISTORY + + def delete_history(self, history): + self.history_deleted = True + + def get_tests_summary(self): + return { + 'cat1': { + '0.2.0': { + 'count': 4, + }, + '0.1.0': { + 'count': 2, + }, + }, + } + + def get_tool_tests(self, tool_id, tool_version=None): + tool_dict = self.get_tests_summary().get(tool_id) + test_defs = [] + for this_tool_version, version_defs in tool_dict.items(): + if tool_version is not None and tool_version != "*" and this_tool_version != tool_version: + continue + + count = version_defs['count'] + for i in range(count): + test_def = { + 'tool_id': tool_id, + 'tool_version': this_tool_version or '0.1.1-default', + } + test_defs.append(test_def) + + if tool_version is None or tool_version != "*": + break + + return test_defs