From 4d332519437f3e8c9665e7ff492b4f0a0579d265 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 2 Nov 2020 14:53:09 -0500 Subject: [PATCH 1/8] Allow test_data API to get tests for all tool versions. --- lib/galaxy/webapps/galaxy/api/tools.py | 18 +++++++++++++++--- lib/galaxy_test/api/test_tools.py | 12 ++++++++++++ .../functional/tools/multiple_versions_v01.xml | 10 ++++++++++ .../functional/tools/multiple_versions_v02.xml | 10 ++++++++++ 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index ee13fd4b858..7bd35cfb3bd 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -193,13 +193,25 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): internals/Pythonisms in a rough way). If this endpoint is being used from outside of scripts shipped with Galaxy let us know and please be prepared for the response from this API to change its format in some ways. + + If tool version is not passed, it is assumed to be latest. Tool version can be + set as '*' to get tests for all configured versions. """ - # TODO: eliminate copy and paste with above code. if 'payload' in kwd: kwd = kwd.get('payload') tool_version = kwd.get('tool_version', None) - tool = self._get_tool(id, tool_version=tool_version, user=trans.user) - return [t.to_dict() for t in tool.tests] + if tool_version == "*": + tools = self.app.toolbox.get_tool(id, get_all_versions=True) + for tool in tools: + if not tool.allow_user_access(user): + raise exceptions.AuthenticationFailed("Access denied, please login for tool with id '%s'." % id) + else: + tools = [self._get_tool(id, tool_version=tool_version, user=trans.user)] + + test_defs = [] + for tool in tools: + test_defs.extend([t.to_dict() for t in tool.tests]) + return test_defs @web.require_admin @expose_api diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 7d40cada9d4..05b81cf554a 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -686,6 +686,18 @@ class ToolsTestCase(ApiTestCase, TestsTools): output1_content = self.dataset_populator.get_history_dataset_content(history_id, dataset=output1) self.assertEqual(output1_content.strip(), "Version " + version) + @skip_without_tool("multiple_versions") + @uses_test_history(require_new=False) + def test_test_by_versions(self, history_id): + test_data_response = self._get("tools/%s/test_data" % "multiple_versions") + test_data_dicts = test_data_response.json() + assert len(test_data_dicts) == 1 + assert test_data_dicts[0]["tool_version"] == "0.2" + + test_data_response = self._get("tools/%s/test_data?tool_version=*" % "multiple_versions") + test_data_dicts = test_data_response.json() + assert len(test_data_dicts) == 2 + @skip_without_tool("multiple_versions") @uses_test_history(require_new=False) def test_show_with_wrong_tool_version_in_tool_id(self, history_id): diff --git a/test/functional/tools/multiple_versions_v01.xml b/test/functional/tools/multiple_versions_v01.xml index 8b05454529f..f7f4580049b 100644 --- a/test/functional/tools/multiple_versions_v01.xml +++ b/test/functional/tools/multiple_versions_v01.xml @@ -8,4 +8,14 @@ + + + + + + + + + + diff --git a/test/functional/tools/multiple_versions_v02.xml b/test/functional/tools/multiple_versions_v02.xml index 7a7a3a14937..5493c7f8140 100644 --- a/test/functional/tools/multiple_versions_v02.xml +++ b/test/functional/tools/multiple_versions_v02.xml @@ -8,4 +8,14 @@ + + + + + + + + + + From be2f25c8b8600d9da86fc730eb4bf9f400feac81 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 3 Nov 2020 10:03:59 -0500 Subject: [PATCH 2/8] Enhancements for galaxy-tool-test script. - Option to skip if reference data is annotated. - Try to build more abstractions to allow reuse with ephemeris. - Enhance to allow parallelization and enhnaced reporting like ephemeris. - Allow testing a whole tool panel. - Fix bugs in imported ephemeris logic (skipped failed tests and skipped tests for in counts/reporting for instance). - Unit tests for abstractions. --- lib/galaxy/tool_util/verify/interactor.py | 31 +- lib/galaxy/tool_util/verify/script.py | 341 ++++++++++++++++++---- lib/galaxy/webapps/galaxy/api/tools.py | 2 +- test/unit/tool_util/test_verify_script.py | 257 ++++++++++++++++ 4 files changed, 562 insertions(+), 69 deletions(-) create mode 100644 test/unit/tool_util/test_verify_script.py diff --git a/lib/galaxy/tool_util/verify/interactor.py b/lib/galaxy/tool_util/verify/interactor.py index 452b7593d09..64a00980d3c 100644 --- a/lib/galaxy/tool_util/verify/interactor.py +++ b/lib/galaxy/tool_util/verify/interactor.py @@ -466,7 +466,7 @@ class GalaxyInteractorApi: return output_data['id'] def delete_history(self, history): - return None + self._delete(f"histories/{history}") def __job_ready(self, job_id, history_id=None): if job_id is None: @@ -876,10 +876,12 @@ def verify_tool(tool_id, tool_version=None, quiet=False, test_history=None, + no_history_cleanup=False, force_path_paste=False, maxseconds=DEFAULT_TOOL_TEST_WAIT, tool_test_dicts=None, client_test_config=None, + skip_with_reference_data=False, skip_on_dynamic_param_errors=False): if resource_parameters is None: resource_parameters = {} @@ -900,20 +902,34 @@ def verify_tool(tool_id, "test_index": test_index, } client_config = client_test_config.get_test_config(job_data) + skip_message = None if client_config is not None: job_data.update(client_config) skip_message = job_data.get("skip") - if skip_message: - job_data["status"] = "skip" - register_job_data(job_data) - return + + if not skip_message and skip_with_reference_data: + required_data_tables = tool_test_dict.get("required_data_tables") + required_loc_files = tool_test_dict.get("required_loc_files") + # TODO: actually hit the API and see if these tables are available. + if required_data_tables: + skip_message = f"Skipping test because of required data tables ({required_data_tables})" + if required_loc_files: + skip_message = f"Skipping test because of required loc files ({required_loc_files})" + + if skip_message: + job_data["status"] = "skip" + register_job_data(job_data) + return tool_test_dict.setdefault('maxseconds', maxseconds) testdef = ToolTestDescription(tool_test_dict) _handle_def_errors(testdef) + created_history = False if test_history is None: - test_history = galaxy_interactor.new_history() + created_history = True + history_name = f"Tool Test History for {tool_id}/{tool_version}-{test_index}" + test_history = galaxy_interactor.new_history(history_name=history_name) # Upload data to test_history, run the tool and check the outputs - record # API input, job info, tool run exception, as well as exceptions related to @@ -979,7 +995,8 @@ def verify_tool(tool_id, job_data["status"] = status register_job_data(job_data) - galaxy_interactor.delete_history(test_history) + if created_history and not no_history_cleanup: + galaxy_interactor.delete_history(test_history) def _handle_def_errors(testdef): diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py index fc7b644137f..0ff0c437fcf 100644 --- a/lib/galaxy/tool_util/verify/script.py +++ b/lib/galaxy/tool_util/verify/script.py @@ -1,8 +1,12 @@ #!/usr/bin/env python import argparse +import datetime as dt import json +import os import sys +from collections import namedtuple +from concurrent.futures import thread, ThreadPoolExecutor import yaml @@ -13,7 +17,252 @@ from galaxy.tool_util.verify.interactor import ( ) DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance.""" -ALL_TESTS = "*all_tests*" +DEFAULT_SUITE_NAME = "Galaxy Tool Tests" +ALL_TESTS = "*" +ALL_TOOLS = "*" +ALL_VERSION = "*" +LATEST_VERSION = None + + +TestReference = namedtuple("TestReference", ["tool_id", "tool_version", "test_index"]) +TestException = namedtuple("TestException", ["tool_id", "exception", "was_recorded"]) + + +class Results: + + def __init__(self, default_suitename, test_json, append=False): + self.test_json = test_json or "-" + test_results = [] + test_exceptions = [] + suitename = default_suitename + if append: + assert test_json != "-" + with open(test_json) as f: + previous_results = json.load(f) + test_results = previous_results["tests"] + if "suitename" in previous_results: + suitename = previous_results["suitename"] + self.test_results = test_results + self.test_exceptions = test_exceptions + self.suitename = suitename + + def register_result(self, result): + self.test_results.append(result) + + def register_exception(self, test_exception): + self.test_exceptions.append(test_exception) + + def write(self): + tests = sorted(self.test_results, key=lambda el: el['id']) + n_passed, n_failures, n_skips = 0, 0, 0 + n_errors = len([e for e in self.test_exceptions if not e.was_recorded]) + for test in tests: + status = test['status'] + if status == "success": + n_passed += 1 + elif status == "error": + n_errors += 1 + elif status == "skip": + n_skips += 1 + elif status == "failure": + n_failures += 1 + report_obj = { + 'version': '0.1', + 'suitename': self.suitename, + 'results': { + 'total': n_passed + n_failures + n_skips + n_errors, + 'errors': n_errors, + 'failures': n_failures, + 'skips': n_skips, + }, + 'tests': tests, + } + if self.test_json == "-": + print(json.dumps(report_obj)) + else: + with open(self.test_json, "w") as f: + json.dump(report_obj, f) + + def info_message(self): + messages = [] + passed_tests = self._tests_with_status('success') + messages.append("Passed tool tests ({0}): {1}".format( + len(passed_tests), + [t["id"] for t in passed_tests] + )) + failed_tests = self._tests_with_status('failure') + messages.append("Failed tool tests ({0}): {1}".format( + len(failed_tests), + [t["id"] for t in failed_tests] + )) + skiped_tests = self._tests_with_status('skip') + messages.append("Skipped tool tests ({0}): {1}".format( + len(skiped_tests), + [t["id"] for t in skiped_tests] + )) + errored_tests = self._tests_with_status('error') + messages.append("Errored tool tests ({0}): {1}".format( + len(errored_tests), + [t["id"] for t in errored_tests] + )) + return "\n".join(messages) + + @property + def success_count(self): + self._tests_with_status('success') + + @property + def skip_count(self): + self._tests_with_status('skip') + + @property + def error_count(self): + return self._tests_with_status('error') + len(self.test_exceptions) + + @property + def failure_count(self): + return self._tests_with_status('failure') + + def _tests_with_status(self, status): + return [t for t in self.test_results if t.get("status") == status] + + +def test_tools( + galaxy_interactor, + test_references, + results, + log=None, + parallel_tests=1, + history_per_test_case=False, + no_history_cleanup=False, + retries=0, + verify_kwds=None, +): + """Run through tool tests and write report. + + Refactor this into Galaxy in 21.01. + """ + verify_kwds = (verify_kwds or {}).copy() + tool_test_start = dt.datetime.now() + history_created = False + if history_per_test_case: + test_history = None + else: + history_created = True + test_history = galaxy_interactor.new_history(history_name=f"History for {results.suitename}") + verify_kwds.update({ + "no_history_cleanup": no_history_cleanup, + "test_history": test_history, + }) + with ThreadPoolExecutor(max_workers=parallel_tests) as executor: + try: + for test_reference in test_references: + _test_tool( + executor=executor, + test_reference=test_reference, + results=results, + galaxy_interactor=galaxy_interactor, + log=log, + retries=retries, + verify_kwds=verify_kwds, + ) + finally: + # Always write report, even if test was cancelled. + try: + executor.shutdown(wait=True) + except KeyboardInterrupt: + executor._threads.clear() + thread._threads_queues.clear() + results.write() + if log: + log.info("Report written to '%s'", os.path.abspath(results.test_json)) + log.info(results.info_message()) + log.info("Total tool test time: {0}".format(dt.datetime.now() - tool_test_start)) + if history_created and not no_history_cleanup: + galaxy_interactor.delete_history(test_history) + + +def _test_tool( + executor, + test_reference, + results, + galaxy_interactor, + log, + retries, + verify_kwds, +): + tool_id = test_reference.tool_id + tool_version = test_reference.tool_version + test_index = test_reference.test_index + # If given a tool_id with a version suffix, strip it off so we can treat tool_version + # correctly at least in client_test_config. + if tool_version and tool_id.endswith("/" + tool_version): + tool_id = tool_id[:-len("/" + tool_version)] + + label_base = tool_id + if tool_version: + label_base += "/" + str(tool_version) + + test_id = label_base + "-" + str(test_index) + + def run_test(): + run_retries = retries + job_data = None + job_exception = None + + def register(job_data_): + nonlocal job_data + job_data = job_data_ + + try: + while run_retries >= 0: + job_exception = None + try: + if log: + log.info("Executing test '%s'", test_id) + verify_tool( + tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version, + register_job_data=register, **verify_kwds + ) + if log: + log.info("Test '%s' passed", test_id) + break + except Exception as e: + if log: + log.warning("Test '%s' failed", test_id, exc_info=True) + + job_exception = e + run_retries -= 1 + finally: + if job_data is not None: + results.register_result(job_data) + if job_exception is not None: + was_recorded = job_data is not None + test_exception = TestException(tool_id, job_exception, was_recorded) + results.register_exception(test_exception) + + executor.submit(run_test) + + +def build_case_references(galaxy_interactor, tool_id=ALL_TOOLS, tool_version=LATEST_VERSION, test_index=ALL_TESTS): + test_references = [] + if tool_id == ALL_TOOLS: + tests_summary = galaxy_interactor.get_tests_summary() + for tool_id, tool_versions_dict in tests_summary.items(): + for tool_version, summary in tool_versions_dict.items(): + for test_index in range(summary["count"]): + test_reference = TestReference(tool_id, tool_version, test_index) + test_references.append(test_reference) + else: + assert tool_id + tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) or {} + for i, tool_test_dict in enumerate(tool_test_dicts): + this_tool_version = tool_test_dict.get("tool_version", tool_version) + this_test_index = i + if test_index == ALL_TESTS or i == test_index: + test_reference = TestReference(tool_id, this_tool_version, this_test_index) + test_references.append(test_reference) + return test_references def main(argv=None): @@ -46,66 +295,31 @@ def main(argv=None): tool_id = args.tool_id tool_version = args.tool_version tools_client_test_config = DictClientTestConfig(client_test_config.get("tools")) + verbose = args.verbose galaxy_interactor = GalaxyInteractorApi(**galaxy_interactor_kwds) - raw_test_index = args.test_index - if raw_test_index == ALL_TESTS: - tool_test_dicts = galaxy_interactor.get_tool_tests(tool_id, tool_version=tool_version) - test_indices = list(range(len(tool_test_dicts))) - else: - test_indices = [int(raw_test_index)] - - test_results = [] - - if args.append: - assert output_json_path != "-" - with open(output_json_path) as f: - previous_results = json.load(f) - test_results = previous_results["tests"] - - exceptions = [] - verbose = args.verbose - for test_index in test_indices: - if tool_version: - tool_id_and_version = f"{tool_id}/{tool_version}" - else: - tool_id_and_version = tool_id - - test_identifier = "tool %s test # %d" % (tool_id_and_version, test_index) - - def register(job_data): - test_results.append({ - 'id': tool_id + "-" + str(test_index), - 'has_data': True, - 'data': job_data, - }) - - try: - verify_tool( - tool_id, galaxy_interactor, test_index=test_index, tool_version=tool_version, - register_job_data=register, quiet=not verbose, force_path_paste=args.force_path_paste, - client_test_config=tools_client_test_config, - ) - - if verbose: - print("%s passed" % test_identifier) - - except Exception as e: - if verbose: - print(f"{test_identifier} failed, {e}") - exceptions.append(e) - - report_obj = { - 'version': '0.1', - 'tests': test_results, - } - if output_json_path: - if output_json_path == "-": - print(json.dumps(report_obj)) - else: - with open(output_json_path, "w") as f: - json.dump(report_obj, f) - + test_references = build_case_references( + galaxy_interactor, + tool_id=tool_id, + tool_version=tool_version, + test_index=args.test_index, + ) + results = Results(args.suite_name, output_json_path, append=args.append) + verify_kwds = dict( + client_test_config=tools_client_test_config, + force_path_paste=args.force_path_paste, + skip_with_reference_data=args.skip_with_reference_data, + quiet=not verbose, + ) + test_tools( + galaxy_interactor, + test_references, + results, + verify_quiet=not verbose, + log=None, + verify_kwds=verify_kwds, + ) + exceptions = results.test_exceptions if exceptions: raise exceptions[0] @@ -116,14 +330,19 @@ def _arg_parser(): parser.add_argument('-k', '--key', default=None, help='Galaxy User API Key') parser.add_argument('-a', '--admin-key', default=None, help='Galaxy Admin API Key') parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.') - parser.add_argument('-t', '--tool-id', default=None, help='Tool ID') - parser.add_argument('--tool-version', default=None, help='Tool Version') + parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID') + parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions') parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.') parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to') parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.") parser.add_argument('-j', '--output-json', default=None, help='output metadata json') parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.") parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing") + parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output") + parser.add_argument('--skip-with-reference-data', default=False, action="store_true", help="Skip tests the Galaxy server believes use data tables or loc files.") + parser.add_argument('--history-per-test-case', default=False, action="store_true", help="Create new history per test case.") + parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.") + parser.add_argument('--retries', default=0, help="Retry failed tests.") return parser diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index 7bd35cfb3bd..4b6adb72495 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -203,7 +203,7 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin): if tool_version == "*": tools = self.app.toolbox.get_tool(id, get_all_versions=True) for tool in tools: - if not tool.allow_user_access(user): + if not tool.allow_user_access(trans.user): raise exceptions.AuthenticationFailed("Access denied, please login for tool with id '%s'." % id) else: tools = [self._get_tool(id, tool_version=tool_version, user=trans.user)] diff --git a/test/unit/tool_util/test_verify_script.py b/test/unit/tool_util/test_verify_script.py new file mode 100644 index 00000000000..58f52ade7b7 --- /dev/null +++ b/test/unit/tool_util/test_verify_script.py @@ -0,0 +1,257 @@ +import json +import os +from tempfile import NamedTemporaryFile +from unittest import mock + +from galaxy.tool_util.verify.script import ( + build_case_references, + Results, + test_tools as run, + TestReference, +) + +VT_PATH = 'galaxy.tool_util.verify.script.verify_tool' +NEW_HISTORY = object() + + +def test_test_tools(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("cat", "0.1.0", 0), + TestReference("cat", "0.1.0", 1), + TestReference("cat", "0.2.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + run( + interactor, + test_references, + results, + ) + calls = mock_verify.call_args_list + assert len(calls) == 3 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + assert interactor.history_created + assert interactor.history_deleted + + +def test_test_tools_no_history_cleanup(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("cat", "0.1.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + run( + interactor, + test_references, + results, + no_history_cleanup=True, + ) + calls = mock_verify.call_args_list + assert len(calls) == 1 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + assert interactor.history_created + assert not interactor.history_deleted + + +def test_test_tool_per_test_history(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("cat", "0.1.0", 0), + TestReference("cat", "0.1.0", 1), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + run( + interactor, + test_references, + results, + history_per_test_case=True, + ) + calls = mock_verify.call_args_list + assert len(calls) == 2 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + assert not interactor.history_created + assert not interactor.history_deleted + + +def test_test_tools_records_exception(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("bad", "0.1.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + + def side_effect(*args, **kwd): + raise Exception("Cow") + + mock_verify.side_effect = side_effect + run( + interactor, + test_references, + results, + ) + calls = mock_verify.call_args_list + assert len(calls) == 1 + assert len(results.test_exceptions) == 1 + assert_results_written(results) + + +def test_test_tools_records_retry_exception(): + interactor = MockGalaxyInteractor() + f = NamedTemporaryFile() + results = Results("my suite", f.name) + test_references = [ + TestReference("bad", "0.1.0", 0), + ] + with mock.patch(VT_PATH) as mock_verify: + assert_results_not_written(results) + + count = 0 + + def side_effect(*args, **kwd): + nonlocal count + raise_exception = count == 0 + count += 1 + if raise_exception: + raise Exception("Cow") + + mock_verify.side_effect = side_effect + run( + interactor, + test_references, + results, + retries=1, + ) + calls = mock_verify.call_args_list + assert len(calls) == 2 + assert len(results.test_exceptions) == 0 + assert_results_written(results) + + +def test_results(): + f = NamedTemporaryFile() + results = Results("my suite", f.name) + results.register_result({"id": "foo", "status": "success"}) + results.write() + message = results.info_message() + + with open(f.name, "r") as f: + report_obj = json.load(f) + assert "tests" in report_obj + assert len(report_obj["tests"]) == 1 + assert report_obj["results"]["total"] == 1 + assert report_obj["results"]["errors"] == 0 + assert report_obj["results"]["skips"] == 0 + + assert "Passed tool tests (1)" in message + assert "Skipped tool tests (0)" in message + + results.register_result({"id": "bar", "status": "skip"}) + results.write() + message = results.info_message() + + with open(f.name, "r") as f: + report_obj = json.load(f) + assert len(report_obj["tests"]) == 2 + assert report_obj["results"]["skips"] == 1 + assert "Passed tool tests (1)" in message + assert "Skipped tool tests (1)" in message + + +def test_build_references(): + interactor = MockGalaxyInteractor() + test_references = build_case_references(interactor) + assert len(test_references) == 6 + + test_references = build_case_references(interactor, 'cat1', tool_version="*") + assert len(test_references) == 6 + + test_references = build_case_references(interactor, 'cat1', tool_version=None) + assert len(test_references) == 4 + + test_references = build_case_references(interactor, 'cat1', tool_version="0.2.0") + assert len(test_references) == 4 + + test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0") + assert len(test_references) == 2 + + test_references = build_case_references(interactor, 'cat1', tool_version="0.1.0", test_index=1) + assert len(test_references) == 1 + + # Specifying an index but not a version, grabs latest version and fills it in. + test_references = build_case_references(interactor, 'cat1', test_index=2) + assert len(test_references) == 1 + test_reference = test_references[0] + assert test_reference.tool_id == "cat1" + assert test_reference.tool_version == "0.2.0" + assert test_reference.test_index == 2 + + +def assert_results_not_written(results): + assert os.stat(results.test_json).st_size == 0 + + +def assert_results_written(results): + assert os.stat(results.test_json).st_size > 0 + with open(results.test_json, "r") as f: + json.load(f) + + +class MockGalaxyInteractor: + + def __init__(self): + self.history_deleted = False + self.history_created = False + + def new_history(self, history_name=""): + self.history_created = True + return NEW_HISTORY + + def delete_history(self, history): + self.history_deleted = True + + def get_tests_summary(self): + return { + 'cat1': { + '0.2.0': { + 'count': 4, + }, + '0.1.0': { + 'count': 2, + }, + }, + } + + def get_tool_tests(self, tool_id, tool_version=None): + tool_dict = self.get_tests_summary().get(tool_id) + test_defs = [] + for this_tool_version, version_defs in tool_dict.items(): + if tool_version is not None and tool_version != "*" and this_tool_version != tool_version: + continue + + count = version_defs['count'] + for i in range(count): + test_def = { + 'tool_id': tool_id, + 'tool_version': this_tool_version or '0.1.1-default', + } + test_defs.append(test_def) + + if tool_version is None or tool_version != "*": + break + + return test_defs From ab142bf9c510d03bda004a88fc870336c2d09c88 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 5 Nov 2020 11:59:19 -0500 Subject: [PATCH 3/8] Rebase into handling multiple versions. --- lib/galaxy/tools/toolbox/base.py | 10 +++++++--- lib/galaxy_test/api/test_tools.py | 2 ++ 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index 8f6f6b0ad88..fd8a4740025 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -519,9 +519,13 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): rval.append(lineage_tool) if not rval: # still no tool, do a deeper search and try to match by old ids - for tool in self._tools_by_id.values(): - if tool.old_id == tool_id: - rval.append(tool) + if get_all_versions and tool_id in self._tool_versions_by_id: + rval.extend(self._tool_versions_by_id[tool_id].values()) + else: + for tool in self._tools_by_id.values(): + if tool.old_id == tool_id: + rval.append(tool) + # if we don't have a lineage_map for this tool we need to sort by version, # so that the last tool in rval is the newest tool. rval.sort(key=lambda t: t.version) diff --git a/lib/galaxy_test/api/test_tools.py b/lib/galaxy_test/api/test_tools.py index 05b81cf554a..b322822a9ef 100644 --- a/lib/galaxy_test/api/test_tools.py +++ b/lib/galaxy_test/api/test_tools.py @@ -690,11 +690,13 @@ class ToolsTestCase(ApiTestCase, TestsTools): @uses_test_history(require_new=False) def test_test_by_versions(self, history_id): test_data_response = self._get("tools/%s/test_data" % "multiple_versions") + test_data_response.raise_for_status() test_data_dicts = test_data_response.json() assert len(test_data_dicts) == 1 assert test_data_dicts[0]["tool_version"] == "0.2" test_data_response = self._get("tools/%s/test_data?tool_version=*" % "multiple_versions") + test_data_response.raise_for_status() test_data_dicts = test_data_response.json() assert len(test_data_dicts) == 2 From 9467106ee01ef5a08ede2e3b67a720a282edaa7b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 5 Nov 2020 15:19:24 -0500 Subject: [PATCH 4/8] Or maybe this? --- lib/galaxy/tools/toolbox/base.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/tools/toolbox/base.py b/lib/galaxy/tools/toolbox/base.py index fd8a4740025..2d80b7cdfd2 100644 --- a/lib/galaxy/tools/toolbox/base.py +++ b/lib/galaxy/tools/toolbox/base.py @@ -519,11 +519,12 @@ class AbstractToolBox(Dictifiable, ManagesIntegratedToolPanelMixin): rval.append(lineage_tool) if not rval: # still no tool, do a deeper search and try to match by old ids + for tool in self._tools_by_id.values(): + if tool.old_id == tool_id: + rval.append(tool) if get_all_versions and tool_id in self._tool_versions_by_id: - rval.extend(self._tool_versions_by_id[tool_id].values()) - else: - for tool in self._tools_by_id.values(): - if tool.old_id == tool_id: + for tool in self._tool_versions_by_id[tool_id].values(): + if tool not in rval: rval.append(tool) # if we don't have a lineage_map for this tool we need to sort by version, From 050141a5f6ef0d7644e5ce584c027f47a7f95845 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 8 Nov 2020 13:12:36 +0100 Subject: [PATCH 5/8] Store test id --- lib/galaxy/tool_util/verify/script.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py index 0ff0c437fcf..83816f3a6aa 100644 --- a/lib/galaxy/tool_util/verify/script.py +++ b/lib/galaxy/tool_util/verify/script.py @@ -213,6 +213,7 @@ def _test_tool( def register(job_data_): nonlocal job_data job_data = job_data_ + job_data['id'] = test_id try: while run_retries >= 0: From 406c3acdcb9b43f519b8f979a419318540c9838f Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 8 Nov 2020 13:12:59 +0100 Subject: [PATCH 6/8] verify_quiet is already in verify_kwds --- lib/galaxy/tool_util/verify/script.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py index 83816f3a6aa..a809aa22300 100644 --- a/lib/galaxy/tool_util/verify/script.py +++ b/lib/galaxy/tool_util/verify/script.py @@ -316,7 +316,6 @@ def main(argv=None): galaxy_interactor, test_references, results, - verify_quiet=not verbose, log=None, verify_kwds=verify_kwds, ) From 64682ebde75bb4f16ab0888091a2ebe1f59c6402 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 8 Nov 2020 13:13:51 +0100 Subject: [PATCH 7/8] Parse --test-index and --retries as integer --- lib/galaxy/tool_util/verify/script.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py index a809aa22300..3c58d81b1a2 100644 --- a/lib/galaxy/tool_util/verify/script.py +++ b/lib/galaxy/tool_util/verify/script.py @@ -332,7 +332,7 @@ def _arg_parser(): parser.add_argument('--force_path_paste', default=False, action="store_true", help='This requires Galaxy-side config option "allow_path_paste" enabled. Allows for fetching test data locally. Only for admins.') parser.add_argument('-t', '--tool-id', default=ALL_TOOLS, help='Tool ID') parser.add_argument('--tool-version', default=None, help='Tool Version (if tool id supplied). Defaults to just latest version, use * to test all versions') - parser.add_argument('-i', '--test-index', default=ALL_TESTS, help='Tool Test Index (starting at 0) - by default all tests will run.') + parser.add_argument('-i', '--test-index', default=ALL_TESTS, type=int, help='Tool Test Index (starting at 0) - by default all tests will run.') parser.add_argument('-o', '--output', default=None, help='directory to dump outputs to') parser.add_argument('--append', default=False, action="store_true", help="Extend a test record json (created with --output-json) with additional tests.") parser.add_argument('-j', '--output-json', default=None, help='output metadata json') @@ -342,7 +342,7 @@ def _arg_parser(): parser.add_argument('--skip-with-reference-data', default=False, action="store_true", help="Skip tests the Galaxy server believes use data tables or loc files.") parser.add_argument('--history-per-test-case', default=False, action="store_true", help="Create new history per test case.") parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.") - parser.add_argument('--retries', default=0, help="Retry failed tests.") + parser.add_argument('--retries', default=0, type=int, help="Retry failed tests.") return parser From 1cf6c980f2877f2709b6c525ce1debdcdaebb015 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Sun, 8 Nov 2020 08:23:51 -0500 Subject: [PATCH 8/8] More fixes for tool test script. --- lib/galaxy/tool_util/verify/script.py | 17 +++++++++++------ test/unit/tool_util/test_verify_script.py | 20 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/tool_util/verify/script.py b/lib/galaxy/tool_util/verify/script.py index 3c58d81b1a2..688608c990c 100644 --- a/lib/galaxy/tool_util/verify/script.py +++ b/lib/galaxy/tool_util/verify/script.py @@ -18,7 +18,7 @@ from galaxy.tool_util.verify.interactor import ( DESCRIPTION = """Script to quickly run a tool test against a running Galaxy instance.""" DEFAULT_SUITE_NAME = "Galaxy Tool Tests" -ALL_TESTS = "*" +ALL_TESTS = -1 ALL_TOOLS = "*" ALL_VERSION = "*" LATEST_VERSION = None @@ -213,7 +213,6 @@ def _test_tool( def register(job_data_): nonlocal job_data job_data = job_data_ - job_data['id'] = test_id try: while run_retries >= 0: @@ -236,7 +235,11 @@ def _test_tool( run_retries -= 1 finally: if job_data is not None: - results.register_result(job_data) + results.register_result({ + "id": test_id, + "has_data": True, + "data": job_data, + }) if job_exception is not None: was_recorded = job_data is not None test_exception = TestException(tool_id, job_exception, was_recorded) @@ -309,7 +312,7 @@ def main(argv=None): verify_kwds = dict( client_test_config=tools_client_test_config, force_path_paste=args.force_path_paste, - skip_with_reference_data=args.skip_with_reference_data, + skip_with_reference_data=not args.with_reference_data, quiet=not verbose, ) test_tools( @@ -339,8 +342,10 @@ def _arg_parser(): parser.add_argument('--verbose', default=False, action="store_true", help="Verbose logging.") parser.add_argument('-c', '--client-test-config', default=None, help="Test config YAML to help with client testing") parser.add_argument('--suite-name', default=DEFAULT_SUITE_NAME, help="Suite name for tool test output") - parser.add_argument('--skip-with-reference-data', default=False, action="store_true", help="Skip tests the Galaxy server believes use data tables or loc files.") - parser.add_argument('--history-per-test-case', default=False, action="store_true", help="Create new history per test case.") + parser.add_argument('--with-reference-data', dest="with_reference_data", default=False, action="store_true") + parser.add_argument('--skip-with-reference-data', dest="with_reference_data", action="store_false", help="Skip tests the Galaxy server believes use data tables or loc files.") + parser.add_argument('--history-per-suite', dest="history_per_test_case", default=False, action="store_false", help="Create new history per test suite (all tests in same history).") + parser.add_argument('--history-per-test-case', dest="history_per_test_case", action="store_true", help="Create new history per test case.") parser.add_argument('--no-history-cleanup', default=False, action="store_true", help="Perserve histories created for testing.") parser.add_argument('--retries', default=0, type=int, help="Retry failed tests.") return parser diff --git a/test/unit/tool_util/test_verify_script.py b/test/unit/tool_util/test_verify_script.py index 58f52ade7b7..357964a9ff1 100644 --- a/test/unit/tool_util/test_verify_script.py +++ b/test/unit/tool_util/test_verify_script.py @@ -4,6 +4,7 @@ from tempfile import NamedTemporaryFile from unittest import mock from galaxy.tool_util.verify.script import ( + _arg_parser, build_case_references, Results, test_tools as run, @@ -14,6 +15,25 @@ VT_PATH = 'galaxy.tool_util.verify.script.verify_tool' NEW_HISTORY = object() +def test_arg_parse(): + parser = _arg_parser() + + # defaults + args = parser.parse_args([]) + assert not args.with_reference_data + assert not args.history_per_test_case + + # skip flags + args = parser.parse_args(["--skip-with-reference-data", "--history-per-test-case"]) + assert not args.with_reference_data + assert args.history_per_test_case + + # enable flags + args = parser.parse_args(["--with-reference-data", "--history-per-suite"]) + assert args.with_reference_data + assert not args.history_per_test_case + + def test_test_tools(): interactor = MockGalaxyInteractor() f = NamedTemporaryFile()