mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-21 13:50:20 +08:00
331 lines
14 KiB
Python
331 lines
14 KiB
Python
from __future__ import print_function
|
|
|
|
import logging
|
|
import new
|
|
import sys
|
|
|
|
try:
|
|
from nose.tools import nottest
|
|
except ImportError:
|
|
def nottest(x):
|
|
return x
|
|
|
|
from base.instrument import register_job_data
|
|
from base.interactor import build_interactor, RunToolException, stage_data_in_history
|
|
from galaxy.tools import DataManagerTool # noqa: I201
|
|
from galaxy.tools.verify.asserts import verify_assertions
|
|
from galaxy.util import bunch
|
|
from .twilltestcase import TwillTestCase
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
toolbox = None
|
|
|
|
# Do not test Data Managers as part of the standard Tool Test Framework.
|
|
TOOL_TYPES_NO_TEST = (DataManagerTool, )
|
|
|
|
|
|
class ToolTestCase(TwillTestCase):
|
|
"""Abstract test case that runs tests based on a `galaxy.tools.test.ToolTest`.
|
|
|
|
Ideally this would be FunctionalTestCase instead of a TwillTestCase but the
|
|
subclass DataManagerToolTestCase requires the use of Twill still.
|
|
"""
|
|
|
|
def do_it(self, testdef, resource_parameters={}):
|
|
"""
|
|
Run through a tool test case.
|
|
"""
|
|
shed_tool_id = self.shed_tool_id
|
|
|
|
self._handle_test_def_errors(testdef)
|
|
|
|
galaxy_interactor = self._galaxy_interactor(testdef)
|
|
|
|
test_history = galaxy_interactor.new_history()
|
|
|
|
stage_data_in_history(galaxy_interactor, testdef.test_data(), test_history, shed_tool_id)
|
|
|
|
# Once data is ready, run the tool and check the outputs - record API
|
|
# input, job info, tool run exception, as well as exceptions related to
|
|
# job output checking and register they with the test plugin so it can
|
|
# record structured information.
|
|
tool_inputs = None
|
|
job_stdio = None
|
|
job_output_exceptions = None
|
|
tool_execution_exception = None
|
|
expected_failure_occurred = False
|
|
try:
|
|
try:
|
|
tool_response = galaxy_interactor.run_tool(testdef, test_history, resource_parameters=resource_parameters)
|
|
data_list, jobs, tool_inputs = tool_response.outputs, tool_response.jobs, tool_response.inputs
|
|
data_collection_list = tool_response.output_collections
|
|
except RunToolException as e:
|
|
tool_inputs = e.inputs
|
|
tool_execution_exception = e
|
|
if not testdef.expect_failure:
|
|
raise e
|
|
else:
|
|
expected_failure_occurred = True
|
|
except Exception as e:
|
|
tool_execution_exception = e
|
|
raise e
|
|
|
|
if not expected_failure_occurred:
|
|
self.assertTrue(data_list or data_collection_list)
|
|
|
|
try:
|
|
job_stdio = self._verify_outputs(testdef, test_history, jobs, shed_tool_id, data_list, data_collection_list, galaxy_interactor)
|
|
except JobOutputsError as e:
|
|
job_stdio = e.job_stdio
|
|
job_output_exceptions = e.output_exceptions
|
|
raise e
|
|
except Exception as e:
|
|
job_output_exceptions = [e]
|
|
raise e
|
|
finally:
|
|
job_data = {}
|
|
if tool_inputs is not None:
|
|
job_data["inputs"] = tool_inputs
|
|
if job_stdio is not None:
|
|
job_data["job"] = job_stdio
|
|
if job_output_exceptions:
|
|
job_data["output_problems"] = [str(_) for _ in job_output_exceptions]
|
|
if tool_execution_exception:
|
|
job_data["execution_problem"] = str(tool_execution_exception)
|
|
register_job_data(job_data)
|
|
|
|
galaxy_interactor.delete_history(test_history)
|
|
|
|
def _galaxy_interactor(self, testdef):
|
|
return build_interactor(self, testdef.interactor)
|
|
|
|
def _handle_test_def_errors(self, testdef):
|
|
# If the test generation had an error, raise
|
|
if testdef.error:
|
|
if testdef.exception:
|
|
raise testdef.exception
|
|
else:
|
|
raise Exception("Test parse failure")
|
|
|
|
def _verify_outputs(self, testdef, history, jobs, shed_tool_id, data_list, data_collection_list, galaxy_interactor):
|
|
assert len(jobs) == 1, "Test framework logic error, somehow tool test resulted in more than one job."
|
|
job = jobs[0]
|
|
|
|
maxseconds = testdef.maxseconds
|
|
if testdef.num_outputs is not None:
|
|
expected = testdef.num_outputs
|
|
actual = len(data_list)
|
|
if expected != actual:
|
|
messaage_template = "Incorrect number of outputs - expected %d, found %s."
|
|
message = messaage_template % (expected, actual)
|
|
raise Exception(message)
|
|
found_exceptions = []
|
|
|
|
def register_exception(e):
|
|
if not found_exceptions:
|
|
# Only print this stuff out once.
|
|
for stream in ['stdout', 'stderr']:
|
|
if stream in job_stdio:
|
|
print(self._format_stream(job_stdio[stream], stream=stream, format=True), file=sys.stderr)
|
|
found_exceptions.append(e)
|
|
|
|
if testdef.expect_failure:
|
|
if testdef.outputs:
|
|
raise Exception("Cannot specify outputs in a test expecting failure.")
|
|
|
|
# Wait for the job to complete and register expections if the final
|
|
# status was not what test was expecting.
|
|
job_failed = False
|
|
try:
|
|
galaxy_interactor.wait_for_job(job['id'], history, maxseconds)
|
|
except Exception as e:
|
|
job_failed = True
|
|
if not testdef.expect_failure:
|
|
found_exceptions.append(e)
|
|
|
|
job_stdio = galaxy_interactor.get_job_stdio(job['id'])
|
|
|
|
if not job_failed and testdef.expect_failure:
|
|
error = AssertionError("Expected job to fail but Galaxy indicated the job successfully completed.")
|
|
register_exception(error)
|
|
|
|
expect_exit_code = testdef.expect_exit_code
|
|
if expect_exit_code is not None:
|
|
exit_code = job_stdio["exit_code"]
|
|
if str(expect_exit_code) != str(exit_code):
|
|
error = AssertionError("Expected job to complete with exit code %s, found %s" % (expect_exit_code, exit_code))
|
|
register_exception(error)
|
|
|
|
for output_index, output_tuple in enumerate(testdef.outputs):
|
|
# Get the correct hid
|
|
name, outfile, attributes = output_tuple
|
|
output_testdef = bunch.Bunch(name=name, outfile=outfile, attributes=attributes)
|
|
try:
|
|
output_data = data_list[name]
|
|
except (TypeError, KeyError):
|
|
# Legacy - fall back on ordered data list access if data_list is
|
|
# just a list (case with twill variant or if output changes its
|
|
# name).
|
|
if hasattr(data_list, "values"):
|
|
output_data = list(data_list.values())[output_index]
|
|
else:
|
|
output_data = data_list[len(data_list) - len(testdef.outputs) + output_index]
|
|
self.assertTrue(output_data is not None)
|
|
try:
|
|
galaxy_interactor.verify_output(history, jobs, output_data, output_testdef=output_testdef, shed_tool_id=shed_tool_id, maxseconds=maxseconds)
|
|
except Exception as e:
|
|
register_exception(e)
|
|
|
|
other_checks = {
|
|
"command_line": "Command produced by the job",
|
|
"stdout": "Standard output of the job",
|
|
"stderr": "Standard error of the job",
|
|
}
|
|
for what, description in other_checks.items():
|
|
if getattr(testdef, what, None) is not None:
|
|
try:
|
|
data = job_stdio[what]
|
|
verify_assertions(data, getattr(testdef, what))
|
|
except AssertionError as err:
|
|
errmsg = '%s different than expected\n' % description
|
|
errmsg += str(err)
|
|
register_exception(AssertionError(errmsg))
|
|
|
|
for output_collection_def in testdef.output_collections:
|
|
try:
|
|
name = output_collection_def.name
|
|
# TODO: data_collection_list is clearly a bad name for dictionary.
|
|
if name not in data_collection_list:
|
|
template = "Failed to find output [%s], tool outputs include [%s]"
|
|
message = template % (name, ",".join(data_collection_list.keys()))
|
|
raise AssertionError(message)
|
|
|
|
# Data collection returned from submission, elements may have been populated after
|
|
# the job completed so re-hit the API for more information.
|
|
data_collection_returned = data_collection_list[name]
|
|
data_collection = galaxy_interactor._get("dataset_collections/%s" % data_collection_returned["id"], data={"instance_type": "history"}).json()
|
|
|
|
def get_element(elements, id):
|
|
for element in elements:
|
|
if element["element_identifier"] == id:
|
|
return element
|
|
return False
|
|
|
|
expected_collection_type = output_collection_def.collection_type
|
|
if expected_collection_type:
|
|
collection_type = data_collection["collection_type"]
|
|
if expected_collection_type != collection_type:
|
|
template = "Expected output collection [%s] to be of type [%s], was of type [%s]."
|
|
message = template % (name, expected_collection_type, collection_type)
|
|
raise AssertionError(message)
|
|
|
|
expected_element_count = output_collection_def.count
|
|
if expected_element_count:
|
|
actual_element_count = len(data_collection["elements"])
|
|
if expected_element_count != actual_element_count:
|
|
template = "Expected output collection [%s] to have %s elements, but it had %s."
|
|
message = template % (name, expected_element_count, actual_element_count)
|
|
raise AssertionError(message)
|
|
|
|
def verify_elements(element_objects, element_tests):
|
|
for element_identifier, (element_outfile, element_attrib) in element_tests.items():
|
|
element = get_element(element_objects, element_identifier)
|
|
if not element:
|
|
template = "Failed to find identifier [%s] for testing, tool generated collection elements [%s]"
|
|
message = template % (element_identifier, element_objects)
|
|
raise AssertionError(message)
|
|
|
|
element_type = element["element_type"]
|
|
if element_type != "dataset_collection":
|
|
hda = element["object"]
|
|
galaxy_interactor.verify_output_dataset(
|
|
history,
|
|
hda_id=hda["id"],
|
|
outfile=element_outfile,
|
|
attributes=element_attrib,
|
|
shed_tool_id=shed_tool_id
|
|
)
|
|
if element_type == "dataset_collection":
|
|
elements = element["object"]["elements"]
|
|
verify_elements(elements, element_attrib.get("elements", {}))
|
|
|
|
verify_elements(data_collection["elements"], output_collection_def.element_tests)
|
|
except Exception as e:
|
|
register_exception(e)
|
|
|
|
if found_exceptions:
|
|
raise JobOutputsError(found_exceptions, job_stdio)
|
|
else:
|
|
return job_stdio
|
|
|
|
def _format_stream(self, output, stream, format):
|
|
output = output or ''
|
|
if format:
|
|
msg = "---------------------- >> begin tool %s << -----------------------\n" % stream
|
|
msg += output + "\n"
|
|
msg += "----------------------- >> end tool %s << ------------------------\n" % stream
|
|
else:
|
|
msg = output
|
|
return msg
|
|
|
|
|
|
class JobOutputsError(AssertionError):
|
|
|
|
def __init__(self, output_exceptions, job_stdio):
|
|
big_message = "\n".join(map(str, output_exceptions))
|
|
super(JobOutputsError, self).__init__(big_message)
|
|
self.job_stdio = job_stdio
|
|
self.output_exceptions = output_exceptions
|
|
|
|
|
|
@nottest
|
|
def build_tests(app=None, testing_shed_tools=False, master_api_key=None, user_api_key=None):
|
|
"""
|
|
If the module level variable `toolbox` is set, generate `ToolTestCase`
|
|
classes for all of its tests and put them into this modules globals() so
|
|
they can be discovered by nose.
|
|
"""
|
|
if app is None:
|
|
return
|
|
|
|
# Push all the toolbox tests to module level
|
|
G = globals()
|
|
|
|
# Eliminate all previous tests from G.
|
|
for key, val in G.items():
|
|
if key.startswith('TestForTool_'):
|
|
del G[key]
|
|
for i, tool_id in enumerate(app.toolbox.tools_by_id):
|
|
tool = app.toolbox.get_tool(tool_id)
|
|
if isinstance(tool, TOOL_TYPES_NO_TEST):
|
|
# We do not test certain types of tools (e.g. Data Manager tools) as part of ToolTestCase
|
|
continue
|
|
if tool.tests:
|
|
shed_tool_id = None if not testing_shed_tools else tool.id
|
|
# Create a new subclass of ToolTestCase, dynamically adding methods
|
|
# named test_tool_XXX that run each test defined in the tool config.
|
|
name = "TestForTool_" + tool.id.replace(' ', '_')
|
|
baseclasses = (ToolTestCase, )
|
|
namespace = dict()
|
|
for j, testdef in enumerate(tool.tests):
|
|
test_function_name = 'test_tool_%06d' % j
|
|
|
|
def make_test_method(td):
|
|
def test_tool(self):
|
|
self.do_it(td)
|
|
test_tool.__name__ = test_function_name
|
|
|
|
return test_tool
|
|
|
|
test_method = make_test_method(testdef)
|
|
test_method.__doc__ = "%s ( %s ) > %s" % (tool.name, tool.id, testdef.name)
|
|
namespace[test_function_name] = test_method
|
|
namespace['shed_tool_id'] = shed_tool_id
|
|
namespace['master_api_key'] = master_api_key
|
|
namespace['user_api_key'] = user_api_key
|
|
# The new.classobj function returns a new class object, with name name, derived
|
|
# from baseclasses (which should be a tuple of classes) and with namespace dict.
|
|
new_class_obj = new.classobj(name, baseclasses, namespace)
|
|
G[name] = new_class_obj
|