diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py index 16afe0f6dca..382c7eee8ce 100644 --- a/doc/parse_gx_xsd.py +++ b/doc/parse_gx_xsd.py @@ -90,28 +90,32 @@ def _build_tag(tag, hide_attributes): tag_help = StringIO() annotation_el = tag_el.find("{http://www.w3.org/2001/XMLSchema}annotation") text = annotation_el.find("{http://www.w3.org/2001/XMLSchema}documentation").text + text = _replace_attribute_list(tag, text, attributes) for line in text.splitlines(): - if line.startswith("$attribute_list:"): - attributes_str, header_level = line.split(":")[1:3] - attribute_names = attributes_str.split(",") - header_level = int(header_level) - text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) if line.startswith("$assertions"): assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']") - assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='TestAssertion']") assertions_buffer = StringIO() assertions_buffer.write(_doc_or_none(assertions_tag)) assertions_buffer.write("\n\n") - assertions_buffer.write("Child Element/Assertion | Details \n") - assertions_buffer.write("--- | ---\n") - elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element") - for element in elements: - doc = _doc_or_none(element) - if doc is None: - doc = _doc_or_none(_type_el(element)) - assert doc is not None, "Documentation for %s is empty" % element.attrib["name"] - doc = doc.strip() - assertions_buffer.write("``{}`` | {}\n".format(element.attrib["name"], doc)) + + assertion_groups = assertions_tag.xpath("xs:choice/xs:group", namespaces={'xs': 'http://www.w3.org/2001/XMLSchema'}) + for group in assertion_groups: + ref = group.attrib['ref'] + assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='" + ref + "']") + doc = _doc_or_none(assertion_tag) + assertions_buffer.write(f"### {doc}\n\n") + elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element") + for element in elements: + doc = _doc_or_none(element) + if doc is None: + doc = _doc_or_none(_type_el(element)) + assert doc is not None, "Documentation for %s is empty" % element.attrib["name"] + doc = doc.strip() + + element_el = _find_tag_el(element) + element_attributes = _find_attributes(element_el) + doc = _replace_attribute_list(element_el, doc, element_attributes) + assertions_buffer.write(f"#### ``{element.attrib['name']}``:\n\n{doc}\n\n") text = text.replace(line, assertions_buffer.getvalue()) tag_help.write(text) best_practices = _get_bp_link(annotation_el) @@ -125,6 +129,20 @@ element [here](%s).""" % best_practices) return tag_help.getvalue() +def _replace_attribute_list(tag, text, attributes): + for line in text.splitlines(): + if not line.startswith("$attribute_list:"): + continue + attributes_str, header_level = line.split(":")[1:3] + if attributes_str == "": + attribute_names = None + else: + attribute_names = attributes_str.split(",") + header_level = int(header_level) + text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) + return text + + def _get_bp_link(annotation_el): anchor = annotation_el.attrib.get("{http://galaxyproject.org/xml/1.0}best_practices", None) link = None @@ -190,8 +208,9 @@ def _find_tag_el(tag): def _type_el(tag): element_type = tag.attrib["type"] - type_el = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType/[@name='%s']" % element_type) or \ - xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}simpleType/[@name='%s']" % element_type) + type_el = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType/[@name='%s']" % element_type) + if type_el is None: + type_el = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}simpleType/[@name='%s']" % element_type) return type_el diff --git a/lib/galaxy/tool_util/linters/tests.py b/lib/galaxy/tool_util/linters/tests.py index 9936147efb8..e6ff65321aa 100644 --- a/lib/galaxy/tool_util/linters/tests.py +++ b/lib/galaxy/tool_util/linters/tests.py @@ -1,5 +1,8 @@ """This module contains a linting functions for tool tests.""" +from inspect import Parameter, signature + from ._util import is_datasource +from ..verify import asserts # Misspelled so as not be picked up by nosetests. @@ -35,9 +38,14 @@ def lint_tsts(tool_xml, lint_ctx): break test_assert = ("assert_stdout", "assert_stderr", "assert_command") for ta in test_assert: - if len(test.findall(ta)) > 0: - has_test = True - break + assertions = test.findall(ta) + if len(assertions) == 0: + continue + if len(assertions) > 1: + lint_ctx.error("Test {test_idx}: More than one {ta} found. Only the first is considered.") + has_test = True + _check_asserts(test_idx, assertions, lint_ctx) + _check_asserts(test_idx, test.findall(".//assert_contents"), lint_ctx) # really simple test that test parameters are also present in the inputs for param in test.findall("param"): @@ -94,6 +102,45 @@ def lint_tsts(tool_xml, lint_ctx): lint_ctx.warn("No valid test(s) found.", line=tests_line, xpath=tests_path) +def _check_asserts(test_idx, assertions, lint_ctx): + """ + assertions is a list of assert_contents, assert_stdout, assert_stderr, assert_command + in practice only for the first case the list may be longer than one + """ + for assertion in assertions: + for i, a in enumerate(assertion.iter()): + if i == 0: # skip root note itself + continue + assert_function_name = "assert_" + a.tag + if assert_function_name not in asserts.assertion_functions: + lint_ctx.error(f"Test {test_idx}: unknown assertion {a.tag}") + continue + assert_function_sig = signature(asserts.assertion_functions[assert_function_name]) + # check type of the attributes (int, float ...) + for attrib in a.attrib: + if attrib not in assert_function_sig.parameters: + lint_ctx.error(f"Test {test_idx}: unknown attribute {attrib} for {a.tag}") + continue + if assert_function_sig.parameters[attrib].annotation is not Parameter.empty: + try: + assert_function_sig.parameters[attrib].annotation(a.attrib[attrib]) + except ValueError: + lint_ctx.error(f"Test {test_idx}: attribute {attrib} for {a.tag} needs to be {assert_function_sig.parameters[attrib].annotation.__name__} got {a.attrib[attrib]}") + # check missing required attributes + for p in assert_function_sig.parameters: + if p in ["output", "output_bytes", "verify_assertions_function", "children"]: + continue + if assert_function_sig.parameters[p].default is Parameter.empty and p not in a.attrib: + lint_ctx.error(f"Test {test_idx}: missing attribute {p} for {a.tag}") + # has_n_lines, has_n_columns, and has_size need to specify n/value, min, or max + if a.tag in ["has_n_lines", "has_n_columns"]: + if "n" not in a.attrib and "min" not in a.attrib and "max" not in a.attrib: + lint_ctx.error(f"Test {test_idx}: {a.tag} needs to specify 'n', 'min', or 'max'") + if a.tag == "has_size": + if "value" not in a.attrib and "min" not in a.attrib and "max" not in a.attrib: + lint_ctx.error(f"Test {test_idx}: {a.tag} needs to specify 'n', 'min', or 'max'") + + def _collect_output_names(tool_xml): output_data_names = [] output_collection_names = [] diff --git a/lib/galaxy/tool_util/verify/asserts/__init__.py b/lib/galaxy/tool_util/verify/asserts/__init__.py index 234e5e929b6..df54c4cdbbf 100644 --- a/lib/galaxy/tool_util/verify/asserts/__init__.py +++ b/lib/galaxy/tool_util/verify/asserts/__init__.py @@ -1,6 +1,6 @@ import logging import sys -from inspect import getfullargspec +from inspect import getfullargspec, getmembers from galaxy.util import unicodify @@ -12,16 +12,19 @@ assertion_module_names = ['text', 'tabular', 'xml', 'hdf5', 'archive', 'size'] # create a new module of assertion functions, create the needed python # source file "test/base/asserts/.py" and add # to the list of assertion module names defined above. -assertion_modules = [] +assertion_functions = {} for assertion_module_name in assertion_module_names: full_assertion_module_name = f"galaxy.tool_util.verify.asserts.{assertion_module_name}" try: # Dynamically import module __import__(full_assertion_module_name) assertion_module = sys.modules[full_assertion_module_name] - assertion_modules.append(assertion_module) except Exception: log.exception('Failed to load assertion module: %s', assertion_module_name) + continue + for member, value in getmembers(assertion_module): + if member.startswith("assert_"): + assertion_functions[member] = value def verify_assertions(data, assertion_description_list): @@ -33,14 +36,11 @@ def verify_assertions(data, assertion_description_list): def verify_assertion(data, assertion_description): tag = assertion_description["tag"] - assert_function_name = f"assert_{tag}" - assert_function = None - for assertion_module in assertion_modules: - if hasattr(assertion_module, assert_function_name): - assert_function = getattr(assertion_module, assert_function_name) + assert_function_name = "assert_" + tag + assert_function = assertion_functions.get(assert_function_name) if assert_function is None: - errmsg = f"Unable to find test function associated with XML tag '{tag}'. Check your tool file syntax." + errmsg = f"Unable to find test function associated with XML tag {tag}. Check your tool file syntax." raise AssertionError(errmsg) assert_function_args = getfullargspec(assert_function).args diff --git a/lib/galaxy/tool_util/verify/asserts/_util.py b/lib/galaxy/tool_util/verify/asserts/_util.py new file mode 100644 index 00000000000..98d63e74a5b --- /dev/null +++ b/lib/galaxy/tool_util/verify/asserts/_util.py @@ -0,0 +1,58 @@ +from math import inf + +from galaxy.util import asbool +from galaxy.util.bytesize import parse_bytesize + + +def _assert_number(count, n, delta, min, max, negate, n_text, min_max_text): + """ + helper function for assering that count is in + - [n-delta:n+delta] + - [min:max] + + raising an assertion error using n_text and min_max_text (resp) + substituting {n}, {delta}, {min}, and {max} + (and keeping potentially present {text} and {output}) + + n, delta, min, max can be suffixed by (K|M|G|T|P|E)i? + """ + negate = asbool(negate) + expected = "Expected" if not negate else "Did not expect" + if n is not None: + n_bytes = parse_bytesize(n) + delta_bytes = parse_bytesize(delta) + assert (not negate) == (abs(count - n_bytes) <= delta_bytes), n_text.format(expected=expected, n=n, delta=delta, text="{text}", output="{output}") + f" found {count}" + if min is not None or max is not None: + if min is None: + min = -inf # also replacing min/max for output + min_bytes = -inf + else: + min_bytes = parse_bytesize(min) + if max is None: + max = inf + max_bytes = inf + else: + max_bytes = parse_bytesize(max) + assert (not negate) == (min_bytes <= count <= max_bytes), min_max_text.format(expected=expected, min=min, max=max, text="{text}", output="{output}") + f" found {count}" + + +def _assert_presence_number(output, text, n, delta, min, max, negate, check_presence_foo, count_foo, presence_text, n_text, min_max_text): + """ + helper function to assert that + - text is present in output using check_presence_foo + this is done only if n, min, and max are None + - text appears a certain number of times, where the count is determined with count foo + + raising an assertion error using presence_text, n_text or min_max_text (resp) + substituting {n}, {delta}, {min}, {max}, {text}, and {output} + + n, delta, min, max can be suffixed by (K|M|G|T|P|E)i? + """ + negate = asbool(negate) + expected = "Expected" if not negate else "Did not expect" + if n is None and min is None and max is None: + assert (not negate) == check_presence_foo(output, text), presence_text.format(expected=expected, output=output, text=text) + try: + _assert_number(count_foo(output, text), n, delta, min, max, negate, n_text, min_max_text) + except AssertionError as e: + raise AssertionError(str(e).format(output=output, text=text)) diff --git a/lib/galaxy/tool_util/verify/asserts/archive.py b/lib/galaxy/tool_util/verify/asserts/archive.py index 840e93dfcb9..fd5caa58f77 100644 --- a/lib/galaxy/tool_util/verify/asserts/archive.py +++ b/lib/galaxy/tool_util/verify/asserts/archive.py @@ -1,37 +1,90 @@ import io import re import tarfile +import tempfile import zipfile - -def _extract_from_tar(tar_temp, path): - for fn in tar_temp.getnames(): - if re.match(path, fn): - # Will only match on first hit, probably fine for now - return tar_temp.extractfile(fn) +from galaxy.util import asbool +from ._util import _assert_presence_number -def _extract_from_zip(zip_temp, path): - for fn in zip_temp.namelist(): - if re.match(path, fn): - # Will only match on first hit, probably fine for now - return zip_temp.open(fn) +def _extract_from_tar(bytes, fn): + with io.BytesIO(bytes) as temp: + with tarfile.open(fileobj=temp, mode='r') as tar_temp: + ti = tar_temp.getmember(fn) + # zip treats directories like empty files. + # so make this consistent for tar + if ti.isdir(): + return "" + with tar_temp.extractfile(fn) as member_fh: + return member_fh.read() -def assert_has_archive_member(output_bytes, path, verify_assertions_function, children): +def _list_from_tar(bytes, path): + lst = list() + with io.BytesIO(bytes) as temp: + with tarfile.open(fileobj=temp, mode='r') as tar_temp: + for fn in tar_temp.getnames(): + if not re.match(path, fn): + continue + lst.append(fn) + return sorted(lst) + + +def _extract_from_zip(bytes, fn): + with io.BytesIO(bytes) as temp: + with zipfile.ZipFile(temp, mode='r') as zip_temp: + with zip_temp.open(fn) as member_fh: + return member_fh.read() + + +def _list_from_zip(bytes, path): + lst = list() + with io.BytesIO(bytes) as temp: + with zipfile.ZipFile(temp, mode='r') as zip_temp: + for fn in zip_temp.namelist(): + if not re.match(path, fn): + continue + lst.append(fn) + return sorted(lst) + + +def assert_has_archive_member(output_bytes, path, verify_assertions_function, children, all="false", n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Recursively checks the specified children assertions against the text of the first element matching the specified path found within the archive. Currently supported formats: .zip, .tar, .tar.gz.""" + all = asbool(all) + extract_foo = None + # from python 3.9 is_tarfile supports file like objects then we do not need + # the tempfile detour but can use io.BytesIO(output_bytes) + with tempfile.NamedTemporaryFile() as tmp: + tmp.write(output_bytes) + tmp.flush() + if zipfile.is_zipfile(tmp.name): + extract_foo = _extract_from_zip + list_foo = _list_from_zip + elif tarfile.is_tarfile(tmp.name): + extract_foo = _extract_from_tar + list_foo = _list_from_tar + assert extract_foo is not None, f"Expected path '{path}' to be an archive" - output_temp = io.BytesIO(output_bytes) - try: # tar / tar.gz - temp = tarfile.open(fileobj=output_temp, mode='r') - contents = _extract_from_tar(temp, path) - except tarfile.TarError: # zip - temp = zipfile.ZipFile(output_temp, mode='r') - contents = _extract_from_zip(temp, path) - finally: - verify_assertions_function(contents.read(), children) - contents.close() - temp.close() - output_temp.close() + # get list of matching file names in archive and check against n, delta, + # min, max (slightly abusing the output and text as well as the function + # parameters) + fns = list_foo(output_bytes, path) + _assert_presence_number(None, path, n, delta, min, max, negate, + lambda o, t: len(fns) > 0, + lambda o, t: len(fns), + "{expected} path '{text}' in archive", + "{expected} {n}+-{delta} matches for path '{text}' in archive", + "{expected} that the number of matches for path '{text}' in archive is in [{min}:{max}]") + + # check sub-assertions on members matching path + for fn in fns: + contents = extract_foo(output_bytes, fn) + try: + verify_assertions_function(contents, children) + except AssertionError as e: + raise AssertionError(f"Archive member '{path}': {str(e)}") + if not all: + break diff --git a/lib/galaxy/tool_util/verify/asserts/hdf5.py b/lib/galaxy/tool_util/verify/asserts/hdf5.py index 36f6814c174..e273aa1ae53 100644 --- a/lib/galaxy/tool_util/verify/asserts/hdf5.py +++ b/lib/galaxy/tool_util/verify/asserts/hdf5.py @@ -20,9 +20,10 @@ def assert_has_h5_attribute(output_bytes, key, value): output_temp = io.BytesIO(output_bytes) local_attrs = h5py.File(output_temp, 'r').attrs assert key in local_attrs and str(local_attrs[key]) == value, ( - f"Not a HDF5 file or H5 attributes do not match:\n\t{local_attrs.items()}\n\n\t({key} : {value})") + f"Not a HDF5 file or H5 attributes do not match:\n\t{list(local_attrs.items())}\n\n\t({key} : {value})") +# TODO the function actually queries groups. so the function and argument name are misleading def assert_has_h5_keys(output_bytes, keys): """ Asserts the specified HDF5 output has the given keys.""" _assert_h5py() diff --git a/lib/galaxy/tool_util/verify/asserts/size.py b/lib/galaxy/tool_util/verify/asserts/size.py index 2ad061247ab..2e16d041f37 100644 --- a/lib/galaxy/tool_util/verify/asserts/size.py +++ b/lib/galaxy/tool_util/verify/asserts/size.py @@ -1,4 +1,12 @@ -def assert_has_size(output_bytes, value, delta=0): - """Asserts the specified output has a size of the specified value""" +from ._util import _assert_number + + +def assert_has_size(output_bytes, value: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): + """ + Asserts the specified output has a size of the specified value, + allowing for absolute (delta) and relative (delta_frac) difference. + """ output_size = len(output_bytes) - assert abs(output_size - int(value)) <= int(delta), f"Expected file size was {value}, actual file size was {output_size} (difference of {delta} accepted)" + _assert_number(output_size, value, delta, min, max, negate, + "{expected} file size of {n}+-{delta}", + "{expected} file size to be in [{min}:{max}]") diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index bac40dce0b0..c16fe6f4732 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -1,19 +1,29 @@ import re +from ._util import _assert_number -def get_first_line(output): - match = re.search("^(.*)$", output, flags=re.MULTILINE) + +def get_first_line(output, comment): + """ + get the first non-comment and non-empty line + """ + if comment != "": + match = re.search(f"^([^{comment}].*)$", output, flags=re.MULTILINE) + else: + match = re.search("^(.+)$", output, flags=re.MULTILINE) if match is None: - return None + return "" else: return match.group(1) -def assert_has_n_columns(output, n, sep='\t'): +def assert_has_n_columns(output, n: int = None, delta: int = 0, min: int = None, max: int = None, sep='\t', comment="", negate: bool = False): """ Asserts the tabular output contains n columns. The optional sep argument specifies the column seperator used to determine the - number of columns.""" - n = int(n) - first_line = get_first_line(output) - assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n - assert len(first_line.split(sep)) == n, "Output does not have %d columns." % n + number of columns. The optional comment argument specifies + comment characters""" + first_line = get_first_line(output, comment) + n_columns = len(first_line.split(sep)) + _assert_number(n_columns, n, delta, min, max, negate, + "{expected} {n}+-{delta} columns in output", + "{expected} the number of columns in output to be in [{min}:{max}]") diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index 003fd48bde9..6215fa26bc5 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -1,54 +1,73 @@ import re +from ._util import _assert_number, _assert_presence_number -def assert_has_text(output, text, n=None): + +def assert_has_text(output, text, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts specified output contains the substring specified by the argument text. The exact number of occurrences can be - optionally specified by the argument n.""" + optionally specified by the argument n""" assert output is not None, "Checking has_text assertion on empty output (None)" - if n is None: - assert output.find(text) >= 0, f"Output file did not contain expected text '{text}' (output '{output}')" - else: - matches = re.findall(re.escape(text), output) - assert len(matches) == int(n), f"Expected {n} matches for '{text}' in output file (output '{output}'); found {len(matches)}" + _assert_presence_number(output, text, n, delta, min, max, negate, + lambda o, t: o.find(t) >= 0, + lambda o, t: len(re.findall(re.escape(t), o)), + "{expected} text '{text}' in output ('{output}')", + "{expected} {n}+-{delta} occurences of '{text}' in output ('{output}')", + "{expected} that the number of occurences of '{text}' in output is in [{min}:{max}] ('{output}')") def assert_not_has_text(output, text): """ Asserts specified output does not contain the substring - specified by the argument text.""" + specified by the argument text""" assert output is not None, "Checking not_has_text assertion on empty output (None)" assert output.find(text) < 0, f"Output file contains unexpected text '{text}'" -def assert_has_line(output, line, n=None): +def assert_has_line(output, line, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output contains the line specified by the argument line. The exact number of occurrences can be optionally - specified by the argument n.""" + specified by the argument n""" assert output is not None, "Checking has_line assertion on empty output (None)" - if n is None: - match = re.search(f"^{re.escape(line)}$", output, flags=re.MULTILINE) - assert match is not None, f"No line of output file was '{line}' (output was '{output}') " - else: - matches = re.findall(f"^{re.escape(line)}$", output, flags=re.MULTILINE) - assert len(matches) == int(n), f"Expected {n} lines matching '{line}' in output file (output was '{output}'); found {len(matches)}" + _assert_presence_number(output, line, n, delta, min, max, negate, + lambda o, l: re.search(f"^{re.escape(l)}$", o, flags=re.MULTILINE) is not None, + lambda o, l: len(re.findall(f"^{re.escape(l)}$", o, flags=re.MULTILINE)), + "{expected} line '{text}' in output ('{output}')", + "{expected} {n}+-{delta} lines '{text}' in output ('{output}')", + "{expected} that the number of lines '{text}' in output is in [{min}:{max}] ('{output}')") -def assert_has_n_lines(output, n): - """Asserts the specified output contains ``n`` lines.""" +def assert_has_n_lines(output, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): + """Asserts the specified output contains ``n`` lines allowing + for a difference in the number of lines (delta) + or relative differebce in the number of lines""" assert output is not None, "Checking has_n_lines assertion on empty output (None)" - n_lines_found = len(output.splitlines()) - assert n_lines_found == int(n), f"Expected {n} lines in output, found {n_lines_found} lines" + count = len(output.splitlines()) + _assert_number(count, n, delta, min, max, negate, + "{expected} {n}+-{delta} lines in the output", + "{expected} the number of line to be in [{min}:{max}]") -def assert_has_text_matching(output, expression): +def assert_has_text_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output contains text matching the - regular expression specified by the argument expression.""" - match = re.search(expression, output) - assert match is not None, f"No text matching expression '{expression}' was found in output file." + regular expression specified by the argument expression. + If n is given the assertion checks for exacly n (nonoverlapping) + occurences. + """ + _assert_presence_number(output, expression, n, delta, min, max, negate, + lambda o, e: re.search(e, o) is not None, + lambda o, e: len(re.findall(e, o)), + "{expected} text matching expression '{text}' in output ('{output}')", + "{expected} {n}+-{delta} (non-overlapping) matches for '{text}' in output ('{output}')", + "{expected} that the number of (non-overlapping) matches for '{text}' in output is in [{min}:{max}] ('{output}')") -def assert_has_line_matching(output, expression): +def assert_has_line_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output contains a line matching the - regular expression specified by the argument expression.""" - match = re.search(f"^{expression}$", output, flags=re.MULTILINE) - assert match is not None, f"No line matching expression '{expression}' was found in output file." + regular expression specified by the argument expression. If n is given + the assertion checks for exactly n occurences.""" + _assert_presence_number(output, expression, n, delta, min, max, negate, + lambda o, e: re.search(f"^{e}$", o, flags=re.MULTILINE) is not None, + lambda o, e: len(re.findall(f"^{e}$", o, flags=re.MULTILINE)), + "{expected} line matching expression '{text}' in output ('{output}')", + "{expected} {n}+-{delta} lines matching for '{text}' in output ('{output}')", + "{expected} that the number of lines matching for '{text}' in output is in [{min}:{max}] ('{output}')") diff --git a/lib/galaxy/tool_util/verify/asserts/xml.py b/lib/galaxy/tool_util/verify/asserts/xml.py index 7835a2dd29e..b8842384b4f 100644 --- a/lib/galaxy/tool_util/verify/asserts/xml.py +++ b/lib/galaxy/tool_util/verify/asserts/xml.py @@ -1,91 +1,105 @@ import re +from lxml.etree import XMLSyntaxError + +from galaxy.tool_util.verify import asserts from galaxy.util import ( + asbool, parse_xml_string, unicodify, ) -# Helper functions used to work with XML output. -def to_xml(output): - return parse_xml_string(output) - - -def xml_find_text(output, path): - xml = to_xml(output) - text = xml.findtext(path) - return text - - -def xml_find(output, path): - xml = to_xml(output) - return xml.find(path) - - def assert_is_valid_xml(output): """ Simple assertion that just verifies the specified output is valid XML.""" try: - to_xml(output) - except Exception as e: - # TODO: Narrow caught exception to just parsing failure + parse_xml_string(output) + except XMLSyntaxError as e: raise AssertionError(f"Expected valid XML, but could not parse output. {unicodify(e)}") -def assert_has_element_with_path(output, path): +def assert_has_element_with_path(output, path, negate: bool = False): """ Asserts the specified output has at least one XML element with a path matching the specified path argument. Valid paths are the simplified subsets of XPath implemented by lxml.etree; - http://effbot.org/zone/element-xpath.htm for more information.""" - if xml_find(output, path) is None: - errmsg = f"Expected to find XML element matching expression {path}, not such match was found." - raise AssertionError(errmsg) + https://lxml.de/xpathxslt.html for more information.""" + assert_xml_element(output, path, negate=negate) -def assert_has_n_elements_with_path(output, path, n): +def assert_has_n_elements_with_path(output, path, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output has exactly n elements matching the path specified.""" - xml = to_xml(output) - n = int(n) - num_elements = len(xml.findall(path)) - if num_elements != n: - errmsg = "Expected to find %d elements with path %s, but %d were found." % (n, path, num_elements) - raise AssertionError(errmsg) + assert_xml_element(output, path, n=n, delta=delta, min=min, max=max, negate=negate) -def assert_element_text_matches(output, path, expression): +def assert_element_text_matches(output, path, expression, negate: bool = False): """ Asserts the text of the first element matching the specified path matches the specified regular expression.""" - text = xml_find_text(output, path) - if re.match(expression, text) is None: - errmsg = f"Expected element with path '{path}' to contain text matching '{expression}', instead text '{text}' was found." - raise AssertionError(errmsg) + sub = {"tag": "has_text_matching", 'attributes': {'expression': expression, 'negate': negate}} + assert_xml_element(output, path, asserts.verify_assertions, [sub]) -def assert_element_text_is(output, path, text): +def assert_element_text_is(output, path, text, negate: bool = False): """ Asserts the text of the first element matching the specified path matches exactly the specified text. """ - assert_element_text_matches(output, path, re.escape(text)) + assert_element_text_matches(output, path, re.escape(text) + "$", negate=negate) -def assert_attribute_matches(output, path, attribute, expression): +def assert_attribute_matches(output, path, attribute, expression, negate: bool = False): """ Asserts the specified attribute of the first element matching the specified path matches the specified regular expression.""" - xml = xml_find(output, path) - attribute_value = xml.attrib[attribute] - if re.match(expression, attribute_value) is None: - errmsg = f"Expected attribute '{attribute}' on element with path '{path}' to match '{expression}', instead attribute value was '{attribute_value}'." - raise AssertionError(errmsg) + sub = {"tag": "has_text_matching", 'attributes': {'expression': expression, 'negate': negate}} + assert_xml_element(output, path, asserts.verify_assertions, [sub], attribute=attribute) -def assert_attribute_is(output, path, attribute, text): +def assert_attribute_is(output, path, attribute, text, negate: bool = False): """ Asserts the specified attribute of the first element matching the specified path matches exactly the specified text.""" - assert_attribute_matches(output, path, attribute, re.escape(text)) + assert_attribute_matches(output, path, attribute, re.escape(text) + "$", negate=negate) -def assert_element_text(output, path, verify_assertions_function, children): +def assert_element_text(output, path, verify_assertions_function, children, negate: bool = False): """ Recursively checks the specified assertions against the text of the first element matching the specified path.""" - text = xml_find_text(output, path) - verify_assertions_function(text, children) + assert_xml_element(output, path, verify_assertions_function, children, negate=negate) + + +def assert_xml_element(output, path, verify_assertions_function=None, children=None, attribute=None, all=False, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): + """ + Check if path occurs in the xml. If n and delta or min and max are given + also the number of occurences is checked. + If there are any sub assertions then check them against + - the element's text if attribute is None + - the content of the attribute + If all is True then the sub assertions are checked for all occurences. + """ + children = children or [] + all = asbool(all) + # assert that path is in output (the specified number of times) + + xml = parse_xml_string(output) + asserts._util._assert_presence_number(xml, path, n, delta, min, max, negate, + lambda x, p: x.find(p) is not None, + lambda x, p: len(x.findall(p)), + "{expected} path '{text}' in xml", + "{expected} {n}+-{delta} occurrences of path '{text}' in xml", + "{expected} that the number of occurences of path '{text}' in xml is in [{min}:{max}]") + + # check sub-assertions + if len(children) == 0 or verify_assertions_function is None: + return + for occ in xml.findall(path): + if attribute is None or attribute == "": + content = occ.text + else: + content = occ.attrib[attribute] + try: + verify_assertions_function(content, children) + except AssertionError as e: + if attribute is not None and attribute != "": + raise AssertionError(f"Attribute '{attribute}' on element with path '{path}': {str(e)}") + else: + raise AssertionError(f"Text of element with path '{path}': {str(e)}") + if not all: + break diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 7dd8dabe92b..c82d22c9ce0 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1778,7 +1778,6 @@ provides a demonstration of using this tag. ``` - ]]> @@ -2016,128 +2015,561 @@ module. ]]> - - - + + + + + + + + - + + + + - - - ``). If the ``text`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> - - - - - - ``).]]> - - - - - - `` ).]]> - - - - - - ``). If the ``line`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> - - - - - - ``.]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - `` ).]]> - - - - - - ``).]]> - - - - - ``).]]> - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - ``.]]> + ``. +Alternatively the range of the expected size can be specified by ``min`` and/or +``max``. + +$attribute_list::5 +]]> + - + - Desired size of the outpyt (in bytes) + Desired size of the output (in bytes), can be suffixed by ``(k|M|G|T|P|E)i?`` - + - Maximum allowed size difference (default is 0). + Maximum allowed size difference (default is 0). The observed size has to be in the range ``value +- delta``. Can be suffixed by ``(k|M|G|T|P|E)i?`` + + + + + Minimum expected size, can be suffixed by ``(k|M|G|T|P|E)i?`` + + + + + Maximum expected size, can be suffixed by ``(k|M|G|T|P|E)i?`` + + + + + + + ``). If the ``text`` is expected to occur a particular number of +times, this value can be specified using ``n``. Optionally also with a certain +``delta``. Alternatively the range of expected occurences can be specified by +``min`` and/or ``max``. + +$attribute_list::5 +]]> + + + + + Text to check for + + + + + + + + ``). + +$attribute_list::5 +]]> + + + + + Text to check for + + + `` ). If the +regular expression is expected to match a particular number of times, this value +can be specified using ``n``. Note only non-overlapping occurences are counted. +Optionally also with a certain ``delta``. Alternatively the range of expected +occurences can be specified by ``min`` and/or ``max``. + +$attribute_list::5 +]]> + + + + + Regular expression to check for + + + + + + + + ``). If the ``line`` is expected +to occur a particular number of times, this value can be specified using ``n``. +Optionally also with a certain ``delta``. Alternatively the range of expected +occurences can be specified by ``min`` and/or ``max``. + +$attribute_list::5 +]]> + + + + + The line to check for + + + + + + + + ``. +Alternatively the range of expected occurences can be specified by ``min`` +and/or ``max``. + +$attribute_list::5 +]]> + + + + + + + + ``). +If a particular number of matching lines is expected, this value can be +specified using ``n``. Optionally also with ``delta``. Alternatively the range +of expected occurences can be specified by ``min`` and/or ``max``. + +$attribute_list::5 +]]> + + + + + Regular expression to check for + + + + + + + + ``) optionally also with +``delta``. Alternatively the range of expected occurences can be specified by +``min`` and/or ``max``. Optionally a column separator (``sep``, default is +``\t``) `and comment character(s) can be specified (``comment``, default is +empty string), then the first non-comment line is used for determining the +number of columns. + +$attribute_list::5 +]]> + + + + + + + + +``` + +With ``n`` and ``delta`` (or ``min`` and ``max``) assertions on the number of +archive members matching ``path`` can be expressed. The following could be used, +e.g., to assert an archive containing n±1 elements out of which at least +4 need to have a ``txt`` extension. + +```xml + + +``` + +In addition the tag can contain additional assertions as child elements about +the first member in the archive matching the regular expression ``path``. For +instance + +```xml + + + +``` + +If the ``all`` attribute is set to ``true`` then all archive members are subject +to the assertions. Note that, archive members matching the ``path`` are sorted +alphabetically. + +The ``negate`` attribute of the ``has_archive_member`` assertion only affects +the asserts on the presence and number of matching archive members, but not any +sub-assertions (which can offer the ``negate`` attribute on their own). The +check if the file is an archive at all, which is also done by the function, is +not affected. + +$attribute_list::5 +]]> + + + + + + + + + + + The regular expression specifying the archive member. + + + + + Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first + + + + + + + ``). + +$attribute_list::5 +]]> + + + + + + + + + + +``` + +With ``negate="true"`` the outcome of the assertions wrt the precence and number +of ``path`` can be negated. If there are any sub assertions then check them against + +- the content of the attribute ``attribute`` +- the element's text if no attribute is given + +```xml + + + + + +``` + +Sub-assertions are not subject to the ``negate`` attribute of ``xml_element``. +If ``all`` is ``true`` then the sub assertions are checked for all occurences. + +Note that all other XML assertions can be expressed by this assertion (Galaxy +also implements the other assertions by calling this one). + +$attribute_list::5 +]]> + + + + + + Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first + + + + + The name of the attribute to apply sub-assertion on. If not given then the element text is used + + + + + + + + +``` + +With ``negate`` the result of the assertion can be inverted. + +$attribute_list::5 +]]> + + + + + + + + +``` + +Alternatively to ``n`` and ``delta`` also the ``min`` and ``max`` attributes +can be used to specify the range of the expected number of occurences. +With ``negate`` the result of the assertion can be inverted. + +$attribute_list::5 +]]> + + + + + + + + + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the matching) can be inverted (the +implicit assertion on the existence of the path is not affected). + +$attribute_list::5 +]]> + + + + + + The regular expression to use. + + + + + + + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the equality) can be inverted (the +implicit assertion on the existence of the path is not affected). + +$attribute_list::5 +]]> + + + + + + Text to check for. + + + + + + + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the matching) can be inverted (the +implicit assertion on the existence of the path is not affected). + +$attribute_list::5 +]]> + + + + + + The regular expression to use. + + + + + + + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the equality) can be inverted (the +implicit assertion on the existence of the path is not affected). + +$attribute_list::5 +]]> + + + + + + Text to check for. + + + + + + + + + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the implicit assertions can be inverted. +The sub-assertions, which have their own ``negate`` attribute, are not affected +by ``negate``. + +$attribute_list::5 +]]> + + + + + + + + - ``).]]> + +``` +$attribute_list::5 +]]> + @@ -2147,7 +2579,16 @@ module. - ``).]]> + +``` + +$attribute_list::5 +]]> + @@ -2160,19 +2601,46 @@ module. - - - ``). Valid archive formats include ``.zip``, ``.tar``, and ``.tar.gz``.]]> - - - - - + + + - The regular expression specifying the archive member. + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - + + + + + + Negate the outcome of the assertion. + + + + + + + + Desired number, can be suffixed by ``(k|M|G|T|P|E)i?`` + + + + + Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?`` + + + + + Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?`` + + + + + Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?`` + + + + Type of comparison to use when comparing test generated output files to expected output files. Currently valid value are -``diff`` (the default), ``re_match``, ``sim_size``, ``re_match_multiline``, -and ``contains``. +``diff`` (the default), ``re_match``, ``re_match_multiline``, +and ``contains``. In addtion there is ``sim_size`` which is discouraged in fafour of a ``has_size`` assertion. @@ -6313,6 +6781,14 @@ and ``contains``. + + + Number of bytes alowing for suffix (k|K|M|G|P|E)i? + + + + + """ +ASSERTS = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +""" + # tool xml for xml_order linter XML_ORDER = """ @@ -873,6 +911,20 @@ TESTS = [ '1 outputs found.' in x.info_messages and len(x.info_messages) == 1 and len(x.valid_messages) == 0 and len(x.warn_messages) == 0 and len(x.error_messages) == 0 ), + ( + ASSERTS, tests.lint_tsts, + lambda x: + 'Test 1: unknown assertion invalid' in x.error_messages + and 'Test 1: unknown attribute invalid_attrib for has_text' in x.error_messages + and 'Test 1: missing attribute text for has_text' in x.error_messages + and 'Test 1: attribute value for has_size needs to be int got 500k' in x.error_messages + and 'Test 1: attribute delta for has_size needs to be int got 1O' in x.error_messages + and 'Test 1: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages + and "Test 1: has_size needs to specify 'n', 'min', or 'max'" in x.error_messages + and "Test 1: has_n_columns needs to specify 'n', 'min', or 'max'" in x.error_messages + and "Test 1: has_n_lines needs to specify 'n', 'min', or 'max'" in x.error_messages + and len(x.warn_messages) == 0 and len(x.error_messages) == 9 + ), ( REPEATS, inputs.lint_repeats, lambda x: @@ -954,7 +1006,7 @@ TESTS = [ 'Unknown tag [wrong_tag] encountered, this may result in a warning in the future.' in x.info_messages and 'Best practice violation [stdio] elements should come before [command]' in x.warn_messages and len(x.info_messages) == 1 and len(x.valid_messages) == 0 and len(x.warn_messages) == 1 and len(x.error_messages) == 0 - ) + ), ] TEST_IDS = [ @@ -997,6 +1049,7 @@ TEST_IDS = [ 'outputs: format="input"', 'outputs collection static elements with format_source', 'outputs discover datatsets with tool provided metadata', + 'outputs: asserts', 'repeats', 'stdio: default for default profile', 'stdio: default for non-legacy profile', @@ -1007,7 +1060,7 @@ TEST_IDS = [ 'tests: without expectations', 'tests: param and output names', 'tests: expecting failure with outputs', - 'xml_order' + 'xml_order', ] diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py new file mode 100644 index 00000000000..6e0889b0758 --- /dev/null +++ b/test/unit/tool_util/verify/test_asserts.py @@ -0,0 +1,1012 @@ +import os +import shutil +import tempfile + +try: + import h5py +except ImportError: + h5py = None +import pytest + +from galaxy.tool_util.parser.xml import __parse_assert_list_from_elem +from galaxy.tool_util.verify import asserts +from galaxy.util import etree + +TABULAR_ASSERTION = """ + + + +""" +TABULAR_CSV_ASSERTION = """ + + + +""" +TABULAR_ASSERTION_COMMENT = """ + + + +""" + +TABULAR_DATA_POS = """1\t2\t3 +""" + +TABULAR_DATA_NEG = """1\t2\t3\t4 +""" + +TABULAR_CSV_DATA = """1,2 +""" + +TABULAR_DATA_COMMENT = """# comment +$ more comment (using a char with meaning wrt regexp) +1\t2\t3 +""" + +TEXT_HAS_TEXT_ASSERTION = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_N = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_N_DELTA = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_MIN_MAX = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_NEGATE = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_N_NEGATE = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE = """ + + + +""" + +TEXT_NOT_HAS_TEXT_ASSERTION = """ + + + +""" + +TEXT_HAS_TEXT_MATCHING_ASSERTION = """ + + + +""" + +TEXT_HAS_TEXT_MATCHING_ASSERTION_N = """ + + + +""" + +TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX = """ + + + +""" + +TEXT_HAS_LINE_ASSERTION = """ + + + +""" +TEXT_HAS_LINE_ASSERTION_N = """ + + + +""" +TEXT_HAS_N_LINES_ASSERTION = """ + + + +""" +TEXT_HAS_N_LINES_ASSERTION_DELTA = """ + + + +""" +TEXT_HAS_LINE_MATCHING_ASSERTION = """ + + + +""" +TEXT_HAS_LINE_MATCHING_ASSERTION_N = """ + + + +""" + +SIZE_HAS_SIZE_ASSERTION = """ + + + +""" +SIZE_HAS_SIZE_ASSERTION_DELTA = """ + + + +""" + +TEXT_DATA_HAS_TEXT = """test text +""" + +TEXT_DATA_HAS_TEXT_NEG = """desired content +is not here +""" + +TEXT_DATA_NONE = None + +TEXT_DATA_EMPTY = "" + + +XML_IS_VALID_XML_ASSERTION = """ + + + +""" +XML_HAS_ELEMENT_WITH_PATH = """ + + + +""" +XML_HAS_N_ELEMENTS_WITH_PATH = """ + + + + +""" +XML_ELEMENT_TEXT_MATCHES = """ + + + +""" +XML_ELEMENT_TEXT_IS = """ + + + +""" +XML_ATTRIBUTE_MATCHES = """ + + + + +""" +XML_ELEMENT_TEXT = """ + + + {content_assert} + + +""" + +XML_XML_ELEMENT = """ + + + {content_assert} + + +""" + +VALID_XML = ''' + + BAR + BAZ + QUX + + + +''' +INVALID_XML = '' + +if h5py is not None: + with tempfile.NamedTemporaryFile(delete=False) as tmp: + h5name = tmp.name + with h5py.File(tmp.name, "w") as h5fh: + h5fh.attrs['myfileattr'] = "myfileattrvalue" + h5fh.attrs['myfileattrint'] = 1 + dset = h5fh.create_dataset("myint", (100,), dtype='i') + dset.attrs['myintattr'] = "myintattrvalue" + grp = h5fh.create_group("mygroup") + grp.attrs['mygroupattr'] = "mygroupattrvalue" + grp.create_dataset("myfloat", (50,), dtype='f') + dset.attrs['myfloatattr'] = "myfloatattrvalue" + with open(h5name, "rb") as h5fh: + H5BYTES = h5fh.read() + os.remove(h5name) + + H5_HAS_H5_KEYS = """ + + + + """ + H5_HAS_H5_KEYS_NEGATIVE = """ + + + + """ + H5_HAS_ATTRIBUTE = """ + + + + + """ + H5_HAS_ATTRIBUTE_NEGATIVE = """ + + + + + """ + +# create a test directory structure for zipping +# might also be done directly with the fipfile/tarfile module without creating +# a tmpdir, but its much harder to create empty directories or symlinks +tmpdir = tempfile.mkdtemp() +for f in ["file1.txt", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: + tmpfile = os.path.join(tmpdir, f) + os.makedirs(os.path.dirname(tmpfile), exist_ok=True) + with open(tmpfile, "w") as fh: + fh.write(f) +os.makedirs(os.path.join(tmpdir, "emptydir")) +os.symlink("testdir/file1.txt", os.path.join(tmpdir, "symlink")) + +with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as ziptmp: + zipname = ziptmp.name + shutil.make_archive(zipname[:-4], "zip", tmpdir) + with open(zipname, "rb") as zfh: + ZIPBYTES = zfh.read() +with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as ziptmp: + zipname = ziptmp.name + shutil.make_archive(zipname[:-7], "gztar", tmpdir) + with open(zipname, "rb") as zfh: + TARBYTES = zfh.read() +shutil.rmtree(tmpdir) + + +with tempfile.NamedTemporaryFile(mode="w", delete=False) as nonarchivetmp: + nonarchivename = nonarchivetmp.name + nonarchivetmp.write("some text") +with open(nonarchivename, "rb") as ntmp: + NONARCHIVE = ntmp.read() + +ARCHIVE_HAS_ARCHIVE_MEMBER = """ + + + {content_assert} + + +""" + +ARCHIVE_HAS_ARCHIVE_MEMBER_N = """ + + + {content_assert} + + +""" + +ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX = """ + + + {content_assert} + + +""" + +TESTS = [ + # test successful assertion + ( + TABULAR_ASSERTION, TABULAR_DATA_POS, + lambda x: len(x) == 0 + ), + # test wrong number of columns + ( + TABULAR_ASSERTION, TABULAR_DATA_NEG, + lambda x: 'Expected 3+-0 columns in output found 4' in x + ), + # test wrong number of columns for csv data + ( + TABULAR_CSV_ASSERTION, TABULAR_CSV_DATA, + lambda x: 'Expected the number of columns in output to be in [3:inf] found 2' in x + ), + # test tabular data with comments + ( + TABULAR_ASSERTION_COMMENT, TABULAR_DATA_COMMENT, + lambda x: len(x) == 0 + ), + # test has_text + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text .. negative test + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "Expected text 'test text' in output ('desired content\nis not here\n')" in x + ), + # test has_text with None output + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_NONE, + lambda x: "Checking has_text assertion on empty output (None)" in x + ), + # test has_text with empty output + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_EMPTY, + lambda x: "Expected text 'test text' in output ('')" in x + ), + # test has_text with n + ( + TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_text with n .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2+-0 occurences of 'test text' in output ('test text\n') found 1" in x + ), + # test has_text with n and delta + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_text with n and delta .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 3+-1 occurences of 'test text' in output ('test text\n') found 1" in x + ), + # test has_text with min max + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX, TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_text with min max .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX, TEXT_DATA_HAS_TEXT, + lambda x: "Expected that the number of occurences of 'test text' in output is in [2:4] ('test text\n') found 1" in x + ), + + + # test has_text negate + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_HAS_TEXT_NEG, + lambda x: len(x) == 0 + ), + # test has_text negate .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: "Did not expect text 'test text' in output ('test text\n')" in x + ), + # test has_text negate with None output .. should have the same output as with negate="false" + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_NONE, + lambda x: "Checking has_text assertion on empty output (None)" in x + ), + # test has_text negate with empty output + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_EMPTY, + lambda x: len(x) == 0 + ), + # test has_text negate with n + ( + TEXT_HAS_TEXT_ASSERTION_N_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text negate with n .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N_NEGATE, TEXT_DATA_HAS_TEXT * 2, + lambda x: "Did not expect 2+-0 occurences of 'test text' in output ('test text\ntest text\n') found 2" in x + ), + # test has_text negate with n and delta + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text negate with n and delta .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE, TEXT_DATA_HAS_TEXT * 2, + lambda x: "Did not expect 3+-1 occurences of 'test text' in output ('test text\ntest text\n') found 2" in x + ), + # test has_text negate with min max + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text negate with min max .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE, TEXT_DATA_HAS_TEXT * 2, + lambda x: "Did not expect that the number of occurences of 'test text' in output is in [2:4] ('test text\ntest text\n') found 2" in x + ), + + + # test not_has_text + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test not_has_text .. negative test + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "Output file contains unexpected text 'not here'" in x + ), + # test not_has_text with None output + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_NONE, + lambda x: "Checking not_has_text assertion on empty output (None)" in x + ), + # test not_has_text with empty output + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_EMPTY, + lambda x: len(x) == 0 + ), + # test has_text_matching + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text_matching .. negative test + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "Expected text matching expression 'te[sx]t' in output ('desired content\nis not here\n')" in x + ), + # test has_text_matching with n + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 4+-0 (non-overlapping) matches for 'te[sx]t' in output ('test text\n') found 2" in x + ), + # test has_text_matching with n + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX, TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX, TEXT_DATA_HAS_TEXT, + lambda x: "Expected that the number of (non-overlapping) matches for 'te[sx]t' in output is in [3:5] ('test text\n') found 2" in x + ), + # test has_line + ( + TEXT_HAS_LINE_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_line .. negative test + ( + TEXT_HAS_LINE_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "Expected line 'test text' in output ('desired content\nis not here\n')" in x + ), + # test has_line with n + ( + TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_line with n .. negative test + ( + TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2+-0 lines 'test text' in output ('test text\n') found 1" in x + ), + # test has_n_lines + ( + TEXT_HAS_N_LINES_ASSERTION.format(n="2"), TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_n_lines .. bytes + ( + TEXT_HAS_N_LINES_ASSERTION.format(n="2ki"), TEXT_DATA_HAS_TEXT * 2048, + lambda x: len(x) == 0 + ), + # test has_n_lines .. negative test + ( + TEXT_HAS_N_LINES_ASSERTION.format(n="2"), TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2+-0 lines in the output found 1" in x + ), + # test has_n_lines ..delta + ( + TEXT_HAS_N_LINES_ASSERTION_DELTA.format(n="3", delta="1"), TEXT_DATA_HAS_TEXT, + lambda x: "Expected 3+-1 lines in the output found 1" in x + ), + # test has_line_matching + ( + TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_line_matching .. negative test + ( + TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "Expected line matching expression 'te[sx]t te[sx]t' in output ('desired content\nis not here\n')" in x + ), + # test has_line_matching n + ( + TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_line_matching n .. negative test + ( + TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2+-0 lines matching for 'te[sx]t te[sx]t' in output ('test text\n') found 1" in x + ), + # test has_size + ( + SIZE_HAS_SIZE_ASSERTION.format(value=10), TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_size .. negative test + ( + SIZE_HAS_SIZE_ASSERTION.format(value="10"), TEXT_DATA_HAS_TEXT * 2, + lambda x: "Expected file size of 10+-0 found 20" in x + ), + # test has_size .. delta + ( + SIZE_HAS_SIZE_ASSERTION_DELTA.format(value="10", delta="10"), TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_size .. bytes suffix + ( + SIZE_HAS_SIZE_ASSERTION_DELTA.format(value="1k", delta="0"), TEXT_DATA_HAS_TEXT * 100, + lambda x: len(x) == 0 + ), + # test has_size .. bytes suffix .. negative + ( + SIZE_HAS_SIZE_ASSERTION_DELTA.format(value="1Mi", delta="10k"), TEXT_DATA_HAS_TEXT * 100, + lambda x: 'Expected file size of 1Mi+-10k found 1000' in x + ), + # test is_valid_xml + ( + XML_IS_VALID_XML_ASSERTION, VALID_XML, + lambda x: len(x) == 0 + ), + # test is_valid_xml .. negative test + ( + XML_IS_VALID_XML_ASSERTION, INVALID_XML, + lambda x: 'Expected valid XML, but could not parse output. Opening and ending tag mismatch: elem line 1 and root, line 1, column 31 (, line 1)' in x + ), + # test has_element_with_path + ( + XML_HAS_ELEMENT_WITH_PATH.format(path="./elem[1]/more"), VALID_XML, + lambda x: len(x) == 0 + ), + ( + XML_HAS_ELEMENT_WITH_PATH.format(path="./elem[@name='foo']"), VALID_XML, + lambda x: len(x) == 0 + ), + ( + XML_HAS_ELEMENT_WITH_PATH.format(path=".//more[@name]"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_element_with_path .. negative test + ( + XML_HAS_ELEMENT_WITH_PATH.format(path="./blah"), VALID_XML, + lambda x: "Expected path './blah' in xml" in x + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem", n="2"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem[1]/more", n="3"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem[@name='foo']/more", n="3"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem[2]/more", n="0"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path .. negative test + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem", n="1"), VALID_XML, + lambda x: "Expected 1+-0 occurrences of path './elem' in xml found 2" in x + ), + # test element_text_matches + ( + XML_ELEMENT_TEXT_MATCHES.format(path="./elem/more", expression="BA(R|Z)"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_matches more specific path + ( + XML_ELEMENT_TEXT_MATCHES.format(path="./elem/more[2]", expression="BA(R|Z)"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_matches .. negative test + ( + XML_ELEMENT_TEXT_MATCHES.format(path="./elem/more", expression="QU(X|Y)"), VALID_XML, + lambda x: "Text of element with path './elem/more': Expected text matching expression 'QU(X|Y)' in output ('BAR')" in x + ), + # test element_text_is + ( + XML_ELEMENT_TEXT_IS.format(path="./elem/more", text="BAR"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_is with more specific path + ( + XML_ELEMENT_TEXT_IS.format(path="./elem/more[@name='baz']", text="BAZ"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_is .. negative test testing that prefix is not accepted + ( + XML_ELEMENT_TEXT_IS.format(path="./elem/more", text="BA"), VALID_XML, + lambda x: "Text of element with path './elem/more': Expected text matching expression 'BA$' in output ('BAR')" in x + ), + # test element_attribute_matches + ( + XML_ATTRIBUTE_MATCHES.format(path="./elem/more", attribute="name", expression="ba(r|z)"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_attribute_matches with more specific path + ( + XML_ATTRIBUTE_MATCHES.format(path="./elem/more[2]", attribute="name", expression="ba(r|z)"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_attribute_matches .. negative test + ( + XML_ATTRIBUTE_MATCHES.format(path="./elem/more", attribute="name", expression="qu(x|y)"), VALID_XML, + lambda x: "Attribute 'name' on element with path './elem/more': Expected text matching expression 'qu(x|y)' in output ('bar')" in x + ), + # test element_text + ( + XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text .. negative + ( + XML_ELEMENT_TEXT.format(path="./absent", content_assert=''), VALID_XML, + lambda x: "Expected path './absent' in xml" in x + ), + # test element_text with sub-assertion + ( + XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text with sub-assertion .. negative + ( + XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, + lambda x: "Text of element with path './elem/more': Expected text 'NOTBAR' in output ('BAR')" in x + ), + # note that xml_element is also tested indirectly by the other xml + # assertions which are all implemented by xml_element + # test xml_element + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="1", min="1", max="3", attribute="", all="false", content_assert='', negate="false"), VALID_XML, + lambda x: len(x) == 0 + ), + # test xml_element testing attribute matching on all matching elements + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="1", min="1", max="3", attribute="name", all="true", content_assert='', negate="false"), VALID_XML, + lambda x: len(x) == 0 + ), + # test xml_element .. failing because of n + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="0", min="1", max="3", attribute="", all="false", content_assert='', negate="false"), VALID_XML, + lambda x: "Expected 2+-0 occurrences of path './/more' in xml found 3" in x + ), + # test xml_element .. failing because of n + ( + XML_XML_ELEMENT.format(path=".//more", n="10000", delta="1", min="1", max="3", attribute="", all="false", content_assert='', negate="true"), VALID_XML, + lambda x: "Did not expect that the number of occurences of path './/more' in xml is in [1:3] found 3" in x + ), + # test xml_element .. failing because of sub assertion + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="1", min="1", max="3", attribute="", all="false", content_assert='', negate="false"), VALID_XML, + lambda x: "Text of element with path './/more': Did not expect text matching expression '(BA[RZ]|QUX)$' in output ('BAR')" in x + ), + + # test has_archive_member with zip + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert="", all="false"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert="", all="false"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with non archive + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="irrelevant", content_assert="", all="false"), NONARCHIVE, + lambda x: "Expected path 'irrelevant' to be an archive" in x + ), + # test has_archive_member with zip on absent member + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert="", all="false"), ZIPBYTES, + lambda x: "Expected path 'absent' in archive" in x + ), + # test has_archive_member with tar on absent member + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert="", all="false"), TARBYTES, + lambda x: "Expected path 'absent' in archive" in x + ), + # test has_archive_member with zip on symlink + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?symlink", content_assert='', all="false"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar on symlink + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?symlink", content_assert='', all="false"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip on a dir member (which are treated like empty files) + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/", content_assert='', all="false"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar on a dir member (which are treated like empty files) + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/", content_assert='', all="false"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip with subassertion (note that archive members are sorted therefor file1 in dir2 is tested) + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/.*\\.txt", content_assert='', all="false"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar with subassertion (note that archive members are sorted therefor file1 in dir2 is tested) + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/.*\\.txt", content_assert='', all="false"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert='', all="false"), ZIPBYTES, + lambda x: "Archive member '(\\./)?testdir/file1.txt': Expected text 'ABSENT' in output ('testdir/file1.txt')" in x + ), + # test has_archive_member with tar with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert='', all="false"), TARBYTES, + lambda x: "Archive member '(\\./)?testdir/file1.txt': Expected text 'ABSENT' in output ('testdir/file1.txt')" in x + ), + # test has_archive_member with zip checking all matches with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar checking all matches with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip checking all matches with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), ZIPBYTES, + lambda x: "Expected text matching expression 'file1\\.txt' in output ('testdir/file2.txt')" + ), + # test has_archive_member with tar checking all matches with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), TARBYTES, + lambda x: "Expected text matching expression 'file1\\.txt' in output ('testdir/file2.txt')" + ), + + # test has_archive_member with zip n+delta with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="3", delta="1"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip n+delta with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="1", delta="1"), ZIPBYTES, + lambda x: "Expected 1+-1 matches for path '.*file.\\.txt' in archive found 4" in x + ), + # test has_archive_member with tar n+delta with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="3", delta="1"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar n+delta with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="1", delta="1"), TARBYTES, + lambda x: "Expected 1+-1 matches for path '.*file.\\.txt' in archive found 4" in x + ), + # test has_archive_member with zip min+max with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="2", max="4"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip min+max with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="0", max="2"), ZIPBYTES, + lambda x: "Expected that the number of matches for path '.*file.\\.txt' in archive is in [0:2] found 4" in x + ), + # test has_archive_member with tar min+max with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="2", max="4"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar min+max with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="0", max="2"), TARBYTES, + lambda x: "Expected that the number of matches for path '.*file.\\.txt' in archive is in [0:2] found 4" in x + ), +] + +if h5py is not None: + H5PY_TESTS = [ + # test has_h5_keys + ( + H5_HAS_H5_KEYS, H5BYTES, + lambda x: len(x) == 0 + ), + # test has_h5_keys .. negative + ( + H5_HAS_H5_KEYS_NEGATIVE, H5BYTES, + lambda x: "Not a HDF5 file or H5 keys missing:\n\t['mygroup', 'mygroup/myfloat', 'myint']\n\t['absent']" in x + ), + # test has_attribute + ( + H5_HAS_ATTRIBUTE, H5BYTES, + lambda x: len(x) == 0 + ), + # test has_attribute .. negative + ( + H5_HAS_ATTRIBUTE_NEGATIVE, H5BYTES, + lambda x: "Not a HDF5 file or H5 attributes do not match:\n\t[('myfileattr', 'myfileattrvalue'), ('myfileattrint', 1)]\n\n\t(myfileattr : wrong)" in x + ), + ] + TESTS.extend(H5PY_TESTS) + + +TEST_IDS = [ + 'has_n_columns success', + 'has_n_columns failure', + 'has_n_columns for csv', + 'has_n_columns with comments', + 'has_text success', + 'has_text failure', + 'has_text None output', + 'has_text empty output', + 'has_text n success', + 'has_text n failure', + 'has_text n delta success', + 'has_text n delta failure', + 'has_text min/max delta success', + 'has_text min/max delta failure', + 'has_text negate success', + 'has_text negate failure', + 'has_text negate None output', + 'has_text negate empty output', + 'has_text negate n success', + 'has_text negate n failure', + 'has_text negate n delta success', + 'has_text negate n delta failure', + 'has_text negate min/max delta success', + 'has_text negate min/max delta failure', + 'not_has_text success', + 'not_has_text failure', + 'not_has_text None output', + 'not_has_text empty output', + 'has_text_matching success', + 'has_text_matching failure', + 'has_text_matching n success', + 'has_text_matching n failure', + 'has_text_matching min/max success', + 'has_text_matching min/max failure', + 'has_line success', + 'has_line failure', + 'has_line n success', + 'has_line n failure', + 'has_n_lines success', + 'has_n_lines n as bytes success', + 'has_n_lines failure', + 'has_n_lines delta', + 'has_line_matching success', + 'has_line_matching failure', + 'has_line_matching n success', + 'has_line_matching n failure', + 'has_size success', + 'has_size failure', + 'has_size delta', + 'has_size with bytes suffix', + 'has_size with bytes suffix failure', + 'is_valid_xml success', + 'is_valid_xml failure', + 'has_element_with_path success 1', + 'has_element_with_path success 2', + 'has_element_with_path success 3', + 'has_element_with_path failure', + 'has_n_elements_with_path success 1', + 'has_n_elements_with_path success 2', + 'has_n_elements_with_path success 3', + 'has_n_elements_with_path success 4', + 'has_n_elements_with_path failure', + 'element_text_matches sucess', + 'element_text_matches sucess (with more specific path)', + 'element_text_matches failure', + 'element_text_is sucess', + 'element_text_is sucess (with more specific path)', + 'element_text_is failure', + 'attribute_matches sucess', + 'attribute_matches sucess (with more specific path)', + 'attribute_matches failure', + 'element_text success', + 'element_text failure', + 'element_text with subassertion sucess', + 'element_text with subassertion failure', + 'xml_element matching text success', + 'xml_element matching attribute success', + 'xml_element failure (due to n)', + 'xml_element failure (due to min/max in combination with negate)', + 'xml_element failure (due to subassertion)', + 'has_archive_member zip', + 'has_archive_member tar', + 'has_archive_member non-archive', + 'has_archive_member zip absent member', + 'has_archive_member tar absent member', + 'has_archive_member zip symlink member', + 'has_archive_member tar symlink member', + 'has_archive_member zip non-file member', + 'has_archive_member tar non-file member', + 'has_archive_member zip with content assertion', + 'has_archive_member tar with content assertion', + 'has_archive_member zip with failing content assertion', + 'has_archive_member tar with failing content assertion', + 'has_archive_member zip all matching with content assertion', + 'has_archive_member tar all matching with content assertion', + 'has_archive_member zip all matching with failing content assertion', + 'has_archive_member tar all matching with failing content assertion', + 'has_archive_member zip n + delta and content assertion', + 'has_archive_member zip n + delta failing and content assertion', + 'has_archive_member tar n + delta and content assertion', + 'has_archive_member tar n + delta failing and content assertion', + 'has_archive_member zip min max and content assertion', + 'has_archive_member zip min max failing and content assertion', + 'has_archive_member tar min max and content assertion', + 'has_archive_member tar min max failing and content assertion', +] + +if h5py is not None: + H5PY_TEST_IDS = [ + 'has_h5_keys', + 'has_h5_keys failure', + 'has_h5_attribute', + 'has_h5_attribute failure', + ] + TEST_IDS.extend(H5PY_TEST_IDS) + + +@pytest.mark.parametrize('assertion_xml,data,assert_func', TESTS, ids=TEST_IDS) +def test_assertions(assertion_xml, data, assert_func): + assertion = etree.fromstring(assertion_xml) + assertion_description = __parse_assert_list_from_elem(assertion) + try: + asserts.verify_assertions(data, assertion_description) + except AssertionError as e: + assert_list = e.args + else: + assert_list = () + assert assert_func(assert_list), assert_list