From 669a79b6bc8492065f68fb91305de67a2a2b9cd2 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 12:30:09 +0200 Subject: [PATCH 01/41] make has_n_columns check only non-empty and non-comment lines wanted to use a has_n_columns assertion for this file: https://github.com/galaxyproject/tools-iuc/blob/master/tools/stacks2/test-data/populations/populations.sumstats.tsv this file might be problematic since Galaxy's tabular data type uses the first comment line for column definitions Also makes the assert text similar to the has_n_lines text --- lib/galaxy/tool_util/verify/asserts/tabular.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index bac40dce0b0..cc41664d431 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -2,7 +2,10 @@ import re def get_first_line(output): - match = re.search("^(.*)$", output, flags=re.MULTILINE) + """ + get the first non-comment and non-empty line + """ + match = re.search("^([^#\n].*)$", output, flags=re.MULTILINE) if match is None: return None else: @@ -16,4 +19,4 @@ def assert_has_n_columns(output, n, sep='\t'): n = int(n) first_line = get_first_line(output) assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n - assert len(first_line.split(sep)) == n, "Output does not have %d columns." % n + assert len(first_line.split(sep)) == n, f"Expected {n} columns in output, found {first_line.split(sep)} lines" \ No newline at end of file From 39589062258fb4ded60c628a88b426454dd4b30f Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 13:04:28 +0200 Subject: [PATCH 02/41] fix linter problem --- lib/galaxy/tool_util/verify/asserts/tabular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index cc41664d431..929053f24ef 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -19,4 +19,4 @@ def assert_has_n_columns(output, n, sep='\t'): n = int(n) first_line = get_first_line(output) assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n - assert len(first_line.split(sep)) == n, f"Expected {n} columns in output, found {first_line.split(sep)} lines" \ No newline at end of file + assert len(first_line.split(sep)) == n, f"Expected {n} columns in output, found {first_line.split(sep)} lines" From 9019a9f1bb47684678e3bf115d6ce252998befd1 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 14:19:23 +0200 Subject: [PATCH 03/41] fix assertion text --- lib/galaxy/tool_util/verify/asserts/tabular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index 929053f24ef..b61bf9447b2 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -19,4 +19,4 @@ def assert_has_n_columns(output, n, sep='\t'): n = int(n) first_line = get_first_line(output) assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n - assert len(first_line.split(sep)) == n, f"Expected {n} columns in output, found {first_line.split(sep)} lines" + assert len(first_line.split(sep)) == n, f"Expected {n} columns in output, found {len(first_line.split(sep))} columns" From 506501197ff0ca545af1783b0f1b39942a6e4749 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 14:19:46 +0200 Subject: [PATCH 04/41] add unit tests for tabular assertions --- test/unit/tool_util/verify/test_asserts.py | 68 ++++++++++++++++++++++ 1 file changed, 68 insertions(+) create mode 100644 test/unit/tool_util/verify/test_asserts.py diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py new file mode 100644 index 00000000000..9e1808f0ac5 --- /dev/null +++ b/test/unit/tool_util/verify/test_asserts.py @@ -0,0 +1,68 @@ +import pytest + +from galaxy.util import etree +from galaxy.tool_util.verify import asserts +from galaxy.tool_util.parser.xml import __parse_assert_list_from_elem + +# def test_foo() + +TABULAR_ASSERTION = """ + + + +""" +TABULAR_CSV_ASSERTION = """ + + + +""" + +TABULAR_DATA_POS = """ +# comment line +1\t2\t3 +""" + +TABULAR_DATA_NEG = """ +# comment line +1\t2\t3\t4 +""" + +TABULAR_CSV_DATA = """ +1,2 +""" + +TESTS = [ + # test successful assertion + ( + TABULAR_ASSERTION, TABULAR_DATA_POS, + lambda x: len(x) == 0 + ), + # test wrong number of columns + ( + TABULAR_ASSERTION, TABULAR_DATA_NEG, + lambda x: 'Expected 3 columns in output, found 4 columns' in x + ), + # test wrong number of columns for csv data + ( + TABULAR_CSV_ASSERTION, TABULAR_CSV_DATA, + lambda x: 'Expected 3 columns in output, found 2 columns' in x + ), +] + +TEST_IDS = [ + 'tabular assertion success', + 'tabular assertion failure', + 'tabular csv assertion', +] + +@pytest.mark.parametrize('assertion_xml,data,assert_func', TESTS, ids=TEST_IDS) +def test_assertions(assertion_xml, data, assert_func): + assertion = etree.fromstring(assertion_xml) + assertion_description = __parse_assert_list_from_elem(assertion) + try: + asserts.verify_assertions(data, assertion_description) + except AssertionError as e: + assert_list = e.args + else: + assert_list = [] + assert assert_func(assert_list) From 4d72a122df28bd31ab77396e99335c74f525ce7b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 14:21:55 +0200 Subject: [PATCH 05/41] document sep attribute of has_n_columns --- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 0ffb5a25046..6c96cfdeec4 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2054,7 +2054,7 @@ module. - ``).]]> + ``). ``sep`` defaults to ``"\t"``.]]> From b2b46140f359d98c5bca85167e5f484b51a9e827 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 14:34:45 +0200 Subject: [PATCH 06/41] fixes --- lib/galaxy/tool_util/verify/asserts/tabular.py | 3 ++- test/unit/tool_util/verify/test_asserts.py | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index b61bf9447b2..f887f28437e 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -19,4 +19,5 @@ def assert_has_n_columns(output, n, sep='\t'): n = int(n) first_line = get_first_line(output) assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n - assert len(first_line.split(sep)) == n, f"Expected {n} columns in output, found {len(first_line.split(sep))} columns" + n_columns = len(first_line.split(sep)) + assert n_columns == n, f"Expected {n} columns in output, found {n_columns} columns" diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index 9e1808f0ac5..91eead1daf9 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -1,8 +1,8 @@ import pytest -from galaxy.util import etree -from galaxy.tool_util.verify import asserts from galaxy.tool_util.parser.xml import __parse_assert_list_from_elem +from galaxy.tool_util.verify import asserts +from galaxy.util import etree # def test_foo() @@ -18,12 +18,12 @@ TABULAR_CSV_ASSERTION = """ """ TABULAR_DATA_POS = """ -# comment line +# comment line 1\t2\t3 """ TABULAR_DATA_NEG = """ -# comment line +# comment line 1\t2\t3\t4 """ @@ -34,17 +34,17 @@ TABULAR_CSV_DATA = """ TESTS = [ # test successful assertion ( - TABULAR_ASSERTION, TABULAR_DATA_POS, + TABULAR_ASSERTION, TABULAR_DATA_POS, lambda x: len(x) == 0 ), # test wrong number of columns ( - TABULAR_ASSERTION, TABULAR_DATA_NEG, + TABULAR_ASSERTION, TABULAR_DATA_NEG, lambda x: 'Expected 3 columns in output, found 4 columns' in x ), # test wrong number of columns for csv data ( - TABULAR_CSV_ASSERTION, TABULAR_CSV_DATA, + TABULAR_CSV_ASSERTION, TABULAR_CSV_DATA, lambda x: 'Expected 3 columns in output, found 2 columns' in x ), ] @@ -55,6 +55,7 @@ TEST_IDS = [ 'tabular csv assertion', ] + @pytest.mark.parametrize('assertion_xml,data,assert_func', TESTS, ids=TEST_IDS) def test_assertions(assertion_xml, data, assert_func): assertion = etree.fromstring(assertion_xml) From 8c3052332f83f8231ad0a1b852c4897edc38c882 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 15:03:48 +0200 Subject: [PATCH 07/41] allow to specify comment char for has_n_columns --- .../tool_util/verify/asserts/tabular.py | 11 +++--- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- test/unit/tool_util/verify/test_asserts.py | 34 ++++++++++++------- 3 files changed, 30 insertions(+), 17 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index f887f28437e..27ce3782d83 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -1,23 +1,26 @@ import re -def get_first_line(output): +def get_first_line(output, comment): """ get the first non-comment and non-empty line """ - match = re.search("^([^#\n].*)$", output, flags=re.MULTILINE) + if comment != "": + match = re.search(f"^([^{comment}].*)$", output, flags=re.MULTILINE) + else: + match = re.search("^(.+)$", output, flags=re.MULTILINE) if match is None: return None else: return match.group(1) -def assert_has_n_columns(output, n, sep='\t'): +def assert_has_n_columns(output, n, sep='\t', comment=""): """ Asserts the tabular output contains n columns. The optional sep argument specifies the column seperator used to determine the number of columns.""" n = int(n) - first_line = get_first_line(output) + first_line = get_first_line(output, comment) assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n n_columns = len(first_line.split(sep)) assert n_columns == n, f"Expected {n} columns in output, found {n_columns} columns" diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 6c96cfdeec4..09a1d493114 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2054,7 +2054,7 @@ module. - ``). ``sep`` defaults to ``"\t"``.]]> + ``). ``sep`` defaults to ``"\t"``. The comment character(s) can be specified with the ``comment`` (default: empty string).]]> diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index 91eead1daf9..d8415b0e961 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -4,8 +4,6 @@ from galaxy.tool_util.parser.xml import __parse_assert_list_from_elem from galaxy.tool_util.verify import asserts from galaxy.util import etree -# def test_foo() - TABULAR_ASSERTION = """ @@ -16,20 +14,26 @@ TABULAR_CSV_ASSERTION = """ """ +TABULAR_ASSERTION_COMMENT = """ + + + +""" -TABULAR_DATA_POS = """ -# comment line +TABULAR_DATA_POS = """1\t2\t3 +""" + +TABULAR_DATA_NEG = """1\t2\t3\t4 +""" + +TABULAR_CSV_DATA = """1,2 +""" + +TABULAR_DATA_COMMENT = """# comment +$ more comment (using a char with meaning wrt regexp) 1\t2\t3 """ -TABULAR_DATA_NEG = """ -# comment line -1\t2\t3\t4 -""" - -TABULAR_CSV_DATA = """ -1,2 -""" TESTS = [ # test successful assertion @@ -47,12 +51,18 @@ TESTS = [ TABULAR_CSV_ASSERTION, TABULAR_CSV_DATA, lambda x: 'Expected 3 columns in output, found 2 columns' in x ), + # test tabular data with comments + ( + TABULAR_ASSERTION_COMMENT, TABULAR_DATA_COMMENT, + lambda x: len(x) == 0 + ), ] TEST_IDS = [ 'tabular assertion success', 'tabular assertion failure', 'tabular csv assertion', + 'tabular with comments assertion', ] From 06937c7a90e00f44c728f338d3bb607ec592b88e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 15:30:01 +0200 Subject: [PATCH 08/41] add AssertHasNColumns type in xsd --- lib/galaxy/tool_util/xsd/galaxy.xsd | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 09a1d493114..be8f88c8511 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2052,11 +2052,7 @@ module. - - - ``). ``sep`` defaults to ``"\t"``. The comment character(s) can be specified with the ``comment`` (default: empty string).]]> - - + @@ -2114,6 +2110,27 @@ module. + + + + ``). Optionally a column separator (``sep``, default is ``\t``) `and comment character(s) can be specified (``comment``, default is empty string).]]> + + + + Desired number of columns + + + + + Column separator + + + + + Comment character(s) + + + ``.]]> From d8cc7e3776ad78db647f45a8e6bd8a191bc424fc Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Sep 2021 17:46:03 +0200 Subject: [PATCH 09/41] add unit tests for text asserts and proper xsd --- .../tool_util/verify/asserts/tabular.py | 3 +- lib/galaxy/tool_util/verify/asserts/text.py | 2 +- lib/galaxy/tool_util/xsd/galaxy.xsd | 106 +++++++--- test/unit/tool_util/verify/test_asserts.py | 186 +++++++++++++++++- 4 files changed, 261 insertions(+), 36 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index 27ce3782d83..a9b2358a03f 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -18,7 +18,8 @@ def get_first_line(output, comment): def assert_has_n_columns(output, n, sep='\t', comment=""): """ Asserts the tabular output contains n columns. The optional sep argument specifies the column seperator used to determine the - number of columns.""" + number of columns. The optional comment argument specifies + comment characters.""" n = int(n) first_line = get_first_line(output, comment) assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index 003fd48bde9..5dfb7b52afc 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -27,7 +27,7 @@ def assert_has_line(output, line, n=None): assert output is not None, "Checking has_line assertion on empty output (None)" if n is None: match = re.search(f"^{re.escape(line)}$", output, flags=re.MULTILINE) - assert match is not None, f"No line of output file was '{line}' (output was '{output}') " + assert match is not None, f"No line of output file was '{line}' (output was '{output}')" else: matches = re.findall(f"^{re.escape(line)}$", output, flags=re.MULTILINE) assert len(matches) == int(n), f"Expected {n} lines matching '{line}' in output file (output was '{output}'); found {len(matches)}" diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index be8f88c8511..988e405280b 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2016,41 +2016,17 @@ module. - - - ``). If the ``text`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> - - + - - - ``).]]> - - + - - - `` ).]]> - - + - - - ``). If the ``line`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> - - + - - - ``.]]> - - + - - - ``).]]> - - + @@ -2111,6 +2087,76 @@ module. + + + ``). If the ``text`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> + + + + Text to check for + + + + + Desired number of occurences of the text + + + + + + ``).]]> + + + + Text to check for + + + + + + `` ).]]> + + + + Regular expression to check for + + + + + + ``). If the ``line`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> + + + + The line to check for + + + + + Desired number of occurences of the line + + + + + + ``.]]> + + + + Desired number of lines + + + + + + ``).]]> + + + + Regular expression to check for + + + ``). Optionally a column separator (``sep``, default is ``\t``) `and comment character(s) can be specified (``comment``, default is empty string).]]> diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index d8415b0e961..358229e2ff9 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -34,6 +34,64 @@ $ more comment (using a char with meaning wrt regexp) 1\t2\t3 """ +TEXT_HAS_TEXT_ASSERTION = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_N = """ + + + +""" + +TEXT_NOT_HAS_TEXT_ASSERTION = """ + + + +""" + +TEXT_HAS_TEXT_MATCHING_ASSERTION = """ + + + +""" +TEXT_HAS_LINE_ASSERTION = """ + + + +""" +TEXT_HAS_LINE_ASSERTION_N = """ + + + +""" +TEXT_HAS_N_LINES_ASSERTION = """ + + + +""" +TEXT_HAS_LINE_MATCHING_ASSERTION = """ + + + +""" + +TEXT_DATA_HAS_TEXT = """test text +""" + +TEXT_DATA_HAS_TEXT_TWO = """test text +test text +""" + +TEXT_DATA_HAS_TEXT_NEG = """desired content +is not here +""" + +TEXT_DATA_NONE = None + +TEXT_DATA_EMPTY = "" TESTS = [ # test successful assertion @@ -56,13 +114,133 @@ TESTS = [ TABULAR_ASSERTION_COMMENT, TABULAR_DATA_COMMENT, lambda x: len(x) == 0 ), + # test has_text + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text .. negative test + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "Output file did not contain expected text 'test text' (output 'desired content\nis not here\n')" in x + ), + # test has_text with None output + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_NONE, + lambda x: "Checking has_text assertion on empty output (None)" in x + ), + # test has_text with empty output + ( + TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_EMPTY, + lambda x: "Output file did not contain expected text 'test text' (output '')" in x + ), + # test has_text with n + ( + TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_text with n .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2 matches for 'test text' in output file (output 'test text\n'); found 1" in x + ), + # test not_has_text + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test not_has_text .. negative test + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "Output file contains unexpected text 'not here'" in x + ), + # test not_has_text with None output + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_NONE, + lambda x: "Checking not_has_text assertion on empty output (None)" in x + ), + # test not_has_text with empty output + ( + TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_EMPTY, + lambda x: len(x) == 0 + ), + # test has_text_matching + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text_matching .. negative test + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "No text matching expression 'te[sx]t' was found in output file." in x + ), + # test has_line + ( + TEXT_HAS_LINE_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_line .. negative test + ( + TEXT_HAS_LINE_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "No line of output file was 'test text' (output was 'desired content\nis not here\n')" in x + ), + # test has_line with n + ( + TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_line with n .. negative test + ( + TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2 lines matching 'test text' in output file (output was 'test text\n'); found 1" in x + ), + # test has_n_lines + ( + TEXT_HAS_N_LINES_ASSERTION, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_n_lines .. negative test + ( + TEXT_HAS_N_LINES_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2 lines in output, found 1 lines" in x + ), + # test has_line_matching + ( + TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_line_matching .. negative test + ( + TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, + lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file." in x + ), ] TEST_IDS = [ - 'tabular assertion success', - 'tabular assertion failure', - 'tabular csv assertion', - 'tabular with comments assertion', + 'has_n_columns success', + 'has_n_columns failure', + 'has_n_columns for csv', + 'has_n_columns with comments', + 'has_text success', + 'has_text failure', + 'has_text None output', + 'has_text empty output', + 'has_text n success', + 'has_text n failure', + 'not_has_text success', + 'not_has_text failure', + 'not_has_text None output', + 'not_has_text empty output', + 'has_text_matching success', + 'has_text_matching failure', + 'has_line success', + 'has_line failure', + 'has_line n success', + 'has_line n failure', + 'has_n_lines success', + 'has_n_lines failure', + 'has_line_matching success', + 'has_line_matching failure', ] From f78f35443f0e4d2e91f36194abddd127c9029c58 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Sep 2021 11:27:06 +0200 Subject: [PATCH 10/41] add delta for has_n_lines and has_size and delta_frac for has_size plus tests and xsd changes --- lib/galaxy/tool_util/verify/asserts/size.py | 15 ++++-- lib/galaxy/tool_util/verify/asserts/text.py | 13 ++++-- lib/galaxy/tool_util/xsd/galaxy.xsd | 14 +++++- test/unit/tool_util/verify/test_asserts.py | 51 +++++++++++++++++++++ 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/size.py b/lib/galaxy/tool_util/verify/asserts/size.py index 2ad061247ab..a8ccc132cd3 100644 --- a/lib/galaxy/tool_util/verify/asserts/size.py +++ b/lib/galaxy/tool_util/verify/asserts/size.py @@ -1,4 +1,13 @@ -def assert_has_size(output_bytes, value, delta=0): - """Asserts the specified output has a size of the specified value""" +def assert_has_size(output_bytes, value, delta=0, delta_frac=None): + """ + Asserts the specified output has a size of the specified value, + allowing for absolute (delta) and relative (delta_frac) difference. + """ output_size = len(output_bytes) - assert abs(output_size - int(value)) <= int(delta), f"Expected file size was {value}, actual file size was {output_size} (difference of {delta} accepted)" + value = int(value) + delta = int(delta) + assert abs(output_size - value) <= delta, f"Expected file size of {value}, actual file size is {output_size} (difference of {delta} accepted)" + if delta_frac is not None: + delta_frac = float(delta_frac) + print(f'{value - (value * delta_frac)} {int(output_size)} {value + (value * delta_frac)}') + assert (value - (value * delta_frac) <= int(output_size) <= value + (value * delta_frac)), f'Expected file size of {value}, actual file size is {output_size} (relative difference of {delta_frac} accepted, i.e. size in {value - (value * delta_frac)}:{value + (value * delta_frac)})' diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index 5dfb7b52afc..b1c5150c6e0 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -33,11 +33,18 @@ def assert_has_line(output, line, n=None): assert len(matches) == int(n), f"Expected {n} lines matching '{line}' in output file (output was '{output}'); found {len(matches)}" -def assert_has_n_lines(output, n): - """Asserts the specified output contains ``n`` lines.""" +def assert_has_n_lines(output, n, delta=0): + """Asserts the specified output contains ``n`` lines allowing + for a difference in the number of lines (delta)""" assert output is not None, "Checking has_n_lines assertion on empty output (None)" n_lines_found = len(output.splitlines()) - assert n_lines_found == int(n), f"Expected {n} lines in output, found {n_lines_found} lines" + delta = int(delta) + if delta == 0: + assert n_lines_found == int(n), f"Expected {n} lines in output, found {n_lines_found} lines" + else: + diff_lines_found = abs(n_lines_found - int(n)) + print(f"{diff_lines_found} {delta}") + assert diff_lines_found <= delta, f"Expected difference of the number of lines is at most {delta}, found {diff_lines_found}." def assert_has_text_matching(output, expression): diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 988e405280b..2a2a1f0af7a 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2139,13 +2139,18 @@ module. - ``.]]> + ``.]]> Desired number of lines + + + Allowed difference in the number of lines + + @@ -2188,7 +2193,12 @@ module. - Maximum allowed size difference (default is 0). + Maximum allowed size difference (default is 0). See also, the documentation of ``delta`` for the test ```` tag, but note the different default value. + + + + + Maximum allowed size relative size difference (default is to not check). See also, the documentation of ``delta_frac`` for the test ```` tag. diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index 358229e2ff9..f19d3235b41 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -72,12 +72,33 @@ TEXT_HAS_N_LINES_ASSERTION = """ """ +TEXT_HAS_N_LINES_ASSERTION_DELTA = """ + + + +""" TEXT_HAS_LINE_MATCHING_ASSERTION = """ """ +SIZE_HAS_SIZE_ASSERTION = """ + + + +""" +SIZE_HAS_SIZE_ASSERTION_DELTA = """ + + + +""" +SIZE_HAS_SIZE_ASSERTION_DELTA_FRAC = """ + + + +""" + TEXT_DATA_HAS_TEXT = """test text """ @@ -204,6 +225,11 @@ TESTS = [ TEXT_HAS_N_LINES_ASSERTION, TEXT_DATA_HAS_TEXT, lambda x: "Expected 2 lines in output, found 1 lines" in x ), + # test has_n_lines .. lines_diff + ( + TEXT_HAS_N_LINES_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT, + lambda x: "Expected difference of the number of lines is at most 1, found 2." in x + ), # test has_line_matching ( TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT, @@ -214,6 +240,26 @@ TESTS = [ TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file." in x ), + # test has_size + ( + SIZE_HAS_SIZE_ASSERTION, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_size .. negative test + ( + SIZE_HAS_SIZE_ASSERTION, TEXT_DATA_HAS_TEXT_TWO, + lambda x: "Expected file size of 10, actual file size is 20 (difference of 0 accepted)" in x + ), + # test has_size .. delta + ( + SIZE_HAS_SIZE_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_size .. delta + ( + SIZE_HAS_SIZE_ASSERTION_DELTA_FRAC, TEXT_DATA_HAS_TEXT_TWO, + lambda x: "Expected file size of 10, actual file size is 20 (relative difference of 0.2 accepted, i.e. size in 8.0:12.0)" in x + ), ] TEST_IDS = [ @@ -239,8 +285,13 @@ TEST_IDS = [ 'has_line n failure', 'has_n_lines success', 'has_n_lines failure', + 'has_n_lines lines_diff', 'has_line_matching success', 'has_line_matching failure', + 'has_size success', + 'has_size failure', + 'has_size delta', + 'has_size delta_frac', ] From f176797a1ac138a8e282b00aafcbaa2eb4200bfc Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Sep 2021 12:04:42 +0200 Subject: [PATCH 11/41] fix xsd --- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 2a2a1f0af7a..71f0c53dadd 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2193,7 +2193,7 @@ module. - Maximum allowed size difference (default is 0). See also, the documentation of ``delta`` for the test ```` tag, but note the different default value. + Maximum allowed size difference (default is 0). See also, the documentation of ``delta`` for the test ``output`` tag, but note the different default value. From 7b33291a159ebe83b8c872cf996dc263ff20fe36 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Sep 2021 12:30:40 +0200 Subject: [PATCH 12/41] add delta_frac also for has_n_lines and - reformulations in xsd file - consistent use of `.` in assertion messages --- lib/galaxy/tool_util/verify/asserts/size.py | 5 ++-- .../tool_util/verify/asserts/tabular.py | 4 +-- lib/galaxy/tool_util/verify/asserts/text.py | 29 +++++++++++-------- lib/galaxy/tool_util/xsd/galaxy.xsd | 11 +++++-- test/unit/tool_util/verify/test_asserts.py | 25 +++++++++++----- 5 files changed, 47 insertions(+), 27 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/size.py b/lib/galaxy/tool_util/verify/asserts/size.py index a8ccc132cd3..1cab50ef765 100644 --- a/lib/galaxy/tool_util/verify/asserts/size.py +++ b/lib/galaxy/tool_util/verify/asserts/size.py @@ -6,8 +6,7 @@ def assert_has_size(output_bytes, value, delta=0, delta_frac=None): output_size = len(output_bytes) value = int(value) delta = int(delta) - assert abs(output_size - value) <= delta, f"Expected file size of {value}, actual file size is {output_size} (difference of {delta} accepted)" + assert abs(output_size - value) <= delta, f"Expected file size of {value}+-{delta}, actual file size is {output_size}" if delta_frac is not None: delta_frac = float(delta_frac) - print(f'{value - (value * delta_frac)} {int(output_size)} {value + (value * delta_frac)}') - assert (value - (value * delta_frac) <= int(output_size) <= value + (value * delta_frac)), f'Expected file size of {value}, actual file size is {output_size} (relative difference of {delta_frac} accepted, i.e. size in {value - (value * delta_frac)}:{value + (value * delta_frac)})' + assert (value - (value * delta_frac) <= int(output_size) <= value + (value * delta_frac)), f"Expected file size of {value}+-{value * delta_frac}, actual file size is {output_size}" diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index a9b2358a03f..cb1f93e3849 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -19,9 +19,9 @@ def assert_has_n_columns(output, n, sep='\t', comment=""): """ Asserts the tabular output contains n columns. The optional sep argument specifies the column seperator used to determine the number of columns. The optional comment argument specifies - comment characters.""" + comment characters""" n = int(n) first_line = get_first_line(output, comment) - assert first_line is not None, "Was expecting output with %d columns, but output was empty." % n + assert first_line is not None, "Was expecting output with %d columns, but output was empty" % n n_columns = len(first_line.split(sep)) assert n_columns == n, f"Expected {n} columns in output, found {n_columns} columns" diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index b1c5150c6e0..927812d2142 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -4,7 +4,7 @@ import re def assert_has_text(output, text, n=None): """ Asserts specified output contains the substring specified by the argument text. The exact number of occurrences can be - optionally specified by the argument n.""" + optionally specified by the argument n""" assert output is not None, "Checking has_text assertion on empty output (None)" if n is None: assert output.find(text) >= 0, f"Output file did not contain expected text '{text}' (output '{output}')" @@ -15,7 +15,7 @@ def assert_has_text(output, text, n=None): def assert_not_has_text(output, text): """ Asserts specified output does not contain the substring - specified by the argument text.""" + specified by the argument text""" assert output is not None, "Checking not_has_text assertion on empty output (None)" assert output.find(text) < 0, f"Output file contains unexpected text '{text}'" @@ -23,7 +23,7 @@ def assert_not_has_text(output, text): def assert_has_line(output, line, n=None): """ Asserts the specified output contains the line specified by the argument line. The exact number of occurrences can be optionally - specified by the argument n.""" + specified by the argument n""" assert output is not None, "Checking has_line assertion on empty output (None)" if n is None: match = re.search(f"^{re.escape(line)}$", output, flags=re.MULTILINE) @@ -33,29 +33,34 @@ def assert_has_line(output, line, n=None): assert len(matches) == int(n), f"Expected {n} lines matching '{line}' in output file (output was '{output}'); found {len(matches)}" -def assert_has_n_lines(output, n, delta=0): +def assert_has_n_lines(output, n, delta=0, delta_frac=None): """Asserts the specified output contains ``n`` lines allowing - for a difference in the number of lines (delta)""" + for a difference in the number of lines (delta) + or relative differebce in the number of lines""" assert output is not None, "Checking has_n_lines assertion on empty output (None)" n_lines_found = len(output.splitlines()) + n = int(n) delta = int(delta) if delta == 0: - assert n_lines_found == int(n), f"Expected {n} lines in output, found {n_lines_found} lines" + assert n_lines_found == n, f"Expected {n} lines in output, found {n_lines_found} lines" else: - diff_lines_found = abs(n_lines_found - int(n)) + diff_lines_found = abs(n_lines_found - n) print(f"{diff_lines_found} {delta}") - assert diff_lines_found <= delta, f"Expected difference of the number of lines is at most {delta}, found {diff_lines_found}." + assert diff_lines_found <= delta, f"Expected {n}+-{delta} lines in the output, found {n_lines_found} lines" + if delta_frac is not None: + delta_frac = float(delta_frac) + assert (n - (n * delta_frac) <= int(n_lines_found) <= n + (n * delta_frac)), f"Expected {n}+-{n * delta_frac} lines in the output, found {n_lines_found} lines" def assert_has_text_matching(output, expression): """ Asserts the specified output contains text matching the - regular expression specified by the argument expression.""" + regular expression specified by the argument expression""" match = re.search(expression, output) - assert match is not None, f"No text matching expression '{expression}' was found in output file." + assert match is not None, f"No text matching expression '{expression}' was found in output file" def assert_has_line_matching(output, expression): """ Asserts the specified output contains a line matching the - regular expression specified by the argument expression.""" + regular expression specified by the argument expression""" match = re.search(f"^{expression}$", output, flags=re.MULTILINE) - assert match is not None, f"No line matching expression '{expression}' was found in output file." + assert match is not None, f"No line matching expression '{expression}' was found in output file" diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 71f0c53dadd..021b7b6b2eb 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2148,7 +2148,12 @@ module. - Allowed difference in the number of lines + Allowed difference in the number of lines. The observed number of lines has to be in the range ``n +- delta``. + + + + + Maximum allowed relative difference of the number of lines (default is to not check). The observed number of lines must be in the range ``n +- n*delta_frac`` @@ -2193,12 +2198,12 @@ module. - Maximum allowed size difference (default is 0). See also, the documentation of ``delta`` for the test ``output`` tag, but note the different default value. + Maximum allowed size difference (default is 0). The observed size has to be in the range ``value +- delta``. - Maximum allowed size relative size difference (default is to not check). See also, the documentation of ``delta_frac`` for the test ```` tag. + Maximum allowed size relative size difference (default is to not check). The observed size must be in the range ``value +- value*delta_frac`` diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index f19d3235b41..e7b6eae5334 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -77,6 +77,11 @@ TEXT_HAS_N_LINES_ASSERTION_DELTA = """ """ +TEXT_HAS_N_LINES_ASSERTION_DELTA_FRAC = """ + + + +""" TEXT_HAS_LINE_MATCHING_ASSERTION = """ @@ -193,7 +198,7 @@ TESTS = [ # test has_text_matching .. negative test ( TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "No text matching expression 'te[sx]t' was found in output file." in x + lambda x: "No text matching expression 'te[sx]t' was found in output file" in x ), # test has_line ( @@ -225,10 +230,15 @@ TESTS = [ TEXT_HAS_N_LINES_ASSERTION, TEXT_DATA_HAS_TEXT, lambda x: "Expected 2 lines in output, found 1 lines" in x ), - # test has_n_lines .. lines_diff + # test has_n_lines ..delta ( TEXT_HAS_N_LINES_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT, - lambda x: "Expected difference of the number of lines is at most 1, found 2." in x + lambda x: "Expected 3+-1 lines in the output, found 1 lines" in x + ), + # test has_n_lines ..delta_frac + ( + TEXT_HAS_N_LINES_ASSERTION_DELTA_FRAC, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 3+-1.002 lines in the output, found 1 lines" in x ), # test has_line_matching ( @@ -238,7 +248,7 @@ TESTS = [ # test has_line_matching .. negative test ( TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file." in x + lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file" in x ), # test has_size ( @@ -248,7 +258,7 @@ TESTS = [ # test has_size .. negative test ( SIZE_HAS_SIZE_ASSERTION, TEXT_DATA_HAS_TEXT_TWO, - lambda x: "Expected file size of 10, actual file size is 20 (difference of 0 accepted)" in x + lambda x: "Expected file size of 10+-0, actual file size is 20" in x ), # test has_size .. delta ( @@ -258,7 +268,7 @@ TESTS = [ # test has_size .. delta ( SIZE_HAS_SIZE_ASSERTION_DELTA_FRAC, TEXT_DATA_HAS_TEXT_TWO, - lambda x: "Expected file size of 10, actual file size is 20 (relative difference of 0.2 accepted, i.e. size in 8.0:12.0)" in x + lambda x: "Expected file size of 10+-2.0, actual file size is 20" in x ), ] @@ -285,7 +295,8 @@ TEST_IDS = [ 'has_line n failure', 'has_n_lines success', 'has_n_lines failure', - 'has_n_lines lines_diff', + 'has_n_lines delta', + 'has_n_lines delta_frac', 'has_line_matching success', 'has_line_matching failure', 'has_size success', From fc244d224fc74d60b2913fa29919e70f0f3d2b15 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 27 Sep 2021 14:59:10 +0200 Subject: [PATCH 13/41] document sim_size as discouraged --- lib/galaxy/tool_util/xsd/galaxy.xsd | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 021b7b6b2eb..09f9c8bed08 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -6359,8 +6359,8 @@ and ``bibtex`` are the only supported options. Type of comparison to use when comparing test generated output files to expected output files. Currently valid value are -``diff`` (the default), ``re_match``, ``sim_size``, ``re_match_multiline``, -and ``contains``. +``diff`` (the default), ``re_match``, ``re_match_multiline``, +and ``contains``. In addtion there is ``sim_size`` which is discouraged in fafour of a ``has_size`` assertion. From 34b65de9658e3e9be5de043122338ce0d43dfd3f Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 20 Oct 2021 19:59:59 +0200 Subject: [PATCH 14/41] simplify assertion code a bit do not store modules, but directly the functions --- .../tool_util/verify/asserts/__init__.py | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/__init__.py b/lib/galaxy/tool_util/verify/asserts/__init__.py index 234e5e929b6..7763a91035c 100644 --- a/lib/galaxy/tool_util/verify/asserts/__init__.py +++ b/lib/galaxy/tool_util/verify/asserts/__init__.py @@ -1,6 +1,6 @@ import logging import sys -from inspect import getfullargspec +from inspect import getfullargspec, getmembers from galaxy.util import unicodify @@ -12,16 +12,20 @@ assertion_module_names = ['text', 'tabular', 'xml', 'hdf5', 'archive', 'size'] # create a new module of assertion functions, create the needed python # source file "test/base/asserts/.py" and add # to the list of assertion module names defined above. -assertion_modules = [] +assertion_functions = {} for assertion_module_name in assertion_module_names: - full_assertion_module_name = f"galaxy.tool_util.verify.asserts.{assertion_module_name}" + full_assertion_module_name = 'galaxy.tool_util.verify.asserts.' + assertion_module_name try: # Dynamically import module __import__(full_assertion_module_name) assertion_module = sys.modules[full_assertion_module_name] - assertion_modules.append(assertion_module) except Exception: log.exception('Failed to load assertion module: %s', assertion_module_name) + continue + for member, value in getmembers(assertion_module): + print(member) + if member.startswith("assert_"): + assertion_functions[member] = value def verify_assertions(data, assertion_description_list): @@ -33,14 +37,11 @@ def verify_assertions(data, assertion_description_list): def verify_assertion(data, assertion_description): tag = assertion_description["tag"] - assert_function_name = f"assert_{tag}" - assert_function = None - for assertion_module in assertion_modules: - if hasattr(assertion_module, assert_function_name): - assert_function = getattr(assertion_module, assert_function_name) + assert_function_name = "assert_" + tag + assert_function = assertion_functions.get(assert_function_name, None) if assert_function is None: - errmsg = f"Unable to find test function associated with XML tag '{tag}'. Check your tool file syntax." + errmsg = "Unable to find test function associated with XML tag '%s'. Check your tool file syntax." % tag raise AssertionError(errmsg) assert_function_args = getfullargspec(assert_function).args From ebda711c2694af7b17884d72202d938391934eae Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 20 Oct 2021 20:01:09 +0200 Subject: [PATCH 15/41] annotate int and float params of assertion functions --- lib/galaxy/tool_util/verify/asserts/size.py | 2 +- lib/galaxy/tool_util/verify/asserts/tabular.py | 2 +- lib/galaxy/tool_util/verify/asserts/text.py | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/size.py b/lib/galaxy/tool_util/verify/asserts/size.py index 1cab50ef765..9a8e4c53c1b 100644 --- a/lib/galaxy/tool_util/verify/asserts/size.py +++ b/lib/galaxy/tool_util/verify/asserts/size.py @@ -1,4 +1,4 @@ -def assert_has_size(output_bytes, value, delta=0, delta_frac=None): +def assert_has_size(output_bytes, value: int, delta: int = 0, delta_frac: float = None): """ Asserts the specified output has a size of the specified value, allowing for absolute (delta) and relative (delta_frac) difference. diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index cb1f93e3849..324b988d592 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -15,7 +15,7 @@ def get_first_line(output, comment): return match.group(1) -def assert_has_n_columns(output, n, sep='\t', comment=""): +def assert_has_n_columns(output, n: int, sep='\t', comment=""): """ Asserts the tabular output contains n columns. The optional sep argument specifies the column seperator used to determine the number of columns. The optional comment argument specifies diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index 927812d2142..48c4c6292e7 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -1,7 +1,7 @@ import re -def assert_has_text(output, text, n=None): +def assert_has_text(output, text, n: int = None): """ Asserts specified output contains the substring specified by the argument text. The exact number of occurrences can be optionally specified by the argument n""" @@ -20,7 +20,7 @@ def assert_not_has_text(output, text): assert output.find(text) < 0, f"Output file contains unexpected text '{text}'" -def assert_has_line(output, line, n=None): +def assert_has_line(output, line, n: int = None): """ Asserts the specified output contains the line specified by the argument line. The exact number of occurrences can be optionally specified by the argument n""" @@ -33,7 +33,7 @@ def assert_has_line(output, line, n=None): assert len(matches) == int(n), f"Expected {n} lines matching '{line}' in output file (output was '{output}'); found {len(matches)}" -def assert_has_n_lines(output, n, delta=0, delta_frac=None): +def assert_has_n_lines(output, n, delta: int = 0, delta_frac: float = None): """Asserts the specified output contains ``n`` lines allowing for a difference in the number of lines (delta) or relative differebce in the number of lines""" From 1e1acf0049d8f617f1d9da42241bf2701788e2ed Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 20 Oct 2021 20:01:48 +0200 Subject: [PATCH 16/41] add linting of assertions --- lib/galaxy/tool_util/linters/tests.py | 44 ++++++++++++++++++++++-- test/unit/tool_util/test_tool_linters.py | 43 +++++++++++++++++++++++ 2 files changed, 84 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/tool_util/linters/tests.py b/lib/galaxy/tool_util/linters/tests.py index 9936147efb8..f8ae7cee97e 100644 --- a/lib/galaxy/tool_util/linters/tests.py +++ b/lib/galaxy/tool_util/linters/tests.py @@ -1,5 +1,8 @@ """This module contains a linting functions for tool tests.""" +from inspect import Parameter, signature + from ._util import is_datasource +from ..verify import asserts # Misspelled so as not be picked up by nosetests. @@ -35,9 +38,14 @@ def lint_tsts(tool_xml, lint_ctx): break test_assert = ("assert_stdout", "assert_stderr", "assert_command") for ta in test_assert: - if len(test.findall(ta)) > 0: - has_test = True - break + assertions = test.findall(ta) + if len(assertions) == 0: + continue + if len(assertions) > 1: + lint_ctx.error("Test {test_idx}: More than one {ta} found. Only the first is considered.") + has_test = True + _check_asserts(test_idx, assertions, lint_ctx) + _check_asserts(test_idx, test.findall(".//assert_contents"), lint_ctx) # really simple test that test parameters are also present in the inputs for param in test.findall("param"): @@ -94,6 +102,36 @@ def lint_tsts(tool_xml, lint_ctx): lint_ctx.warn("No valid test(s) found.", line=tests_line, xpath=tests_path) +def _check_asserts(test_idx, assertions, lint_ctx): + """ + assertions is a list of assert_contents, assert_stdout, assert_stderr, assert_command + in practice only for the first case the list may be longer than one + """ + for assertion in assertions: + for i, a in enumerate(assertion.iter()): + if i == 0: # skip root note itself + continue + assert_function_name = "assert_" + a.tag + if assert_function_name not in asserts.assertion_functions: + lint_ctx.error(f"Test {test_idx}: unknown assertion {a.tag}") + continue + assert_function_sig = signature(asserts.assertion_functions[assert_function_name]) + for attrib in a.attrib: + if attrib not in assert_function_sig.parameters: + lint_ctx.error(f"Test {test_idx}: unknown attribute {attrib} for {a.tag}") + continue + if assert_function_sig.parameters[attrib].annotation is not Parameter.empty: + try: + assert_function_sig.parameters[attrib].annotation(a.attrib[attrib]) + except ValueError: + lint_ctx.error(f"Test {test_idx}: attribute {attrib} for {a.tag} needs to be {assert_function_sig.parameters[attrib].annotation.__name__} got {a.attrib[attrib]}") + for p in assert_function_sig.parameters: + if p in ["output", "output_bytes", "verify_assertions_function", "children"]: + continue + if assert_function_sig.parameters[p].default is Parameter.empty and p not in a.attrib: + lint_ctx.error(f"Test {test_idx}: missing attribute {p} for {a.tag}") + + def _collect_output_names(tool_xml): output_data_names = [] output_collection_names = [] diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index 0bf37661827..3aa79df96dc 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -547,6 +547,34 @@ TESTS_EXPECT_FAILURE_OUTPUT = """ +ASSERTS = """ + + + + + + + + + + + + + + + + + + + + + + + + + + + """ # tool xml for xml_order linter @@ -954,6 +982,20 @@ TESTS = [ 'Unknown tag [wrong_tag] encountered, this may result in a warning in the future.' in x.info_messages and 'Best practice violation [stdio] elements should come before [command]' in x.warn_messages and len(x.info_messages) == 1 and len(x.valid_messages) == 0 and len(x.warn_messages) == 1 and len(x.error_messages) == 0 + "Test 1: Cannot specify outputs in a test expecting failure." in x.error_messages + and len(x.warn_messages) == 0 and len(x.error_messages) == 1 + ), + ( + ASSERTS, tests.lint_tsts, + lambda x: + 'Test 0: unknown assertion invalid' in x.error_messages + and 'Test 0: unknown attribute invalid_attrib for has_text' in x.error_messages + and 'Test 0: missing attribute text for has_text' in x.error_messages + and 'Test 0: attribute value for has_size needs to be int got 500k' in x.error_messages + and 'Test 0: attribute delta for has_size needs to be int got 1O' in x.error_messages + and 'Test 0: attribute delta_frac for has_size needs to be float got 10,0' in x.error_messages + and 'Test 0: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages + and len(x.warn_messages) == 0 and len(x.error_messages) == 4 ) ] @@ -1008,6 +1050,7 @@ TEST_IDS = [ 'tests: param and output names', 'tests: expecting failure with outputs', 'xml_order' + 'asserts' ] From 00948543d4a6b47220c8d37b59a8ca6ed0de58e7 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 21 Oct 2021 10:21:28 +0200 Subject: [PATCH 17/41] remove print and fix unit test --- lib/galaxy/tool_util/verify/asserts/__init__.py | 3 +-- test/unit/tool_util/test_tool_linters.py | 6 ++++-- test/unit/tool_util/verify/test_asserts.py | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/__init__.py b/lib/galaxy/tool_util/verify/asserts/__init__.py index 7763a91035c..49cd0267b47 100644 --- a/lib/galaxy/tool_util/verify/asserts/__init__.py +++ b/lib/galaxy/tool_util/verify/asserts/__init__.py @@ -23,7 +23,6 @@ for assertion_module_name in assertion_module_names: log.exception('Failed to load assertion module: %s', assertion_module_name) continue for member, value in getmembers(assertion_module): - print(member) if member.startswith("assert_"): assertion_functions[member] = value @@ -41,7 +40,7 @@ def verify_assertion(data, assertion_description): assert_function = assertion_functions.get(assert_function_name, None) if assert_function is None: - errmsg = "Unable to find test function associated with XML tag '%s'. Check your tool file syntax." % tag + errmsg = f"Unable to find test function associated with XML tag {tag}. Check your tool file syntax." raise AssertionError(errmsg) assert_function_args = getfullargspec(assert_function).args diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index 3aa79df96dc..8b216695730 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -547,6 +547,8 @@ TESTS_EXPECT_FAILURE_OUTPUT = """ +""" + ASSERTS = """ @@ -995,8 +997,8 @@ TESTS = [ and 'Test 0: attribute delta for has_size needs to be int got 1O' in x.error_messages and 'Test 0: attribute delta_frac for has_size needs to be float got 10,0' in x.error_messages and 'Test 0: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages - and len(x.warn_messages) == 0 and len(x.error_messages) == 4 - ) + and len(x.warn_messages) == 0 and len(x.error_messages) == 7 + ), ] TEST_IDS = [ diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index e7b6eae5334..e0b3cf45cb7 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -315,5 +315,5 @@ def test_assertions(assertion_xml, data, assert_func): except AssertionError as e: assert_list = e.args else: - assert_list = [] + assert_list = () assert assert_func(assert_list) From c5f2ac556b30d21613e48713449555fa5b1a3e07 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 18 Dec 2021 16:01:52 +0100 Subject: [PATCH 18/41] fix numbers in assertion test --- test/unit/tool_util/test_tool_linters.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index 8b216695730..28fd2387c45 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -990,13 +990,13 @@ TESTS = [ ( ASSERTS, tests.lint_tsts, lambda x: - 'Test 0: unknown assertion invalid' in x.error_messages - and 'Test 0: unknown attribute invalid_attrib for has_text' in x.error_messages - and 'Test 0: missing attribute text for has_text' in x.error_messages - and 'Test 0: attribute value for has_size needs to be int got 500k' in x.error_messages - and 'Test 0: attribute delta for has_size needs to be int got 1O' in x.error_messages - and 'Test 0: attribute delta_frac for has_size needs to be float got 10,0' in x.error_messages - and 'Test 0: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages + 'Test 1: unknown assertion invalid' in x.error_messages + and 'Test 1: unknown attribute invalid_attrib for has_text' in x.error_messages + and 'Test 1: missing attribute text for has_text' in x.error_messages + and 'Test 1: attribute value for has_size needs to be int got 500k' in x.error_messages + and 'Test 1: attribute delta for has_size needs to be int got 1O' in x.error_messages + and 'Test 1: attribute delta_frac for has_size needs to be float got 10,0' in x.error_messages + and 'Test 1: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages and len(x.warn_messages) == 0 and len(x.error_messages) == 7 ), ] From 9c27b6b4e2fea126a7c3bd7b9461c2c3f5a59f64 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Sat, 18 Dec 2021 18:22:04 +0100 Subject: [PATCH 19/41] add n for assert_has_text_matching and assert_has_line_matching --- lib/galaxy/tool_util/verify/asserts/text.py | 30 +++++++++++----- lib/galaxy/tool_util/xsd/galaxy.xsd | 10 ++++++ test/unit/tool_util/verify/test_asserts.py | 38 ++++++++++++++++++++- 3 files changed, 68 insertions(+), 10 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index 48c4c6292e7..4ad52e058a2 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -10,7 +10,7 @@ def assert_has_text(output, text, n: int = None): assert output.find(text) >= 0, f"Output file did not contain expected text '{text}' (output '{output}')" else: matches = re.findall(re.escape(text), output) - assert len(matches) == int(n), f"Expected {n} matches for '{text}' in output file (output '{output}'); found {len(matches)}" + assert len(matches) == int(n), f"Expected {n} occurences of '{text}' in output file (output '{output}'); found {len(matches)}" def assert_not_has_text(output, text): @@ -52,15 +52,27 @@ def assert_has_n_lines(output, n, delta: int = 0, delta_frac: float = None): assert (n - (n * delta_frac) <= int(n_lines_found) <= n + (n * delta_frac)), f"Expected {n}+-{n * delta_frac} lines in the output, found {n_lines_found} lines" -def assert_has_text_matching(output, expression): +def assert_has_text_matching(output, expression, n: int = None): """ Asserts the specified output contains text matching the - regular expression specified by the argument expression""" - match = re.search(expression, output) - assert match is not None, f"No text matching expression '{expression}' was found in output file" + regular expression specified by the argument expression. + If n is given the assertion checks for exacly n (nonoverlapping) + occurences. + """ + if n is None: + match = re.search(expression, output) + assert match is not None, f"No text matching expression '{expression}' was found in output file" + else: + matches = re.findall(expression, output) + assert len(matches) == int(n), f"Expected {n} (non-overlapping) matches for '{expression}' in output file (output '{output}'); found {len(matches)}" -def assert_has_line_matching(output, expression): +def assert_has_line_matching(output, expression, n: int = None): """ Asserts the specified output contains a line matching the - regular expression specified by the argument expression""" - match = re.search(f"^{expression}$", output, flags=re.MULTILINE) - assert match is not None, f"No line matching expression '{expression}' was found in output file" + regular expression specified by the argument expression. If n is given + the assertion checks for exactly n occurences.""" + if n is None: + match = re.search(f"^{expression}$", output, flags=re.MULTILINE) + assert match is not None, f"No line matching expression '{expression}' was found in output file" + else: + matches = re.findall(f"^{expression}$", output, flags=re.MULTILINE) + assert len(matches) == int(n), f"Expected {n} lines matching for '{expression}' in output file (output '{output}'); found {len(matches)}" diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 09f9c8bed08..bd9288ecec0 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2121,6 +2121,11 @@ module. Regular expression to check for + + + Desired number of non-overlapping matches of the expression + + @@ -2166,6 +2171,11 @@ module. Regular expression to check for + + + Desired number of lines matching the expression + + diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index e0b3cf45cb7..adcef764744 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -57,6 +57,13 @@ TEXT_HAS_TEXT_MATCHING_ASSERTION = """ """ + +TEXT_HAS_TEXT_MATCHING_ASSERTION_N = """ + + + +""" + TEXT_HAS_LINE_ASSERTION = """ @@ -87,6 +94,11 @@ TEXT_HAS_LINE_MATCHING_ASSERTION = """ """ +TEXT_HAS_LINE_MATCHING_ASSERTION_N = """ + + + +""" SIZE_HAS_SIZE_ASSERTION = """ @@ -168,7 +180,7 @@ TESTS = [ # test has_text with n .. negative test ( TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2 matches for 'test text' in output file (output 'test text\n'); found 1" in x + lambda x: "Expected 2 occurences of 'test text' in output file (output 'test text\n'); found 1" in x ), # test not_has_text ( @@ -200,6 +212,16 @@ TESTS = [ TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, lambda x: "No text matching expression 'te[sx]t' was found in output file" in x ), + # test has_text_matching with n + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 4 (non-overlapping) matches for 'te[sx]t' in output file (output 'test text\n'); found 2" in x + ), # test has_line ( TEXT_HAS_LINE_ASSERTION, TEXT_DATA_HAS_TEXT, @@ -250,6 +272,16 @@ TESTS = [ TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file" in x ), + # test has_line_matching n + ( + TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_line_matching n .. negative test + ( + TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 2 lines matching for 'te[sx]t te[sx]t' in output file (output 'test text\n'); found 1" in x + ), # test has_size ( SIZE_HAS_SIZE_ASSERTION, TEXT_DATA_HAS_TEXT, @@ -289,6 +321,8 @@ TEST_IDS = [ 'not_has_text empty output', 'has_text_matching success', 'has_text_matching failure', + 'has_text_matching n success', + 'has_text_matching n failure', 'has_line success', 'has_line failure', 'has_line n success', @@ -299,6 +333,8 @@ TEST_IDS = [ 'has_n_lines delta_frac', 'has_line_matching success', 'has_line_matching failure', + 'has_line_matching n success', + 'has_line_matching n failure', 'has_size success', 'has_size failure', 'has_size delta', From 77d1f3b45b772f504274b977a230e22e8412ecc7 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 20 Dec 2021 13:35:26 +0100 Subject: [PATCH 20/41] add delta min max to text, tabular, and size assertions for tabular and size n/value becomes optional and remove delta_frac --- lib/galaxy/tool_util/linters/tests.py | 9 + lib/galaxy/tool_util/verify/asserts/_util.py | 25 +++ lib/galaxy/tool_util/verify/asserts/size.py | 14 +- .../tool_util/verify/asserts/tabular.py | 12 +- lib/galaxy/tool_util/verify/asserts/text.py | 93 +++++---- lib/galaxy/tool_util/xsd/galaxy.xsd | 195 +++++++++++++++--- test/unit/tool_util/test_tool_linters.py | 15 +- test/unit/tool_util/verify/test_asserts.py | 100 ++++++--- 8 files changed, 340 insertions(+), 123 deletions(-) create mode 100644 lib/galaxy/tool_util/verify/asserts/_util.py diff --git a/lib/galaxy/tool_util/linters/tests.py b/lib/galaxy/tool_util/linters/tests.py index f8ae7cee97e..e6ff65321aa 100644 --- a/lib/galaxy/tool_util/linters/tests.py +++ b/lib/galaxy/tool_util/linters/tests.py @@ -116,6 +116,7 @@ def _check_asserts(test_idx, assertions, lint_ctx): lint_ctx.error(f"Test {test_idx}: unknown assertion {a.tag}") continue assert_function_sig = signature(asserts.assertion_functions[assert_function_name]) + # check type of the attributes (int, float ...) for attrib in a.attrib: if attrib not in assert_function_sig.parameters: lint_ctx.error(f"Test {test_idx}: unknown attribute {attrib} for {a.tag}") @@ -125,11 +126,19 @@ def _check_asserts(test_idx, assertions, lint_ctx): assert_function_sig.parameters[attrib].annotation(a.attrib[attrib]) except ValueError: lint_ctx.error(f"Test {test_idx}: attribute {attrib} for {a.tag} needs to be {assert_function_sig.parameters[attrib].annotation.__name__} got {a.attrib[attrib]}") + # check missing required attributes for p in assert_function_sig.parameters: if p in ["output", "output_bytes", "verify_assertions_function", "children"]: continue if assert_function_sig.parameters[p].default is Parameter.empty and p not in a.attrib: lint_ctx.error(f"Test {test_idx}: missing attribute {p} for {a.tag}") + # has_n_lines, has_n_columns, and has_size need to specify n/value, min, or max + if a.tag in ["has_n_lines", "has_n_columns"]: + if "n" not in a.attrib and "min" not in a.attrib and "max" not in a.attrib: + lint_ctx.error(f"Test {test_idx}: {a.tag} needs to specify 'n', 'min', or 'max'") + if a.tag == "has_size": + if "value" not in a.attrib and "min" not in a.attrib and "max" not in a.attrib: + lint_ctx.error(f"Test {test_idx}: {a.tag} needs to specify 'n', 'min', or 'max'") def _collect_output_names(tool_xml): diff --git a/lib/galaxy/tool_util/verify/asserts/_util.py b/lib/galaxy/tool_util/verify/asserts/_util.py new file mode 100644 index 00000000000..09a319534a9 --- /dev/null +++ b/lib/galaxy/tool_util/verify/asserts/_util.py @@ -0,0 +1,25 @@ +from math import inf + + +def _assert_number(count, n, delta, min, max, n_text, min_max_text): + """ + helper function for assering that count is in + - n +- delta + - min:max + + raising an assertion error using n_text and min_max_text (resp) + substituting {n}, {delta}, {min}, and {max} + (and keeping potentially present {text} and {output}) + """ + if n is not None: + assert abs(count - int(n)) <= int(delta), n_text.format(n=n, delta=delta, text="{text}", output="{output}") + f" found {count}" + if min is not None or max is not None: + if min is None: + min = -inf + else: + min = int(min) + if max is None: + max = inf + else: + max = int(max) + assert min <= count <= max, min_max_text.format(min=min, max=max, text="{text}", output="{output}") + f" found {count}" diff --git a/lib/galaxy/tool_util/verify/asserts/size.py b/lib/galaxy/tool_util/verify/asserts/size.py index 9a8e4c53c1b..acac5e9ca76 100644 --- a/lib/galaxy/tool_util/verify/asserts/size.py +++ b/lib/galaxy/tool_util/verify/asserts/size.py @@ -1,12 +1,12 @@ -def assert_has_size(output_bytes, value: int, delta: int = 0, delta_frac: float = None): +from ._util import _assert_number + + +def assert_has_size(output_bytes, value: int = None, delta: int = 0, min: int = None, max: int = None): """ Asserts the specified output has a size of the specified value, allowing for absolute (delta) and relative (delta_frac) difference. """ output_size = len(output_bytes) - value = int(value) - delta = int(delta) - assert abs(output_size - value) <= delta, f"Expected file size of {value}+-{delta}, actual file size is {output_size}" - if delta_frac is not None: - delta_frac = float(delta_frac) - assert (value - (value * delta_frac) <= int(output_size) <= value + (value * delta_frac)), f"Expected file size of {value}+-{value * delta_frac}, actual file size is {output_size}" + _assert_number(output_size, value, delta, min, max, + "Expected file size of {n}+-{delta}", + "Expected file size to be in [{min}:{max}]") diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index 324b988d592..9bffc7c89f5 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -1,5 +1,7 @@ import re +from ._util import _assert_number + def get_first_line(output, comment): """ @@ -10,18 +12,18 @@ def get_first_line(output, comment): else: match = re.search("^(.+)$", output, flags=re.MULTILINE) if match is None: - return None + return "" else: return match.group(1) -def assert_has_n_columns(output, n: int, sep='\t', comment=""): +def assert_has_n_columns(output, n: int = None, delta: int = 0, min: int = None, max: int = None, sep='\t', comment=""): """ Asserts the tabular output contains n columns. The optional sep argument specifies the column seperator used to determine the number of columns. The optional comment argument specifies comment characters""" - n = int(n) first_line = get_first_line(output, comment) - assert first_line is not None, "Was expecting output with %d columns, but output was empty" % n n_columns = len(first_line.split(sep)) - assert n_columns == n, f"Expected {n} columns in output, found {n_columns} columns" + _assert_number(n_columns, n, delta, min, max, + "Expected {n}+-{delta} columns in output", + "Expected the number of columns in output to be in [{min}:{max}]") diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index 4ad52e058a2..de1a6cc764f 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -1,16 +1,37 @@ import re +from ._util import _assert_number -def assert_has_text(output, text, n: int = None): + +def _assert_presence_number(output, text, n, delta, min, max, check_presence_foo, count_foo, presence_text, n_text, min_max_text): + """ + helper function to assert that + - text is present in output using check_presence_foo + this is done only if n, min, and max are None + - text appears a certain number of times, where the count is determined with count foo + + raising an assertion error using presence_text, n_text or min_max_text (resp) + substituting {n}, {delta}, {min}, {max}, {text}, and {output} + """ + if n is None and min is None and max is None: + assert check_presence_foo(output, text), presence_text.format(output=output, text=text) + try: + _assert_number(count_foo(output, text), n, delta, min, max, n_text, min_max_text) + except AssertionError as e: + raise AssertionError(str(e).format(output=output, text=text)) + + +def assert_has_text(output, text, n: int = None, delta: int = 0, min: int = None, max: int = None): """ Asserts specified output contains the substring specified by the argument text. The exact number of occurrences can be optionally specified by the argument n""" assert output is not None, "Checking has_text assertion on empty output (None)" - if n is None: - assert output.find(text) >= 0, f"Output file did not contain expected text '{text}' (output '{output}')" - else: - matches = re.findall(re.escape(text), output) - assert len(matches) == int(n), f"Expected {n} occurences of '{text}' in output file (output '{output}'); found {len(matches)}" + _assert_presence_number(output, text, n, delta, min, max, + lambda o, t: o.find(t) >= 0, + lambda o, t: len(re.findall(re.escape(t), o)), + "Output file did not contain expected text '{text}' (output '{output}')", + "Expected {n}+-{delta} occurences of '{text}' in output file (output '{output}')", + "Expected that the number of occurences of '{text}' in output file is in [{min}:{max}] (output '{output}')") def assert_not_has_text(output, text): @@ -20,59 +41,51 @@ def assert_not_has_text(output, text): assert output.find(text) < 0, f"Output file contains unexpected text '{text}'" -def assert_has_line(output, line, n: int = None): +def assert_has_line(output, line, n: int = None, delta: int = 0, min: int = None, max: int = None): """ Asserts the specified output contains the line specified by the argument line. The exact number of occurrences can be optionally specified by the argument n""" assert output is not None, "Checking has_line assertion on empty output (None)" - if n is None: - match = re.search(f"^{re.escape(line)}$", output, flags=re.MULTILINE) - assert match is not None, f"No line of output file was '{line}' (output was '{output}')" - else: - matches = re.findall(f"^{re.escape(line)}$", output, flags=re.MULTILINE) - assert len(matches) == int(n), f"Expected {n} lines matching '{line}' in output file (output was '{output}'); found {len(matches)}" + _assert_presence_number(output, line, n, delta, min, max, + lambda o, l: re.search(f"^{re.escape(l)}$", o, flags=re.MULTILINE) is not None, + lambda o, l: len(re.findall(f"^{re.escape(l)}$", o, flags=re.MULTILINE)), + "No line of output file was '{text}' (output was '{output}')", + "Expected {n}+-{delta} lines '{text}' in output file (output was '{output}')", + "Expected that the number of lines '{text}' in output file is in [{min}:{max}] (output '{output}')") -def assert_has_n_lines(output, n, delta: int = 0, delta_frac: float = None): +def assert_has_n_lines(output, n: int = None, delta: int = 0, min: int = None, max: int = None): """Asserts the specified output contains ``n`` lines allowing for a difference in the number of lines (delta) or relative differebce in the number of lines""" assert output is not None, "Checking has_n_lines assertion on empty output (None)" - n_lines_found = len(output.splitlines()) - n = int(n) - delta = int(delta) - if delta == 0: - assert n_lines_found == n, f"Expected {n} lines in output, found {n_lines_found} lines" - else: - diff_lines_found = abs(n_lines_found - n) - print(f"{diff_lines_found} {delta}") - assert diff_lines_found <= delta, f"Expected {n}+-{delta} lines in the output, found {n_lines_found} lines" - if delta_frac is not None: - delta_frac = float(delta_frac) - assert (n - (n * delta_frac) <= int(n_lines_found) <= n + (n * delta_frac)), f"Expected {n}+-{n * delta_frac} lines in the output, found {n_lines_found} lines" + count = len(output.splitlines()) + _assert_number(count, n, delta, min, max, + "Expected {n}+-{delta} lines in the output", + "Expected the number of line to be in [{min}:{max}]") -def assert_has_text_matching(output, expression, n: int = None): +def assert_has_text_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None): """ Asserts the specified output contains text matching the regular expression specified by the argument expression. If n is given the assertion checks for exacly n (nonoverlapping) occurences. """ - if n is None: - match = re.search(expression, output) - assert match is not None, f"No text matching expression '{expression}' was found in output file" - else: - matches = re.findall(expression, output) - assert len(matches) == int(n), f"Expected {n} (non-overlapping) matches for '{expression}' in output file (output '{output}'); found {len(matches)}" + _assert_presence_number(output, expression, n, delta, min, max, + lambda o, e: re.search(e, o) is not None, + lambda o, e: len(re.findall(e, o)), + "No text matching expression '{text}' was found in output file (output '{output}')", + "Expected {n}+-{delta} (non-overlapping) matches for '{text}' in output file (output '{output}')", + "Expected that the number of (non-overlapping) matches for '{text}' in output file is in [{min}:{max}] (output '{output}')") -def assert_has_line_matching(output, expression, n: int = None): +def assert_has_line_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None): """ Asserts the specified output contains a line matching the regular expression specified by the argument expression. If n is given the assertion checks for exactly n occurences.""" - if n is None: - match = re.search(f"^{expression}$", output, flags=re.MULTILINE) - assert match is not None, f"No line matching expression '{expression}' was found in output file" - else: - matches = re.findall(f"^{expression}$", output, flags=re.MULTILINE) - assert len(matches) == int(n), f"Expected {n} lines matching for '{expression}' in output file (output '{output}'); found {len(matches)}" + _assert_presence_number(output, expression, n, delta, min, max, + lambda o, e: re.search(f"^{e}$", o, flags=re.MULTILINE) is not None, + lambda o, e: len(re.findall(f"^{e}$", o, flags=re.MULTILINE)), + "No line matching expression '{text}' was found in output file (output '{output}')", + "Expected {n}+-{delta} lines matching for '{text}' in output file (output '{output}')", + "Expected that the number of lines matching for '{text}' in output file is in [{min}:{max}] (output '{output}')") diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index bd9288ecec0..82c165c7b3a 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2016,20 +2016,13 @@ module. - - - - - - - - - - - - - - + + + + + + + ``).]]> @@ -2082,25 +2075,46 @@ module. - - + - ``). If the ``text`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> + ``). If the ``text`` is expected to occur a particular number of +times, this value can be specified using ``n``. Optionally also with a certain +``delta``. Alternatively the range of expected occurences can be specified by +``min`` and/or ``max``. +]]> + Text to check for - + Desired number of occurences of the text + + + Allowed difference wrt. n + + + + + Minimum number of occurences + + + + + Maximum number of occurences + + @@ -2114,72 +2128,159 @@ module. - `` ).]]> + `` ). If the +regular expression is expected to match a particular number of times, this value +can be specified using ``n``. Note only non-overlapping occurences are counted. +Optionally also with a certain ``delta``. Alternatively the range of expected +occurences can be specified by ``min`` and/or ``max``. +]]> + Regular expression to check for - + Desired number of non-overlapping matches of the expression + + + Allowed difference wrt. n + + + + + Minimum number of occurences + + + + + Maximum number of occurences + + - ``). If the ``line`` is expected to occur a particular number of times, this value can be specified using ``n``.]]> + ``). If the ``line`` is expected +to occur a particular number of times, this value can be specified using ``n``. +Optionally also with a certain ``delta``. Alternatively the range of expected +occurences can be specified by ``min`` and/or ``max``. +]]> + The line to check for - + Desired number of occurences of the line + + + Allowed difference wrt. n + + + + + Minimum number of occurences + + + + + Maximum number of occurences + + - ``.]]> + ``. +Alternatively the range of expected occurences can be specified by ``min`` +and/or ``max``. +]]> + - + Desired number of lines - + Allowed difference in the number of lines. The observed number of lines has to be in the range ``n +- delta``. - + - Maximum allowed relative difference of the number of lines (default is to not check). The observed number of lines must be in the range ``n +- n*delta_frac`` + Minimum number of lines + + + + + Maximum number of lines - ``).]]> + ``). +If a particular number of matching lines is expected, this value can be +specified using ``n``. Optionally also with ``delta``. Alternatively the range +of expected occurences can be specified by ``min`` and/or ``max``. +]]> + Regular expression to check for - + Desired number of lines matching the expression + + + Allowed difference in the number of matching lines. The observed number of matching lines has to be in the range ``n +- delta``. + + + + + Minimum number of matching lines + + + + + Maximum number of matching lines + + - ``). Optionally a column separator (``sep``, default is ``\t``) `and comment character(s) can be specified (``comment``, default is empty string).]]> + ``) optionally also with +``delta``. Alternatively the range of expected occurences can be specified by +``min`` and/or ``max``. Optionally a column separator (``sep``, default is +``\t``) `and comment character(s) can be specified (``comment``, default is +empty string), then the first non-comment line is used for determining the +number of columns. +]]> + @@ -2196,12 +2297,33 @@ module. Comment character(s) + + + Allowed difference in the number of columns. The observed number of columns has to be in the range ``n +- delta``. + + + + + Minimum number of columns + + + + + Maximum number of columns + + - ``.]]> + ``. +Alternatively the range of the expected size can be specified by ``min`` and/or +``max``. +]]> + - + Desired size of the outpyt (in bytes) @@ -2211,9 +2333,14 @@ module. Maximum allowed size difference (default is 0). The observed size has to be in the range ``value +- delta``. - + - Maximum allowed size relative size difference (default is to not check). The observed size must be in the range ``value +- value*delta_frac`` + Minimum expected size + + + + + Maximum expected size diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index 28fd2387c45..aed939641ed 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -568,12 +568,19 @@ ASSERTS = """ - + + + + + + + + @@ -995,9 +1002,11 @@ TESTS = [ and 'Test 1: missing attribute text for has_text' in x.error_messages and 'Test 1: attribute value for has_size needs to be int got 500k' in x.error_messages and 'Test 1: attribute delta for has_size needs to be int got 1O' in x.error_messages - and 'Test 1: attribute delta_frac for has_size needs to be float got 10,0' in x.error_messages and 'Test 1: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages - and len(x.warn_messages) == 0 and len(x.error_messages) == 7 + and "Test 1: has_size needs to specify 'n', 'min', or 'max'" in x.error_messages + and "Test 1: has_n_columns needs to specify 'n', 'min', or 'max'" in x.error_messages + and "Test 1: has_n_lines needs to specify 'n', 'min', or 'max'" in x.error_messages + and len(x.warn_messages) == 0 and len(x.error_messages) == 9 ), ] diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index adcef764744..e2d6af8aa7b 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -11,7 +11,7 @@ TABULAR_ASSERTION = """ """ TABULAR_CSV_ASSERTION = """ - + """ TABULAR_ASSERTION_COMMENT = """ @@ -46,6 +46,18 @@ TEXT_HAS_TEXT_ASSERTION_N = """ """ +TEXT_HAS_TEXT_ASSERTION_N_DELTA = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_MIN_MAX = """ + + + +""" + TEXT_NOT_HAS_TEXT_ASSERTION = """ @@ -64,6 +76,12 @@ TEXT_HAS_TEXT_MATCHING_ASSERTION_N = """ """ +TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX = """ + + + +""" + TEXT_HAS_LINE_ASSERTION = """ @@ -84,11 +102,6 @@ TEXT_HAS_N_LINES_ASSERTION_DELTA = """ """ -TEXT_HAS_N_LINES_ASSERTION_DELTA_FRAC = """ - - - -""" TEXT_HAS_LINE_MATCHING_ASSERTION = """ @@ -110,11 +123,6 @@ SIZE_HAS_SIZE_ASSERTION_DELTA = """ """ -SIZE_HAS_SIZE_ASSERTION_DELTA_FRAC = """ - - - -""" TEXT_DATA_HAS_TEXT = """test text """ @@ -140,12 +148,12 @@ TESTS = [ # test wrong number of columns ( TABULAR_ASSERTION, TABULAR_DATA_NEG, - lambda x: 'Expected 3 columns in output, found 4 columns' in x + lambda x: 'Expected 3+-0 columns in output found 4' in x ), # test wrong number of columns for csv data ( TABULAR_CSV_ASSERTION, TABULAR_CSV_DATA, - lambda x: 'Expected 3 columns in output, found 2 columns' in x + lambda x: 'Expected the number of columns in output to be in [3:inf] found 2' in x ), # test tabular data with comments ( @@ -180,7 +188,27 @@ TESTS = [ # test has_text with n .. negative test ( TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2 occurences of 'test text' in output file (output 'test text\n'); found 1" in x + lambda x: "Expected 2+-0 occurences of 'test text' in output file (output 'test text\n') found 1" in x + ), + # test has_text with n + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_text with n .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT, + lambda x: "Expected 3+-1 occurences of 'test text' in output file (output 'test text\n') found 1" in x + ), + # test has_text with min max + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_text with min max .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX, TEXT_DATA_HAS_TEXT, + lambda x: "Expected that the number of occurences of 'test text' in output file is in [2:4] (output 'test text\n') found 1" in x ), # test not_has_text ( @@ -210,7 +238,7 @@ TESTS = [ # test has_text_matching .. negative test ( TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "No text matching expression 'te[sx]t' was found in output file" in x + lambda x: "No text matching expression 'te[sx]t' was found in output file (output 'desired content\nis not here\n')" in x ), # test has_text_matching with n ( @@ -220,7 +248,17 @@ TESTS = [ # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) ( TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 4 (non-overlapping) matches for 'te[sx]t' in output file (output 'test text\n'); found 2" in x + lambda x: "Expected 4+-0 (non-overlapping) matches for 'te[sx]t' in output file (output 'test text\n') found 2" in x + ), + # test has_text_matching with n + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX, TEXT_DATA_HAS_TEXT_TWO, + lambda x: len(x) == 0 + ), + # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) + ( + TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX, TEXT_DATA_HAS_TEXT, + lambda x: "Expected that the number of (non-overlapping) matches for 'te[sx]t' in output file is in [3:5] (output 'test text\n') found 2" in x ), # test has_line ( @@ -240,7 +278,7 @@ TESTS = [ # test has_line with n .. negative test ( TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2 lines matching 'test text' in output file (output was 'test text\n'); found 1" in x + lambda x: "Expected 2+-0 lines 'test text' in output file (output was 'test text\n') found 1" in x ), # test has_n_lines ( @@ -250,17 +288,12 @@ TESTS = [ # test has_n_lines .. negative test ( TEXT_HAS_N_LINES_ASSERTION, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2 lines in output, found 1 lines" in x + lambda x: "Expected 2+-0 lines in the output found 1" in x ), # test has_n_lines ..delta ( TEXT_HAS_N_LINES_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 3+-1 lines in the output, found 1 lines" in x - ), - # test has_n_lines ..delta_frac - ( - TEXT_HAS_N_LINES_ASSERTION_DELTA_FRAC, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 3+-1.002 lines in the output, found 1 lines" in x + lambda x: "Expected 3+-1 lines in the output found 1" in x ), # test has_line_matching ( @@ -270,7 +303,7 @@ TESTS = [ # test has_line_matching .. negative test ( TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file" in x + lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file (output 'desired content\nis not here\n')" in x ), # test has_line_matching n ( @@ -280,7 +313,7 @@ TESTS = [ # test has_line_matching n .. negative test ( TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2 lines matching for 'te[sx]t te[sx]t' in output file (output 'test text\n'); found 1" in x + lambda x: "Expected 2+-0 lines matching for 'te[sx]t te[sx]t' in output file (output 'test text\n') found 1" in x ), # test has_size ( @@ -290,18 +323,13 @@ TESTS = [ # test has_size .. negative test ( SIZE_HAS_SIZE_ASSERTION, TEXT_DATA_HAS_TEXT_TWO, - lambda x: "Expected file size of 10+-0, actual file size is 20" in x + lambda x: "Expected file size of 10+-0 found 20" in x ), # test has_size .. delta ( SIZE_HAS_SIZE_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT_TWO, lambda x: len(x) == 0 ), - # test has_size .. delta - ( - SIZE_HAS_SIZE_ASSERTION_DELTA_FRAC, TEXT_DATA_HAS_TEXT_TWO, - lambda x: "Expected file size of 10+-2.0, actual file size is 20" in x - ), ] TEST_IDS = [ @@ -315,6 +343,10 @@ TEST_IDS = [ 'has_text empty output', 'has_text n success', 'has_text n failure', + 'has_text n delta success', + 'has_text n delta failure', + 'has_text min/max delta success', + 'has_text min/max delta failure', 'not_has_text success', 'not_has_text failure', 'not_has_text None output', @@ -323,6 +355,8 @@ TEST_IDS = [ 'has_text_matching failure', 'has_text_matching n success', 'has_text_matching n failure', + 'has_text_matching min/max success', + 'has_text_matching min/max failure', 'has_line success', 'has_line failure', 'has_line n success', @@ -330,7 +364,6 @@ TEST_IDS = [ 'has_n_lines success', 'has_n_lines failure', 'has_n_lines delta', - 'has_n_lines delta_frac', 'has_line_matching success', 'has_line_matching failure', 'has_line_matching n success', @@ -338,7 +371,6 @@ TEST_IDS = [ 'has_size success', 'has_size failure', 'has_size delta', - 'has_size delta_frac', ] From 6a4381cc785c1dc1cd7ee130078b25e3aed24e07 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 22 Dec 2021 21:58:10 +0100 Subject: [PATCH 21/41] xml assertions: tests, small improvements + xsd reformatting - added unit tests for all xml assertions - restructured xsd: - grouped general, text, tabular, archive, xml, and h5 assertions - full definition of all assertions in xsd - allow for "recursion" in xsd for has_archive_member and element_text - restructured schema doc: instead of the table its now a definition list (no more horizontal scrolling in the wide table) also here assertions are grouped --- doc/parse_gx_xsd.py | 26 +- lib/galaxy/tool_util/verify/asserts/xml.py | 24 +- lib/galaxy/tool_util/xsd/galaxy.xsd | 318 +++++++++++++++------ test/unit/tool_util/test_tool_linters.py | 7 +- test/unit/tool_util/verify/test_asserts.py | 287 ++++++++++++++++++- 5 files changed, 527 insertions(+), 135 deletions(-) diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py index 16afe0f6dca..b0ac29972d6 100644 --- a/doc/parse_gx_xsd.py +++ b/doc/parse_gx_xsd.py @@ -98,20 +98,24 @@ def _build_tag(tag, hide_attributes): text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) if line.startswith("$assertions"): assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']") - assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='TestAssertion']") assertions_buffer = StringIO() assertions_buffer.write(_doc_or_none(assertions_tag)) assertions_buffer.write("\n\n") - assertions_buffer.write("Child Element/Assertion | Details \n") - assertions_buffer.write("--- | ---\n") - elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element") - for element in elements: - doc = _doc_or_none(element) - if doc is None: - doc = _doc_or_none(_type_el(element)) - assert doc is not None, "Documentation for %s is empty" % element.attrib["name"] - doc = doc.strip() - assertions_buffer.write("``{}`` | {}\n".format(element.attrib["name"], doc)) + + assertion_groups = assertions_tag.xpath("xs:sequence/xs:group", namespaces={'xs': 'http://www.w3.org/2001/XMLSchema'}) + for group in assertion_groups: + ref = group.attrib['ref'] + assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='" + ref + "']") + doc = _doc_or_none(assertion_tag) + assertions_buffer.write(f"### {doc}\n\n") + elements = assertion_tag.findall("{http://www.w3.org/2001/XMLSchema}choice/{http://www.w3.org/2001/XMLSchema}element") + for element in elements: + doc = _doc_or_none(element) + if doc is None: + doc = _doc_or_none(_type_el(element)) + assert doc is not None, "Documentation for %s is empty" % element.attrib["name"] + doc = doc.strip() + assertions_buffer.write(f"``{element.attrib['name']}``\n: {doc}\n\n") text = text.replace(line, assertions_buffer.getvalue()) tag_help.write(text) best_practices = _get_bp_link(annotation_el) diff --git a/lib/galaxy/tool_util/verify/asserts/xml.py b/lib/galaxy/tool_util/verify/asserts/xml.py index 7835a2dd29e..b649c599846 100644 --- a/lib/galaxy/tool_util/verify/asserts/xml.py +++ b/lib/galaxy/tool_util/verify/asserts/xml.py @@ -1,5 +1,7 @@ import re +from lxml.etree import XMLSyntaxError + from galaxy.util import ( parse_xml_string, unicodify, @@ -27,8 +29,7 @@ def assert_is_valid_xml(output): is valid XML.""" try: to_xml(output) - except Exception as e: - # TODO: Narrow caught exception to just parsing failure + except XMLSyntaxError as e: raise AssertionError(f"Expected valid XML, but could not parse output. {unicodify(e)}") @@ -36,10 +37,9 @@ def assert_has_element_with_path(output, path): """ Asserts the specified output has at least one XML element with a path matching the specified path argument. Valid paths are the simplified subsets of XPath implemented by lxml.etree; - http://effbot.org/zone/element-xpath.htm for more information.""" - if xml_find(output, path) is None: - errmsg = f"Expected to find XML element matching expression {path}, not such match was found." - raise AssertionError(errmsg) + https://lxml.de/xpathxslt.html for more information.""" + print(xml_find(output, path)) + assert xml_find(output, path) is not None, f"Expected to find XML element matching expression {path}, not such match was found." def assert_has_n_elements_with_path(output, path, n): @@ -48,18 +48,14 @@ def assert_has_n_elements_with_path(output, path, n): xml = to_xml(output) n = int(n) num_elements = len(xml.findall(path)) - if num_elements != n: - errmsg = "Expected to find %d elements with path %s, but %d were found." % (n, path, num_elements) - raise AssertionError(errmsg) + assert num_elements == n, f"Expected to find {n} elements with path {path}, but {num_elements} were found." def assert_element_text_matches(output, path, expression): """ Asserts the text of the first element matching the specified path matches the specified regular expression.""" text = xml_find_text(output, path) - if re.match(expression, text) is None: - errmsg = f"Expected element with path '{path}' to contain text matching '{expression}', instead text '{text}' was found." - raise AssertionError(errmsg) + assert re.match(expression, text), f"Expected element with path '{path}' to contain text matching '{expression}', instead text '{text}' was found." def assert_element_text_is(output, path, text): @@ -73,9 +69,7 @@ def assert_attribute_matches(output, path, attribute, expression): the specified path matches the specified regular expression.""" xml = xml_find(output, path) attribute_value = xml.attrib[attribute] - if re.match(expression, attribute_value) is None: - errmsg = f"Expected attribute '{attribute}' on element with path '{path}' to match '{expression}', instead attribute value was '{attribute_value}'." - raise AssertionError(errmsg) + assert re.match(expression, attribute_value), f"Expected attribute '{attribute}' on element with path '{path}' to match '{expression}', instead attribute value was '{attribute_value}'." def assert_attribute_is(output, path, attribute, text): diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 82c165c7b3a..07e46ce30b5 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2010,86 +2010,112 @@ module. ]]> - - - + + + + + + + + - + + + + + + + + + + + + - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - ``).]]> - - - - - - `` ).]]> - - - - - - ``).]]> - - - - - ``).]]> - - - - - - - - - + - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ``). If the ``text`` is expected to occur a particular number of -times, this value can be specified using ``n``. Optionally also with a certain -``delta``. Alternatively the range of expected occurences can be specified by -``min`` and/or ``max``. +Asserts the output has a specific size (in bytes) of ``value`` plus minus +``delta``, e.g. ````. +Alternatively the range of the expected size can be specified by ``min`` and/or +``max``. ]]> + + + Desired size of the outpyt (in bytes) + + + + + Maximum allowed size difference (default is 0). The observed size has to be in the range ``value +- delta``. + + + + + Minimum expected size + + + + + Maximum expected size + + + + + + ``). If the ``text`` is expected to occur a particular number of times, this value can be specified using ``n``. Optionally also with a certain ``delta``. Alternatively the range of expected occurences can be specified by ``min`` and/or ``max``.]]> + + Text to check for @@ -2313,34 +2339,155 @@ number of columns. - + + + ``). Valid archive formats include ``.zip``, ``.tar``, and ``.tar.gz``.]]> + + + + + + + + + + + The regular expression specifying the archive member. + + + + ``. -Alternatively the range of the expected size can be specified by ``min`` and/or -``max``. +Asserts the output is a valid XML file (e.g. ````). ]]> - + + + + ``). +]]> + + + - Desired size of the outpyt (in bytes) + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - + + + + ``). +]]> + + + - Maximum allowed size difference (default is 0). The observed size has to be in the range ``value +- delta``. + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - + - Minimum expected size + The number of times the path should appear. - + + + + ``). +]]> + + + - Maximum expected size + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. + + + + + The regular expression to use. + + + + + + ``). +]]> + + + + + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. + + + + + Text to check for. + + + + + + ``). +]]> + + + + + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. + + + + + The regular expression to use. + + + + + + `` ). +]]> + + + + + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. + + + + + Text to check for. + + + + + + ``).]]> + + + + + + + + Text to check for @@ -2369,19 +2516,6 @@ Alternatively the range of the expected size can be specified by ``min`` and/or - - - ``). Valid archive formats include ``.zip``, ``.tar``, and ``.tar.gz``.]]> - - - - - - - The regular expression specifying the archive member. - - - - + + @@ -566,7 +567,7 @@ ASSERTS = """ - + @@ -574,7 +575,7 @@ ASSERTS = """ - + diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index e2d6af8aa7b..0bc68f13fde 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -58,6 +58,30 @@ TEXT_HAS_TEXT_ASSERTION_MIN_MAX = """ """ +TEXT_HAS_TEXT_ASSERTION_NEGATE = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_N_NEGATE = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE = """ + + + +""" + +TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE = """ + + + +""" + TEXT_NOT_HAS_TEXT_ASSERTION = """ @@ -139,6 +163,102 @@ TEXT_DATA_NONE = None TEXT_DATA_EMPTY = "" + +XML_IS_VALID_XML_ASSERTION = """ + + + +""" +XML_HAS_ELEMENT_WITH_PATH = """ + + + + + + +""" +XML_HAS_ELEMENT_WITH_PATH = """ + + + + + + +""" +XML_HAS_ELEMENT_WITH_PATH_NEGATIVE = """ + + + +""" +XML_HAS_N_ELEMENTS_WITH_PATH = """ + + + + + + + +""" +XML_HAS_N_ELEMENTS_WITH_PATH_NEGATIVE = """ + + + + +""" +XML_ELEMENT_TEXT_MATCHES = """ + + + + + +""" +XML_ELEMENT_TEXT_MATCHES_NEGATIVE = """ + + + + +""" +XML_ATTRIBUTE_MATCHES = """ + + + + + +""" +XML_ATTRIBUTE_MATCHES_NEGATIVE = """ + + + + +""" +XML_ELEMENT_TEXT = """ + + + + + + +""" +XML_ELEMENT_TEXT_NEGATIVE = """ + + + + + + +""" + +VALID_XML = ''' + + BAR + BAZ + QUX + + + +''' +INVALID_XML = '' + TESTS = [ # test successful assertion ( @@ -160,6 +280,9 @@ TESTS = [ TABULAR_ASSERTION_COMMENT, TABULAR_DATA_COMMENT, lambda x: len(x) == 0 ), + + + # test has_text ( TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT, @@ -168,7 +291,7 @@ TESTS = [ # test has_text .. negative test ( TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "Output file did not contain expected text 'test text' (output 'desired content\nis not here\n')" in x + lambda x: "Expected text 'test text' in output ('desired content\nis not here\n')" in x ), # test has_text with None output ( @@ -178,7 +301,7 @@ TESTS = [ # test has_text with empty output ( TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_EMPTY, - lambda x: "Output file did not contain expected text 'test text' (output '')" in x + lambda x: "Expected text 'test text' in output ('')" in x ), # test has_text with n ( @@ -188,17 +311,17 @@ TESTS = [ # test has_text with n .. negative test ( TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2+-0 occurences of 'test text' in output file (output 'test text\n') found 1" in x + lambda x: "Expected 2+-0 occurences of 'test text' in output ('test text\n') found 1" in x ), - # test has_text with n + # test has_text with n and delta ( TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT_TWO, lambda x: len(x) == 0 ), - # test has_text with n .. negative test + # test has_text with n and delta .. negative test ( TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 3+-1 occurences of 'test text' in output file (output 'test text\n') found 1" in x + lambda x: "Expected 3+-1 occurences of 'test text' in output ('test text\n') found 1" in x ), # test has_text with min max ( @@ -208,8 +331,62 @@ TESTS = [ # test has_text with min max .. negative test ( TEXT_HAS_TEXT_ASSERTION_MIN_MAX, TEXT_DATA_HAS_TEXT, - lambda x: "Expected that the number of occurences of 'test text' in output file is in [2:4] (output 'test text\n') found 1" in x + lambda x: "Expected that the number of occurences of 'test text' in output is in [2:4] ('test text\n') found 1" in x ), + + + # test has_text negate + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_HAS_TEXT_NEG, + lambda x: len(x) == 0 + ), + # test has_text negate .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: "Did not expect text 'test text' in output ('test text\n')" in x + ), + # test has_text negate with None output .. should have the same output as with negate="false" + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_NONE, + lambda x: "Checking has_text assertion on empty output (None)" in x + ), + # test has_text negate with empty output + ( + TEXT_HAS_TEXT_ASSERTION_NEGATE, TEXT_DATA_EMPTY, + lambda x: len(x) == 0 + ), + # test has_text negate with n + ( + TEXT_HAS_TEXT_ASSERTION_N_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text negate with n .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N_NEGATE, TEXT_DATA_HAS_TEXT_TWO, + lambda x: "Did not expect 2+-0 occurences of 'test text' in output ('test text\ntest text\n') found 2" in x + ), + # test has_text negate with n and delta + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text negate with n and delta .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE, TEXT_DATA_HAS_TEXT_TWO, + lambda x: "Did not expect 3+-1 occurences of 'test text' in output ('test text\ntest text\n') found 2" in x + ), + # test has_text negate with min max + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE, TEXT_DATA_HAS_TEXT, + lambda x: len(x) == 0 + ), + # test has_text negate with min max .. negative test + ( + TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE, TEXT_DATA_HAS_TEXT_TWO, + lambda x: "Did not expect that the number of occurences of 'test text' in output is in [2:4] ('test text\ntest text\n') found 2" in x + ), + + # test not_has_text ( TEXT_NOT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT, @@ -238,7 +415,7 @@ TESTS = [ # test has_text_matching .. negative test ( TEXT_HAS_TEXT_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "No text matching expression 'te[sx]t' was found in output file (output 'desired content\nis not here\n')" in x + lambda x: "Expected text matching expression 'te[sx]t' in output ('desired content\nis not here\n')" in x ), # test has_text_matching with n ( @@ -248,7 +425,7 @@ TESTS = [ # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) ( TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 4+-0 (non-overlapping) matches for 'te[sx]t' in output file (output 'test text\n') found 2" in x + lambda x: "Expected 4+-0 (non-overlapping) matches for 'te[sx]t' in output ('test text\n') found 2" in x ), # test has_text_matching with n ( @@ -258,7 +435,7 @@ TESTS = [ # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) ( TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX, TEXT_DATA_HAS_TEXT, - lambda x: "Expected that the number of (non-overlapping) matches for 'te[sx]t' in output file is in [3:5] (output 'test text\n') found 2" in x + lambda x: "Expected that the number of (non-overlapping) matches for 'te[sx]t' in output is in [3:5] ('test text\n') found 2" in x ), # test has_line ( @@ -268,7 +445,7 @@ TESTS = [ # test has_line .. negative test ( TEXT_HAS_LINE_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "No line of output file was 'test text' (output was 'desired content\nis not here\n')" in x + lambda x: "Expected line 'test text' in output ('desired content\nis not here\n')" in x ), # test has_line with n ( @@ -278,7 +455,7 @@ TESTS = [ # test has_line with n .. negative test ( TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2+-0 lines 'test text' in output file (output was 'test text\n') found 1" in x + lambda x: "Expected 2+-0 lines 'test text' in output ('test text\n') found 1" in x ), # test has_n_lines ( @@ -303,7 +480,7 @@ TESTS = [ # test has_line_matching .. negative test ( TEXT_HAS_LINE_MATCHING_ASSERTION, TEXT_DATA_HAS_TEXT_NEG, - lambda x: "No line matching expression 'te[sx]t te[sx]t' was found in output file (output 'desired content\nis not here\n')" in x + lambda x: "Expected line matching expression 'te[sx]t te[sx]t' in output ('desired content\nis not here\n')" in x ), # test has_line_matching n ( @@ -313,7 +490,7 @@ TESTS = [ # test has_line_matching n .. negative test ( TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT, - lambda x: "Expected 2+-0 lines matching for 'te[sx]t te[sx]t' in output file (output 'test text\n') found 1" in x + lambda x: "Expected 2+-0 lines matching for 'te[sx]t te[sx]t' in output ('test text\n') found 1" in x ), # test has_size ( @@ -330,6 +507,66 @@ TESTS = [ SIZE_HAS_SIZE_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT_TWO, lambda x: len(x) == 0 ), + # test is_valid_xml + ( + XML_IS_VALID_XML_ASSERTION, VALID_XML, + lambda x: len(x) == 0 + ), + # test is_valid_xml .. negative test + ( + XML_IS_VALID_XML_ASSERTION, INVALID_XML, + lambda x: 'Expected valid XML, but could not parse output. Opening and ending tag mismatch: elem line 1 and root, line 1, column 31 (, line 1)' in x + ), + # test has_element_with_path + ( + XML_HAS_ELEMENT_WITH_PATH, VALID_XML, + lambda x: len(x) == 0 + ), + # test has_element_with_path .. negative test + ( + XML_HAS_ELEMENT_WITH_PATH_NEGATIVE, VALID_XML, + lambda x: 'Expected to find XML element matching expression ./blah, not such match was found.' in x + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH, VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path .. negative test + ( + XML_HAS_N_ELEMENTS_WITH_PATH_NEGATIVE, VALID_XML, + lambda x: 'Expected to find 1 elements with path ./elem, but 2 were found.' in x + ), + # test element_text_matches + ( + XML_ELEMENT_TEXT_MATCHES, VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_matches .. negative test + ( + XML_ELEMENT_TEXT_MATCHES_NEGATIVE, VALID_XML, + lambda x: "Expected element with path './elem/more' to contain text matching 'QU(X|Y)', instead text 'BAR' was found." in x + ), + # test element_attribute_matches + ( + XML_ATTRIBUTE_MATCHES, VALID_XML, + lambda x: len(x) == 0 + ), + # test element_attribute_matches .. negative test + ( + XML_ATTRIBUTE_MATCHES_NEGATIVE, VALID_XML, + lambda x: "Expected attribute 'name' on element with path './elem/more' to match 'QU(X|Y)', instead attribute value was 'bar'." in x + ), + # test element_text + ( + XML_ELEMENT_TEXT, VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text + ( + XML_ELEMENT_TEXT_NEGATIVE, VALID_XML, + lambda x: "Expected text 'NOTBAR' in output ('BAR')" in x + ), ] TEST_IDS = [ @@ -347,6 +584,16 @@ TEST_IDS = [ 'has_text n delta failure', 'has_text min/max delta success', 'has_text min/max delta failure', + 'has_text negate success', + 'has_text negate failure', + 'has_text negate None output', + 'has_text negate empty output', + 'has_text negate n success', + 'has_text negate n failure', + 'has_text negate n delta success', + 'has_text negate n delta failure', + 'has_text negate min/max delta success', + 'has_text negate min/max delta failure', 'not_has_text success', 'not_has_text failure', 'not_has_text None output', @@ -371,6 +618,18 @@ TEST_IDS = [ 'has_size success', 'has_size failure', 'has_size delta', + 'is_valid_xml success', + 'is_valid_xml failure', + 'has_element_with_path success', + 'has_element_with_path failure', + 'has_n_elements_with_path success', + 'has_n_elements_with_path failure', + 'element_text_matches sucess', + 'element_text_matches failure', + 'attribute_matches sucess', + 'attribute_matches failure', + 'element_text sucess', + 'element_text failure', ] From c1e1fd04823690a213a52e02b020ee491a3d227c Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Dec 2021 11:17:52 +0100 Subject: [PATCH 22/41] add negate attribute to text, size, tabular assertions - assertions messages have been reformulated to make them negatable (which also fixes the unit tests for the previous commit which did parallel edits to the test) --- lib/galaxy/tool_util/verify/asserts/_util.py | 10 +++- lib/galaxy/tool_util/verify/asserts/size.py | 8 +-- .../tool_util/verify/asserts/tabular.py | 8 +-- lib/galaxy/tool_util/verify/asserts/text.py | 59 ++++++++++--------- 4 files changed, 47 insertions(+), 38 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/_util.py b/lib/galaxy/tool_util/verify/asserts/_util.py index 09a319534a9..46e3b635f18 100644 --- a/lib/galaxy/tool_util/verify/asserts/_util.py +++ b/lib/galaxy/tool_util/verify/asserts/_util.py @@ -1,7 +1,9 @@ from math import inf +from galaxy.util import asbool -def _assert_number(count, n, delta, min, max, n_text, min_max_text): + +def _assert_number(count, n, delta, min, max, negate, n_text, min_max_text): """ helper function for assering that count is in - n +- delta @@ -11,8 +13,10 @@ def _assert_number(count, n, delta, min, max, n_text, min_max_text): substituting {n}, {delta}, {min}, and {max} (and keeping potentially present {text} and {output}) """ + negate = asbool(negate) + expected = "Expected" if not negate else "Did not expect" if n is not None: - assert abs(count - int(n)) <= int(delta), n_text.format(n=n, delta=delta, text="{text}", output="{output}") + f" found {count}" + assert (not negate) == (abs(count - int(n)) <= int(delta)), n_text.format(expected=expected, n=n, delta=delta, text="{text}", output="{output}") + f" found {count}" if min is not None or max is not None: if min is None: min = -inf @@ -22,4 +26,4 @@ def _assert_number(count, n, delta, min, max, n_text, min_max_text): max = inf else: max = int(max) - assert min <= count <= max, min_max_text.format(min=min, max=max, text="{text}", output="{output}") + f" found {count}" + assert (not negate) == (min <= count <= max), min_max_text.format(expected=expected, min=min, max=max, text="{text}", output="{output}") + f" found {count}" diff --git a/lib/galaxy/tool_util/verify/asserts/size.py b/lib/galaxy/tool_util/verify/asserts/size.py index acac5e9ca76..2e16d041f37 100644 --- a/lib/galaxy/tool_util/verify/asserts/size.py +++ b/lib/galaxy/tool_util/verify/asserts/size.py @@ -1,12 +1,12 @@ from ._util import _assert_number -def assert_has_size(output_bytes, value: int = None, delta: int = 0, min: int = None, max: int = None): +def assert_has_size(output_bytes, value: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output has a size of the specified value, allowing for absolute (delta) and relative (delta_frac) difference. """ output_size = len(output_bytes) - _assert_number(output_size, value, delta, min, max, - "Expected file size of {n}+-{delta}", - "Expected file size to be in [{min}:{max}]") + _assert_number(output_size, value, delta, min, max, negate, + "{expected} file size of {n}+-{delta}", + "{expected} file size to be in [{min}:{max}]") diff --git a/lib/galaxy/tool_util/verify/asserts/tabular.py b/lib/galaxy/tool_util/verify/asserts/tabular.py index 9bffc7c89f5..c16fe6f4732 100644 --- a/lib/galaxy/tool_util/verify/asserts/tabular.py +++ b/lib/galaxy/tool_util/verify/asserts/tabular.py @@ -17,13 +17,13 @@ def get_first_line(output, comment): return match.group(1) -def assert_has_n_columns(output, n: int = None, delta: int = 0, min: int = None, max: int = None, sep='\t', comment=""): +def assert_has_n_columns(output, n: int = None, delta: int = 0, min: int = None, max: int = None, sep='\t', comment="", negate: bool = False): """ Asserts the tabular output contains n columns. The optional sep argument specifies the column seperator used to determine the number of columns. The optional comment argument specifies comment characters""" first_line = get_first_line(output, comment) n_columns = len(first_line.split(sep)) - _assert_number(n_columns, n, delta, min, max, - "Expected {n}+-{delta} columns in output", - "Expected the number of columns in output to be in [{min}:{max}]") + _assert_number(n_columns, n, delta, min, max, negate, + "{expected} {n}+-{delta} columns in output", + "{expected} the number of columns in output to be in [{min}:{max}]") diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index de1a6cc764f..ab5f40f6898 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -1,9 +1,10 @@ import re +from galaxy.util import asbool from ._util import _assert_number -def _assert_presence_number(output, text, n, delta, min, max, check_presence_foo, count_foo, presence_text, n_text, min_max_text): +def _assert_presence_number(output, text, n, delta, min, max, negate, check_presence_foo, count_foo, presence_text, n_text, min_max_text): """ helper function to assert that - text is present in output using check_presence_foo @@ -13,25 +14,29 @@ def _assert_presence_number(output, text, n, delta, min, max, check_presence_foo raising an assertion error using presence_text, n_text or min_max_text (resp) substituting {n}, {delta}, {min}, {max}, {text}, and {output} """ + print(f"{text} {output} eval {check_presence_foo(output, text)}") + negate = asbool(negate) + expected = "Expected" if not negate else "Did not expect" if n is None and min is None and max is None: - assert check_presence_foo(output, text), presence_text.format(output=output, text=text) + print(f"negate {negate} presence {check_presence_foo(output, text)}") + assert (not negate) == check_presence_foo(output, text), presence_text.format(expected=expected, output=output, text=text) try: - _assert_number(count_foo(output, text), n, delta, min, max, n_text, min_max_text) + _assert_number(count_foo(output, text), n, delta, min, max, negate, n_text, min_max_text) except AssertionError as e: raise AssertionError(str(e).format(output=output, text=text)) -def assert_has_text(output, text, n: int = None, delta: int = 0, min: int = None, max: int = None): +def assert_has_text(output, text, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts specified output contains the substring specified by the argument text. The exact number of occurrences can be optionally specified by the argument n""" assert output is not None, "Checking has_text assertion on empty output (None)" - _assert_presence_number(output, text, n, delta, min, max, + _assert_presence_number(output, text, n, delta, min, max, negate, lambda o, t: o.find(t) >= 0, lambda o, t: len(re.findall(re.escape(t), o)), - "Output file did not contain expected text '{text}' (output '{output}')", - "Expected {n}+-{delta} occurences of '{text}' in output file (output '{output}')", - "Expected that the number of occurences of '{text}' in output file is in [{min}:{max}] (output '{output}')") + "{expected} text '{text}' in output ('{output}')", + "{expected} {n}+-{delta} occurences of '{text}' in output ('{output}')", + "{expected} that the number of occurences of '{text}' in output is in [{min}:{max}] ('{output}')") def assert_not_has_text(output, text): @@ -41,51 +46,51 @@ def assert_not_has_text(output, text): assert output.find(text) < 0, f"Output file contains unexpected text '{text}'" -def assert_has_line(output, line, n: int = None, delta: int = 0, min: int = None, max: int = None): +def assert_has_line(output, line, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output contains the line specified by the argument line. The exact number of occurrences can be optionally specified by the argument n""" assert output is not None, "Checking has_line assertion on empty output (None)" - _assert_presence_number(output, line, n, delta, min, max, + _assert_presence_number(output, line, n, delta, min, max, negate, lambda o, l: re.search(f"^{re.escape(l)}$", o, flags=re.MULTILINE) is not None, lambda o, l: len(re.findall(f"^{re.escape(l)}$", o, flags=re.MULTILINE)), - "No line of output file was '{text}' (output was '{output}')", - "Expected {n}+-{delta} lines '{text}' in output file (output was '{output}')", - "Expected that the number of lines '{text}' in output file is in [{min}:{max}] (output '{output}')") + "{expected} line '{text}' in output ('{output}')", + "{expected} {n}+-{delta} lines '{text}' in output ('{output}')", + "{expected} that the number of lines '{text}' in output is in [{min}:{max}] ('{output}')") -def assert_has_n_lines(output, n: int = None, delta: int = 0, min: int = None, max: int = None): +def assert_has_n_lines(output, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """Asserts the specified output contains ``n`` lines allowing for a difference in the number of lines (delta) or relative differebce in the number of lines""" assert output is not None, "Checking has_n_lines assertion on empty output (None)" count = len(output.splitlines()) - _assert_number(count, n, delta, min, max, - "Expected {n}+-{delta} lines in the output", - "Expected the number of line to be in [{min}:{max}]") + _assert_number(count, n, delta, min, max, negate, + "{expected} {n}+-{delta} lines in the output", + "{expected} the number of line to be in [{min}:{max}]") -def assert_has_text_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None): +def assert_has_text_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output contains text matching the regular expression specified by the argument expression. If n is given the assertion checks for exacly n (nonoverlapping) occurences. """ - _assert_presence_number(output, expression, n, delta, min, max, + _assert_presence_number(output, expression, n, delta, min, max, negate, lambda o, e: re.search(e, o) is not None, lambda o, e: len(re.findall(e, o)), - "No text matching expression '{text}' was found in output file (output '{output}')", - "Expected {n}+-{delta} (non-overlapping) matches for '{text}' in output file (output '{output}')", - "Expected that the number of (non-overlapping) matches for '{text}' in output file is in [{min}:{max}] (output '{output}')") + "{expected} text matching expression '{text}' in output ('{output}')", + "{expected} {n}+-{delta} (non-overlapping) matches for '{text}' in output ('{output}')", + "{expected} that the number of (non-overlapping) matches for '{text}' in output is in [{min}:{max}] ('{output}')") -def assert_has_line_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None): +def assert_has_line_matching(output, expression, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output contains a line matching the regular expression specified by the argument expression. If n is given the assertion checks for exactly n occurences.""" - _assert_presence_number(output, expression, n, delta, min, max, + _assert_presence_number(output, expression, n, delta, min, max, negate, lambda o, e: re.search(f"^{e}$", o, flags=re.MULTILINE) is not None, lambda o, e: len(re.findall(f"^{e}$", o, flags=re.MULTILINE)), - "No line matching expression '{text}' was found in output file (output '{output}')", - "Expected {n}+-{delta} lines matching for '{text}' in output file (output '{output}')", - "Expected that the number of lines matching for '{text}' in output file is in [{min}:{max}] (output '{output}')") + "{expected} line matching expression '{text}' in output ('{output}')", + "{expected} {n}+-{delta} lines matching for '{text}' in output ('{output}')", + "{expected} that the number of lines matching for '{text}' in output is in [{min}:{max}] ('{output}')") From ec4b38493d45198c6971ccf55f820a81e23573db Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Dec 2021 18:56:35 +0100 Subject: [PATCH 23/41] xsd: use AttributeGroup for some assertion attributes to make the xsd more compact and less redundant --- lib/galaxy/tool_util/xsd/galaxy.xsd | 188 ++++++++-------------------- 1 file changed, 55 insertions(+), 133 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 07e46ce30b5..63746e908b6 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2092,7 +2092,7 @@ Alternatively the range of the expected size can be specified by ``min`` and/or - Desired size of the outpyt (in bytes) + Desired size of the output (in bytes) @@ -2110,10 +2110,49 @@ Alternatively the range of the expected size can be specified by ``min`` and/or Maximum expected size + + + + + + Negate the outcome of the assertion. + + + + + + + + Desired number + + + + + Allowed difference with respect to n (default: 0) + + + + + Minimum number (default: -infinity) + + + + + Maximum number (default: infinity) + + + + - ``). If the ``text`` is expected to occur a particular number of times, this value can be specified using ``n``. Optionally also with a certain ``delta``. Alternatively the range of expected occurences can be specified by ``min`` and/or ``max``.]]> + ``). If the ``text`` is expected to occur a particular number of +times, this value can be specified using ``n``. Optionally also with a certain +``delta``. Alternatively the range of expected occurences can be specified by +``min`` and/or ``max``. +]]> @@ -2121,26 +2160,8 @@ Alternatively the range of the expected size can be specified by ``min`` and/or Text to check for - - - Desired number of occurences of the text - - - - - Allowed difference wrt. n - - - - - Minimum number of occurences - - - - - Maximum number of occurences - - + + @@ -2169,26 +2190,8 @@ occurences can be specified by ``min`` and/or ``max``. Regular expression to check for - - - Desired number of non-overlapping matches of the expression - - - - - Allowed difference wrt. n - - - - - Minimum number of occurences - - - - - Maximum number of occurences - - + + @@ -2206,26 +2209,8 @@ occurences can be specified by ``min`` and/or ``max``. The line to check for - - - Desired number of occurences of the line - - - - - Allowed difference wrt. n - - - - - Minimum number of occurences - - - - - Maximum number of occurences - - + + @@ -2237,26 +2222,8 @@ and/or ``max``. ]]> - - - Desired number of lines - - - - - Allowed difference in the number of lines. The observed number of lines has to be in the range ``n +- delta``. - - - - - Minimum number of lines - - - - - Maximum number of lines - - + + @@ -2274,26 +2241,8 @@ of expected occurences can be specified by ``min`` and/or ``max``. Regular expression to check for - - - Desired number of lines matching the expression - - - - - Allowed difference in the number of matching lines. The observed number of matching lines has to be in the range ``n +- delta``. - - - - - Minimum number of matching lines - - - - - Maximum number of matching lines - - + + @@ -2308,36 +2257,8 @@ number of columns. ]]> - - - Desired number of columns - - - - - Column separator - - - - - Comment character(s) - - - - - Allowed difference in the number of columns. The observed number of columns has to be in the range ``n +- delta``. - - - - - Minimum number of columns - - - - - Maximum number of columns - - + + @@ -2503,7 +2424,7 @@ XPath-like ``path`` is the specified ``text`` (e.g. `` - ``).]]> + ``).]]> @@ -2516,6 +2437,7 @@ XPath-like ``path`` is the specified ``text`` (e.g. `` Date: Thu, 23 Dec 2021 18:59:59 +0100 Subject: [PATCH 24/41] add tests for hdf5 and archive asserts and some fixes archive asserts: - check for absent members directly in the assertion - before the _extract... functions returned `None` which had no effect if there is no included assertion (probably this is also the reason why some of the text assertions check for None) - so now the assertion can be used without included sub-assertions to check for member presence - fix for non-file members: the behavior for zip and tar was different hdf5 asserts: - fix assertion text --- .../tool_util/verify/asserts/archive.py | 38 +++-- lib/galaxy/tool_util/verify/asserts/hdf5.py | 3 +- test/unit/tool_util/verify/test_asserts.py | 134 +++++++++++++++++- 3 files changed, 156 insertions(+), 19 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/archive.py b/lib/galaxy/tool_util/verify/asserts/archive.py index 840e93dfcb9..0c4aa6acd66 100644 --- a/lib/galaxy/tool_util/verify/asserts/archive.py +++ b/lib/galaxy/tool_util/verify/asserts/archive.py @@ -6,16 +6,22 @@ import zipfile def _extract_from_tar(tar_temp, path): for fn in tar_temp.getnames(): - if re.match(path, fn): - # Will only match on first hit, probably fine for now - return tar_temp.extractfile(fn) + if not re.match(path, fn): + continue + # Will only match on first hit, probably fine for now + # if called on a dir zip.open returns a handle to an empty string + # in contrast tar.extractfile returns None, the following + # if makes the _extract... functions behave equally in this case + if not tar_temp.getmember(fn).isfile(): + return io.StringIO("") + return tar_temp.extractfile(fn) def _extract_from_zip(zip_temp, path): for fn in zip_temp.namelist(): - if re.match(path, fn): - # Will only match on first hit, probably fine for now - return zip_temp.open(fn) + if not re.match(path, fn): + continue + return zip_temp.open(fn) def assert_has_archive_member(output_bytes, path, verify_assertions_function, children): @@ -23,15 +29,17 @@ def assert_has_archive_member(output_bytes, path, verify_assertions_function, ch the first element matching the specified path found within the archive. Currently supported formats: .zip, .tar, .tar.gz.""" - output_temp = io.BytesIO(output_bytes) - try: # tar / tar.gz - temp = tarfile.open(fileobj=output_temp, mode='r') - contents = _extract_from_tar(temp, path) - except tarfile.TarError: # zip - temp = zipfile.ZipFile(output_temp, mode='r') - contents = _extract_from_zip(temp, path) - finally: + with io.BytesIO(output_bytes) as output_temp: + try: # tar / tar.gz + temp = tarfile.open(fileobj=output_temp, mode='r') + contents = _extract_from_tar(temp, path) + except tarfile.TarError: # zip + try: + temp = zipfile.ZipFile(output_temp, mode='r') + contents = _extract_from_zip(temp, path) + except zipfile.BadZipFile: + raise AssertionError(f"Expected path '{path}' to be an archive") + assert contents is not None, f"Expected path '{path}' in archive" verify_assertions_function(contents.read(), children) contents.close() temp.close() - output_temp.close() diff --git a/lib/galaxy/tool_util/verify/asserts/hdf5.py b/lib/galaxy/tool_util/verify/asserts/hdf5.py index 36f6814c174..e273aa1ae53 100644 --- a/lib/galaxy/tool_util/verify/asserts/hdf5.py +++ b/lib/galaxy/tool_util/verify/asserts/hdf5.py @@ -20,9 +20,10 @@ def assert_has_h5_attribute(output_bytes, key, value): output_temp = io.BytesIO(output_bytes) local_attrs = h5py.File(output_temp, 'r').attrs assert key in local_attrs and str(local_attrs[key]) == value, ( - f"Not a HDF5 file or H5 attributes do not match:\n\t{local_attrs.items()}\n\n\t({key} : {value})") + f"Not a HDF5 file or H5 attributes do not match:\n\t{list(local_attrs.items())}\n\n\t({key} : {value})") +# TODO the function actually queries groups. so the function and argument name are misleading def assert_has_h5_keys(output_bytes, keys): """ Asserts the specified HDF5 output has the given keys.""" _assert_h5py() diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index 0bc68f13fde..8227e4aac88 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -1,3 +1,6 @@ +import tempfile + +import h5py import pytest from galaxy.tool_util.parser.xml import __parse_assert_list_from_elem @@ -259,6 +262,56 @@ VALID_XML = ''' ''' INVALID_XML = '' +with tempfile.NamedTemporaryFile() as tmp: + with h5py.File(tmp.name, "w") as h5fh: + h5fh.attrs['myfileattr'] = "myfileattrvalue" + h5fh.attrs['myfileattrint'] = 1 + dset = h5fh.create_dataset("myint", (100,), dtype='i') + dset.attrs['myintattr'] = "myintattrvalue" + grp = h5fh.create_group("mygroup") + grp.attrs['mygroupattr'] = "mygroupattrvalue" + grp.create_dataset("myfloat", (50,), dtype='f') + dset.attrs['myfloatattr'] = "myfloatattrvalue" + H5BYTES = open(tmp.name, "rb").read() + +H5_HAS_H5_KEYS = """ + + + +""" +H5_HAS_H5_KEYS_NEGATIVE = """ + + + +""" +H5_HAS_ATTRIBUTE = """ + + + + +""" +H5_HAS_ATTRIBUTE_NEGATIVE = """ + + + + +""" + +with open("test-data/example-bag.zip", "rb") as zipfh: + ZIPBYTES = zipfh.read() +with open("test-data/testdir1.tar.gz", "rb") as tarfh: + TARBYTES = tarfh.read() +with open("test-data/1.bed", "rb") as fh: + NONARCHIVE = fh.read() + +ARCHIVE_HAS_ARCHIVE_MEMBER = """ + + + {content_assert} + + +""" + TESTS = [ # test successful assertion ( @@ -280,9 +333,6 @@ TESTS = [ TABULAR_ASSERTION_COMMENT, TABULAR_DATA_COMMENT, lambda x: len(x) == 0 ), - - - # test has_text ( TEXT_HAS_TEXT_ASSERTION, TEXT_DATA_HAS_TEXT, @@ -567,6 +617,71 @@ TESTS = [ XML_ELEMENT_TEXT_NEGATIVE, VALID_XML, lambda x: "Expected text 'NOTBAR' in output ('BAR')" in x ), + # test has_h5_keys + ( + H5_HAS_H5_KEYS, H5BYTES, + lambda x: len(x) == 0 + ), + # test has_h5_keys .. negative + ( + H5_HAS_H5_KEYS_NEGATIVE, H5BYTES, + lambda x: "Not a HDF5 file or H5 keys missing:\n\t['mygroup', 'mygroup/myfloat', 'myint']\n\t['absent']" in x + ), + # test has_attribute + ( + H5_HAS_ATTRIBUTE, H5BYTES, + lambda x: len(x) == 0 + ), + # test has_attribute .. negative + ( + H5_HAS_ATTRIBUTE_NEGATIVE, H5BYTES, + lambda x: "Not a HDF5 file or H5 attributes do not match:\n\t[('myfileattr', 'myfileattrvalue'), ('myfileattrint', 1)]\n\n\t(myfileattr : wrong)" in x + ), + # test has_archive_member with zip + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="test-bag-fetch-http/bag-info.txt", content_assert=""), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir1/dir1/file3", content_assert=""), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with non archive + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="irrelevant", content_assert=""), NONARCHIVE, + lambda x: "Expected path 'irrelevant' to be an archive" in x + ), + # test has_archive_member with zip on absent member + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert=""), ZIPBYTES, + lambda x: "Expected path 'absent' in archive" in x + ), + # test has_archive_member with tar on absent member + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert=""), TARBYTES, + lambda x: "Expected path 'absent' in archive" in x + ), + # test has_archive_member with zip on a dir member + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="test-bag-fetch-http/", content_assert=""), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar on a dir member + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir1/dir1", content_assert=""), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="test-bag-fetch-http/bag-info.txt", content_assert=''), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir1/dir1/file3", content_assert=''), TARBYTES, + lambda x: len(x) == 0 + ), ] TEST_IDS = [ @@ -630,6 +745,19 @@ TEST_IDS = [ 'attribute_matches failure', 'element_text sucess', 'element_text failure', + 'has_h5_keys', + 'has_h5_keys failure', + 'has_h5_attribute', + 'has_h5_attribute failure', + 'has_archive_member zip', + 'has_archive_member tar', + 'has_archive_member non-archive', + 'has_archive_member zip absent member', + 'has_archive_member tar absent member', + 'has_archive_member zip non-file member', + 'has_archive_member tar non-file member', + 'has_archive_member zip with content assertion', + 'has_archive_member tar with content assertion', ] From 381480775c06394c3829f63a6093715adfaa28d6 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Dec 2021 19:41:03 +0100 Subject: [PATCH 25/41] xml assert: element_text check if path exists --- lib/galaxy/tool_util/verify/asserts/xml.py | 1 + test/unit/tool_util/verify/test_asserts.py | 32 ++++++++++++---------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/xml.py b/lib/galaxy/tool_util/verify/asserts/xml.py index b649c599846..4a04866c2fa 100644 --- a/lib/galaxy/tool_util/verify/asserts/xml.py +++ b/lib/galaxy/tool_util/verify/asserts/xml.py @@ -82,4 +82,5 @@ def assert_element_text(output, path, verify_assertions_function, children): """ Recursively checks the specified assertions against the text of the first element matching the specified path.""" text = xml_find_text(output, path) + assert text is not None, f"Expected path '{path}' in xml" verify_assertions_function(text, children) diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index 8227e4aac88..787be6e0375 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -237,16 +237,8 @@ XML_ATTRIBUTE_MATCHES_NEGATIVE = """ XML_ELEMENT_TEXT = """ - - - - -""" -XML_ELEMENT_TEXT_NEGATIVE = """ - - - - + + {content_assert} """ @@ -609,12 +601,22 @@ TESTS = [ ), # test element_text ( - XML_ELEMENT_TEXT, VALID_XML, + XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, lambda x: len(x) == 0 ), - # test element_text + # test element_text .. negative ( - XML_ELEMENT_TEXT_NEGATIVE, VALID_XML, + XML_ELEMENT_TEXT.format(path="./absent", content_assert=''), VALID_XML, + lambda x: "Expected path './absent' in xml" in x + ), + # test element_text with sub-assertion + ( + XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text with sub-assertion .. negative + ( + XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, lambda x: "Expected text 'NOTBAR' in output ('BAR')" in x ), # test has_h5_keys @@ -743,8 +745,10 @@ TEST_IDS = [ 'element_text_matches failure', 'attribute_matches sucess', 'attribute_matches failure', - 'element_text sucess', + 'element_text success', 'element_text failure', + 'element_text with subassertion sucess', + 'element_text with subassertion failure', 'has_h5_keys', 'has_h5_keys failure', 'has_h5_attribute', From 00eb1f45a583ed5e5c154ab36c37f3caa4e6cab1 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Dec 2021 19:42:03 +0100 Subject: [PATCH 26/41] reformulate has_archive_member docs --- lib/galaxy/tool_util/xsd/galaxy.xsd | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 63746e908b6..36cee8ccdf3 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2262,7 +2262,17 @@ number of columns. - ``). Valid archive formats include ``.zip``, ``.tar``, and ``.tar.gz``.]]> + ``). Note that if this member is a directory then it is +considered like an empty file. +]]> @@ -2400,7 +2410,13 @@ XPath-like ``path`` is the specified ``text`` (e.g. `` - ``).]]> + ``). +]]> From 388c99a70425b3fa8a7bcdbb889989bb155d208a Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 23 Dec 2021 20:53:07 +0100 Subject: [PATCH 27/41] conditionally execute h5 assertion tetst if h5py module is available. otherwise package tests are failing --- test/unit/tool_util/verify/test_asserts.py | 131 ++++++++++++--------- 1 file changed, 73 insertions(+), 58 deletions(-) diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index 787be6e0375..bd505e025c3 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -1,6 +1,9 @@ import tempfile -import h5py +try: + import h5py +except ImportError: + h5py = None import pytest from galaxy.tool_util.parser.xml import __parse_assert_list_from_elem @@ -254,40 +257,41 @@ VALID_XML = ''' ''' INVALID_XML = '' -with tempfile.NamedTemporaryFile() as tmp: - with h5py.File(tmp.name, "w") as h5fh: - h5fh.attrs['myfileattr'] = "myfileattrvalue" - h5fh.attrs['myfileattrint'] = 1 - dset = h5fh.create_dataset("myint", (100,), dtype='i') - dset.attrs['myintattr'] = "myintattrvalue" - grp = h5fh.create_group("mygroup") - grp.attrs['mygroupattr'] = "mygroupattrvalue" - grp.create_dataset("myfloat", (50,), dtype='f') - dset.attrs['myfloatattr'] = "myfloatattrvalue" - H5BYTES = open(tmp.name, "rb").read() +if h5py is not None: + with tempfile.NamedTemporaryFile() as tmp: + with h5py.File(tmp.name, "w") as h5fh: + h5fh.attrs['myfileattr'] = "myfileattrvalue" + h5fh.attrs['myfileattrint'] = 1 + dset = h5fh.create_dataset("myint", (100,), dtype='i') + dset.attrs['myintattr'] = "myintattrvalue" + grp = h5fh.create_group("mygroup") + grp.attrs['mygroupattr'] = "mygroupattrvalue" + grp.create_dataset("myfloat", (50,), dtype='f') + dset.attrs['myfloatattr'] = "myfloatattrvalue" + H5BYTES = open(tmp.name, "rb").read() -H5_HAS_H5_KEYS = """ - - - -""" -H5_HAS_H5_KEYS_NEGATIVE = """ - - - -""" -H5_HAS_ATTRIBUTE = """ - - - - -""" -H5_HAS_ATTRIBUTE_NEGATIVE = """ - - - - -""" + H5_HAS_H5_KEYS = """ + + + + """ + H5_HAS_H5_KEYS_NEGATIVE = """ + + + + """ + H5_HAS_ATTRIBUTE = """ + + + + + """ + H5_HAS_ATTRIBUTE_NEGATIVE = """ + + + + + """ with open("test-data/example-bag.zip", "rb") as zipfh: ZIPBYTES = zipfh.read() @@ -619,26 +623,6 @@ TESTS = [ XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, lambda x: "Expected text 'NOTBAR' in output ('BAR')" in x ), - # test has_h5_keys - ( - H5_HAS_H5_KEYS, H5BYTES, - lambda x: len(x) == 0 - ), - # test has_h5_keys .. negative - ( - H5_HAS_H5_KEYS_NEGATIVE, H5BYTES, - lambda x: "Not a HDF5 file or H5 keys missing:\n\t['mygroup', 'mygroup/myfloat', 'myint']\n\t['absent']" in x - ), - # test has_attribute - ( - H5_HAS_ATTRIBUTE, H5BYTES, - lambda x: len(x) == 0 - ), - # test has_attribute .. negative - ( - H5_HAS_ATTRIBUTE_NEGATIVE, H5BYTES, - lambda x: "Not a HDF5 file or H5 attributes do not match:\n\t[('myfileattr', 'myfileattrvalue'), ('myfileattrint', 1)]\n\n\t(myfileattr : wrong)" in x - ), # test has_archive_member with zip ( ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="test-bag-fetch-http/bag-info.txt", content_assert=""), ZIPBYTES, @@ -686,6 +670,32 @@ TESTS = [ ), ] +if h5py is not None: + H5PY_TESTS = [ + # test has_h5_keys + ( + H5_HAS_H5_KEYS, H5BYTES, + lambda x: len(x) == 0 + ), + # test has_h5_keys .. negative + ( + H5_HAS_H5_KEYS_NEGATIVE, H5BYTES, + lambda x: "Not a HDF5 file or H5 keys missing:\n\t['mygroup', 'mygroup/myfloat', 'myint']\n\t['absent']" in x + ), + # test has_attribute + ( + H5_HAS_ATTRIBUTE, H5BYTES, + lambda x: len(x) == 0 + ), + # test has_attribute .. negative + ( + H5_HAS_ATTRIBUTE_NEGATIVE, H5BYTES, + lambda x: "Not a HDF5 file or H5 attributes do not match:\n\t[('myfileattr', 'myfileattrvalue'), ('myfileattrint', 1)]\n\n\t(myfileattr : wrong)" in x + ), + ] + TESTS.extend(H5PY_TESTS) + + TEST_IDS = [ 'has_n_columns success', 'has_n_columns failure', @@ -749,10 +759,6 @@ TEST_IDS = [ 'element_text failure', 'element_text with subassertion sucess', 'element_text with subassertion failure', - 'has_h5_keys', - 'has_h5_keys failure', - 'has_h5_attribute', - 'has_h5_attribute failure', 'has_archive_member zip', 'has_archive_member tar', 'has_archive_member non-archive', @@ -764,6 +770,15 @@ TEST_IDS = [ 'has_archive_member tar with content assertion', ] +if h5py is not None: + H5PY_TEST_IDS = [ + 'has_h5_keys', + 'has_h5_keys failure', + 'has_h5_attribute', + 'has_h5_attribute failure', + ] + TEST_IDS.extend(H5PY_TEST_IDS) + @pytest.mark.parametrize('assertion_xml,data,assert_func', TESTS, ids=TEST_IDS) def test_assertions(assertion_xml, data, assert_func): From 57cd7700a241e32cc80a66131b75f6633a41d62e Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Fri, 24 Dec 2021 10:32:34 +0100 Subject: [PATCH 28/41] test archive assert using no test data but just create mock files on the fly --- test/unit/tool_util/verify/test_asserts.py | 55 ++++++++++++++++------ 1 file changed, 41 insertions(+), 14 deletions(-) diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index bd505e025c3..f4963e31fe9 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -1,4 +1,7 @@ +import os +import tarfile import tempfile +import zipfile try: import h5py @@ -258,7 +261,8 @@ VALID_XML = ''' INVALID_XML = '' if h5py is not None: - with tempfile.NamedTemporaryFile() as tmp: + with tempfile.NamedTemporaryFile(delete=False) as tmp: + h5name = tmp.name with h5py.File(tmp.name, "w") as h5fh: h5fh.attrs['myfileattr'] = "myfileattrvalue" h5fh.attrs['myfileattrint'] = 1 @@ -268,7 +272,9 @@ if h5py is not None: grp.attrs['mygroupattr'] = "mygroupattrvalue" grp.create_dataset("myfloat", (50,), dtype='f') dset.attrs['myfloatattr'] = "myfloatattrvalue" - H5BYTES = open(tmp.name, "rb").read() + with open(h5name, "rb") as h5fh: + H5BYTES = h5fh.read() + os.remove(h5name) H5_HAS_H5_KEYS = """ @@ -293,12 +299,33 @@ if h5py is not None: """ -with open("test-data/example-bag.zip", "rb") as zipfh: - ZIPBYTES = zipfh.read() -with open("test-data/testdir1.tar.gz", "rb") as tarfh: - TARBYTES = tarfh.read() -with open("test-data/1.bed", "rb") as fh: - NONARCHIVE = fh.read() +with tempfile.NamedTemporaryFile(delete=False) as ziptmp: + zipname = ziptmp.name + with zipfile.ZipFile(ziptmp, mode='w') as zipfh: + for f in ["file1", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: + zipfh.writestr(f, f) +with open(zipname, "rb") as zfh: + ZIPBYTES = zfh.read() +os.remove(zipname) + +with tempfile.NamedTemporaryFile(delete=False) as tartmp: + tarname = tartmp.name + with tarfile.open(name=None, mode='w:gz', fileobj=tartmp) as tarfh: + for f in ["file1", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: + with tempfile.NamedTemporaryFile("w") as tmptmp: + tmptmpname = tmptmp.name + tmptmp.write(f) + tarfh.add(tmptmpname, arcname=f) +with open(tarname, "rb") as tfh: + TARBYTES = tfh.read() +os.remove(tarname) + + +with tempfile.NamedTemporaryFile(mode="w", delete=False) as nonarchivetmp: + nonarchivename = nonarchivetmp.name + nonarchivetmp.write("some text") +with open(nonarchivename, "rb") as ntmp: + NONARCHIVE = ntmp.read() ARCHIVE_HAS_ARCHIVE_MEMBER = """ @@ -625,12 +652,12 @@ TESTS = [ ), # test has_archive_member with zip ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="test-bag-fetch-http/bag-info.txt", content_assert=""), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=""), ZIPBYTES, lambda x: len(x) == 0 ), # test has_archive_member with tar ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir1/dir1/file3", content_assert=""), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=""), TARBYTES, lambda x: len(x) == 0 ), # test has_archive_member with non archive @@ -650,22 +677,22 @@ TESTS = [ ), # test has_archive_member with zip on a dir member ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="test-bag-fetch-http/", content_assert=""), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/", content_assert=""), ZIPBYTES, lambda x: len(x) == 0 ), # test has_archive_member with tar on a dir member ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir1/dir1", content_assert=""), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/", content_assert=""), TARBYTES, lambda x: len(x) == 0 ), # test has_archive_member with zip ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="test-bag-fetch-http/bag-info.txt", content_assert=''), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert='testdir/file1.txt'), ZIPBYTES, lambda x: len(x) == 0 ), # test has_archive_member with tar ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir1/dir1/file3", content_assert=''), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert='testdir/file1.txt'), TARBYTES, lambda x: len(x) == 0 ), ] From 33ec189dcb1766db9c8ab71ab2a531e9811a2c09 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 29 Dec 2021 12:16:13 +0100 Subject: [PATCH 29/41] fix/complete archive test for sub assertions --- test/unit/tool_util/verify/test_asserts.py | 28 +++++++++++++++------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index f4963e31fe9..97e3bb8f138 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -302,7 +302,7 @@ if h5py is not None: with tempfile.NamedTemporaryFile(delete=False) as ziptmp: zipname = ziptmp.name with zipfile.ZipFile(ziptmp, mode='w') as zipfh: - for f in ["file1", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: + for f in ["file1.txt", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: zipfh.writestr(f, f) with open(zipname, "rb") as zfh: ZIPBYTES = zfh.read() @@ -311,11 +311,11 @@ os.remove(zipname) with tempfile.NamedTemporaryFile(delete=False) as tartmp: tarname = tartmp.name with tarfile.open(name=None, mode='w:gz', fileobj=tartmp) as tarfh: - for f in ["file1", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: - with tempfile.NamedTemporaryFile("w") as tmptmp: + for f in ["file1.txt", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: + with tempfile.NamedTemporaryFile("w", delete=False) as tmptmp: tmptmpname = tmptmp.name tmptmp.write(f) - tarfh.add(tmptmpname, arcname=f) + tarfh.add(tmptmpname, arcname=f) with open(tarname, "rb") as tfh: TARBYTES = tfh.read() os.remove(tarname) @@ -685,16 +685,26 @@ TESTS = [ ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/", content_assert=""), TARBYTES, lambda x: len(x) == 0 ), - # test has_archive_member with zip + # test has_archive_member with zip with subassertion ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert='testdir/file1.txt'), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), ZIPBYTES, lambda x: len(x) == 0 ), - # test has_archive_member with tar + # test has_archive_member with tar with subassertion ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert='testdir/file1.txt'), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), TARBYTES, lambda x: len(x) == 0 ), + # test has_archive_member with zip with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), ZIPBYTES, + lambda x: "Expected text 'ABSENT' in output ('testdir/file1.txt')" in x + ), + # test has_archive_member with tar with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), TARBYTES, + lambda x: "Expected text 'ABSENT' in output ('testdir/file1.txt')" in x + ), ] if h5py is not None: @@ -795,6 +805,8 @@ TEST_IDS = [ 'has_archive_member tar non-file member', 'has_archive_member zip with content assertion', 'has_archive_member tar with content assertion', + 'has_archive_member zip with failing content assertion', + 'has_archive_member tar with failing content assertion', ] if h5py is not None: From 96de9ed5b51603580cc237438a2526a3d7a4e22b Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 29 Dec 2021 10:33:17 +0100 Subject: [PATCH 30/41] add all attribute to archive asserts allowing to check sub assertions for all archive members matching path --- .../tool_util/verify/asserts/archive.py | 71 +++++++++++-------- lib/galaxy/tool_util/xsd/galaxy.xsd | 5 ++ 2 files changed, 46 insertions(+), 30 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/archive.py b/lib/galaxy/tool_util/verify/asserts/archive.py index 0c4aa6acd66..31c332ddb41 100644 --- a/lib/galaxy/tool_util/verify/asserts/archive.py +++ b/lib/galaxy/tool_util/verify/asserts/archive.py @@ -1,45 +1,56 @@ import io +import os import re import tarfile +import tempfile import zipfile -def _extract_from_tar(tar_temp, path): - for fn in tar_temp.getnames(): - if not re.match(path, fn): - continue - # Will only match on first hit, probably fine for now - # if called on a dir zip.open returns a handle to an empty string - # in contrast tar.extractfile returns None, the following - # if makes the _extract... functions behave equally in this case - if not tar_temp.getmember(fn).isfile(): - return io.StringIO("") - return tar_temp.extractfile(fn) +def _extract_from_tar(temp, path): + with tarfile.open(fileobj=temp, mode='r') as tar_temp: + for fn in tar_temp.getnames(): + if not re.match(path, fn): + continue + # if called on a dir zip.open returns a handle to an empty string + # in contrast tar.extractfile returns None, the following + # if makes the _extract... functions behave equally in this case + if not tar_temp.getmember(fn).isfile(): + yield io.StringIO("") + yield tar_temp.extractfile(fn) -def _extract_from_zip(zip_temp, path): - for fn in zip_temp.namelist(): - if not re.match(path, fn): - continue - return zip_temp.open(fn) +def _extract_from_zip(temp, path): + with zipfile.ZipFile(temp, mode='r') as zip_temp: + for fn in zip_temp.namelist(): + if not re.match(path, fn): + continue + yield zip_temp.open(fn) -def assert_has_archive_member(output_bytes, path, verify_assertions_function, children): +def assert_has_archive_member(output_bytes, path, verify_assertions_function, children, all=False): """ Recursively checks the specified children assertions against the text of the first element matching the specified path found within the archive. Currently supported formats: .zip, .tar, .tar.gz.""" + # from python 3.9 is_tarfile supports file like objects then we do not need + # the tempfile detour but can use io.BytesIO(output_bytes) + with tempfile.NamedTemporaryFile(delete=False) as tmp: + tmpname = tmp.name + tmp.write(output_bytes) + zipfoo = None + if zipfile.is_zipfile(tmp.name): + zipfoo = _extract_from_zip + if tarfile.is_tarfile(tmp.name): + zipfoo = _extract_from_tar + os.remove(tmpname) + assert zipfoo is not None, f"Expected path '{path}' to be an archive" + with io.BytesIO(output_bytes) as output_temp: - try: # tar / tar.gz - temp = tarfile.open(fileobj=output_temp, mode='r') - contents = _extract_from_tar(temp, path) - except tarfile.TarError: # zip - try: - temp = zipfile.ZipFile(output_temp, mode='r') - contents = _extract_from_zip(temp, path) - except zipfile.BadZipFile: - raise AssertionError(f"Expected path '{path}' to be an archive") - assert contents is not None, f"Expected path '{path}' in archive" - verify_assertions_function(contents.read(), children) - contents.close() - temp.close() + haspath = False + for contents in zipfoo(output_temp, path): + haspath = True + verify_assertions_function(contents.read(), children) + contents.close() + if not all: + break + assert haspath, f"Expected path '{path}' in archive" \ No newline at end of file diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 36cee8ccdf3..1d87dfff94a 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2286,6 +2286,11 @@ considered like an empty file. The regular expression specifying the archive member. + + + Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first + + From 4496311bb210f287dac6a95a9eb708cff58715a9 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 30 Dec 2021 19:14:35 +0100 Subject: [PATCH 31/41] followup on xsd restructuring doc parsing: asserts are now choice --- doc/parse_gx_xsd.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py index b0ac29972d6..b3e6e977b03 100644 --- a/doc/parse_gx_xsd.py +++ b/doc/parse_gx_xsd.py @@ -102,8 +102,9 @@ def _build_tag(tag, hide_attributes): assertions_buffer.write(_doc_or_none(assertions_tag)) assertions_buffer.write("\n\n") - assertion_groups = assertions_tag.xpath("xs:sequence/xs:group", namespaces={'xs': 'http://www.w3.org/2001/XMLSchema'}) + assertion_groups = assertions_tag.xpath("xs:choice/xs:group", namespaces={'xs': 'http://www.w3.org/2001/XMLSchema'}) for group in assertion_groups: + sys.stderr.write(f"{group}") ref = group.attrib['ref'] assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='" + ref + "']") doc = _doc_or_none(assertion_tag) @@ -115,7 +116,7 @@ def _build_tag(tag, hide_attributes): doc = _doc_or_none(_type_el(element)) assert doc is not None, "Documentation for %s is empty" % element.attrib["name"] doc = doc.strip() - assertions_buffer.write(f"``{element.attrib['name']}``\n: {doc}\n\n") + assertions_buffer.write(f"#### ``{element.attrib['name']}``:\n\n{doc}\n\n") text = text.replace(line, assertions_buffer.getvalue()) tag_help.write(text) best_practices = _get_bp_link(annotation_el) From 85f0de0d1435cc3ad03668f8a770b98e9f0ca007 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 30 Dec 2021 19:15:43 +0100 Subject: [PATCH 32/41] add n, delta, min, max for archive assertions --- lib/galaxy/tool_util/verify/asserts/_util.py | 22 +++ .../tool_util/verify/asserts/archive.py | 109 +++++++---- lib/galaxy/tool_util/verify/asserts/text.py | 25 +-- lib/galaxy/tool_util/xsd/galaxy.xsd | 49 ++++- test/unit/tool_util/verify/test_asserts.py | 179 ++++++++++++++---- 5 files changed, 276 insertions(+), 108 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/_util.py b/lib/galaxy/tool_util/verify/asserts/_util.py index 46e3b635f18..5a64746310a 100644 --- a/lib/galaxy/tool_util/verify/asserts/_util.py +++ b/lib/galaxy/tool_util/verify/asserts/_util.py @@ -27,3 +27,25 @@ def _assert_number(count, n, delta, min, max, negate, n_text, min_max_text): else: max = int(max) assert (not negate) == (min <= count <= max), min_max_text.format(expected=expected, min=min, max=max, text="{text}", output="{output}") + f" found {count}" + + +def _assert_presence_number(output, text, n, delta, min, max, negate, check_presence_foo, count_foo, presence_text, n_text, min_max_text): + """ + helper function to assert that + - text is present in output using check_presence_foo + this is done only if n, min, and max are None + - text appears a certain number of times, where the count is determined with count foo + + raising an assertion error using presence_text, n_text or min_max_text (resp) + substituting {n}, {delta}, {min}, {max}, {text}, and {output} + """ + print(f"{text} {output} eval {check_presence_foo(output, text)}") + negate = asbool(negate) + expected = "Expected" if not negate else "Did not expect" + if n is None and min is None and max is None: + print(f"negate {negate} presence {check_presence_foo(output, text)}") + assert (not negate) == check_presence_foo(output, text), presence_text.format(expected=expected, output=output, text=text) + try: + _assert_number(count_foo(output, text), n, delta, min, max, negate, n_text, min_max_text) + except AssertionError as e: + raise AssertionError(str(e).format(output=output, text=text)) diff --git a/lib/galaxy/tool_util/verify/asserts/archive.py b/lib/galaxy/tool_util/verify/asserts/archive.py index 31c332ddb41..4a832e79c5c 100644 --- a/lib/galaxy/tool_util/verify/asserts/archive.py +++ b/lib/galaxy/tool_util/verify/asserts/archive.py @@ -1,56 +1,87 @@ import io -import os import re import tarfile import tempfile import zipfile - -def _extract_from_tar(temp, path): - with tarfile.open(fileobj=temp, mode='r') as tar_temp: - for fn in tar_temp.getnames(): - if not re.match(path, fn): - continue - # if called on a dir zip.open returns a handle to an empty string - # in contrast tar.extractfile returns None, the following - # if makes the _extract... functions behave equally in this case - if not tar_temp.getmember(fn).isfile(): - yield io.StringIO("") - yield tar_temp.extractfile(fn) +from galaxy.util import asbool +from ._util import _assert_presence_number -def _extract_from_zip(temp, path): - with zipfile.ZipFile(temp, mode='r') as zip_temp: - for fn in zip_temp.namelist(): - if not re.match(path, fn): - continue - yield zip_temp.open(fn) +def _extract_from_tar(bytes, fn): + with io.BytesIO(bytes) as temp: + with tarfile.open(fileobj=temp, mode='r') as tar_temp: + ti = tar_temp.getmember(fn) + # zip treats directories like empty files. + # so make this consistent for tar + if ti.isdir(): + return "" + with tar_temp.extractfile(fn) as member_fh: + return member_fh.read() -def assert_has_archive_member(output_bytes, path, verify_assertions_function, children, all=False): +def _list_from_tar(bytes, path): + lst = list() + with io.BytesIO(bytes) as temp: + with tarfile.open(fileobj=temp, mode='r') as tar_temp: + for fn in tar_temp.getnames(): + if not re.match(path, fn): + continue + lst.append(fn) + return sorted(lst) + + +def _extract_from_zip(bytes, fn): + with io.BytesIO(bytes) as temp: + with zipfile.ZipFile(temp, mode='r') as zip_temp: + with zip_temp.open(fn) as member_fh: + return member_fh.read() + + +def _list_from_zip(bytes, path): + lst = list() + with io.BytesIO(bytes) as temp: + with zipfile.ZipFile(temp, mode='r') as zip_temp: + for fn in zip_temp.namelist(): + if not re.match(path, fn): + continue + lst.append(fn) + return sorted(lst) + + +def assert_has_archive_member(output_bytes, path, verify_assertions_function, children, all="false", n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Recursively checks the specified children assertions against the text of the first element matching the specified path found within the archive. Currently supported formats: .zip, .tar, .tar.gz.""" - + all = asbool(all) + extract_foo = None # from python 3.9 is_tarfile supports file like objects then we do not need # the tempfile detour but can use io.BytesIO(output_bytes) - with tempfile.NamedTemporaryFile(delete=False) as tmp: - tmpname = tmp.name + with tempfile.NamedTemporaryFile() as tmp: tmp.write(output_bytes) - zipfoo = None - if zipfile.is_zipfile(tmp.name): - zipfoo = _extract_from_zip - if tarfile.is_tarfile(tmp.name): - zipfoo = _extract_from_tar - os.remove(tmpname) - assert zipfoo is not None, f"Expected path '{path}' to be an archive" + tmp.flush() + if zipfile.is_zipfile(tmp.name): + extract_foo = _extract_from_zip + list_foo = _list_from_zip + elif tarfile.is_tarfile(tmp.name): + extract_foo = _extract_from_tar + list_foo = _list_from_tar + assert extract_foo is not None, f"Expected path '{path}' to be an archive" - with io.BytesIO(output_bytes) as output_temp: - haspath = False - for contents in zipfoo(output_temp, path): - haspath = True - verify_assertions_function(contents.read(), children) - contents.close() - if not all: - break - assert haspath, f"Expected path '{path}' in archive" \ No newline at end of file + # get list of matching file names in archive and check against n, delta, + # min, max (slightly abusing the output and text as well as the function + # parameters) + fns = list_foo(output_bytes, path) + _assert_presence_number(None, path, n, delta, min, max, negate, + lambda o, t: len(fns) > 0, + lambda o, t: len(fns), + "{expected} path '{text}' in archive", + "{expected} {n}+-{delta} matches for path '{text}' in archive", + "{expected} that the number of matches for path '{text}' in archive is in [{min}:{max}]") + + # check sub-assertions on members matching path + for fn in fns: + contents = extract_foo(output_bytes, fn) + verify_assertions_function(contents, children) + if not all: + break diff --git a/lib/galaxy/tool_util/verify/asserts/text.py b/lib/galaxy/tool_util/verify/asserts/text.py index ab5f40f6898..6215fa26bc5 100644 --- a/lib/galaxy/tool_util/verify/asserts/text.py +++ b/lib/galaxy/tool_util/verify/asserts/text.py @@ -1,29 +1,6 @@ import re -from galaxy.util import asbool -from ._util import _assert_number - - -def _assert_presence_number(output, text, n, delta, min, max, negate, check_presence_foo, count_foo, presence_text, n_text, min_max_text): - """ - helper function to assert that - - text is present in output using check_presence_foo - this is done only if n, min, and max are None - - text appears a certain number of times, where the count is determined with count foo - - raising an assertion error using presence_text, n_text or min_max_text (resp) - substituting {n}, {delta}, {min}, {max}, {text}, and {output} - """ - print(f"{text} {output} eval {check_presence_foo(output, text)}") - negate = asbool(negate) - expected = "Expected" if not negate else "Did not expect" - if n is None and min is None and max is None: - print(f"negate {negate} presence {check_presence_foo(output, text)}") - assert (not negate) == check_presence_foo(output, text), presence_text.format(expected=expected, output=output, text=text) - try: - _assert_number(count_foo(output, text), n, delta, min, max, negate, n_text, min_max_text) - except AssertionError as e: - raise AssertionError(str(e).format(output=output, text=text)) +from ._util import _assert_number, _assert_presence_number def assert_has_text(output, text, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 1d87dfff94a..1da0775be5e 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2265,13 +2265,47 @@ number of columns. ``). Note that if this member is a directory then it is -considered like an empty file. +the compressed file (remember that "matching" means it is checked if a prefix of +the full path of an archive member is described by the regular expression). +Valid archive formats include ``.zip``, ``.tar``, and ``.tar.gz``. Note that +depending on the archive creation method: + +- full paths of the members may be prefixed with ``./`` +- directories may be treated as empty files + +```xml + +``` + +With ``n`` and ``delta`` (or ``min`` and ``max``) assertions on the number of +archive members matching ``path`` can be expressed. The following could be used, +e.g., to assert an archive containing n±1 elements out of which at least +4 need to have a ``txt`` extension. + +```xml + + +``` + +In addition the tag can contain additional assertions as child elements about +the first member in the archive matching the regular expression ``path``. For +instance + +```xml + + + +``` + +If the ``all`` attribute is set to ``true`` then all archive members are subject +to the assertions. Note that, archive members matching the ``path`` are sorted +alphabetically. + +The ``negate`` attribute of the ``has_archive_member`` assertion only affects +the asserts on the presence and number of matching archive members, but not any +sub-assertions (which can offer the ``negate`` attribute on their own). The +check if the file is an archive at all, which is also done by the function, is +not affected. ]]> @@ -2291,6 +2325,7 @@ considered like an empty file. Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first + diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index 97e3bb8f138..f7e5eb8f729 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -1,7 +1,6 @@ import os -import tarfile +import shutil import tempfile -import zipfile try: import h5py @@ -299,26 +298,29 @@ if h5py is not None: """ -with tempfile.NamedTemporaryFile(delete=False) as ziptmp: - zipname = ziptmp.name - with zipfile.ZipFile(ziptmp, mode='w') as zipfh: - for f in ["file1.txt", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: - zipfh.writestr(f, f) -with open(zipname, "rb") as zfh: - ZIPBYTES = zfh.read() -os.remove(zipname) +# create a test directory structure for zipping +# might also be done directly with the fipfile/tarfile module without creating +# a tmpdir, but its much harder to create empty directories or symlinks +tmpdir = tempfile.mkdtemp() +for f in ["file1.txt", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: + tmpfile = os.path.join(tmpdir, f) + os.makedirs(os.path.dirname(tmpfile), exist_ok=True) + with open(tmpfile, "w") as fh: + fh.write(f) +os.makedirs(os.path.join(tmpdir, "emptydir")) +os.symlink("testdir/file1.txt", os.path.join(tmpdir, "symlink")) -with tempfile.NamedTemporaryFile(delete=False) as tartmp: - tarname = tartmp.name - with tarfile.open(name=None, mode='w:gz', fileobj=tartmp) as tarfh: - for f in ["file1.txt", "testdir/file1.txt", "testdir/file2.txt", "testdir/dir2/file1.txt"]: - with tempfile.NamedTemporaryFile("w", delete=False) as tmptmp: - tmptmpname = tmptmp.name - tmptmp.write(f) - tarfh.add(tmptmpname, arcname=f) -with open(tarname, "rb") as tfh: - TARBYTES = tfh.read() -os.remove(tarname) +with tempfile.NamedTemporaryFile(suffix=".zip", delete=False) as ziptmp: + zipname = ziptmp.name + shutil.make_archive(zipname[:-4], "zip", tmpdir) + with open(zipname, "rb") as zfh: + ZIPBYTES = zfh.read() +with tempfile.NamedTemporaryFile(suffix=".tar.gz", delete=False) as ziptmp: + zipname = ziptmp.name + shutil.make_archive(zipname[:-7], "gztar", tmpdir) + with open(zipname, "rb") as zfh: + TARBYTES = zfh.read() +shutil.rmtree(tmpdir) with tempfile.NamedTemporaryFile(mode="w", delete=False) as nonarchivetmp: @@ -329,7 +331,23 @@ with open(nonarchivename, "rb") as ntmp: ARCHIVE_HAS_ARCHIVE_MEMBER = """ - + + {content_assert} + + +""" + +ARCHIVE_HAS_ARCHIVE_MEMBER_N = """ + + + {content_assert} + + +""" + +ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX = """ + + {content_assert} @@ -652,59 +670,130 @@ TESTS = [ ), # test has_archive_member with zip ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=""), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert="", all="false"), ZIPBYTES, lambda x: len(x) == 0 ), # test has_archive_member with tar ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=""), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert="", all="false"), TARBYTES, lambda x: len(x) == 0 ), # test has_archive_member with non archive ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="irrelevant", content_assert=""), NONARCHIVE, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="irrelevant", content_assert="", all="false"), NONARCHIVE, lambda x: "Expected path 'irrelevant' to be an archive" in x ), # test has_archive_member with zip on absent member ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert=""), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert="", all="false"), ZIPBYTES, lambda x: "Expected path 'absent' in archive" in x ), # test has_archive_member with tar on absent member ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert=""), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="absent", content_assert="", all="false"), TARBYTES, lambda x: "Expected path 'absent' in archive" in x ), - # test has_archive_member with zip on a dir member + # test has_archive_member with zip on symlink ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/", content_assert=""), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?symlink", content_assert='', all="false"), ZIPBYTES, lambda x: len(x) == 0 ), - # test has_archive_member with tar on a dir member + # test has_archive_member with tar on symlink ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/", content_assert=""), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?symlink", content_assert='', all="false"), TARBYTES, lambda x: len(x) == 0 ), - # test has_archive_member with zip with subassertion + # test has_archive_member with zip on a dir member (which are treated like empty files) ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/", content_assert='', all="false"), ZIPBYTES, lambda x: len(x) == 0 ), - # test has_archive_member with tar with subassertion + # test has_archive_member with tar on a dir member (which are treated like empty files) ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/", content_assert='', all="false"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip with subassertion (note that archive members are sorted therefor file1 in dir2 is tested) + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/.*\\.txt", content_assert='', all="false"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar with subassertion (note that archive members are sorted therefor file1 in dir2 is tested) + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/.*\\.txt", content_assert='', all="false"), TARBYTES, lambda x: len(x) == 0 ), # test has_archive_member with zip with failing subassertion ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), ZIPBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert='', all="false"), ZIPBYTES, lambda x: "Expected text 'ABSENT' in output ('testdir/file1.txt')" in x ), # test has_archive_member with tar with failing subassertion ( - ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="testdir/file1.txt", content_assert=''), TARBYTES, + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert='', all="false"), TARBYTES, lambda x: "Expected text 'ABSENT' in output ('testdir/file1.txt')" in x ), + # test has_archive_member with zip checking all matches with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar checking all matches with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip checking all matches with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), ZIPBYTES, + lambda x: "Expected text matching expression 'file1\\.txt' in output ('testdir/file2.txt')" + ), + # test has_archive_member with tar checking all matches with failing subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER.format(path=".*file.\\.txt", content_assert='', all="true"), TARBYTES, + lambda x: "Expected text matching expression 'file1\\.txt' in output ('testdir/file2.txt')" + ), + + # test has_archive_member with zip n+delta with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="3", delta="1"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip n+delta with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="1", delta="1"), ZIPBYTES, + lambda x: "Expected 1+-1 matches for path '.*file.\\.txt' in archive found 4" in x + ), + # test has_archive_member with tar n+delta with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="3", delta="1"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar n+delta with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_N.format(path=".*file.\\.txt", content_assert='', n="1", delta="1"), TARBYTES, + lambda x: "Expected 1+-1 matches for path '.*file.\\.txt' in archive found 4" in x + ), + # test has_archive_member with zip min+max with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="2", max="4"), ZIPBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with zip min+max with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="0", max="2"), ZIPBYTES, + lambda x: "Expected that the number of matches for path '.*file.\\.txt' in archive is in [0:2] found 4" in x + ), + # test has_archive_member with tar min+max with subassertion + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="2", max="4"), TARBYTES, + lambda x: len(x) == 0 + ), + # test has_archive_member with tar min+max with subassertion .. negative + ( + ARCHIVE_HAS_ARCHIVE_MEMBER_MINMAX.format(path=".*file.\\.txt", content_assert='', min="0", max="2"), TARBYTES, + lambda x: "Expected that the number of matches for path '.*file.\\.txt' in archive is in [0:2] found 4" in x + ), ] if h5py is not None: @@ -801,12 +890,26 @@ TEST_IDS = [ 'has_archive_member non-archive', 'has_archive_member zip absent member', 'has_archive_member tar absent member', + 'has_archive_member zip symlink member', + 'has_archive_member tar symlink member', 'has_archive_member zip non-file member', 'has_archive_member tar non-file member', 'has_archive_member zip with content assertion', 'has_archive_member tar with content assertion', 'has_archive_member zip with failing content assertion', 'has_archive_member tar with failing content assertion', + 'has_archive_member zip all matching with content assertion', + 'has_archive_member tar all matching with content assertion', + 'has_archive_member zip all matching with failing content assertion', + 'has_archive_member tar all matching with failing content assertion', + 'has_archive_member zip n + delta and content assertion', + 'has_archive_member zip n + delta failing and content assertion', + 'has_archive_member tar n + delta and content assertion', + 'has_archive_member tar n + delta failing and content assertion', + 'has_archive_member zip min max and content assertion', + 'has_archive_member zip min max failing and content assertion', + 'has_archive_member tar min max and content assertion', + 'has_archive_member tar min max failing and content assertion', ] if h5py is not None: @@ -829,4 +932,4 @@ def test_assertions(assertion_xml, data, assert_func): assert_list = e.args else: assert_list = () - assert assert_func(assert_list) + assert assert_func(assert_list), assert_list From 617a417a61c3e6c64d95a4ba1e681674637eacf9 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 3 Jan 2022 11:15:32 +0100 Subject: [PATCH 33/41] fix element_text_matches and attribute_matches the assertions also accepted that the given text is a prefix of the attribute/text --- lib/galaxy/tool_util/verify/asserts/xml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/xml.py b/lib/galaxy/tool_util/verify/asserts/xml.py index 4a04866c2fa..5bcdb6a0805 100644 --- a/lib/galaxy/tool_util/verify/asserts/xml.py +++ b/lib/galaxy/tool_util/verify/asserts/xml.py @@ -61,7 +61,7 @@ def assert_element_text_matches(output, path, expression): def assert_element_text_is(output, path, text): """ Asserts the text of the first element matching the specified path matches exactly the specified text. """ - assert_element_text_matches(output, path, re.escape(text)) + assert_element_text_matches(output, path, re.escape(text) + "$") def assert_attribute_matches(output, path, attribute, expression): @@ -75,7 +75,7 @@ def assert_attribute_matches(output, path, attribute, expression): def assert_attribute_is(output, path, attribute, text): """ Asserts the specified attribute of the first element matching the specified path matches exactly the specified text.""" - assert_attribute_matches(output, path, attribute, re.escape(text)) + assert_attribute_matches(output, path, attribute, re.escape(text) + "$") def assert_element_text(output, path, verify_assertions_function, children): From 10ac6c8c8cb7f64ef9008d579774acfd61080274 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Mon, 3 Jan 2022 14:21:43 +0100 Subject: [PATCH 34/41] introduce xml_element assertion which generalizes all other xml assertions --- lib/galaxy/tool_util/verify/asserts/_util.py | 2 - lib/galaxy/tool_util/verify/asserts/xml.py | 81 +++++---- lib/galaxy/tool_util/xsd/galaxy.xsd | 56 ++++++ test/unit/tool_util/verify/test_asserts.py | 177 +++++++++++++------ 4 files changed, 226 insertions(+), 90 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/_util.py b/lib/galaxy/tool_util/verify/asserts/_util.py index 5a64746310a..3874c58dbad 100644 --- a/lib/galaxy/tool_util/verify/asserts/_util.py +++ b/lib/galaxy/tool_util/verify/asserts/_util.py @@ -39,11 +39,9 @@ def _assert_presence_number(output, text, n, delta, min, max, negate, check_pres raising an assertion error using presence_text, n_text or min_max_text (resp) substituting {n}, {delta}, {min}, {max}, {text}, and {output} """ - print(f"{text} {output} eval {check_presence_foo(output, text)}") negate = asbool(negate) expected = "Expected" if not negate else "Did not expect" if n is None and min is None and max is None: - print(f"negate {negate} presence {check_presence_foo(output, text)}") assert (not negate) == check_presence_foo(output, text), presence_text.format(expected=expected, output=output, text=text) try: _assert_number(count_foo(output, text), n, delta, min, max, negate, n_text, min_max_text) diff --git a/lib/galaxy/tool_util/verify/asserts/xml.py b/lib/galaxy/tool_util/verify/asserts/xml.py index 5bcdb6a0805..c3f7d11fd69 100644 --- a/lib/galaxy/tool_util/verify/asserts/xml.py +++ b/lib/galaxy/tool_util/verify/asserts/xml.py @@ -2,33 +2,19 @@ import re from lxml.etree import XMLSyntaxError +from galaxy.tool_util.verify import asserts from galaxy.util import ( + asbool, parse_xml_string, unicodify, ) -# Helper functions used to work with XML output. -def to_xml(output): - return parse_xml_string(output) - - -def xml_find_text(output, path): - xml = to_xml(output) - text = xml.findtext(path) - return text - - -def xml_find(output, path): - xml = to_xml(output) - return xml.find(path) - - def assert_is_valid_xml(output): """ Simple assertion that just verifies the specified output is valid XML.""" try: - to_xml(output) + parse_xml_string(output) except XMLSyntaxError as e: raise AssertionError(f"Expected valid XML, but could not parse output. {unicodify(e)}") @@ -38,24 +24,20 @@ def assert_has_element_with_path(output, path): path matching the specified path argument. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information.""" - print(xml_find(output, path)) - assert xml_find(output, path) is not None, f"Expected to find XML element matching expression {path}, not such match was found." + assert_xml_element(output, path) def assert_has_n_elements_with_path(output, path, n): """ Asserts the specified output has exactly n elements matching the path specified.""" - xml = to_xml(output) - n = int(n) - num_elements = len(xml.findall(path)) - assert num_elements == n, f"Expected to find {n} elements with path {path}, but {num_elements} were found." + assert_xml_element(output, path, n=n) def assert_element_text_matches(output, path, expression): """ Asserts the text of the first element matching the specified path matches the specified regular expression.""" - text = xml_find_text(output, path) - assert re.match(expression, text), f"Expected element with path '{path}' to contain text matching '{expression}', instead text '{text}' was found." + sub = {"tag": "has_text_matching", 'attributes': {'expression': expression}} + assert_xml_element(output, path, asserts.verify_assertions, [sub]) def assert_element_text_is(output, path, text): @@ -67,9 +49,8 @@ def assert_element_text_is(output, path, text): def assert_attribute_matches(output, path, attribute, expression): """ Asserts the specified attribute of the first element matching the specified path matches the specified regular expression.""" - xml = xml_find(output, path) - attribute_value = xml.attrib[attribute] - assert re.match(expression, attribute_value), f"Expected attribute '{attribute}' on element with path '{path}' to match '{expression}', instead attribute value was '{attribute_value}'." + sub = {"tag": "has_text_matching", 'attributes': {'expression': expression}} + assert_xml_element(output, path, asserts.verify_assertions, [sub], attribute=attribute) def assert_attribute_is(output, path, attribute, text): @@ -81,6 +62,44 @@ def assert_attribute_is(output, path, attribute, text): def assert_element_text(output, path, verify_assertions_function, children): """ Recursively checks the specified assertions against the text of the first element matching the specified path.""" - text = xml_find_text(output, path) - assert text is not None, f"Expected path '{path}' in xml" - verify_assertions_function(text, children) + assert_xml_element(output, path, verify_assertions_function, children) + + +def assert_xml_element(output, path, verify_assertions_function=None, children=None, attribute=None, all=False, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): + """ + Check if path occurs in the xml. If n and delta or min and max are given + also the number of occurences is checked. + If there are any sub assertions then check them against + - the element's text if attribute is None + - the content of the attribute + If all is True then the sub assertions are checked for all occurences. + """ + children = children or [] + all = asbool(all) + # assert that path is in output (the specified number of times) + + xml = parse_xml_string(output) + asserts._util._assert_presence_number(xml, path, n, delta, min, max, negate, + lambda x, p: x.find(p) is not None, + lambda x, p: len(x.findall(p)), + "{expected} path '{text}' in xml", + "{expected} {n}+-{delta} occurrences of path '{text}' in xml", + "{expected} that the number of occurences of path '{text}' in xml is in [{min}:{max}]") + + # check sub-assertions + if len(children) == 0 or verify_assertions_function is None: + return + for occ in xml.findall(path): + if attribute is None or attribute == "": + content = occ.text + else: + content = occ.attrib[attribute] + try: + verify_assertions_function(content, children) + except AssertionError as e: + if attribute is not None and attribute != "": + raise AssertionError(f"Attribute '{attribute}' on element with path '{path}': {str(e)}") + else: + raise AssertionError(f"Text of element with path '{path}': {str(e)}") + if not all: + break diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 1da0775be5e..77b0ede64b5 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2062,6 +2062,7 @@ module. + @@ -2335,6 +2336,61 @@ Asserts the output is a valid XML file (e.g. ````). + + + + + + + +``` + +With ``negate="true"`` the outcome of the assertions wrt the precence and number +of ``path`` can be negated. If there are any sub assertions then check them against + +- the content of the attribute ``attribute`` +- the element's text if no attribute is given + +```xml + + + + + +``` + +Sub-assertions are not subject to the ``negate`` attribute of ``xml_element``. +If ``all`` is ``true`` then the sub assertions are checked for all occurences. + +Note that all other XML assertions can be expressed by this assertion (Galaxy +also implements the other assertions by calling this one). +]]> + + + + + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. + + + + + Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first + + + + + The name of the attribute to apply sub assertion on. If not given then the element text is used. + + + + + - - - - - -""" -XML_HAS_ELEMENT_WITH_PATH = """ - - - - - - -""" -XML_HAS_ELEMENT_WITH_PATH_NEGATIVE = """ - - + """ XML_HAS_N_ELEMENTS_WITH_PATH = """ - - - - - -""" -XML_HAS_N_ELEMENTS_WITH_PATH_NEGATIVE = """ - - - + """ XML_ELEMENT_TEXT_MATCHES = """ - - - + """ -XML_ELEMENT_TEXT_MATCHES_NEGATIVE = """ +XML_ELEMENT_TEXT_IS = """ - - + """ XML_ATTRIBUTE_MATCHES = """ - - - -""" -XML_ATTRIBUTE_MATCHES_NEGATIVE = """ - - - + """ XML_ELEMENT_TEXT = """ - {content_assert} """ +XML_XML_ELEMENT = """ + + + {content_assert} + + +""" + VALID_XML = ''' BAR @@ -610,43 +582,91 @@ TESTS = [ ), # test has_element_with_path ( - XML_HAS_ELEMENT_WITH_PATH, VALID_XML, + XML_HAS_ELEMENT_WITH_PATH.format(path="./elem[1]/more"), VALID_XML, + lambda x: len(x) == 0 + ), + ( + XML_HAS_ELEMENT_WITH_PATH.format(path="./elem[@name='foo']"), VALID_XML, + lambda x: len(x) == 0 + ), + ( + XML_HAS_ELEMENT_WITH_PATH.format(path=".//more[@name]"), VALID_XML, lambda x: len(x) == 0 ), # test has_element_with_path .. negative test ( - XML_HAS_ELEMENT_WITH_PATH_NEGATIVE, VALID_XML, - lambda x: 'Expected to find XML element matching expression ./blah, not such match was found.' in x + XML_HAS_ELEMENT_WITH_PATH.format(path="./blah"), VALID_XML, + lambda x: "Expected path './blah' in xml" in x ), # test has_n_elements_with_path ( - XML_HAS_N_ELEMENTS_WITH_PATH, VALID_XML, + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem", n="2"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem[1]/more", n="3"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem[@name='foo']/more", n="3"), VALID_XML, + lambda x: len(x) == 0 + ), + # test has_n_elements_with_path + ( + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem[2]/more", n="0"), VALID_XML, lambda x: len(x) == 0 ), # test has_n_elements_with_path .. negative test ( - XML_HAS_N_ELEMENTS_WITH_PATH_NEGATIVE, VALID_XML, - lambda x: 'Expected to find 1 elements with path ./elem, but 2 were found.' in x + XML_HAS_N_ELEMENTS_WITH_PATH.format(path="./elem", n="1"), VALID_XML, + lambda x: "Expected 1+-0 occurrences of path './elem' in xml found 2" in x ), # test element_text_matches ( - XML_ELEMENT_TEXT_MATCHES, VALID_XML, + XML_ELEMENT_TEXT_MATCHES.format(path="./elem/more", expression="BA(R|Z)"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_matches more specific path + ( + XML_ELEMENT_TEXT_MATCHES.format(path="./elem/more[2]", expression="BA(R|Z)"), VALID_XML, lambda x: len(x) == 0 ), # test element_text_matches .. negative test ( - XML_ELEMENT_TEXT_MATCHES_NEGATIVE, VALID_XML, - lambda x: "Expected element with path './elem/more' to contain text matching 'QU(X|Y)', instead text 'BAR' was found." in x + XML_ELEMENT_TEXT_MATCHES.format(path="./elem/more", expression="QU(X|Y)"), VALID_XML, + lambda x: "Text of element with path './elem/more': Expected text matching expression 'QU(X|Y)' in output ('BAR')" in x + ), + # test element_text_is + ( + XML_ELEMENT_TEXT_IS.format(path="./elem/more", text="BAR"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_is with more specific path + ( + XML_ELEMENT_TEXT_IS.format(path="./elem/more[@name='baz']", text="BAZ"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_text_is .. negative test testing that prefix is not accepted + ( + XML_ELEMENT_TEXT_IS.format(path="./elem/more", text="BA"), VALID_XML, + lambda x: "Text of element with path './elem/more': Expected text matching expression 'BA$' in output ('BAR')" in x ), # test element_attribute_matches ( - XML_ATTRIBUTE_MATCHES, VALID_XML, + XML_ATTRIBUTE_MATCHES.format(path="./elem/more", attribute="name", expression="ba(r|z)"), VALID_XML, + lambda x: len(x) == 0 + ), + # test element_attribute_matches with more specific path + ( + XML_ATTRIBUTE_MATCHES.format(path="./elem/more[2]", attribute="name", expression="ba(r|z)"), VALID_XML, lambda x: len(x) == 0 ), # test element_attribute_matches .. negative test ( - XML_ATTRIBUTE_MATCHES_NEGATIVE, VALID_XML, - lambda x: "Expected attribute 'name' on element with path './elem/more' to match 'QU(X|Y)', instead attribute value was 'bar'." in x + XML_ATTRIBUTE_MATCHES.format(path="./elem/more", attribute="name", expression="qu(x|y)"), VALID_XML, + lambda x: "Attribute 'name' on element with path './elem/more': Expected text matching expression 'qu(x|y)' in output ('bar')" in x ), # test element_text ( @@ -666,8 +686,36 @@ TESTS = [ # test element_text with sub-assertion .. negative ( XML_ELEMENT_TEXT.format(path="./elem/more", content_assert=''), VALID_XML, - lambda x: "Expected text 'NOTBAR' in output ('BAR')" in x + lambda x: "Text of element with path './elem/more': Expected text 'NOTBAR' in output ('BAR')" in x ), + # note that xml_element is also tested indirectly by the other xml + # assertions which are all implemented by xml_element + # test xml_element + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="1", min="1", max="3", attribute="", all="false", content_assert='', negate="false"), VALID_XML, + lambda x: len(x) == 0 + ), + # test xml_element testing attribute matching on all matching elements + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="1", min="1", max="3", attribute="name", all="true", content_assert='', negate="false"), VALID_XML, + lambda x: len(x) == 0 + ), + # test xml_element .. failing because of n + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="0", min="1", max="3", attribute="", all="false", content_assert='', negate="false"), VALID_XML, + lambda x: "Expected 2+-0 occurrences of path './/more' in xml found 3" in x + ), + # test xml_element .. failing because of n + ( + XML_XML_ELEMENT.format(path=".//more", n="10000", delta="1", min="1", max="3", attribute="", all="false", content_assert='', negate="true"), VALID_XML, + lambda x: "Did not expect that the number of occurences of path './/more' in xml is in [1:3] found 3" in x + ), + # test xml_element .. failing because of sub assertion + ( + XML_XML_ELEMENT.format(path=".//more", n="2", delta="1", min="1", max="3", attribute="", all="false", content_assert='', negate="false"), VALID_XML, + lambda x: "Text of element with path './/more': Did not expect text matching expression '(BA[RZ]|QUX)$' in output ('BAR')" in x + ), + # test has_archive_member with zip ( ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert="", all="false"), ZIPBYTES, @@ -873,18 +921,33 @@ TEST_IDS = [ 'has_size delta', 'is_valid_xml success', 'is_valid_xml failure', - 'has_element_with_path success', + 'has_element_with_path success 1', + 'has_element_with_path success 2', + 'has_element_with_path success 3', 'has_element_with_path failure', - 'has_n_elements_with_path success', + 'has_n_elements_with_path success 1', + 'has_n_elements_with_path success 2', + 'has_n_elements_with_path success 3', + 'has_n_elements_with_path success 4', 'has_n_elements_with_path failure', 'element_text_matches sucess', + 'element_text_matches sucess (with more specific path)', 'element_text_matches failure', + 'element_text_is sucess', + 'element_text_is sucess (with more specific path)', + 'element_text_is failure', 'attribute_matches sucess', + 'attribute_matches sucess (with more specific path)', 'attribute_matches failure', 'element_text success', 'element_text failure', 'element_text with subassertion sucess', 'element_text with subassertion failure', + 'xml_element matching text success', + 'xml_element matching attribute success', + 'xml_element failure (due to n)', + 'xml_element failure (due to min/max in combination with negate)', + 'xml_element failure (due to subassertion)', 'has_archive_member zip', 'has_archive_member tar', 'has_archive_member non-archive', From f3a20a4b687980f4b95ad30ca798c44a4a1cfa24 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Jan 2022 10:08:17 +0100 Subject: [PATCH 35/41] more specific error message for archive assertions and - has_n_elements_with_path add delta, min, max - negate for all xml assertions (except is_valid) --- .../tool_util/verify/asserts/archive.py | 5 +- lib/galaxy/tool_util/verify/asserts/xml.py | 28 +-- lib/galaxy/tool_util/xsd/galaxy.xsd | 215 ++++++++++-------- test/unit/tool_util/verify/test_asserts.py | 4 +- 4 files changed, 141 insertions(+), 111 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/archive.py b/lib/galaxy/tool_util/verify/asserts/archive.py index 4a832e79c5c..fd5caa58f77 100644 --- a/lib/galaxy/tool_util/verify/asserts/archive.py +++ b/lib/galaxy/tool_util/verify/asserts/archive.py @@ -82,6 +82,9 @@ def assert_has_archive_member(output_bytes, path, verify_assertions_function, ch # check sub-assertions on members matching path for fn in fns: contents = extract_foo(output_bytes, fn) - verify_assertions_function(contents, children) + try: + verify_assertions_function(contents, children) + except AssertionError as e: + raise AssertionError(f"Archive member '{path}': {str(e)}") if not all: break diff --git a/lib/galaxy/tool_util/verify/asserts/xml.py b/lib/galaxy/tool_util/verify/asserts/xml.py index c3f7d11fd69..b8842384b4f 100644 --- a/lib/galaxy/tool_util/verify/asserts/xml.py +++ b/lib/galaxy/tool_util/verify/asserts/xml.py @@ -19,50 +19,50 @@ def assert_is_valid_xml(output): raise AssertionError(f"Expected valid XML, but could not parse output. {unicodify(e)}") -def assert_has_element_with_path(output, path): +def assert_has_element_with_path(output, path, negate: bool = False): """ Asserts the specified output has at least one XML element with a path matching the specified path argument. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information.""" - assert_xml_element(output, path) + assert_xml_element(output, path, negate=negate) -def assert_has_n_elements_with_path(output, path, n): +def assert_has_n_elements_with_path(output, path, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): """ Asserts the specified output has exactly n elements matching the path specified.""" - assert_xml_element(output, path, n=n) + assert_xml_element(output, path, n=n, delta=delta, min=min, max=max, negate=negate) -def assert_element_text_matches(output, path, expression): +def assert_element_text_matches(output, path, expression, negate: bool = False): """ Asserts the text of the first element matching the specified path matches the specified regular expression.""" - sub = {"tag": "has_text_matching", 'attributes': {'expression': expression}} + sub = {"tag": "has_text_matching", 'attributes': {'expression': expression, 'negate': negate}} assert_xml_element(output, path, asserts.verify_assertions, [sub]) -def assert_element_text_is(output, path, text): +def assert_element_text_is(output, path, text, negate: bool = False): """ Asserts the text of the first element matching the specified path matches exactly the specified text. """ - assert_element_text_matches(output, path, re.escape(text) + "$") + assert_element_text_matches(output, path, re.escape(text) + "$", negate=negate) -def assert_attribute_matches(output, path, attribute, expression): +def assert_attribute_matches(output, path, attribute, expression, negate: bool = False): """ Asserts the specified attribute of the first element matching the specified path matches the specified regular expression.""" - sub = {"tag": "has_text_matching", 'attributes': {'expression': expression}} + sub = {"tag": "has_text_matching", 'attributes': {'expression': expression, 'negate': negate}} assert_xml_element(output, path, asserts.verify_assertions, [sub], attribute=attribute) -def assert_attribute_is(output, path, attribute, text): +def assert_attribute_is(output, path, attribute, text, negate: bool = False): """ Asserts the specified attribute of the first element matching the specified path matches exactly the specified text.""" - assert_attribute_matches(output, path, attribute, re.escape(text) + "$") + assert_attribute_matches(output, path, attribute, re.escape(text) + "$", negate=negate) -def assert_element_text(output, path, verify_assertions_function, children): +def assert_element_text(output, path, verify_assertions_function, children, negate: bool = False): """ Recursively checks the specified assertions against the text of the first element matching the specified path.""" - assert_xml_element(output, path, verify_assertions_function, children) + assert_xml_element(output, path, verify_assertions_function, children, negate=negate) def assert_xml_element(output, path, verify_assertions_function=None, children=None, attribute=None, all=False, n: int = None, delta: int = 0, min: int = None, max: int = None, negate: bool = False): diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 77b0ede64b5..05c433dad08 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1772,7 +1772,6 @@ provides a demonstration of using this tag. ``` - ]]> @@ -2113,38 +2112,6 @@ Alternatively the range of the expected size can be specified by ``min`` and/or - - - - - Negate the outcome of the assertion. - - - - - - - - Desired number - - - - - Allowed difference with respect to n (default: 0) - - - - - Minimum number (default: -infinity) - - - - - Maximum number (default: infinity) - - - - - - - Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - - + - Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first + Check the sub-assertions for all paths matching the path. Default: false, i.e. only the first - The name of the attribute to apply sub assertion on. If not given then the element text is used. + The name of the attribute to apply sub-assertion on. If not given then the element text is used @@ -2395,134 +2358,159 @@ also implements the other assertions by calling this one). ``). +XPath-like ``path``, e.g. + +```xml + +``` + +With ``negate`` the result of the assertion can be inverted. ]]> - - - Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - - + + ``). +Asserts the XML output contains the specified number (``n``, optionally with ``delta``) of elements (or +tags) with the specified XPath-like ``path``, e.g. + +```xml + +``` + +Alternatively to ``n`` and ``delta`` also the ``min`` and ``max`` attributes +can be used to specify the range of the expected number of occurences. +With ``negate`` the result of the assertion can be inverted. ]]> - - - Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - - - - - The number of times the path should appear. - - + + + ``). +matches the regular expression defined by ``expression``, e.g. + +```xml + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the matching) can be inverted (the +implicit assertion on the existence of the path is not affected). ]]> - - - Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - - + The regular expression to use. + ``). +the specified ``text``, e.g. + +```xml + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the equality) can be inverted (the +implicit assertion on the existence of the path is not affected). ]]> - - - Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - - + Text to check for. + ``). +XPath-like ``path`` matches the regular expression specified by ``expression``, e.g. + +```xml + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the matching) can be inverted (the +implicit assertion on the existence of the path is not affected). ]]> - - - Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - - + The regular expression to use. + `` ). +XPath-like ``path`` is the specified ``text``, e.g. + +```xml + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the assertion (on the equality) can be inverted (the +implicit assertion on the existence of the path is not affected). ]]> - - - Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. - - + Text to check for. + ``). +XPath-like ``path``, e.g. +```xml + + +``` + +The assertion implicitly also asserts that an element matching ``path`` exists. +With ``negate`` the result of the implicit assertions can be inverted. +The sub-assertions, which have their own ``negate`` attribute, are not affected +by ``negate``. + ]]> - - - Text to check for - - + + @@ -2549,6 +2537,45 @@ text="EDK72998.1" />``). + + + + + Path to check for. Valid paths are the simplified subsets of XPath implemented by lxml.etree; https://lxml.de/xpathxslt.html for more information. + + + + + + + + Negate the outcome of the assertion. + + + + + + + + Desired number + + + + + Allowed difference with respect to n (default: 0) + + + + + Minimum number (default: -infinity) + + + + + Maximum number (default: infinity) + + + diff --git a/test/unit/tool_util/verify/test_asserts.py b/test/unit/tool_util/verify/test_asserts.py index b040578c3f5..d95ae6d78d2 100644 --- a/test/unit/tool_util/verify/test_asserts.py +++ b/test/unit/tool_util/verify/test_asserts.py @@ -774,12 +774,12 @@ TESTS = [ # test has_archive_member with zip with failing subassertion ( ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert='', all="false"), ZIPBYTES, - lambda x: "Expected text 'ABSENT' in output ('testdir/file1.txt')" in x + lambda x: "Archive member '(\\./)?testdir/file1.txt': Expected text 'ABSENT' in output ('testdir/file1.txt')" in x ), # test has_archive_member with tar with failing subassertion ( ARCHIVE_HAS_ARCHIVE_MEMBER.format(path="(\\./)?testdir/file1.txt", content_assert='', all="false"), TARBYTES, - lambda x: "Expected text 'ABSENT' in output ('testdir/file1.txt')" in x + lambda x: "Archive member '(\\./)?testdir/file1.txt': Expected text 'ABSENT' in output ('testdir/file1.txt')" in x ), # test has_archive_member with zip checking all matches with subassertion ( From d89987c74842d4638a0f62ece12b16a3917432c3 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Tue, 4 Jan 2022 19:14:55 +0100 Subject: [PATCH 36/41] schema doc: add attribute table to all assertions --- doc/parse_gx_xsd.py | 30 +++++++++---- lib/galaxy/tool_util/xsd/galaxy.xsd | 66 +++++++++++++++++++++++++++-- 2 files changed, 84 insertions(+), 12 deletions(-) diff --git a/doc/parse_gx_xsd.py b/doc/parse_gx_xsd.py index b3e6e977b03..382c7eee8ce 100644 --- a/doc/parse_gx_xsd.py +++ b/doc/parse_gx_xsd.py @@ -90,12 +90,8 @@ def _build_tag(tag, hide_attributes): tag_help = StringIO() annotation_el = tag_el.find("{http://www.w3.org/2001/XMLSchema}annotation") text = annotation_el.find("{http://www.w3.org/2001/XMLSchema}documentation").text + text = _replace_attribute_list(tag, text, attributes) for line in text.splitlines(): - if line.startswith("$attribute_list:"): - attributes_str, header_level = line.split(":")[1:3] - attribute_names = attributes_str.split(",") - header_level = int(header_level) - text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) if line.startswith("$assertions"): assertions_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType[@name='TestAssertions']") assertions_buffer = StringIO() @@ -104,7 +100,6 @@ def _build_tag(tag, hide_attributes): assertion_groups = assertions_tag.xpath("xs:choice/xs:group", namespaces={'xs': 'http://www.w3.org/2001/XMLSchema'}) for group in assertion_groups: - sys.stderr.write(f"{group}") ref = group.attrib['ref'] assertion_tag = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}group[@name='" + ref + "']") doc = _doc_or_none(assertion_tag) @@ -116,6 +111,10 @@ def _build_tag(tag, hide_attributes): doc = _doc_or_none(_type_el(element)) assert doc is not None, "Documentation for %s is empty" % element.attrib["name"] doc = doc.strip() + + element_el = _find_tag_el(element) + element_attributes = _find_attributes(element_el) + doc = _replace_attribute_list(element_el, doc, element_attributes) assertions_buffer.write(f"#### ``{element.attrib['name']}``:\n\n{doc}\n\n") text = text.replace(line, assertions_buffer.getvalue()) tag_help.write(text) @@ -130,6 +129,20 @@ element [here](%s).""" % best_practices) return tag_help.getvalue() +def _replace_attribute_list(tag, text, attributes): + for line in text.splitlines(): + if not line.startswith("$attribute_list:"): + continue + attributes_str, header_level = line.split(":")[1:3] + if attributes_str == "": + attribute_names = None + else: + attribute_names = attributes_str.split(",") + header_level = int(header_level) + text = text.replace(line, _build_attributes_table(tag, attributes, attribute_names=attribute_names, header_level=header_level)) + return text + + def _get_bp_link(annotation_el): anchor = annotation_el.attrib.get("{http://galaxyproject.org/xml/1.0}best_practices", None) link = None @@ -195,8 +208,9 @@ def _find_tag_el(tag): def _type_el(tag): element_type = tag.attrib["type"] - type_el = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType/[@name='%s']" % element_type) or \ - xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}simpleType/[@name='%s']" % element_type) + type_el = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}complexType/[@name='%s']" % element_type) + if type_el is None: + type_el = xmlschema_doc.find("//{http://www.w3.org/2001/XMLSchema}simpleType/[@name='%s']" % element_type) return type_el diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 05c433dad08..69eea6b401b 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2087,6 +2087,8 @@ Asserts the output has a specific size (in bytes) of ``value`` plus minus ``delta``, e.g. ````. Alternatively the range of the expected size can be specified by ``min`` and/or ``max``. + +$attribute_list::5 ]]> @@ -2120,6 +2122,8 @@ text="chr7">``). If the ``text`` is expected to occur a particular number of times, this value can be specified using ``n``. Optionally also with a certain ``delta``. Alternatively the range of expected occurences can be specified by ``min`` and/or ``max``. + +$attribute_list::5 ]]> @@ -2133,7 +2137,12 @@ times, this value can be specified using ``n``. Optionally also with a certain - ``).]]> + ``). + +$attribute_list::5 +]]> + @@ -2150,6 +2159,8 @@ regular expression is expected to match a particular number of times, this value can be specified using ``n``. Note only non-overlapping occurences are counted. Optionally also with a certain ``delta``. Alternatively the range of expected occurences can be specified by ``min`` and/or ``max``. + +$attribute_list::5 ]]> @@ -2169,6 +2180,8 @@ Asserts a line matching the specified string (``line``) appears in the output to occur a particular number of times, this value can be specified using ``n``. Optionally also with a certain ``delta``. Alternatively the range of expected occurences can be specified by ``min`` and/or ``max``. + +$attribute_list::5 ]]> @@ -2187,6 +2200,8 @@ Asserts that an output contains ``n`` lines, allowing for a difference of ``delta`` (default is 0), e.g. ````. Alternatively the range of expected occurences can be specified by ``min`` and/or ``max``. + +$attribute_list::5 ]]> @@ -2201,6 +2216,8 @@ appears in the output (e.g. ````) optionally also with ``\t``) `and comment character(s) can be specified (``comment``, default is empty string), then the first non-comment line is used for determining the number of columns. + +$attribute_list::5 ]]> @@ -2274,6 +2293,8 @@ the asserts on the presence and number of matching archive members, but not any sub-assertions (which can offer the ``negate`` attribute on their own). The check if the file is an archive at all, which is also done by the function, is not affected. + +$attribute_list::5 ]]> @@ -2299,6 +2320,8 @@ not affected. ``). + +$attribute_list::5 ]]> @@ -2337,6 +2360,8 @@ If ``all`` is ``true`` then the sub assertions are checked for all occurences. Note that all other XML assertions can be expressed by this assertion (Galaxy also implements the other assertions by calling this one). + +$attribute_list::5 ]]> @@ -2365,6 +2390,8 @@ XPath-like ``path``, e.g. ``` With ``negate`` the result of the assertion can be inverted. + +$attribute_list::5 ]]> @@ -2384,6 +2411,8 @@ tags) with the specified XPath-like ``path``, e.g. Alternatively to ``n`` and ``delta`` also the ``min`` and ``max`` attributes can be used to specify the range of the expected number of occurences. With ``negate`` the result of the assertion can be inverted. + +$attribute_list::5 ]]> @@ -2404,6 +2433,8 @@ matches the regular expression defined by ``expression``, e.g. The assertion implicitly also asserts that an element matching ``path`` exists. With ``negate`` the result of the assertion (on the matching) can be inverted (the implicit assertion on the existence of the path is not affected). + +$attribute_list::5 ]]> @@ -2428,6 +2459,8 @@ the specified ``text``, e.g. The assertion implicitly also asserts that an element matching ``path`` exists. With ``negate`` the result of the assertion (on the equality) can be inverted (the implicit assertion on the existence of the path is not affected). + +$attribute_list::5 ]]> @@ -2452,6 +2485,8 @@ XPath-like ``path`` matches the regular expression specified by ``expression``, The assertion implicitly also asserts that an element matching ``path`` exists. With ``negate`` the result of the assertion (on the matching) can be inverted (the implicit assertion on the existence of the path is not affected). + +$attribute_list::5 ]]> @@ -2476,6 +2511,8 @@ XPath-like ``path`` is the specified ``text``, e.g. The assertion implicitly also asserts that an element matching ``path`` exists. With ``negate`` the result of the assertion (on the equality) can be inverted (the implicit assertion on the existence of the path is not affected). + +$attribute_list::5 ]]> @@ -2493,16 +2530,19 @@ implicit assertion on the existence of the path is not affected). This tag allows the developer to recurisively specify additional assertions as child elements about just the text contained in the element specified by the XPath-like ``path``, e.g. + ```xml -``` + +``` The assertion implicitly also asserts that an element matching ``path`` exists. With ``negate`` the result of the implicit assertions can be inverted. The sub-assertions, which have their own ``negate`` attribute, are not affected by ``negate``. +$attribute_list::5 ]]> @@ -2514,7 +2554,16 @@ by ``negate``. - ``).]]> + +``` +$attribute_list::5 +]]> + @@ -2524,7 +2573,16 @@ by ``negate``. - ``).]]> + +``` + +$attribute_list::5 +]]> + From f8d0a71e6a2ce40b071e1ac29e2cf0dce46f5a12 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 5 Jan 2022 10:39:43 +0100 Subject: [PATCH 37/41] xsd fix --- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 69eea6b401b..befeb5096a2 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2495,8 +2495,8 @@ $attribute_list::5 The regular expression to use. - + From a86b05e6acdb1a615197035d3b37e112ffa88284 Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Wed, 5 Jan 2022 12:49:23 +0100 Subject: [PATCH 38/41] allow byte suffixed for numbers in asserts --- lib/galaxy/tool_util/verify/asserts/_util.py | 23 +++++--- lib/galaxy/tool_util/xsd/galaxy.xsd | 40 ++++++++----- test/unit/tool_util/verify/test_asserts.py | 62 ++++++++++++-------- 3 files changed, 78 insertions(+), 47 deletions(-) diff --git a/lib/galaxy/tool_util/verify/asserts/_util.py b/lib/galaxy/tool_util/verify/asserts/_util.py index 3874c58dbad..98d63e74a5b 100644 --- a/lib/galaxy/tool_util/verify/asserts/_util.py +++ b/lib/galaxy/tool_util/verify/asserts/_util.py @@ -1,32 +1,39 @@ from math import inf from galaxy.util import asbool +from galaxy.util.bytesize import parse_bytesize def _assert_number(count, n, delta, min, max, negate, n_text, min_max_text): """ helper function for assering that count is in - - n +- delta - - min:max + - [n-delta:n+delta] + - [min:max] raising an assertion error using n_text and min_max_text (resp) substituting {n}, {delta}, {min}, and {max} (and keeping potentially present {text} and {output}) + + n, delta, min, max can be suffixed by (K|M|G|T|P|E)i? """ negate = asbool(negate) expected = "Expected" if not negate else "Did not expect" if n is not None: - assert (not negate) == (abs(count - int(n)) <= int(delta)), n_text.format(expected=expected, n=n, delta=delta, text="{text}", output="{output}") + f" found {count}" + n_bytes = parse_bytesize(n) + delta_bytes = parse_bytesize(delta) + assert (not negate) == (abs(count - n_bytes) <= delta_bytes), n_text.format(expected=expected, n=n, delta=delta, text="{text}", output="{output}") + f" found {count}" if min is not None or max is not None: if min is None: - min = -inf + min = -inf # also replacing min/max for output + min_bytes = -inf else: - min = int(min) + min_bytes = parse_bytesize(min) if max is None: max = inf + max_bytes = inf else: - max = int(max) - assert (not negate) == (min <= count <= max), min_max_text.format(expected=expected, min=min, max=max, text="{text}", output="{output}") + f" found {count}" + max_bytes = parse_bytesize(max) + assert (not negate) == (min_bytes <= count <= max_bytes), min_max_text.format(expected=expected, min=min, max=max, text="{text}", output="{output}") + f" found {count}" def _assert_presence_number(output, text, n, delta, min, max, negate, check_presence_foo, count_foo, presence_text, n_text, min_max_text): @@ -38,6 +45,8 @@ def _assert_presence_number(output, text, n, delta, min, max, negate, check_pres raising an assertion error using presence_text, n_text or min_max_text (resp) substituting {n}, {delta}, {min}, {max}, {text}, and {output} + + n, delta, min, max can be suffixed by (K|M|G|T|P|E)i? """ negate = asbool(negate) expected = "Expected" if not negate else "Did not expect" diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index befeb5096a2..12aab318d76 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -2092,24 +2092,24 @@ $attribute_list::5 ]]> - + - Desired size of the output (in bytes) + Desired size of the output (in bytes), can be suffixed by ``(k|M|G|T|P|E)i?`` - + - Maximum allowed size difference (default is 0). The observed size has to be in the range ``value +- delta``. + Maximum allowed size difference (default is 0). The observed size has to be in the range ``value +- delta``. Can be suffixed by ``(k|M|G|T|P|E)i?`` - + - Minimum expected size + Minimum expected size, can be suffixed by ``(k|M|G|T|P|E)i?`` - + - Maximum expected size + Maximum expected size, can be suffixed by ``(k|M|G|T|P|E)i?`` @@ -2613,24 +2613,24 @@ $attribute_list::5 - + - Desired number + Desired number, can be suffixed by ``(k|M|G|T|P|E)i?`` - + - Allowed difference with respect to n (default: 0) + Allowed difference with respect to n (default: 0), can be suffixed by ``(k|M|G|T|P|E)i?`` - + - Minimum number (default: -infinity) + Minimum number (default: -infinity), can be suffixed by ``(k|M|G|T|P|E)i?`` - + - Maximum number (default: infinity) + Maximum number (default: infinity), can be suffixed by ``(k|M|G|T|P|E)i?`` @@ -6775,6 +6775,14 @@ and ``contains``. In addtion there is ``sim_size`` which is discouraged in fafou + + + Number of bytes alowing for suffix (k|K|M|G|P|E)i? + + + + + - + """ TEXT_HAS_N_LINES_ASSERTION_DELTA = """ - + """ TEXT_HAS_LINE_MATCHING_ASSERTION = """ @@ -147,22 +147,18 @@ TEXT_HAS_LINE_MATCHING_ASSERTION_N = """ SIZE_HAS_SIZE_ASSERTION = """ - + """ SIZE_HAS_SIZE_ASSERTION_DELTA = """ - + """ TEXT_DATA_HAS_TEXT = """test text """ -TEXT_DATA_HAS_TEXT_TWO = """test text -test text -""" - TEXT_DATA_HAS_TEXT_NEG = """desired content is not here """ @@ -368,7 +364,7 @@ TESTS = [ ), # test has_text with n ( - TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), # test has_text with n .. negative test @@ -378,7 +374,7 @@ TESTS = [ ), # test has_text with n and delta ( - TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_ASSERTION_N_DELTA, TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), # test has_text with n and delta .. negative test @@ -388,7 +384,7 @@ TESTS = [ ), # test has_text with min max ( - TEXT_HAS_TEXT_ASSERTION_MIN_MAX, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_ASSERTION_MIN_MAX, TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), # test has_text with min max .. negative test @@ -425,7 +421,7 @@ TESTS = [ ), # test has_text negate with n .. negative test ( - TEXT_HAS_TEXT_ASSERTION_N_NEGATE, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_ASSERTION_N_NEGATE, TEXT_DATA_HAS_TEXT * 2, lambda x: "Did not expect 2+-0 occurences of 'test text' in output ('test text\ntest text\n') found 2" in x ), # test has_text negate with n and delta @@ -435,7 +431,7 @@ TESTS = [ ), # test has_text negate with n and delta .. negative test ( - TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_ASSERTION_N_DELTA_NEGATE, TEXT_DATA_HAS_TEXT * 2, lambda x: "Did not expect 3+-1 occurences of 'test text' in output ('test text\ntest text\n') found 2" in x ), # test has_text negate with min max @@ -445,7 +441,7 @@ TESTS = [ ), # test has_text negate with min max .. negative test ( - TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_ASSERTION_MIN_MAX_NEGATE, TEXT_DATA_HAS_TEXT * 2, lambda x: "Did not expect that the number of occurences of 'test text' in output is in [2:4] ('test text\ntest text\n') found 2" in x ), @@ -482,7 +478,7 @@ TESTS = [ ), # test has_text_matching with n ( - TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) @@ -492,7 +488,7 @@ TESTS = [ ), # test has_text_matching with n ( - TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_TEXT_MATCHING_ASSERTION_MINMAX, TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), # test has_text_matching with n .. negative test (using the test text where "te[sx]st" appears twice) @@ -512,7 +508,7 @@ TESTS = [ ), # test has_line with n ( - TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_LINE_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), # test has_line with n .. negative test @@ -522,17 +518,22 @@ TESTS = [ ), # test has_n_lines ( - TEXT_HAS_N_LINES_ASSERTION, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_N_LINES_ASSERTION.format(n="2"), TEXT_DATA_HAS_TEXT * 2, + lambda x: len(x) == 0 + ), + # test has_n_lines .. bytes + ( + TEXT_HAS_N_LINES_ASSERTION.format(n="2ki"), TEXT_DATA_HAS_TEXT * 2048, lambda x: len(x) == 0 ), # test has_n_lines .. negative test ( - TEXT_HAS_N_LINES_ASSERTION, TEXT_DATA_HAS_TEXT, + TEXT_HAS_N_LINES_ASSERTION.format(n="2"), TEXT_DATA_HAS_TEXT, lambda x: "Expected 2+-0 lines in the output found 1" in x ), # test has_n_lines ..delta ( - TEXT_HAS_N_LINES_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT, + TEXT_HAS_N_LINES_ASSERTION_DELTA.format(n="3", delta="1"), TEXT_DATA_HAS_TEXT, lambda x: "Expected 3+-1 lines in the output found 1" in x ), # test has_line_matching @@ -547,7 +548,7 @@ TESTS = [ ), # test has_line_matching n ( - TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT_TWO, + TEXT_HAS_LINE_MATCHING_ASSERTION_N, TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), # test has_line_matching n .. negative test @@ -557,19 +558,29 @@ TESTS = [ ), # test has_size ( - SIZE_HAS_SIZE_ASSERTION, TEXT_DATA_HAS_TEXT, + SIZE_HAS_SIZE_ASSERTION.format(value=10), TEXT_DATA_HAS_TEXT, lambda x: len(x) == 0 ), # test has_size .. negative test ( - SIZE_HAS_SIZE_ASSERTION, TEXT_DATA_HAS_TEXT_TWO, + SIZE_HAS_SIZE_ASSERTION.format(value="10"), TEXT_DATA_HAS_TEXT * 2, lambda x: "Expected file size of 10+-0 found 20" in x ), # test has_size .. delta ( - SIZE_HAS_SIZE_ASSERTION_DELTA, TEXT_DATA_HAS_TEXT_TWO, + SIZE_HAS_SIZE_ASSERTION_DELTA.format(value="10", delta="10"), TEXT_DATA_HAS_TEXT * 2, lambda x: len(x) == 0 ), + # test has_size .. bytes suffix + ( + SIZE_HAS_SIZE_ASSERTION_DELTA.format(value="1k", delta="0"), TEXT_DATA_HAS_TEXT * 100, + lambda x: len(x) == 0 + ), + # test has_size .. bytes suffix .. negative + ( + SIZE_HAS_SIZE_ASSERTION_DELTA.format(value="1Mi", delta="10k"), TEXT_DATA_HAS_TEXT * 100, + lambda x: 'Expected file size of 1Mi+-10k found 1000' in x + ), # test is_valid_xml ( XML_IS_VALID_XML_ASSERTION, VALID_XML, @@ -910,6 +921,7 @@ TEST_IDS = [ 'has_line n success', 'has_line n failure', 'has_n_lines success', + 'has_n_lines n as bytes success', 'has_n_lines failure', 'has_n_lines delta', 'has_line_matching success', @@ -919,6 +931,8 @@ TEST_IDS = [ 'has_size success', 'has_size failure', 'has_size delta', + 'has_size with bytes suffix', + 'has_size with bytes suffix failure', 'is_valid_xml success', 'is_valid_xml failure', 'has_element_with_path success 1', From b128f77293a8fea9b70b99fd1a27cb3915f7c3e8 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 17 Jan 2022 20:15:03 +0100 Subject: [PATCH 39/41] Update lib/galaxy/tool_util/verify/asserts/__init__.py Co-authored-by: Marius van den Beek --- lib/galaxy/tool_util/verify/asserts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/asserts/__init__.py b/lib/galaxy/tool_util/verify/asserts/__init__.py index 49cd0267b47..f5690b821ab 100644 --- a/lib/galaxy/tool_util/verify/asserts/__init__.py +++ b/lib/galaxy/tool_util/verify/asserts/__init__.py @@ -37,7 +37,7 @@ def verify_assertions(data, assertion_description_list): def verify_assertion(data, assertion_description): tag = assertion_description["tag"] assert_function_name = "assert_" + tag - assert_function = assertion_functions.get(assert_function_name, None) + assert_function = assertion_functions.get(assert_function_name) if assert_function is None: errmsg = f"Unable to find test function associated with XML tag {tag}. Check your tool file syntax." From 5899e3c6806287c9c0a52135b2adb506d7389ea5 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 17 Jan 2022 20:16:18 +0100 Subject: [PATCH 40/41] Update lib/galaxy/tool_util/verify/asserts/__init__.py --- lib/galaxy/tool_util/verify/asserts/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/asserts/__init__.py b/lib/galaxy/tool_util/verify/asserts/__init__.py index f5690b821ab..df54c4cdbbf 100644 --- a/lib/galaxy/tool_util/verify/asserts/__init__.py +++ b/lib/galaxy/tool_util/verify/asserts/__init__.py @@ -14,7 +14,7 @@ assertion_module_names = ['text', 'tabular', 'xml', 'hdf5', 'archive', 'size'] # to the list of assertion module names defined above. assertion_functions = {} for assertion_module_name in assertion_module_names: - full_assertion_module_name = 'galaxy.tool_util.verify.asserts.' + assertion_module_name + full_assertion_module_name = f"galaxy.tool_util.verify.asserts.{assertion_module_name}" try: # Dynamically import module __import__(full_assertion_module_name) From 79b00df1acc39cb27260919b3338c3ff6d4fccbc Mon Sep 17 00:00:00 2001 From: Matthias Bernt Date: Thu, 20 Jan 2022 12:07:21 +0100 Subject: [PATCH 41/41] fix rebase errors --- test/unit/tool_util/test_tool_linters.py | 34 +++++++++++------------- 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/test/unit/tool_util/test_tool_linters.py b/test/unit/tool_util/test_tool_linters.py index 6ae96efdfee..cee5d85f5d5 100644 --- a/test/unit/tool_util/test_tool_linters.py +++ b/test/unit/tool_util/test_tool_linters.py @@ -911,6 +911,20 @@ TESTS = [ '1 outputs found.' in x.info_messages and len(x.info_messages) == 1 and len(x.valid_messages) == 0 and len(x.warn_messages) == 0 and len(x.error_messages) == 0 ), + ( + ASSERTS, tests.lint_tsts, + lambda x: + 'Test 1: unknown assertion invalid' in x.error_messages + and 'Test 1: unknown attribute invalid_attrib for has_text' in x.error_messages + and 'Test 1: missing attribute text for has_text' in x.error_messages + and 'Test 1: attribute value for has_size needs to be int got 500k' in x.error_messages + and 'Test 1: attribute delta for has_size needs to be int got 1O' in x.error_messages + and 'Test 1: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages + and "Test 1: has_size needs to specify 'n', 'min', or 'max'" in x.error_messages + and "Test 1: has_n_columns needs to specify 'n', 'min', or 'max'" in x.error_messages + and "Test 1: has_n_lines needs to specify 'n', 'min', or 'max'" in x.error_messages + and len(x.warn_messages) == 0 and len(x.error_messages) == 9 + ), ( REPEATS, inputs.lint_repeats, lambda x: @@ -992,22 +1006,6 @@ TESTS = [ 'Unknown tag [wrong_tag] encountered, this may result in a warning in the future.' in x.info_messages and 'Best practice violation [stdio] elements should come before [command]' in x.warn_messages and len(x.info_messages) == 1 and len(x.valid_messages) == 0 and len(x.warn_messages) == 1 and len(x.error_messages) == 0 - "Test 1: Cannot specify outputs in a test expecting failure." in x.error_messages - and len(x.warn_messages) == 0 and len(x.error_messages) == 1 - ), - ( - ASSERTS, tests.lint_tsts, - lambda x: - 'Test 1: unknown assertion invalid' in x.error_messages - and 'Test 1: unknown attribute invalid_attrib for has_text' in x.error_messages - and 'Test 1: missing attribute text for has_text' in x.error_messages - and 'Test 1: attribute value for has_size needs to be int got 500k' in x.error_messages - and 'Test 1: attribute delta for has_size needs to be int got 1O' in x.error_messages - and 'Test 1: unknown attribute invalid_attrib_also_checked_in_nested_asserts for not_has_text' in x.error_messages - and "Test 1: has_size needs to specify 'n', 'min', or 'max'" in x.error_messages - and "Test 1: has_n_columns needs to specify 'n', 'min', or 'max'" in x.error_messages - and "Test 1: has_n_lines needs to specify 'n', 'min', or 'max'" in x.error_messages - and len(x.warn_messages) == 0 and len(x.error_messages) == 9 ), ] @@ -1051,6 +1049,7 @@ TEST_IDS = [ 'outputs: format="input"', 'outputs collection static elements with format_source', 'outputs discover datatsets with tool provided metadata', + 'outputs: asserts', 'repeats', 'stdio: default for default profile', 'stdio: default for non-legacy profile', @@ -1061,8 +1060,7 @@ TEST_IDS = [ 'tests: without expectations', 'tests: param and output names', 'tests: expecting failure with outputs', - 'xml_order' - 'asserts' + 'xml_order', ]