From 10c2715d7ab57f76477b3e3fab72bd6cadeb9767 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 24 Feb 2020 15:44:31 +0100 Subject: [PATCH 1/8] make delta (sim_size) default consistent --- lib/galaxy/tool_util/verify/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 0a2a554da0d..0b36227e840 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -131,7 +131,7 @@ def verify( elif compare == 're_match_multiline': files_re_match_multiline(local_name, temp_name, attributes=attributes) elif compare == 'sim_size': - delta = attributes.get('delta', '100') + delta = attributes.get('delta', '10000') s1 = len(output_content) s2 = os.path.getsize(local_name) if abs(s1 - s2) > int(delta): From e47ab7a13aac8f1d85fac267f98a761fd5554e3e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 24 Feb 2020 17:42:30 +0100 Subject: [PATCH 2/8] use a constant to specify default --- lib/galaxy/tool_util/parser/util.py | 2 ++ lib/galaxy/tool_util/parser/xml.py | 2 ++ lib/galaxy/tool_util/verify/__init__.py | 3 ++- 3 files changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/parser/util.py b/lib/galaxy/tool_util/parser/util.py index 41c25186c14..b2cdfd2bac3 100644 --- a/lib/galaxy/tool_util/parser/util.py +++ b/lib/galaxy/tool_util/parser/util.py @@ -1,5 +1,7 @@ from collections import OrderedDict +DEFAULT_DELTA = "10000" + def is_dict(item): return isinstance(item, dict) or isinstance(item, OrderedDict) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index f38301762e0..90a3ec45083 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -9,6 +9,7 @@ from math import isinf import packaging.version from galaxy.tool_util.deps import requirements +from galaxy.tool_util.parser import DEFAULT_DELTA from galaxy.util import string_as_bool, xml_text, xml_to_string from .interface import ( InputSource, @@ -613,6 +614,7 @@ def __parse_test_attributes(output_elem, attrib, parse_elements=False, parse_dis attributes['lines_diff'] = int(attrib.pop('lines_diff', '0')) # Allow a file size to vary if sim_size compare attributes['delta'] = int(attrib.pop('delta', '10000')) + attributes['delta'] = int(attrib.pop('delta', DEFAULT_DELTA)) attributes['sort'] = string_as_bool(attrib.pop('sort', False)) attributes['decompress'] = string_as_bool(attrib.pop('decompress', False)) extra_files = [] diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 0b36227e840..5c8bb414f29 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -16,6 +16,7 @@ try: except ImportError: pysam = None +from galaxy.tool_util.parser import DEFAULT_DELTA from galaxy.util import unicodify from galaxy.util.compression_utils import get_fileobj from .asserts import verify_assertions @@ -131,7 +132,7 @@ def verify( elif compare == 're_match_multiline': files_re_match_multiline(local_name, temp_name, attributes=attributes) elif compare == 'sim_size': - delta = attributes.get('delta', '10000') + delta = int(attributes.get('delta', DEFAULT_DELTA)) s1 = len(output_content) s2 = os.path.getsize(local_name) if abs(s1 - s2) > int(delta): From 0e98dbd778fbef80a2c48bc3fd2816b24cbe22de Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 24 Feb 2020 17:57:28 +0100 Subject: [PATCH 3/8] fix imports --- lib/galaxy/tool_util/parser/xml.py | 2 +- lib/galaxy/tool_util/verify/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 90a3ec45083..9bdc060cfa1 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -9,7 +9,7 @@ from math import isinf import packaging.version from galaxy.tool_util.deps import requirements -from galaxy.tool_util.parser import DEFAULT_DELTA +from galaxy.tool_util.parser.util import DEFAULT_DELTA from galaxy.util import string_as_bool, xml_text, xml_to_string from .interface import ( InputSource, diff --git a/lib/galaxy/tool_util/verify/__init__.py b/lib/galaxy/tool_util/verify/__init__.py index 5c8bb414f29..966f13f9060 100644 --- a/lib/galaxy/tool_util/verify/__init__.py +++ b/lib/galaxy/tool_util/verify/__init__.py @@ -16,7 +16,7 @@ try: except ImportError: pysam = None -from galaxy.tool_util.parser import DEFAULT_DELTA +from galaxy.tool_util.parser.util import DEFAULT_DELTA from galaxy.util import unicodify from galaxy.util.compression_utils import get_fileobj from .asserts import verify_assertions From dc3c9009f13b2b944c301ff1766ba3b6afccf12e Mon Sep 17 00:00:00 2001 From: M Bernt Date: Mon, 24 Feb 2020 18:30:37 +0100 Subject: [PATCH 4/8] also use the default for yaml parser --- lib/galaxy/tool_util/parser/yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index c271a9a1cc6..ce2a910c4d0 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -1,8 +1,8 @@ from collections import OrderedDict import packaging.version - from galaxy.tool_util.deps import requirements +from galaxy.tool_util.parser.util import DEFAULT_DELTA from .interface import ( InputSource, PageSource, @@ -218,7 +218,7 @@ def _parse_test(i, test_dict): defaults = { 'compare': 'diff', 'lines_diff': 0, - 'delta': 1000, + 'delta': DEFAULT_DELTA, 'sort': False, } # TODO From 28aaa8834445071096635bb378d6ae79e5856c6b Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 26 Feb 2020 10:37:01 +0100 Subject: [PATCH 5/8] apply int to default also in yaml parser --- lib/galaxy/tool_util/parser/yaml.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index ce2a910c4d0..bcdd9528c9b 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -218,7 +218,7 @@ def _parse_test(i, test_dict): defaults = { 'compare': 'diff', 'lines_diff': 0, - 'delta': DEFAULT_DELTA, + 'delta': int(DEFAULT_DELTA), 'sort': False, } # TODO From cfeff793782c9a984e8f0b64c1b68323bc0dc265 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 26 Feb 2020 12:51:56 +0100 Subject: [PATCH 6/8] remove remnant line Co-Authored-By: Nicola Soranzo --- lib/galaxy/tool_util/parser/xml.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/tool_util/parser/xml.py b/lib/galaxy/tool_util/parser/xml.py index 9bdc060cfa1..110524b4659 100644 --- a/lib/galaxy/tool_util/parser/xml.py +++ b/lib/galaxy/tool_util/parser/xml.py @@ -613,7 +613,6 @@ def __parse_test_attributes(output_elem, attrib, parse_elements=False, parse_dis # Number of lines to allow to vary in logs (for dates, etc) attributes['lines_diff'] = int(attrib.pop('lines_diff', '0')) # Allow a file size to vary if sim_size compare - attributes['delta'] = int(attrib.pop('delta', '10000')) attributes['delta'] = int(attrib.pop('delta', DEFAULT_DELTA)) attributes['sort'] = string_as_bool(attrib.pop('sort', False)) attributes['decompress'] = string_as_bool(attrib.pop('decompress', False)) From 45eb41a80eddb842f0d41ae218d82dbaa43437f5 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 26 Feb 2020 12:53:56 +0100 Subject: [PATCH 7/8] add default to docs and restore empty line to separate third-party from "internal" imports --- lib/galaxy/tool_util/parser/yaml.py | 1 + lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/parser/yaml.py b/lib/galaxy/tool_util/parser/yaml.py index bcdd9528c9b..4ee12798997 100644 --- a/lib/galaxy/tool_util/parser/yaml.py +++ b/lib/galaxy/tool_util/parser/yaml.py @@ -1,6 +1,7 @@ from collections import OrderedDict import packaging.version + from galaxy.tool_util.deps import requirements from galaxy.tool_util.parser.util import DEFAULT_DELTA from .interface import ( diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 6ae0e3e102c..5d4fd5253d2 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1314,7 +1314,7 @@ When this attribute is true and ``compare`` is set to ``diff``, try to decompres - If ``compare`` is set to ``sim_size``, this is the number of bytes different allowed. + If ``compare`` is set to ``sim_size``, this is the number of bytes different allowed (default 10.000). From 04515f11bd5d5a3f046f6247445d9f843bb597e6 Mon Sep 17 00:00:00 2001 From: M Bernt Date: Wed, 26 Feb 2020 12:57:23 +0100 Subject: [PATCH 8/8] Correct number Co-Authored-By: Nicola Soranzo --- lib/galaxy/tool_util/xsd/galaxy.xsd | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/tool_util/xsd/galaxy.xsd b/lib/galaxy/tool_util/xsd/galaxy.xsd index 5d4fd5253d2..a812c6dc04d 100644 --- a/lib/galaxy/tool_util/xsd/galaxy.xsd +++ b/lib/galaxy/tool_util/xsd/galaxy.xsd @@ -1314,7 +1314,7 @@ When this attribute is true and ``compare`` is set to ``diff``, try to decompres - If ``compare`` is set to ``sim_size``, this is the number of bytes different allowed (default 10.000). + If ``compare`` is set to ``sim_size``, this is the number of bytes different allowed (default 10000).