From a398127ede922d504ec7f1c39e66df22a9f0a16d Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 23 May 2019 13:43:57 +0200 Subject: [PATCH] Work with bytestrings in `files_contains`, `files_re_match_*` But read utf-8 unicode in `lines_diff`. This will replace non-utf8 characters and seems like a reasonable tradeoff for text files mixed with byte contents, like pdfs. --- lib/galaxy/tools/verify/__init__.py | 31 ++++++++++++++++------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/tools/verify/__init__.py b/lib/galaxy/tools/verify/__init__.py index 6e873818b28..a58cb39bcf2 100644 --- a/lib/galaxy/tools/verify/__init__.py +++ b/lib/galaxy/tools/verify/__init__.py @@ -3,7 +3,6 @@ import difflib import filecmp import hashlib -import io import logging import os import os.path @@ -206,6 +205,9 @@ def files_diff(file1, file2, attributes=None): compressed_formats = None else: compressed_formats = [] + + # Open expected contents and history data in binary mode and run lines through unicodify, + # which will replace non utf-8 characters. local_file = [unicodify(l) for l in get_fileobj(file1, mode='rb', compressed_formats=compressed_formats)] history_data = [unicodify(l) for l in get_fileobj(file2, mode='rb', compressed_formats=compressed_formats)] is_pdf = file1.endswith('.pdf') or file2.endswith('.pdf') @@ -261,9 +263,9 @@ def files_diff(file1, file2, attributes=None): def files_re_match(file1, file2, attributes=None): """Check the contents of 2 files for differences using re.match.""" - local_file = io.open(file1, 'r', encoding='utf-8', errors='replace').readlines() # regex file - history_data = io.open(file2, 'r', encoding='utf-8', errors='replace').readlines() - assert len(local_file) == len(history_data), 'Data File and Regular Expression File contain a different number of lines (%d != %d)\nHistory Data (first 40 lines):\n%s' % (len(local_file), len(history_data), ''.join(history_data[:40])) + local_file = open(file1, 'rb').readlines() # regex file + history_data = open(file2, 'rb').readlines() + assert len(local_file) == len(history_data), 'Data File and Regular Expression File contain a different number of lines (%d != %d)\nHistory Data (first 40 lines):\n%s' % (len(local_file), len(history_data), ''.join(unicodify(history_data[:40]))) if attributes is None: attributes = {} if attributes.get('sort', False): @@ -272,24 +274,25 @@ def files_re_match(file1, file2, attributes=None): line_diff_count = 0 diffs = [] for regex_line, data_line in zip(local_file, history_data): - if not re.match(regex_line.rstrip('\r\n'), data_line.rstrip('\r\n')): + if not re.match(regex_line.rstrip(b'\r\n'), data_line.rstrip(b'\r\n')): line_diff_count += 1 - diffs.append('Regular Expression: %s, Data file: %s\n' % (regex_line.rstrip('\r\n'), data_line.rstrip('\r\n'))) + diffs.append('Regular Expression: %s, Data file: %s\n' % (unicodify(regex_line).rstrip('\r\n'), + unicodify(data_line).rstrip('\r\n'))) if line_diff_count > lines_diff: raise AssertionError("Regular expression did not match data file (allowed variants=%i):\n%s" % (lines_diff, "".join(diffs))) def files_re_match_multiline(file1, file2, attributes=None): """Check the contents of 2 files for differences using re.match in multiline mode.""" - local_file = io.open(file1, 'r', encoding='utf-8', errors='replace').read() # regex file + local_file = open(file1, 'rb').read() # regex file if attributes is None: attributes = {} if attributes.get('sort', False): - history_data = io.open(file2, 'r', encoding='utf-8', errors='replace').readlines() + history_data = open(file2, 'rb').readlines() history_data.sort() - history_data = ''.join(history_data) + history_data = b''.join(history_data) else: - history_data = io.open(file2, 'r', encoding='utf-8', errors='replace').read() + history_data = open(file2, 'rb').read() # lines_diff not applicable to multiline matching assert re.match(local_file, history_data, re.MULTILINE), "Multiline Regular expression did not match data file" @@ -298,14 +301,14 @@ def files_contains(file1, file2, attributes=None): """Check the contents of file2 for substrings found in file1, on a per-line basis.""" if attributes is None: attributes = {} - local_file = io.open(file1, 'r', encoding='utf-8', errors='replace').readlines() # regex file + local_file = open(file1, 'rb').readlines() # regex file # TODO: allow forcing ordering of contains - history_data = io.open(file2, 'r', encoding='utf-8', errors='replace').read() + history_data = open(file2, 'rb').read() lines_diff = int(attributes.get('lines_diff', 0)) line_diff_count = 0 for contains in local_file: - contains = contains.rstrip('\r\n') + contains = contains.rstrip(b'\r\n') if contains not in history_data: line_diff_count += 1 if line_diff_count > lines_diff: - raise AssertionError("Failed to find '%s' in history data. (lines_diff=%i):\n" % (contains, lines_diff)) + raise AssertionError("Failed to find '%s' in history data. (lines_diff=%i):\n" % (unicodify(contains), lines_diff))