From 0638240c82c5f84d295b1aee467c699181658dd5 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 May 2019 11:24:44 +0200 Subject: [PATCH 1/4] Operate on blocks directly --- lib/galaxy/datatypes/sniff.py | 39 +++++++++++++++++++++-------------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 6d352abb08d..35f7dba4a5f 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -14,7 +14,11 @@ import sys import tempfile import zipfile -from six import StringIO, text_type +from six import ( + PY3, + StringIO, + text_type, +) from six.moves import filter from six.moves.urllib.request import urlopen @@ -110,27 +114,30 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", """ fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir) i = 0 + if PY3: + NEWLINE_BYTE = 10 + CR_BYTE = 13 + else: + NEWLINE_BYTE = "\n" + CR_BYTE = "\r" with io.open(fd, mode="wb") as fp: with io.open(fname, mode="rb") as fi: - line = b'' - converted_line = b"" last_char = None block = fi.read(block_size) + last_block = b"" while block: - if last_char == "\r" and block.startswith(b"\n"): - # last block ended with "\r", new block startswith "\n" - # since we replace "\r" with "\n" in the previous iteration we skip the first byte + if last_char == CR_BYTE and block.startswith(b"\n"): + # Last block ended with CR, new block startswith newline. + # Since we replace CR with newline in the previous iteration we skip the first byte block = block[1:] - # splitlines(True) splits at line terminators but keeps them so we can replace them - lines = block.splitlines(True) - for line in lines: - converted_line = line.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - if b"\n" in converted_line: - i += 1 - fp.write(converted_line) - last_char = util.unicodify(line, error='replace')[-1] - block = fi.read(block_size) - if not converted_line.endswith(b"\n"): + if block: + last_char = block[-1] + block = block.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + fp.write(block) + i += block.count(b"\n") + last_block = block + block = fi.read(block_size) + if last_block and last_block[-1] != NEWLINE_BYTE: i += 1 fp.write(b"\n") if in_place: From b67ddbf03664b55d9b37ae491d38e45f96e67c76 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 May 2019 16:14:44 +0200 Subject: [PATCH 2/4] Drop (seemingly?) unused sep2tabs function --- lib/galaxy/datatypes/sniff.py | 33 ------------------------------- test/unit/datatypes/test_sniff.py | 23 --------------------- 2 files changed, 56 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 35f7dba4a5f..2048291a9c9 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -148,39 +148,6 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", return (i, temp_name) -def sep2tabs(fname, in_place=True, patt=r"\s+", tmp_dir=None, tmp_prefix="gxupload"): - """ - Transforms in place a 'sep' separated file to a tab separated one - """ - regexp = re.compile(patt) - fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir) - with io.open(fd, mode="w", encoding='utf-8') as fp: - i = None - for i, line in enumerate(io.open(fname, encoding='utf-8', newline='')): - if line.endswith("\r"): - line = line.rstrip('\r') - elems = regexp.split(line) - fp.write(u"%s\r" % '\t'.join(elems)) - elif line.endswith("\r\n"): - line = line.rstrip('\r\n') - elems = regexp.split(line) - fp.write(u"%s\r\n" % '\t'.join(elems)) - else: - line = line.rstrip('\n') - elems = regexp.split(line) - fp.write(u"%s\n" % '\t'.join(elems)) - if i is None: - i = 0 - else: - i += 1 - if in_place: - shutil.move(temp_name, fname) - # Return number of lines in file. - return (i, None) - else: - return (i, temp_name) - - def convert_newlines_sep2tabs(fname, in_place=True, patt=r"\s+", tmp_dir=None, tmp_prefix="gxupload"): """ Combines above methods: convert_newlines() and sep2tabs() diff --git a/test/unit/datatypes/test_sniff.py b/test/unit/datatypes/test_sniff.py index 4df85cdb9fa..fab1794c8d0 100644 --- a/test/unit/datatypes/test_sniff.py +++ b/test/unit/datatypes/test_sniff.py @@ -7,18 +7,9 @@ from galaxy.datatypes.sniff import ( convert_newlines, convert_newlines_sep2tabs, get_test_fname, - sep2tabs, ) -def assert_converts_to_1234_sep2tabs(content, line_ending="\n"): - with tempfile.NamedTemporaryFile(delete=False, mode='w') as tf: - tf.write(content) - rval = sep2tabs(tf.name, tmp_prefix="gxtest", tmp_dir=tempfile.gettempdir()) - assert rval == (2, None), rval - assert '1\t2%s3\t4%s' % (line_ending, line_ending) == io.open(tf.name, newline='').read() - - def assert_converts_to_1234_convert_sep2tabs(content, expected='1\t2\n3\t4\n', line_ending="\n"): with tempfile.NamedTemporaryFile(delete=False, mode='w') as tf: tf.write(content) @@ -73,20 +64,6 @@ def test_convert_newlines_non_utf(): assert open(new_file, "rb").read() == open(get_test_fname("1imzml"), "rb").read() -@pytest.mark.parametrize('source,line_ending', [ - ("1 2\n3 4\n", None), - ("1 2\n3 4\n", None), - ("1\t2\n3\t4\n", None), - ("1\t2\r3\t4\r", '\r'), - ("1\t2\r\n3\t4\r\n", '\r\n'), -]) -def test_sep2tabs(source, line_ending): - if line_ending: - assert_converts_to_1234_sep2tabs(source, line_ending) - else: - assert_converts_to_1234_sep2tabs(source) - - @pytest.mark.parametrize('source,expected', [ ("1 2\n3 4\n", None), ("1 2\n3 4\n", None), From 9c06527bbb4beddb127fcd78c85349550b4ec7e9 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 May 2019 17:38:40 +0200 Subject: [PATCH 3/4] Simplify convert_newlines_sep2tabs Just call convert_newlines with a compiled regex --- lib/galaxy/datatypes/sniff.py | 24 +++++------------------- test/unit/datatypes/test_sniff.py | 7 +++---- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 2048291a9c9..ff2d69acc7b 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -107,7 +107,7 @@ def stream_to_file(stream, suffix='', prefix='', dir=None, text=False, **kwd): return stream_to_open_named_file(stream, fd, temp_name, **kwd) -def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", block_size=128 * 1024): +def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", block_size=128 * 1024, regexp=None): """ Converts in place a file from universal line endings to Posix line endings. @@ -133,6 +133,8 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", if block: last_char = block[-1] block = block.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + if regexp: + block = b"\t".join(regexp.split(block)) fp.write(block) i += block.count(b"\n") last_block = block @@ -148,7 +150,7 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", return (i, temp_name) -def convert_newlines_sep2tabs(fname, in_place=True, patt=r"\s+", tmp_dir=None, tmp_prefix="gxupload"): +def convert_newlines_sep2tabs(fname, in_place=True, patt=br"[^\S\n]+", tmp_dir=None, tmp_prefix="gxupload"): """ Combines above methods: convert_newlines() and sep2tabs() so that files do not need to be read twice @@ -162,23 +164,7 @@ def convert_newlines_sep2tabs(fname, in_place=True, patt=r"\s+", tmp_dir=None, t '1\\t2\\n3\\t4\\n' """ regexp = re.compile(patt) - fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir) - with io.open(fd, mode="wt", encoding='utf-8') as fp: - i = None - for i, line in enumerate(io.open(fname, encoding='utf-8')): - line = line.rstrip('\r\n') - elems = regexp.split(line) - fp.write(u"%s\n" % '\t'.join(elems)) - if i is None: - i = 0 - else: - i = i + 1 - if in_place: - shutil.move(temp_name, fname) - # Return number of lines in file. - return (i, None) - else: - return (i, temp_name) + return convert_newlines(fname, in_place, tmp_dir, tmp_prefix, regexp=regexp) def iter_headers(fname_or_file_prefix, sep, count=60, comment_designator=None): diff --git a/test/unit/datatypes/test_sniff.py b/test/unit/datatypes/test_sniff.py index fab1794c8d0..faf75443281 100644 --- a/test/unit/datatypes/test_sniff.py +++ b/test/unit/datatypes/test_sniff.py @@ -1,4 +1,3 @@ -import io import tempfile import pytest @@ -10,12 +9,12 @@ from galaxy.datatypes.sniff import ( ) -def assert_converts_to_1234_convert_sep2tabs(content, expected='1\t2\n3\t4\n', line_ending="\n"): +def assert_converts_to_1234_convert_sep2tabs(content, expected='1\t2\n3\t4\n'): with tempfile.NamedTemporaryFile(delete=False, mode='w') as tf: tf.write(content) rval = convert_newlines_sep2tabs(tf.name, tmp_prefix="gxtest", tmp_dir=tempfile.gettempdir()) - assert rval == (2, None), rval assert expected == open(tf.name).read() + assert rval == (2, None), rval def assert_converts_to_1234_convert(content, block_size=1024): @@ -23,9 +22,9 @@ def assert_converts_to_1234_convert(content, block_size=1024): with open(fname, 'w') as fh: fh.write(content) rval = convert_newlines(fname, tmp_prefix="gxtest", tmp_dir=tempfile.gettempdir(), block_size=block_size) - assert rval == (2, None), "rval != %s for %s" % (rval, content) actual_contents = open(fname).read() assert '1 2\n3 4\n' == actual_contents, actual_contents + assert rval == (2, None), "rval != %s for %s" % (rval, content) @pytest.mark.parametrize('source,block_size', [ From ecc453b646f222d235c289f3cd001a98ba0934ce Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Fri, 17 May 2019 17:53:47 +0200 Subject: [PATCH 4/4] Group with statements and clean up docstring --- lib/galaxy/datatypes/sniff.py | 46 +++++++++++++++++------------------ 1 file changed, 22 insertions(+), 24 deletions(-) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index ff2d69acc7b..d35c83ffd4c 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -120,28 +120,27 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", else: NEWLINE_BYTE = "\n" CR_BYTE = "\r" - with io.open(fd, mode="wb") as fp: - with io.open(fname, mode="rb") as fi: - last_char = None - block = fi.read(block_size) - last_block = b"" - while block: - if last_char == CR_BYTE and block.startswith(b"\n"): - # Last block ended with CR, new block startswith newline. - # Since we replace CR with newline in the previous iteration we skip the first byte - block = block[1:] - if block: - last_char = block[-1] - block = block.replace(b"\r\n", b"\n").replace(b"\r", b"\n") - if regexp: - block = b"\t".join(regexp.split(block)) - fp.write(block) - i += block.count(b"\n") - last_block = block - block = fi.read(block_size) - if last_block and last_block[-1] != NEWLINE_BYTE: - i += 1 - fp.write(b"\n") + with io.open(fd, mode="wb") as fp, io.open(fname, mode="rb") as fi: + last_char = None + block = fi.read(block_size) + last_block = b"" + while block: + if last_char == CR_BYTE and block.startswith(b"\n"): + # Last block ended with CR, new block startswith newline. + # Since we replace CR with newline in the previous iteration we skip the first byte + block = block[1:] + if block: + last_char = block[-1] + block = block.replace(b"\r\n", b"\n").replace(b"\r", b"\n") + if regexp: + block = b"\t".join(regexp.split(block)) + fp.write(block) + i += block.count(b"\n") + last_block = block + block = fi.read(block_size) + if last_block and last_block[-1] != NEWLINE_BYTE: + i += 1 + fp.write(b"\n") if in_place: shutil.move(temp_name, fname) # Return number of lines in file. @@ -152,8 +151,7 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload", def convert_newlines_sep2tabs(fname, in_place=True, patt=br"[^\S\n]+", tmp_dir=None, tmp_prefix="gxupload"): """ - Combines above methods: convert_newlines() and sep2tabs() - so that files do not need to be read twice + Converts newlines in a file to posix newlines and replaces spaces with tabs. >>> fname = get_test_fname('temp.txt') >>> with open(fname, 'wt') as fh: