diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py
index d01fd7bb55d..a629fc15695 100644
--- a/lib/galaxy/datatypes/sniff.py
+++ b/lib/galaxy/datatypes/sniff.py
@@ -139,26 +139,52 @@ def convert_newlines(fname, in_place=True, tmp_dir=None, tmp_prefix="gxupload",
"""
fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
i = 0
- with io.open(fd, mode="wt", encoding='utf-8') as fp:
- with io.open(fname, encoding='utf-8') as fi:
- partial_line = False
- while True:
- line = fi.readline(block_size)
- if not line:
- if partial_line:
- fp.write(u"\n")
+ try:
+ with io.open(fd, mode="wt", encoding='utf-8') as fp:
+ with io.open(fname, encoding='utf-8') as fi:
+ partial_line = False
+ while True:
+ line = fi.readline(block_size)
+ if not line:
+ if partial_line:
+ fp.write(u"\n")
+ i += 1
+ break
+
+ if line[-1] == u"\n":
+ partial_line = False
+ fp.write(line)
i += 1
- break
+ continue
- if line[-1] == u"\n":
- partial_line = False
+ # We have a block... maybe at the end of the file.
+ partial_line = True
fp.write(line)
- i += 1
- continue
+ except UnicodeDecodeError:
+ fd, temp_name = tempfile.mkstemp(prefix=tmp_prefix, dir=tmp_dir)
+ i = 0
+ with io.open(fd, mode="wb") as fp:
+ with io.open(fname, mode="rb") as fi:
+ partial_line = False
+ while True:
+ line = fi.readline(block_size)
+ if not line:
+ if partial_line:
+ fp.write(b"\n")
+ i += 1
+ break
- # We have a block... maybe at the end of the file.
- partial_line = True
- fp.write(line)
+ strip_line = line.rstrip(b"\r\n")
+ if len(strip_line) != len(line):
+ partial_line = False
+ fp.write(strip_line)
+ fp.write(b"\n")
+ i += 1
+ continue
+
+ # We have a block... maybe at the end of the file.
+ partial_line = True
+ fp.write(line)
if in_place:
shutil.move(temp_name, fname)
diff --git a/lib/galaxy/datatypes/test/1.imzml b/lib/galaxy/datatypes/test/1.imzml
new file mode 100644
index 00000000000..6b45d2a3667
--- /dev/null
+++ b/lib/galaxy/datatypes/test/1.imzml
@@ -0,0 +1,380 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/test/dosimzml b/lib/galaxy/datatypes/test/dosimzml
new file mode 100644
index 00000000000..6ff4f5c078d
--- /dev/null
+++ b/lib/galaxy/datatypes/test/dosimzml
@@ -0,0 +1,380 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/test/unit/datatypes/test_sniff.py b/test/unit/datatypes/test_sniff.py
index a41f891dba3..7f6370f7df9 100644
--- a/test/unit/datatypes/test_sniff.py
+++ b/test/unit/datatypes/test_sniff.py
@@ -1,6 +1,11 @@
import tempfile
-from galaxy.datatypes.sniff import convert_newlines_sep2tabs, sep2tabs
+from galaxy.datatypes.sniff import (
+ convert_newlines,
+ convert_newlines_sep2tabs,
+ get_test_fname,
+ sep2tabs,
+)
def assert_converts_to_1234_sep2tabs(content, line_ending="\n"):
@@ -23,6 +28,13 @@ def assert_converts_to_1234_convert_sep2tabs(content, expected='1\t2\n3\t4\n', l
assert expected == open(tf.name).read()
+def test_convert_newlines_non_utf():
+ fname = get_test_fname("dosimzml")
+ rval = convert_newlines(fname, tmp_prefix="gxtest", tmp_dir=tempfile.gettempdir(), in_place=False)
+ new_file = rval[1]
+ assert open(new_file, "rb").read() == open(get_test_fname("1.imzml"), "rb").read()
+
+
def test_sep2tabs():
assert_converts_to_1234_sep2tabs("1 2\n3 4\n")
assert_converts_to_1234_sep2tabs("1 2\n3 4\n")