From c4a2d8c2189423e45124d847928431a726587eec Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 22 Nov 2017 18:22:54 +0000 Subject: [PATCH 1/7] Fix html sniffer to not return True in case of exception --- lib/galaxy/datatypes/text.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 33bdec72306..a34e1b0dcaa 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -50,13 +50,10 @@ class Html(Text): True """ headers = iter_headers(filename, None) - try: - for i, hdr in enumerate(headers): - if hdr and hdr[0].lower().find('') >= 0: - return True - return False - except Exception: - return True + for i, hdr in enumerate(headers): + if hdr and hdr[0].lower().find('') >= 0: + return True + return False class Json(Text): From 1ba863d4d6af388aaf5911b1f57a1ce59eb20f09 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Wed, 22 Nov 2017 19:09:24 +0000 Subject: [PATCH 2/7] Remove duplicated functions Added in commits 777d6a155c369e49127084b63dee5701da7589f1 and 917d5a72e83741594912578ee14d1df19f5974c9 --- lib/tool_shed/util/tool_util.py | 36 +-------------------------------- 1 file changed, 1 insertion(+), 35 deletions(-) diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index b9aef505bdf..da0518b1659 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -4,8 +4,8 @@ import shutil import galaxy.tools from galaxy import util +from galaxy.datatypes.sniff import is_column_based from galaxy.util import checkers -from galaxy.util import unicodify from galaxy.util.expressions import ExpressionContext from galaxy.web.form_builder import SelectField from tool_shed.util import basic_util @@ -125,20 +125,6 @@ def generate_message_for_invalid_tools(app, invalid_file_tups, repository, metad return message -def get_headers(fname, sep, count=60, is_multi_byte=False): - """Returns a list with the first 'count' lines split by 'sep'.""" - headers = [] - for idx, line in enumerate(open(fname)): - line = line.rstrip('\n\r') - if is_multi_byte: - line = unicodify(line, 'utf-8') - sep = sep.encode('utf-8') - headers.append(line.split(sep)) - if idx == count: - break - return headers - - def get_tool_path_install_dir(partial_install_dir, shed_tool_conf_dict, tool_dict, config_elems): for elem in config_elems: if elem.tag == 'tool': @@ -184,26 +170,6 @@ def handle_missing_index_file(app, tool_path, sample_files, repository_tools_tup return repository_tools_tups, sample_files_copied -def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False): - """See if the file is column based with respect to a separator.""" - headers = get_headers(fname, sep, is_multi_byte=is_multi_byte) - count = 0 - if not headers: - return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) > 1: - count = len(hdr) - break - if count < 2: - return False - for hdr in headers[skip:]: - if hdr and hdr[0] and not hdr[0].startswith('#'): - if len(hdr) != count: - return False - return True - - def is_data_index_sample_file(file_path): """ Attempt to determine if a .sample file is appropriate for copying to ~/tool-data when From 0df99d89d7d749df8d3432a3506d76b2461e8bfe Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 24 Nov 2017 14:50:56 +0000 Subject: [PATCH 3/7] Improve and test the PDF sniffer Read just the first 4 bytes in binary mode. --- lib/galaxy/datatypes/images.py | 11 +- lib/galaxy/datatypes/sniff.py | 3 + lib/galaxy/datatypes/test/454Score.pdf | 545 +++++++++++++++++++++++++ 3 files changed, 550 insertions(+), 9 deletions(-) create mode 100644 lib/galaxy/datatypes/test/454Score.pdf diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index 6c7e457a9ee..46659607d41 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -7,7 +7,6 @@ import zipfile from six.moves.urllib.parse import quote_plus from galaxy.datatypes.binary import Binary -from galaxy.datatypes.sniff import get_headers from galaxy.datatypes.text import Html as HtmlFromText from galaxy.util import nice_size from galaxy.util.image_util import check_image_type @@ -160,14 +159,8 @@ class Pdf(Image): def sniff(self, filename): """Determine if the file is in pdf format.""" - headers = get_headers(filename, None, 1) - try: - if headers[0][0].startswith("%PDF"): - return True - else: - return False - except IndexError: - return False + with open(filename, 'rb') as fh: + return fh.read(4) == b"%PDF" Binary.register_sniffable_binary_format("pdf", "pdf", Pdf) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 970beefa63d..23c79061bcb 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -393,6 +393,9 @@ def guess_ext(fname, sniff_order, is_multi_byte=False): >>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom') >>> guess_ext(fname, sniff_order) 'biom2' + >>> fname = get_test_fname('454Score.pdf') + >>> guess_ext(fname, sniff_order) + 'pdf' """ file_ext = None for datatype in sniff_order: diff --git a/lib/galaxy/datatypes/test/454Score.pdf b/lib/galaxy/datatypes/test/454Score.pdf new file mode 100644 index 00000000000..1199be2305a --- /dev/null +++ b/lib/galaxy/datatypes/test/454Score.pdf @@ -0,0 +1,545 @@ +%PDF-1.1 +%ρ\r +1 0 obj +<< +/CreationDate (D:20080403110358) +/ModDate (D:20080403110358) +/Title (R Graphics Output) +/Producer (R 2.6.2) +/Creator (R) +>> +endobj +2 0 obj +<< +/Type /Catalog +/Pages 3 0 R +>> +endobj +5 0 obj +<< +/Type /Font +/Subtype /Type1 +/Name /F1 +/BaseFont /ZapfDingbats +>> +endobj +6 0 obj +<< +/Type /Page +/Parent 3 0 R +/Contents 7 0 R +/Resources 4 0 R +>> +endobj +7 0 obj +<< +/Length 8 0 R +>> +stream +q +Q q 59.04 73.44 342.72 299.52 re W n +0.000 0.000 0.000 RG +2.25 w +[] 0 d +1 J +1 j +10.00 M +73.40 149.79 m 86.76 149.79 l S +0.75 w +[ 3.00 5.00] 0 d +80.08 100.85 m 80.08 149.79 l S +80.08 296.61 m 80.08 263.98 l S +0.75 w +[] 0 d +76.74 100.85 m 83.42 100.85 l S +76.74 296.61 m 83.42 296.61 l S +73.40 149.79 m +86.76 149.79 l +86.76 263.98 l +73.40 263.98 l +73.40 149.79 l +S +2.25 w +[] 0 d +90.11 280.30 m 103.47 280.30 l S +0.75 w +[ 3.00 5.00] 0 d +96.79 263.98 m 96.79 280.30 l S +96.79 296.61 m 96.79 296.61 l S +0.75 w +[] 0 d +93.45 263.98 m 100.13 263.98 l S +93.45 296.61 m 100.13 296.61 l S +90.11 280.30 m +103.47 280.30 l +103.47 296.61 l +90.11 296.61 l +90.11 280.30 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 93.82 342.96 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +106.81 280.30 m 120.17 280.30 l S +0.75 w +[ 3.00 5.00] 0 d +113.49 263.98 m 113.49 263.98 l S +113.49 280.30 m 113.49 280.30 l S +0.75 w +[] 0 d +110.15 263.98 m 116.83 263.98 l S +110.15 280.30 m 116.83 280.30 l S +106.81 263.98 m +120.17 263.98 l +120.17 280.30 l +106.81 280.30 l +106.81 263.98 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 110.53 179.82 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +123.51 247.67 m 136.87 247.67 l S +0.75 w +[ 3.00 5.00] 0 d +130.19 247.67 m 130.19 247.67 l S +130.19 247.67 m 130.19 247.67 l S +0.75 w +[] 0 d +126.85 247.67 m 133.53 247.67 l S +126.85 247.67 m 133.53 247.67 l S +123.51 247.67 m +136.87 247.67 l +136.87 247.67 l +123.51 247.67 l +123.51 247.67 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 127.23 261.39 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +140.21 280.30 m 153.57 280.30 l S +0.75 w +[ 3.00 5.00] 0 d +146.89 231.36 m 146.89 247.67 l S +146.89 361.87 m 146.89 296.61 l S +0.75 w +[] 0 d +143.55 231.36 m 150.23 231.36 l S +143.55 361.87 m 150.23 361.87 l S +140.21 247.67 m +153.57 247.67 l +153.57 296.61 l +140.21 296.61 l +140.21 247.67 l +S +2.25 w +[] 0 d +156.91 280.30 m 170.27 280.30 l S +0.75 w +[ 3.00 5.00] 0 d +163.59 198.73 m 163.59 231.36 l S +163.59 280.30 m 163.59 280.30 l S +0.75 w +[] 0 d +160.25 198.73 m 166.93 198.73 l S +160.25 280.30 m 166.93 280.30 l S +156.91 231.36 m +170.27 231.36 l +170.27 280.30 l +156.91 280.30 l +156.91 231.36 l +S +2.25 w +[] 0 d +173.61 280.30 m 186.98 280.30 l S +0.75 w +[ 3.00 5.00] 0 d +180.29 247.67 m 180.29 263.98 l S +180.29 280.30 m 180.29 280.30 l S +0.75 w +[] 0 d +176.95 247.67 m 183.64 247.67 l S +176.95 280.30 m 183.64 280.30 l S +173.61 263.98 m +186.98 263.98 l +186.98 280.30 l +173.61 280.30 l +173.61 263.98 l +S +2.25 w +[] 0 d +190.32 247.67 m 203.68 247.67 l S +0.75 w +[ 3.00 5.00] 0 d +197.00 247.67 m 197.00 247.67 l S +197.00 263.98 m 197.00 263.98 l S +0.75 w +[] 0 d +193.66 247.67 m 200.34 247.67 l S +193.66 263.98 m 200.34 263.98 l S +190.32 247.67 m +203.68 247.67 l +203.68 263.98 l +190.32 263.98 l +190.32 247.67 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 194.03 294.02 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +207.02 263.98 m 220.38 263.98 l S +0.75 w +[ 3.00 5.00] 0 d +213.70 247.67 m 213.70 247.67 l S +213.70 263.98 m 213.70 263.98 l S +0.75 w +[] 0 d +210.36 247.67 m 217.04 247.67 l S +210.36 263.98 m 217.04 263.98 l S +207.02 247.67 m +220.38 247.67 l +220.38 263.98 l +207.02 263.98 l +207.02 247.67 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 210.74 342.96 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +223.72 263.98 m 237.08 263.98 l S +0.75 w +[ 3.00 5.00] 0 d +230.40 263.98 m 230.40 263.98 l S +230.40 280.30 m 230.40 280.30 l S +0.75 w +[] 0 d +227.06 263.98 m 233.74 263.98 l S +227.06 280.30 m 233.74 280.30 l S +223.72 263.98 m +237.08 263.98 l +237.08 280.30 l +223.72 280.30 l +223.72 263.98 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 227.44 163.51 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +240.42 280.30 m 253.78 280.30 l S +0.75 w +[ 3.00 5.00] 0 d +247.10 263.98 m 247.10 263.98 l S +247.10 280.30 m 247.10 280.30 l S +0.75 w +[] 0 d +243.76 263.98 m 250.44 263.98 l S +243.76 280.30 m 250.44 280.30 l S +240.42 263.98 m +253.78 263.98 l +253.78 280.30 l +240.42 280.30 l +240.42 263.98 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 244.14 179.82 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +257.12 263.98 m 270.48 263.98 l S +0.75 w +[ 3.00 5.00] 0 d +263.80 247.67 m 263.80 247.67 l S +263.80 280.30 m 263.80 280.30 l S +0.75 w +[] 0 d +260.46 247.67 m 267.14 247.67 l S +260.46 280.30 m 267.14 280.30 l S +257.12 247.67 m +270.48 247.67 l +270.48 280.30 l +257.12 280.30 l +257.12 247.67 l +S +2.25 w +[] 0 d +273.82 247.67 m 287.19 247.67 l S +0.75 w +[ 3.00 5.00] 0 d +280.51 84.53 m 280.51 182.42 l S +280.51 296.61 m 280.51 296.61 l S +0.75 w +[] 0 d +277.16 84.53 m 283.85 84.53 l S +277.16 296.61 m 283.85 296.61 l S +273.82 182.42 m +287.19 182.42 l +287.19 296.61 l +273.82 296.61 l +273.82 182.42 l +S +2.25 w +[] 0 d +290.53 280.30 m 303.89 280.30 l S +0.75 w +[ 3.00 5.00] 0 d +297.21 280.30 m 297.21 280.30 l S +297.21 280.30 m 297.21 280.30 l S +0.75 w +[] 0 d +293.87 280.30 m 300.55 280.30 l S +293.87 280.30 m 300.55 280.30 l S +290.53 280.30 m +303.89 280.30 l +303.89 280.30 l +290.53 280.30 l +290.53 280.30 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 294.25 294.02 Tm (l) Tj 0 Tr +/F1 1 Tf 1 Tr 7.48 0 0 7.48 294.25 228.76 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +307.23 263.98 m 320.59 263.98 l S +0.75 w +[ 3.00 5.00] 0 d +313.91 247.67 m 313.91 247.67 l S +313.91 280.30 m 313.91 280.30 l S +0.75 w +[] 0 d +310.57 247.67 m 317.25 247.67 l S +310.57 280.30 m 317.25 280.30 l S +307.23 247.67 m +320.59 247.67 l +320.59 280.30 l +307.23 280.30 l +307.23 247.67 l +S +2.25 w +[] 0 d +323.93 231.36 m 337.29 231.36 l S +0.75 w +[ 3.00 5.00] 0 d +330.61 198.73 m 330.61 215.04 l S +330.61 231.36 m 330.61 231.36 l S +0.75 w +[] 0 d +327.27 198.73 m 333.95 198.73 l S +327.27 231.36 m 333.95 231.36 l S +323.93 215.04 m +337.29 215.04 l +337.29 231.36 l +323.93 231.36 l +323.93 215.04 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 327.65 261.39 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +340.63 263.98 m 353.99 263.98 l S +0.75 w +[ 3.00 5.00] 0 d +347.31 263.98 m 347.31 263.98 l S +347.31 263.98 m 347.31 263.98 l S +0.75 w +[] 0 d +343.97 263.98 m 350.65 263.98 l S +343.97 263.98 m 350.65 263.98 l S +340.63 263.98 m +353.99 263.98 l +353.99 263.98 l +340.63 263.98 l +340.63 263.98 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 344.35 179.82 Tm (l) Tj 0 Tr +/F1 1 Tf 1 Tr 7.48 0 0 7.48 344.35 277.70 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +357.33 263.98 m 370.69 263.98 l S +0.75 w +[ 3.00 5.00] 0 d +364.01 247.67 m 364.01 247.67 l S +364.01 280.30 m 364.01 280.30 l S +0.75 w +[] 0 d +360.67 247.67 m 367.35 247.67 l S +360.67 280.30 m 367.35 280.30 l S +357.33 247.67 m +370.69 247.67 l +370.69 280.30 l +357.33 280.30 l +357.33 247.67 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 361.05 81.94 Tm (l) Tj 0 Tr +2.25 w +[] 0 d +ET +374.04 263.98 m 387.40 263.98 l S +0.75 w +[ 3.00 5.00] 0 d +380.72 263.98 m 380.72 263.98 l S +380.72 296.61 m 380.72 296.61 l S +0.75 w +[] 0 d +377.38 263.98 m 384.06 263.98 l S +377.38 296.61 m 384.06 296.61 l S +374.04 263.98 m +387.40 263.98 l +387.40 296.61 l +374.04 296.61 l +374.04 263.98 l +S +BT +/F1 1 Tf 1 Tr 7.48 0 0 7.48 377.75 163.51 Tm (l) Tj 0 Tr +ET +Q q +0.000 0.000 0.000 RG +0.75 w +[] 0 d +1 J +1 j +10.00 M +59.04 133.47 m 59.04 296.61 l S +59.04 133.47 m 51.84 133.47 l S +59.04 215.04 m 51.84 215.04 l S +59.04 296.61 m 51.84 296.61 l S +BT +0.000 0.000 0.000 rg +/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 126.80 Tm (20) Tj +/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 208.37 Tm (25) Tj +/F2 1 Tf 0.00 12.00 -12.00 0.00 41.76 289.94 Tm (30) Tj +ET +Q q +BT +0.000 0.000 0.000 rg +/F3 1 Tf 14.00 0.00 -0.00 14.00 147.76 397.45 Tm (boxplot of quality scores) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 130.36 18.72 Tm (position within read \(% of total length\)) Tj +ET +Q q +0.000 0.000 0.000 RG +0.75 w +[] 0 d +1 J +1 j +10.00 M +59.04 73.44 m +401.76 73.44 l +401.76 372.96 l +59.04 372.96 l +59.04 73.44 l +S +63.38 73.44 m 380.72 73.44 l S +63.38 73.44 m 63.38 66.24 l S +80.08 73.44 m 80.08 66.24 l S +96.79 73.44 m 96.79 66.24 l S +113.49 73.44 m 113.49 66.24 l S +130.19 73.44 m 130.19 66.24 l S +146.89 73.44 m 146.89 66.24 l S +163.59 73.44 m 163.59 66.24 l S +180.29 73.44 m 180.29 66.24 l S +197.00 73.44 m 197.00 66.24 l S +213.70 73.44 m 213.70 66.24 l S +230.40 73.44 m 230.40 66.24 l S +247.10 73.44 m 247.10 66.24 l S +263.80 73.44 m 263.80 66.24 l S +280.51 73.44 m 280.51 66.24 l S +297.21 73.44 m 297.21 66.24 l S +313.91 73.44 m 313.91 66.24 l S +330.61 73.44 m 330.61 66.24 l S +347.31 73.44 m 347.31 66.24 l S +364.01 73.44 m 364.01 66.24 l S +380.72 73.44 m 380.72 66.24 l S +BT +0.000 0.000 0.000 rg +/F2 1 Tf 12.00 0.00 -0.00 12.00 60.05 47.52 Tm (0) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 76.75 47.52 Tm (5) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 106.82 47.52 Tm (15) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 140.22 47.52 Tm (25) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 173.62 47.52 Tm (35) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 207.03 47.52 Tm (45) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 240.43 47.52 Tm (55) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 273.83 47.52 Tm (65) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 307.24 47.52 Tm (75) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 340.64 47.52 Tm (85) Tj +/F2 1 Tf 12.00 0.00 -0.00 12.00 374.04 47.52 Tm (95) Tj +ET +Q +endstream +endobj +8 0 obj +8714 +endobj +3 0 obj +<< +/Type /Pages +/Kids [ +6 0 R +] +/Count 1 +/MediaBox [0 0 432 432] +>> +endobj +4 0 obj +<< +/ProcSet [/PDF /Text] +/Font << /F1 5 0 R /F2 10 0 R /F3 11 0 R >> +/ExtGState << >> +>> +endobj +9 0 obj +<< +/Type /Encoding +/BaseEncoding /WinAnsiEncoding +/Differences [ 45/minus 96/quoteleft +144/dotlessi /grave /acute /circumflex /tilde /macron /breve /dotaccent +/dieresis /.notdef /ring /cedilla /.notdef /hungarumlaut /ogonek /caron /space] +>> +endobj +10 0 obj << +/Type /Font +/Subtype /Type1 +/Name /F2 +/BaseFont /Helvetica +/Encoding 9 0 R +>> endobj +11 0 obj << +/Type /Font +/Subtype /Type1 +/Name /F3 +/BaseFont /Helvetica-Bold +/Encoding 9 0 R +>> endobj +xref +0 12 +0000000000 65535 f +0000000021 00000 n +0000000163 00000 n +0000009162 00000 n +0000009245 00000 n +0000000212 00000 n +0000000295 00000 n +0000000375 00000 n +0000009142 00000 n +0000009349 00000 n +0000009606 00000 n +0000009703 00000 n +trailer +<< +/Size 12 +/Info 1 0 R +/Root 2 0 R +>> +startxref +9805 +%%EOF From 4a19bf62e35d52236a1fe5572a19e24ac20c3efe Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 24 Nov 2017 10:16:17 +0000 Subject: [PATCH 4/7] Add missing newline at end of some test files It gets added anyway when the file is uploaded to Galaxy due to the default "Use POSIX standard" (to_posix_lines) option. Many tests which simply copy them with cat and compare them would fail a strict diff test. --- test-data/1.fastqsanger | 2 +- test-data/1.fastqsolexa | 2 +- test-data/simple_line_alternative.txt | 2 +- test-data/simple_lines_interleaved.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test-data/1.fastqsanger b/test-data/1.fastqsanger index 3a1f7901974..1ec085aa290 100644 --- a/test-data/1.fastqsanger +++ b/test-data/1.fastqsanger @@ -5,4 +5,4 @@ AATACTTTCGGCGCCCTAAACCAGCTCACTGGGG @1831_573_1050/1 TTTATGGGTATGGCCGCTCACAGGCCAGCGGCCT + -;@@17?@=>7??@A8?==@4A?A4)&+.'&+'1, \ No newline at end of file +;@@17?@=>7??@A8?==@4A?A4)&+.'&+'1, diff --git a/test-data/1.fastqsolexa b/test-data/1.fastqsolexa index 01ddd7bed12..23d6e5f80d0 100644 --- a/test-data/1.fastqsolexa +++ b/test-data/1.fastqsolexa @@ -5,4 +5,4 @@ GAATTGATCAGGACATAGGACAACTGTAGGCACCAT @HANNIBAL_1_FC302VTAAXX:2:1:156:340 GAGTTCTCGTCGCCTGTAGGCACCATCAATCGTATG +HANNIBAL_1_FC302VTAAXX:2:1:156:340 -40 15 40 17 6 36 40 40 40 25 40 9 35 33 40 14 14 18 15 17 19 28 31 4 24 18 27 14 15 18 2 8 12 8 11 9 \ No newline at end of file +40 15 40 17 6 36 40 40 40 25 40 9 35 33 40 14 14 18 15 17 19 28 31 4 24 18 27 14 15 18 2 8 12 8 11 9 diff --git a/test-data/simple_line_alternative.txt b/test-data/simple_line_alternative.txt index 6966fea9758..56aa6e445c4 100644 --- a/test-data/simple_line_alternative.txt +++ b/test-data/simple_line_alternative.txt @@ -1 +1 @@ -This is a different line of text. \ No newline at end of file +This is a different line of text. diff --git a/test-data/simple_lines_interleaved.txt b/test-data/simple_lines_interleaved.txt index a1ec72a3503..9fb64962198 100644 --- a/test-data/simple_lines_interleaved.txt +++ b/test-data/simple_lines_interleaved.txt @@ -1,4 +1,4 @@ This is a line of text. This is a different line of text. This is a line of text. -This is a different line of text. \ No newline at end of file +This is a different line of text. From 5a2cb8546948c2f57d780201c1252d55269fcfe3 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 27 Nov 2017 16:48:24 +0000 Subject: [PATCH 5/7] Fix `SnpSiftDbNSFP.generate_primary_file()` to return a string Also use a `with` statement to open/close a file in `SnpSiftDbNSFP.regenerate_primary_file()` . --- lib/galaxy/datatypes/text.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index a34e1b0dcaa..5882593cb03 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -500,20 +500,19 @@ class SnpSiftDbNSFP(Text): This is called only at upload to write the html file cannot rename the datasets here - they come with the default unfortunately """ - self.regenerate_primary_file(dataset) + return 'SnpSiftDbNSFP Composite Dataset' def regenerate_primary_file(self, dataset): """ cannot do this until we are setting metadata """ annotations = "dbNSFP Annotations: %s\n" % ','.join(dataset.metadata.annotation) - f = open(dataset.file_name, 'a') - if dataset.metadata.bgzip: - bn = dataset.metadata.bgzip - f.write(bn) - f.write('\n') - f.write(annotations) - f.close() + with open(dataset.file_name, 'a') as f: + if dataset.metadata.bgzip: + bn = dataset.metadata.bgzip + f.write(bn) + f.write('\n') + f.write(annotations) def set_meta(self, dataset, overwrite=True, **kwd): try: From 6b730ae961c12375bd37e870cc52e0f77e4c0e17 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 27 Nov 2017 19:57:29 +0000 Subject: [PATCH 6/7] Add a datatype to sniff_order only if it has a sniff() method This reduces the number of default sniffers from 412 to 246. --- lib/galaxy/datatypes/registry.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 4fd441e5863..08ea3d4f544 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -314,13 +314,14 @@ class Registry(object): def append_to_sniff_order(): # Just in case any supported data types are not included in the config's sniff_order section. for ext, datatype in self.datatypes_by_extension.items(): - included = False - for atype in self.sniff_order: - if isinstance(atype, datatype.__class__): - included = True - break - if not included: - self.sniff_order.append(datatype) + if hasattr(datatype, 'sniff'): + included = False + for atype in self.sniff_order: + if isinstance(atype, datatype.__class__): + included = True + break + if not included: + self.sniff_order.append(datatype) append_to_sniff_order() def _load_build_sites(self, root): From 59f44cf2be48abd4bed0f7c6ddf7cc0aee5ad1f5 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 23 Nov 2017 17:29:50 +0000 Subject: [PATCH 7/7] Update `get_fileobj()` to use utf-8 encoding in text mode Also, merge its 3 parameters `gzip_only`, `bz2_only`, `zip_only` into `compressed_formats` (a list of allowed formats). As a consequence of the changes in `get_fileobj()`, update: - `files_diff()` - `get_file_peek()`, which now determines that a file is binary when a `UnicodeDecodeError` exception is raised and doesn't need `is_multi_byte` any more - `iter_headers()` and `get_headers()`, which now return Unicode and don't need `is_multi_byte` parameter any more As a consequence of the changes in `get_file_peek()`, update: - `set_peek()`, which now doesn't need `is_multi_byte` any more As a consequence of the changes in `get_headers()`, update: - `guess_ext` and `is_column_based()`, which now determine that a file is binary when a `UnicodeDecodeError` exception is raised and don't need `is_multi_byte` any more As a consequence of the changes to `guess_ext`, update: - `handle_uploaded_dataset_file() doesn't need `is_multi_byte` any more Also, remove duplicated calls to `get_file_peek()` in lib/galaxy/datatypes/molecules.py and lib/galaxy/datatypes/msa.py The `is_multi_byte` was not removed from the signature of `get_file_peek()` and `set_peek()` in order to preserve compatibility for ToolShed datatypes, thanks @jmchilton for the review. --- lib/galaxy/datatypes/annotation.py | 2 +- lib/galaxy/datatypes/binary.py | 2 +- lib/galaxy/datatypes/blast.py | 2 +- .../datatypes/constructive_solid_geometry.py | 4 +- lib/galaxy/datatypes/data.py | 39 +++--- lib/galaxy/datatypes/graph.py | 4 +- lib/galaxy/datatypes/molecules.py | 20 ++- lib/galaxy/datatypes/msa.py | 8 +- lib/galaxy/datatypes/plant_tribes.py | 26 ++-- lib/galaxy/datatypes/proteomics.py | 10 +- lib/galaxy/datatypes/sequence.py | 6 +- lib/galaxy/datatypes/sniff.py | 56 +++----- lib/galaxy/datatypes/tabular.py | 4 +- lib/galaxy/datatypes/text.py | 10 +- lib/galaxy/datatypes/triples.py | 14 +- lib/galaxy/datatypes/xml.py | 10 +- lib/galaxy/jobs/__init__.py | 10 +- lib/galaxy/model/__init__.py | 4 +- .../versions/0005_cleanup_datasets_fix.py | 4 +- lib/galaxy/tools/verify/__init__.py | 121 +++++++++--------- lib/galaxy/util/compression_utils.py | 37 ++++-- tools/data_source/data_source.py | 2 +- tools/data_source/upload.py | 2 +- 23 files changed, 186 insertions(+), 211 deletions(-) diff --git a/lib/galaxy/datatypes/annotation.py b/lib/galaxy/datatypes/annotation.py index 9a0717b5218..d5dd97cde80 100644 --- a/lib/galaxy/datatypes/annotation.py +++ b/lib/galaxy/datatypes/annotation.py @@ -14,7 +14,7 @@ class SnapHmm(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "SNAP HMM model" else: dataset.peek = 'file does not exist' diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 1d69cbba5fb..e3a99c931bb 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -1088,7 +1088,7 @@ class TwoBit(Binary): dataset.peek = "Binary TwoBit format nucleotide file" dataset.blurb = nice_size(dataset.get_size()) else: - return super(TwoBit, self).set_peek(dataset, is_multi_byte) + return super(TwoBit, self).set_peek(dataset) def display_peek(self, dataset): try: diff --git a/lib/galaxy/datatypes/blast.py b/lib/galaxy/datatypes/blast.py index 107202610f7..480c1ef1d67 100644 --- a/lib/galaxy/datatypes/blast.py +++ b/lib/galaxy/datatypes/blast.py @@ -53,7 +53,7 @@ class BlastXml(GenericXml): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = 'NCBI Blast XML data' else: dataset.peek = 'file does not exist' diff --git a/lib/galaxy/datatypes/constructive_solid_geometry.py b/lib/galaxy/datatypes/constructive_solid_geometry.py index 40325ffa5a0..9a31979afb3 100644 --- a/lib/galaxy/datatypes/constructive_solid_geometry.py +++ b/lib/galaxy/datatypes/constructive_solid_geometry.py @@ -102,7 +102,7 @@ class Ply(object): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "Faces: %s, Vertices: %s" % (str(dataset.metadata.face), str(dataset.metadata.vertex)) else: dataset.peek = 'File does not exist' @@ -429,7 +429,7 @@ class Vtk(object): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = self.get_blurb(dataset) else: dataset.peek = 'File does not exist' diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 1ae1b583f17..0931e2bd005 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -195,7 +195,12 @@ class Data(object): max_optional_metadata_filesize = property(get_max_optional_metadata_filesize, set_max_optional_metadata_filesize) def set_peek(self, dataset, is_multi_byte=False): - """Set the peek and blurb text""" + """ + Set the peek and blurb text + + :param is_multi_byte: deprecated + :type is_multi_byte: bool + """ if not dataset.dataset.purged: dataset.peek = '' dataset.blurb = 'data' @@ -838,7 +843,7 @@ class Text(Data): """ if not dataset.dataset.purged: # The file must exist on disk for the get_file_peek() method - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap) + dataset.peek = get_file_peek(dataset.file_name, WIDTH=WIDTH, skipchars=skipchars, line_wrap=line_wrap) if line_count is None: # See if line_count is stored in the metadata if dataset.metadata.data_lines: @@ -1046,7 +1051,10 @@ def get_test_fname(fname): def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipchars=None, line_wrap=True): """ - Returns the first LINE_COUNT lines wrapped to WIDTH + Returns the first LINE_COUNT lines wrapped to WIDTH. + + :param is_multi_byte: deprecated + :type is_multi_byte: bool >>> fname = get_test_fname('4.bed') >>> get_file_peek(fname, LINE_COUNT=1) @@ -1061,20 +1069,12 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc skipchars = [] lines = [] count = 0 - file_type = None - data_checked = False with compression_utils.get_fileobj(file_name, "U") as temp: while count < LINE_COUNT: - line = temp.readline(WIDTH) - if line and not is_multi_byte and not data_checked: - # See if we have a compressed or binary file - for char in line: - if ord(char) > 128: - file_type = 'binary' - break - data_checked = True - if file_type == 'binary': - break + try: + line = temp.readline(WIDTH) + except UnicodeDecodeError: + return "binary file" if not line_wrap: if line.endswith('\n'): line = line[:-1] @@ -1091,11 +1091,4 @@ def get_file_peek(file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skipc if not skip_line: lines.append(line) count += 1 - if file_type == 'binary': - text = "%s file" % file_type - else: - try: - text = util.unicodify('\n'.join(lines)) - except UnicodeDecodeError: - text = "binary/unknown file" - return text + return '\n'.join(lines) diff --git a/lib/galaxy/datatypes/graph.py b/lib/galaxy/datatypes/graph.py index cb50fb81d29..e8448fd155e 100644 --- a/lib/galaxy/datatypes/graph.py +++ b/lib/galaxy/datatypes/graph.py @@ -27,7 +27,7 @@ class Xgmml(xml.GenericXml): Set the peek and blurb text """ if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'XGMML data' else: dataset.peek = 'file does not exist' @@ -73,7 +73,7 @@ class Sif(tabular.Tabular): Set the peek and blurb text """ if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'SIF data' else: dataset.peek = 'file does not exist' diff --git a/lib/galaxy/datatypes/molecules.py b/lib/galaxy/datatypes/molecules.py index e7ee4734804..c3e00f33229 100644 --- a/lib/galaxy/datatypes/molecules.py +++ b/lib/galaxy/datatypes/molecules.py @@ -61,12 +61,11 @@ class GenericMolFile(data.Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) if (dataset.metadata.number_of_molecules == 1): dataset.blurb = "1 molecule" else: dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' @@ -471,7 +470,7 @@ class PHAR(GenericMolFile): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "pharmacophore" else: dataset.peek = 'file does not exist' @@ -524,7 +523,7 @@ class PDB(GenericMolFile): if not dataset.dataset.purged: atom_numbers = count_special_lines("^ATOM", dataset.file_name) hetatm_numbers = count_special_lines("^HETATM", dataset.file_name) - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "%s atoms and %s HET-atoms" % (atom_numbers, hetatm_numbers) else: dataset.peek = 'file does not exist' @@ -575,7 +574,7 @@ class PDBQT(GenericMolFile): if not dataset.dataset.purged: root_numbers = count_special_lines("^ROOT", dataset.file_name) branch_numbers = count_special_lines("^BRANCH", dataset.file_name) - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "%s roots and %s branches" % (root_numbers, branch_numbers) else: dataset.peek = 'file does not exist' @@ -587,7 +586,7 @@ class grd(data.Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "grids for docking" else: dataset.peek = 'file does not exist' @@ -621,12 +620,11 @@ class InChI(Tabular): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) if (dataset.metadata.number_of_molecules == 1): dataset.blurb = "1 molecule" else: dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' @@ -666,12 +664,11 @@ class SMILES(Tabular): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) if dataset.metadata.number_of_molecules == 1: dataset.blurb = "1 molecule" else: dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' @@ -727,12 +724,11 @@ class CML(GenericXml): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) if (dataset.metadata.number_of_molecules == 1): dataset.blurb = "1 molecule" else: dataset.blurb = "%s molecules" % dataset.metadata.number_of_molecules - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disk' diff --git a/lib/galaxy/datatypes/msa.py b/lib/galaxy/datatypes/msa.py index cc26438ceca..3985ef95488 100644 --- a/lib/galaxy/datatypes/msa.py +++ b/lib/galaxy/datatypes/msa.py @@ -17,7 +17,7 @@ class Hmmer(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "HMMER Database" else: dataset.peek = 'file does not exist' @@ -104,12 +104,11 @@ class Stockholm_1_0(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) if (dataset.metadata.number_of_models == 1): dataset.blurb = "1 alignment" else: dataset.blurb = "%s alignments" % dataset.metadata.number_of_models - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disc' @@ -187,12 +186,11 @@ class MauveXmfa(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) if (dataset.metadata.number_of_models == 1): dataset.blurb = "1 alignment" else: dataset.blurb = "%s alignments" % dataset.metadata.number_of_models - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) else: dataset.peek = 'file does not exist' dataset.blurb = 'file purged from disc' diff --git a/lib/galaxy/datatypes/plant_tribes.py b/lib/galaxy/datatypes/plant_tribes.py index d275e1aedf1..983600a86b9 100644 --- a/lib/galaxy/datatypes/plant_tribes.py +++ b/lib/galaxy/datatypes/plant_tribes.py @@ -23,7 +23,7 @@ class Smat(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "ESTScan scores matrices" else: dataset.peek = 'file does not exist' @@ -125,7 +125,7 @@ class PlantTribesKsComponents(Tabular): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) if (dataset.metadata.number_comp == 1): dataset.blurb = "1 significant component" else: @@ -159,7 +159,7 @@ class PlantTribesOrtho(PlantTribes): file_ext = "ptortho" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesOrtho, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesOrtho, self).set_peek(dataset) dataset.blurb = "Proteins orthogroup fasta files: %d items" % dataset.metadata.num_files @@ -171,7 +171,7 @@ class PlantTribesOrthoCodingSequence(PlantTribes): file_ext = "ptorthocs" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesOrthoCodingSequence, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesOrthoCodingSequence, self).set_peek(dataset) dataset.blurb = "Protein and coding sequences orthogroup fasta files: %d items" % dataset.metadata.num_files @@ -182,7 +182,7 @@ class PlantTribesTargetedGeneFamilies(PlantTribes): file_ext = "pttgf" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesTargetedGeneFamilies, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesTargetedGeneFamilies, self).set_peek(dataset) dataset.blurb = "Targeted gene families" @@ -194,7 +194,7 @@ class PlantTribesPhylogeneticTree(PlantTribes): file_ext = "pttree" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesPhylogeneticTree, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesPhylogeneticTree, self).set_peek(dataset) dataset.blurb = "Phylogenetic trees: %d items" % dataset.metadata.num_files @@ -205,7 +205,7 @@ class PlantTribesPhylip(PlantTribes): file_ext = "ptphylip" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesPhylip, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesPhylip, self).set_peek(dataset) dataset.blurb = "Orthogroup phylip multiple sequence alignments: %d items" % dataset.metadata.num_files @@ -216,7 +216,7 @@ class PlantTribesMultipleSequenceAlignment(PlantTribes): file_ext = "ptalign" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesMultipleSequenceAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesMultipleSequenceAlignment, self).set_peek(dataset) dataset.blurb = "Proteins orthogroup alignments: %d items" % dataset.metadata.num_files @@ -227,7 +227,7 @@ class PlantTribesMultipleSequenceAlignmentCodonAlignment(PlantTribes): file_ext = "ptalignca" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesMultipleSequenceAlignmentCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesMultipleSequenceAlignmentCodonAlignment, self).set_peek(dataset) dataset.blurb = "Protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files @@ -238,7 +238,7 @@ class PlantTribesMultipleSequenceAlignmentTrimmed(PlantTribes): file_ext = "ptaligntrimmed" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesMultipleSequenceAlignmentTrimmed, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesMultipleSequenceAlignmentTrimmed, self).set_peek(dataset) dataset.blurb = "Trimmed proteins orthogroup alignments: %d items" % dataset.metadata.num_files @@ -249,7 +249,7 @@ class PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment(PlantTribes): file_ext = "ptaligntrimmedca" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesMultipleSequenceAlignmentTrimmedCodonAlignment, self).set_peek(dataset) dataset.blurb = "Trimmed protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files @@ -260,7 +260,7 @@ class PlantTribesMultipleSequenceAlignmentFiltered(PlantTribes): file_ext = "ptalignfiltered" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesMultipleSequenceAlignmentFiltered, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesMultipleSequenceAlignmentFiltered, self).set_peek(dataset) dataset.blurb = "Filtered proteins orthogroup alignments: %d items" % dataset.metadata.num_files @@ -271,5 +271,5 @@ class PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment(PlantTribes): file_ext = "ptalignfilteredca" def set_peek(self, dataset, is_multi_byte=False): - super(PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment, self).set_peek(dataset, is_multi_byte=is_multi_byte) + super(PlantTribesMultipleSequenceAlignmentFilteredCodonAlignment, self).set_peek(dataset) dataset.blurb = "Filtered protein and coding sequences orthogroup alignments: %d items" % dataset.metadata.num_files diff --git a/lib/galaxy/datatypes/proteomics.py b/lib/galaxy/datatypes/proteomics.py index 27e923c2b3b..932e0fe90fa 100644 --- a/lib/galaxy/datatypes/proteomics.py +++ b/lib/galaxy/datatypes/proteomics.py @@ -114,7 +114,7 @@ class ProteomicsXml(GenericXml): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = self.blurb else: dataset.peek = 'file does not exist' @@ -221,7 +221,7 @@ class Mgf(Text): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'mgf Mascot Generic Format' else: dataset.peek = 'file does not exist' @@ -249,7 +249,7 @@ class MascotDat(Text): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'mascotdat Mascot Search Results' else: dataset.peek = 'file does not exist' @@ -334,7 +334,7 @@ class SPLibNoIndex(Text): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'Spectral Library without index files' else: dataset.peek = 'file does not exist' @@ -374,7 +374,7 @@ class SPLib(Msp): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'splib Spectral Library Format' else: dataset.peek = 'file does not exist' diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index ec47b34235d..fe521fb49b5 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -61,7 +61,7 @@ class SequenceSplitLocations(data.Text): try: parsed_data = json.load(open(dataset.file_name)) # dataset.peek = json.dumps(data, sort_keys=True, indent=4) - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = '%d sections' % len(parsed_data['sections']) except Exception: dataset.peek = 'Not FQTOC file' @@ -112,7 +112,7 @@ class Sequence(data.Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) if dataset.metadata.sequences: dataset.blurb = "%s sequences" % util.commaify(str(dataset.metadata.sequences)) else: @@ -861,7 +861,7 @@ class Maf(Alignment): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: # The file must exist on disk for the get_file_peek() method - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) if dataset.metadata.blocks: dataset.blurb = "%s blocks" % util.commaify(str(dataset.metadata.blocks)) else: diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 23c79061bcb..e8de38168a3 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -20,7 +20,6 @@ from galaxy.datatypes.binary import Binary from galaxy.util import ( compression_utils, multi_byte, - unicodify ) from galaxy.util.checkers import ( check_binary, @@ -204,17 +203,11 @@ def convert_newlines_sep2tabs(fname, in_place=True, patt="\\s+", tmp_dir=None, t return (i + 1, temp_name) -def iter_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=None): +def iter_headers(fname, sep, count=60, comment_designator=None): with compression_utils.get_fileobj(fname) as in_file: idx = 0 for line in in_file: line = line.rstrip('\n\r') - if is_multi_byte: - # TODO: fix this - sep is never found in line - line = unicodify(line, 'utf-8') - sep = sep.encode('utf-8') - if comment_designator is not None and comment_designator != '': - comment_designator = comment_designator.encode('utf-8') if comment_designator is not None and comment_designator != '' and line.startswith(comment_designator): continue yield line.split(sep) @@ -223,22 +216,22 @@ def iter_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=N break -def get_headers(fname, sep, count=60, is_multi_byte=False, comment_designator=None): +def get_headers(fname, sep, count=60, comment_designator=None): """ Returns a list with the first 'count' lines split by 'sep', ignoring lines starting with 'comment_designator' >>> fname = get_test_fname('complete.bed') - >>> get_headers(fname,'\\t') - [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']] + >>> get_headers(fname,'\\t') == [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']] + True >>> fname = get_test_fname('test.gff') - >>> get_headers(fname, '\\t', count=5, comment_designator='#') - [[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']] + >>> get_headers(fname, '\\t', count=5, comment_designator='#') == [[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']] + True """ - return list(iter_headers(fname=fname, sep=sep, count=count, is_multi_byte=is_multi_byte, comment_designator=comment_designator)) + return list(iter_headers(fname=fname, sep=sep, count=count, comment_designator=comment_designator)) -def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False): +def is_column_based(fname, sep='\t', skip=0): """ Checks whether the file is column based with respect to a separator (defaults to tab separator). @@ -266,7 +259,10 @@ def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False): >>> is_column_based(fname) True """ - headers = get_headers(fname, sep, is_multi_byte=is_multi_byte) + try: + headers = get_headers(fname, sep) + except UnicodeDecodeError: + return False count = 0 if not headers: return False @@ -284,7 +280,7 @@ def is_column_based(fname, sep='\t', skip=0, is_multi_byte=False): return True -def guess_ext(fname, sniff_order, is_multi_byte=False): +def guess_ext(fname, sniff_order): """ Returns an extension that can be used in the datatype factory to generate a data for the 'fname' file @@ -417,28 +413,16 @@ def guess_ext(fname, sniff_order, is_multi_byte=False): # to tsv but it doesn't have a sniffer - is TSV was sniffed just check # if it is an okay tabular and use that instead. if file_ext == 'tsv': - if is_column_based(fname, '\t', 1, is_multi_byte=is_multi_byte): + if is_column_based(fname, '\t', 1): file_ext = 'tabular' if file_ext is not None: return file_ext - headers = get_headers(fname, None) - is_binary = False - if is_multi_byte: - is_binary = False - else: - for hdr in headers: - for char in hdr: - # old behavior had 'char' possibly having length > 1, - # need to determine when/if this occurs - is_binary = util.is_binary(char) - if is_binary: - break - if is_binary: - break - if is_binary: + try: + get_headers(fname, None) + except UnicodeDecodeError: return 'data' # default binary data type file extension - if is_column_based(fname, '\t', 1, is_multi_byte=is_multi_byte): + if is_column_based(fname, '\t', 1): return 'tabular' # default tabular data type file extension return 'txt' # default text data type file extension @@ -492,14 +476,14 @@ def handle_compressed_file(filename, datatypes_registry, ext='auto'): return is_valid, ext -def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto', is_multi_byte=False): +def handle_uploaded_dataset_file(filename, datatypes_registry, ext='auto'): is_valid, ext = handle_compressed_file(filename, datatypes_registry, ext=ext) if not is_valid: raise InappropriateDatasetContentError('The compressed uploaded file contains inappropriate content.') if ext in AUTO_DETECT_EXTENSIONS: - ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order, is_multi_byte=is_multi_byte) + ext = guess_ext(filename, sniff_order=datatypes_registry.sniff_order) if check_binary(filename): if not Binary.is_ext_unsniffable(ext) and not datatypes_registry.get_datatype_by_extension(ext).sniff(filename): diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 731ec358464..7f94565fb89 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -51,7 +51,7 @@ class TabularData(data.Text): raise NotImplementedError def set_peek(self, dataset, line_count=None, is_multi_byte=False, WIDTH=256, skipchars=None): - super(TabularData, self).set_peek(dataset, line_count=line_count, is_multi_byte=is_multi_byte, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False) + super(TabularData, self).set_peek(dataset, line_count=line_count, WIDTH=WIDTH, skipchars=skipchars, line_wrap=False) if dataset.metadata.comment_lines: dataset.blurb = "%s, %s comments" % (dataset.blurb, util.commaify(str(dataset.metadata.comment_lines))) @@ -825,7 +825,7 @@ class Eland(Tabular): - LANE, TILEm X, Y, INDEX, READ_NO, SEQ, QUAL, POSITION, *STRAND, FILT must be correct - We will only check that up to the first 5 alignments are correctly formatted. """ - with compression_utils.get_fileobj(filename, gzip_only=True) as fh: + with compression_utils.get_fileobj(filename, compressed_formats=['gzip']) as fh: count = 0 while True: line = fh.readline() diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 5882593cb03..761e6a6ea78 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -62,7 +62,7 @@ class Json(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "JavaScript Object Notation (JSON)" else: dataset.peek = 'file does not exist' @@ -110,7 +110,7 @@ class Ipynb(Json): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "Jupyter Notebook" else: dataset.peek = 'file does not exist' @@ -183,7 +183,7 @@ class Biom1(Json): MetadataElement(name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) def set_peek(self, dataset, is_multi_byte=False): - super(Biom1, self).set_peek(dataset, is_multi_byte) + super(Biom1, self).set_peek(dataset) if not dataset.dataset.purged: dataset.blurb = "Biological Observation Matrix v1" @@ -267,7 +267,7 @@ class Obo(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "Open Biomedical Ontology (OBO)" else: dataset.peek = 'file does not exist' @@ -306,7 +306,7 @@ class Arff(Text): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = get_file_peek(dataset.file_name) dataset.blurb = "Attribute-Relation File Format (ARFF)" dataset.blurb += ", %s comments, %s attributes" % (dataset.metadata.comment_lines, dataset.metadata.columns) else: diff --git a/lib/galaxy/datatypes/triples.py b/lib/galaxy/datatypes/triples.py index 377a47efd3d..c8ce7631d60 100644 --- a/lib/galaxy/datatypes/triples.py +++ b/lib/galaxy/datatypes/triples.py @@ -31,7 +31,7 @@ class Triples(data.Data): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'Triple data' else: dataset.peek = 'file does not exist' @@ -55,7 +55,7 @@ class NTriples(data.Text, Triples): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'N-Triples triple data' else: dataset.peek = 'file does not exist' @@ -78,7 +78,7 @@ class N3(data.Text, Triples): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'Notation-3 Triple data' else: dataset.peek = 'file does not exist' @@ -105,7 +105,7 @@ class Turtle(data.Text, Triples): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'Turtle triple data' else: dataset.peek = 'file does not exist' @@ -132,7 +132,7 @@ class Rdf(xml.GenericXml, Triples): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'RDF/XML triple data' else: dataset.peek = 'file does not exist' @@ -158,7 +158,7 @@ class Jsonld(text.Json, Triples): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'JSON-LD triple data' else: dataset.peek = 'file does not exist' @@ -181,7 +181,7 @@ class HDT(binary.Binary, Triples): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'HDT triple data' else: dataset.peek = 'file does not exist' diff --git a/lib/galaxy/datatypes/xml.py b/lib/galaxy/datatypes/xml.py index aba2272c2a7..18ab8009b8f 100644 --- a/lib/galaxy/datatypes/xml.py +++ b/lib/galaxy/datatypes/xml.py @@ -21,7 +21,7 @@ class GenericXml(data.Text): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'XML data' else: dataset.peek = 'file does not exist' @@ -68,7 +68,7 @@ class MEMEXml(GenericXml): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'MEME XML data' else: dataset.peek = 'file does not exist' @@ -85,7 +85,7 @@ class CisML(GenericXml): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'CisML data' else: dataset.peek = 'file does not exist' @@ -104,7 +104,7 @@ class Phyloxml(GenericXml): def set_peek(self, dataset, is_multi_byte=False): """Set the peek and blurb text""" if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = 'Phyloxml data' else: dataset.peek = 'file does not exist' @@ -139,7 +139,7 @@ class Owl(GenericXml): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = data.get_file_peek(dataset.file_name, is_multi_byte=is_multi_byte) + dataset.peek = data.get_file_peek(dataset.file_name) dataset.blurb = "Web Ontology Language OWL" else: dataset.peek = 'file does not exist' diff --git a/lib/galaxy/jobs/__init__.py b/lib/galaxy/jobs/__init__.py index 58f393eab03..8c82a17da06 100644 --- a/lib/galaxy/jobs/__init__.py +++ b/lib/galaxy/jobs/__init__.py @@ -1287,15 +1287,9 @@ class JobWrapper(object, HasResourceParameters): dataset.metadata.from_JSON_dict(output_filename, path_rewriter=path_rewriter) try: assert context.get('line_count', None) is not None - if (not dataset.datatype.composite_type and dataset.dataset.is_multi_byte()) or self.tool.is_multi_byte: - dataset.set_peek(line_count=context['line_count'], is_multi_byte=True) - else: - dataset.set_peek(line_count=context['line_count']) + dataset.set_peek(line_count=context['line_count']) except Exception: - if (not dataset.datatype.composite_type and dataset.dataset.is_multi_byte()) or self.tool.is_multi_byte: - dataset.set_peek(is_multi_byte=True) - else: - dataset.set_peek() + dataset.set_peek() else: # Handle an empty dataset. dataset.blurb = "empty" diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 147090a6171..01689a1a555 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -2071,8 +2071,8 @@ class DatasetInstance(object): """Data consists of multi-byte characters""" return self.dataset.is_multi_byte() - def set_peek(self, is_multi_byte=False): - return self.datatype.set_peek(self, is_multi_byte=is_multi_byte) + def set_peek(self): + return self.datatype.set_peek(self) def init_meta(self, copy_from=None): return self.datatype.init_meta(self, copy_from=copy_from) diff --git a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py index 4a86c22d983..c40ec37e151 100644 --- a/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py +++ b/lib/galaxy/model/migrate/versions/0005_cleanup_datasets_fix.py @@ -241,8 +241,8 @@ class DatasetInstance(object): """Saves the data on the disc""" self.datatype.set_raw_data(self, data) - def set_peek(self, is_multi_byte=False): - return self.datatype.set_peek(self, is_multi_byte=is_multi_byte) + def set_peek(self): + return self.datatype.set_peek(self) def init_meta(self, copy_from=None): return self.datatype.init_meta(self, copy_from=copy_from) diff --git a/lib/galaxy/tools/verify/__init__.py b/lib/galaxy/tools/verify/__init__.py index c2f873dfa45..82fbfce9eb4 100644 --- a/lib/galaxy/tools/verify/__init__.py +++ b/lib/galaxy/tools/verify/__init__.py @@ -168,77 +168,74 @@ def files_diff(file1, file2, attributes=None): if (line.startswith('+') and not line.startswith('+++')) or (line.startswith('-') and not line.startswith('---')): count += 1 return count + if not filecmp.cmp(file1, file2): - files_differ = False if attributes is None: attributes = {} decompress = attributes.get("decompress", None) - if not decompress: - local_file = open(file1, 'U').readlines() - history_data = open(file2, 'U').readlines() + if decompress: + # None means all compressed formats are allowed + compressed_formats = None else: - local_file = get_fileobj(file1, 'U').readlines() - history_data = get_fileobj(file2, 'U').readlines() + compressed_formats = [] + is_pdf = False + try: + local_file = get_fileobj(file1, 'U', compressed_formats=compressed_formats).readlines() + history_data = get_fileobj(file2, 'U', compressed_formats=compressed_formats).readlines() + except UnicodeDecodeError: + if file1.endswith('.pdf') or file2.endswith('.pdf'): + is_pdf = True + local_file = open(file1, 'rb').readlines() + history_data = open(file2, 'rb').readlines() + else: + raise AssertionError("Binary data detected, not displaying diff") if attributes.get('sort', False): history_data.sort() - # Why even bother with the check loop below, why not just use the diff output? This seems wasteful. - if len(local_file) == len(history_data): - for i in range(len(history_data)): - if local_file[i].rstrip('\r\n') != history_data[i].rstrip('\r\n'): - files_differ = True - break - else: - files_differ = True - if files_differ: - allowed_diff_count = int(attributes.get('lines_diff', 0)) - diff = list(difflib.unified_diff(local_file, history_data, "local_file", "history_data")) - diff_lines = get_lines_diff(diff) - if diff_lines > allowed_diff_count: - if 'GALAXY_TEST_RAW_DIFF' in os.environ: - diff_slice = diff + allowed_diff_count = int(attributes.get('lines_diff', 0)) + diff = list(difflib.unified_diff(local_file, history_data, "local_file", "history_data")) + diff_lines = get_lines_diff(diff) + if diff_lines > allowed_diff_count: + if 'GALAXY_TEST_RAW_DIFF' in os.environ: + diff_slice = diff + else: + if len(diff) < 60: + diff_slice = diff[0:40] else: - if len(diff) < 60: - diff_slice = diff[0:40] - else: - diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:] - # FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff - # due to unknown desired behavior when used in conjunction with a non-zero lines_diff - # PDF forgiveness can probably be handled better by not special casing by __extension__ here - # and instead using lines_diff or a regular expression matching - # or by creating and using a specialized pdf comparison function - if file1.endswith('.pdf') or file2.endswith('.pdf'): - # PDF files contain creation dates, modification dates, ids and descriptions that change with each - # new file, so we need to handle these differences. As long as the rest of the PDF file does - # not differ we're ok. - valid_diff_strs = ['description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator'] - valid_diff = False - invalid_diff_lines = 0 - for line in diff_slice: - # Make sure to lower case strings before checking. - line = line.lower() - # Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n' - if (line.startswith('+') or line.startswith('-')) and line.find('local_file') < 0 and line.find('history_data') < 0: - for vdf in valid_diff_strs: - if line.find(vdf) < 0: - valid_diff = False - else: - valid_diff = True - # Stop checking as soon as we know we have a valid difference - break - if not valid_diff: - invalid_diff_lines += 1 - log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1, file2, allowed_diff_count, diff_lines, invalid_diff_lines)) - if invalid_diff_lines > allowed_diff_count: - # Print out diff_slice so we can see what failed - log.info("###### diff_slice ######") - raise AssertionError("".join(diff_slice)) - else: - log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1, file2, allowed_diff_count, diff_lines)) - for line in diff_slice: - for char in line: - if ord(char) > 128: - raise AssertionError("Binary data detected, not displaying diff") + diff_slice = diff[:25] + ["********\n", "*SNIP *\n", "********\n"] + diff[-25:] + # FIXME: This pdf stuff is rather special cased and has not been updated to consider lines_diff + # due to unknown desired behavior when used in conjunction with a non-zero lines_diff + # PDF forgiveness can probably be handled better by not special casing by __extension__ here + # and instead using lines_diff or a regular expression matching + # or by creating and using a specialized pdf comparison function + if is_pdf: + # PDF files contain creation dates, modification dates, ids and descriptions that change with each + # new file, so we need to handle these differences. As long as the rest of the PDF file does + # not differ we're ok. + valid_diff_strs = ['description', 'createdate', 'creationdate', 'moddate', 'id', 'producer', 'creator'] + valid_diff = False + invalid_diff_lines = 0 + for line in diff_slice: + # Make sure to lower case strings before checking. + line = line.lower() + # Diff lines will always start with a + or - character, but handle special cases: '--- local_file \n', '+++ history_data \n' + if (line.startswith('+') or line.startswith('-')) and line.find('local_file') < 0 and line.find('history_data') < 0: + for vdf in valid_diff_strs: + if line.find(vdf) < 0: + valid_diff = False + else: + valid_diff = True + # Stop checking as soon as we know we have a valid difference + break + if not valid_diff: + invalid_diff_lines += 1 + log.info('## files diff on %s and %s lines_diff=%d, found diff = %d, found pdf invalid diff = %d' % (file1, file2, allowed_diff_count, diff_lines, invalid_diff_lines)) + if invalid_diff_lines > allowed_diff_count: + # Print out diff_slice so we can see what failed + log.info("###### diff_slice ######") raise AssertionError("".join(diff_slice)) + else: + log.info('## files diff on %s and %s lines_diff=%d, found diff = %d' % (file1, file2, allowed_diff_count, diff_lines)) + raise AssertionError("".join(diff_slice)) def files_re_match(file1, file2, attributes=None): diff --git a/lib/galaxy/util/compression_utils.py b/lib/galaxy/util/compression_utils.py index 8d46e80f8b4..7f9ebd4356f 100644 --- a/lib/galaxy/util/compression_utils.py +++ b/lib/galaxy/util/compression_utils.py @@ -1,4 +1,5 @@ import gzip +import io import sys import zipfile @@ -9,32 +10,44 @@ from .checkers import ( if sys.version_info < (3, 3): import bz2file as bz2 + gzip.GzipFile.read1 = gzip.GzipFile.read # workaround for https://bugs.python.org/issue12591 else: import bz2 -def get_fileobj(filename, mode="r", gzip_only=False, bz2_only=False, zip_only=False): +def get_fileobj(filename, mode="r", compressed_formats=None): """ - Returns a fileobj. If the file is compressed, return appropriate file reader. + Returns a fileobj. If the file is compressed, return an appropriate file + reader. In text mode, always use 'utf-8' encoding. :param filename: path to file that should be opened :param mode: mode to pass to opener - :param gzip_only: only open file if file is gzip compressed or not compressed - :param bz2_only: only open file if file is bz2 compressed or not compressed - :param zip_only: only open file if file is zip compressed or not compressed + :param compressed_formats: list of allowed compressed file formats among + 'bz2', 'gzip' and 'zip'. If left to None, all 3 formats are allowed """ + if compressed_formats is None: + compressed_formats = ['bz2', 'gzip', 'zip'] + # Remove 't' from mode, which may cause an error for compressed files + mode = mode.replace('t', '') # the various compression readers don't support 'U' mode, # so we open in 'r'. if mode == 'U': cmode = 'r' else: cmode = mode - if not bz2_only and not zip_only and is_gzip(filename): - return gzip.GzipFile(filename, cmode) - if not gzip_only and not zip_only and is_bz2(filename): - return bz2.BZ2File(filename, cmode) - if not bz2_only and not gzip_only and zipfile.is_zipfile(filename): + if 'gzip' in compressed_formats and is_gzip(filename): + fh = gzip.GzipFile(filename, cmode) + elif 'bz2' in compressed_formats and is_bz2(filename): + fh = bz2.BZ2File(filename, cmode) + elif 'zip' in compressed_formats and zipfile.is_zipfile(filename): # Return fileobj for the first file in a zip file. with zipfile.ZipFile(filename, cmode) as zh: - return zh.open(zh.namelist()[0], cmode) - return open(filename, mode) + fh = zh.open(zh.namelist()[0], cmode) + elif 'b' in mode: + return open(filename, mode) + else: + return io.open(filename, mode, encoding='utf-8') + if 'b' not in mode: + return io.TextIOWrapper(fh, encoding='utf-8') + else: + return fh diff --git a/tools/data_source/data_source.py b/tools/data_source/data_source.py index 93a2c62bee5..01768c69a94 100644 --- a/tools/data_source/data_source.py +++ b/tools/data_source/data_source.py @@ -105,7 +105,7 @@ def __main__(): # here import checks that upload tool performs if enhanced_handling: try: - ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict['ext'], is_multi_byte=is_multi_byte) + ext = sniff.handle_uploaded_dataset_file(filename, datatypes_registry, ext=data_dict['ext']) except Exception as e: stop_err(str(e)) info = dict(type='dataset', diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 03d57ae7d15..1bc55f0469b 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -124,7 +124,7 @@ def add_file(dataset, registry, json_file, output_path): # Is dataset content multi-byte? elif dataset.is_multi_byte: data_type = 'multi-byte char' - ext = sniff.guess_ext(dataset.path, registry.sniff_order, is_multi_byte=True) + ext = sniff.guess_ext(dataset.path, registry.sniff_order) # Is dataset content supported sniffable binary? else: # FIXME: This ignores the declared sniff order in datatype_conf.xml