From c9db3f56bbe70ea3d578d16985c5c38c73ec5e42 Mon Sep 17 00:00:00 2001 From: Christian-B Date: Mon, 23 Nov 2015 14:28:43 +0000 Subject: [PATCH 1/7] Alternative stricter csv formats --- lib/galaxy/datatypes/tabular.py | 142 ++++++++++++++++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 2c4bf3be072..fbb66d6da9c 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -875,6 +875,9 @@ class CSV( TabularData ): Delimiter-separated table data. This includes CSV, TSV and other dialects understood by the Python 'csv' module https://docs.python.org/2/library/csv.html + + WARNING: This type is BUGGY it is kept purely for backward compatability + It will incorrectly sniff tab seperated files for which the get_meta method fails! """ delimiter = ',' file_ext = 'csv' # File extension @@ -905,6 +908,7 @@ class CSV( TabularData ): return 'str' def sniff( self, filename ): + log.info ("all csv sniff called") """ Return True if if recognizes dialect and header. """ if not csv.Sniffer().has_header(open(filename, 'r').read(self.peek_size)): return False @@ -942,6 +946,144 @@ class CSV( TabularData ): dataset.metadata.delimiter = reader.dialect.delimiter +@dataproviders.decorators.has_dataproviders +class Base_CSV( CSV ): + """ + Delimiter-separated table data. + This includes CSV, TSV and other dialects understood by the + Python 'csv' module https://docs.python.org/2/library/csv.html + Must be extended to define the dialect to use, strict_width: and file_ext. + See Python module csv for documentation of dialect settings + """ + #dialect Set by subclass + #file_ext Set by subclass + #strict_width Set by subclass + #If set sniff fails is a single row is incorrect. + #Python's csv is more tollerant + big_peek_size = 10240 # Large File chunk used for sniffing CSV dialect + + def sniff( self, filename ): + """ Return True if if recognizes dialect and header. """ + try: + #check the dialect works + reader = csv.reader(open(filename, 'r'), self.dialect) + #Check we can read header and get columns + header_row = reader.next() + if len(header_row) < 2: + #No columns so not seperated by this dialect. + return False + + #check all rows can be read as otherwise set_meta throws an exception + if self.strict_width: + num_columns = len(header_row) + for data_row in reader: + #All columns must be the same length + if num_columns != len(data_row): + return False + else: + #Check the next row as it is used by set_meta + data_row = reader.next() + if len(data_row) < 2: + #No columns so not seperated by this dialect. + return False + #ignore the length in the rest + for data_row in reader: + pass + + #Optional: Check Python's csv comes up with a similar dialect + auto_dialect = csv.Sniffer().sniff(open(filename, 'r').read(self.big_peek_size)) + if (auto_dialect.delimiter != self.dialect.delimiter): + return False + if (auto_dialect.quotechar != self.dialect.quotechar): + return False + """ + Not checking for other dialect options + They may be mis detected from just the sample. + Or not effect the read such as doublequote + + Optional: Check for headers as in the past. + Note No way around Python's csv calling Sniffer.sniff again. + Note Without checking the dialect returned by sniff + this test may be checking the wrong dialect. + """ + if not csv.Sniffer().has_header(open(filename, 'r').read(self.big_peek_size)): + return False + + return True + except: + #Not readable by Python's csv using this dialect + return False + + def set_meta( self, dataset, **kwd ): + with open(dataset.file_name, 'r') as csvfile: + # Parse file with the correct dialect + reader = csv.reader(csvfile, self.dialect) + data_row = None + header_row = None + try: + header_row = reader.next() + data_row = reader.next() + for row in reader: + pass + except csv.Error as e: + raise Exception('CSV reader error - line %d: %s' % (reader.line_num, e)) + + # Guess column types + column_types = [] + for cell in data_row: + column_types.append(self.guess_type(cell)) + + # Set metadata + dataset.metadata.data_lines = reader.line_num - 1 + dataset.metadata.comment_lines = 1 + dataset.metadata.column_types = column_types + dataset.metadata.columns = max( len( header_row ), len( data_row ) ) + dataset.metadata.column_names = header_row + dataset.metadata.delimiter = reader.dialect.delimiter + + +@dataproviders.decorators.has_dataproviders +class Excell_CSV( Base_CSV ): + """ + Comma separated table data. + Only sniffs comma seperated files with at least 2 columns + """ + + def __init__(self, **kwd): + Base_CSV.__init__( self, **kwd ) + self.dialect = csv.excel # This is the default + #delimiter = ',' + #quotechar = '"' + #doublequote = True + #skipinitialspace = False + self.file_ext = 'csv' # File extension + self.strict_width = False # Previous csv type did not check column width + + +@dataproviders.decorators.has_dataproviders +class Excell_TSV( Base_CSV ): + """ + Comma separated table data. + Only sniff tab seperated files with at least two columns + + Note: Use of this datatype is optional as the general tabular format will handle most tab seperated files. + This datatye would only be required for dataset with tabs INSIDE double quotes. + + This datatype currently does not support tsv files where the header has one column less to indicate first column is row names + This kind of file is handled fine by tabular. + """ + + def __init__(self, **kwd): + Base_CSV.__init__( self, **kwd ) + self.dialect = csv.excel_tab + #delimiter = '\t' + #quotechar = '"' + #doublequote = True + #skipinitialspace = False + self.file_ext = 'tsv' # File extension + self.strict_width = True # Leave files with different width to tabular + + class ConnectivityTable( Tabular ): edam_format = "format_3309" file_ext = "ct" From fa7ccdf4b07598f5ebdfb1400f0bec2e2dabc164 Mon Sep 17 00:00:00 2001 From: Christian-B Date: Mon, 23 Nov 2015 14:49:18 +0000 Subject: [PATCH 2/7] Flake8 fix --- lib/galaxy/datatypes/tabular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index fbb66d6da9c..d90bcb1c227 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -908,7 +908,7 @@ class CSV( TabularData ): return 'str' def sniff( self, filename ): - log.info ("all csv sniff called") + log.info( "all csv sniff called" ) """ Return True if if recognizes dialect and header. """ if not csv.Sniffer().has_header(open(filename, 'r').read(self.peek_size)): return False From 79a7e31da7e6151c64b32f911fc60fcbdb78bba2 Mon Sep 17 00:00:00 2001 From: Christian-B Date: Mon, 23 Nov 2015 15:54:37 +0000 Subject: [PATCH 3/7] Remove bug log statement --- lib/galaxy/datatypes/tabular.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index d90bcb1c227..2456557cf88 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -908,7 +908,6 @@ class CSV( TabularData ): return 'str' def sniff( self, filename ): - log.info( "all csv sniff called" ) """ Return True if if recognizes dialect and header. """ if not csv.Sniffer().has_header(open(filename, 'r').read(self.peek_size)): return False From 7a960b08521ffa00e2984434aa870a49a96da0b5 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 4 Jan 2016 13:42:10 +0000 Subject: [PATCH 4/7] Small tweaks to CSV fixes by @Christian-B. - Update sample datatype conf. - Use Python naming conventions and PEP-8 style fixes. --- config/datatypes_conf.xml.sample | 8 +++--- lib/galaxy/datatypes/tabular.py | 43 +++++++++++--------------------- 2 files changed, 19 insertions(+), 32 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index d3505088219..87e29c3239f 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -51,9 +51,8 @@ - - - + + @@ -494,7 +493,8 @@ - + + diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 2456557cf88..92d6e833e10 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -946,7 +946,7 @@ class CSV( TabularData ): @dataproviders.decorators.has_dataproviders -class Base_CSV( CSV ): +class BaseCSV( CSV ): """ Delimiter-separated table data. This includes CSV, TSV and other dialects understood by the @@ -954,42 +954,37 @@ class Base_CSV( CSV ): Must be extended to define the dialect to use, strict_width: and file_ext. See Python module csv for documentation of dialect settings """ - #dialect Set by subclass - #file_ext Set by subclass - #strict_width Set by subclass - #If set sniff fails is a single row is incorrect. - #Python's csv is more tollerant big_peek_size = 10240 # Large File chunk used for sniffing CSV dialect def sniff( self, filename ): """ Return True if if recognizes dialect and header. """ try: - #check the dialect works + # check the dialect works reader = csv.reader(open(filename, 'r'), self.dialect) - #Check we can read header and get columns + # Check we can read header and get columns header_row = reader.next() if len(header_row) < 2: - #No columns so not seperated by this dialect. + # No columns so not seperated by this dialect. return False - #check all rows can be read as otherwise set_meta throws an exception + # check all rows can be read as otherwise set_meta throws an exception if self.strict_width: num_columns = len(header_row) for data_row in reader: - #All columns must be the same length + # All columns must be the same length if num_columns != len(data_row): return False else: - #Check the next row as it is used by set_meta + # Check the next row as it is used by set_meta data_row = reader.next() if len(data_row) < 2: - #No columns so not seperated by this dialect. + # No columns so not seperated by this dialect. return False - #ignore the length in the rest + # ignore the length in the rest for data_row in reader: pass - #Optional: Check Python's csv comes up with a similar dialect + # Optional: Check Python's csv comes up with a similar dialect auto_dialect = csv.Sniffer().sniff(open(filename, 'r').read(self.big_peek_size)) if (auto_dialect.delimiter != self.dialect.delimiter): return False @@ -1010,7 +1005,7 @@ class Base_CSV( CSV ): return True except: - #Not readable by Python's csv using this dialect + # Not readable by Python's csv using this dialect return False def set_meta( self, dataset, **kwd ): @@ -1042,25 +1037,21 @@ class Base_CSV( CSV ): @dataproviders.decorators.has_dataproviders -class Excell_CSV( Base_CSV ): +class ExcelCSV( BaseCSV ): """ Comma separated table data. Only sniffs comma seperated files with at least 2 columns """ def __init__(self, **kwd): - Base_CSV.__init__( self, **kwd ) + BaseCSV.__init__( self, **kwd ) self.dialect = csv.excel # This is the default - #delimiter = ',' - #quotechar = '"' - #doublequote = True - #skipinitialspace = False self.file_ext = 'csv' # File extension self.strict_width = False # Previous csv type did not check column width @dataproviders.decorators.has_dataproviders -class Excell_TSV( Base_CSV ): +class ExcelTSV( BaseCSV ): """ Comma separated table data. Only sniff tab seperated files with at least two columns @@ -1073,12 +1064,8 @@ class Excell_TSV( Base_CSV ): """ def __init__(self, **kwd): - Base_CSV.__init__( self, **kwd ) + BaseCSV.__init__( self, **kwd ) self.dialect = csv.excel_tab - #delimiter = '\t' - #quotechar = '"' - #doublequote = True - #skipinitialspace = False self.file_ext = 'tsv' # File extension self.strict_width = True # Leave files with different width to tabular From de506dec0c52e655a3fdff96368a288b6c454cde Mon Sep 17 00:00:00 2001 From: Christian-B Date: Tue, 23 Feb 2016 09:04:15 +0000 Subject: [PATCH 5/7] fixed spelling found by @nsoranzo --- lib/galaxy/datatypes/tabular.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 92d6e833e10..44dbb4eb15f 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -876,8 +876,8 @@ class CSV( TabularData ): This includes CSV, TSV and other dialects understood by the Python 'csv' module https://docs.python.org/2/library/csv.html - WARNING: This type is BUGGY it is kept purely for backward compatability - It will incorrectly sniff tab seperated files for which the get_meta method fails! + WARNING: This type is BUGGY it is kept purely for backward compatibility + It will incorrectly sniff tab separated files for which the get_meta method fails! """ delimiter = ',' file_ext = 'csv' # File extension @@ -964,7 +964,7 @@ class BaseCSV( CSV ): # Check we can read header and get columns header_row = reader.next() if len(header_row) < 2: - # No columns so not seperated by this dialect. + # No columns so not separated by this dialect. return False # check all rows can be read as otherwise set_meta throws an exception @@ -978,7 +978,7 @@ class BaseCSV( CSV ): # Check the next row as it is used by set_meta data_row = reader.next() if len(data_row) < 2: - # No columns so not seperated by this dialect. + # No columns so not separated by this dialect. return False # ignore the length in the rest for data_row in reader: @@ -1040,7 +1040,7 @@ class BaseCSV( CSV ): class ExcelCSV( BaseCSV ): """ Comma separated table data. - Only sniffs comma seperated files with at least 2 columns + Only sniffs comma separated files with at least 2 columns """ def __init__(self, **kwd): @@ -1054,10 +1054,10 @@ class ExcelCSV( BaseCSV ): class ExcelTSV( BaseCSV ): """ Comma separated table data. - Only sniff tab seperated files with at least two columns + Only sniff tab separated files with at least two columns - Note: Use of this datatype is optional as the general tabular format will handle most tab seperated files. - This datatye would only be required for dataset with tabs INSIDE double quotes. + Note: Use of this datatype is optional as the general tabular format will handle most tab separated files. + This datatype would only be required for dataset with tabs INSIDE double quotes. This datatype currently does not support tsv files where the header has one column less to indicate first column is row names This kind of file is handled fine by tabular. From b8cb5432a204742956267ba5b76f6eb4804983b4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Tue, 1 Mar 2016 07:41:18 +0000 Subject: [PATCH 6/7] Add test for #1818 to PR #1156. --- lib/galaxy/datatypes/sniff.py | 3 + lib/galaxy/datatypes/test/issue1818.tabular | 84 +++++++++++++++++++++ 2 files changed, 87 insertions(+) create mode 100644 lib/galaxy/datatypes/test/issue1818.tabular diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index d85807d1309..df9024bb6b8 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -324,6 +324,9 @@ def guess_ext( fname, sniff_order, is_multi_byte=False ): >>> fname = get_test_fname('test.mz5') >>> guess_ext(fname, sniff_order) 'h5' + >>> fname = get_test_fname('issue1818.tabular') + >>> guess_ext(fname, sniff_order) + 'tabular' """ for datatype in sniff_order: """ diff --git a/lib/galaxy/datatypes/test/issue1818.tabular b/lib/galaxy/datatypes/test/issue1818.tabular new file mode 100644 index 00000000000..cd575ebcab0 --- /dev/null +++ b/lib/galaxy/datatypes/test/issue1818.tabular @@ -0,0 +1,84 @@ +Name Major Score +Ignatius Engineering 83 +Austin Life-Sciences 91 +Zackery Engineering 54 +Marques Arts 58 +Darren Business 94 +Darius Social-Sciences 51 +Thanh Engineering 53 +Joe'Quann Engineering 75 +Bryan Arts 68 +Devin Engineering 92 +Joseph Social-Sciences 61 +Joshua Life-Sciences 14 +Ja'Won Social-Sciences 37 +Tyreque Arts 74 +Sage Arts 55 +Antonio Engineering 88 +Michael Engineering 39 +Randy Social-Sciences 68 +Dilan Health-Medicine 84 +Omar Engineering 99 +Zachary Arts 80 +Faison Engineering 47 +Angel Health-Medicine 100 +Gabriel Health-Medicine 100 +John Life-Sciences 70 +Leonard Business 87 +Juan Business 79 +Jonathan Health-Medicine 100 +Christopher Life-Sciences 59 +Brandon Life-Sciences 72 +D'Angelo Health-Medicine 90 +Justin Social-Sciences 90 +Israel Health-Medicine 81 +William Arts 46 +David Social-Sciences 69 +Drake Social-Sciences 59 +Drake Social-Sciences 76 +Nathan Arts 71 +Trevon Arts 74 +Aaron Business 83 +Daniel Health-Medicine 91 +Kevin Health-Medicine 100 +Antonio Engineering 56 +Donovan Arts 75 +Kerris Business 82 +Andre Health-Medicine 72 +Dakota Business 83 +Aaron Life-Sciences 58 +Walter Arts 75 +Isaiah Arts 80 +Christian Life-Sciences 67 +Dalton Health-Medicine 100 +Jesse Social-Sciences 32 +Diego Health-Medicine 82 +Nathen Life-Sciences 46 +Anthony Life-Sciences 32 +Christian Business 88 +David Business 92 +Avery Engineering 51 +Paul Arts 63 +Derek Arts 60 +Levi Arts 76 +Lance Social-Sciences 65 +Sonny Engineering 50 +Shawn Arts 65 +Leonardo Engineering 78 +Yeng Life-Sciences 39 +Leroy Social-Sciences 74 +Gurnam Life-Sciences 66 +Fernando Arts 78 +Williams Social-Sciences 62 +Roberto Arts 65 +Teriuse Business 94 +Nathaniel Arts 88 +Chase Social-Sciences 27 +Caleb Business 87 +Tysza Business 92 +Nico Arts 59 +Manuel Social-Sciences 61 +Patrick Health-Medicine 92 +Peter Health-Medicine 86 +Allen Life-Sciences 50 +Joel Social-Sciences 72 From 0029949ed2ead12735ce052aca328cc783139da6 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 7 Apr 2016 09:02:25 -0400 Subject: [PATCH 7/7] More fixups for specialized, more correct tabular datatypes. Followed through with @nsoranzo's comment: 1. overwrite CSV.sniff() and CSV.set_meta() methods with the ones from BaseCSV 2. remove BaseCSV class and rename CSV to BaseCSV 3. rename ExcelCSV class to CSV Also renamed ExcelTSV to TSV and corrected another bug in sample datatypes noticed by @nsoranzo. --- config/datatypes_conf.xml.sample | 8 ++--- lib/galaxy/datatypes/tabular.py | 61 ++++---------------------------- 2 files changed, 10 insertions(+), 59 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index d1a8dacd765..44aee3e5531 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -52,8 +52,8 @@ - - + + @@ -570,8 +570,8 @@ - - + + diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 75257f1d9f1..4c5038c17ed 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -875,18 +875,18 @@ class FeatureLocationIndex( Tabular ): @dataproviders.decorators.has_dataproviders -class CSV( TabularData ): +class BaseCSV( TabularData ): """ Delimiter-separated table data. This includes CSV, TSV and other dialects understood by the Python 'csv' module https://docs.python.org/2/library/csv.html - - WARNING: This type is BUGGY it is kept purely for backward compatibility - It will incorrectly sniff tab separated files for which the get_meta method fails! + Must be extended to define the dialect to use, strict_width: and file_ext. + See Python module csv for documentation of dialect settings """ delimiter = ',' file_ext = 'csv' # File extension peek_size = 1024 # File chunk used for sniffing CSV dialect + big_peek_size = 10240 # Large File chunk used for sniffing CSV dialect def is_int( self, column_text ): try: @@ -912,55 +912,6 @@ class CSV( TabularData ): else: return 'str' - def sniff( self, filename ): - """ Return True if if recognizes dialect and header. """ - if not csv.Sniffer().has_header(open(filename, 'r').read(self.peek_size)): - return False - # Fetch at least three consecutive lines to be reasonably sure - reader = csv.reader(open(filename, 'r')) - for i in range(0, 3): - reader.next() - return True - - def set_meta( self, dataset, **kwd ): - with open(dataset.file_name, 'r') as csvfile: - # Parse file - reader = csv.reader(csvfile) - data_row = None - header_row = None - try: - header_row = reader.next() - data_row = reader.next() - for row in reader: - pass - except csv.Error as e: - raise Exception('CSV reader error - line %d: %s' % (reader.line_num, e)) - - # Guess column types - column_types = [] - for cell in data_row: - column_types.append(self.guess_type(cell)) - - # Set metadata - dataset.metadata.data_lines = reader.line_num - 1 - dataset.metadata.comment_lines = 1 - dataset.metadata.column_types = column_types - dataset.metadata.columns = max( len( header_row ), len( data_row ) ) - dataset.metadata.column_names = header_row - dataset.metadata.delimiter = reader.dialect.delimiter - - -@dataproviders.decorators.has_dataproviders -class BaseCSV( CSV ): - """ - Delimiter-separated table data. - This includes CSV, TSV and other dialects understood by the - Python 'csv' module https://docs.python.org/2/library/csv.html - Must be extended to define the dialect to use, strict_width: and file_ext. - See Python module csv for documentation of dialect settings - """ - big_peek_size = 10240 # Large File chunk used for sniffing CSV dialect - def sniff( self, filename ): """ Return True if if recognizes dialect and header. """ try: @@ -1042,7 +993,7 @@ class BaseCSV( CSV ): @dataproviders.decorators.has_dataproviders -class ExcelCSV( BaseCSV ): +class CSV( BaseCSV ): """ Comma separated table data. Only sniffs comma separated files with at least 2 columns @@ -1056,7 +1007,7 @@ class ExcelCSV( BaseCSV ): @dataproviders.decorators.has_dataproviders -class ExcelTSV( BaseCSV ): +class TSV( BaseCSV ): """ Comma separated table data. Only sniff tab separated files with at least two columns