From 0029949ed2ead12735ce052aca328cc783139da6 Mon Sep 17 00:00:00 2001
From: John Chilton <jmchilton@gmail.com>
Date: Thu, 7 Apr 2016 09:02:25 -0400
Subject: [PATCH] More fixups for specialized, more correct tabular datatypes.

Followed through with @nsoranzo's comment:

1. overwrite CSV.sniff() and CSV.set_meta() methods with the ones from BaseCSV
2. remove BaseCSV class and rename CSV to BaseCSV
3. rename ExcelCSV class to CSV

Also renamed ExcelTSV to TSV and corrected another bug in sample datatypes noticed by @nsoranzo.
---
 config/datatypes_conf.xml.sample |  8 ++---
 lib/galaxy/datatypes/tabular.py  | 61 ++++----------------------------
 2 files changed, 10 insertions(+), 59 deletions(-)

diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample
index d1a8dacd765..44aee3e5531 100644
--- a/config/datatypes_conf.xml.sample
+++ b/config/datatypes_conf.xml.sample
@@ -52,8 +52,8 @@
       <converter file="interval_to_tabix_converter.xml" target_datatype="tabix" depends_on="bgzip"/>
       <converter file="bed_gff_or_vcf_to_bigwig_converter.xml" target_datatype="bigwig"/>
     </datatype>
-    <datatype extension="csv" type="galaxy.datatypes.tabular:ExcelCSV" display_in_upload="true" />
-    <datatype extension="tsv" type="galaxy.datatypes.tabular:ExcelCSV" display_in_upload="true" />
+    <datatype extension="csv" type="galaxy.datatypes.tabular:CSV" display_in_upload="true" />
+    <datatype extension="tsv" type="galaxy.datatypes.tabular:TSV" display_in_upload="true" />
     <datatype extension="customtrack" type="galaxy.datatypes.interval:CustomTrack"/>
     <datatype extension="bowtie_color_index" type="galaxy.datatypes.ngsindex:BowtieColorIndex" mimetype="text/html" display_in_upload="False"/>
     <datatype extension="bowtie_base_index" type="galaxy.datatypes.ngsindex:BowtieBaseIndex" mimetype="text/html" display_in_upload="False"/>
@@ -570,8 +570,8 @@
     <sniffer type="galaxy.datatypes.sequence:RNADotPlotMatrix"/>
     <sniffer type="galaxy.datatypes.sequence:DotBracket"/>
     <sniffer type="galaxy.datatypes.tabular:ConnectivityTable"/>
-    <sniffer type="galaxy.datatypes.tabular:ExcelCSV"/>
-    <sniffer type="galaxy.datatypes.tabular:ExcelTSV"/>
+    <sniffer type="galaxy.datatypes.tabular:CSV"/>
+    <sniffer type="galaxy.datatypes.tabular:TSV"/>
     <sniffer type="galaxy.datatypes.msa:Hmmer2" />
     <sniffer type="galaxy.datatypes.msa:Hmmer3" />
     <sniffer type="galaxy.datatypes.msa:Stockholm_1_0" />
diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py
index 75257f1d9f1..4c5038c17ed 100644
--- a/lib/galaxy/datatypes/tabular.py
+++ b/lib/galaxy/datatypes/tabular.py
@@ -875,18 +875,18 @@ class FeatureLocationIndex( Tabular ):
 
 
 @dataproviders.decorators.has_dataproviders
-class CSV( TabularData ):
+class BaseCSV( TabularData ):
     """
     Delimiter-separated table data.
     This includes CSV, TSV and other dialects understood by the
     Python 'csv' module https://docs.python.org/2/library/csv.html
-
-    WARNING: This type is BUGGY it is kept purely for backward compatibility
-    It will incorrectly sniff tab separated files for which the get_meta method fails!
+    Must be extended to define the dialect to use, strict_width: and file_ext.
+    See Python module csv for documentation of dialect settings
     """
     delimiter = ','
     file_ext = 'csv'  # File extension
     peek_size = 1024  # File chunk used for sniffing CSV dialect
+    big_peek_size = 10240  # Large File chunk used for sniffing CSV dialect
 
     def is_int( self, column_text ):
         try:
@@ -912,55 +912,6 @@ class CSV( TabularData ):
         else:
             return 'str'
 
-    def sniff( self, filename ):
-        """ Return True if if recognizes dialect and header. """
-        if not csv.Sniffer().has_header(open(filename, 'r').read(self.peek_size)):
-            return False
-        # Fetch at least three consecutive lines to be reasonably sure
-        reader = csv.reader(open(filename, 'r'))
-        for i in range(0, 3):
-            reader.next()
-        return True
-
-    def set_meta( self, dataset, **kwd ):
-        with open(dataset.file_name, 'r') as csvfile:
-            # Parse file
-            reader = csv.reader(csvfile)
-            data_row = None
-            header_row = None
-            try:
-                header_row = reader.next()
-                data_row = reader.next()
-                for row in reader:
-                    pass
-            except csv.Error as e:
-                raise Exception('CSV reader error - line %d: %s' % (reader.line_num, e))
-
-            # Guess column types
-            column_types = []
-            for cell in data_row:
-                column_types.append(self.guess_type(cell))
-
-            # Set metadata
-            dataset.metadata.data_lines = reader.line_num - 1
-            dataset.metadata.comment_lines = 1
-            dataset.metadata.column_types = column_types
-            dataset.metadata.columns = max( len( header_row ), len( data_row ) )
-            dataset.metadata.column_names = header_row
-            dataset.metadata.delimiter = reader.dialect.delimiter
-
-
-@dataproviders.decorators.has_dataproviders
-class BaseCSV( CSV ):
-    """
-    Delimiter-separated table data.
-    This includes CSV, TSV and other dialects understood by the
-    Python 'csv' module https://docs.python.org/2/library/csv.html
-    Must be extended to define the dialect to use, strict_width: and file_ext.
-    See Python module csv for documentation of dialect settings
-    """
-    big_peek_size = 10240  # Large File chunk used for sniffing CSV dialect
-
     def sniff( self, filename ):
         """ Return True if if recognizes dialect and header. """
         try:
@@ -1042,7 +993,7 @@ class BaseCSV( CSV ):
 
 
 @dataproviders.decorators.has_dataproviders
-class ExcelCSV( BaseCSV ):
+class CSV( BaseCSV ):
     """
     Comma separated table data.
     Only sniffs comma separated files with at least 2 columns
@@ -1056,7 +1007,7 @@ class ExcelCSV( BaseCSV ):
 
 
 @dataproviders.decorators.has_dataproviders
-class ExcelTSV( BaseCSV ):
+class TSV( BaseCSV ):
     """
     Comma separated table data.
     Only sniff tab separated files with at least two columns