Enhanced the Filter tool to handle invalid data and to display more useful, informative messages to the user.

tools/stats/filtering.py

@@ -1,14 +1,23 @@
-# this is a tool that takes a textfile as input and 
-# creates filters columns based on certain properties
+#!/usr/bin/env python
+"""
+This tool takes a tab-delimited textfile as input and creates filters on columns based on certain properties.
+The tool will skip over invalid lines within the file, informing the user about the number of lines skipped.
+Invalid lines are those that do not follow the standard defined when the get_wrap_func function (immediately below)
+is applied to the first uncommented line in the input file.
+"""
 import sys, sets, re
 
 def get_wrap_func(value):
+    """
+    Determine the data type of each column in the input file
+    (valid data types for columns are either string or float)
+    """
     try:
         check = float(value)
         return 'float(%s)'
     except:
         return 'str(%s)'
-
+    
 def stop_err(msg):
     sys.stderr.write(msg)
     sys.exit()
@@ -16,7 +25,7 @@ def stop_err(msg):
 # we expect 4 parameters
 if len(sys.argv) != 4:
     print sys.argv
-    stop_err('Usage: python filter.py input_file ouput_file condition')
+    stop_err('Usage: python filtering.py input_file ouput_file condition')
 #debug 
 #cond_text = "(c2-c3) < 115487120 and c1=='chr7' "
 #sys.argv.extend( [ 'a.txt', 'b.txt', cond_text ])
@@ -33,14 +42,11 @@ mapped_str = {
     '__ge__': '>=',
     '__sq__': '\'',
     '__dq__': '"',
-
 }
 for key, value in mapped_str.items():
     cond_text = cond_text.replace(key, value)
 
-#
-# safety measures
-#
+# Safety measures
 safe_words = sets.Set( "c chr str float int split map lambda and or len not intronic intergenic proximal distal scaffold chrX chrY chrUn random contig ctg ctgY ctgX".split() )
 try:
     # filter on words
@@ -51,9 +57,9 @@ try:
 except Exception, e:
     stop_err("Cannot recognize the word %s in condition %s" % (e, cond_text) )
 
-#
-# guess how many columns there are and what wrappers are appropriate for each
-#
+"""
+Determine the number of columns in the input file and the data type for each
+"""
 elems = []
 fp = open(inp_file)
 while 1:
@@ -69,54 +75,63 @@ if not elems:
 if len(elems) == 1:
     if len(line.split()) != 1:
         stop_err('This tool can only be run on tab delimited files')
-#
-# prepare the variable names and wrappers
-#
+
+"""
+Prepare the column variable names and wrappers for column data types
+"""
 cols, funcs = [], []
 for ind, elem in enumerate(elems):
     name = 'c%d' % ( ind + 1 )
     cols.append(name)
     funcs.append(get_wrap_func(elem) % name)
 
-col   = ', '.join(cols)
-func  = ', '.join(funcs)
+col = ', '.join(cols)
+func = ', '.join(funcs)
 assign = "%s = line.split('\\t')" % col
-wrap   = "%s = %s" % (col, func)
-
+wrap = "%s = %s" % (col, func)
 flags = []
+skipped_lines = 0
+first_invalid_line = 0
+invalid_line = None
+
+# Read and filter input file, skipping invalid lines
 code = '''
-for line in file(inp_file):
+for i, line in enumerate( open( inp_file )):
     line = line.strip()
     if line and line[0] != '#':
-        %s
-        %s
-        if %s:
-            flags.append(True)
-        else:
+        try:
+            %s
+            %s
+            if %s:
+                flags.append(True)
+            else:
+                flags.append(False)
+        except:
+            skipped_lines += 1
             flags.append(False)
+            if not invalid_line:
+                first_invalid_line = i
+                invalid_line = line
     else:
-        flags.append(False)        
+        flags.append(False)
 ''' % (assign, wrap, cond_text)
 
-#
-# execute code
-#
-try:
-    exec code
-except Exception, e:
-    stop_err('Code %s raised error: %s' % (code, e))
+exec code
 
-#
-# create output
-#
+# Write filtered output file
 fp = open(out_file, 'wt')
 keep = 0
+total = 0
 for flag, line in zip(flags, file(inp_file)):
+    total += 1
     if flag:
         fp.write(line)
         keep  += 1
 fp.close()
 
-print 'Filtering with %s ' % cond_text
-print 'Kept %d lines (%4.2f%% of total)' % ( keep, 100.0*keep/len(flags) )
-
+print 'Filtering with %s, ' % cond_text
+print 'kept %4.2f%% of %d original lines.  ' % ( 100.0*keep/len(flags), total )
+if skipped_lines > 0:
+    print 'Skipped %d invalid lines in file starting with line # %d, data: %s' % ( skipped_lines, first_invalid_line, invalid_line )
+    
+    

tools/stats/filtering.xml

@@ -1,6 +1,6 @@
 <tool id="Filter1" name="Filter">
   <description>data on any column using simple expressions</description>
-  <command interpreter="python">
+  <command interpreter="python2.4">
     filtering.py $input $out_file1 "$cond"
   </command>
   <inputs>