Enhanced column_maker.py to gracefully handle invalid data in input file and to display more meaningful info to user.

tools/stats/column_maker.py

@@ -1,4 +1,10 @@
-# this is a tool that creates another column
+#!/usr/bin/env python
+"""
+This tool takes a tab-delimited textfile as input and creates another column in the file which is the result of
+a computation performed on every row in the original file.  The tool will skip over invalid lines within the file,
+informing the user about the number of lines skipped.  Invalid lines are those that do not follow the standard 
+defined when the get_wrap_func function (immediately below) is applied to the first uncommented line in the input file.
+"""
 import sys, sets, re, os.path
 
 def get_wrap_func(value):
@@ -20,9 +26,6 @@ def stop_err(msg):
 if len(sys.argv) != 4:
     print sys.argv
     stop_err('Usage: python column_maker.py input_file ouput_file condition')
-#debug 
-#cond_text = "(c2-c3) < 115487120 and c1=='chr7' "
-#sys.argv.extend( [ 'a.txt', 'b.txt', cond_text ])
 
 inp_file  = sys.argv[1]
 out_file  = sys.argv[2]
@@ -36,14 +39,11 @@ mapped_str = {
     '__ge__': '>=',
     '__sq__': '\'',
     '__dq__': '"',
-
 }
 for key, value in mapped_str.items():
     cond_text = cond_text.replace(key, value)
 
-#
 # safety measures
-#
 safe_words = sets.Set( "c chr str float int split map lambda and or len".split() )
 try:
     # filter on words
@@ -87,36 +87,51 @@ col   = ', '.join(cols)
 func  = ', '.join(funcs)
 assign = "%s = line.split('\\t')" % col
 wrap   = "%s = %s" % (col, func)
-
+skipped_lines = 0
+first_invalid_line = 0
+invalid_line = None
+flags = []
 cols = []
+
+# Read input file, skipping invalid lines, and perform computation that will result in a new column
 code = '''
-for line in file(inp_file):
+for i, line in enumerate( open( inp_file )):
     line = line.strip()
     if line and not line.startswith( '#' ):
-        %s
-        %s
-        cols.append(%s)
+        try:
+            %s
+            %s
+            cols.append(%s)
+            flags.append(True)
+        except:
+            skipped_lines += 1
+            cols.append('')
+            flags.append(False)
+            if not invalid_line:
+                first_invalid_line = i + 1
+                invalid_line = line
     else:
         cols.append('')
+        flags.append(False)
 ''' % (assign, wrap, cond_text)
 
-#
-# execute code
-#
-try:
-    exec code
-except Exception, e:
-    stop_err('Code %s raised error: %s' % (code, e))
+exec code
 
-#
-# create output
-#
+# Write output file
 fp = open(out_file, 'wt')
-for value, line in zip(cols, file(inp_file)):
-    line = line.strip()
-    line = '%s\t%s\n' % (line, value)
-    fp.write(line)
+keep = 0
+total = 0
+for flag, value, line in zip(flags, cols, file(inp_file)):
+    total += 1
+    if flag:
+        line = line.strip()
+        line = '%s\t%s\n' % (line, value)
+        fp.write(line)
+        keep += 1
 fp.close()
 
 print 'Creating column %d with expression %s' % (len(elems)+1, cond_text)
-
+if skipped_lines > 0:
+    print ', kept %d of %d original lines.  ' % ( keep, total )
+    print 'Skipped %d invalid lines in file starting with line # %d, data: %s' % ( skipped_lines, first_invalid_line, invalid_line )
+    

tools/stats/column_maker.xml

@@ -1,6 +1,6 @@
 <tool id="Add a column1" name="Compute">
   <description>an expression on every row</description>
-  <command interpreter="python">
+  <command interpreter="python2.4">
     column_maker.py $input $out_file1 "$cond"
   </command>
   <inputs>