From 312620d2eb60b3400d2fe61b175ed40cb0b194d5 Mon Sep 17 00:00:00 2001
From: Greg Von Kuster <greg@bx.psu.edu>
Date: Tue, 21 Aug 2007 14:55:36 +0000
Subject: [PATCH] Fixes to various tools: scatterplot, cor, grouping

---
 tools/filters/sorter.xml           |  3 +-
 tools/plotting/scatterplot.py      | 71 +++++++++++++++++++++---------
 tools/plotting/scatterplot.xml     | 22 +++++----
 tools/plotting/scatterplot_code.py | 39 ++++++++++++++++
 tools/stats/cor.py                 |  4 +-
 tools/stats/cor.xml                |  2 +-
 tools/stats/cor_code.py            | 39 ++++++++++++++--
 tools/stats/grouping.py            |  6 ++-
 8 files changed, 147 insertions(+), 39 deletions(-)
 create mode 100644 tools/plotting/scatterplot_code.py

diff --git a/tools/filters/sorter.xml b/tools/filters/sorter.xml
index 3891ff45b6c..de0030bf615 100644
--- a/tools/filters/sorter.xml
+++ b/tools/filters/sorter.xml
@@ -6,8 +6,7 @@
       <param format="tabular" name="input" type="data" label="Sort Query"/>
     </page>
     <page>
-      <param name="column" size="4" type="integer" value="1" label="on column"/>
-      <param name="column" label="on columns" type="select" dynamic_options="get_columns( input )" />
+      <param name="column" label="on column" type="select" dynamic_options="get_columns( input )" />
       <param name="order" type="select" label="in">
         <option value="ASC">Ascending order</option>
         <option value="DESC">Descending order</option>
diff --git a/tools/plotting/scatterplot.py b/tools/plotting/scatterplot.py
index c556a391a43..82e452c2633 100644
--- a/tools/plotting/scatterplot.py
+++ b/tools/plotting/scatterplot.py
@@ -6,7 +6,7 @@ from rpy import *
 
 def fail( message ):
     print >> sys.stderr, message
-    return -1
+    sys.exit()
 
 def main():
 
@@ -20,30 +20,59 @@ def main():
     ylab = sys.argv[7]
 
     matrix = []
-    for i, line in enumerate( open( sys.argv[1] ) ):
-        # Skip comments
-        if line.startswith( '#' ): 
-            continue
-        # Extract values and convert to floats
-        row = []
-        for column in columns:
-            fields = line.split( "\t" )
-            if len( fields ) <= column:
-                return fail( "No column %d on line %d" % ( column+1, i ) )
-            val = fields[column]
-            if val.lower() == "na": 
-                row.append( float( "nan" ) )
-            else:
-                try:
-                    row.append( float( fields[column] ) )
-                except ValueError:
-                    return fail( "Value '%s' in column %d on line %d is not numeric" % ( fields[column], column+1, i ) )
-        matrix.append( row )
+    skipped_lines = 0
+    first_invalid_line = 0
+    invalid_value = ''
+    invalid_column = 0
+
+    for i, line in enumerate( file ( sys.argv[1] ) ):
+        valid = True
+        line = line.rstrip('\r\n')
+        if line and not line.startswith( '#' ): 
+            # Extract values and convert to floats
+            row = []
+            for column in columns:
+                if not valid:
+                    break
+                fields = line.split( "\t" )
+                if len( fields ) <= column:
+                    return fail( "Column %d on line %d missing, line: %s" % ( column+1, i, line ) )
+                val = fields[column]
+                if val.lower() == "na": 
+                    row.append( float( "nan" ) )
+                else:
+                    try:
+                        row.append( float( fields[column] ) )
+                    except:
+                        valid = False
+                        skipped_lines += 1
+                        if not first_invalid_line:
+                            first_invalid_line = i+1
+                            invalid_value = fields[column]
+                            invalid_column = column+1
+        else:
+            valid = False
+            skipped_lines += 1
+            if not first_invalid_line:
+                first_invalid_line = i+1
+
+        if valid:
+            matrix.append( row )
         
     r.pdf( out_fname, 8, 8 )
     r.plot( array( matrix ), type="p", main=title, xlab=xlab, ylab=ylab, col="blue", pch=19 )
     r.dev_off()
+
+    msg = "--Scatter plot on "
+    for i,col in enumerate(columns):
+        col += 1
+        msg += "c%d, " %col
+    if skipped_lines > 0:
+        msg += " skipped %d lines starting with line #%d.  Value '%s' in column %d is not numeric." % ( skipped_lines, first_invalid_line, invalid_value, invalid_column )
+
+    print msg
+
     r.quit( save="no" )
-    
+
 if __name__ == "__main__":
     main()
diff --git a/tools/plotting/scatterplot.xml b/tools/plotting/scatterplot.xml
index 3fa9f277ccd..d4780447a7e 100644
--- a/tools/plotting/scatterplot.xml
+++ b/tools/plotting/scatterplot.xml
@@ -2,12 +2,16 @@
   <description>of two numeric columns</description>
   <command interpreter="python">scatterplot.py $input $out_file1 $col1 $col2 "$title" "$xlab" "$ylab"</command>
   <inputs>
-    <param format="text" name="input" type="data" label="Dataset" help="Query missing? See TIP below"/>  
-    <param name="col1" size="4" type="integer" value="5" label="Column for x axis"/>
-    <param name="col2" size="4" type="integer" value="6" label="Column for y axis"/>
-    <param name="title" size="30" type="text" value="Scatterplot" label="Plot title"/>
-    <param name="xlab" size="30" type="text" value="V1" label="Label for x axis"/>
-    <param name="ylab" size="30" type="text" value="V2" label="Label for y axis"/>
+    <page>
+      <param name="input" format="tabular" type="data" label="Dataset" help="Query missing? See TIP below"/>
+    </page>
+    <page>
+      <param name="col1" type="select" label="Column for x axis" dynamic_options="get_columns( input )" />
+      <param name="col2" type="select" label="Column for y axis" dynamic_options="get_columns( input )" />
+      <param name="title" size="30" type="text" value="Scatterplot" label="Plot title"/>
+      <param name="xlab" size="30" type="text" value="V1" label="Label for x axis"/>
+      <param name="ylab" size="30" type="text" value="V2" label="Label for y axis"/>
+    </page>
   </inputs>
   <outputs>
     <data format="pdf" name="out_file1" />
@@ -22,9 +26,10 @@
 
 **Syntax**
 
-A scatter plot reveals relationships or association between two variables. This tool creates a simple scatterplot between two variables of a selected Query.
+This tool creates a simple scatter plot between two variables containing numeric values of a selected query. 
+
+- All invalid, blank and comment lines in the query are skipped.  The number of skipped lines is displayed in the resulting history item.
 
-- **Column for x axis** and **Column for x axis** columns are referenced with a number, it start with 1.
 - **Plot title** The scatterplot title
 - **Label for x axis** and **Label for y axis** The labels for x and y axis of the scatterplot.
 
@@ -50,4 +55,5 @@ A scatter plot reveals relationships or association between two variables. This
 .. image:: ../static/images/scatterplot.png
 
 </help>
+  <code file="scatterplot_code.py" />
 </tool>
diff --git a/tools/plotting/scatterplot_code.py b/tools/plotting/scatterplot_code.py
new file mode 100644
index 00000000000..ce0b42f775c
--- /dev/null
+++ b/tools/plotting/scatterplot_code.py
@@ -0,0 +1,39 @@
+
+def get_columns( input ):
+    columns = []
+    elems = []
+
+    for i, line in enumerate( file ( input.file_name ) ):
+        valid = True
+        if line and not line.startswith( '#' ): 
+            line = line.rstrip('\r\n')
+            elems = line.split( '\t' )
+    
+            """
+            Since this tool requires users to select only those columns
+            that contain numerical values, we'll restrict the column select
+            list appropriately.
+            """
+            if len(elems) > 0:
+                for col in range(1, input.metadata.columns+1):
+                    try:
+                        val = float(elems[col-1])
+                    except:
+                        val = elems[col-1]
+                        if val:
+                            if val.strip().lower() != "na":
+                                valid = False
+                        else:
+                            valid = False
+                    if valid:
+                        option = "c" + str(col)
+                        columns.append((option,str(col),False))
+            if len(columns) > 0:
+                """
+                We have our select list built, so we can break out of the outer most for loop
+                """
+                break 
+        if i == 30:
+            break # Hopefully we never get here...
+
+    return columns
\ No newline at end of file
diff --git a/tools/stats/cor.py b/tools/stats/cor.py
index 4eb6ba3279e..2329233f205 100644
--- a/tools/stats/cor.py
+++ b/tools/stats/cor.py
@@ -52,10 +52,10 @@ def main():
                         except:
                             valid = False
                             skipped_lines += 1
-                            invalid_value = fields[column]
-                            invalid_column = column+1
                             if not first_invalid_line:
                                 first_invalid_line = i+1
+                                invalid_value = fields[column]
+                                invalid_column = column+1
         else:
             valid = False
             skipped_lines += 1
diff --git a/tools/stats/cor.xml b/tools/stats/cor.xml
index f4d010256fe..409d09b923b 100644
--- a/tools/stats/cor.xml
+++ b/tools/stats/cor.xml
@@ -6,7 +6,7 @@
       <param format="tabular" name="input1" type="data" label="Dataset" help="Query missing? See TIP below"/>
     </page>
     <page>
-      <param name="columns" label="Columns" type="select" multiple="True" dynamic_options="get_columns( input1 )" help="Multi-select list - hold the appropriate key while clicking to select multiple columns" />
+      <param name="columns" label="Numerical columns" type="select" multiple="True" dynamic_options="get_columns( input1 )" help="Multi-select list - hold the appropriate key while clicking to select multiple columns" />
       <param name="method" type="select" label="Method">
         <option value="pearson">Pearson</option>
         <option value="kendall">Kendall rank</option>
diff --git a/tools/stats/cor_code.py b/tools/stats/cor_code.py
index 6477df1b695..ce0b42f775c 100644
--- a/tools/stats/cor_code.py
+++ b/tools/stats/cor_code.py
@@ -1,8 +1,39 @@
 
-def get_columns( input1 ):
+def get_columns( input ):
     columns = []
+    elems = []
+
+    for i, line in enumerate( file ( input.file_name ) ):
+        valid = True
+        if line and not line.startswith( '#' ): 
+            line = line.rstrip('\r\n')
+            elems = line.split( '\t' )
+    
+            """
+            Since this tool requires users to select only those columns
+            that contain numerical values, we'll restrict the column select
+            list appropriately.
+            """
+            if len(elems) > 0:
+                for col in range(1, input.metadata.columns+1):
+                    try:
+                        val = float(elems[col-1])
+                    except:
+                        val = elems[col-1]
+                        if val:
+                            if val.strip().lower() != "na":
+                                valid = False
+                        else:
+                            valid = False
+                    if valid:
+                        option = "c" + str(col)
+                        columns.append((option,str(col),False))
+            if len(columns) > 0:
+                """
+                We have our select list built, so we can break out of the outer most for loop
+                """
+                break 
+        if i == 30:
+            break # Hopefully we never get here...
 
-    for col in range(1, input1.metadata.columns+1):
-        option = "c" + str(col)
-        columns.append((option,str(col),False))
     return columns
\ No newline at end of file
diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py
index 02b7d9398a7..c13a5075627 100644
--- a/tools/stats/grouping.py
+++ b/tools/stats/grouping.py
@@ -75,6 +75,8 @@ prev_vals = []
 skipped_lines = 0
 first_invalid_line = 0
 invalid_line = ''
+invalid_value = ''
+invalid_column = 0
 fout = open(sys.argv[1], "w")
 
 for ii, line in enumerate( file( tmpfile.name )):
@@ -105,6 +107,8 @@ for ii, line in enumerate( file( tmpfile.name )):
                                 skipped_lines += 1
                                 if not first_invalid_line:
                                     first_invalid_line = ii+1
+                                    invalid_value = fields[col]
+                                    invalid_column = col+1
                         if valid:
                             prev_vals[i].append(fields[col].strip())
                 else:   
@@ -181,6 +185,6 @@ for i,op in enumerate(ops):
         op = 'concat'
     msg += op + "[c" + cols[i] + "] "
 if skipped_lines > 0:
-    msg+= "--skipped %d blank/comment/invalid lines starting with line #%d.  " %( skipped_lines, first_invalid_line )
+    msg+= "--skipped %d invalid lines starting with line %d.  Value '%s' in column %d is not numeric." % ( skipped_lines, first_invalid_line, invalid_value, invalid_column )
 
 print msg