Fix try-except-finally bug to be compatible with Python 2.4 in several tools and twilltestcase.py. Updated liftOver test to replace hard-coded (incorrect) path and modified code file to allow for new path. Improved bowtie error handling.

test/base/twilltestcase.py

@@ -633,24 +633,26 @@ class TwillTestCase( unittest.TestCase ):
             data = self.last_page()
             file( temp_name, 'wb' ).write(data)
             try:
-                if attributes is None:
-                    attributes = {}
-                compare = attributes.get( 'compare', 'diff' )
-                extra_files = attributes.get( 'extra_files', None )
-                if compare == 'diff':
-                    self.files_diff( local_name, temp_name, attributes=attributes )
-                elif compare == 're_match':
-                    self.files_re_match( local_name, temp_name, attributes=attributes )
-                elif compare == 're_match_multiline':
-                    self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
-                else:
-                    raise Exception, 'Unimplemented Compare type: %s' % compare
-                if extra_files:
-                    self.verify_extra_files_content( extra_files, elem.get( 'id' ) )
-            except AssertionError, err:
-                errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare )
-                errmsg += str( err )
-                raise AssertionError( errmsg )
+                # have to nest try-except in try-finally to handle 2.4
+                try:
+                    if attributes is None:
+                        attributes = {}
+                    compare = attributes.get( 'compare', 'diff' )
+                    extra_files = attributes.get( 'extra_files', None )
+                    if compare == 'diff':
+                        self.files_diff( local_name, temp_name, attributes=attributes )
+                    elif compare == 're_match':
+                        self.files_re_match( local_name, temp_name, attributes=attributes )
+                    elif compare == 're_match_multiline':
+                        self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
+                    else:
+                        raise Exception, 'Unimplemented Compare type: %s' % compare
+                    if extra_files:
+                        self.verify_extra_files_content( extra_files, elem.get( 'id' ) )
+                except AssertionError, err:
+                    errmsg = 'History item %s different than expected, difference (using %s):\n' % ( hid, compare )
+                    errmsg += str( err )
+                    raise AssertionError( errmsg )
             finally:
                 os.remove( temp_name )
 
@@ -676,21 +678,23 @@ class TwillTestCase( unittest.TestCase ):
         data = self.last_page()
         file( temp_name, 'wb' ).write( data )
         try:
-            if attributes is None:
-                attributes = {}
-            compare = attributes.get( 'compare', 'diff' )
-            if compare == 'diff':
-                self.files_diff( local_name, temp_name, attributes=attributes )
-            elif compare == 're_match':
-                self.files_re_match( local_name, temp_name, attributes=attributes )
-            elif compare == 're_match_multiline':
-                self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
-            else:
-                raise Exception, 'Unimplemented Compare type: %s' % compare
-        except AssertionError, err:
-            errmsg = 'Composite file (%s) of History item %s different than expected, difference (using %s):\n' % ( base_name, hda_id, compare )
-            errmsg += str( err )
-            raise AssertionError( errmsg )
+            # have to nest try-except in try-finally to handle 2.4
+            try:
+                if attributes is None:
+                    attributes = {}
+                compare = attributes.get( 'compare', 'diff' )
+                if compare == 'diff':
+                    self.files_diff( local_name, temp_name, attributes=attributes )
+                elif compare == 're_match':
+                    self.files_re_match( local_name, temp_name, attributes=attributes )
+                elif compare == 're_match_multiline':
+                    self.files_re_match_multiline( local_name, temp_name, attributes=attributes )
+                else:
+                    raise Exception, 'Unimplemented Compare type: %s' % compare
+            except AssertionError, err:
+                errmsg = 'Composite file (%s) of History item %s different than expected, difference (using %s):\n' % ( base_name, hda_id, compare )
+                errmsg += str( err )
+                raise AssertionError( errmsg )
         finally:
             os.remove( temp_name )
 

tools/extract/liftOver_wrapper.py

@@ -4,7 +4,7 @@
 Converts coordinates from one build/assembly to another using liftOver binary and mapping files downloaded from UCSC.
 """
 
-import sys, os, string
+import os, string, subprocess, sys
 import tempfile
 import re
 
@@ -51,15 +51,20 @@ except:
 if in_dbkey == "?": 
     stop_err( "Input dataset genome build unspecified, click the pencil icon in the history item to specify it." )
 
-
 if not os.path.isfile( mapfilepath ):
     stop_err( "%s mapping is not currently available."  % ( mapfilepath.split('/')[-1].split('.')[0] ) )
 
 safe_infile = safe_bed_file(infile)
-cmd_line = "liftOver -minMatch=" + str(minMatch) + " " + safe_infile + " " + mapfilepath + " " + outfile1 + " " + outfile2 + "  > /dev/null 2>&1"
+cmd_line = "liftOver -minMatch=" + str(minMatch) + " " + safe_infile + " " + mapfilepath + " " + outfile1 + " " + outfile2 + "  > /dev/null"
 try:
-    os.system( cmd_line )
-except Exception, exc:
-    stop_err( "Exception caught attempting conversion: %s"  % str( exc ) )
+    # have to nest try-except in try-finally to handle 2.4
+    try:
+        proc = subprocess.Popen( args=cmd_line, shell=True, stderr=subprocess.PIPE )
+        returncode = proc.wait()
+        stderr = proc.stderr.read()
+        if returncode != 0:
+            raise Exception, stderr
+    except Exception, e:
+        raise Exception, 'Exception caught attempting conversion: ' + str( e )
 finally:
     os.remove(safe_infile)

tools/extract/liftOver_wrapper.xml

@@ -23,19 +23,16 @@
   <requirements>
     <requirement type="binary">liftOver</requirement>
   </requirements>
-  
   <tests>
     <test>
       <param name="input" value="5.bed" dbkey="hg18" ftype="bed" />
-      <param name="to_dbkey" value="/galaxy/data/hg18/liftOver/hg18ToPanTro2.over.chain" />
+      <param name="to_dbkey" value="panTro2" />
       <param name="minMatch" value="0.95" />
       <output name="out_file1" file="5_liftover_mapped.bed"/>
       <output name="out_file2" file="5_liftover_unmapped.bed"/>
     </test>
   </tests>
-
   <help>
-
 .. class:: warningmark
 
 Make sure that the genome build of the input dataset is specified (click the pencil icon in the history item to set it if necessary).
@@ -71,6 +68,6 @@ will produce the following hg18 intervals::
     chrX  158279  160020  AK097346  0  +
     chrX  160024  169033  AK074528  0  -
 
-</help>
-<code file="liftOver_wrapper_code.py"/>
+  </help>
+  <code file="liftOver_wrapper_code.py" />
 </tool>

tools/extract/liftOver_wrapper_code.py

@@ -1,8 +1,10 @@
+import os
+
 def exec_before_job(app, inp_data, out_data, param_dict, tool):
     #Assuming the path of the form liftOverDirectory/hg18ToHg17.over.chain (This is how the mapping chain files from UCSC look.)
-    to_dbkey = param_dict['to_dbkey'].split('.')[0].split('To')[1]
+    #allows for . in path
+    to_dbkey = os.path.split(param_dict['to_dbkey'])[1].split('.')[0].split('To')[1]
     to_dbkey = to_dbkey[0].lower()+to_dbkey[1:]
     out_data['out_file1'].set_dbkey(to_dbkey)
     out_data['out_file1'].name = out_data['out_file1'].name + " [ MAPPED COORDINATES ]"
     out_data['out_file2'].name = out_data['out_file2'].name + " [ UNMAPPED COORDINATES ]"
-    

tools/samtools/sam_pileup.py

@@ -78,30 +78,32 @@ def __main__():
     #prepare basic pileup command
     cmd = 'samtools pileup %s -f %s %s > %s'
     try:
-        #index reference if necessary and prepare pileup command
-        if options.ref == 'indexed':
-            if not os.path.exists( "%s.fai" % seqPath ):
-                raise Exception, "No sequences are available for '%s', request them by reporting this error." % options.dbkey
-            cmd = cmd % ( opts, seqPath, tmpf0bam_name, options.output1 )
-        elif options.ref == 'history':
-            os.symlink( options.ownFile, tmpf1_name )
-            cmdIndex = 'samtools faidx %s' % ( tmpf1_name )
-            proc = subprocess.Popen( args=cmdIndex, shell=True, cwd=tmpDir, stderr=subprocess.PIPE )
+        # have to nest try-except in try-finally to handle 2.4
+        try:
+            #index reference if necessary and prepare pileup command
+            if options.ref == 'indexed':
+                if not os.path.exists( "%s.fai" % seqPath ):
+                    raise Exception, "No sequences are available for '%s', request them by reporting this error." % options.dbkey
+                cmd = cmd % ( opts, seqPath, tmpf0bam_name, options.output1 )
+            elif options.ref == 'history':
+                os.symlink( options.ownFile, tmpf1_name )
+                cmdIndex = 'samtools faidx %s' % ( tmpf1_name )
+                proc = subprocess.Popen( args=cmdIndex, shell=True, cwd=tmpDir, stderr=subprocess.PIPE )
+                returncode = proc.wait()
+                stderr = proc.stderr.read()
+                #did index succeed?
+                if returncode != 0:
+                    raise Exception, 'Error creating index file\n' + stderr
+                cmd = cmd % ( opts, tmpf1_name, tmpf0bam_name, options.output1 )
+            #perform pileup command
+            proc = subprocess.Popen( args=cmd, shell=True, cwd=tmpDir, stderr=subprocess.PIPE )
             returncode = proc.wait()
+            #did it succeed?
             stderr = proc.stderr.read()
-            #did index succeed?
             if returncode != 0:
-                raise Exception, 'Error creating index file\n' + stderr
-            cmd = cmd % ( opts, tmpf1_name, tmpf0bam_name, options.output1 )
-        #perform pileup command
-        proc = subprocess.Popen( args=cmd, shell=True, cwd=tmpDir, stderr=subprocess.PIPE )
-        returncode = proc.wait()
-        #did it succeed?
-        stderr = proc.stderr.read()
-        if returncode != 0:
-            raise Exception, stderr
-    except Exception, e:
-        stop_err( 'Error running Samtools pileup tool\n' + str( e ) )
+                raise Exception, stderr
+        except Exception, e:
+            stop_err( 'Error running Samtools pileup tool\n' + str( e ) )
     finally:
         #clean up temp files
         if os.path.exists( tmpDir ):

tools/sr_mapping/bowtie_wrapper.py

@@ -2,7 +2,7 @@
 
 """
 Runs Bowtie on single-end or paired-end data.
-For use with Bowtie v. 0.12.1
+For use with Bowtie v. 0.12.3
 
 usage: bowtie_wrapper.py [options]
     -t, --threads=t: The number of threads to run
@@ -58,12 +58,12 @@ usage: bowtie_wrapper.py [options]
     -H, --suppressHeader=H: Suppress header
 """
 
-import optparse, os, shutil, sys, tempfile
+import optparse, os, shutil, subprocess, sys, tempfile
 
 def stop_err( msg ):
-    sys.stderr.write( "%s\n" % msg )
+    sys.stderr.write( '%s\n' % msg )
     sys.exit()
- 
+
 def __main__():
     #Parse Command Line
     parser = optparse.OptionParser()
@@ -119,6 +119,7 @@ def __main__():
     parser.add_option( '-x', '--indexSettings', dest='index_settings', help='Whether or not indexing options are to be set' )
     parser.add_option( '-H', '--suppressHeader', dest='suppressHeader', help='Suppress header' )
     (options, args) = parser.parse_args()
+    stdout = ''
     # make temp directory for placement of indices and copy reference file there if necessary
     tmp_index_dir = tempfile.mkdtemp()
     # get type of data (solid or solexa)
@@ -187,17 +188,25 @@ def __main__():
                                   iseed, icutoff, colorspace )
             except ValueError:
                 indexing_cmds = '%s' % colorspace
+        ref_file = tempfile.NamedTemporaryFile( dir=tmp_index_dir )
+        ref_file_name = ref_file.name
+        ref_file.close()
+        os.symlink( options.ref, ref_file_name )
+        cmd1 = 'bowtie-build %s -f %s %s' % ( indexing_cmds, ref_file_name, ref_file_name )
         try:
-            shutil.copy( options.ref, tmp_index_dir )
-        except Exception, e:
-            stop_err( 'Error creating temp directory for indexing purposes\n' + str( e ) )
-        options.ref = os.path.join( tmp_index_dir, os.path.split( options.ref )[1] )
-        cmd1 = 'bowtie-build %s -f %s %s 2> /dev/null' % ( indexing_cmds, options.ref, options.ref )
-        try:
-            os.chdir( tmp_index_dir )
-            os.system( cmd1 )
+            proc = subprocess.Popen( args=cmd1, shell=True, cwd=tmp_index_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE )
+            returncode = proc.wait()
+            stderr = proc.stderr.read()
+            if returncode != 0:
+                raise Exception, stderr
         except Exception, e:
+            # clean up temp dir
+            if os.path.exists( tmp_index_dir ):
+                shutil.rmtree( tmp_index_dir )
             stop_err( 'Error indexing reference sequence\n' + str( e ) )
+        stdout += 'File indexed. '
+    else:
+        ref_file_name = options.ref
     # set up aligning and generate aligning command options
     # automatically set threads in both cases
     if options.suppressHeader == 'true':
@@ -328,19 +337,34 @@ def __main__():
                               best, strata, offrate, seed, colorspace, snpphred, snpfrac, 
                               keepends, options.threads, suppressHeader )
         except ValueError, e:
+            # clean up temp dir
+            if os.path.exists( tmp_index_dir ):
+                shutil.rmtree( tmp_index_dir )
             stop_err( 'Something is wrong with the alignment parameters and the alignment could not be run\n' + str( e ) )
-    # prepare actual aligning commands
-    if options.paired == 'paired':
-        cmd2 = 'bowtie %s %s -1 %s -2 %s > %s 2> /dev/null' % ( aligning_cmds, options.ref, options.input1, options.input2, options.output ) 
-    else:
-        cmd2 = 'bowtie %s %s %s > %s 2> /dev/null' % ( aligning_cmds, options.ref, options.input1, options.output ) 
-    # align
     try:
-        os.system( cmd2 )
-    except Exception, e:
-        stop_err( 'Error aligning sequence\n' + str( e ) )
-    # clean up temp dir
-    if os.path.exists( tmp_index_dir ):
-        shutil.rmtree( tmp_index_dir )
+        # have to nest try-except in try-finally to handle 2.4
+        try:
+            # prepare actual aligning commands
+            if options.paired == 'paired':
+                cmd2 = 'bowtie %s %s -1 %s -2 %s > %s' % ( aligning_cmds, ref_file_name, options.input1, options.input2, options.output )
+            else:
+                cmd2 = 'bowtie %s %s %s > %s' % ( aligning_cmds, ref_file_name, options.input1, options.output )
+            # align
+            proc = subprocess.Popen( args=cmd2, shell=True, cwd=tmp_index_dir, stderr=subprocess.PIPE )
+            returncode = proc.wait()
+            stderr = proc.stderr.read()
+            if returncode != 0:
+                raise Exception, stderr
+            # check that there are results in the output file
+            if len( open( options.output, 'rb' ).read().strip() ) == 0:
+                raise Exception, 'The output file is empty, there may be an error with your input file or settings.'
+        except Exception, e:
+            stop_err( 'Error aligning sequence. ' + str( e ) )
+    finally:
+        # clean up temp dir
+        if os.path.exists( tmp_index_dir ):
+            shutil.rmtree( tmp_index_dir )
+    stdout += 'Sequence file aligned.\n'
+    sys.stdout.write( stdout )
 
 if __name__=="__main__": __main__()

tools/sr_mapping/bwa_wrapper.py

@@ -152,55 +152,57 @@ def __main__():
         cmd3 = 'bwa samse %s %s %s %s >> %s' % ( gen_alignment_cmds, ref_file_name, tmp_align_out_name, options.fastq, options.output )
     # perform alignments
     try:
-        # align
+        # need to nest try-except in try-finally to handle 2.4
         try:
-            proc = subprocess.Popen( args=cmd2, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE )
-            returncode = proc.wait()
-            stderr = proc.stderr.read()
-            if returncode != 0:
-                raise Exception, stderr
-        except Exception, e:
-            raise Exception, 'Error aligning sequence. ' + str( e )
-        # and again if paired data
-        try:
-            if cmd2b: 
-                proc = subprocess.Popen( args=cmd2b, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE )
+            # align
+            try:
+                proc = subprocess.Popen( args=cmd2, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE )
                 returncode = proc.wait()
                 stderr = proc.stderr.read()
                 if returncode != 0:
                     raise Exception, stderr
-        except Exception, e:
-            raise Exception, 'Error aligning second sequence. ' + str( e )
-        # generate align
-        try:
-            proc = subprocess.Popen( args=cmd3, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE )
-            returncode = proc.wait()
-            stderr = proc.stderr.read()
-            if returncode != 0:
-                raise Exception, stderr
-        except Exception, e:
-            raise Exception, 'Error generating alignments. ' + str( e ) 
-        # remove header if necessary
-        if options.suppressHeader == 'true':
-            tmp_out = tempfile.NamedTemporaryFile( dir=tmp_dir)
-            tmp_out_name = tmp_out.name
-            tmp_out.close()
-            try:
-                shutil.move( options.output, tmp_out_name )
             except Exception, e:
-                raise Exception, 'Error moving output file before removing headers. ' + str( e )
-            fout = file( options.output, 'w' )
-            for line in file( tmp_out.name, 'r' ):
-                if not ( line.startswith( '@HD' ) or line.startswith( '@SQ' ) or line.startswith( '@RG' ) or line.startswith( '@PG' ) or line.startswith( '@CO' ) ):
-                    fout.write( line )
-            fout.close()
-        # check that there are results in the output file
-        if os.path.getsize( options.output ) > 0:
-            sys.stdout.write( 'BWA run on %s-end data' % options.genAlignType )
-        else:
-            raise Exception, 'The output file is empty. You may simply have no matches, or there may be an error with your input file or settings.'
-    except Exception, e:
-        stop_err( 'The alignment failed.\n' + str( e ) )
+                raise Exception, 'Error aligning sequence. ' + str( e )
+            # and again if paired data
+            try:
+                if cmd2b: 
+                    proc = subprocess.Popen( args=cmd2b, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE )
+                    returncode = proc.wait()
+                    stderr = proc.stderr.read()
+                    if returncode != 0:
+                        raise Exception, stderr
+            except Exception, e:
+                raise Exception, 'Error aligning second sequence. ' + str( e )
+            # generate align
+            try:
+                proc = subprocess.Popen( args=cmd3, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE )
+                returncode = proc.wait()
+                stderr = proc.stderr.read()
+                if returncode != 0:
+                    raise Exception, stderr
+            except Exception, e:
+                raise Exception, 'Error generating alignments. ' + str( e ) 
+            # remove header if necessary
+            if options.suppressHeader == 'true':
+                tmp_out = tempfile.NamedTemporaryFile( dir=tmp_dir)
+                tmp_out_name = tmp_out.name
+                tmp_out.close()
+                try:
+                    shutil.move( options.output, tmp_out_name )
+                except Exception, e:
+                    raise Exception, 'Error moving output file before removing headers. ' + str( e )
+                fout = file( options.output, 'w' )
+                for line in file( tmp_out.name, 'r' ):
+                    if not ( line.startswith( '@HD' ) or line.startswith( '@SQ' ) or line.startswith( '@RG' ) or line.startswith( '@PG' ) or line.startswith( '@CO' ) ):
+                        fout.write( line )
+                fout.close()
+            # check that there are results in the output file
+            if os.path.getsize( options.output ) > 0:
+                sys.stdout.write( 'BWA run on %s-end data' % options.genAlignType )
+            else:
+                raise Exception, 'The output file is empty. You may simply have no matches, or there may be an error with your input file or settings.'
+        except Exception, e:
+            stop_err( 'The alignment failed.\n' + str( e ) )
     finally:
         # clean up temp dir
         if os.path.exists( tmp_index_dir ):