diff --git a/tools/metag_tools/megablast_wrapper.py b/tools/metag_tools/megablast_wrapper.py index 2cc409834e0..34a57894c8a 100644 --- a/tools/metag_tools/megablast_wrapper.py +++ b/tools/metag_tools/megablast_wrapper.py @@ -74,24 +74,33 @@ def __main__(): print megablast_command + tmp = tempfile.NamedTemporaryFile().name try: - proc = subprocess.Popen( args=megablast_command, shell=True, stderr=subprocess.PIPE ) + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=megablast_command, shell=True, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr + if os.path.exists( tmp ): + os.unlink( tmp ) except Exception, e: if os.path.exists( mega_temp_output ): os.unlink( mega_temp_output ) + if os.path.exists( tmp ): + os.unlink( tmp ) stop_err( 'Error indexing reference sequence. ' + str( e ) ) output = open( output_filename, 'w' ) diff --git a/tools/samtools/sam_merge.py b/tools/samtools/sam_merge.py index 60204880baa..5e5f2f5b614 100644 --- a/tools/samtools/sam_merge.py +++ b/tools/samtools/sam_merge.py @@ -9,7 +9,7 @@ usage: %prog [options] [input3[,input4[,input5[,...]]]] """ -import os, subprocess, sys +import os, subprocess, sys, tempfile def stop_err( msg ): sys.stderr.write( '%s\n' % msg ) @@ -22,22 +22,31 @@ def __main__(): stop_err( 'There are not enough files to merge' ) filenames = sys.argv[3:] cmd = 'samtools merge %s %s %s' % ( outfile, infile, ' '.join( filenames ) ) + tmp = tempfile.NamedTemporaryFile().name try: - proc = subprocess.Popen( args=cmd, shell=True, stderr=subprocess.PIPE ) + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd, shell=True, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr + if os.path.exists( tmp ): + os.unlink( tmp ) except Exception, e: + if os.path.exists( tmp ): + os.unlink( tmp ) stop_err( 'Error running SAMtools merge tool\n' + str( e ) ) if os.path.getsize( outfile ) > 0: sys.stdout.write( '%s files merged.' % ( len( sys.argv ) - 2 ) ) diff --git a/tools/samtools/sam_pileup.py b/tools/samtools/sam_pileup.py index 4b57b391585..238bffa9685 100644 --- a/tools/samtools/sam_pileup.py +++ b/tools/samtools/sam_pileup.py @@ -88,36 +88,46 @@ def __main__(): elif options.ref == 'history': os.symlink( options.ownFile, tmpf1_name ) cmdIndex = 'samtools faidx %s' % ( tmpf1_name ) - proc = subprocess.Popen( args=cmdIndex, shell=True, cwd=tmpDir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmpDir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmdIndex, shell=True, cwd=tmpDir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() #did index succeed? if returncode != 0: raise Exception, 'Error creating index file\n' + stderr cmd = cmd % ( opts, tmpf1_name, tmpf0bam_name, options.output1 ) #perform pileup command - proc = subprocess.Popen( args=cmd, shell=True, cwd=tmpDir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmpDir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd, shell=True, cwd=tmpDir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() #did it succeed? # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr except Exception, e: diff --git a/tools/samtools/sam_to_bam.py b/tools/samtools/sam_to_bam.py index 7a2cabfe569..4111444dfbb 100644 --- a/tools/samtools/sam_to_bam.py +++ b/tools/samtools/sam_to_bam.py @@ -75,18 +75,23 @@ def __main__(): os.symlink( options.ref_file, fai_index_file_base ) fai_index_file_path = '%s.fai' % fai_index_file_base command = 'samtools faidx %s' % fai_index_file_base - proc = subprocess.Popen( args=command, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=command, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr if len( open( fai_index_file_path ).read().strip() ) == 0: @@ -104,18 +109,23 @@ def __main__(): # IMPORTANT NOTE: for some reason the samtools view command gzips the resulting bam file without warning, # and the docs do not currently state that this occurs ( very bad ). command = 'samtools view -bt %s -o %s %s' % ( fai_index_file_path, tmp_aligns_file_name, options.input1 ) - proc = subprocess.Popen( args=command, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=command, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr if len( open( tmp_aligns_file_name ).read() ) == 0: @@ -133,18 +143,23 @@ def __main__(): tmp_sorted_aligns_file_name = tmp_sorted_aligns_file.name tmp_sorted_aligns_file.close() command = 'samtools sort %s %s' % ( tmp_aligns_file_name, tmp_sorted_aligns_file_name ) - proc = subprocess.Popen( args=command, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=command, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr except Exception, e: diff --git a/tools/sr_mapping/bowtie_wrapper.py b/tools/sr_mapping/bowtie_wrapper.py index 5d66450df1e..b2718d46aec 100644 --- a/tools/sr_mapping/bowtie_wrapper.py +++ b/tools/sr_mapping/bowtie_wrapper.py @@ -186,25 +186,34 @@ def __main__(): ( iautoB, ipacked, ibmax, ibmaxdivn, idcv, inodc, inoref, options.ioffrate, iftab, intoa, iendian, iseed, icutoff, colorspace ) - except ValueError: - indexing_cmds = '%s' % colorspace + except ValueError, e: + # clean up temp dir + if os.path.exists( tmp_index_dir ): + shutil.rmtree( tmp_index_dir ) + stop_err( 'Something is wrong with the indexing parameters and the indexing and alignment could not be run\n' + str( e ) ) ref_file = tempfile.NamedTemporaryFile( dir=tmp_index_dir ) ref_file_name = ref_file.name ref_file.close() os.symlink( options.ref, ref_file_name ) cmd1 = 'bowtie-build %s -f %s %s' % ( indexing_cmds, ref_file_name, ref_file_name ) try: - proc = subprocess.Popen( args=cmd1, shell=True, cwd=tmp_index_dir, stderr=subprocess.PIPE, stdout=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_index_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd1, shell=True, cwd=tmp_index_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() + # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr except Exception, e: @@ -358,18 +367,23 @@ def __main__(): else: cmd2 = 'bowtie %s %s %s > %s' % ( aligning_cmds, ref_file_name, options.input1, options.output ) # align - proc = subprocess.Popen( args=cmd2, shell=True, cwd=tmp_index_dir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_index_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd2, shell=True, cwd=tmp_index_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr # check that there are results in the output file diff --git a/tools/sr_mapping/bowtie_wrapper.xml b/tools/sr_mapping/bowtie_wrapper.xml index eb3bb78c737..e2db54c5a93 100644 --- a/tools/sr_mapping/bowtie_wrapper.xml +++ b/tools/sr_mapping/bowtie_wrapper.xml @@ -181,8 +181,8 @@ --maxAlignAttempt="None" --forwardAlign="None" --reverseAlign="None" - #end if - #end if + #end if + #end if diff --git a/tools/sr_mapping/bwa_wrapper.py b/tools/sr_mapping/bwa_wrapper.py index b53f4f6dc86..89acfeb2984 100644 --- a/tools/sr_mapping/bwa_wrapper.py +++ b/tools/sr_mapping/bwa_wrapper.py @@ -91,18 +91,23 @@ def __main__(): indexing_cmds = '-a %s' % indexingAlg cmd1 = 'bwa index %s %s' % ( indexing_cmds, ref_file_name ) try: - proc = subprocess.Popen( args=cmd1, shell=True, cwd=tmp_index_dir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_index_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd1, shell=True, cwd=tmp_index_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' buffsize = 1048576 try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr except Exception, e: @@ -166,52 +171,67 @@ def __main__(): try: # align try: - proc = subprocess.Popen( args=cmd2, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd2, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr except Exception, e: raise Exception, 'Error aligning sequence. ' + str( e ) # and again if paired data try: - if cmd2b: - proc = subprocess.Popen( args=cmd2b, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE ) + if cmd2b: + tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd2b, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr except Exception, e: raise Exception, 'Error aligning second sequence. ' + str( e ) # generate align try: - proc = subprocess.Popen( args=cmd3, shell=True, cwd=tmp_dir, stderr=subprocess.PIPE ) + tmp = tempfile.NamedTemporaryFile( dir=tmp_dir ).name + tmp_stderr = open( tmp, 'wb' ) + proc = subprocess.Popen( args=cmd3, shell=True, cwd=tmp_dir, stderr=tmp_stderr.fileno() ) returncode = proc.wait() + tmp_stderr.close() # get stderr, allowing for case where it's very large + tmp_stderr = open( tmp, 'rb' ) stderr = '' try: while True: - stderr += proc.stderr.read( buffsize ) + stderr += tmp_stderr.read( buffsize ) if not stderr or len( stderr ) % buffsize != 0: break except OverflowError: pass + tmp_stderr.close() if returncode != 0: raise Exception, stderr except Exception, e: