diff --git a/.ci/jenkins/main-tools-py3/run_tests.sh b/.ci/jenkins/main-tools-py3/run_tests.sh new file mode 100644 index 00000000000..442b15cdf2d --- /dev/null +++ b/.ci/jenkins/main-tools-py3/run_tests.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +./run_tests.sh --dockerize --python3 --db postgres --clean_pyc -main "$@" diff --git a/.ci/jenkins/main-tools/run_tests.sh b/.ci/jenkins/main-tools/run_tests.sh new file mode 100644 index 00000000000..8b9edccddec --- /dev/null +++ b/.ci/jenkins/main-tools/run_tests.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +./run_tests.sh --dockerize --db postgres --external_tmp --clean_pyc -main "$@" diff --git a/lib/galaxy/datatypes/util/gff_util.py b/lib/galaxy/datatypes/util/gff_util.py index e81296a8cf7..421a5886c75 100644 --- a/lib/galaxy/datatypes/util/gff_util.py +++ b/lib/galaxy/datatypes/util/gff_util.py @@ -188,7 +188,7 @@ class GFFReaderWrapper(NiceReaderWrapper): if not self.seed_interval: while not self.seed_interval: try: - self.seed_interval = GenomicIntervalReader.next(self) + self.seed_interval = super(GenomicIntervalReader, self).__next__() except ParseError as e: handle_parse_error(e) # TODO: When no longer supporting python 2.4 use finally: @@ -215,7 +215,7 @@ class GFFReaderWrapper(NiceReaderWrapper): feature_intervals.append(self.seed_interval) while True: try: - interval = GenomicIntervalReader.next(self) + interval = super(GenomicIntervalReader, self).__next__() raw_size += len(self.current_line) except StopIteration as e: # No more intervals to read, but last feature needs to be diff --git a/lib/galaxy/tools/util/maf_utilities.py b/lib/galaxy/tools/util/maf_utilities.py index d3abbd9529d..b2c25abb38d 100644 --- a/lib/galaxy/tools/util/maf_utilities.py +++ b/lib/galaxy/tools/util/maf_utilities.py @@ -5,6 +5,7 @@ Provides wrappers and utilities for working with MAF files and alignments. # Dan Blankenberg from __future__ import print_function +import functools import logging import os import resource @@ -655,7 +656,7 @@ def sort_block_components_by_block(block1, block2): # orders the components in block1 by the index of the component in block2 # block1 must be a subset of block2 # occurs in-place - return block1.components.sort(cmp=lambda x, y: block2.components.index(x) - block2.components.index(y)) + return block1.components.sort(key=functools.cmp_to_key(lambda x, y: block2.components.index(x) - block2.components.index(y))) def get_species_in_maf(maf_filename): diff --git a/lib/galaxy/util/ucsc.py b/lib/galaxy/util/ucsc.py index 1542542b95d..295fa4608ad 100644 --- a/lib/galaxy/util/ucsc.py +++ b/lib/galaxy/util/ucsc.py @@ -18,19 +18,22 @@ class UCSCOutWrapper(object): def __iter__(self): return self - def next(self): + def __next__(self): if self.lookahead is None: - line = self.other.next() + line = next(self.other) else: line = self.lookahead self.lookahead = None if line.startswith("----------"): - next_line = self.other.next() + next_line = next(self.other) if next_line.startswith("Reached output limit"): raise UCSCLimitException(next_line.strip()) else: self.lookahead = next_line return line + def next(self): + return self.__next__() + def readline(self): return self.next() diff --git a/test/functional/known_broken_tools.txt b/test/functional/known_broken_tools.txt index 506709a36fb..a7914ac942e 100644 --- a/test/functional/known_broken_tools.txt +++ b/test/functional/known_broken_tools.txt @@ -1,11 +1,15 @@ # Deprecated, IUC provides same tool, kept for backwards compat? Extract genomic DNA 1 -# Relies on data table, but data table not available +# Relies on data table or loc file, but data table not available Interval2Maf1 Interval2Maf_pairwise1 maf_stats1 Interval_Maf_Merged_Fasta2 +GeneBed_Maf_Fasta2 # No dependency available hgv_lps # Some weird parameter issue maf_by_block_number1 +# Requires hg17.len to be present +bed_to_bigBed +wig_to_bigWig diff --git a/tools/filters/bed_to_bigbed.xml b/tools/filters/bed_to_bigbed.xml index fd8d0017c19..d09064e9b44 100644 --- a/tools/filters/bed_to_bigbed.xml +++ b/tools/filters/bed_to_bigbed.xml @@ -39,12 +39,12 @@ - + - + diff --git a/tools/filters/join.py b/tools/filters/join.py index 75fd2af482a..e2094bc9b63 100644 --- a/tools/filters/join.py +++ b/tools/filters/join.py @@ -79,7 +79,7 @@ class SortedOffsets(OffsetList): def __init__(self, indexed_filename, column, split=None): OffsetList.__init__(self, os.stat(indexed_filename).st_size) self.indexed_filename = indexed_filename - self.indexed_file = open(indexed_filename, 'rb') + self.indexed_file = open(indexed_filename, 'r') self.column = column self.split = split self.last_identifier = None @@ -149,7 +149,7 @@ class SortedOffsets(OffsetList): class OffsetIndex: def __init__(self, filename, column, split=None, index_depth=3): self.filename = filename - self.file = open(filename, 'rb') + self.file = open(filename, 'r') self.column = column self.split = split self._offsets = {} @@ -243,7 +243,7 @@ class BufferedIndex: def __init__(self, filename, column, split=None, buffer=1000000, index_depth=3): self.index = OffsetIndex(filename, column, split, index_depth) self.buffered_offsets = {} - f = open(filename, 'rb') + f = open(filename, 'r') offset = f.tell() identified_offset_count = 1 while True: @@ -296,9 +296,9 @@ def join_files(filename1, column1, filename2, column2, out_filename, split=None, if fill_options is None: fill_options = Bunch(fill_unjoined_only=True, file1_columns=None, file2_columns=None) keep_headers_done = False - out = open(out_filename, 'w+b') + out = open(out_filename, 'w') index = BufferedIndex(filename2, column2, split, buffer, index_depth) - for line1 in open(filename1, 'rb'): + for line1 in open(filename1, 'r'): if keep_headers and not keep_headers_done: header1 = line1 with open(filename2, 'r') as file2: diff --git a/tools/filters/joinWrapper.py b/tools/filters/joinWrapper.py index ad6ecb075bd..b308d46e968 100644 --- a/tools/filters/joinWrapper.py +++ b/tools/filters/joinWrapper.py @@ -53,7 +53,7 @@ def main(): if ret == 0: cl = subprocess.Popen(["join", "--version"], stdout=subprocess.PIPE) (stdout, _) = cl.communicate() - version_line = stdout.split("\n")[0] + version_line = stdout.decode('utf-8').split("\n")[0] (version, _) = version_line.split()[-1].split(".") if int(version) >= 7: flags = "--nocheck-order" diff --git a/tools/filters/secure_hash_message_digest.py b/tools/filters/secure_hash_message_digest.py index 16efe6bac88..f3b6ba7a2a1 100644 --- a/tools/filters/secure_hash_message_digest.py +++ b/tools/filters/secure_hash_message_digest.py @@ -29,19 +29,18 @@ def __main__(): assert options.input, "You must provide an input filename." assert options.output, "You must provide an output filename." - input = open(options.input) - while True: - chunk = input.read(CHUNK_SIZE) - if chunk: - for algorithm in algorithms.values(): - algorithm.update(chunk) - else: - break + with open(options.input, 'rb') as fh: + while True: + chunk = fh.read(CHUNK_SIZE) + if chunk: + for algorithm in algorithms.values(): + algorithm.update(chunk) + else: + break - output = open(options.output, 'wb') - output.write('#%s\n' % ('\t'.join(algorithms.keys()))) - output.write('%s\n' % ('\t'.join(x.hexdigest() for x in algorithms.values()))) - output.close() + with open(options.output, 'w') as output: + output.write('#%s\n' % ('\t'.join(algorithms.keys()))) + output.write('%s\n' % ('\t'.join(x.hexdigest() for x in algorithms.values()))) if __name__ == "__main__": diff --git a/tools/filters/sff_extractor.xml b/tools/filters/sff_extractor.xml index 2beb7548b21..d26b6e5ca2e 100644 --- a/tools/filters/sff_extractor.xml +++ b/tools/filters/sff_extractor.xml @@ -1,9 +1,12 @@ - - #if str($fastq_output) == "fastq_false" #sff_extract.py $clip --seq_file=$out_file3 --qual_file=$out_file4 --xml_file=$out_file2 $input - #elif str($fastq_output) == "fastq_true" #sff_extract.py $clip --fastq --seq_file=$out_file1 --xml_file=$out_file2 $input - #end if# + + python + + +#if str($fastq_output) == "fastq_false" #python '$__tool_directory__/sff_extract.py' $clip --seq_file='$out_file3' --qual_file='$out_file4' --xml_file='$out_file2' '$input' +#elif str($fastq_output) == "fastq_true" #python '$__tool_directory__/sff_extract.py' $clip --fastq --seq_file='$out_file1' --xml_file='$out_file2' '$input' +#end if# diff --git a/tools/filters/tailWrapper.xml b/tools/filters/tailWrapper.xml index 1a7d7789ad5..0865145dc88 100644 --- a/tools/filters/tailWrapper.xml +++ b/tools/filters/tailWrapper.xml @@ -1,6 +1,7 @@ lines from a dataset - tailWrapper.pl $input $lineNum $out_file1 + +perl '$__tool_directory__/tailWrapper.pl' '$input' $lineNum '$out_file1' diff --git a/tools/filters/uniq.xml b/tools/filters/uniq.xml index 63ed13f176a..6e9556effff 100644 --- a/tools/filters/uniq.xml +++ b/tools/filters/uniq.xml @@ -1,6 +1,9 @@ occurrences of each record - uniq.py -i $input -o $out_file1 -c "$column" -d $delim -s $sorting + + coreutils + + python '$__tool_directory__/uniq.py' -i '$input' -o '$out_file1' -c '$column' -d $delim -s $sorting diff --git a/tools/filters/wc_gnu.xml b/tools/filters/wc_gnu.xml index 4c4f4fbd1a4..1688e70b073 100644 --- a/tools/filters/wc_gnu.xml +++ b/tools/filters/wc_gnu.xml @@ -4,11 +4,11 @@ #set $word_to_arg = { 'characters':'m', 'words':'w', 'lines':'l' } #set $arg_order = [ 'lines', 'words', 'characters' ] #if not isinstance( $options.value, list ): - #set $args = [ $options.value ] + #set global $args = [ $options.value ] #else: - #set $args = $options.value + #set global $args = $options.value #end if - #if $include_header.value: + #if $include_header.value echo "#${ "\t".join( [ i for i in $arg_order if i in $args ] ) }" > $out_file1 && #end if diff --git a/tools/filters/wig_to_bigwig.xml b/tools/filters/wig_to_bigwig.xml index a9c0d55edf6..7db6785af6a 100644 --- a/tools/filters/wig_to_bigwig.xml +++ b/tools/filters/wig_to_bigwig.xml @@ -11,7 +11,7 @@ - + - + diff --git a/tools/maf/maf_limit_to_species.py b/tools/maf/maf_limit_to_species.py index bcc4936d876..52e76719f63 100644 --- a/tools/maf/maf_limit_to_species.py +++ b/tools/maf/maf_limit_to_species.py @@ -46,7 +46,8 @@ def main(): maf_reader.close() maf_writer.close() - print("Restricted to species: %s." % ", ".join(species)) + if species: + print("Restricted to species: %s." % ", ".join(species)) print("%i MAF blocks have been kept." % maf_blocks_kept) diff --git a/tools/maf/maf_split_by_species.xml b/tools/maf/maf_split_by_species.xml index a8bd6ba0069..c73a2f81d86 100644 --- a/tools/maf/maf_split_by_species.xml +++ b/tools/maf/maf_split_by_species.xml @@ -18,12 +18,12 @@ - + - + diff --git a/tools/maf/maf_to_bed.py b/tools/maf/maf_to_bed.py index f05d468b7eb..6347924cf2a 100644 --- a/tools/maf/maf_to_bed.py +++ b/tools/maf/maf_to_bed.py @@ -6,6 +6,7 @@ from __future__ import print_function import os import sys +from collections import OrderedDict from bx.align import maf @@ -23,14 +24,15 @@ def __main__(): primary_spec = None if "None" in species: - species = set() + species = OrderedDict() try: for i, m in enumerate(maf.Reader(open(input_filename, 'r'))): for c in m.components: spec, chrom = maf.src_split(c.src) if not spec or not chrom: - spec = chrom = c.src - species.add(spec) + spec = c.src + species[spec] = None + species = species.keys() except Exception: print("Invalid MAF file specified", file=sys.stderr) return @@ -44,36 +46,35 @@ def __main__(): out_files[spec] = open(output_filename, 'w') primary_spec = spec else: - out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_bed_%s' % (output_id, spec, spec)), 'wb+') + out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_bed_%s' % (output_id, spec, spec)), 'w+') num_species = len(species) print("Restricted to species:", ",".join(species)) - file_in = open(input_filename, 'r') - maf_reader = maf.Reader(file_in) + with open(input_filename, 'r') as file_in: + maf_reader = maf.Reader(file_in) - block_num = -1 + block_num = -1 - for i, m in enumerate(maf_reader): - block_num += 1 - if "None" not in species: - m = m.limit_to_species(species) - l = m.components - if len(l) < num_species and partial == "partial_disallowed": - continue - for c in l: - spec, chrom = maf.src_split(c.src) - if not spec or not chrom: - spec = chrom = c.src - if spec not in out_files.keys(): - out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_bed_%s' % (output_id, spec, spec)), 'wb+') + for i, m in enumerate(maf_reader): + block_num += 1 + if "None" not in species: + m = m.limit_to_species(species) + l = m.components + if len(l) < num_species and partial == "partial_disallowed": + continue + for c in l: + spec, chrom = maf.src_split(c.src) + if not spec or not chrom: + spec = chrom = c.src + if spec not in out_files.keys(): + out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_bed_%s' % (output_id, spec, spec)), 'wb+') - if c.strand == "-": - out_files[spec].write(chrom + "\t" + str(c.src_size - c.end) + "\t" + str(c.src_size - c.start) + "\t" + spec + "_" + str(block_num) + "\t" + "0\t" + c.strand + "\n") - else: - out_files[spec].write(chrom + "\t" + str(c.start) + "\t" + str(c.end) + "\t" + spec + "_" + str(block_num) + "\t" + "0\t" + c.strand + "\n") + if c.strand == "-": + out_files[spec].write(chrom + "\t" + str(c.src_size - c.end) + "\t" + str(c.src_size - c.start) + "\t" + spec + "_" + str(block_num) + "\t" + "0\t" + c.strand + "\n") + else: + out_files[spec].write(chrom + "\t" + str(c.start) + "\t" + str(c.end) + "\t" + spec + "_" + str(block_num) + "\t" + "0\t" + c.strand + "\n") - file_in.close() for file_out in out_files.keys(): out_files[file_out].close() diff --git a/tools/maf/maf_to_bed.xml b/tools/maf/maf_to_bed.xml index 2464296cc2a..e7e56b6d32d 100644 --- a/tools/maf/maf_to_bed.xml +++ b/tools/maf/maf_to_bed.xml @@ -22,7 +22,7 @@ - + diff --git a/tools/maf/maf_to_fasta_multiple_sets.py b/tools/maf/maf_to_fasta_multiple_sets.py index 5d551f4a258..713b48c3861 100755 --- a/tools/maf/maf_to_fasta_multiple_sets.py +++ b/tools/maf/maf_to_fasta_multiple_sets.py @@ -6,6 +6,7 @@ Read a maf and output a multiple block fasta file. from __future__ import print_function import sys +from collections import OrderedDict from bx.align import maf @@ -51,7 +52,8 @@ def __main__(): spec_counts[spec] = 0 else: spec_counts[spec] += 1 - file_out.write("%s\n" % maf_utilities.get_fasta_header(component, {'block_index': block_num, 'species': spec, 'sequence_index': spec_counts[spec]}, suffix="%s_%i_%i" % (spec, block_num, spec_counts[spec]))) + d = OrderedDict([('block_index', block_num), ('species', spec), ('sequence_index', spec_counts[spec])]) + file_out.write("%s\n" % maf_utilities.get_fasta_header(component, d, suffix="%s_%i_%i" % (spec, block_num, spec_counts[spec]))) file_out.write("%s\n" % component.text) file_out.write("\n") file_out.close() diff --git a/tools/maf/maf_to_interval.py b/tools/maf/maf_to_interval.py index 9f51f7c7300..e9e6508b954 100644 --- a/tools/maf/maf_to_interval.py +++ b/tools/maf/maf_to_interval.py @@ -35,9 +35,9 @@ def __main__(): all_species.sort() for spec in species: if spec == primary_spec: - out_files[spec] = open(output_filename, 'wb+') + out_files[spec] = open(output_filename, 'w+') else: - out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_interval_%s' % (output_id, spec, spec)), 'wb+') + out_files[spec] = open(os.path.join(database_tmp_dir, 'primary_%s_%s_visible_interval_%s' % (output_id, spec, spec)), 'w+') out_files[spec].write('#chrom\tstart\tend\tstrand\tscore\tname\t%s\n' % ('\t'.join(all_species))) num_species = len(all_species) diff --git a/tools/maf/maf_to_interval.xml b/tools/maf/maf_to_interval.xml index 9188ef91268..86df1d109fc 100644 --- a/tools/maf/maf_to_interval.xml +++ b/tools/maf/maf_to_interval.xml @@ -26,7 +26,7 @@ - + diff --git a/tools/next_gen_conversion/solid2fastq.xml b/tools/next_gen_conversion/solid2fastq.xml index f88c4148b23..9d1372dd967 100644 --- a/tools/next_gen_conversion/solid2fastq.xml +++ b/tools/next_gen_conversion/solid2fastq.xml @@ -46,9 +46,9 @@ - + - + @@ -58,9 +58,9 @@ - + - + diff --git a/tools/phenotype_association/ldtools.xml b/tools/phenotype_association/ldtools.xml index 9fc188ea934..653d9d0f4e5 100644 --- a/tools/phenotype_association/ldtools.xml +++ b/tools/phenotype_association/ldtools.xml @@ -1,8 +1,10 @@ linkage disequilibrium and tag SNPs - - - ldtools_wrapper.sh rsquare=$rsquare freq=$freq input=$input output=$output + + python + + +'$__tool_directory__/ldtools_wrapper.sh' rsquare=$rsquare freq=$freq input='$input' output='$output' diff --git a/tools/phenotype_association/ldtools_wrapper.sh b/tools/phenotype_association/ldtools_wrapper.sh index fb70a36fc02..331d01bf025 100755 --- a/tools/phenotype_association/ldtools_wrapper.sh +++ b/tools/phenotype_association/ldtools_wrapper.sh @@ -5,7 +5,7 @@ set -e -export PATH=$PATH:$(dirname $0) +SCRIPT_DIR=$(dirname $0) ## pagetag options input= @@ -46,19 +46,16 @@ do done ## run pagetag -pagetag.py --rsquare $rsquare --freq $freq $input snps.txt neighborhood.txt &> /dev/null +python $SCRIPT_DIR/pagetag.py --rsquare $rsquare --freq $freq "$input" snps.txt neighborhood.txt if [ $? -ne 0 ]; then - echo "failed: pagetag.py --rsquare $rsquare --freq $freq $input snps.txt neighborhood.txt" + echo "failed: pagetag.py --rsquare $rsquare --freq $freq \"$input\" snps.txt neighborhood.txt" exit 1 fi ## run sentag -senatag.py neighborhood.txt snps.txt > $output 2> /dev/null +python $SCRIPT_DIR/senatag.py neighborhood.txt snps.txt > "$output" if [ $? -ne 0 ]; then echo "failed: senatag.py neighborhood.txt snps.txt" exit 1 fi -## cleanup -rm -f snps.txt neighborhood.txt - diff --git a/tools/phenotype_association/linkToGProfile.pl b/tools/phenotype_association/linkToGProfile.pl index 0f0ee242e09..2d6e8100482 100755 --- a/tools/phenotype_association/linkToGProfile.pl +++ b/tools/phenotype_association/linkToGProfile.pl @@ -77,7 +77,7 @@ if (@pos) { $params{"region_query"} = 1; } open(FH, ">", $out) or die "Couldn't open $out, $!\n"; print FH "g:Profiler link\n"; -print FH '
'; +print FH '', "\n"; foreach my $k (keys %params) { print FH "\n"; } diff --git a/tools/phenotype_association/linkToGProfile.xml b/tools/phenotype_association/linkToGProfile.xml index a21a00d3c8c..908099e8270 100644 --- a/tools/phenotype_association/linkToGProfile.xml +++ b/tools/phenotype_association/linkToGProfile.xml @@ -1,8 +1,8 @@ tools for functional profiling of gene lists - - linkToGProfile.pl $input $type $out_file1 -region=$region -gene=$genes -chr=${input.metadata.chromCol} -start=${input.metadata.startCol} -end=${input.metadata.endCol} + +perl '$__tool_directory__/linkToGProfile.pl' '$input' $type '$out_file1' -region='$region' -gene=$genes -chr=${input.metadata.chromCol} -start=${input.metadata.startCol} -end=${input.metadata.endCol} @@ -34,10 +34,10 @@ - + - - + + diff --git a/tools/phenotype_association/senatag.py b/tools/phenotype_association/senatag.py index 1fffd53ee83..d4637838d2b 100755 --- a/tools/phenotype_association/senatag.py +++ b/tools/phenotype_association/senatag.py @@ -25,6 +25,7 @@ from __future__ import print_function import heapq import os +from functools import total_ordering from getopt import getopt, GetoptError from sys import argv, exit, stderr @@ -35,6 +36,7 @@ __email__ = "ratan@bx.psu.edu" debug_flag = False +@total_ordering class node: def __init__(self, name): self.name = name @@ -50,8 +52,14 @@ class node: num += 1 return num - def __cmp__(self, other): - return other.num_not_visited() - self.num_not_visited() + def __eq__(self, other): + return self.num_not_visited() == other.num_not_visited() + + def __ne__(self, other): + return not (self == other) + + def __lt__(self, other): + return other.num_not_visited() < self.num_not_visited() def __str__(self): return self.name @@ -165,7 +173,7 @@ def main(ldfile, snpsfile, required, excluded): neighbors[t.name] = list(set(ns)) # find the tag SNPs for this graph - data = g.nodes.values()[:] + data = list(g.nodes.values())[:] heapq.heapify(data) while data: diff --git a/tools/plotting/boxplot.xml b/tools/plotting/boxplot.xml index 38fc474b8bb..415e02e2f8c 100644 --- a/tools/plotting/boxplot.xml +++ b/tools/plotting/boxplot.xml @@ -1,8 +1,8 @@ - + of quality statistics gnuplot < '$gnuplot_commands' 2>&1 || echo "Error running gnuplot." >&2 - gnuplot + gnuplot @@ -44,7 +44,7 @@ set offsets 1, 1, 1, 1 plot '${input_file}' using ${xcol}:${q1col}:${lwcol}:${rwcol}:${q3col} with candlesticks lt 1 lw 1 title 'Quartiles' whiskerbars, \ '' using ${xcol}:${medcol}:${medcol}:${medcol}:${medcol} with candlesticks lt -1 lw 2 title 'Medians'\ #if str( $use_outliers['use_outliers_type'] ) == 'use_outliers': -, "< python -c \"for xval, yvals in [ ( fields[${xcol} - 1], fields[${use_outliers['outliercol']} - 1].split( ',' ) ) for fields in [ line.rstrip( '\\n\\r' ).split( '\\t' ) for line in open( '${input_file}' ) if not line.startswith( '#' ) ] if len( fields ) > max( ${xcol} - 1, ${use_outliers['outliercol']} - 1 ) ]: print '\\n'.join( [ '%s\\t%s' % ( xval, yval ) for yval in yvals if yval ] )\"" using 1:2 with points pt 29 title 'Outliers' +, "< python -c \"for xval, yvals in [ ( fields[${xcol} - 1], fields[${use_outliers['outliercol']} - 1].split( ',' ) ) for fields in [ line.rstrip( '\\n\\r' ).split( '\\t' ) for line in open( '${input_file}' ) if not line.startswith( '#' ) ] if len( fields ) > max( ${xcol} - 1, ${use_outliers['outliercol']} - 1 ) ]: print('\\n'.join( [ '%s\\t%s' % ( xval, yval ) for yval in yvals if yval ] ))\"" using 1:2 with points pt 29 title 'Outliers' #end if @@ -66,7 +66,7 @@ plot '${input_file}' using ${xcol}:${q1col}:${lwcol}:${rwcol}:${q3col} with cand - + diff --git a/tools/solid_tools/solid_qual_stats.py b/tools/solid_tools/solid_qual_stats.py index bc172e29f50..4dd0e992644 100644 --- a/tools/solid_tools/solid_qual_stats.py +++ b/tools/solid_tools/solid_qual_stats.py @@ -26,109 +26,110 @@ def unzip(filename): def __main__(): infile_score_name = sys.argv[1].strip() - fout = open(sys.argv[2].strip(), 'r+w') + with open(sys.argv[2].strip(), 'w') as fout: - if zipfile.is_zipfile(infile_score_name): - infile_name = unzip(infile_score_name) - else: - infile_name = infile_score_name + if zipfile.is_zipfile(infile_score_name): + infile_name = unzip(infile_score_name) + else: + infile_name = infile_score_name - readlen = None - invalid_lines = 0 - j = 0 - for line in open(infile_name): - line = line.strip() - if not(line) or line.startswith("#") or line.startswith(">"): - continue - elems = line.split() - try: - for item in elems: - int(item) - if not readlen: - readlen = len(elems) - if len(elems) != readlen: - print("Note: Reads in the input dataset are of variable lengths.") - j += 1 - except ValueError: - invalid_lines += 1 - if j > 10: - break + readlen = None + invalid_lines = 0 + j = 0 + with open(infile_name) as fin: + for line in fin: + line = line.strip() + if not(line) or line.startswith("#") or line.startswith(">"): + continue + elems = line.split() + try: + for item in elems: + int(item) + if not readlen: + readlen = len(elems) + if len(elems) != readlen: + print("Note: Reads in the input dataset are of variable lengths.") + j += 1 + except ValueError: + invalid_lines += 1 + if j > 10: + break - position_dict = {} - print("column\tcount\tmin\tmax\tsum\tmean\tQ1\tmed\tQ3\tIQR\tlW\trW", file=fout) - for k, line in enumerate(open(infile_name)): - line = line.strip() - if not(line) or line.startswith("#") or line.startswith(">"): - continue - elems = line.split() - if position_dict == {}: - for pos in range(readlen): - position_dict[pos] = [0] * QUAL_UPPER_BOUND - if len(elems) != readlen: - invalid_lines += 1 - continue - for ind, item in enumerate(elems): - try: - item = int(item) - position_dict[ind][item] += 1 - except Exception: - pass + position_dict = {} + print("column\tcount\tmin\tmax\tsum\tmean\tQ1\tmed\tQ3\tIQR\tlW\trW", file=fout) + for k, line in enumerate(open(infile_name)): + line = line.strip() + if not(line) or line.startswith("#") or line.startswith(">"): + continue + elems = line.split() + if position_dict == {}: + for pos in range(readlen): + position_dict[pos] = [0] * QUAL_UPPER_BOUND + if len(elems) != readlen: + invalid_lines += 1 + continue + for ind, item in enumerate(elems): + try: + item = int(item) + position_dict[ind][item] += 1 + except Exception: + pass - invalid_positions = 0 - for pos in position_dict: - carr = position_dict[pos] # count array for position pos - total = sum(carr) # number of bases found in this column. - med_elem = int(round(total / 2.0)) - lowest = None # Lowest quality score value found in this column. - highest = None # Highest quality score value found in this column. - median = None # Median quality score value found in this column. - qsum = 0.0 # Sum of quality score values for this column. - q1 = None # 1st quartile quality score. - q3 = None # 3rd quartile quality score. - q1_elem = int(round((total + 1) / 4.0)) - q3_elem = int(round((total + 1) * 3 / 4.0)) + invalid_positions = 0 + for pos in position_dict: + carr = position_dict[pos] # count array for position pos + total = sum(carr) # number of bases found in this column. + med_elem = int(round(total / 2.0)) + lowest = None # Lowest quality score value found in this column. + highest = None # Highest quality score value found in this column. + median = None # Median quality score value found in this column. + qsum = 0.0 # Sum of quality score values for this column. + q1 = None # 1st quartile quality score. + q3 = None # 3rd quartile quality score. + q1_elem = int(round((total + 1) / 4.0)) + q3_elem = int(round((total + 1) * 3 / 4.0)) - try: - for ind, cnt in enumerate(carr): - qsum += ind * cnt + try: + for ind, cnt in enumerate(carr): + qsum += ind * cnt - if cnt != 0: - highest = ind + if cnt != 0: + highest = ind - if lowest is None and cnt != 0: # first non-zero count - lowest = ind + if lowest is None and cnt != 0: # first non-zero count + lowest = ind - if q1 is None: - if sum(carr[:ind + 1]) >= q1_elem: - q1 = ind + if q1 is None: + if sum(carr[:ind + 1]) >= q1_elem: + q1 = ind - if median is None: - if sum(carr[:ind + 1]) < med_elem: - continue - median = ind - if total % 2 == 0: # even number of elements - median2 = median - if sum(carr[:ind + 1]) < med_elem + 1: - for ind2, elem in enumerate(carr[ind + 1:]): - if elem != 0: - median2 = ind + ind2 + 1 - break - median = (median + median2) / 2.0 + if median is None: + if sum(carr[:ind + 1]) < med_elem: + continue + median = ind + if total % 2 == 0: # even number of elements + median2 = median + if sum(carr[:ind + 1]) < med_elem + 1: + for ind2, elem in enumerate(carr[ind + 1:]): + if elem != 0: + median2 = ind + ind2 + 1 + break + median = (median + median2) / 2.0 - if q3 is None: - if sum(carr[:ind + 1]) >= q3_elem: - q3 = ind + if q3 is None: + if sum(carr[:ind + 1]) >= q3_elem: + q3 = ind - mean = qsum / total # Mean quality score value for this column. - iqr = q3 - q1 - left_whisker = max(q1 - 1.5 * iqr, lowest) - right_whisker = min(q3 + 1.5 * iqr, highest) + mean = qsum / total # Mean quality score value for this column. + iqr = q3 - q1 + left_whisker = max(q1 - 1.5 * iqr, lowest) + right_whisker = min(q3 + 1.5 * iqr, highest) - print("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (pos + 1, total, lowest, highest, qsum, mean, q1, median, q3, iqr, left_whisker, right_whisker), file=fout) - except Exception: - invalid_positions += 1 - nullvals = ['NA'] * 11 - print("%s\t%s" % (pos + 1, '\t'.join(nullvals)), file=fout) + print("%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s" % (pos + 1, total, lowest, highest, qsum, mean, q1, median, q3, iqr, left_whisker, right_whisker), file=fout) + except Exception: + invalid_positions += 1 + nullvals = ['NA'] * 11 + print("%s\t%s" % (pos + 1, '\t'.join(nullvals)), file=fout) if invalid_lines: print("Skipped %d reads as invalid." % invalid_lines) diff --git a/tools/solid_tools/solid_qual_stats.xml b/tools/solid_tools/solid_qual_stats.xml index c2018592f8c..6da89ed5d4f 100644 --- a/tools/solid_tools/solid_qual_stats.xml +++ b/tools/solid_tools/solid_qual_stats.xml @@ -1,7 +1,7 @@ for SOLiD data - solid_qual_stats.py $input $output1 - + +python '$__tool_directory__/solid_qual_stats.py' '$input' '$output1' @@ -11,7 +11,7 @@ - + diff --git a/tools/stats/aggregate_binned_scores_in_intervals.xml b/tools/stats/aggregate_binned_scores_in_intervals.xml index 827bd805e2a..7e204c8ef67 100644 --- a/tools/stats/aggregate_binned_scores_in_intervals.xml +++ b/tools/stats/aggregate_binned_scores_in_intervals.xml @@ -38,6 +38,7 @@ python '$__tool_directory__/aggregate_scores_in_intervals.py' + - + diff --git a/tools/stats/grouping.py b/tools/stats/grouping.py index c2ecde15e43..a003219217e 100644 --- a/tools/stats/grouping.py +++ b/tools/stats/grouping.py @@ -72,7 +72,7 @@ def main(): except Exception: stop_err("Group column not specified.") - tmpfile = tempfile.NamedTemporaryFile() + tmpfile = tempfile.NamedTemporaryFile(mode='r') try: """ diff --git a/tools/stats/gsummary.xml b/tools/stats/gsummary.xml index 86c74073597..f7168d2400b 100644 --- a/tools/stats/gsummary.xml +++ b/tools/stats/gsummary.xml @@ -1,10 +1,10 @@ - + for any numerical column topic_2269 - rpy2 + rpy2