diff --git a/.github/workflows/converter_tests.yaml b/.github/workflows/converter_tests.yaml
new file mode 100644
index 00000000000..a8cd3f3f7a4
--- /dev/null
+++ b/.github/workflows/converter_tests.yaml
@@ -0,0 +1,29 @@
+name: Converter tests
+on: [push, pull_request]
+jobs:
+
+ test:
+ name: Test
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - uses: actions/checkout@v1
+ with:
+ fetch-depth: 1
+ - uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Cache venv dir
+ uses: actions/cache@v1
+ id: pip-cache
+ with:
+ path: ~/.cache/pip
+ key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+ - name: fetch test data
+ run: git clone https://github.com/galaxyproject/galaxy-test-data && cp -R galaxy-test-data/* test-data
+ - name: Install planemo
+ run: pip install planemo
+ - name: Run tests
+ run: 'planemo test --galaxy_python_version ${{ matrix.python-version }} --galaxy_root . lib/galaxy/datatypes/converters/*xml'
diff --git a/client/styleguide.config.js b/client/styleguide.config.js
index d3d7d64d7e9..511dc134efc 100644
--- a/client/styleguide.config.js
+++ b/client/styleguide.config.js
@@ -2,8 +2,9 @@ const path = require("path");
const glob = require("glob");
const fs = require("fs");
const merge = require("webpack-merge");
+const baseConfig = require("./webpack.config.js")
-let webpackConfig = require("./webpack.config.js");
+const webpackConfig = baseConfig();
const fileLoaderTest = /\.(png|jpg|jpeg|gif|svg|woff|woff2|ttf|eot)(\?.*$|$)/;
@@ -37,9 +38,9 @@ const fileLoaderConfigRule = { rules: [{ test: fileLoaderTest, use: ["file-loade
webpackConfig.module = merge.smart(webpackConfig.module, fileLoaderConfigRule);
webpackConfig.output.publicPath = "";
-webpackConfig.resolve.modules.push( path.join(__dirname, "galaxy/style/scss") );
+webpackConfig.resolve.modules.push(path.join(__dirname, "galaxy/style/scss"));
-galaxyStyleDocs = []
+const galaxyStyleDocs = [];
glob.sync("./galaxy/docs/galaxy-*.md").forEach(file => {
const name = file.match(/galaxy-(\w+).md/)[1];
galaxyStyleDocs.push({ name: name, content: file });
@@ -47,26 +48,29 @@ glob.sync("./galaxy/docs/galaxy-*.md").forEach(file => {
const sections = [
{
- name: 'Components',
- // Components that are directories will get their own section
- sections: glob.sync("./galaxy/scripts/components/*").map(file => {
- if (fs.lstatSync(file).isDirectory()) {
- return {
- name: path.basename(file),
- components: file + "/**/*.vue"
- };
- }
- }).filter( v => v ),
- // ...while top level components are handled here.
- components: './galaxy/scripts/components/*.vue',
+ name: "Galaxy styles",
+ sections: galaxyStyleDocs
},
{
- name: 'Galaxy styles',
- sections: galaxyStyleDocs
+ name: "Basic Bootstrap Styles",
+ content: "./galaxy/docs/bootstrap.md"
},
{
- name: "Basic Bootstrap Styles",
- content: "./galaxy/docs/bootstrap.md"
+ name: "Components",
+ // Components that are directories will get their own section
+ sections: glob
+ .sync("./galaxy/scripts/components/*")
+ .map(file => {
+ if (fs.lstatSync(file).isDirectory()) {
+ return {
+ name: path.basename(file),
+ components: file + "/**/*.vue"
+ };
+ }
+ })
+ .filter(v => v),
+ // ...while top level components are handled here.
+ components: "./galaxy/scripts/components/*.vue"
}
];
diff --git a/client/webpack.config.js b/client/webpack.config.js
index 2c45fbb7499..4ea1b1e85c9 100644
--- a/client/webpack.config.js
+++ b/client/webpack.config.js
@@ -15,7 +15,7 @@ module.exports = (env = {}, argv = {}) => {
// environment name based on -d, -p, webpack flag
const targetEnv = argv.mode || "development";
- let buildconfig = {
+ const buildconfig = {
entry: {
login: ["polyfills", "bundleEntries", "entry/login"],
analysis: ["polyfills", "bundleEntries", "entry/analysis"],
diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
index 9f5db55df75..f2aacad6d03 100644
--- a/lib/galaxy/config/sample/datatypes_conf.xml.sample
+++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -142,6 +142,7 @@
+
diff --git a/lib/galaxy/datatypes/converters/bam_to_bai.xml b/lib/galaxy/datatypes/converters/bam_to_bai.xml
index 1587ff83cab..65ace10fc93 100644
--- a/lib/galaxy/datatypes/converters/bam_to_bai.xml
+++ b/lib/galaxy/datatypes/converters/bam_to_bai.xml
@@ -1,14 +1,20 @@
-
+
- samtools
+ samtools
- samtools index '$input1' '$output1'
+ samtools index -@ \${GALAXY_SLOTS:-1} '$input1' '$output1'
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
index 7f8d8bcfeb3..53abc0a9c5c 100644
--- a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
@@ -1,8 +1,9 @@
- ucsc-bedgraphtobigwig
- bedtools
+ ucsc-bedgraphtobigwig
+ bedtools
+ coreutils
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml b/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml
index 83bf34f11f7..b3b79270ef6 100644
--- a/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml
+++ b/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml
@@ -12,6 +12,12 @@ bcftools view -o '$output1' -O u '$input1'
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml b/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml
index faf09918f64..a7f5b48813f 100644
--- a/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml
+++ b/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml
@@ -12,6 +12,12 @@ bcftools view -o '$output1' -O b '$input1'
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
index cd64113d236..2b2d7ab6f53 100644
--- a/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
@@ -38,6 +38,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml
index 08d91cddf72..270b98bcf16 100644
--- a/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py' -P bed '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml b/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml
index 2c3c55541ac..eb7ac28ea03 100644
--- a/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml
@@ -8,6 +8,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml b/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml
index 26a4eeb2f2d..79ed48a9ef6 100644
--- a/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/bed_to_gff_converter.py' '$input1' '$output1'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml
index bb171c0cd0b..98618ef4c68 100644
--- a/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_interval_index_converter.py' '$input1' '$output1'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml
index 72c2b908de5..cfedf148a3b 100644
--- a/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P bed '$input1' '$bgzip' '$output1'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py
deleted file mode 100644
index 096b6a01254..00000000000
--- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import division
-
-import sys
-
-from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict
-from six import Iterator
-
-BLOCK_SIZE = 100
-
-
-class BedGraphReader(Iterator):
- def __init__(self, f):
- self.f = f
-
- def __iter__(self):
- return self
-
- def __next__(self):
- while True:
- line = self.f.readline()
- if not line:
- raise StopIteration()
- if line.isspace():
- continue
- if line[0] == "#":
- continue
- if line[0].isalpha():
- if line.startswith("track") or line.startswith("browser"):
- continue
-
- feature = line.strip().split()
- chrom = feature[0]
- chrom_start = int(feature[1])
- chrom_end = int(feature[2])
- score = float(feature[3])
- return chrom, chrom_start, chrom_end, None, score
-
-
-def main():
- input_fname = sys.argv[1]
- out_fname = sys.argv[2]
-
- reader = BedGraphReader(open(input_fname))
-
- # Fill array from reader
- d = array_tree_dict_from_reader(reader, {}, block_size=BLOCK_SIZE)
-
- for array_tree in d.values():
- array_tree.root.build_summary()
-
- FileArrayTreeDict.dict_to_file(d, open(out_fname, "w"))
-
-
-if __name__ == "__main__":
- main()
diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.xml b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.xml
deleted file mode 100644
index f01a0408f02..00000000000
--- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
- python '$__tool_directory__/bedgraph_to_array_tree_converter.py' '$input' '$output'
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
index f842d0201f5..b984cd3d43f 100644
--- a/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
@@ -10,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/biom1_to_biom2.xml b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml
index 8dc76d09e26..672bba616ea 100644
--- a/lib/galaxy/datatypes/converters/biom1_to_biom2.xml
+++ b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml
@@ -9,6 +9,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/biom2_to_biom1.xml b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml
index 559763ff74f..a280bd84432 100644
--- a/lib/galaxy/datatypes/converters/biom2_to_biom1.xml
+++ b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml
@@ -9,6 +9,16 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml b/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml
index 3d5ee318b78..ebe660218a0 100644
--- a/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml
+++ b/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml
@@ -1,4 +1,7 @@
-
+
+
+ bzip2
+
'$output1'
]]>
diff --git a/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml b/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml
index 440e72e4e81..a877bcd3320 100644
--- a/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml
+++ b/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ pysam
+
diff --git a/lib/galaxy/datatypes/converters/csv_to_tabular.xml b/lib/galaxy/datatypes/converters/csv_to_tabular.xml
index 4a00b67e406..95809c39c80 100644
--- a/lib/galaxy/datatypes/converters/csv_to_tabular.xml
+++ b/lib/galaxy/datatypes/converters/csv_to_tabular.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/tabular_csv.py' -o '$tabular' -i '$csv'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/dbkeys.loc.test b/lib/galaxy/datatypes/converters/dbkeys.loc.test
new file mode 100644
index 00000000000..2c13efb0fc0
--- /dev/null
+++ b/lib/galaxy/datatypes/converters/dbkeys.loc.test
@@ -0,0 +1,2 @@
+#
+hg17 hg17 ${__HERE__}/hg17.len
diff --git a/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml
index babbe100a48..15518840912 100644
--- a/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py'
-c ${input1.metadata.chromCol}
diff --git a/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml
index d3506653e84..8ba65850ba0 100644
--- a/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py'
-c ${input1.metadata.chromCol}
diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
index 419e04c2ec8..65706581c4b 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
@@ -11,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml
index 8b191f1376e..7482cd03c28 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml
@@ -18,6 +18,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml
index 81ebd75a23a..d72edeae150 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml
@@ -19,6 +19,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_fai.xml b/lib/galaxy/datatypes/converters/fasta_to_fai.xml
index b8d1d3b5b46..6d572fdde85 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_fai.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_fai.xml
@@ -12,6 +12,12 @@ samtools faidx temp.fasta
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_len.xml b/lib/galaxy/datatypes/converters/fasta_to_len.xml
index 74f021cd753..03dd3672463 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_len.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_len.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/fasta_to_len.py' '$input' '$output' 0
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml
index d87ae808905..f9722c7d493 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/fasta_to_tabular_converter.py' '$input' '$output'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py
index 5a90201de8f..f6cab4c12fe 100644
--- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py
+++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.py
@@ -20,30 +20,26 @@ def main():
"""
input_fname = sys.argv[1]
if is_gzip(input_fname):
- print('Conversion is only possible for uncompressed files')
- sys.exit(1)
-
- out_file = open(sys.argv[2], 'w')
+ sys.exit('Conversion is only possible for uncompressed files')
current_line = 0
sequences = 1000000
lines_per_chunk = 4 * sequences
chunk_begin = 0
- in_file = open(input_fname)
+ with open(input_fname) as in_file, open(sys.argv[2], 'w') as out_file:
+ out_file.write('{"sections" : [')
- out_file.write('{"sections" : [')
+ for line in in_file:
+ current_line += 1
+ if 0 == current_line % lines_per_chunk:
+ chunk_end = in_file.tell()
+ out_file.write('{"start":"%s","end":"%s","sequences":"%s"},' % (chunk_begin, chunk_end, sequences))
+ chunk_begin = chunk_end
- for line in in_file:
- current_line += 1
- if 0 == current_line % lines_per_chunk:
- chunk_end = in_file.tell()
- out_file.write('{"start":"%s","end":"%s","sequences":"%s"},' % (chunk_begin, chunk_end, sequences))
- chunk_begin = chunk_end
-
- chunk_end = in_file.tell()
- out_file.write('{"start":"%s","end":"%s","sequences":"%s"}' % (chunk_begin, chunk_end, (current_line % lines_per_chunk) / 4))
- out_file.write(']}\n')
+ chunk_end = in_file.tell()
+ out_file.write('{"start":"%s","end":"%s","sequences":"%s"}' % (chunk_begin, chunk_end, (current_line % lines_per_chunk) / 4))
+ out_file.write(']}\n')
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml
index d8bb8ad377f..ca11e91cc4e 100644
--- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml
+++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml
@@ -1,4 +1,7 @@
-
+
+
+ galaxy-util
+
python '$__tool_directory__/fastq_to_fqtoc.py' '$input1' '$output1'
@@ -6,6 +9,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py
index 8ddb749ec91..86b0cd9a1cc 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py
@@ -19,8 +19,7 @@ assert sys.version_info[:2] >= (2, 4)
def stop_err(msg):
- sys.stderr.write("%s" % msg)
- sys.exit()
+ sys.exit("%s" % msg)
def __main__():
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml
index b4b1c89b00c..ff528b6f65f 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml
@@ -1,5 +1,8 @@
-
+
converts Fastqsolexa file to Fasta format
+
+ python
+
python '$__tool_directory__/fastqsolexa_to_fasta_converter.py' '$input' '$output'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py
index e0aba894622..90fc5527ca7 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py
@@ -19,8 +19,7 @@ assert sys.version_info[:2] >= (2, 4)
def stop_err(msg):
- sys.stderr.write("%s" % msg)
- sys.exit()
+ sys.exit("%s" % msg)
def __main__():
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml
index b5ac900ae5e..eb0757db83b 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ python
+
python '$__tool_directory__/fastqsolexa_to_qual_converter.py' '$input1' '$output1' ${input1.extension}
@@ -6,6 +9,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml b/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml
index 4835288abe9..1208586b05d 100644
--- a/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/gff_to_bed_converter.py' '$input1' '$output1'
@@ -8,6 +8,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml
index f26f8049bb8..3dcfa1d18bd 100644
--- a/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py' -P gff '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml b/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
index 0cd2b0e46ed..cba80687cf1 100644
--- a/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/interval_to_fli.py' -F $input1.extension '$input1' '$output1'
@@ -8,6 +8,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py
index 6dda96b944f..2f5e131e2c9 100644
--- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py
@@ -34,7 +34,7 @@ def main():
# not included in the index.
offset += feature.raw_size
- index.write(open(out_fname, "w"))
+ index.write(open(out_fname, "wb"))
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml
index f5971eb197a..47a6e72d269 100644
--- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/gff_to_interval_index_converter.py' '$input1' '$output1'
@@ -7,6 +7,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml
index ad0a6059970..06ffce81787 100644
--- a/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P gff '$input1' '$bgzip' '$output1'
@@ -8,6 +11,13 @@
+
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/hg17.len b/lib/galaxy/datatypes/converters/hg17.len
new file mode 100644
index 00000000000..801131eeb22
--- /dev/null
+++ b/lib/galaxy/datatypes/converters/hg17.len
@@ -0,0 +1,25 @@
+dummy_chr 100000000
+super_1 100000000
+chr1 100000000
+chr7 200000000
+chrX 200000000
+phiX174 100000000
+random_phiX_region_1 100000000
+random_phiX_region_2 100000000
+random_phiX_region_3 100000000
+random_phiX_region_4 100000000
+random_phiX_region_5 100000000
+random_phiX_region_6 100000000
+random_phiX_region_7 100000000
+random_phiX_region_8 100000000
+random_phiX_region_9 100000000
+random_phiX_region_10 100000000
+random_phiX_region_11 100000000
+random_phiX_region_12 100000000
+random_phiX_region_13 100000000
+random_phiX_region_14 100000000
+random_phiX_region_15 100000000
+random_phiX_region_16 100000000
+random_phiX_region_17 100000000
+random_phiX_region_18 100000000
+random_phiX_region_19 100000000
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml
index 26bc669653a..ace96313d74 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_bedstrict_converter.py'
'$output1' '$input1' ${input1.metadata.chromCol}
@@ -13,6 +16,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml
index 3130f5d8865..77f39d8dfa9 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_bedstrict_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension} 6
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py
index 38fc4ed3679..57505520611 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py
@@ -10,8 +10,7 @@ assert sys.version_info[:2] >= (2, 6)
def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit()
+ sys.exit(msg)
def __main__():
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml
index 11109b7d3e3..5ff627e8174 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml
@@ -1,8 +1,7 @@
-
+
- python
- bx-python
+ bx-python
python '$__tool_directory__/interval_to_bed_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol}
@@ -11,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py
index 7a6dc10358f..e309292db44 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py
@@ -10,8 +10,7 @@ assert sys.version_info[:2] >= (2, 6)
def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit()
+ sys.exit(msg)
def force_bed_field_count(fields, region_count, force_num_columns):
diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml
index 47b3a7a4e4d..33203ae06d6 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_bedstrict_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension}
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml
index 615170f893a..020f539d501 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py'
-c ${input1.metadata.chromCol}
@@ -13,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
index 05654055e55..684001e6a03 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
@@ -1,8 +1,9 @@
- ucsc-bedgraphtobigwig
- bedtools
+ ucsc-bedgraphtobigwig
+ bedtools
+ coreutils
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py
deleted file mode 100644
index aff6b296a11..00000000000
--- a/lib/galaxy/datatypes/converters/interval_to_coverage.py
+++ /dev/null
@@ -1,151 +0,0 @@
-#!/usr/bin/env python
-"""
-Converter to generate 3 (or 4) column base-pair coverage from an interval file.
-
-usage: %prog bed_file out_file
- -1, --cols1=N,N,N,N: Columns for chrom, start, end, strand in interval file
- -2, --cols2=N,N,N,N: Columns for chrom, start, end, strand in coverage file
-"""
-import subprocess
-import tempfile
-from bisect import bisect
-from os import environ
-
-from bx.cookbook import doc_optparse
-from bx.intervals import io
-
-INTERVAL_METADATA = ('chromCol',
- 'startCol',
- 'endCol',
- 'strandCol',)
-
-COVERAGE_METADATA = ('chromCol',
- 'positionCol',
- 'forwardCol',
- 'reverseCol',)
-
-
-def main(interval, coverage):
- """
- Uses a sliding window of partitions to count coverages.
- Every interval record adds its start and end to the partitions. The result
- is a list of partitions, or every position that has a (maybe) different
- number of basepairs covered. We don't worry about merging because we pop
- as the sorted intervals are read in. As the input start positions exceed
- the partition positions in partitions, coverages are kicked out in bulk.
- """
- partitions = []
- forward_covs = []
- reverse_covs = []
- chrom = None
- lastchrom = None
- for record in interval:
- chrom = record.chrom
- if lastchrom and not lastchrom == chrom and partitions:
- for partition in range(0, len(partitions) - 1):
- forward = forward_covs[partition]
- reverse = reverse_covs[partition]
- if forward + reverse > 0:
- coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
- forward=forward, reverse=reverse)
- partitions = []
- forward_covs = []
- reverse_covs = []
-
- start_index = bisect(partitions, record.start)
- forward = int(record.strand == "+")
- reverse = int(record.strand == "-")
- forward_base = 0
- reverse_base = 0
- if start_index > 0:
- forward_base = forward_covs[start_index - 1]
- reverse_base = reverse_covs[start_index - 1]
- partitions.insert(start_index, record.start)
- forward_covs.insert(start_index, forward_base)
- reverse_covs.insert(start_index, reverse_base)
- end_index = bisect(partitions, record.end)
- for index in range(start_index, end_index):
- forward_covs[index] += forward
- reverse_covs[index] += reverse
- partitions.insert(end_index, record.end)
- forward_covs.insert(end_index, forward_covs[end_index - 1] - forward)
- reverse_covs.insert(end_index, reverse_covs[end_index - 1] - reverse)
-
- if partitions:
- for partition in range(0, start_index):
- forward = forward_covs[partition]
- reverse = reverse_covs[partition]
- if forward + reverse > 0:
- coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
- forward=forward, reverse=reverse)
- partitions = partitions[start_index:]
- forward_covs = forward_covs[start_index:]
- reverse_covs = reverse_covs[start_index:]
-
- lastchrom = chrom
-
- # Finish the last chromosome
- if partitions:
- for partition in range(0, len(partitions) - 1):
- forward = forward_covs[partition]
- reverse = reverse_covs[partition]
- if forward + reverse > 0:
- coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
- forward=forward, reverse=reverse)
-
-
-class CoverageWriter(object):
- def __init__(self, out_stream=None, chromCol=0, positionCol=1, forwardCol=2, reverseCol=3):
- self.out_stream = out_stream
- self.reverseCol = reverseCol
- self.nlines = 0
- positions = {str(chromCol): '%(chrom)s',
- str(positionCol): '%(position)d',
- str(forwardCol): '%(forward)d',
- str(reverseCol): '%(reverse)d'}
- if reverseCol < 0:
- self.template = "%(0)s\t%(1)s\t%(2)s\n" % positions
- else:
- self.template = "%(0)s\t%(1)s\t%(2)s\t%(3)s\n" % positions
-
- def write(self, **kwargs):
- if self.reverseCol < 0:
- kwargs['forward'] += kwargs['reverse']
- posgen = kwargs['position']
- for position in posgen:
- kwargs['position'] = position
- self.out_stream.write(self.template % kwargs)
-
- def close(self):
- self.out_stream.flush()
- self.out_stream.close()
-
-
-if __name__ == "__main__":
- options, args = doc_optparse.parse(__doc__)
- try:
- chr_col_1, start_col_1, end_col_1, strand_col_1 = [int(x) - 1 for x in options.cols1.split(',')]
- chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x) - 1 for x in options.cols2.split(',')]
- in_fname, out_fname = args
- except Exception:
- doc_optparse.exception()
-
- # Sort through a tempfile first
- with tempfile.NamedTemporaryFile(mode="r") as temp_file:
- environ['LC_ALL'] = 'POSIX'
- subprocess.check_call([
- 'sort', '-f', '-n', '-k', chr_col_1 + 1, '-k', start_col_1 + 1, '-k', end_col_1 + 1, '-o', temp_file.name, in_fname
- ])
-
- coverage = CoverageWriter(out_stream=open(out_fname, "a"),
- chromCol=chr_col_2, positionCol=position_col_2,
- forwardCol=forward_col_2, reverseCol=reverse_col_2, )
- temp_file.seek(0)
- interval = io.NiceReaderWrapper(temp_file,
- chrom_col=chr_col_1,
- start_col=start_col_1,
- end_col=end_col_1,
- strand_col=strand_col_1,
- fix_strand=True)
- main(interval, coverage)
- coverage.close()
diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.xml b/lib/galaxy/datatypes/converters/interval_to_coverage.xml
deleted file mode 100644
index 1b3a139fec9..00000000000
--- a/lib/galaxy/datatypes/converters/interval_to_coverage.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
- python '$__tool_directory__/interval_to_coverage.py' '$input1' '$output1'
- -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol}
- -2 ${output1.metadata.chromCol},${output1.metadata.positionCol},${output1.metadata.forwardCol},${output1.metadata.reverseCol}
-
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py
index 54dbd2d16ed..c5f7b72cb85 100644
--- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py
@@ -33,18 +33,20 @@ def main():
# Do conversion.
index = Indexes()
offset = 0
- for line in open(input_fname, "r"):
- feature = line.strip().split()
- if not feature or feature[0].startswith("track") or feature[0].startswith("#"):
+ with open(input_fname) as in_fh:
+ for line in in_fh:
+ feature = line.strip().split()
+ if not feature or feature[0].startswith("track") or feature[0].startswith("#"):
+ offset += len(line)
+ continue
+ chrom = feature[options.chrom_col]
+ chrom_start = int(feature[options.start_col])
+ chrom_end = int(feature[options.end_col])
+ index.add(chrom, chrom_start, chrom_end, offset)
offset += len(line)
- continue
- chrom = feature[options.chrom_col]
- chrom_start = int(feature[options.start_col])
- chrom_end = int(feature[options.end_col])
- index.add(chrom, chrom_start, chrom_end, offset)
- offset += len(line)
- index.write(open(output_fname, "w"))
+ with open(output_fname, 'wb') as out:
+ index.write(out)
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml
index 67a480fe4e7..9c8d1a949c4 100644
--- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_interval_index_converter.py'
-c ${input1.metadata.chromCol}
@@ -13,6 +16,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py
index faf988da48e..a4fec2bef4d 100644
--- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py
@@ -42,7 +42,7 @@ def to_tabix(bgzip_fname, out_fname, preset=None, chrom_col=None, start_col=None
start_col=(start_col - 1), end_col=(end_col - 1),
keep_original=True, index=out_fname, force=True)
if os.path.getsize(out_fname) == 0:
- sys.stderr.write("The converted tabix index file is empty, meaning the input data is invalid.")
+ sys.exit("The converted tabix index file is empty, meaning the input data is invalid.")
return bgzip_fname
diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml
index 1e5d9974524..d99ee11d89b 100644
--- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py'
@@ -14,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/len_to_linecount.xml b/lib/galaxy/datatypes/converters/len_to_linecount.xml
index 9f638bfbda7..db451e80bf8 100644
--- a/lib/galaxy/datatypes/converters/len_to_linecount.xml
+++ b/lib/galaxy/datatypes/converters/len_to_linecount.xml
@@ -1,6 +1,9 @@
-
+
+
+ gawk
+
'$output'
@@ -12,6 +15,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py
index 2bc185310bf..81be44df3c5 100644
--- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py
+++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py
@@ -42,13 +42,11 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
try:
mf = open(inmap, 'r')
except Exception:
- sys.stderr.write('%s cannot open inmap file %s - do you have permission?\n' % (prog, inmap))
- sys.exit(1)
+ sys.exit('%s cannot open inmap file %s - do you have permission?\n' % (prog, inmap))
try:
rsl = [x.split()[1] for x in mf]
except Exception:
- sys.stderr.write('## cannot parse %s' % inmap)
- sys.exit(1)
+ sys.exit('## cannot parse %s' % inmap)
try:
os.makedirs(outfilepath)
except Exception:
@@ -85,8 +83,7 @@ def main():
"""
nparm = 3
if len(sys.argv) < nparm:
- sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
- sys.exit(1)
+ sys.exit('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
inpedfilepath = sys.argv[1]
outhtmlname = sys.argv[2]
outfilepath = sys.argv[3]
diff --git a/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml b/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml
index 16d0cab9686..da15908c80a 100644
--- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml
+++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/lped_to_fped_converter.py' '$input1.extra_files_path/$input1.metadata.base_name' '$output1' '$output1.files_path'
diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
index 68fc19bc9c3..4201bc8fe22 100644
--- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
+++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
@@ -87,8 +87,7 @@ def main():
"""
nparm = 4
if len(sys.argv) < nparm:
- sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
- sys.exit(1)
+ sys.exit('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
inpedfilepath = sys.argv[1]
outhtmlname = sys.argv[2]
outfilepath = sys.argv[3]
diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml
index 40001ae9af8..fe434710368 100644
--- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml
+++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/lped_to_pbed_converter.py' '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' 'plink'
diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml
index 725d9789a47..824d8fc0f95 100644
--- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml
+++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/maf_to_fasta_converter.py' '$output1' '$input1'
@@ -7,6 +7,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml b/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml
index fc2a04b06f7..6405dfb097d 100644
--- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml
+++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/maf_to_interval_converter.py' '$output1' '$input1' '${input1.metadata.dbkey}'
@@ -7,6 +7,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml
index 74486fe30d4..3ae5c09c564 100644
--- a/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml
+++ b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ unzip
+
$output1
]]>
diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml
index dfe22d6ae12..c48372b818f 100644
--- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml
+++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/pbed_ldreduced_converter.py' '$input1.extra_files_path/$input1.metadata.base_name' '60' '55' '0.1' '$output1' '$output1.files_path' 'plink'
diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
index e61aba09556..f7ed7689c80 100644
--- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
+++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
@@ -53,8 +53,7 @@ def main():
"""
nparm = 4
if len(sys.argv) < nparm:
- sys.stderr.write('PBED to LPED converter called with %s - needs %d parameters \n' % (sys.argv, nparm))
- sys.exit(1)
+ sys.exit('PBED to LPED converter called with %s - needs %d parameters \n' % (sys.argv, nparm))
inpedfilepath = sys.argv[1]
outhtmlname = sys.argv[2]
outfilepath = sys.argv[3]
diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml
index 0b19e2f20e3..0a308fd9b04 100644
--- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml
+++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/pbed_to_lped_converter.py' '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' plink
diff --git a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml
index 99d31e7eb98..4f573473314 100644
--- a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml
+++ b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml
@@ -1,5 +1,8 @@
-
+
converter
+
+ python
+
python '$__tool_directory__/picard_interval_list_to_bed6_converter.py' '$input' '$output'
diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
index 02d781e1742..103e0c5f4ba 100644
--- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
@@ -22,14 +22,16 @@ def main():
# Do conversion.
index = Indexes()
offset = 0
- for line in open(input_fname, "r"):
- chrom, start = line.split()[0:2]
- # Pileup format is 1-based.
- start = int(start) - 1
- index.add(chrom, start, start + 1, offset)
- offset += len(line)
+ with open(input_fname) as in_fh:
+ for line in in_fh:
+ chrom, start = line.split()[0:2]
+ # Pileup format is 1-based.
+ start = int(start) - 1
+ index.add(chrom, start, start + 1, offset)
+ offset += len(line)
- index.write(open(output_fname, "w"))
+ with open(output_fname, 'wb') as out:
+ index.write(out)
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
index a500ed866ff..7b960385095 100644
--- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/pileup_to_interval_index_converter.py'
'$input' '$output'
@@ -10,6 +13,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml
index 2d1d9db87ed..6f0d114422f 100644
--- a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml
+++ b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml
@@ -1,5 +1,8 @@
-
+
converts 2 or 3 column sequence taxonomy file to a 2 column mothur taxonomy_outline format
+
+ python
+
python '$__tool_directory__/ref_to_seq_taxonomy_converter.py' '$input' '$output'
@@ -9,4 +12,4 @@
-
\ No newline at end of file
+
diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py
deleted file mode 100644
index 66962d13392..00000000000
--- a/lib/galaxy/datatypes/converters/sam_to_bam.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env python
-# Dan Blankenberg
-
-"""
-A wrapper script for converting SAM to BAM, with sorting.
-%prog input_filename.sam output_filename.bam
-"""
-import optparse
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-
-import packaging.version
-
-CHUNK_SIZE = 2 ** 20 # 1mb
-
-
-def cleanup_before_exit(tmp_dir):
- if tmp_dir and os.path.exists(tmp_dir):
- shutil.rmtree(tmp_dir)
-
-
-def cmd_exists(cmd):
- # http://stackoverflow.com/questions/5226958/which-equivalent-function-in-python
- for path in os.environ["PATH"].split(":"):
- if os.path.exists(os.path.join(path, cmd)):
- return True
- return False
-
-
-def _get_samtools_version():
- version = '0.0.0'
- if not cmd_exists('samtools'):
- raise Exception('This tool needs samtools, but it is not on PATH.')
- # Get the version of samtools via --version-only, if available
- p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- output, error = p.communicate()
- # --version-only is available
- # Format is +htslib-
- if p.returncode == 0:
- version = output.split('+')[0]
- return version
-
- output = subprocess.Popen(['samtools'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[1]
- lines = output.split('\n')
- for line in lines:
- if line.lower().startswith('version'):
- # Assuming line looks something like: version: 0.1.12a (r862)
- version = line.split()[1]
- break
- return version
-
-
-def __main__():
- # Parse Command Line
- parser = optparse.OptionParser()
- (options, args) = parser.parse_args()
-
- assert len(args) == 2, 'You must specify the input and output filenames'
- input_filename, output_filename = args
-
- tmp_dir = tempfile.mkdtemp(prefix='tmp-sam_to_bam_converter-')
-
- # convert to SAM
- unsorted_bam_filename = os.path.join(tmp_dir, 'unsorted.bam')
- unsorted_stderr_filename = os.path.join(tmp_dir, 'unsorted.stderr')
- proc = subprocess.Popen(['samtools', 'view', '-bS', input_filename],
- stdout=open(unsorted_bam_filename, 'wb'),
- stderr=open(unsorted_stderr_filename, 'wb'),
- cwd=tmp_dir)
- return_code = proc.wait()
- if return_code:
- stderr_target = sys.stderr
- else:
- stderr_target = sys.stdout
- with open(unsorted_stderr_filename) as stderr:
- while True:
- chunk = stderr.read(CHUNK_SIZE)
- if chunk:
- stderr_target.write(chunk)
- else:
- break
-
- # sort sam, so indexing will not fail
- sorted_stderr_filename = os.path.join(tmp_dir, 'sorted.stderr')
- sorting_prefix = os.path.join(tmp_dir, 'sorted_bam')
- # samtools changed sort command arguments (starting from version 1.3)
- samtools_version = packaging.version.parse(_get_samtools_version())
- if samtools_version < packaging.version.parse('1.0'):
- sort_args = ['-o', unsorted_bam_filename, sorting_prefix]
- else:
- sort_args = ['-T', sorting_prefix, unsorted_bam_filename]
- proc = subprocess.Popen(['samtools', 'sort'] + sort_args,
- stdout=open(output_filename, 'wb'),
- stderr=open(sorted_stderr_filename, 'wb'),
- cwd=tmp_dir)
- return_code = proc.wait()
-
- if return_code:
- stderr_target = sys.stderr
- else:
- stderr_target = sys.stdout
- with open(sorted_stderr_filename) as stderr:
- while True:
- chunk = stderr.read(CHUNK_SIZE)
- if chunk:
- stderr_target.write(chunk)
- else:
- break
-
- cleanup_before_exit(tmp_dir)
-
-
-if __name__ == "__main__":
- __main__()
diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.xml b/lib/galaxy/datatypes/converters/sam_to_bam.xml
deleted file mode 100644
index 9a32a1d988e..00000000000
--- a/lib/galaxy/datatypes/converters/sam_to_bam.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-
-
-
-
-
- samtools
-
- python '$__tool_directory__/sam_to_bam.py' '$input1' '$output'
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
index 434c44b6a12..94ab9981480 100644
--- a/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
@@ -20,6 +20,12 @@ samtools view -bh '$input' | bedtools genomecov -bg -split -ibam stdin
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml b/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml
index 2a99a793d27..46fa4af4ae7 100644
--- a/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml
+++ b/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml
@@ -17,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tabular_csv.py b/lib/galaxy/datatypes/converters/tabular_csv.py
index 0b18e1571e5..a323385037a 100644
--- a/lib/galaxy/datatypes/converters/tabular_csv.py
+++ b/lib/galaxy/datatypes/converters/tabular_csv.py
@@ -25,19 +25,17 @@ def main():
def convert_to_tsv(input_fname, output_fname):
- with open(input_fname, 'rb') as csvfile:
- with open(output_fname, 'wb') as ofh:
- reader = csv.reader(csvfile)
- for line in reader:
- ofh.write('\t'.join(line) + '\n')
+ with open(input_fname, newline="") as csvfile, open(output_fname, 'w') as ofh:
+ reader = csv.reader(csvfile)
+ for line in reader:
+ ofh.write('\t'.join(line) + '\n')
def convert_to_csv(input_fname, output_fname):
- with open(input_fname, 'rb') as tabfile:
- with open(output_fname, 'wb') as ofh:
- writer = csv.writer(ofh, delimiter=',')
- for line in tabfile.readlines():
- writer.writerow(line.strip().split('\t'))
+ with open(input_fname) as tabfile, open(output_fname, 'w', newline='') as ofh:
+ writer = csv.writer(ofh, delimiter=',')
+ for line in tabfile.readlines():
+ writer.writerow(line.strip().split('\t'))
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/tabular_to_csv.xml b/lib/galaxy/datatypes/converters/tabular_to_csv.xml
index 2c853109184..c449bc3614e 100644
--- a/lib/galaxy/datatypes/converters/tabular_to_csv.xml
+++ b/lib/galaxy/datatypes/converters/tabular_to_csv.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/tabular_csv.py' --from-tabular -i '$tabular' -o '$csv'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml
index d82911bc912..5d61c6119a4 100644
--- a/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml
+++ b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/tabular_to_dbnsfp.py' '$input' '$dbnsfp.extra_files_path/dbNSFP.gz'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tar_to_directory.xml b/lib/galaxy/datatypes/converters/tar_to_directory.xml
index 6865b86ee1d..ba4e21a81b7 100644
--- a/lib/galaxy/datatypes/converters/tar_to_directory.xml
+++ b/lib/galaxy/datatypes/converters/tar_to_directory.xml
@@ -1,5 +1,8 @@
-
+
+
+ galay-util
+
mkdir '$output1.files_path';
cd '$output1.files_path';
@@ -11,6 +14,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml b/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml
index 1e2889a921c..f47ee1efbff 100644
--- a/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml
+++ b/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml
@@ -17,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml b/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml
index ced46c0cc31..c2a4d8c53b7 100644
--- a/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml
+++ b/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml
@@ -18,6 +18,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tool_data_table_conf.xml.test b/lib/galaxy/datatypes/converters/tool_data_table_conf.xml.test
new file mode 100644
index 00000000000..d88d1d880ef
--- /dev/null
+++ b/lib/galaxy/datatypes/converters/tool_data_table_conf.xml.test
@@ -0,0 +1,7 @@
+
+
+
+ value, name, len_path
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml
index 64deb6df678..cd5b477bc37 100644
--- a/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P 'vcf' '' '$input1' '$output1'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml
index 79a09003382..be9d42400c3 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py' -P vcf '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py
index a635db88aab..8f5e24402af 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py
@@ -18,15 +18,17 @@ def main():
# Do conversion.
index = Indexes()
- reader = galaxy_utils.sequence.vcf.Reader(open(in_file))
- offset = reader.metadata_len
- for vcf_line in reader:
- # VCF format provides a chrom and 1-based position for each variant.
- # IntervalIndex expects 0-based coordinates.
- index.add(vcf_line.chrom, vcf_line.pos - 1, vcf_line.pos, offset)
- offset += len(vcf_line.raw_line)
+ with open(in_file) as in_fh:
+ reader = galaxy_utils.sequence.vcf.Reader(in_fh)
+ offset = reader.metadata_len
+ for vcf_line in reader:
+ # VCF format provides a chrom and 1-based position for each variant.
+ # IntervalIndex expects 0-based coordinates.
+ index.add(vcf_line.chrom, vcf_line.pos - 1, vcf_line.pos, offset)
+ offset += len(vcf_line.raw_line)
- index.write(open(out_file, "w"))
+ with open(out_file, "wb") as out_fh:
+ index.write(out_fh)
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml
index 03bf17d9709..9c2c96fa559 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml
@@ -1,5 +1,9 @@
-
+
__NOT_USED_CURRENTLY_FOR_CONVERTERS__
+
+ bx-python
+ galaxy_sequence_utils
+
python '$__tool_directory__/vcf_to_interval_index_converter.py' '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml
index 41394394572..984695d4b39 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P vcf '$input1' '$bgzip' '$output1'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml
index 6b015752ca9..4a78f175c11 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/vcf_to_vcf_bgzip.py' '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
index 862016e9151..4666b41ed99 100644
--- a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
@@ -15,6 +15,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py
deleted file mode 100644
index 6dc81144ea6..00000000000
--- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import division
-
-import sys
-
-from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict
-from bx.arrays.wiggle import WiggleReader
-
-BLOCK_SIZE = 100
-
-
-def main():
-
- input_fname = sys.argv[1]
- out_fname = sys.argv[2]
-
- reader = WiggleReader(open(input_fname))
-
- # Fill array from reader
- d = array_tree_dict_from_reader(reader, {}, block_size=BLOCK_SIZE)
-
- for array_tree in d.values():
- array_tree.root.build_summary()
-
- FileArrayTreeDict.dict_to_file(d, open(out_fname, "w"))
-
-
-if __name__ == "__main__":
- main()
diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.xml b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.xml
deleted file mode 100644
index 5ae0d98fd08..00000000000
--- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
- python '$__tool_directory__/wiggle_to_array_tree_converter.py' '$input' '$output'
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py
index ac965e105ae..b2c3229bc86 100644
--- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py
+++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.py
@@ -11,40 +11,20 @@ import sys
import bx.wiggle
-from galaxy.util import unicodify
from galaxy.util.ucsc import (
UCSCLimitException,
UCSCOutWrapper
)
-def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit(1)
-
-
def main():
- if len(sys.argv) > 1:
- in_file = open(sys.argv[1])
- else:
- in_file = open(sys.stdin)
-
- if len(sys.argv) > 2:
- out_file = open(sys.argv[2], "w")
- else:
- out_file = sys.stdout
-
- try:
- for fields in bx.wiggle.IntervalReader(UCSCOutWrapper(in_file)):
- out_file.write("%s\n" % "\t".join(map(str, fields)))
- except UCSCLimitException:
- # Wiggle data was truncated, at the very least need to warn the user.
- print('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.')
- except ValueError as e:
- stop_err(unicodify(e))
- finally:
- in_file.close()
- out_file.close()
+ with open(sys.argv[1]) as in_file, open(sys.argv[2], "w") as out_file:
+ try:
+ for fields in bx.wiggle.IntervalReader(UCSCOutWrapper(in_file)):
+ out_file.write("%s\n" % "\t".join(map(str, fields)))
+ except UCSCLimitException:
+ # Wiggle data was truncated, at the very least need to warn the user.
+ sys.stderr.write('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.')
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml
index 29c494189ad..0ed14968083 100644
--- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml
+++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml
@@ -1,6 +1,10 @@
-
+
+
+ galaxy-util
+ bx-python
+
python '$__tool_directory__/wiggle_to_simple_converter.py' '$input' '$out_file1'
@@ -8,4 +12,10 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py
index 11f2fb2f27b..67c01e39f57 100644
--- a/lib/galaxy/datatypes/registry.py
+++ b/lib/galaxy/datatypes/registry.py
@@ -46,6 +46,7 @@ class Registry(object):
self.datatype_converters = OrderedDict()
# Converters defined in local datatypes_conf.xml
self.converters = []
+ self.converter_tools = set()
# Converters defined in datatypes_conf.xml included in installed tool shed repositories.
self.proprietary_converters = []
self.converter_deps = {}
@@ -614,6 +615,7 @@ class Registry(object):
try:
config_path = os.path.join(converter_path, tool_config)
converter = toolbox.load_tool(config_path, use_cached=use_cached)
+ self.converter_tools.add(converter)
if installed_repository_dict:
# If the converter is included in an installed tool shed repository, set the tool
# shed related tool attributes.
diff --git a/lib/galaxy/datatypes/util/maf_utilities.py b/lib/galaxy/datatypes/util/maf_utilities.py
index 801d7a4d623..2600f0adc99 100644
--- a/lib/galaxy/datatypes/util/maf_utilities.py
+++ b/lib/galaxy/datatypes/util/maf_utilities.py
@@ -56,8 +56,7 @@ def get_species_in_block(block):
def tool_fail(msg="Unknown Error"):
- print("Fatal Error: %s" % msg, file=sys.stderr)
- sys.exit()
+ sys.exit("Fatal Error: %s" % msg)
class TempFileHandler(object):
@@ -101,11 +100,11 @@ class TempFileHandler(object):
else:
raise e
tmp_file.close()
- self.files.append(open(filename, 'w+b'))
+ self.files.append(open(filename, 'w'))
else:
while True:
try:
- self.files[index] = open(self.files[index].name, 'r+b')
+ self.files[index] = open(self.files[index].name, 'r')
break
except OSError as e:
if self.open_file_indexes and e.errno == EMFILE:
diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index 20c30ec9aee..aa9834a996e 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -28,7 +28,7 @@ boltons==19.3.0
boto3==1.9.114
boto==2.49.0
botocore==1.12.253
-bx-python==0.8.6
+bx-python==0.8.8
bz2file==0.98 ; python_version < '3.3'
cachecontrol==0.11.7
cachetools==3.1.1
diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py
index d8fb528ee0b..6699b80432d 100644
--- a/lib/galaxy/managers/context.py
+++ b/lib/galaxy/managers/context.py
@@ -180,12 +180,24 @@ class ProvidesHistoryContext(object):
# The API presents a Bunch for a history. Until the API is
# more fully featured for handling this, also return None.
return None
- datasets = self.sa_session.query(self.app.model.HistoryDatasetAssociation) \
- .filter_by(deleted=False, history_id=self.history.id, extension="len")
+ non_ready_or_ok = set(self.app.model.Dataset.non_ready_states)
+ non_ready_or_ok.add(self.app.model.HistoryDatasetAssociation.states.OK)
+ datasets = self.sa_session.query(
+ self.app.model.HistoryDatasetAssociation
+ ).filter_by(
+ deleted=False,
+ history_id=self.history.id,
+ extension="len"
+ ).filter(
+ self.app.model.HistoryDatasetAssociation._state.in_(non_ready_or_ok),
+ )
+ valid_ds = None
for ds in datasets:
- if dbkey == ds.dbkey:
- return ds
- return None
+ if ds.dbkey == dbkey:
+ if ds.state == self.app.model.HistoryDatasetAssociation.states.OK:
+ return ds
+ valid_ds = ds
+ return valid_ds
@property
def db_builds(self):
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index ff464135a9f..6af6043e374 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -127,13 +127,9 @@ GALAXY_LIB_TOOLS_UNVERSIONED = [
"send_to_cloud",
"__DATA_FETCH__",
# Legacy tools bundled with Galaxy.
- "vcf_to_maf_customtrack1",
"laj_1",
- "secure_hash_message_digest",
- "join1",
"gff2bed1",
"gff_filter_by_feature_count",
- "aggregate_scores_in_intervals2",
"Interval_Maf_Merged_Fasta2",
"GeneBed_Maf_Fasta2",
"maf_stats1",
@@ -146,16 +142,12 @@ GALAXY_LIB_TOOLS_UNVERSIONED = [
"MAF_split_blocks_by_species1",
"MAF_Limit_To_Species1",
"maf_by_block_number1",
- "wiggle2simple1",
# Converters
"CONVERTER_bed_to_fli_0",
- "CONVERTER_fastq_to_fqtoc0",
"CONVERTER_gff_to_fli_0",
"CONVERTER_gff_to_interval_index_0",
"CONVERTER_maf_to_fasta_0",
"CONVERTER_maf_to_interval_0",
- "CONVERTER_wiggle_to_interval_0",
- "CONVERTER_tar_to_directory",
# Tools improperly migrated to the tool shed (devteam)
"qualityFilter",
"winSplitter",
@@ -189,6 +181,14 @@ GALAXY_LIB_TOOLS_VERSIONED = {
"PEsortedSAM2readprofile": packaging.version.parse("1.1.1"),
"sam_to_bam": packaging.version.parse("1.1.3"),
"sam_pileup": packaging.version.parse("1.1.3"),
+ "vcf_to_maf_customtrack1": packaging.version.parse("1.0.1"),
+ "secure_hash_message_digest": packaging.version.parse("0.0.2"),
+ "join1": packaging.version.parse("2.1.3"),
+ "wiggle2simple1": packaging.version.parse("1.0.1"),
+ "CONVERTER_wiggle_to_interval_0": packaging.version.parse("1.0.1"),
+ "aggregate_scores_in_intervals2": packaging.version.parse("1.1.4"),
+ "CONVERTER_fastq_to_fqtoc0": packaging.version.parse("1.0.1"),
+ "CONVERTER_tar_to_directory": packaging.version.parse("1.0.1"),
}
@@ -537,6 +537,10 @@ class Tool(Dictifiable):
tool_versions = self.tool_versions
return not tool_versions or self.version == self.tool_versions[-1]
+ @property
+ def is_datatype_converter(self):
+ return self in self.app.datatypes_registry.converter_tools
+
@property
def tool_shed_repository(self):
# If this tool is included in an installed tool shed repository, return it.
diff --git a/lib/galaxy/tools/cache.py b/lib/galaxy/tools/cache.py
index 768d1b1ab07..36ce418ab75 100644
--- a/lib/galaxy/tools/cache.py
+++ b/lib/galaxy/tools/cache.py
@@ -149,16 +149,20 @@ class ToolShedRepositoryCache(object):
self.repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
def rebuild(self):
- self.repositories = self.app.install_model.context.current.query(self.app.install_model.ToolShedRepository).options(
- defer(self.app.install_model.ToolShedRepository.metadata),
- joinedload('tool_dependencies').subqueryload('tool_shed_repository').options(
- defer(self.app.install_model.ToolShedRepository.metadata)
- ),
- ).all()
- repos_by_tuple = defaultdict(list)
- for repository in self.repositories + self.local_repositories:
- repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
- self.repos_by_tuple = repos_by_tuple
+ try:
+ session = self.app.install_model.context.current.session_factory()
+ self.repositories = session.query(self.app.install_model.ToolShedRepository).options(
+ defer(self.app.install_model.ToolShedRepository.metadata),
+ joinedload('tool_dependencies').subqueryload('tool_shed_repository').options(
+ defer(self.app.install_model.ToolShedRepository.metadata)
+ ),
+ ).all()
+ repos_by_tuple = defaultdict(list)
+ for repository in self.repositories + self.local_repositories:
+ repos_by_tuple[(repository.tool_shed, repository.owner, repository.name)].append(repository)
+ self.repos_by_tuple = repos_by_tuple
+ finally:
+ session.close()
def get_installed_repository(self, tool_shed=None, name=None, owner=None, installed_changeset_revision=None, changeset_revision=None, repository_id=None):
if repository_id:
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index ad4f918e0ba..90208e09da8 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -172,15 +172,16 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
"""
test_counts_by_tool = {}
for id, tool in self.app.toolbox.tools():
- tests = tool.tests
- if tests:
- if tool.id not in test_counts_by_tool:
- test_counts_by_tool[tool.id] = {}
- available_versions = test_counts_by_tool[tool.id]
- available_versions[tool.version] = {
- "tool_name": tool.name,
- "count": len(tests),
- }
+ if not tool.is_datatype_converter:
+ tests = tool.tests
+ if tests:
+ if tool.id not in test_counts_by_tool:
+ test_counts_by_tool[tool.id] = {}
+ available_versions = test_counts_by_tool[tool.id]
+ available_versions[tool.version] = {
+ "tool_name": tool.name,
+ "count": len(tests),
+ }
return test_counts_by_tool
@expose_api_raw_anonymous_and_sessionless
diff --git a/scripts/get_uwsgi_args.py b/scripts/get_uwsgi_args.py
index 3206711ee66..61e804990c5 100644
--- a/scripts/get_uwsgi_args.py
+++ b/scripts/get_uwsgi_args.py
@@ -23,7 +23,7 @@ ALIASES = {
'module': ('mount',), # mount is not actually an alias for module, but we don't want to set module if mount is set
}
DEFAULT_ARGS = {
- '_all_': ('pythonpath', 'threads', 'buffer-size', 'http', 'static-map', 'static-safe', 'die-on-term', 'hook-master-start', 'enable-threads', 'umask'),
+ '_all_': ('pythonpath', 'threads', 'buffer-size', 'http', 'static-map', 'die-on-term', 'hook-master-start', 'enable-threads', 'umask'),
'galaxy': ('py-call-osafterfork',),
'reports': (),
'tool_shed': ('cron',),
@@ -93,7 +93,6 @@ def _get_uwsgi_args(cliargs, kwargs):
'http': 'localhost:{port}'.format(port=DEFAULT_PORTS[cliargs.app]),
'static-map': ('/static={here}/static'.format(here=os.getcwd()),
'/favicon.ico={here}/static/favicon.ico'.format(here=os.getcwd())),
- 'static-safe': ('{here}/client/galaxy/images'.format(here=os.getcwd())),
'die-on-term': True,
'enable-threads': True,
'hook-master-start': ('unix_signal:2 gracefully_kill_them_all',
@@ -120,6 +119,8 @@ def _get_uwsgi_args(cliargs, kwargs):
if hmr_server.lower() in ['1', 'true', 'default']:
hmr_server = "http:127.0.0.1:8081"
__add_arg(args, 'route', '^/static/dist/ {hmr_server}'.format(hmr_server=hmr_server))
+ # We always want to append client/galaxy/images as static-safe.
+ __add_arg(args, 'static-safe', '{here}/client/galaxy/images'.format(here=os.getcwd()))
for arg in DEFAULT_ARGS['_all_'] + DEFAULT_ARGS[cliargs.app]:
if not __arg_set(arg, uwsgi_kwargs):
diff --git a/test/unit/tools/conftest.py b/test/unit/tools/conftest.py
index a7de3f768bc..ab8d702131e 100644
--- a/test/unit/tools/conftest.py
+++ b/test/unit/tools/conftest.py
@@ -65,4 +65,13 @@ def create_repo(app, changeset, installed_changeset, config_filename=None):
repository.deleted = False
repository.uninstalled = False
app.install_model.context.add(repository)
+ app.install_model.context.flush()
+ tool_dependency = tool_shed_install.ToolDependency(
+ name='Name',
+ version='100',
+ type='package',
+ status='ok',
+ tool_shed_repository_id=repository.id,
+ )
+ app.install_model.context.add(tool_dependency)
return repository
diff --git a/test/unit/tools/test_tool_shed_repository_cache.py b/test/unit/tools/test_tool_shed_repository_cache.py
index 383d8351191..ae99298292e 100644
--- a/test/unit/tools/test_tool_shed_repository_cache.py
+++ b/test/unit/tools/test_tool_shed_repository_cache.py
@@ -1,4 +1,5 @@
import pytest
+from sqlalchemy.orm.exc import DetachedInstanceError
from .conftest import create_repo
@@ -54,3 +55,20 @@ def test_get_installed_repository(tool_shed_repository_cache, repos, tool_conf_r
assert repo
else:
assert repo is None
+
+
+def test_repo_cache_expunge(tool_shed_repository_cache, repos):
+ tool_shed_repository_cache.rebuild()
+ assert len(tool_shed_repository_cache.repositories) == 10
+ # Modify and commit a repo, will expire in memory attributes of orm objects (unless using a different session)
+ repo = tool_shed_repository_cache.repositories[0]
+ repo.name = 'new name'
+ tool_shed_repository_cache.app.install_model.session.flush()
+ # remove session, should demonstrate the separate session is in use
+ tool_shed_repository_cache.app.install_model.session.remove()
+ assert repo.changeset_revision == "1"
+ assert repo.tool_dependencies[0].name == 'Name'
+ with pytest.raises(DetachedInstanceError):
+ # Make sure this still raises DetachedInstanceError,
+ # keeping this in memory would be expensive
+ repo.metadata
diff --git a/tools/filters/bed_to_gff_converter.py b/tools/filters/bed_to_gff_converter.py
index 768891fdcf8..fb7e0bd258f 100644
--- a/tools/filters/bed_to_gff_converter.py
+++ b/tools/filters/bed_to_gff_converter.py
@@ -12,60 +12,59 @@ def __main__():
output_name = sys.argv[2]
skipped_lines = 0
first_skipped_line = 0
- out = open(output_name, 'w')
i = 0
- for i, line in enumerate(open(input_name)):
- complete_bed = False
- line = line.rstrip('\r\n')
- if line and not line.startswith('#') and not line.startswith('track') and not line.startswith('browser'):
- try:
- elems = line.split('\t')
- if len(elems) == 12:
- complete_bed = True
- chrom = elems[0]
- if complete_bed:
- feature = "mRNA"
- else:
+ with open(output_name, 'w') as out, open(input_name) as fh_in:
+ for i, line in enumerate(fh_in):
+ complete_bed = False
+ line = line.rstrip('\r\n')
+ if line and not line.startswith('#') and not line.startswith('track') and not line.startswith('browser'):
+ try:
+ elems = line.split('\t')
+ if len(elems) == 12:
+ complete_bed = True
+ chrom = elems[0]
+ if complete_bed:
+ feature = "mRNA"
+ else:
+ try:
+ feature = elems[3]
+ except Exception:
+ feature = 'feature%d' % (i + 1)
+ start = int(elems[1]) + 1
+ end = int(elems[2])
try:
- feature = elems[3]
+ score = elems[4]
except Exception:
- feature = 'feature%d' % (i + 1)
- start = int(elems[1]) + 1
- end = int(elems[2])
- try:
- score = elems[4]
+ score = '0'
+ try:
+ strand = elems[5]
+ except Exception:
+ strand = '+'
+ try:
+ group = elems[3]
+ except Exception:
+ group = 'group%d' % (i + 1)
+ if complete_bed:
+ out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n' % (chrom, feature, start, end, score, strand, feature, group))
+ else:
+ out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n' % (chrom, feature, start, end, score, strand, group))
+ if complete_bed:
+ # We have all the info necessary to annotate exons for genes and mRNAs
+ block_count = int(elems[9])
+ block_sizes = elems[10].split(',')
+ block_starts = elems[11].split(',')
+ for j in range(block_count):
+ exon_start = int(start) + int(block_starts[j])
+ exon_end = exon_start + int(block_sizes[j]) - 1
+ out.write('%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n' % (chrom, exon_start, exon_end, score, strand, group))
except Exception:
- score = '0'
- try:
- strand = elems[5]
- except Exception:
- strand = '+'
- try:
- group = elems[3]
- except Exception:
- group = 'group%d' % (i + 1)
- if complete_bed:
- out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n' % (chrom, feature, start, end, score, strand, feature, group))
- else:
- out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n' % (chrom, feature, start, end, score, strand, group))
- if complete_bed:
- # We have all the info necessary to annotate exons for genes and mRNAs
- block_count = int(elems[9])
- block_sizes = elems[10].split(',')
- block_starts = elems[11].split(',')
- for j in range(block_count):
- exon_start = int(start) + int(block_starts[j])
- exon_end = exon_start + int(block_sizes[j]) - 1
- out.write('%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n' % (chrom, exon_start, exon_end, score, strand, group))
- except Exception:
+ skipped_lines += 1
+ if not first_skipped_line:
+ first_skipped_line = i + 1
+ else:
skipped_lines += 1
if not first_skipped_line:
first_skipped_line = i + 1
- else:
- skipped_lines += 1
- if not first_skipped_line:
- first_skipped_line = i + 1
- out.close()
info_msg = "%i lines converted to GFF version 2. " % (i + 1 - skipped_lines)
if skipped_lines > 0:
info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line)
diff --git a/tools/filters/gff_to_bed_converter.py b/tools/filters/gff_to_bed_converter.py
index 1326188dd9b..929347ce959 100644
--- a/tools/filters/gff_to_bed_converter.py
+++ b/tools/filters/gff_to_bed_converter.py
@@ -55,29 +55,50 @@ def __main__():
output_name = sys.argv[2]
skipped_lines = 0
first_skipped_line = 0
- out = open(output_name, 'w')
i = 0
cur_transcript_chrome = None
cur_transcript_id = None
cur_transcript_strand = None
cur_transcripts_blocks = [] # (start, end) for each block.
- for i, line in enumerate(open(input_name)):
- line = line.rstrip('\r\n')
- if line and not line.startswith('#'):
- try:
- # GFF format: chrom source, name, chromStart, chromEnd, score, strand, attributes
- elems = line.split('\t')
- start = str(int(elems[3]) - 1)
- coords = [int(start), int(elems[4])]
- strand = elems[6]
- if strand not in ['+', '-']:
- strand = '+'
- attributes = parse_gff_attributes(elems[8])
- t_id = attributes.get("transcript_id", None)
+ with open(output_name, 'w') as out, open(input_name) as in_fh:
+ for i, line in enumerate(in_fh):
+ line = line.rstrip('\r\n')
+ if line and not line.startswith('#'):
+ try:
+ # GFF format: chrom source, name, chromStart, chromEnd, score, strand, attributes
+ elems = line.split('\t')
+ start = str(int(elems[3]) - 1)
+ coords = [int(start), int(elems[4])]
+ strand = elems[6]
+ if strand not in ['+', '-']:
+ strand = '+'
+ attributes = parse_gff_attributes(elems[8])
+ t_id = attributes.get("transcript_id", None)
+
+ if not t_id:
+ #
+ # No transcript ID, so write last transcript and write current line as its own line.
+ #
+
+ # Write previous transcript.
+ if cur_transcript_id:
+ # Write BED entry.
+ out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
+
+ # Replace any spaces in the name with underscores so UCSC will not complain.
+ name = elems[2].replace(" ", "_")
+ out.write(get_bed_line(elems[0], name, strand, [coords]))
+ continue
+
+ # There is a transcript ID, so process line at transcript level.
+ if t_id == cur_transcript_id:
+ # Line is element of transcript and will be a block in the BED entry.
+ cur_transcripts_blocks.append(coords)
+ continue
- if not t_id:
#
- # No transcript ID, so write last transcript and write current line as its own line.
+ # Line is part of new transcript; write previous transcript and start
+ # new transcript.
#
# Write previous transcript.
@@ -85,47 +106,25 @@ def __main__():
# Write BED entry.
out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
- # Replace any spaces in the name with underscores so UCSC will not complain.
- name = elems[2].replace(" ", "_")
- out.write(get_bed_line(elems[0], name, strand, [coords]))
- continue
-
- # There is a transcript ID, so process line at transcript level.
- if t_id == cur_transcript_id:
- # Line is element of transcript and will be a block in the BED entry.
+ # Start new transcript.
+ cur_transcript_chrome = elems[0]
+ cur_transcript_id = t_id
+ cur_transcript_strand = strand
+ cur_transcripts_blocks = []
cur_transcripts_blocks.append(coords)
- continue
-
- #
- # Line is part of new transcript; write previous transcript and start
- # new transcript.
- #
-
- # Write previous transcript.
- if cur_transcript_id:
- # Write BED entry.
- out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
-
- # Start new transcript.
- cur_transcript_chrome = elems[0]
- cur_transcript_id = t_id
- cur_transcript_strand = strand
- cur_transcripts_blocks = []
- cur_transcripts_blocks.append(coords)
- except Exception:
+ except Exception:
+ skipped_lines += 1
+ if not first_skipped_line:
+ first_skipped_line = i + 1
+ else:
skipped_lines += 1
if not first_skipped_line:
first_skipped_line = i + 1
- else:
- skipped_lines += 1
- if not first_skipped_line:
- first_skipped_line = i + 1
- # Write last transcript.
- if cur_transcript_id:
- # Write BED entry.
- out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
- out.close()
+ # Write last transcript.
+ if cur_transcript_id:
+ # Write BED entry.
+ out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
info_msg = "%i lines converted to BED. " % (i + 1 - skipped_lines)
if skipped_lines > 0:
info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line)
diff --git a/tools/filters/joiner.xml b/tools/filters/joiner.xml
index 642d1e8b429..a1fc30f18bf 100644
--- a/tools/filters/joiner.xml
+++ b/tools/filters/joiner.xml
@@ -1,15 +1,15 @@
-
+
side by side on a specified field
- python
+ galaxy-util
python '$__tool_directory__/join.py' '$input1' '$input2' $field1 $field2 '$out_file1' $unmatched $partial --index_depth=3 --buffer=50000000 --fill_options_file=$fill_options_file $header
- <%
+
#set $__fill_options = {}
#if $fill_empty_columns['fill_empty_columns_switch'] == 'fill_empty':
#set $__fill_options['fill_unjoined_only'] = $fill_empty_columns['fill_columns_by'].value == 'fill_unjoined_only'
@@ -30,7 +30,7 @@ import json
#end if
#end if
${json.dumps( __fill_options )}
-
+ ]]>
@@ -174,7 +174,7 @@ ${json.dumps( __fill_options )}
-
+ Convert*
-----
@@ -221,6 +221,6 @@ Joining the 4th column of Dataset1 with the 1st column of Dataset2, while keepin
chr1 50 80 geneB geneB Foxp2
chr5 10 40 geneL
-
+]]>
diff --git a/tools/filters/secure_hash_message_digest.xml b/tools/filters/secure_hash_message_digest.xml
index e20ce80969a..50554c7181d 100644
--- a/tools/filters/secure_hash_message_digest.xml
+++ b/tools/filters/secure_hash_message_digest.xml
@@ -1,6 +1,10 @@
-
+
on a dataset
- secure_hash_message_digest.py --input "${input1}" --output "${out_file1}"
+
+ python
+
+
+ python '$__tool_directory__/secure_hash_message_digest.py' --input '${input1}' --output '${out_file1}'
#if $algorithms.value:
#for $algorithm in str( $algorithms ).split( "," ):
--algorithm "${algorithm}"
diff --git a/tools/filters/wiggle_to_simple.py b/tools/filters/wiggle_to_simple.py
index f1fcf045c8f..d619a8dc3f7 100755
--- a/tools/filters/wiggle_to_simple.py
+++ b/tools/filters/wiggle_to_simple.py
@@ -10,38 +10,20 @@ import sys
import bx.wiggle
-from galaxy.util.ucsc import UCSCLimitException, UCSCOutWrapper
-
-
-def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit()
+from galaxy.util.ucsc import (
+ UCSCLimitException,
+ UCSCOutWrapper
+)
def main():
- if len(sys.argv) > 1:
- in_file = open(sys.argv[1])
- else:
- in_file = open(sys.stdin)
-
- if len(sys.argv) > 2:
- out_file = open(sys.argv[2], "w")
- else:
- out_file = sys.stdout
-
- try:
- for fields in bx.wiggle.IntervalReader(UCSCOutWrapper(in_file)):
- out_file.write("%s\n" % "\t".join(map(str, fields)))
- except UCSCLimitException:
- # Wiggle data was truncated, at the very least need to warn the user.
- print('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.')
- except ValueError as e:
- in_file.close()
- out_file.close()
- stop_err(str(e))
-
- in_file.close()
- out_file.close()
+ with open(sys.argv[1]) as in_file, open(sys.argv[2], "w") as out_file:
+ try:
+ for fields in bx.wiggle.IntervalReader(UCSCOutWrapper(in_file)):
+ out_file.write("%s\n" % "\t".join(map(str, fields)))
+ except UCSCLimitException:
+ # Wiggle data was truncated, at the very least need to warn the user.
+ sys.stderr.write('Encountered message from UCSC: "Reached output limit of 100000 data values", so be aware your data was truncated.')
if __name__ == "__main__":
diff --git a/tools/filters/wiggle_to_simple.xml b/tools/filters/wiggle_to_simple.xml
index 991ce655148..b4362369cae 100644
--- a/tools/filters/wiggle_to_simple.xml
+++ b/tools/filters/wiggle_to_simple.xml
@@ -1,6 +1,10 @@
-
+
converter
- wiggle_to_simple.py $input $out_file1
+
+ galaxy-util
+ bx-python
+
+ python '$__tool_directory__/wiggle_to_simple.py' '$input' '$out_file1'
diff --git a/tools/maf/interval_maf_to_merged_fasta.py b/tools/maf/interval_maf_to_merged_fasta.py
index 54ad28a4156..6df4bc88cdf 100644
--- a/tools/maf/interval_maf_to_merged_fasta.py
+++ b/tools/maf/interval_maf_to_merged_fasta.py
@@ -35,8 +35,7 @@ from galaxy.tools.util import maf_utilities
def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit()
+ sys.exit(msg)
def __main__():
diff --git a/tools/maf/vcf_to_maf_customtrack.py b/tools/maf/vcf_to_maf_customtrack.py
index 5c7a2bce376..5b89907c5e1 100644
--- a/tools/maf/vcf_to_maf_customtrack.py
+++ b/tools/maf/vcf_to_maf_customtrack.py
@@ -13,51 +13,43 @@ UNKNOWN_NUCLEOTIDE = '*'
class PopulationVCFParser(Iterator):
def __init__(self, reader, name):
- self.reader = reader
+ self.reader = iter(reader)
self.name = name
self.counter = 0
- def __next__(self):
- rval = []
- vc = next(self.reader)
- for i, allele in enumerate(vc.alt):
- rval.append(('%s_%i.%i' % (self.name, i + 1, self.counter + 1), allele))
- self.counter += 1
- return (vc, rval)
-
def __iter__(self):
- while True:
- yield next(self)
+ for vc in self.reader:
+ rval = []
+ for i, allele in enumerate(vc.alt):
+ rval.append(('%s_%i.%i' % (self.name, i + 1, self.counter + 1), allele))
+ self.counter += 1
+ yield (vc, rval)
class SampleVCFParser(Iterator):
def __init__(self, reader):
- self.reader = reader
+ self.reader = iter(reader)
self.counter = 0
- def __next__(self):
- rval = []
- vc = next(self.reader)
- alleles = [vc.ref] + vc.alt
-
- if 'GT' in vc.format:
- gt_index = vc.format.index('GT')
- for sample_name, sample_value in zip(vc.sample_names, vc.sample_values):
- gt_indexes = []
- for i in sample_value[gt_index].replace('|', '/').replace('\\', '/').split('/'): # Do we need to consider phase here?
- try:
- gt_indexes.append(int(i))
- except Exception:
- gt_indexes.append(None)
- for i, allele_i in enumerate(gt_indexes):
- if allele_i is not None:
- rval.append(('%s_%i.%i' % (sample_name, i + 1, self.counter + 1), alleles[allele_i]))
- self.counter += 1
- return (vc, rval)
-
def __iter__(self):
- while True:
- yield next(self)
+ for vc in self.reader:
+ rval = []
+ alleles = [vc.ref] + vc.alt
+
+ if 'GT' in vc.format:
+ gt_index = vc.format.index('GT')
+ for sample_name, sample_value in zip(vc.sample_names, vc.sample_values):
+ gt_indexes = []
+ for i in sample_value[gt_index].replace('|', '/').replace('\\', '/').split('/'): # Do we need to consider phase here?
+ try:
+ gt_indexes.append(int(i))
+ except Exception:
+ gt_indexes.append(None)
+ for i, allele_i in enumerate(gt_indexes):
+ if allele_i is not None:
+ rval.append(('%s_%i.%i' % (sample_name, i + 1, self.counter + 1), alleles[allele_i]))
+ self.counter += 1
+ yield (vc, rval)
def main():
@@ -77,7 +69,7 @@ def main():
if not (options.population ^ options.sample):
parser.error('You must specify either a per population conversion or a per sample conversion, but not both')
- out = open(args.pop(0), 'wb')
+ out = open(args.pop(0), 'w')
out.write('track name="%s" visibility=pack\n' % options.name.replace("\"", "'"))
maf_writer = bx.align.maf.Writer(out)
diff --git a/tools/maf/vcf_to_maf_customtrack.xml b/tools/maf/vcf_to_maf_customtrack.xml
index c10618ac41e..39b97ebe761 100644
--- a/tools/maf/vcf_to_maf_customtrack.xml
+++ b/tools/maf/vcf_to_maf_customtrack.xml
@@ -1,5 +1,9 @@
-
+
for display at UCSC
+
+ galaxy_sequence_utils
+ bx-python
+
macros.xml
@@ -43,21 +47,21 @@ ${vcf_source_type.vcf_source} -n '$track_name'
-
+
**What it does**
diff --git a/tools/stats/aggregate_binned_scores_in_intervals.xml b/tools/stats/aggregate_binned_scores_in_intervals.xml
index 7e204c8ef67..fa8e1d111d8 100644
--- a/tools/stats/aggregate_binned_scores_in_intervals.xml
+++ b/tools/stats/aggregate_binned_scores_in_intervals.xml
@@ -1,5 +1,9 @@
-
+
Appends the average, min, max of datapoints per interval
+
+ galaxy-util
+ bx-python
+
python '$__tool_directory__/aggregate_scores_in_intervals.py'
#if $score_source_type.score_source == "user"
diff --git a/tools/stats/aggregate_scores_in_intervals.py b/tools/stats/aggregate_scores_in_intervals.py
index 9c0cceba15b..2c5e8d7ad45 100755
--- a/tools/stats/aggregate_scores_in_intervals.py
+++ b/tools/stats/aggregate_scores_in_intervals.py
@@ -99,8 +99,7 @@ class FileBinnedArrayDir(Mapping):
def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit()
+ sys.exit(msg)
def load_scores_wiggle(fname, chrom_buffer_size=3):