diff --git a/.github/workflows/converter_tests.yaml b/.github/workflows/converter_tests.yaml
new file mode 100644
index 00000000000..a8cd3f3f7a4
--- /dev/null
+++ b/.github/workflows/converter_tests.yaml
@@ -0,0 +1,29 @@
+name: Converter tests
+on: [push, pull_request]
+jobs:
+
+ test:
+ name: Test
+ runs-on: ubuntu-18.04
+ strategy:
+ matrix:
+ python-version: [3.7]
+ steps:
+ - uses: actions/checkout@v1
+ with:
+ fetch-depth: 1
+ - uses: actions/setup-python@v1
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Cache venv dir
+ uses: actions/cache@v1
+ id: pip-cache
+ with:
+ path: ~/.cache/pip
+ key: pip-cache-${{ matrix.python-version }}-${{ hashFiles('requirements.txt') }}
+ - name: fetch test data
+ run: git clone https://github.com/galaxyproject/galaxy-test-data && cp -R galaxy-test-data/* test-data
+ - name: Install planemo
+ run: pip install planemo
+ - name: Run tests
+ run: 'planemo test --galaxy_python_version ${{ matrix.python-version }} --galaxy_root . lib/galaxy/datatypes/converters/*xml'
diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample
index a085297fb9f..75bff794326 100644
--- a/lib/galaxy/config/sample/datatypes_conf.xml.sample
+++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample
@@ -142,6 +142,7 @@
+
diff --git a/lib/galaxy/datatypes/converters/bam_to_bai.xml b/lib/galaxy/datatypes/converters/bam_to_bai.xml
index 1587ff83cab..65ace10fc93 100644
--- a/lib/galaxy/datatypes/converters/bam_to_bai.xml
+++ b/lib/galaxy/datatypes/converters/bam_to_bai.xml
@@ -1,14 +1,20 @@
-
+
- samtools
+ samtools
- samtools index '$input1' '$output1'
+ samtools index -@ \${GALAXY_SLOTS:-1} '$input1' '$output1'
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
index 7f8d8bcfeb3..53abc0a9c5c 100644
--- a/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/bam_to_bigwig_converter.xml
@@ -1,8 +1,9 @@
- ucsc-bedgraphtobigwig
- bedtools
+ ucsc-bedgraphtobigwig
+ bedtools
+ coreutils
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml b/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml
index 83bf34f11f7..b3b79270ef6 100644
--- a/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml
+++ b/lib/galaxy/datatypes/converters/bcf_to_bcf_uncompressed_converter.xml
@@ -12,6 +12,12 @@ bcftools view -o '$output1' -O u '$input1'
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml b/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml
index faf09918f64..a7f5b48813f 100644
--- a/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml
+++ b/lib/galaxy/datatypes/converters/bcf_uncompressed_to_bcf_converter.xml
@@ -12,6 +12,12 @@ bcftools view -o '$output1' -O b '$input1'
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
index cd64113d236..2b2d7ab6f53 100644
--- a/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_gff_or_vcf_to_bigwig_converter.xml
@@ -38,6 +38,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml
index 08d91cddf72..270b98bcf16 100644
--- a/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py' -P bed '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml b/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml
index 2c3c55541ac..eb7ac28ea03 100644
--- a/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_fli_converter.xml
@@ -8,6 +8,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml b/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml
index 26a4eeb2f2d..79ed48a9ef6 100644
--- a/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_gff_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/bed_to_gff_converter.py' '$input1' '$output1'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml
index bb171c0cd0b..98618ef4c68 100644
--- a/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_interval_index_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_interval_index_converter.py' '$input1' '$output1'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml
index 72c2b908de5..cfedf148a3b 100644
--- a/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/bed_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P bed '$input1' '$bgzip' '$output1'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py
deleted file mode 100644
index 096b6a01254..00000000000
--- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.py
+++ /dev/null
@@ -1,57 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import division
-
-import sys
-
-from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict
-from six import Iterator
-
-BLOCK_SIZE = 100
-
-
-class BedGraphReader(Iterator):
- def __init__(self, f):
- self.f = f
-
- def __iter__(self):
- return self
-
- def __next__(self):
- while True:
- line = self.f.readline()
- if not line:
- raise StopIteration()
- if line.isspace():
- continue
- if line[0] == "#":
- continue
- if line[0].isalpha():
- if line.startswith("track") or line.startswith("browser"):
- continue
-
- feature = line.strip().split()
- chrom = feature[0]
- chrom_start = int(feature[1])
- chrom_end = int(feature[2])
- score = float(feature[3])
- return chrom, chrom_start, chrom_end, None, score
-
-
-def main():
- input_fname = sys.argv[1]
- out_fname = sys.argv[2]
-
- reader = BedGraphReader(open(input_fname))
-
- # Fill array from reader
- d = array_tree_dict_from_reader(reader, {}, block_size=BLOCK_SIZE)
-
- for array_tree in d.values():
- array_tree.root.build_summary()
-
- FileArrayTreeDict.dict_to_file(d, open(out_fname, "w"))
-
-
-if __name__ == "__main__":
- main()
diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.xml b/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.xml
deleted file mode 100644
index f01a0408f02..00000000000
--- a/lib/galaxy/datatypes/converters/bedgraph_to_array_tree_converter.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
- python '$__tool_directory__/bedgraph_to_array_tree_converter.py' '$input' '$output'
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
index f842d0201f5..b984cd3d43f 100644
--- a/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/bedgraph_to_bigwig_converter.xml
@@ -10,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/biom1_to_biom2.xml b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml
index 8dc76d09e26..672bba616ea 100644
--- a/lib/galaxy/datatypes/converters/biom1_to_biom2.xml
+++ b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml
@@ -9,6 +9,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/biom2_to_biom1.xml b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml
index 559763ff74f..a280bd84432 100644
--- a/lib/galaxy/datatypes/converters/biom2_to_biom1.xml
+++ b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml
@@ -9,6 +9,16 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml b/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml
index 3d5ee318b78..ebe660218a0 100644
--- a/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml
+++ b/lib/galaxy/datatypes/converters/bz2_to_uncompressed.xml
@@ -1,4 +1,7 @@
-
+
+
+ bzip2
+
'$output1'
]]>
diff --git a/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml b/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml
index 440e72e4e81..a877bcd3320 100644
--- a/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml
+++ b/lib/galaxy/datatypes/converters/cram_to_bam_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ pysam
+
diff --git a/lib/galaxy/datatypes/converters/csv_to_tabular.xml b/lib/galaxy/datatypes/converters/csv_to_tabular.xml
index 4a00b67e406..95809c39c80 100644
--- a/lib/galaxy/datatypes/converters/csv_to_tabular.xml
+++ b/lib/galaxy/datatypes/converters/csv_to_tabular.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/tabular_csv.py' -o '$tabular' -i '$csv'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/dbkeys.loc.test b/lib/galaxy/datatypes/converters/dbkeys.loc.test
new file mode 100644
index 00000000000..2c13efb0fc0
--- /dev/null
+++ b/lib/galaxy/datatypes/converters/dbkeys.loc.test
@@ -0,0 +1,2 @@
+#
+hg17 hg17 ${__HERE__}/hg17.len
diff --git a/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml
index babbe100a48..15518840912 100644
--- a/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/encodepeak_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py'
-c ${input1.metadata.chromCol}
diff --git a/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml
index d3506653e84..8ba65850ba0 100644
--- a/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/encodepeak_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py'
-c ${input1.metadata.chromCol}
diff --git a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
index 419e04c2ec8..65706581c4b 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_2bit.xml
@@ -11,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml
index 8b191f1376e..7482cd03c28 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_bowtie_base_index_converter.xml
@@ -18,6 +18,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml
index 81ebd75a23a..d72edeae150 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_bowtie_color_index_converter.xml
@@ -19,6 +19,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_fai.xml b/lib/galaxy/datatypes/converters/fasta_to_fai.xml
index b8d1d3b5b46..6d572fdde85 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_fai.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_fai.xml
@@ -12,6 +12,12 @@ samtools faidx temp.fasta
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_len.xml b/lib/galaxy/datatypes/converters/fasta_to_len.xml
index 74f021cd753..03dd3672463 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_len.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_len.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/fasta_to_len.py' '$input' '$output' 0
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml
index d87ae808905..f9722c7d493 100644
--- a/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml
+++ b/lib/galaxy/datatypes/converters/fasta_to_tabular_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/fasta_to_tabular_converter.py' '$input' '$output'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml
index d8bb8ad377f..bd19fc6d117 100644
--- a/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml
+++ b/lib/galaxy/datatypes/converters/fastq_to_fqtoc.xml
@@ -6,6 +6,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py
index 8ddb749ec91..86b0cd9a1cc 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.py
@@ -19,8 +19,7 @@ assert sys.version_info[:2] >= (2, 4)
def stop_err(msg):
- sys.stderr.write("%s" % msg)
- sys.exit()
+ sys.exit("%s" % msg)
def __main__():
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml
index b4b1c89b00c..ff528b6f65f 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_fasta_converter.xml
@@ -1,5 +1,8 @@
-
+
converts Fastqsolexa file to Fasta format
+
+ python
+
python '$__tool_directory__/fastqsolexa_to_fasta_converter.py' '$input' '$output'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py
index e0aba894622..90fc5527ca7 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.py
@@ -19,8 +19,7 @@ assert sys.version_info[:2] >= (2, 4)
def stop_err(msg):
- sys.stderr.write("%s" % msg)
- sys.exit()
+ sys.exit("%s" % msg)
def __main__():
diff --git a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml
index b5ac900ae5e..eb0757db83b 100644
--- a/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml
+++ b/lib/galaxy/datatypes/converters/fastqsolexa_to_qual_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ python
+
python '$__tool_directory__/fastqsolexa_to_qual_converter.py' '$input1' '$output1' ${input1.extension}
@@ -6,6 +9,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml b/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml
index 4835288abe9..1208586b05d 100644
--- a/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_bed_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/gff_to_bed_converter.py' '$input1' '$output1'
@@ -8,6 +8,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml
index f26f8049bb8..3dcfa1d18bd 100644
--- a/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py' -P gff '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml b/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
index 0cd2b0e46ed..cba80687cf1 100644
--- a/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_fli_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/interval_to_fli.py' -F $input1.extension '$input1' '$output1'
@@ -8,6 +8,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py
index 6dda96b944f..2f5e131e2c9 100644
--- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.py
@@ -34,7 +34,7 @@ def main():
# not included in the index.
offset += feature.raw_size
- index.write(open(out_fname, "w"))
+ index.write(open(out_fname, "wb"))
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml
index f5971eb197a..47a6e72d269 100644
--- a/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_interval_index_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/gff_to_interval_index_converter.py' '$input1' '$output1'
@@ -7,6 +7,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml
index ad0a6059970..06ffce81787 100644
--- a/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/gff_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P gff '$input1' '$bgzip' '$output1'
@@ -8,6 +11,13 @@
+
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/hg17.len b/lib/galaxy/datatypes/converters/hg17.len
new file mode 100644
index 00000000000..801131eeb22
--- /dev/null
+++ b/lib/galaxy/datatypes/converters/hg17.len
@@ -0,0 +1,25 @@
+dummy_chr 100000000
+super_1 100000000
+chr1 100000000
+chr7 200000000
+chrX 200000000
+phiX174 100000000
+random_phiX_region_1 100000000
+random_phiX_region_2 100000000
+random_phiX_region_3 100000000
+random_phiX_region_4 100000000
+random_phiX_region_5 100000000
+random_phiX_region_6 100000000
+random_phiX_region_7 100000000
+random_phiX_region_8 100000000
+random_phiX_region_9 100000000
+random_phiX_region_10 100000000
+random_phiX_region_11 100000000
+random_phiX_region_12 100000000
+random_phiX_region_13 100000000
+random_phiX_region_14 100000000
+random_phiX_region_15 100000000
+random_phiX_region_16 100000000
+random_phiX_region_17 100000000
+random_phiX_region_18 100000000
+random_phiX_region_19 100000000
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml
index 26bc669653a..ace96313d74 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bed12_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_bedstrict_converter.py'
'$output1' '$input1' ${input1.metadata.chromCol}
@@ -13,6 +16,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml
index 3130f5d8865..77f39d8dfa9 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bed6_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_bedstrict_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension} 6
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py
index 38fc4ed3679..57505520611 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.py
@@ -10,8 +10,7 @@ assert sys.version_info[:2] >= (2, 6)
def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit()
+ sys.exit(msg)
def __main__():
diff --git a/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml
index 11109b7d3e3..5ff627e8174 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bed_converter.xml
@@ -1,8 +1,7 @@
-
+
- python
- bx-python
+ bx-python
python '$__tool_directory__/interval_to_bed_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol}
@@ -11,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py
index 7a6dc10358f..e309292db44 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.py
@@ -10,8 +10,7 @@ assert sys.version_info[:2] >= (2, 6)
def stop_err(msg):
- sys.stderr.write(msg)
- sys.exit()
+ sys.exit(msg)
def force_bed_field_count(fields, region_count, force_num_columns):
diff --git a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml
index 47b3a7a4e4d..33203ae06d6 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bedstrict_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_bedstrict_converter.py' '$output1' '$input1' ${input1.metadata.chromCol} ${input1.metadata.startCol} ${input1.metadata.endCol} ${input1.metadata.strandCol} ${input1.metadata.nameCol} ${input1.extension}
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml
index 615170f893a..020f539d501 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py'
-c ${input1.metadata.chromCol}
@@ -13,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
index 05654055e55..684001e6a03 100644
--- a/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_bigwig_converter.xml
@@ -1,8 +1,9 @@
- ucsc-bedgraphtobigwig
- bedtools
+ ucsc-bedgraphtobigwig
+ bedtools
+ coreutils
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.py b/lib/galaxy/datatypes/converters/interval_to_coverage.py
deleted file mode 100644
index aff6b296a11..00000000000
--- a/lib/galaxy/datatypes/converters/interval_to_coverage.py
+++ /dev/null
@@ -1,151 +0,0 @@
-#!/usr/bin/env python
-"""
-Converter to generate 3 (or 4) column base-pair coverage from an interval file.
-
-usage: %prog bed_file out_file
- -1, --cols1=N,N,N,N: Columns for chrom, start, end, strand in interval file
- -2, --cols2=N,N,N,N: Columns for chrom, start, end, strand in coverage file
-"""
-import subprocess
-import tempfile
-from bisect import bisect
-from os import environ
-
-from bx.cookbook import doc_optparse
-from bx.intervals import io
-
-INTERVAL_METADATA = ('chromCol',
- 'startCol',
- 'endCol',
- 'strandCol',)
-
-COVERAGE_METADATA = ('chromCol',
- 'positionCol',
- 'forwardCol',
- 'reverseCol',)
-
-
-def main(interval, coverage):
- """
- Uses a sliding window of partitions to count coverages.
- Every interval record adds its start and end to the partitions. The result
- is a list of partitions, or every position that has a (maybe) different
- number of basepairs covered. We don't worry about merging because we pop
- as the sorted intervals are read in. As the input start positions exceed
- the partition positions in partitions, coverages are kicked out in bulk.
- """
- partitions = []
- forward_covs = []
- reverse_covs = []
- chrom = None
- lastchrom = None
- for record in interval:
- chrom = record.chrom
- if lastchrom and not lastchrom == chrom and partitions:
- for partition in range(0, len(partitions) - 1):
- forward = forward_covs[partition]
- reverse = reverse_covs[partition]
- if forward + reverse > 0:
- coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
- forward=forward, reverse=reverse)
- partitions = []
- forward_covs = []
- reverse_covs = []
-
- start_index = bisect(partitions, record.start)
- forward = int(record.strand == "+")
- reverse = int(record.strand == "-")
- forward_base = 0
- reverse_base = 0
- if start_index > 0:
- forward_base = forward_covs[start_index - 1]
- reverse_base = reverse_covs[start_index - 1]
- partitions.insert(start_index, record.start)
- forward_covs.insert(start_index, forward_base)
- reverse_covs.insert(start_index, reverse_base)
- end_index = bisect(partitions, record.end)
- for index in range(start_index, end_index):
- forward_covs[index] += forward
- reverse_covs[index] += reverse
- partitions.insert(end_index, record.end)
- forward_covs.insert(end_index, forward_covs[end_index - 1] - forward)
- reverse_covs.insert(end_index, reverse_covs[end_index - 1] - reverse)
-
- if partitions:
- for partition in range(0, start_index):
- forward = forward_covs[partition]
- reverse = reverse_covs[partition]
- if forward + reverse > 0:
- coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
- forward=forward, reverse=reverse)
- partitions = partitions[start_index:]
- forward_covs = forward_covs[start_index:]
- reverse_covs = reverse_covs[start_index:]
-
- lastchrom = chrom
-
- # Finish the last chromosome
- if partitions:
- for partition in range(0, len(partitions) - 1):
- forward = forward_covs[partition]
- reverse = reverse_covs[partition]
- if forward + reverse > 0:
- coverage.write(chrom=chrom, position=range(partitions[partition], partitions[partition + 1]),
- forward=forward, reverse=reverse)
-
-
-class CoverageWriter(object):
- def __init__(self, out_stream=None, chromCol=0, positionCol=1, forwardCol=2, reverseCol=3):
- self.out_stream = out_stream
- self.reverseCol = reverseCol
- self.nlines = 0
- positions = {str(chromCol): '%(chrom)s',
- str(positionCol): '%(position)d',
- str(forwardCol): '%(forward)d',
- str(reverseCol): '%(reverse)d'}
- if reverseCol < 0:
- self.template = "%(0)s\t%(1)s\t%(2)s\n" % positions
- else:
- self.template = "%(0)s\t%(1)s\t%(2)s\t%(3)s\n" % positions
-
- def write(self, **kwargs):
- if self.reverseCol < 0:
- kwargs['forward'] += kwargs['reverse']
- posgen = kwargs['position']
- for position in posgen:
- kwargs['position'] = position
- self.out_stream.write(self.template % kwargs)
-
- def close(self):
- self.out_stream.flush()
- self.out_stream.close()
-
-
-if __name__ == "__main__":
- options, args = doc_optparse.parse(__doc__)
- try:
- chr_col_1, start_col_1, end_col_1, strand_col_1 = [int(x) - 1 for x in options.cols1.split(',')]
- chr_col_2, position_col_2, forward_col_2, reverse_col_2 = [int(x) - 1 for x in options.cols2.split(',')]
- in_fname, out_fname = args
- except Exception:
- doc_optparse.exception()
-
- # Sort through a tempfile first
- with tempfile.NamedTemporaryFile(mode="r") as temp_file:
- environ['LC_ALL'] = 'POSIX'
- subprocess.check_call([
- 'sort', '-f', '-n', '-k', chr_col_1 + 1, '-k', start_col_1 + 1, '-k', end_col_1 + 1, '-o', temp_file.name, in_fname
- ])
-
- coverage = CoverageWriter(out_stream=open(out_fname, "a"),
- chromCol=chr_col_2, positionCol=position_col_2,
- forwardCol=forward_col_2, reverseCol=reverse_col_2, )
- temp_file.seek(0)
- interval = io.NiceReaderWrapper(temp_file,
- chrom_col=chr_col_1,
- start_col=start_col_1,
- end_col=end_col_1,
- strand_col=strand_col_1,
- fix_strand=True)
- main(interval, coverage)
- coverage.close()
diff --git a/lib/galaxy/datatypes/converters/interval_to_coverage.xml b/lib/galaxy/datatypes/converters/interval_to_coverage.xml
deleted file mode 100644
index 1b3a139fec9..00000000000
--- a/lib/galaxy/datatypes/converters/interval_to_coverage.xml
+++ /dev/null
@@ -1,16 +0,0 @@
-
-
-
- python '$__tool_directory__/interval_to_coverage.py' '$input1' '$output1'
- -1 ${input1.metadata.chromCol},${input1.metadata.startCol},${input1.metadata.endCol},${input1.metadata.strandCol}
- -2 ${output1.metadata.chromCol},${output1.metadata.positionCol},${output1.metadata.forwardCol},${output1.metadata.reverseCol}
-
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py
index 54dbd2d16ed..c5f7b72cb85 100644
--- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.py
@@ -33,18 +33,20 @@ def main():
# Do conversion.
index = Indexes()
offset = 0
- for line in open(input_fname, "r"):
- feature = line.strip().split()
- if not feature or feature[0].startswith("track") or feature[0].startswith("#"):
+ with open(input_fname) as in_fh:
+ for line in in_fh:
+ feature = line.strip().split()
+ if not feature or feature[0].startswith("track") or feature[0].startswith("#"):
+ offset += len(line)
+ continue
+ chrom = feature[options.chrom_col]
+ chrom_start = int(feature[options.start_col])
+ chrom_end = int(feature[options.end_col])
+ index.add(chrom, chrom_start, chrom_end, offset)
offset += len(line)
- continue
- chrom = feature[options.chrom_col]
- chrom_start = int(feature[options.start_col])
- chrom_end = int(feature[options.end_col])
- index.add(chrom, chrom_start, chrom_end, offset)
- offset += len(line)
- index.write(open(output_fname, "w"))
+ with open(output_fname, 'wb') as out:
+ index.write(out)
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml
index 67a480fe4e7..9c8d1a949c4 100644
--- a/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_interval_index_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/interval_to_interval_index_converter.py'
-c ${input1.metadata.chromCol}
@@ -13,6 +16,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py
index faf988da48e..a4fec2bef4d 100644
--- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py
+++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.py
@@ -42,7 +42,7 @@ def to_tabix(bgzip_fname, out_fname, preset=None, chrom_col=None, start_col=None
start_col=(start_col - 1), end_col=(end_col - 1),
keep_original=True, index=out_fname, force=True)
if os.path.getsize(out_fname) == 0:
- sys.stderr.write("The converted tabix index file is empty, meaning the input data is invalid.")
+ sys.exit("The converted tabix index file is empty, meaning the input data is invalid.")
return bgzip_fname
diff --git a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml
index 1e5d9974524..d99ee11d89b 100644
--- a/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/interval_to_tabix_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py'
@@ -14,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/len_to_linecount.xml b/lib/galaxy/datatypes/converters/len_to_linecount.xml
index 9f638bfbda7..db451e80bf8 100644
--- a/lib/galaxy/datatypes/converters/len_to_linecount.xml
+++ b/lib/galaxy/datatypes/converters/len_to_linecount.xml
@@ -1,6 +1,9 @@
-
+
+
+ gawk
+
'$output'
@@ -12,6 +15,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py
index 2bc185310bf..81be44df3c5 100644
--- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.py
+++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.py
@@ -42,13 +42,11 @@ def rgConv(inpedfilepath, outhtmlname, outfilepath):
try:
mf = open(inmap, 'r')
except Exception:
- sys.stderr.write('%s cannot open inmap file %s - do you have permission?\n' % (prog, inmap))
- sys.exit(1)
+ sys.exit('%s cannot open inmap file %s - do you have permission?\n' % (prog, inmap))
try:
rsl = [x.split()[1] for x in mf]
except Exception:
- sys.stderr.write('## cannot parse %s' % inmap)
- sys.exit(1)
+ sys.exit('## cannot parse %s' % inmap)
try:
os.makedirs(outfilepath)
except Exception:
@@ -85,8 +83,7 @@ def main():
"""
nparm = 3
if len(sys.argv) < nparm:
- sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
- sys.exit(1)
+ sys.exit('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
inpedfilepath = sys.argv[1]
outhtmlname = sys.argv[2]
outfilepath = sys.argv[3]
diff --git a/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml b/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml
index 16d0cab9686..da15908c80a 100644
--- a/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml
+++ b/lib/galaxy/datatypes/converters/lped_to_fped_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/lped_to_fped_converter.py' '$input1.extra_files_path/$input1.metadata.base_name' '$output1' '$output1.files_path'
diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
index 68fc19bc9c3..4201bc8fe22 100644
--- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
+++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.py
@@ -87,8 +87,7 @@ def main():
"""
nparm = 4
if len(sys.argv) < nparm:
- sys.stderr.write('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
- sys.exit(1)
+ sys.exit('## %s called with %s - needs %d parameters \n' % (prog, sys.argv, nparm))
inpedfilepath = sys.argv[1]
outhtmlname = sys.argv[2]
outfilepath = sys.argv[3]
diff --git a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml
index 40001ae9af8..fe434710368 100644
--- a/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml
+++ b/lib/galaxy/datatypes/converters/lped_to_pbed_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/lped_to_pbed_converter.py' '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' 'plink'
diff --git a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml
index 725d9789a47..824d8fc0f95 100644
--- a/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml
+++ b/lib/galaxy/datatypes/converters/maf_to_fasta_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/maf_to_fasta_converter.py' '$output1' '$input1'
@@ -7,6 +7,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml b/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml
index fc2a04b06f7..6405dfb097d 100644
--- a/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml
+++ b/lib/galaxy/datatypes/converters/maf_to_interval_converter.xml
@@ -1,4 +1,4 @@
-
+
python '$__tool_directory__/maf_to_interval_converter.py' '$output1' '$input1' '${input1.metadata.dbkey}'
@@ -7,6 +7,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml
index 74486fe30d4..3ae5c09c564 100644
--- a/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml
+++ b/lib/galaxy/datatypes/converters/neostorezip_to_neostore_converter.xml
@@ -1,4 +1,7 @@
-
+
+
+ unzip
+
$output1
]]>
diff --git a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml
index dfe22d6ae12..c48372b818f 100644
--- a/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml
+++ b/lib/galaxy/datatypes/converters/pbed_ldreduced_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/pbed_ldreduced_converter.py' '$input1.extra_files_path/$input1.metadata.base_name' '60' '55' '0.1' '$output1' '$output1.files_path' 'plink'
diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
index e61aba09556..f7ed7689c80 100644
--- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
+++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.py
@@ -53,8 +53,7 @@ def main():
"""
nparm = 4
if len(sys.argv) < nparm:
- sys.stderr.write('PBED to LPED converter called with %s - needs %d parameters \n' % (sys.argv, nparm))
- sys.exit(1)
+ sys.exit('PBED to LPED converter called with %s - needs %d parameters \n' % (sys.argv, nparm))
inpedfilepath = sys.argv[1]
outhtmlname = sys.argv[2]
outfilepath = sys.argv[3]
diff --git a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml
index 0b19e2f20e3..0a308fd9b04 100644
--- a/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml
+++ b/lib/galaxy/datatypes/converters/pbed_to_lped_converter.xml
@@ -1,6 +1,9 @@
-
+
+
+ python
+
python '$__tool_directory__/pbed_to_lped_converter.py' '$input1.extra_files_path/$input1.metadata.base_name'
'$output1' '$output1.files_path' plink
diff --git a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml
index 99d31e7eb98..4f573473314 100644
--- a/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml
+++ b/lib/galaxy/datatypes/converters/picard_interval_list_to_bed6_converter.xml
@@ -1,5 +1,8 @@
-
+
converter
+
+ python
+
python '$__tool_directory__/picard_interval_list_to_bed6_converter.py' '$input' '$output'
diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
index 02d781e1742..103e0c5f4ba 100644
--- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.py
@@ -22,14 +22,16 @@ def main():
# Do conversion.
index = Indexes()
offset = 0
- for line in open(input_fname, "r"):
- chrom, start = line.split()[0:2]
- # Pileup format is 1-based.
- start = int(start) - 1
- index.add(chrom, start, start + 1, offset)
- offset += len(line)
+ with open(input_fname) as in_fh:
+ for line in in_fh:
+ chrom, start = line.split()[0:2]
+ # Pileup format is 1-based.
+ start = int(start) - 1
+ index.add(chrom, start, start + 1, offset)
+ offset += len(line)
- index.write(open(output_fname, "w"))
+ with open(output_fname, 'wb') as out:
+ index.write(out)
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
index a500ed866ff..7b960385095 100644
--- a/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/pileup_to_interval_index_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ bx-python
+
python '$__tool_directory__/pileup_to_interval_index_converter.py'
'$input' '$output'
@@ -10,6 +13,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml
index 2d1d9db87ed..6f0d114422f 100644
--- a/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml
+++ b/lib/galaxy/datatypes/converters/ref_to_seq_taxonomy_converter.xml
@@ -1,5 +1,8 @@
-
+
converts 2 or 3 column sequence taxonomy file to a 2 column mothur taxonomy_outline format
+
+ python
+
python '$__tool_directory__/ref_to_seq_taxonomy_converter.py' '$input' '$output'
@@ -9,4 +12,4 @@
-
\ No newline at end of file
+
diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.py b/lib/galaxy/datatypes/converters/sam_to_bam.py
deleted file mode 100644
index 66962d13392..00000000000
--- a/lib/galaxy/datatypes/converters/sam_to_bam.py
+++ /dev/null
@@ -1,117 +0,0 @@
-#!/usr/bin/env python
-# Dan Blankenberg
-
-"""
-A wrapper script for converting SAM to BAM, with sorting.
-%prog input_filename.sam output_filename.bam
-"""
-import optparse
-import os
-import shutil
-import subprocess
-import sys
-import tempfile
-
-import packaging.version
-
-CHUNK_SIZE = 2 ** 20 # 1mb
-
-
-def cleanup_before_exit(tmp_dir):
- if tmp_dir and os.path.exists(tmp_dir):
- shutil.rmtree(tmp_dir)
-
-
-def cmd_exists(cmd):
- # http://stackoverflow.com/questions/5226958/which-equivalent-function-in-python
- for path in os.environ["PATH"].split(":"):
- if os.path.exists(os.path.join(path, cmd)):
- return True
- return False
-
-
-def _get_samtools_version():
- version = '0.0.0'
- if not cmd_exists('samtools'):
- raise Exception('This tool needs samtools, but it is not on PATH.')
- # Get the version of samtools via --version-only, if available
- p = subprocess.Popen(['samtools', '--version-only'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
- output, error = p.communicate()
- # --version-only is available
- # Format is +htslib-
- if p.returncode == 0:
- version = output.split('+')[0]
- return version
-
- output = subprocess.Popen(['samtools'], stderr=subprocess.PIPE, stdout=subprocess.PIPE).communicate()[1]
- lines = output.split('\n')
- for line in lines:
- if line.lower().startswith('version'):
- # Assuming line looks something like: version: 0.1.12a (r862)
- version = line.split()[1]
- break
- return version
-
-
-def __main__():
- # Parse Command Line
- parser = optparse.OptionParser()
- (options, args) = parser.parse_args()
-
- assert len(args) == 2, 'You must specify the input and output filenames'
- input_filename, output_filename = args
-
- tmp_dir = tempfile.mkdtemp(prefix='tmp-sam_to_bam_converter-')
-
- # convert to SAM
- unsorted_bam_filename = os.path.join(tmp_dir, 'unsorted.bam')
- unsorted_stderr_filename = os.path.join(tmp_dir, 'unsorted.stderr')
- proc = subprocess.Popen(['samtools', 'view', '-bS', input_filename],
- stdout=open(unsorted_bam_filename, 'wb'),
- stderr=open(unsorted_stderr_filename, 'wb'),
- cwd=tmp_dir)
- return_code = proc.wait()
- if return_code:
- stderr_target = sys.stderr
- else:
- stderr_target = sys.stdout
- with open(unsorted_stderr_filename) as stderr:
- while True:
- chunk = stderr.read(CHUNK_SIZE)
- if chunk:
- stderr_target.write(chunk)
- else:
- break
-
- # sort sam, so indexing will not fail
- sorted_stderr_filename = os.path.join(tmp_dir, 'sorted.stderr')
- sorting_prefix = os.path.join(tmp_dir, 'sorted_bam')
- # samtools changed sort command arguments (starting from version 1.3)
- samtools_version = packaging.version.parse(_get_samtools_version())
- if samtools_version < packaging.version.parse('1.0'):
- sort_args = ['-o', unsorted_bam_filename, sorting_prefix]
- else:
- sort_args = ['-T', sorting_prefix, unsorted_bam_filename]
- proc = subprocess.Popen(['samtools', 'sort'] + sort_args,
- stdout=open(output_filename, 'wb'),
- stderr=open(sorted_stderr_filename, 'wb'),
- cwd=tmp_dir)
- return_code = proc.wait()
-
- if return_code:
- stderr_target = sys.stderr
- else:
- stderr_target = sys.stdout
- with open(sorted_stderr_filename) as stderr:
- while True:
- chunk = stderr.read(CHUNK_SIZE)
- if chunk:
- stderr_target.write(chunk)
- else:
- break
-
- cleanup_before_exit(tmp_dir)
-
-
-if __name__ == "__main__":
- __main__()
diff --git a/lib/galaxy/datatypes/converters/sam_to_bam.xml b/lib/galaxy/datatypes/converters/sam_to_bam.xml
deleted file mode 100644
index 9a32a1d988e..00000000000
--- a/lib/galaxy/datatypes/converters/sam_to_bam.xml
+++ /dev/null
@@ -1,20 +0,0 @@
-
-
-
-
-
- samtools
-
- python '$__tool_directory__/sam_to_bam.py' '$input1' '$output'
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
index 434c44b6a12..94ab9981480 100644
--- a/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/sam_to_bigwig_converter.xml
@@ -20,6 +20,12 @@ samtools view -bh '$input' | bedtools genomecov -bg -split -ibam stdin
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml b/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml
index 2a99a793d27..46fa4af4ae7 100644
--- a/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml
+++ b/lib/galaxy/datatypes/converters/sam_to_unsorted_bam.xml
@@ -17,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tabular_csv.py b/lib/galaxy/datatypes/converters/tabular_csv.py
index 0b18e1571e5..a323385037a 100644
--- a/lib/galaxy/datatypes/converters/tabular_csv.py
+++ b/lib/galaxy/datatypes/converters/tabular_csv.py
@@ -25,19 +25,17 @@ def main():
def convert_to_tsv(input_fname, output_fname):
- with open(input_fname, 'rb') as csvfile:
- with open(output_fname, 'wb') as ofh:
- reader = csv.reader(csvfile)
- for line in reader:
- ofh.write('\t'.join(line) + '\n')
+ with open(input_fname, newline="") as csvfile, open(output_fname, 'w') as ofh:
+ reader = csv.reader(csvfile)
+ for line in reader:
+ ofh.write('\t'.join(line) + '\n')
def convert_to_csv(input_fname, output_fname):
- with open(input_fname, 'rb') as tabfile:
- with open(output_fname, 'wb') as ofh:
- writer = csv.writer(ofh, delimiter=',')
- for line in tabfile.readlines():
- writer.writerow(line.strip().split('\t'))
+ with open(input_fname) as tabfile, open(output_fname, 'w', newline='') as ofh:
+ writer = csv.writer(ofh, delimiter=',')
+ for line in tabfile.readlines():
+ writer.writerow(line.strip().split('\t'))
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/tabular_to_csv.xml b/lib/galaxy/datatypes/converters/tabular_to_csv.xml
index 2c853109184..c449bc3614e 100644
--- a/lib/galaxy/datatypes/converters/tabular_to_csv.xml
+++ b/lib/galaxy/datatypes/converters/tabular_to_csv.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/tabular_csv.py' --from-tabular -i '$tabular' -o '$csv'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml
index d82911bc912..5d61c6119a4 100644
--- a/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml
+++ b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml
@@ -1,5 +1,8 @@
-
+
+
+ python
+
python '$__tool_directory__/tabular_to_dbnsfp.py' '$input' '$dbnsfp.extra_files_path/dbNSFP.gz'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tar_to_directory.xml b/lib/galaxy/datatypes/converters/tar_to_directory.xml
index 6865b86ee1d..0b8c18c1ded 100644
--- a/lib/galaxy/datatypes/converters/tar_to_directory.xml
+++ b/lib/galaxy/datatypes/converters/tar_to_directory.xml
@@ -11,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml b/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml
index 1e2889a921c..f47ee1efbff 100644
--- a/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml
+++ b/lib/galaxy/datatypes/converters/to_coordinate_sorted_bam.xml
@@ -17,6 +17,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml b/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml
index ced46c0cc31..c2a4d8c53b7 100644
--- a/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml
+++ b/lib/galaxy/datatypes/converters/to_qname_sorted_bam.xml
@@ -18,6 +18,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/tool_data_table_conf.xml.test b/lib/galaxy/datatypes/converters/tool_data_table_conf.xml.test
new file mode 100644
index 00000000000..d88d1d880ef
--- /dev/null
+++ b/lib/galaxy/datatypes/converters/tool_data_table_conf.xml.test
@@ -0,0 +1,7 @@
+
+
+
+ value, name, len_path
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml
index 64deb6df678..cd5b477bc37 100644
--- a/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_bgzip_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P 'vcf' '' '$input1' '$output1'
@@ -7,6 +10,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml
index 79a09003382..be9d42400c3 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/bgzip.py' -P vcf '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py
index a635db88aab..8f5e24402af 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py
+++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.py
@@ -18,15 +18,17 @@ def main():
# Do conversion.
index = Indexes()
- reader = galaxy_utils.sequence.vcf.Reader(open(in_file))
- offset = reader.metadata_len
- for vcf_line in reader:
- # VCF format provides a chrom and 1-based position for each variant.
- # IntervalIndex expects 0-based coordinates.
- index.add(vcf_line.chrom, vcf_line.pos - 1, vcf_line.pos, offset)
- offset += len(vcf_line.raw_line)
+ with open(in_file) as in_fh:
+ reader = galaxy_utils.sequence.vcf.Reader(in_fh)
+ offset = reader.metadata_len
+ for vcf_line in reader:
+ # VCF format provides a chrom and 1-based position for each variant.
+ # IntervalIndex expects 0-based coordinates.
+ index.add(vcf_line.chrom, vcf_line.pos - 1, vcf_line.pos, offset)
+ offset += len(vcf_line.raw_line)
- index.write(open(out_file, "w"))
+ with open(out_file, "wb") as out_fh:
+ index.write(out_fh)
if __name__ == "__main__":
diff --git a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml
index 03bf17d9709..9c2c96fa559 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_interval_index_converter.xml
@@ -1,5 +1,9 @@
-
+
__NOT_USED_CURRENTLY_FOR_CONVERTERS__
+
+ bx-python
+ galaxy_sequence_utils
+
python '$__tool_directory__/vcf_to_interval_index_converter.py' '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml
index 41394394572..984695d4b39 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_tabix_converter.xml
@@ -1,5 +1,8 @@
-
+
+
+ pysam
+
python '$__tool_directory__/interval_to_tabix_converter.py' -P vcf '$input1' '$bgzip' '$output1'
@@ -8,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml
index 6b015752ca9..4a78f175c11 100644
--- a/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml
+++ b/lib/galaxy/datatypes/converters/vcf_to_vcf_bgzip_converter.xml
@@ -1,5 +1,9 @@
-
+
+
+ pysam
+ coreutils
+
python '$__tool_directory__/vcf_to_vcf_bgzip.py' '$input1' '$output1'
@@ -7,6 +11,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
index 862016e9151..4666b41ed99 100644
--- a/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
+++ b/lib/galaxy/datatypes/converters/wig_to_bigwig_converter.xml
@@ -15,6 +15,12 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py
deleted file mode 100644
index 6dc81144ea6..00000000000
--- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.py
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env python
-
-from __future__ import division
-
-import sys
-
-from bx.arrays.array_tree import array_tree_dict_from_reader, FileArrayTreeDict
-from bx.arrays.wiggle import WiggleReader
-
-BLOCK_SIZE = 100
-
-
-def main():
-
- input_fname = sys.argv[1]
- out_fname = sys.argv[2]
-
- reader = WiggleReader(open(input_fname))
-
- # Fill array from reader
- d = array_tree_dict_from_reader(reader, {}, block_size=BLOCK_SIZE)
-
- for array_tree in d.values():
- array_tree.root.build_summary()
-
- FileArrayTreeDict.dict_to_file(d, open(out_fname, "w"))
-
-
-if __name__ == "__main__":
- main()
diff --git a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.xml b/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.xml
deleted file mode 100644
index 5ae0d98fd08..00000000000
--- a/lib/galaxy/datatypes/converters/wiggle_to_array_tree_converter.xml
+++ /dev/null
@@ -1,12 +0,0 @@
-
-
- python '$__tool_directory__/wiggle_to_array_tree_converter.py' '$input' '$output'
-
-
-
-
-
-
-
-
-
diff --git a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml
index 29c494189ad..0273777e20f 100644
--- a/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml
+++ b/lib/galaxy/datatypes/converters/wiggle_to_simple_converter.xml
@@ -8,4 +8,10 @@
+
+
+
+
+
+
diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py
index 036bd1a82d4..0336c5cb2ac 100644
--- a/lib/galaxy/datatypes/registry.py
+++ b/lib/galaxy/datatypes/registry.py
@@ -45,6 +45,7 @@ class Registry(object):
self.datatype_converters = OrderedDict()
# Converters defined in local datatypes_conf.xml
self.converters = []
+ self.converter_tools = set()
# Converters defined in datatypes_conf.xml included in installed tool shed repositories.
self.proprietary_converters = []
self.converter_deps = {}
@@ -612,6 +613,7 @@ class Registry(object):
try:
config_path = os.path.join(converter_path, tool_config)
converter = toolbox.load_tool(config_path, use_cached=use_cached)
+ self.converter_tools.add(converter)
if installed_repository_dict:
# If the converter is included in an installed tool shed repository, set the tool
# shed related tool attributes.
diff --git a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
index 1807fb9ba43..b22a0104668 100644
--- a/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
+++ b/lib/galaxy/dependencies/pipfiles/default/pinned-requirements.txt
@@ -28,7 +28,7 @@ boltons==19.3.0
boto3==1.9.114
boto==2.49.0
botocore==1.12.253
-bx-python==0.8.6
+bx-python==0.8.8
bz2file==0.98 ; python_version < '3.3'
cachecontrol==0.11.7
cachetools==3.1.1
diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py
index 686e4ce796b..64c3b059a2a 100755
--- a/lib/galaxy/tools/__init__.py
+++ b/lib/galaxy/tools/__init__.py
@@ -537,6 +537,10 @@ class Tool(Dictifiable):
tool_versions = self.tool_versions
return not tool_versions or self.version == self.tool_versions[-1]
+ @property
+ def is_datatype_converter(self):
+ return self in self.app.datatypes_registry.converter_tools
+
@property
def tool_shed_repository(self):
# If this tool is included in an installed tool shed repository, return it.
diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py
index ad4f918e0ba..90208e09da8 100644
--- a/lib/galaxy/webapps/galaxy/api/tools.py
+++ b/lib/galaxy/webapps/galaxy/api/tools.py
@@ -172,15 +172,16 @@ class ToolsController(BaseAPIController, UsesVisualizationMixin):
"""
test_counts_by_tool = {}
for id, tool in self.app.toolbox.tools():
- tests = tool.tests
- if tests:
- if tool.id not in test_counts_by_tool:
- test_counts_by_tool[tool.id] = {}
- available_versions = test_counts_by_tool[tool.id]
- available_versions[tool.version] = {
- "tool_name": tool.name,
- "count": len(tests),
- }
+ if not tool.is_datatype_converter:
+ tests = tool.tests
+ if tests:
+ if tool.id not in test_counts_by_tool:
+ test_counts_by_tool[tool.id] = {}
+ available_versions = test_counts_by_tool[tool.id]
+ available_versions[tool.version] = {
+ "tool_name": tool.name,
+ "count": len(tests),
+ }
return test_counts_by_tool
@expose_api_raw_anonymous_and_sessionless
diff --git a/tools/filters/bed_to_gff_converter.py b/tools/filters/bed_to_gff_converter.py
index 768891fdcf8..fb7e0bd258f 100644
--- a/tools/filters/bed_to_gff_converter.py
+++ b/tools/filters/bed_to_gff_converter.py
@@ -12,60 +12,59 @@ def __main__():
output_name = sys.argv[2]
skipped_lines = 0
first_skipped_line = 0
- out = open(output_name, 'w')
i = 0
- for i, line in enumerate(open(input_name)):
- complete_bed = False
- line = line.rstrip('\r\n')
- if line and not line.startswith('#') and not line.startswith('track') and not line.startswith('browser'):
- try:
- elems = line.split('\t')
- if len(elems) == 12:
- complete_bed = True
- chrom = elems[0]
- if complete_bed:
- feature = "mRNA"
- else:
+ with open(output_name, 'w') as out, open(input_name) as fh_in:
+ for i, line in enumerate(fh_in):
+ complete_bed = False
+ line = line.rstrip('\r\n')
+ if line and not line.startswith('#') and not line.startswith('track') and not line.startswith('browser'):
+ try:
+ elems = line.split('\t')
+ if len(elems) == 12:
+ complete_bed = True
+ chrom = elems[0]
+ if complete_bed:
+ feature = "mRNA"
+ else:
+ try:
+ feature = elems[3]
+ except Exception:
+ feature = 'feature%d' % (i + 1)
+ start = int(elems[1]) + 1
+ end = int(elems[2])
try:
- feature = elems[3]
+ score = elems[4]
except Exception:
- feature = 'feature%d' % (i + 1)
- start = int(elems[1]) + 1
- end = int(elems[2])
- try:
- score = elems[4]
+ score = '0'
+ try:
+ strand = elems[5]
+ except Exception:
+ strand = '+'
+ try:
+ group = elems[3]
+ except Exception:
+ group = 'group%d' % (i + 1)
+ if complete_bed:
+ out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n' % (chrom, feature, start, end, score, strand, feature, group))
+ else:
+ out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n' % (chrom, feature, start, end, score, strand, group))
+ if complete_bed:
+ # We have all the info necessary to annotate exons for genes and mRNAs
+ block_count = int(elems[9])
+ block_sizes = elems[10].split(',')
+ block_starts = elems[11].split(',')
+ for j in range(block_count):
+ exon_start = int(start) + int(block_starts[j])
+ exon_end = exon_start + int(block_sizes[j]) - 1
+ out.write('%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n' % (chrom, exon_start, exon_end, score, strand, group))
except Exception:
- score = '0'
- try:
- strand = elems[5]
- except Exception:
- strand = '+'
- try:
- group = elems[3]
- except Exception:
- group = 'group%d' % (i + 1)
- if complete_bed:
- out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s %s;\n' % (chrom, feature, start, end, score, strand, feature, group))
- else:
- out.write('%s\tbed2gff\t%s\t%d\t%d\t%s\t%s\t.\t%s;\n' % (chrom, feature, start, end, score, strand, group))
- if complete_bed:
- # We have all the info necessary to annotate exons for genes and mRNAs
- block_count = int(elems[9])
- block_sizes = elems[10].split(',')
- block_starts = elems[11].split(',')
- for j in range(block_count):
- exon_start = int(start) + int(block_starts[j])
- exon_end = exon_start + int(block_sizes[j]) - 1
- out.write('%s\tbed2gff\texon\t%d\t%d\t%s\t%s\t.\texon %s;\n' % (chrom, exon_start, exon_end, score, strand, group))
- except Exception:
+ skipped_lines += 1
+ if not first_skipped_line:
+ first_skipped_line = i + 1
+ else:
skipped_lines += 1
if not first_skipped_line:
first_skipped_line = i + 1
- else:
- skipped_lines += 1
- if not first_skipped_line:
- first_skipped_line = i + 1
- out.close()
info_msg = "%i lines converted to GFF version 2. " % (i + 1 - skipped_lines)
if skipped_lines > 0:
info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line)
diff --git a/tools/filters/gff_to_bed_converter.py b/tools/filters/gff_to_bed_converter.py
index 1326188dd9b..929347ce959 100644
--- a/tools/filters/gff_to_bed_converter.py
+++ b/tools/filters/gff_to_bed_converter.py
@@ -55,29 +55,50 @@ def __main__():
output_name = sys.argv[2]
skipped_lines = 0
first_skipped_line = 0
- out = open(output_name, 'w')
i = 0
cur_transcript_chrome = None
cur_transcript_id = None
cur_transcript_strand = None
cur_transcripts_blocks = [] # (start, end) for each block.
- for i, line in enumerate(open(input_name)):
- line = line.rstrip('\r\n')
- if line and not line.startswith('#'):
- try:
- # GFF format: chrom source, name, chromStart, chromEnd, score, strand, attributes
- elems = line.split('\t')
- start = str(int(elems[3]) - 1)
- coords = [int(start), int(elems[4])]
- strand = elems[6]
- if strand not in ['+', '-']:
- strand = '+'
- attributes = parse_gff_attributes(elems[8])
- t_id = attributes.get("transcript_id", None)
+ with open(output_name, 'w') as out, open(input_name) as in_fh:
+ for i, line in enumerate(in_fh):
+ line = line.rstrip('\r\n')
+ if line and not line.startswith('#'):
+ try:
+ # GFF format: chrom source, name, chromStart, chromEnd, score, strand, attributes
+ elems = line.split('\t')
+ start = str(int(elems[3]) - 1)
+ coords = [int(start), int(elems[4])]
+ strand = elems[6]
+ if strand not in ['+', '-']:
+ strand = '+'
+ attributes = parse_gff_attributes(elems[8])
+ t_id = attributes.get("transcript_id", None)
+
+ if not t_id:
+ #
+ # No transcript ID, so write last transcript and write current line as its own line.
+ #
+
+ # Write previous transcript.
+ if cur_transcript_id:
+ # Write BED entry.
+ out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
+
+ # Replace any spaces in the name with underscores so UCSC will not complain.
+ name = elems[2].replace(" ", "_")
+ out.write(get_bed_line(elems[0], name, strand, [coords]))
+ continue
+
+ # There is a transcript ID, so process line at transcript level.
+ if t_id == cur_transcript_id:
+ # Line is element of transcript and will be a block in the BED entry.
+ cur_transcripts_blocks.append(coords)
+ continue
- if not t_id:
#
- # No transcript ID, so write last transcript and write current line as its own line.
+ # Line is part of new transcript; write previous transcript and start
+ # new transcript.
#
# Write previous transcript.
@@ -85,47 +106,25 @@ def __main__():
# Write BED entry.
out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
- # Replace any spaces in the name with underscores so UCSC will not complain.
- name = elems[2].replace(" ", "_")
- out.write(get_bed_line(elems[0], name, strand, [coords]))
- continue
-
- # There is a transcript ID, so process line at transcript level.
- if t_id == cur_transcript_id:
- # Line is element of transcript and will be a block in the BED entry.
+ # Start new transcript.
+ cur_transcript_chrome = elems[0]
+ cur_transcript_id = t_id
+ cur_transcript_strand = strand
+ cur_transcripts_blocks = []
cur_transcripts_blocks.append(coords)
- continue
-
- #
- # Line is part of new transcript; write previous transcript and start
- # new transcript.
- #
-
- # Write previous transcript.
- if cur_transcript_id:
- # Write BED entry.
- out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
-
- # Start new transcript.
- cur_transcript_chrome = elems[0]
- cur_transcript_id = t_id
- cur_transcript_strand = strand
- cur_transcripts_blocks = []
- cur_transcripts_blocks.append(coords)
- except Exception:
+ except Exception:
+ skipped_lines += 1
+ if not first_skipped_line:
+ first_skipped_line = i + 1
+ else:
skipped_lines += 1
if not first_skipped_line:
first_skipped_line = i + 1
- else:
- skipped_lines += 1
- if not first_skipped_line:
- first_skipped_line = i + 1
- # Write last transcript.
- if cur_transcript_id:
- # Write BED entry.
- out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
- out.close()
+ # Write last transcript.
+ if cur_transcript_id:
+ # Write BED entry.
+ out.write(get_bed_line(cur_transcript_chrome, cur_transcript_id, cur_transcript_strand, cur_transcripts_blocks))
info_msg = "%i lines converted to BED. " % (i + 1 - skipped_lines)
if skipped_lines > 0:
info_msg += "Skipped %d blank/comment/invalid lines starting with line #%d." % (skipped_lines, first_skipped_line)