diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index f7ea64b8e65..de6ee4e31f5 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -88,6 +88,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + @@ -645,6 +675,8 @@ + + diff --git a/lib/galaxy/datatypes/converters/fastqbz2_to_fastq.xml b/lib/galaxy/datatypes/converters/fastqbz2_to_fastq.xml new file mode 100644 index 00000000000..432c3614495 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqbz2_to_fastq.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqcssangerbz2_to_fastqcssanger.xml b/lib/galaxy/datatypes/converters/fastqcssangerbz2_to_fastqcssanger.xml new file mode 100644 index 00000000000..c617dd25b9e --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqcssangerbz2_to_fastqcssanger.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml b/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml new file mode 100644 index 00000000000..d047195abb0 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml b/lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml new file mode 100644 index 00000000000..62adae69ec8 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqilluminabz2_to_fastqillumina.xml b/lib/galaxy/datatypes/converters/fastqilluminabz2_to_fastqillumina.xml new file mode 100644 index 00000000000..23e8bc5ed4c --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqilluminabz2_to_fastqillumina.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml b/lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml new file mode 100644 index 00000000000..ab2252e8342 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqsangerbz2_to_fastqsanger.xml b/lib/galaxy/datatypes/converters/fastqsangerbz2_to_fastqsanger.xml new file mode 100644 index 00000000000..1f8e9dde17b --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqsangerbz2_to_fastqsanger.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml b/lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml new file mode 100644 index 00000000000..8c9ed6c30a2 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqsolexabz2_to_fastqsolexa.xml b/lib/galaxy/datatypes/converters/fastqsolexabz2_to_fastqsolexa.xml new file mode 100644 index 00000000000..ae35f1c189d --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqsolexabz2_to_fastqsolexa.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml b/lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml new file mode 100644 index 00000000000..3b929fbe6f4 --- /dev/null +++ b/lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml @@ -0,0 +1,11 @@ + diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index d584ee2bced..5c64f34fde5 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -2,6 +2,7 @@ Sequence classes """ +import bz2 import gzip import json import logging @@ -15,10 +16,14 @@ import bx.align.maf from galaxy import util from galaxy.datatypes import metadata +from galaxy.datatypes.binary import Binary from galaxy.datatypes.metadata import MetadataElement from galaxy.datatypes.sniff import get_headers from galaxy.util import nice_size -from galaxy.util.checkers import is_gzip +from galaxy.util.checkers import ( + is_bz2, + is_gzip +) from galaxy.util.image_util import check_image_type from . import data @@ -553,8 +558,8 @@ class csFasta( Sequence ): return Sequence.set_meta( self, dataset, **kwd ) -class Fastq ( Sequence ): - """Class representing a generic FASTQ sequence""" +class BaseFastq ( Sequence ): + """Base class for FastQ sequences""" edam_format = "format_1930" file_ext = "fastq" @@ -571,24 +576,35 @@ class Fastq ( Sequence ): data_lines = 0 sequences = 0 seq_counter = 0 # blocks should be 4 lines long - for line in open( dataset.file_name ): - line = line.strip() - if line and line.startswith( '#' ) and not data_lines: - # We don't count comment lines for sequence data types - continue - seq_counter += 1 - data_lines += 1 - if line and line.startswith( '@' ): - if seq_counter >= 4: - # count previous block - # blocks should be 4 lines long - sequences += 1 - seq_counter = 1 - if seq_counter >= 4: - # count final block - sequences += 1 - dataset.metadata.data_lines = data_lines - dataset.metadata.sequences = sequences + compressed_gzip = is_gzip(dataset.file_name) + compressed_bzip2 = is_bz2(dataset.file_name) + try: + if compressed_gzip: + in_file = gzip.GzipFile(dataset.file_name) + elif compressed_bzip2: + in_file = bz2.BZ2File(dataset.file_name) + else: + in_file = open(dataset.file_name) + for line in in_file: + line = line.strip() + if line and line.startswith( '#' ) and not data_lines: + # We don't count comment lines for sequence data types + continue + seq_counter += 1 + data_lines += 1 + if line and line.startswith( '@' ): + if seq_counter >= 4: + # count previous block + # blocks should be 4 lines long + sequences += 1 + seq_counter = 1 + if seq_counter >= 4: + # count final block + sequences += 1 + dataset.metadata.data_lines = data_lines + dataset.metadata.sequences = sequences + finally: + in_file.close() def sniff( self, filename ): """ @@ -606,6 +622,9 @@ class Fastq ( Sequence ): >>> Fastq().sniff( fname ) True """ + compressed = is_gzip(filename) or is_bz2(filename) + if compressed and not isinstance(self, Binary): + return False headers = get_headers( filename, None ) bases_regexp = re.compile( "^[NGTAC]*" ) # check that first block looks like a fastq block @@ -668,6 +687,12 @@ class Fastq ( Sequence ): process_split_file = staticmethod(process_split_file) +class Fastq( BaseFastq ): + """Class representing a generic FASTQ sequence""" + edam_format = "format_1930" + file_ext = "fastq" + + class FastqSanger( Fastq ): """Class representing a FASTQ sequence ( the Sanger variant )""" edam_format = "format_1932" @@ -691,6 +716,94 @@ class FastqCSSanger( Fastq ): file_ext = "fastqcssanger" +class FastqGz ( BaseFastq, Binary ): + """Class representing a generic compressed FASTQ sequence""" + edam_format = "format_1930" + file_ext = "fastq.gz" + + +Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz) + + +class FastqSangerGz( FastqGz ): + """Class representing a compressed FASTQ sequence ( the Sanger variant )""" + edam_format = "format_1932" + file_ext = "fastqsanger.gz" + + +Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz) + + +class FastqSolexaGz( FastqGz ): + """Class representing a compressed FASTQ sequence ( the Solexa variant )""" + edam_format = "format_1933" + file_ext = "fastqsolexa.gz" + + +Binary.register_sniffable_binary_format("fastqsolexa.gz", "fastqsolexa.gz", FastqSolexaGz) + + +class FastqIlluminaGz( FastqGz ): + """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )""" + edam_format = "format_1931" + file_ext = "fastqillumina.gz" + + +Binary.register_sniffable_binary_format("fastqillumina.gz", "fastqillumina.gz", FastqIlluminaGz) + + +class FastqCSSangerGz( FastqGz ): + """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )""" + file_ext = "fastqcssanger.gz" + + +Binary.register_sniffable_binary_format("fastqcssanger.gz", "fastqcssanger.gz", FastqCSSangerGz) + + +class FastqBz2 ( BaseFastq, Binary ): + """Class representing a generic compressed FASTQ sequence""" + edam_format = "format_1930" + file_ext = "fastq.gz" + + +Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz) + + +class FastqSangerBz2( FastqBz2 ): + """Class representing a compressed FASTQ sequence ( the Sanger variant )""" + edam_format = "format_1932" + file_ext = "fastqsanger.bz2" + + +Binary.register_sniffable_binary_format("fastqsanger.bz2", "fastqsanger.bz2", FastqSangerBz2) + + +class FastqSolexaBz2( FastqBz2 ): + """Class representing a compressed FASTQ sequence ( the Solexa variant )""" + edam_format = "format_1933" + file_ext = "fastqsolexa.bz2" + + +Binary.register_sniffable_binary_format("fastqsolexa.bz2", "fastqsolexa.bz2", FastqSolexaBz2) + + +class FastqIlluminaBz2( FastqBz2 ): + """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )""" + edam_format = "format_1931" + file_ext = "fastqillumina.bz2" + + +Binary.register_sniffable_binary_format("fastqillumina.bz2", "fastqillumina.bz2", FastqIlluminaBz2) + + +class FastqCSSangerBz2( FastqBz2 ): + """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )""" + file_ext = "fastqcssanger.bz2" + + +Binary.register_sniffable_binary_format("fastqcssanger.bz2", "fastqcssanger.bz2", FastqCSSangerBz2) + + class Maf( Alignment ): """Class describing a Maf alignment""" edam_format = "format_3008" diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 7188bafdba5..f012ff420c1 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -4,6 +4,7 @@ File format detector from __future__ import absolute_import import gzip +import bz2 import logging import os import re @@ -18,7 +19,12 @@ from six import text_type from galaxy import util from galaxy.util import multi_byte from galaxy.util import unicodify -from galaxy.util.checkers import check_binary, check_html, is_gzip +from galaxy.util.checkers import ( + check_binary, + check_html, + is_bz2, + is_gzip +) from galaxy.datatypes.binary import Binary log = logging.getLogger(__name__) @@ -198,15 +204,26 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ): [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']] """ headers = [] - for idx, line in enumerate(open(fname)): - line = line.rstrip('\n\r') - if is_multi_byte: - # TODO: fix this - sep is never found in line - line = unicodify( line, 'utf-8' ) - sep = sep.encode( 'utf-8' ) - headers.append( line.split(sep) ) - if idx == count: - break + compressed_gzip = is_gzip(fname) + compressed_bzip2 = is_bz2(fname) + try: + if compressed_gzip: + in_file = gzip.GzipFile(fname, 'r') + elif compressed_bzip2: + in_file = bz2.BZ2File(fname, 'r') + else: + in_file = open(fname, 'rt') + for idx, line in enumerate(in_file): + line = line.rstrip('\n\r') + if is_multi_byte: + # TODO: fix this - sep is never found in line + line = unicodify( line, 'utf-8' ) + sep = sep.encode( 'utf-8' ) + headers.append( line.split(sep) ) + if idx == count: + break + finally: + in_file.close() return headers @@ -475,7 +492,7 @@ def handle_uploaded_dataset_file( filename, datatypes_registry, ext='auto', is_m AUTO_DETECT_EXTENSIONS = [ 'auto' ] # should 'data' also cause auto detect? DECOMPRESSION_FUNCTIONS = dict( gzip=gzip.GzipFile ) COMPRESSION_CHECK_FUNCTIONS = [ ( 'gzip', is_gzip ) ] -COMPRESSION_DATATYPES = dict( gzip=[ 'bam' ] ) +COMPRESSION_DATATYPES = dict( gzip=[ 'bam', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz', 'fastq.bz2', 'fastqsanger.bz2', 'fastqillumina.bz2', 'fastqsolexa.bz2', 'fastqcssanger.bz2' ] ) COMPRESSED_EXTENSIONS = [] for exts in COMPRESSION_DATATYPES.values(): COMPRESSED_EXTENSIONS.extend( exts ) diff --git a/test-data/1.fastqsanger.bz2 b/test-data/1.fastqsanger.bz2 new file mode 100644 index 00000000000..78d42ab6194 Binary files /dev/null and b/test-data/1.fastqsanger.bz2 differ diff --git a/test-data/1.fastqsanger.gz b/test-data/1.fastqsanger.gz new file mode 100644 index 00000000000..56dbcd413a1 Binary files /dev/null and b/test-data/1.fastqsanger.gz differ diff --git a/test/functional/tools/compressed_bz2_fastq_conversion.xml b/test/functional/tools/compressed_bz2_fastq_conversion.xml new file mode 100644 index 00000000000..f020715fde8 --- /dev/null +++ b/test/functional/tools/compressed_bz2_fastq_conversion.xml @@ -0,0 +1,19 @@ + + + cat '$input1' > '$out_file1' + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/compressed_bz2_fastq_no_conversion.xml b/test/functional/tools/compressed_bz2_fastq_no_conversion.xml new file mode 100644 index 00000000000..0d46ad0e39c --- /dev/null +++ b/test/functional/tools/compressed_bz2_fastq_no_conversion.xml @@ -0,0 +1,19 @@ + + + cat '$input1' > '$out_file1' + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/compressed_gz_fastq_conversion.xml b/test/functional/tools/compressed_gz_fastq_conversion.xml new file mode 100644 index 00000000000..af08f5ef375 --- /dev/null +++ b/test/functional/tools/compressed_gz_fastq_conversion.xml @@ -0,0 +1,19 @@ + + + cat '$input1' > '$out_file1' + + + + + + + + + + + + + + + + diff --git a/test/functional/tools/compressed_gz_fastq_no_conversion.xml b/test/functional/tools/compressed_gz_fastq_no_conversion.xml new file mode 100644 index 00000000000..2d2f764b44b --- /dev/null +++ b/test/functional/tools/compressed_gz_fastq_no_conversion.xml @@ -0,0 +1,19 @@ + + + cat '$input1' > '$out_file1' + + + + + + + + + + + + + + + +