Merge pull request #3145 from mvdbeek/fastqgz

Support for gzipped fastq formats

config/datatypes_conf.xml.sample

@@ -88,6 +88,36 @@
     <datatype extension="fastqillumina" type="galaxy.datatypes.sequence:FastqIllumina" display_in_upload="true">
         <converter file="fastq_to_fqtoc.xml" target_datatype="fqtoc"/>
     </datatype>
+    <datatype extension="fastq.gz" type="galaxy.datatypes.sequence:FastqGz" display_in_upload="true">
+        <converter file="fastqgz_to_fastq.xml" target_datatype="fastq"/>
+    </datatype>
+    <datatype extension="fastqsanger.gz" type="galaxy.datatypes.sequence:FastqSangerGz" display_in_upload="true">
+        <converter file="fastqsangergz_to_fastqsanger.xml" target_datatype="fastqsanger"/>
+    </datatype>
+    <datatype extension="fastqsolexa.gz" type="galaxy.datatypes.sequence:FastqSolexaGz" display_in_upload="true">
+        <converter file="fastqsolexagz_to_fastqsolexa.xml" target_datatype="fastqsolexa"/>
+    </datatype>
+    <datatype extension="fastqcssanger.gz" type="galaxy.datatypes.sequence:FastqCSSangerGz" display_in_upload="true">
+        <converter file="fastqcssangergz_to_fastqcssanger.xml" target_datatype="fastqcssanger"/>
+    </datatype>
+    <datatype extension="fastqillumina.gz" type="galaxy.datatypes.sequence:FastqIlluminaGz" display_in_upload="true">
+        <converter file="fastqilluminagz_to_fastqillumina.xml" target_datatype="fastqillumina"/>
+    </datatype>
+    <datatype extension="fastq.bz2" type="galaxy.datatypes.sequence:FastqBz2" display_in_upload="true">
+        <converter file="fastqbz2_to_fastq.xml" target_datatype="fastq"/>
+    </datatype>
+    <datatype extension="fastqsanger.bz2" type="galaxy.datatypes.sequence:FastqSangerBz2" display_in_upload="true">
+        <converter file="fastqsangerbz2_to_fastqsanger.xml" target_datatype="fastqsanger"/>
+    </datatype>
+    <datatype extension="fastqsolexa.bz2" type="galaxy.datatypes.sequence:FastqSolexaBz2" display_in_upload="true">
+        <converter file="fastqsolexabz2_to_fastqsolexa.xml" target_datatype="fastqsolexa"/>
+    </datatype>
+    <datatype extension="fastqcssanger.bz2" type="galaxy.datatypes.sequence:FastqCSSangerBz2" display_in_upload="true">
+        <converter file="fastqcssangerbz2_to_fastqcssanger.xml" target_datatype="fastqcssanger"/>
+    </datatype>
+    <datatype extension="fastqillumina.bz2" type="galaxy.datatypes.sequence:FastqIlluminaBz2" display_in_upload="true">
+        <converter file="fastqilluminabz2_to_fastqillumina.xml" target_datatype="fastqillumina"/>
+    </datatype>
     <datatype extension="fqtoc" type="galaxy.datatypes.sequence:SequenceSplitLocations" display_in_upload="true"/>
     <datatype extension="eland" type="galaxy.datatypes.tabular:Eland" display_in_upload="true"/>
     <datatype extension="elandmulti" type="galaxy.datatypes.tabular:ElandMulti" display_in_upload="true"/>
@@ -645,6 +675,8 @@
     <!-- TODO: see molecules.py <sniffer type="galaxy.datatypes.molecules:SMILES"/>-->
     <sniffer type="galaxy.datatypes.sequence:Fasta"/>
     <sniffer type="galaxy.datatypes.sequence:Fastq"/>
+    <sniffer type="galaxy.datatypes.sequence:FastqGz"/>
+    <sniffer type="galaxy.datatypes.sequence:FastqBz2"/>
     <sniffer type="galaxy.datatypes.interval:Wiggle"/>
     <sniffer type="galaxy.datatypes.text:Html"/>
     <sniffer type="galaxy.datatypes.images:Pdf"/>

lib/galaxy/datatypes/converters/fastqbz2_to_fastq.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqbz2_to_fastq" name="Convert fastq.bz2 files to fastq" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastq.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastq" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqcssangerbz2_to_fastqcssanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqcssangerbz2_to_fastqcssanger" name="Convert fastqcssanger.bz2 files to fastqcssanger" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqcssanger.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqcssanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqcssangergz_to_fastqcssanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqcssangergz_to_fastqcssanger" name="Convert fastqcssanger.gz files to fastqcssanger" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqcssanger.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqcssanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqgz_to_fastq.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqgz_to_fastq" name="Convert fastq.gz files to fastq" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastq.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastq" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqilluminabz2_to_fastqillumina.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqilluminabz2_to_fastqillumina" name="Convert fastqillumina.bz2 files to fastqillumina" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqillumina.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqillumina" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqilluminagz_to_fastqillumina.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqilluminagz_to_fastqillumina" name="Convert fastqillumina.gz files to fastqillumina" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqillumina.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqillumina" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsangerbz2_to_fastqsanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsangerbz2_to_fastqsanger" name="Convert fastqsanger.bz2 files to fastqsanger" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsanger.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsangergz_to_fastqsanger.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsangergz_to_fastqsanger" name="Convert fastqsanger.gz files to fastqsanger" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsanger.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsanger" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsolexabz2_to_fastqsolexa.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsolexabz2_to_fastqsolexa" name="Convert fastqsolexa.bz2 files to fastqsolexa" version="1.0.0" hidden="true">
+  <command>bzip2 -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsolexa.bz2" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsolexa" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/converters/fastqsolexagz_to_fastqsolexa.xml

@@ -0,0 +1,11 @@
+<tool id="CONVERTER_fastqsolexagz_to_fastqsolexa" name="Convert fastqsolexa.gz files to fastqsolexa" version="1.0.0" hidden="true">
+  <command>gzip -dcf '$input1' > '$output1'</command>
+  <inputs>
+    <param format="fastqsolexa.gz" name="input1" type="data" label="Choose FASTQ file"/>
+  </inputs>
+  <outputs>
+    <data format="fastqsolexa" name="output1"/>
+  </outputs>
+  <help>
+  </help>
+</tool>

lib/galaxy/datatypes/sequence.py

@@ -2,6 +2,7 @@
 Sequence classes
 """
 
+import bz2
 import gzip
 import json
 import logging
@@ -15,10 +16,14 @@ import bx.align.maf
 
 from galaxy import util
 from galaxy.datatypes import metadata
+from galaxy.datatypes.binary import Binary
 from galaxy.datatypes.metadata import MetadataElement
 from galaxy.datatypes.sniff import get_headers
 from galaxy.util import nice_size
-from galaxy.util.checkers import is_gzip
+from galaxy.util.checkers import (
+    is_bz2,
+    is_gzip
+)
 from galaxy.util.image_util import check_image_type
 
 from . import data
@@ -553,8 +558,8 @@ class csFasta( Sequence ):
         return Sequence.set_meta( self, dataset, **kwd )
 
 
-class Fastq ( Sequence ):
-    """Class representing a generic FASTQ sequence"""
+class BaseFastq ( Sequence ):
+    """Base class for FastQ sequences"""
     edam_format = "format_1930"
     file_ext = "fastq"
 
@@ -571,24 +576,35 @@ class Fastq ( Sequence ):
         data_lines = 0
         sequences = 0
         seq_counter = 0     # blocks should be 4 lines long
-        for line in open( dataset.file_name ):
-            line = line.strip()
-            if line and line.startswith( '#' ) and not data_lines:
-                # We don't count comment lines for sequence data types
-                continue
-            seq_counter += 1
-            data_lines += 1
-            if line and line.startswith( '@' ):
-                if seq_counter >= 4:
-                    # count previous block
-                    # blocks should be 4 lines long
-                    sequences += 1
-                    seq_counter = 1
-        if seq_counter >= 4:
-            # count final block
-            sequences += 1
-        dataset.metadata.data_lines = data_lines
-        dataset.metadata.sequences = sequences
+        compressed_gzip = is_gzip(dataset.file_name)
+        compressed_bzip2 = is_bz2(dataset.file_name)
+        try:
+            if compressed_gzip:
+                in_file = gzip.GzipFile(dataset.file_name)
+            elif compressed_bzip2:
+                in_file = bz2.BZ2File(dataset.file_name)
+            else:
+                in_file = open(dataset.file_name)
+            for line in in_file:
+                line = line.strip()
+                if line and line.startswith( '#' ) and not data_lines:
+                    # We don't count comment lines for sequence data types
+                    continue
+                seq_counter += 1
+                data_lines += 1
+                if line and line.startswith( '@' ):
+                    if seq_counter >= 4:
+                        # count previous block
+                        # blocks should be 4 lines long
+                        sequences += 1
+                        seq_counter = 1
+            if seq_counter >= 4:
+                # count final block
+                sequences += 1
+            dataset.metadata.data_lines = data_lines
+            dataset.metadata.sequences = sequences
+        finally:
+            in_file.close()
 
     def sniff( self, filename ):
         """
@@ -606,6 +622,9 @@ class Fastq ( Sequence ):
         >>> Fastq().sniff( fname )
         True
         """
+        compressed = is_gzip(filename) or is_bz2(filename)
+        if compressed and not isinstance(self, Binary):
+            return False
         headers = get_headers( filename, None )
         bases_regexp = re.compile( "^[NGTAC]*" )
         # check that first block looks like a fastq block
@@ -668,6 +687,12 @@ class Fastq ( Sequence ):
     process_split_file = staticmethod(process_split_file)
 
 
+class Fastq( BaseFastq ):
+    """Class representing a generic FASTQ sequence"""
+    edam_format = "format_1930"
+    file_ext = "fastq"
+
+
 class FastqSanger( Fastq ):
     """Class representing a FASTQ sequence ( the Sanger variant )"""
     edam_format = "format_1932"
@@ -691,6 +716,94 @@ class FastqCSSanger( Fastq ):
     file_ext = "fastqcssanger"
 
 
+class FastqGz ( BaseFastq, Binary ):
+    """Class representing a generic compressed FASTQ sequence"""
+    edam_format = "format_1930"
+    file_ext = "fastq.gz"
+
+
+Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
+
+
+class FastqSangerGz( FastqGz ):
+    """Class representing a compressed FASTQ sequence ( the Sanger variant )"""
+    edam_format = "format_1932"
+    file_ext = "fastqsanger.gz"
+
+
+Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz)
+
+
+class FastqSolexaGz( FastqGz ):
+    """Class representing a compressed FASTQ sequence ( the Solexa variant )"""
+    edam_format = "format_1933"
+    file_ext = "fastqsolexa.gz"
+
+
+Binary.register_sniffable_binary_format("fastqsolexa.gz", "fastqsolexa.gz", FastqSolexaGz)
+
+
+class FastqIlluminaGz( FastqGz ):
+    """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )"""
+    edam_format = "format_1931"
+    file_ext = "fastqillumina.gz"
+
+
+Binary.register_sniffable_binary_format("fastqillumina.gz", "fastqillumina.gz", FastqIlluminaGz)
+
+
+class FastqCSSangerGz( FastqGz ):
+    """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )"""
+    file_ext = "fastqcssanger.gz"
+
+
+Binary.register_sniffable_binary_format("fastqcssanger.gz", "fastqcssanger.gz", FastqCSSangerGz)
+
+
+class FastqBz2 ( BaseFastq, Binary ):
+    """Class representing a generic compressed FASTQ sequence"""
+    edam_format = "format_1930"
+    file_ext = "fastq.gz"
+
+
+Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz)
+
+
+class FastqSangerBz2( FastqBz2 ):
+    """Class representing a compressed FASTQ sequence ( the Sanger variant )"""
+    edam_format = "format_1932"
+    file_ext = "fastqsanger.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqsanger.bz2", "fastqsanger.bz2", FastqSangerBz2)
+
+
+class FastqSolexaBz2( FastqBz2 ):
+    """Class representing a compressed FASTQ sequence ( the Solexa variant )"""
+    edam_format = "format_1933"
+    file_ext = "fastqsolexa.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqsolexa.bz2", "fastqsolexa.bz2", FastqSolexaBz2)
+
+
+class FastqIlluminaBz2( FastqBz2 ):
+    """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )"""
+    edam_format = "format_1931"
+    file_ext = "fastqillumina.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqillumina.bz2", "fastqillumina.bz2", FastqIlluminaBz2)
+
+
+class FastqCSSangerBz2( FastqBz2 ):
+    """Class representing a Color Space compressed FASTQ sequence ( e.g a SOLiD variant )"""
+    file_ext = "fastqcssanger.bz2"
+
+
+Binary.register_sniffable_binary_format("fastqcssanger.bz2", "fastqcssanger.bz2", FastqCSSangerBz2)
+
+
 class Maf( Alignment ):
     """Class describing a Maf alignment"""
     edam_format = "format_3008"

lib/galaxy/datatypes/sniff.py

@@ -4,6 +4,7 @@ File format detector
 from __future__ import absolute_import
 
 import gzip
+import bz2
 import logging
 import os
 import re
@@ -18,7 +19,12 @@ from six import text_type
 from galaxy import util
 from galaxy.util import multi_byte
 from galaxy.util import unicodify
-from galaxy.util.checkers import check_binary, check_html, is_gzip
+from galaxy.util.checkers import (
+    check_binary,
+    check_html,
+    is_bz2,
+    is_gzip
+)
 from galaxy.datatypes.binary import Binary
 
 log = logging.getLogger(__name__)
@@ -198,15 +204,26 @@ def get_headers( fname, sep, count=60, is_multi_byte=False ):
     [['chr7', '127475281', '127491632', 'NM_000230', '0', '+', '127486022', '127488767', '0', '3', '29,172,3225,', '0,10713,13126,'], ['chr7', '127486011', '127488900', 'D49487', '0', '+', '127486022', '127488767', '0', '2', '155,490,', '0,2399']]
     """
     headers = []
-    for idx, line in enumerate(open(fname)):
-        line = line.rstrip('\n\r')
-        if is_multi_byte:
-            # TODO: fix this - sep is never found in line
-            line = unicodify( line, 'utf-8' )
-            sep = sep.encode( 'utf-8' )
-        headers.append( line.split(sep) )
-        if idx == count:
-            break
+    compressed_gzip = is_gzip(fname)
+    compressed_bzip2 = is_bz2(fname)
+    try:
+        if compressed_gzip:
+            in_file = gzip.GzipFile(fname, 'r')
+        elif compressed_bzip2:
+            in_file = bz2.BZ2File(fname, 'r')
+        else:
+            in_file = open(fname, 'rt')
+        for idx, line in enumerate(in_file):
+            line = line.rstrip('\n\r')
+            if is_multi_byte:
+                # TODO: fix this - sep is never found in line
+                line = unicodify( line, 'utf-8' )
+                sep = sep.encode( 'utf-8' )
+            headers.append( line.split(sep) )
+            if idx == count:
+                break
+    finally:
+        in_file.close()
     return headers
 
 
@@ -475,7 +492,7 @@ def handle_uploaded_dataset_file( filename, datatypes_registry, ext='auto', is_m
 AUTO_DETECT_EXTENSIONS = [ 'auto' ]  # should 'data' also cause auto detect?
 DECOMPRESSION_FUNCTIONS = dict( gzip=gzip.GzipFile )
 COMPRESSION_CHECK_FUNCTIONS = [ ( 'gzip', is_gzip ) ]
-COMPRESSION_DATATYPES = dict( gzip=[ 'bam' ] )
+COMPRESSION_DATATYPES = dict( gzip=[ 'bam', 'fastq.gz', 'fastqsanger.gz', 'fastqillumina.gz', 'fastqsolexa.gz', 'fastqcssanger.gz', 'fastq.bz2', 'fastqsanger.bz2', 'fastqillumina.bz2', 'fastqsolexa.bz2', 'fastqcssanger.bz2' ] )
 COMPRESSED_EXTENSIONS = []
 for exts in COMPRESSION_DATATYPES.values():
     COMPRESSED_EXTENSIONS.extend( exts )

test/functional/tools/compressed_bz2_fastq_conversion.xml

@@ -0,0 +1,19 @@
+<tool id="compressed_bz2_fastq_conversion" name="Confirm fastq.bz2 is converted to fastq on demand">
+    <command>
+        cat '$input1' > '$out_file1'
+    </command>
+    <inputs>
+        <param name="input1" type="data" format="fastq" label="Concatenate Dataset"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="fastq"/>
+    </outputs>
+    <tests>
+        <test>
+            <param name="input1" value="1.fastqsanger.bz2"/>
+            <output name="out_file1" file="1.fastqsanger"/>
+        </test>
+    </tests>
+    <help>
+    </help>
+</tool>

test/functional/tools/compressed_bz2_fastq_no_conversion.xml

@@ -0,0 +1,19 @@
+<tool id="compressed_bz2_fastq_cat" name="Confirm fastq.bz2 is not uncompressed">
+    <command>
+        cat '$input1' > '$out_file1'
+    </command>
+    <inputs>
+        <param name="input1" type="data" format="fastq.bz2" label="Concatenate Dataset"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="fastq.bz2"/>
+    </outputs>
+    <tests>
+        <test>
+            <param name="input1" value="1.fastqsanger.bz2"/>
+            <output name="out_file1" file="1.fastqsanger.bz2" ftype="fastq.bz2"/>
+        </test>
+    </tests>
+    <help>
+    </help>
+</tool>

test/functional/tools/compressed_gz_fastq_conversion.xml

@@ -0,0 +1,19 @@
+<tool id="compressed_gz_fastq_conversion" name="Confirm fastq.gz is converted to fastq on demand">
+    <command>
+        cat '$input1' > '$out_file1'
+    </command>
+    <inputs>
+        <param name="input1" type="data" format="fastq" label="Concatenate Dataset"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="fastq"/>
+    </outputs>
+    <tests>
+        <test>
+            <param name="input1" value="1.fastqsanger.gz"/>
+            <output name="out_file1" file="1.fastqsanger"/>
+        </test>
+    </tests>
+    <help>
+    </help>
+</tool>

test/functional/tools/compressed_gz_fastq_no_conversion.xml

@@ -0,0 +1,19 @@
+<tool id="compressed_gz_fastq_cat" name="Confirm fastq.gz is not uncompressed">
+    <command>
+        cat '$input1' > '$out_file1'
+    </command>
+    <inputs>
+        <param name="input1" type="data" format="fastq.gz" label="Concatenate Dataset"/>
+    </inputs>
+    <outputs>
+        <data name="out_file1" format="fastq.gz"/>
+    </outputs>
+    <tests>
+        <test>
+            <param name="input1" value="1.fastqsanger.gz"/>
+            <output name="out_file1" file="1.fastqsanger.gz" ftype="fastq.gz"/>
+        </test>
+    </tests>
+    <help>
+    </help>
+</tool>