From 782d7db6beaeafa0cf494e4d036f4e65ef2eacbe Mon Sep 17 00:00:00 2001 From: Wen-Yu Chung Date: Fri, 6 Jun 2008 18:52:11 +0000 Subject: [PATCH] Update fastq format. Now we only support FastqSolexa variants. If the quality scores are presented as characters, the integer values are obtained by their ascii code subtract 64. --- datatype_converters_conf.xml.sample | 4 +- .../converters/fastq_to_fasta_converter.xml | 2 +- .../converters/fastq_to_qual_converter.py | 10 +-- .../converters/fastq_to_qual_converter.xml | 2 +- lib/galaxy/datatypes/registry.py | 4 +- lib/galaxy/datatypes/sequence.py | 82 ++++--------------- tools/data_source/upload.xml | 35 ++++---- tools/metag_tools/fastq_to_fasta_qual.py | 8 +- tools/metag_tools/fastq_to_fasta_qual.xml | 54 +++++------- universe_wsgi.ini.sample | 1 - 10 files changed, 70 insertions(+), 132 deletions(-) diff --git a/datatype_converters_conf.xml.sample b/datatype_converters_conf.xml.sample index 0df68f95cf9..2ef823b5268 100644 --- a/datatype_converters_conf.xml.sample +++ b/datatype_converters_conf.xml.sample @@ -2,8 +2,8 @@ - - + + diff --git a/lib/galaxy/datatypes/converters/fastq_to_fasta_converter.xml b/lib/galaxy/datatypes/converters/fastq_to_fasta_converter.xml index fa42db08a4b..4bfe43b9fde 100644 --- a/lib/galaxy/datatypes/converters/fastq_to_fasta_converter.xml +++ b/lib/galaxy/datatypes/converters/fastq_to_fasta_converter.xml @@ -2,7 +2,7 @@ converts Fastq file to Fasta format fastq_to_fasta_converter.py $input $output - + diff --git a/lib/galaxy/datatypes/converters/fastq_to_qual_converter.py b/lib/galaxy/datatypes/converters/fastq_to_qual_converter.py index c400c4ae618..24f20858de8 100644 --- a/lib/galaxy/datatypes/converters/fastq_to_qual_converter.py +++ b/lib/galaxy/datatypes/converters/fastq_to_qual_converter.py @@ -28,7 +28,7 @@ def __main__(): datatype = sys.argv[3] qual_title_startswith = '' seq_title_startswith = '' - default_coding_value = 33 + default_coding_value = 64 fastq_block_lines = 0 for i, line in enumerate( file( infile_name ) ): @@ -75,10 +75,8 @@ def __main__(): if fastq_integer: # digits qual = line - else: # ascii - if datatype == 'fastqsolexa': - outfile_score.close() - stop_err( "This tool currently only works with the fastq solexa variant if the socres are integers, not ascii." ) + else: + # ascii quality_score_length = len( line ) if quality_score_length == read_length + 1: quality_score_startswith = ord( line[0:1] ) @@ -88,7 +86,7 @@ def __main__(): else: stop_err( 'Invalid fastq format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d ).' % ( i + 1, quality_score_length, read_length ) ) for j, char in enumerate( line ): - score = ord( char ) - quality_score_startswith # 33 + score = ord( char ) - quality_score_startswith # 64 qual = "%s%s " % ( qual, str( score ) ) outfile_score.write( '%s\n' % qual ) diff --git a/lib/galaxy/datatypes/converters/fastq_to_qual_converter.xml b/lib/galaxy/datatypes/converters/fastq_to_qual_converter.xml index e2d340af6b5..6e485085e29 100644 --- a/lib/galaxy/datatypes/converters/fastq_to_qual_converter.xml +++ b/lib/galaxy/datatypes/converters/fastq_to_qual_converter.xml @@ -1,7 +1,7 @@ fastq_to_qual_converter.py $input1 $output1 $input1.extension - + diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 2029b9da025..82559c74408 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -67,7 +67,7 @@ class Registry( object ): 'binseq.zip' : images.Binseq(), 'customtrack' : interval.CustomTrack(), 'fasta' : sequence.Fasta(), - 'fastq' : sequence.Fastq(), + 'fastqsolexa' : sequence.FastqSolexa(), 'gff' : interval.Gff(), 'gff3' : interval.Gff3(), 'interval' : interval.Interval(), @@ -89,7 +89,7 @@ class Registry( object ): 'binseq.zip' : 'application/zip', 'customtrack' : 'text/plain', 'fasta' : 'text/plain', - 'fastq' : 'text/plain', + 'fastqsolexa' : 'text/plain', 'gff' : 'text/plain', 'gff3' : 'text/plain', 'interval' : 'text/plain', diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index a8b130fbc8c..a79b2760865 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -89,61 +89,6 @@ class Fasta( Sequence ): except: return False -class Fastq( Sequence ): - """Class representing a FASTQ sequence ( the Sanger/Standard variant )""" - file_ext = "fastq" - - def set_peek( self, dataset ): - Sequence.set_peek( self, dataset ) - count = 0 - size = 0 - bases_regexp = re.compile("^[NGTAC]*$") - for line in file( dataset.file_name ): - if line and line.startswith( ">" ): - count += 1 - elif bases_regexp.match( line ): - line = line.strip() - size += len( line ) - if count == 1: - dataset.blurb = '%d bases' % size - else: - dataset.blurb = '%d sequences' % count - - def sniff(self, filename): - """ - Determines whether the file is in fastq format ( the Sanger/Standard variant ) - For details, see http://maq.sourceforge.net/fastq.shtml - - Note: There are two kinds of FASTQ files, known as "Sanger" (sometimes called "Standard") and Solexa - These differ in the representation of the quality scores - - >>> fname = get_test_fname( '1.fastq' ) - >>> Fastq().sniff( fname ) - True - >>> fname = get_test_fname( '1.fastqsolexa' ) - >>> Fastq().sniff( fname ) - False - """ - headers = get_headers( filename, None ) - bases_regexp = re.compile( "^[NGTAC]*$" ) - try: - if len( headers ) >= 4 and headers[0][0] and headers[0][0][0] == "@" and headers[2][0] and headers[2][0][0] == "+" and headers[1][0] and headers[3][0]: - # Check the sequence line, make sure it contains only G/C/A/T/N - if not bases_regexp.match( headers[1][0] ): - return False - # The quality score line - qscore = headers[3][0] - # In Standard/Sanger format, the quality score is a single string, whose length should be equal to the length of the sequence - if len( qscore ) != len( headers[1][0] ): - return False - #Check the quality score values - in Sanger/Standard these should be ASCII characters between "!" (0x21) and "~" (0x7E) - for x in qscore: - if ord( x ) < 0x21 or ord( x ) > 0x7e: - return False - return True - return False - except: - return False class FastqSolexa( Sequence ): """Class representing a FASTQ sequence ( the Solexa variant )""" @@ -154,7 +99,7 @@ class FastqSolexa( Sequence ): count = size = 0 bases_regexp = re.compile("^[NGTAC]*$") for line in file( dataset.file_name ): - if line and line[0] == ">": + if line and line[0] == "@": count += 1 elif bases_regexp.match(line): line = line.strip() @@ -174,7 +119,7 @@ class FastqSolexa( Sequence ): >>> fname = get_test_fname( '1.fastq' ) >>> FastqSolexa().sniff( fname ) - False + True >>> fname = get_test_fname( '1.fastqsolexa' ) >>> FastqSolexa().sniff( fname ) True @@ -186,17 +131,20 @@ class FastqSolexa( Sequence ): # Check the sequence line, make sure it contains only G/C/A/T/N if not bases_regexp.match( headers[1][0] ): return False - qscore = headers[3] - # In Solexa format, the quality score is a list of numbers, whose length should be equal to the length of the sequence - if len( qscore ) != len( headers[1][0] ): - return False - # Check the quality score values - in Solexa/FASTQ these should be valid decimal numbers - # (if "x" is not a valid number, "int" will raise an exception) - for x in qscore: - try: - check = int( x ) - except: + + # Check quality score: integer or ascii char. + try: + check = int(headers[3][0]) + qscore_int = True + except: + qscore_int = False + + if qscore_int: + if len( headers[3] ) != len( headers[1][0] ): return False + else: + if len( headers[3][0] ) != len( headers[1][0] ): + return False return True return False except: diff --git a/tools/data_source/upload.xml b/tools/data_source/upload.xml index 852f0d28bb8..ec735fc368b 100644 --- a/tools/data_source/upload.xml +++ b/tools/data_source/upload.xml @@ -89,23 +89,30 @@ A sequence in FASTA format consists of a single-line description, followed by li ----- -**Fastq** +**FastqSolexa** -Fastq format stores sequences and Phred qualities in a single file. We define Fastq as the Sanger/Standard variant:: +Fastq format stores sequences and quality scores in a single file. We define FastqSolexa as the Illumina (Solexa) variant:: - @EAS54_6_R1_2_1_413_324 - CCCTTCTTGTCTTCAGCGTTTCTCC - + - ;;3;;;;;;;;;;;;7;;;;;;;88 - @EAS54_6_R1_2_1_540_792 - TTGGCAGGCCAAGGCCGATGGATCA - + - ;;;;;;;;;;;7;;;;;-;;;3;83 - @EAS54_6_R1_2_1_443_348 - GTTGCTTCTGGCGTGGGTGGGGGGG - +EAS54_6_R1_2_1_443_348 - ;;;;;;;;;;;9;7;;.7;393333 + @seq1 + GACAGCTTGGTTTTTAGTGAGTTGTTCCTTTCTTT + +seq1 + hhhhhhhhhhhhhhhhhhhhhhhhhhPW@hhhhhh + @seq2 + GCAATGACGGCAGCAATAAACTCAACAGGTGCTGG + +seq2 + hhhhhhhhhhhhhhYhhahhhhWhAhFhSIJGChO + +Or:: + @seq1 + GAATTGATCAGGACATAGGACAACTGTAGGCACCAT + +seq1 + 40 40 40 40 35 40 40 40 25 40 40 26 40 9 33 11 40 35 17 40 40 33 40 7 9 15 3 22 15 30 11 17 9 4 9 4 + @seq2 + GAGTTCTCGTCGCCTGTAGGCACCATCAATCGTATG + +seq2 + 40 15 40 17 6 36 40 40 40 25 40 9 35 33 40 14 14 18 15 17 19 28 31 4 24 18 27 14 15 18 2 8 12 8 11 9 + ----- **Gff** diff --git a/tools/metag_tools/fastq_to_fasta_qual.py b/tools/metag_tools/fastq_to_fasta_qual.py index 843c17eb7a0..e61d5617fe2 100644 --- a/tools/metag_tools/fastq_to_fasta_qual.py +++ b/tools/metag_tools/fastq_to_fasta_qual.py @@ -36,7 +36,7 @@ def __main__(): datatype = sys.argv[4] seq_title_startswith = '' qual_title_startswith = '' - default_coding_value = 33 + default_coding_value = 64 fastq_block_lines = 0 for i, line in enumerate( file( infile_name ) ): @@ -92,10 +92,6 @@ def __main__(): # digits qual = line else: - if datatype == 'fastqsolexa': - outfile_seq.close() - outfile_score.close() - stop_err( "This tool currently only works with the fastq solexa variant if the socres are integers, not ascii." ) # ascii quality_score_length = len( line ) if quality_score_length == read_length + 1: @@ -107,7 +103,7 @@ def __main__(): else: stop_err( 'Invalid fastq format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d ).' % ( i + 1, quality_score_length, read_length ) ) for j, char in enumerate( line ): - score = ord( char ) - qual_score_startswith # 33 + score = ord( char ) - qual_score_startswith # 64 qual = "%s%s " % ( qual, str( score ) ) outfile_score.write( '%s\n' % qual ) diff --git a/tools/metag_tools/fastq_to_fasta_qual.xml b/tools/metag_tools/fastq_to_fasta_qual.xml index bd9a7385994..9063de09ae2 100644 --- a/tools/metag_tools/fastq_to_fasta_qual.xml +++ b/tools/metag_tools/fastq_to_fasta_qual.xml @@ -2,7 +2,7 @@ extracts sequences and quality scores from FASTQ data fastq_to_fasta_qual.py $input1 $output1 $output2 $input1.extension - + @@ -11,11 +11,11 @@ - + - + @@ -23,13 +23,13 @@ .. class:: warningmark -IMPORTANT: With the Fastq Solexa variant, this tool currently only works with data where the quality scores are integers, ASCII quality scores are not supported. +IMPORTANT: This tool currently only support data where the quality scores are integers or ASCII quality scores with base 64. ----- **What it does** -This tool extracts sequences and quality scores from FASTQ data ( both Sanger/Standard and Solexa variants ), producing a FASTA dataset and a QUAL dataset. With the Solexa variant, this tool currently only works with data where the quality scores are integers, ASCII quality scores are not supported. +This tool extracts sequences and quality scores from FASTQ data ( Solexa variants ), producing a FASTA dataset and a QUAL dataset. ----- @@ -37,38 +37,28 @@ This tool extracts sequences and quality scores from FASTQ data ( both Sanger/St - Converting the following Sanger/Standard fastq data:: - @EAS54_6_R1_2_1_413_324 - CCCTTCTTGTCTTCAGCGTTTCTCC - + - ;;3;;;;;;;;;;;;7;;;;;;;88 - @EAS54_6_R1_2_1_540_792 - TTGGCAGGCCAAGGCCGATGGATCA - + - ;;;;;;;;;;;7;;;;;-;;;3;83 - @EAS54_6_R1_2_1_443_348 - GTTGCTTCTGGCGTGGGTGGGGGGG - +EAS54_6_R1_2_1_443_348 - ;;;;;;;;;;;9;7;;.7;393333 - + @seq1 + GACAGCTTGGTTTTTAGTGAGTTGTTCCTTTCTTT + +seq1 + hhhhhhhhhhhhhhhhhhhhhhhhhhPW@hhhhhh + @seq2 + GCAATGACGGCAGCAATAAACTCAACAGGTGCTGG + +seq2 + hhhhhhhhhhhhhhYhhahhhhWhAhFhSIJGChO - will extract the following sequences:: - >EAS54_6_R1_2_1_413_324 - CCCTTCTTGTCTTCAGCGTTTCTCC - >EAS54_6_R1_2_1_540_792 - TTGGCAGGCCAAGGCCGATGGATCA - >EAS54_6_R1_2_1_443_348 - GTTGCTTCTGGCGTGGGTGGGGGGG - + >seq1 + GACAGCTTGGTTTTTAGTGAGTTGTTCCTTTCTTT + >seq2 + GCAATGACGGCAGCAATAAACTCAACAGGTGCTGG + - and quality scores:: - >EAS54_6_R1_2_1_413_324 - 26 26 18 26 26 26 26 26 26 26 26 26 26 26 26 22 26 26 26 26 26 26 26 23 23 - >EAS54_6_R1_2_1_540_792 - 26 26 26 26 26 26 26 26 26 26 26 22 26 26 26 26 26 12 26 26 26 18 26 23 18 - >EAS54_6_R1_2_1_443_348 - 26 26 26 26 26 26 26 26 26 26 26 24 26 22 26 26 13 22 26 18 24 18 18 18 18 - + >seq1 + 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 16 23 0 40 40 40 40 40 40 + >seq2 + 40 40 40 40 40 40 40 40 40 40 40 40 40 40 25 40 40 33 40 40 40 40 23 40 1 40 6 40 19 9 10 7 3 40 15 **Example2** diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 99a922b13e3..39242d103e3 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -175,7 +175,6 @@ binseq.zip = galaxy.datatypes.images:Binseq,application/zip,display_in_upload customtrack = galaxy.datatypes.interval:CustomTrack data = galaxy.datatypes.data:Data,application/octet-stream fasta = galaxy.datatypes.sequence:Fasta,display_in_upload -fastq = galaxy.datatypes.sequence:Fastq,display_in_upload fastqsolexa = galaxy.datatypes.sequence:FastqSolexa,display_in_upload gff = galaxy.datatypes.interval:Gff,display_in_upload gff3 = galaxy.datatypes.interval:Gff3,display_in_upload