opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update fastq format.

Browse Source 
Now we only support FastqSolexa variants.
If the quality scores are presented as characters,
the integer values are obtained by their ascii code subtract 64.
This commit is contained in:
Wen-Yu Chung
2008-06-06 18:52:11 +00:00
parent e767816519
commit 782d7db6be
10 changed files with 70 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
datatype_converters_conf.xml.sample
+2 -2
View File
@@ -2,8 +2,8 @@
<converters>
<converter file="bed_to_gff_converter.xml" source_datatype="bed" target_datatype="gff"/>
<converter file="fasta_to_tabular_converter.xml" source_datatype="fasta" target_datatype="tabular"/>
<converter file="fastq_to_fasta_converter.xml" source_datatype="fastq,fastqsolexa" target_datatype="fasta"/>
<converter file="fastq_to_qual_converter.xml" source_datatype="fastq,fastqsolexa" target_datatype="qual"/>
<converter file="fastq_to_fasta_converter.xml" source_datatype="fastqsolexa" target_datatype="fasta"/>
<converter file="fastq_to_qual_converter.xml" source_datatype="fastqsolexa" target_datatype="qual"/>
<converter file="gff_to_bed_converter.xml" source_datatype="gff" target_datatype="bed"/>
<converter file="interval_to_bed_converter.xml" source_datatype="interval" target_datatype="bed"/>
<converter file="maf_to_fasta_converter.xml" source_datatype="maf" target_datatype="fasta"/>
lib/galaxy/datatypes/converters/fastq_to_fasta_converter.xml
+1 -1
View File
@@ -2,7 +2,7 @@
<description>converts Fastq file to Fasta format</description>
<command interpreter="python">fastq_to_fasta_converter.py $input $output</command>
<inputs>
<param name="input" type="data" format="fastq,fastqsolexa" label="Choose Fastq file"/>
<param name="input" type="data" format="fastqsolexa" label="Choose Fastq file"/>
</inputs>
<outputs>
<data name="output" format="fasta"/>
lib/galaxy/datatypes/converters/fastq_to_qual_converter.py
+4 -6
View File
@@ -28,7 +28,7 @@ def __main__():
datatype = sys.argv[3]
qual_title_startswith = ''
seq_title_startswith = ''
default_coding_value = 33
default_coding_value = 64
fastq_block_lines = 0
for i, line in enumerate( file( infile_name ) ):
@@ -75,10 +75,8 @@ def __main__():
if fastq_integer: # digits
qual = line
else: # ascii
if datatype == 'fastqsolexa':
outfile_score.close()
stop_err( "This tool currently only works with the fastq solexa variant if the socres are integers, not ascii." )
else:
# ascii
quality_score_length = len( line )
if quality_score_length == read_length + 1:
quality_score_startswith = ord( line[0:1] )
@@ -88,7 +86,7 @@ def __main__():
else:
stop_err( 'Invalid fastq format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d ).' % ( i + 1, quality_score_length, read_length ) )
for j, char in enumerate( line ):
score = ord( char ) - quality_score_startswith # 33
score = ord( char ) - quality_score_startswith # 64
qual = "%s%s " % ( qual, str( score ) )
outfile_score.write( '%s\n' % qual )
lib/galaxy/datatypes/converters/fastq_to_qual_converter.xml
+1 -1
View File
@@ -1,7 +1,7 @@
<tool id="CONVERTER_fastq_to_qual_0" name="Convert Fastq to Qual">
<command interpreter="python">fastq_to_qual_converter.py $input1 $output1 $input1.extension</command>
<inputs>
<param format="fastq,fastqsolexa" name="input1" type="data" label="Choose Fastq file"/>
<param format="fastqsolexa" name="input1" type="data" label="Choose Fastq file"/>
</inputs>
<outputs>
<data format="qual" name="output1" />
lib/galaxy/datatypes/registry.py
+2 -2
View File
@@ -67,7 +67,7 @@ class Registry( object ):
'binseq.zip' : images.Binseq(),
'customtrack' : interval.CustomTrack(),
'fasta' : sequence.Fasta(),
'fastq' : sequence.Fastq(),
'fastqsolexa' : sequence.FastqSolexa(),
'gff' : interval.Gff(),
'gff3' : interval.Gff3(),
'interval' : interval.Interval(),
@@ -89,7 +89,7 @@ class Registry( object ):
'binseq.zip' : 'application/zip',
'customtrack' : 'text/plain',
'fasta' : 'text/plain',
'fastq' : 'text/plain',
'fastqsolexa' : 'text/plain',
'gff' : 'text/plain',
'gff3' : 'text/plain',
'interval' : 'text/plain',
lib/galaxy/datatypes/sequence.py
+15 -67
View File
@@ -89,61 +89,6 @@ class Fasta( Sequence ):
except:
return False
class Fastq( Sequence ):
"""Class representing a FASTQ sequence ( the Sanger/Standard variant )"""
file_ext = "fastq"
def set_peek( self, dataset ):
Sequence.set_peek( self, dataset )
count = 0
size = 0
bases_regexp = re.compile("^[NGTAC]*$")
for line in file( dataset.file_name ):
if line and line.startswith( ">" ):
count += 1
elif bases_regexp.match( line ):
line = line.strip()
size += len( line )
if count == 1:
dataset.blurb = '%d bases' % size
else:
dataset.blurb = '%d sequences' % count
def sniff(self, filename):
"""
Determines whether the file is in fastq format ( the Sanger/Standard variant )
For details, see http://maq.sourceforge.net/fastq.shtml
Note: There are two kinds of FASTQ files, known as "Sanger" (sometimes called "Standard") and Solexa
These differ in the representation of the quality scores
>>> fname = get_test_fname( '1.fastq' )
>>> Fastq().sniff( fname )
True
>>> fname = get_test_fname( '1.fastqsolexa' )
>>> Fastq().sniff( fname )
False
"""
headers = get_headers( filename, None )
bases_regexp = re.compile( "^[NGTAC]*$" )
try:
if len( headers ) >= 4 and headers[0][0] and headers[0][0][0] == "@" and headers[2][0] and headers[2][0][0] == "+" and headers[1][0] and headers[3][0]:
# Check the sequence line, make sure it contains only G/C/A/T/N
if not bases_regexp.match( headers[1][0] ):
return False
# The quality score line
qscore = headers[3][0]
# In Standard/Sanger format, the quality score is a single string, whose length should be equal to the length of the sequence
if len( qscore ) != len( headers[1][0] ):
return False
#Check the quality score values - in Sanger/Standard these should be ASCII characters between "!" (0x21) and "~" (0x7E)
for x in qscore:
if ord( x ) < 0x21 or ord( x ) > 0x7e:
return False
return True
return False
except:
return False
class FastqSolexa( Sequence ):
"""Class representing a FASTQ sequence ( the Solexa variant )"""
@@ -154,7 +99,7 @@ class FastqSolexa( Sequence ):
count = size = 0
bases_regexp = re.compile("^[NGTAC]*$")
for line in file( dataset.file_name ):
if line and line[0] == ">":
if line and line[0] == "@":
count += 1
elif bases_regexp.match(line):
line = line.strip()
@@ -174,7 +119,7 @@ class FastqSolexa( Sequence ):
>>> fname = get_test_fname( '1.fastq' )
>>> FastqSolexa().sniff( fname )
False
True
>>> fname = get_test_fname( '1.fastqsolexa' )
>>> FastqSolexa().sniff( fname )
True
@@ -186,17 +131,20 @@ class FastqSolexa( Sequence ):
# Check the sequence line, make sure it contains only G/C/A/T/N
if not bases_regexp.match( headers[1][0] ):
return False
qscore = headers[3]
# In Solexa format, the quality score is a list of numbers, whose length should be equal to the length of the sequence
if len( qscore ) != len( headers[1][0] ):
return False
# Check the quality score values - in Solexa/FASTQ these should be valid decimal numbers
# (if "x" is not a valid number, "int" will raise an exception)
for x in qscore:
try:
check = int( x )
except:
# Check quality score: integer or ascii char.
try:
check = int(headers[3][0])
qscore_int = True
except:
qscore_int = False
if qscore_int:
if len( headers[3] ) != len( headers[1][0] ):
return False
else:
if len( headers[3][0] ) != len( headers[1][0] ):
return False
return True
return False
except:
tools/data_source/upload.xml
+21 -14
View File
@@ -89,23 +89,30 @@ A sequence in FASTA format consists of a single-line description, followed by li
-----
**Fastq**
**FastqSolexa**
Fastq format stores sequences and Phred qualities in a single file. We define Fastq as the Sanger/Standard variant::
Fastq format stores sequences and quality scores in a single file. We define FastqSolexa as the Illumina (Solexa) variant::
@EAS54_6_R1_2_1_413_324
CCCTTCTTGTCTTCAGCGTTTCTCC
+
;;3;;;;;;;;;;;;7;;;;;;;88
@EAS54_6_R1_2_1_540_792
TTGGCAGGCCAAGGCCGATGGATCA
+
;;;;;;;;;;;7;;;;;-;;;3;83
@EAS54_6_R1_2_1_443_348
GTTGCTTCTGGCGTGGGTGGGGGGG
+EAS54_6_R1_2_1_443_348
;;;;;;;;;;;9;7;;.7;393333
@seq1
GACAGCTTGGTTTTTAGTGAGTTGTTCCTTTCTTT
+seq1
hhhhhhhhhhhhhhhhhhhhhhhhhhPW@hhhhhh
@seq2
GCAATGACGGCAGCAATAAACTCAACAGGTGCTGG
+seq2
hhhhhhhhhhhhhhYhhahhhhWhAhFhSIJGChO
Or::
@seq1
GAATTGATCAGGACATAGGACAACTGTAGGCACCAT
+seq1
40 40 40 40 35 40 40 40 25 40 40 26 40 9 33 11 40 35 17 40 40 33 40 7 9 15 3 22 15 30 11 17 9 4 9 4
@seq2
GAGTTCTCGTCGCCTGTAGGCACCATCAATCGTATG
+seq2
40 15 40 17 6 36 40 40 40 25 40 9 35 33 40 14 14 18 15 17 19 28 31 4 24 18 27 14 15 18 2 8 12 8 11 9
-----
**Gff**
tools/metag_tools/fastq_to_fasta_qual.py
+2 -6
View File
@@ -36,7 +36,7 @@ def __main__():
datatype = sys.argv[4]
seq_title_startswith = ''
qual_title_startswith = ''
default_coding_value = 33
default_coding_value = 64
fastq_block_lines = 0
for i, line in enumerate( file( infile_name ) ):
@@ -92,10 +92,6 @@ def __main__():
# digits
qual = line
else:
if datatype == 'fastqsolexa':
outfile_seq.close()
outfile_score.close()
stop_err( "This tool currently only works with the fastq solexa variant if the socres are integers, not ascii." )
# ascii
quality_score_length = len( line )
if quality_score_length == read_length + 1:
@@ -107,7 +103,7 @@ def __main__():
else:
stop_err( 'Invalid fastq format at line %d: the number of quality scores ( %d ) is not the same as bases ( %d ).' % ( i + 1, quality_score_length, read_length ) )
for j, char in enumerate( line ):
score = ord( char ) - qual_score_startswith # 33
score = ord( char ) - qual_score_startswith # 64
qual = "%s%s " % ( qual, str( score ) )
outfile_score.write( '%s\n' % qual )
tools/metag_tools/fastq_to_fasta_qual.xml
+22 -32
View File
@@ -2,7 +2,7 @@
<description>extracts sequences and quality scores from FASTQ data</description>
<command interpreter="python">fastq_to_fasta_qual.py $input1 $output1 $output2 $input1.extension</command>
<inputs>
<param name="input1" type="data" format="fastq,fastqsolexa" label="Fastq file"/>
<param name="input1" type="data" format="fastqsolexa" label="Fastq file"/>
</inputs>
<outputs>
<data name="output1" format="fasta"/>
@@ -11,11 +11,11 @@
<tests>
<!-- NOTE: this tool generates 2 output files, but our functional tests currently only handle the last one generated -->
<test>
<param name="input1" value="1.fastq" ftype="fastq" />
<param name="input1" value="1.fastq" ftype="fastqsolexa" />
<output name="output1" file="fastq_to_fasta_qual_out2.fasta" />
</test>
<test>
<param name="input1" value="1.fastqsolexa" ftype="fastq" />
<param name="input1" value="1.fastqsolexa" ftype="fastqsolexa" />
<output name="output1" file="fastq_to_fasta_qual_out4.fasta" />
</test>
</tests>
@@ -23,13 +23,13 @@
.. class:: warningmark
IMPORTANT: With the Fastq Solexa variant, this tool currently only works with data where the quality scores are integers, ASCII quality scores are not supported.
IMPORTANT: This tool currently only support data where the quality scores are integers or ASCII quality scores with base 64.
-----
**What it does**
This tool extracts sequences and quality scores from FASTQ data ( both Sanger/Standard and Solexa variants ), producing a FASTA dataset and a QUAL dataset. With the Solexa variant, this tool currently only works with data where the quality scores are integers, ASCII quality scores are not supported.
This tool extracts sequences and quality scores from FASTQ data ( Solexa variants ), producing a FASTA dataset and a QUAL dataset.
-----
@@ -37,38 +37,28 @@ This tool extracts sequences and quality scores from FASTQ data ( both Sanger/St
- Converting the following Sanger/Standard fastq data::
@EAS54_6_R1_2_1_413_324
CCCTTCTTGTCTTCAGCGTTTCTCC
+
;;3;;;;;;;;;;;;7;;;;;;;88
@EAS54_6_R1_2_1_540_792
TTGGCAGGCCAAGGCCGATGGATCA
+
;;;;;;;;;;;7;;;;;-;;;3;83
@EAS54_6_R1_2_1_443_348
GTTGCTTCTGGCGTGGGTGGGGGGG
+EAS54_6_R1_2_1_443_348
;;;;;;;;;;;9;7;;.7;393333
@seq1
GACAGCTTGGTTTTTAGTGAGTTGTTCCTTTCTTT
+seq1
hhhhhhhhhhhhhhhhhhhhhhhhhhPW@hhhhhh
@seq2
GCAATGACGGCAGCAATAAACTCAACAGGTGCTGG
+seq2
hhhhhhhhhhhhhhYhhahhhhWhAhFhSIJGChO
- will extract the following sequences::
>EAS54_6_R1_2_1_413_324
CCCTTCTTGTCTTCAGCGTTTCTCC
>EAS54_6_R1_2_1_540_792
TTGGCAGGCCAAGGCCGATGGATCA
>EAS54_6_R1_2_1_443_348
GTTGCTTCTGGCGTGGGTGGGGGGG
>seq1
GACAGCTTGGTTTTTAGTGAGTTGTTCCTTTCTTT
>seq2
GCAATGACGGCAGCAATAAACTCAACAGGTGCTGG
- and quality scores::
>EAS54_6_R1_2_1_413_324
26 26 18 26 26 26 26 26 26 26 26 26 26 26 26 22 26 26 26 26 26 26 26 23 23
>EAS54_6_R1_2_1_540_792
26 26 26 26 26 26 26 26 26 26 26 22 26 26 26 26 26 12 26 26 26 18 26 23 18
>EAS54_6_R1_2_1_443_348
26 26 26 26 26 26 26 26 26 26 26 24 26 22 26 26 13 22 26 18 24 18 18 18 18
>seq1
40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 40 16 23 0 40 40 40 40 40 40
>seq2
40 40 40 40 40 40 40 40 40 40 40 40 40 40 25 40 40 33 40 40 40 40 23 40 1 40 6 40 19 9 10 7 3 40 15
**Example2**
universe_wsgi.ini.sample
-1
View File
@@ -175,7 +175,6 @@ binseq.zip = galaxy.datatypes.images:Binseq,application/zip,display_in_upload
customtrack = galaxy.datatypes.interval:CustomTrack
data = galaxy.datatypes.data:Data,application/octet-stream
fasta = galaxy.datatypes.sequence:Fasta,display_in_upload
fastq = galaxy.datatypes.sequence:Fastq,display_in_upload
fastqsolexa = galaxy.datatypes.sequence:FastqSolexa,display_in_upload
gff = galaxy.datatypes.interval:Gff,display_in_upload
gff3 = galaxy.datatypes.interval:Gff3,display_in_upload