From 694d8c85551013a894a491d4be9e5a46b72fb199 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Mon, 26 Jun 2017 13:55:12 +0200 Subject: [PATCH 1/9] Sniff FastqSanger before Fastq, in the former case checking the quality scores for compatibility --- lib/galaxy/datatypes/registry.py | 1 + lib/galaxy/datatypes/sequence.py | 18 +++++++++++++++++- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index fcb1884d970..4be4249130e 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -776,6 +776,7 @@ class Registry( object ): qualityscore.QualityScoreSOLiD(), qualityscore.QualityScore454(), sequence.Fasta(), + sequence.FastqSanger(), sequence.Fastq(), interval.Wiggle(), text.Html(), diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 7dc6feaa6eb..b44e8c59476 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -612,7 +612,13 @@ class BaseFastq ( Sequence ): compressed = is_gzip(filename) or is_bz2(filename) if compressed and not isinstance(self, Binary): return False - headers = get_headers( filename, None ) + headers = get_headers( filename, None, count=1000 ) + + # If this is a FastqSanger-derived class, then check to see if the base qualities match + if isinstance(self, FastqSanger): + if not self.sangerQualities(headers): + return False + bases_regexp = re.compile( "^[NGTAC]*" ) # check that first block looks like a fastq block try: @@ -698,6 +704,16 @@ class FastqSanger( Fastq ): """Class representing a FASTQ sequence ( the Sanger variant )""" edam_format = "format_1932" file_ext = "fastqsanger" + if not self.sangerQualities(headers): + + @staticmethod + def sangerQualities( lines ): + """Presuming lines are lines from a fastq file, return True if the qualities are compatible with sanger encoding""" + for line in lines[0::4]: + _ = [ord( c ) for c in line] + if max( _ ) > ord( 'J' ): + return False + return True class FastqSolexa( Fastq ): From 0a942d79a326c3b2cfb8d7cbee6a341e4600a77f Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Mon, 26 Jun 2017 16:36:17 +0200 Subject: [PATCH 2/9] Checking the quality lines will make fastqsanger checks rather more relevant. --- config/datatypes_conf.xml.sample | 3 +++ lib/galaxy/datatypes/sequence.py | 20 ++++++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 9c80d81ddbb..2a1cc9f846c 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -691,6 +691,9 @@ + + + diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index b44e8c59476..86930a7bfdb 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -615,7 +615,7 @@ class BaseFastq ( Sequence ): headers = get_headers( filename, None, count=1000 ) # If this is a FastqSanger-derived class, then check to see if the base qualities match - if isinstance(self, FastqSanger): + if isinstance(self, FastqSanger) or isinstance(self, FastqSangerGz) or isinstance(self, FastqSangerBz2): if not self.sangerQualities(headers): return False @@ -693,6 +693,15 @@ class BaseFastq ( Sequence ): return True process_split_file = staticmethod(process_split_file) + @staticmethod + def sangerQualities( lines ): + """Presuming lines are lines from a fastq file, return True if the qualities are compatible with sanger encoding""" + for line in lines[3::4]: + _ = [ord( c ) for c in line[0]] + if max( _ ) > ord( 'M' ): + return False + return True + class Fastq( BaseFastq ): """Class representing a generic FASTQ sequence""" @@ -706,15 +715,6 @@ class FastqSanger( Fastq ): file_ext = "fastqsanger" if not self.sangerQualities(headers): - @staticmethod - def sangerQualities( lines ): - """Presuming lines are lines from a fastq file, return True if the qualities are compatible with sanger encoding""" - for line in lines[0::4]: - _ = [ord( c ) for c in line] - if max( _ ) > ord( 'J' ): - return False - return True - class FastqSolexa( Fastq ): """Class representing a FASTQ sequence ( the Solexa variant )""" From bc13a16f83d9d74774751fbe5c4d7d3b2b3c6ecc Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Mon, 26 Jun 2017 22:18:38 +0200 Subject: [PATCH 3/9] Add the compressed datatypes to the sniffer lsit --- lib/galaxy/datatypes/registry.py | 9 +++++---- lib/galaxy/datatypes/sequence.py | 22 +++++++++++++--------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 4be4249130e..e317b02351c 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -677,6 +677,8 @@ class Registry( object ): def set_default_values( self ): # Default values. if not self.datatypes_by_extension: + # 'fastq' : sequence.Fastq(), + # 'fastqsanger' : sequence.FastqSanger(), self.datatypes_by_extension = { 'ab1' : binary.Ab1(), 'axt' : sequence.Axt(), @@ -688,8 +690,6 @@ class Registry( object ): 'db3' : binary.SQlite(), 'fasta' : sequence.Fasta(), 'eland' : tabular.Eland(), - 'fastq' : sequence.Fastq(), - 'fastqsanger' : sequence.FastqSanger(), 'gemini.sqlite' : binary.GeminiSQLite(), 'gtf' : interval.Gtf(), 'gff' : interval.Gff(), @@ -761,6 +761,9 @@ class Registry( object ): # Default values - the order in which we attempt to determine data types is critical # because some formats are much more flexibly defined than others. if len( self.sniff_order ) < 1: + #after fasta + # sequence.FastqSanger(), + # sequence.Fastq(), self.sniff_order = [ binary.Bam(), binary.Sff(), @@ -776,8 +779,6 @@ class Registry( object ): qualityscore.QualityScoreSOLiD(), qualityscore.QualityScore454(), sequence.Fasta(), - sequence.FastqSanger(), - sequence.Fastq(), interval.Wiggle(), text.Html(), sequence.Axt(), diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 86930a7bfdb..900f7c59fd8 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -618,6 +618,9 @@ class BaseFastq ( Sequence ): if isinstance(self, FastqSanger) or isinstance(self, FastqSangerGz) or isinstance(self, FastqSangerBz2): if not self.sangerQualities(headers): return False + else: + if self.sangerQualities(headers): + return False bases_regexp = re.compile( "^[NGTAC]*" ) # check that first block looks like a fastq block @@ -713,7 +716,6 @@ class FastqSanger( Fastq ): """Class representing a FASTQ sequence ( the Sanger variant )""" edam_format = "format_1932" file_ext = "fastqsanger" - if not self.sangerQualities(headers): class FastqSolexa( Fastq ): @@ -746,10 +748,6 @@ class FastqGz ( BaseFastq, Binary ): return BaseFastq.sniff( self, filename ) -if SNIFF_COMPRESSED_FASTQS: - Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz) - - class FastqSangerGz( FastqGz ): """Class representing a compressed FASTQ sequence ( the Sanger variant )""" edam_format = "format_1932" @@ -762,6 +760,11 @@ class FastqSolexaGz( FastqGz ): file_ext = "fastqsolexa.gz" +if SNIFF_COMPRESSED_FASTQS: + Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz) + Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz) + + class FastqIlluminaGz( FastqGz ): """Class representing a compressed FASTQ sequence ( the Illumina 1.3+ variant )""" edam_format = "format_1931" @@ -786,16 +789,17 @@ class FastqBz2 ( BaseFastq, Binary ): return BaseFastq.sniff( self, filename ) -if SNIFF_COMPRESSED_FASTQS: - Binary.register_sniffable_binary_format("fastq.bz2", "fastq.bz2", FastqBz2) - - class FastqSangerBz2( FastqBz2 ): """Class representing a compressed FASTQ sequence ( the Sanger variant )""" edam_format = "format_1932" file_ext = "fastqsanger.bz2" +if SNIFF_COMPRESSED_FASTQS: + Binary.register_sniffable_binary_format("fastq.bz2", "fastq.bz2", FastqBz2) + Binary.register_sniffable_binary_format("fastqsanger.bz2", "fastqsanger.bz2", FastqSangerBz2) + + class FastqSolexaBz2( FastqBz2 ): """Class representing a compressed FASTQ sequence ( the Solexa variant )""" edam_format = "format_1933" From ab1127334f1c0e2d43c4fcccb9259c33fdaf235b Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Tue, 27 Jun 2017 09:44:39 +0200 Subject: [PATCH 4/9] Remove debugging stuff, since everything works inside of docker --- lib/galaxy/datatypes/registry.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index e317b02351c..406b7ee0620 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -677,8 +677,6 @@ class Registry( object ): def set_default_values( self ): # Default values. if not self.datatypes_by_extension: - # 'fastq' : sequence.Fastq(), - # 'fastqsanger' : sequence.FastqSanger(), self.datatypes_by_extension = { 'ab1' : binary.Ab1(), 'axt' : sequence.Axt(), @@ -689,6 +687,8 @@ class Registry( object ): 'csfasta' : sequence.csFasta(), 'db3' : binary.SQlite(), 'fasta' : sequence.Fasta(), + 'fastq' : sequence.Fastq(), + 'fastqsanger' : sequence.FastqSanger(), 'eland' : tabular.Eland(), 'gemini.sqlite' : binary.GeminiSQLite(), 'gtf' : interval.Gtf(), @@ -761,9 +761,6 @@ class Registry( object ): # Default values - the order in which we attempt to determine data types is critical # because some formats are much more flexibly defined than others. if len( self.sniff_order ) < 1: - #after fasta - # sequence.FastqSanger(), - # sequence.Fastq(), self.sniff_order = [ binary.Bam(), binary.Sff(), @@ -779,6 +776,8 @@ class Registry( object ): qualityscore.QualityScoreSOLiD(), qualityscore.QualityScore454(), sequence.Fasta(), + sequence.FastqSanger(), + sequence.Fastq(), interval.Wiggle(), text.Html(), sequence.Axt(), From 798b73eb688c5b7786fe2b9ffe69967ded3d7c63 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Tue, 27 Jun 2017 09:46:05 +0200 Subject: [PATCH 5/9] Fix line reordering --- lib/galaxy/datatypes/registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 406b7ee0620..4be4249130e 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -687,9 +687,9 @@ class Registry( object ): 'csfasta' : sequence.csFasta(), 'db3' : binary.SQlite(), 'fasta' : sequence.Fasta(), + 'eland' : tabular.Eland(), 'fastq' : sequence.Fastq(), 'fastqsanger' : sequence.FastqSanger(), - 'eland' : tabular.Eland(), 'gemini.sqlite' : binary.GeminiSQLite(), 'gtf' : interval.Gtf(), 'gff' : interval.Gff(), From 8eaede31102b16f09abcbc8f0f06c6eab75d49bb Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Tue, 27 Jun 2017 10:11:30 +0200 Subject: [PATCH 6/9] Fix sequence.py unit test --- lib/galaxy/datatypes/sequence.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 900f7c59fd8..7af84ed57bf 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -603,10 +603,10 @@ class BaseFastq ( Sequence ): >>> from galaxy.datatypes.sniff import get_test_fname >>> fname = get_test_fname( '1.fastqsanger' ) - >>> Fastq().sniff( fname ) + >>> FastqSanger().sniff( fname ) True >>> fname = get_test_fname( '2.fastqsanger' ) - >>> Fastq().sniff( fname ) + >>> FastqSanger().sniff( fname ) True """ compressed = is_gzip(filename) or is_bz2(filename) From f3853a09562af8d7539740344433c4a8dc94aa36 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Tue, 27 Jun 2017 11:01:46 +0200 Subject: [PATCH 7/9] Test a non-fastqsanger file and ensure it doesn't pass the sniff test. --- lib/galaxy/datatypes/sequence.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 7af84ed57bf..4ca9b919a6c 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -608,6 +608,11 @@ class BaseFastq ( Sequence ): >>> fname = get_test_fname( '2.fastqsanger' ) >>> FastqSanger().sniff( fname ) True + >>> fname = get_test_fname( '2.fastq' ) + >>> Fastq().sniff( fname ) + True + >>> FastqSanger().sniff( fname ) + False """ compressed = is_gzip(filename) or is_bz2(filename) if compressed and not isinstance(self, Binary): From 907bd53776eccec0e8d3fadfb69223dadcc48600 Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Tue, 27 Jun 2017 11:11:19 +0200 Subject: [PATCH 8/9] Make the phred range stricter and more explicit. --- lib/galaxy/datatypes/sequence.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 4ca9b919a6c..016e7309294 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -705,8 +705,7 @@ class BaseFastq ( Sequence ): def sangerQualities( lines ): """Presuming lines are lines from a fastq file, return True if the qualities are compatible with sanger encoding""" for line in lines[3::4]: - _ = [ord( c ) for c in line[0]] - if max( _ ) > ord( 'M' ): + if not all(_ >= '!' and _ <= 'M' for _ in line[0]): return False return True From e21e9ec15fec5d3d1e47cd3c3001d9be24fbf12c Mon Sep 17 00:00:00 2001 From: Devon Ryan Date: Tue, 27 Jun 2017 15:21:34 +0200 Subject: [PATCH 9/9] Ugh, there were multiple places where the sniffing order was specified (binary formats are different). --- lib/galaxy/datatypes/sequence.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 016e7309294..0c4cde56a13 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -623,9 +623,6 @@ class BaseFastq ( Sequence ): if isinstance(self, FastqSanger) or isinstance(self, FastqSangerGz) or isinstance(self, FastqSangerBz2): if not self.sangerQualities(headers): return False - else: - if self.sangerQualities(headers): - return False bases_regexp = re.compile( "^[NGTAC]*" ) # check that first block looks like a fastq block @@ -765,8 +762,8 @@ class FastqSolexaGz( FastqGz ): if SNIFF_COMPRESSED_FASTQS: - Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz) Binary.register_sniffable_binary_format("fastqsanger.gz", "fastqsanger.gz", FastqSangerGz) + Binary.register_sniffable_binary_format("fastq.gz", "fastq.gz", FastqGz) class FastqIlluminaGz( FastqGz ): @@ -800,8 +797,8 @@ class FastqSangerBz2( FastqBz2 ): if SNIFF_COMPRESSED_FASTQS: - Binary.register_sniffable_binary_format("fastq.bz2", "fastq.bz2", FastqBz2) Binary.register_sniffable_binary_format("fastqsanger.bz2", "fastqsanger.bz2", FastqSangerBz2) + Binary.register_sniffable_binary_format("fastq.bz2", "fastq.bz2", FastqBz2) class FastqSolexaBz2( FastqBz2 ):