From a1cc059df12167a447460830b11b0147dc1e5793 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 18 Oct 2022 11:19:05 -0400 Subject: [PATCH 1/4] Add support for the PSL data format --- .../config/sample/datatypes_conf.xml.sample | 2 + lib/galaxy/datatypes/registry.py | 3 + lib/galaxy/datatypes/sniff.py | 6 + lib/galaxy/datatypes/tabular.py | 166 ++++++++++++++++++ lib/galaxy/datatypes/test/1.psl | 6 + lib/galaxy/datatypes/test/2.psl | 5 + 6 files changed, 188 insertions(+) create mode 100644 lib/galaxy/datatypes/test/1.psl create mode 100644 lib/galaxy/datatypes/test/2.psl diff --git a/lib/galaxy/config/sample/datatypes_conf.xml.sample b/lib/galaxy/config/sample/datatypes_conf.xml.sample index f0b6878311f..9221cf9a2cb 100644 --- a/lib/galaxy/config/sample/datatypes_conf.xml.sample +++ b/lib/galaxy/config/sample/datatypes_conf.xml.sample @@ -397,6 +397,7 @@ + @@ -1120,6 +1121,7 @@ + diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 746761d9a74..4fa5b3d9e27 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -711,6 +711,7 @@ class Registry: "customtrack": interval.CustomTrack(), "csfasta": sequence.csFasta(), "fasta": sequence.Fasta(), + "psl": tabular.Psl(), "eland": tabular.Eland(), "fastq": sequence.Fastq(), "fastqsanger": sequence.FastqSanger(), @@ -759,6 +760,7 @@ class Registry: "maf": "text/plain", "memexml": "application/xml", "pileup": "text/plain", + "psl": "text/plain", "qualsolid": "text/plain", "qualsolexa": "text/plain", "qual454": "text/plain", @@ -803,6 +805,7 @@ class Registry: interval.Gff(), interval.Gff3(), tabular.Pileup(), + tabular.Psl(), interval.Interval(), tabular.Sam(), tabular.Eland(), diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 5839b5a2927..9f557c85a2c 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -298,6 +298,12 @@ def guess_ext(fname_or_file_prefix: Union[str, "FilePrefix"], sniff_order, is_bi >>> fname = get_test_fname('megablast_xml_parser_test1.blastxml') >>> guess_ext(fname, sniff_order) 'blastxml' + >>> fname = get_test_fname('1.psl') + >>> guess_ext(fname, sniff_order) + 'psl' + >>> fname = get_test_fname('2.psl') + >>> guess_ext(fname, sniff_order) + 'psl' >>> fname = get_test_fname('interval.interval') >>> guess_ext(fname, sniff_order) 'interval' diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 23c82defe3f..218ee1cdd71 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1779,3 +1779,169 @@ class CMAP(TabularData): dataset.metadata.column_types = cleaned_column_types dataset.metadata.columns = number_of_columns dataset.metadata.delimiter = "\t" + + +@build_sniff_from_prefix +class Psl(Tabular): + """Tab delimited data in psl format.""" + + edam_format = "format_3007" + file_ext = "psl" + line_class = "assemblies" + data_sources = {"data": "tabix"} + + def __init__(self, **kwd): + """Initialize psl datatype""" + super().__init__(**kwd) + self.column_names = [ + "matches", + "misMatches", + "repMatches", + "nCount", + "qNumInsert", + "qBaseInsert", + "tNumInsert", + "tBaseInsert", + "strand", + "qName", + "qSize", + "qStart", + "qEnd", + "tName", + "tSize", + "tStart", + "tEnd", + "blockCount", + "blockSizes", + "qStarts", + "tStarts" + ] + + def display_peek(self, dataset): + """Returns formated html of peek""" + return self.make_html_table(dataset, column_names=self.column_names) + + def sniff_prefix(self, file_prefix: FilePrefix): + """ + PSL lines represent alignments, and are typically generated + by BLAT. Each line consists of 21 required fields, and track + lines may optionally be used to provide more information. + + Fields are tab-separated, and all 21 are required. + Although not part of the formal PSL specification, track lines + may be used to further configure sets of features. Track lines + are placed at the beginning of the list of features they are + to affect. + + Rules for sniffing as True:: + + - There must be 21 columns on each fields line + - matches, misMatches repMatches, nCount, qNumInsert, + qBaseInsert, tNumInsert, tBaseInsert, strand, qSize, qStart, + qEnd, tName, tSize, tStart, tEnd, blockCount, blockSizes, + qStarts, tStarts must be correct + - We will only check that up to the first 10 alignments are + correctly formatted. + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( '1.psl' ) + >>> Psl().sniff( fname ) + True + >>> fname = get_test_fname( '2.psl' ) + >>> Psl().sniff( fname ) + True + >>> fname = get_test_fname( 'interval.interval' ) + >>> Psl().sniff( fname ) + False + >>> fname = get_test_fname( '2.txt' ) + >>> Psl().sniff( fname ) + False + >>> fname = get_test_fname( 'test_tab2.tabular' ) + >>> Psl().sniff( fname ) + False + >>> fname = get_test_fname( 'mothur_datatypetest_true.mothur.ref.taxonomy' ) + >>> Psl().sniff( fname ) + False + """ + count = 0 + for line in file_prefix.line_iterator(): + line = line.strip() + if not line: + break + if line: + if line.startswith("browser") or line.startswith("track"): + # Skip track lines. + continue + items = line.split("\t") + if len(items) != 21: + return False + # matches + if int(items[0]) < 0: + raise Exception("Out of range") + # misMatches + if int(items[1]) < 0: + raise Exception("Out of range") + # repMatches + if int(items[2]) < 0: + raise Exception("Out of range") + # nCount + if int(items[3]) < 0: + raise Exception("Out of range") + # qNumInsert + if int(items[4]) < 0: + raise Exception("Out of range") + # qBaseInsert + if int(items[5]) < 0: + raise Exception("Out of range") + # tNumInsert + if int(items[6]) < 0: + raise Exception("Out of range") + # tBaseInsert + if int(items[7]) < 0: + raise Exception("Out of range") + # strand + if items[8] not in ["-", "+", "+-", "-+"]: + raise Exception("Invalid strand") + # qSize + if int(items[10]) < 0: + raise Exception("Out of range") + # qStart + if int(items[11]) < 0: + raise Exception("Out of range") + # qEnd + if int(items[12]) < 0: + raise Exception("Out of range") + # tSize + if int(items[14]) < 0: + raise Exception("Out of range") + # tStart + if int(items[15]) < 0: + raise Exception("Out of range") + # tEnd + if int(items[16]) < 0: + raise Exception("Out of range") + # blockCount + if int(items[17]) < 0: + raise Exception("Out of range") + # blockSizes + s = items[18].rstrip(',') + s_items = s.split(',') + for item in s_items: + if int(item) < 0: + raise Exception("Out of range") + # qStarts + s = items[19].rstrip(',') + s_items = s.split(',') + for item in s_items: + if int(item) < 0: + raise Exception("Out of range") + # tStarts + s = items[20].rstrip(',') + s_items = s.split(',') + for item in s_items: + if int(item) < 0: + raise Exception("Out of range") + count += 1 + if count == 10: + break + if count > 0: + return True diff --git a/lib/galaxy/datatypes/test/1.psl b/lib/galaxy/datatypes/test/1.psl new file mode 100644 index 00000000000..c67415a6c1a --- /dev/null +++ b/lib/galaxy/datatypes/test/1.psl @@ -0,0 +1,6 @@ +browser position chr22:13073000-13074000 +browser hide all +track name=fishBlats description="Fish BLAT" visibility=2 useScore=1 +59 9 0 0 1 823 1 96 +- FS_CONTIG_48080_1 1955 171 1062 chr22 47748585 13073589 13073753 2 48,20, 171,1042, 34674832,34674976, +59 7 0 0 1 55 1 55 +- FS_CONTIG_26780_1 2825 2456 2577 chr22 47748585 13073626 13073747 2 21,45, 2456,2532, 34674838,34674914, +59 7 0 0 1 55 1 55 -+ FS_CONTIG_26780_1 2825 2455 2676 chr22 47748585 13073727 13073848 2 45,21, 249,349, 13073727,13073827 diff --git a/lib/galaxy/datatypes/test/2.psl b/lib/galaxy/datatypes/test/2.psl new file mode 100644 index 00000000000..d95570c67ef --- /dev/null +++ b/lib/galaxy/datatypes/test/2.psl @@ -0,0 +1,5 @@ +2890 41 0 0 25 41 25 32 + NZ_CP026740.1 98205 66320 69251 PS02056|SAMN29228906_5 3054 125 3047 51 72,57,4,9,13,13,86,4,44,4,19,2,1,52,7,8,4,181,108,189,50,41,3,231,2,3,19,3,5,61,136,1,8,30,18,1,42,26,55,119,2,380,3,109,5,566,2,2,45,11,34, 66320,66392,66449,66454,66464,66477,66490,66577,66581,66626,66630,66649,66653,66654,66706,66714,66723,66727,66908,67016,67206,67257,67298,67303,67540,67542,67545,67565,67569,67574,67635,67771,67774,67782,67813,67832,67834,67877,67903,67959,68079,68081,68462,68465,68575,68580,69154,69157,69159,69206,69217, 125,198,256,260,269,283,297,383,388,432,437,457,459,461,514,521,529,534,716,826,1015,1065,1108,1111,1342,1346,1351,1370,1373,1380,1442,1579,1580,1589,1619,1637,1638,1680,1708,1763,1882,1885,2265,2269,2378,2384,2950,2952,2955,3000,3013, +1513 16 0 0 12 16 14 20 - NZ_CP026740.1 98205 32103 33632 PS02056|SAMN29228906_6 1548 13 1546 27 94,6,159,153,17,2,74,7,6,28,147,6,1,7,202,4,3,164,1,1,2,276,3,83,4,21,42, 32103,32197,32203,32362,32515,32532,32535,32609,32618,32626,32654,32802,32808,32810,32818,33020,33026,33030,33194,33195,33197,33201,33477,33481,33565,33569,33590, 13,108,115,275,430,448,450,525,532,538,567,714,723,724,731,935,939,942,1107,1110,1111,1113,1390,1393,1476,1482,1504, +762 108 0 0 14 108 13 132 + NZ_CP026740.1 98205 69627 70497 PS02056|SAMN29228906_3 6215 1049 1943 28 81,1,45,4,35,2,66,1,2,99,41,2,5,4,40,76,3,3,72,4,3,2,7,71,5,11,3,74, 69627,69711,69713,69758,69762,69797,69799,69865,69868,69870,69969,70011,70014,70019,70023,70073,70150,70153,70157,70295,70304,70311,70314,70332,70404,70409,70420,70423, 1049,1130,1131,1177,1182,1218,1321,1388,1389,1392,1492,1533,1535,1543,1548,1588,1664,1669,1672,1744,1748,1751,1753,1760,1831,1837,1857,1869, +77 0 0 0 0 0 0 0 + NZ_CP026740.1 98205 70504 70581 PS02056|SAMN29228906_3 6215 134 211 1 77, 70504, 134, +2890 41 0 0 25 41 25 32 + CP026740.1 98205 66320 69251 PS02056|SAMN29228906_5 3054 125 3047 51 72,57,4,9,13,13,86,4,44,4,19,2,1,52,7,8,4,181,108,189,50,41,3,231,2,3,19,3,5,61,136,1,8,30,18,1,42,26,55,119,2,380,3,109,5,566,2,2,45,11,34, 66320,66392,66449,66454,66464,66477,66490,66577,66581,66626,66630,66649,66653,66654,66706,66714,66723,66727,66908,67016,67206,67257,67298,67303,67540,67542,67545,67565,67569,67574,67635,67771,67774,67782,67813,67832,67834,67877,67903,67959,68079,68081,68462,68465,68575,68580,69154,69157,69159,69206,69217, 125,198,256,260,269,283,297,383,388,432,437,457,459,461,514,521,529,534,716,826,1015,1065,1108,1111,1342,1346,1351,1370,1373,1380,1442,1579,1580,1589,1619,1637,1638,1680,1708,1763,1882,1885,2265,2269,2378,2384,2950,2952,2955,3000,3013, From fd2d4083eb7c9637e093406c277eb016d0b3fe42 Mon Sep 17 00:00:00 2001 From: greg Date: Tue, 18 Oct 2022 12:30:51 -0400 Subject: [PATCH 2/4] Attempt to fix lint problems --- lib/galaxy/datatypes/tabular.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 218ee1cdd71..473576da9fe 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1814,7 +1814,7 @@ class Psl(Tabular): "blockCount", "blockSizes", "qStarts", - "tStarts" + "tStarts", ] def display_peek(self, dataset): @@ -1923,20 +1923,20 @@ class Psl(Tabular): if int(items[17]) < 0: raise Exception("Out of range") # blockSizes - s = items[18].rstrip(',') - s_items = s.split(',') + s = items[18].rstrip(",") + s_items = s.split(",") for item in s_items: if int(item) < 0: raise Exception("Out of range") # qStarts - s = items[19].rstrip(',') - s_items = s.split(',') + s = items[19].rstrip(",") + s_items = s.split(",") for item in s_items: if int(item) < 0: raise Exception("Out of range") # tStarts - s = items[20].rstrip(',') - s_items = s.split(',') + s = items[20].rstrip(",") + s_items = s.split(",") for item in s_items: if int(item) < 0: raise Exception("Out of range") From fa410aa2d6d4355115855f6047770d4384175b02 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 21 Nov 2022 09:18:36 -0500 Subject: [PATCH 3/4] Review fixes --- lib/galaxy/datatypes/registry.py | 3 -- lib/galaxy/datatypes/tabular.py | 83 +++++++------------------------- 2 files changed, 18 insertions(+), 68 deletions(-) diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 4fa5b3d9e27..746761d9a74 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -711,7 +711,6 @@ class Registry: "customtrack": interval.CustomTrack(), "csfasta": sequence.csFasta(), "fasta": sequence.Fasta(), - "psl": tabular.Psl(), "eland": tabular.Eland(), "fastq": sequence.Fastq(), "fastqsanger": sequence.FastqSanger(), @@ -760,7 +759,6 @@ class Registry: "maf": "text/plain", "memexml": "application/xml", "pileup": "text/plain", - "psl": "text/plain", "qualsolid": "text/plain", "qualsolexa": "text/plain", "qual454": "text/plain", @@ -805,7 +803,6 @@ class Registry: interval.Gff(), interval.Gff3(), tabular.Pileup(), - tabular.Psl(), interval.Interval(), tabular.Sam(), tabular.Eland(), diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 473576da9fe..4c38d82dc51 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1817,10 +1817,6 @@ class Psl(Tabular): "tStarts", ] - def display_peek(self, dataset): - """Returns formated html of peek""" - return self.make_html_table(dataset, column_names=self.column_names) - def sniff_prefix(self, file_prefix: FilePrefix): """ PSL lines represent alignments, and are typically generated @@ -1862,6 +1858,11 @@ class Psl(Tabular): >>> Psl().sniff( fname ) False """ + def check_items(s): + s_items = s.split(",") + for item in s_items: + if int(item) < 0: + raise Exception("Out of range") count = 0 for line in file_prefix.line_iterator(): line = line.strip() @@ -1874,72 +1875,24 @@ class Psl(Tabular): items = line.split("\t") if len(items) != 21: return False - # matches - if int(items[0]) < 0: - raise Exception("Out of range") - # misMatches - if int(items[1]) < 0: - raise Exception("Out of range") - # repMatches - if int(items[2]) < 0: - raise Exception("Out of range") - # nCount - if int(items[3]) < 0: - raise Exception("Out of range") - # qNumInsert - if int(items[4]) < 0: - raise Exception("Out of range") - # qBaseInsert - if int(items[5]) < 0: - raise Exception("Out of range") - # tNumInsert - if int(items[6]) < 0: - raise Exception("Out of range") - # tBaseInsert - if int(items[7]) < 0: - raise Exception("Out of range") + # tName is a string + items.pop(13) + # qName is a string + items.pop(9) # strand - if items[8] not in ["-", "+", "+-", "-+"]: + if items.pop(8) not in ["-", "+", "+-", "-+"]: raise Exception("Invalid strand") - # qSize - if int(items[10]) < 0: - raise Exception("Out of range") - # qStart - if int(items[11]) < 0: - raise Exception("Out of range") - # qEnd - if int(items[12]) < 0: - raise Exception("Out of range") - # tSize - if int(items[14]) < 0: - raise Exception("Out of range") - # tStart - if int(items[15]) < 0: - raise Exception("Out of range") - # tEnd - if int(items[16]) < 0: - raise Exception("Out of range") - # blockCount - if int(items[17]) < 0: - raise Exception("Out of range") # blockSizes - s = items[18].rstrip(",") - s_items = s.split(",") - for item in s_items: - if int(item) < 0: - raise Exception("Out of range") + s = items.pop(15).rstrip(",") + check_items(s) # qStarts - s = items[19].rstrip(",") - s_items = s.split(",") - for item in s_items: - if int(item) < 0: - raise Exception("Out of range") + s = items.pop(15).rstrip(",") + check_items(s) # tStarts - s = items[20].rstrip(",") - s_items = s.split(",") - for item in s_items: - if int(item) < 0: - raise Exception("Out of range") + s = items.pop(15).rstrip(",") + check_items(s) + if any(int(item) < 0 for item in items): + raise Exception("Out of range") count += 1 if count == 10: break From 771934d18f255d2ab2a9b234a425529993b3ab22 Mon Sep 17 00:00:00 2001 From: greg Date: Mon, 21 Nov 2022 09:32:50 -0500 Subject: [PATCH 4/4] Lint fix attempt --- lib/galaxy/datatypes/tabular.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index 4c38d82dc51..0765d932045 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -1858,11 +1858,13 @@ class Psl(Tabular): >>> Psl().sniff( fname ) False """ + def check_items(s): s_items = s.split(",") for item in s_items: if int(item) < 0: raise Exception("Out of range") + count = 0 for line in file_prefix.line_iterator(): line = line.strip()