From 7b1798349673fa0d16a305b010e944a272024328 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 16 Jan 2015 21:38:18 +0100 Subject: [PATCH 1/6] add SnpEffDb datatype --- lib/galaxy/datatypes/text.py | 50 +++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index c872c042c89..2291434b279 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -18,7 +18,6 @@ import re import logging log = logging.getLogger(__name__) - class Json( Text ): file_ext = "json" @@ -259,3 +258,52 @@ class Arff( Text ): dataset.metadata.comment_lines = comment_lines dataset.metadata.columns = column_count + +class SnpEffDb( Text ): + """Class describing a SnpEff genome build""" + file_ext = "snpeffdb" + MetadataElement( name="genome_version", default=None, desc="Genome Version", readonly=True, visible=True, no_value=None ) + MetadataElement( name="regulation", default=[], desc="Regulation Names", readonly=True, visible=True, no_value=[], optional=True) + MetadataElement( name="annotation", default=[], desc="Annotation Names", readonly=True, visible=True, no_value=[], optional=True) + + def __init__( self, **kwd ): + Text.__init__( self, **kwd ) + + def set_meta( self, dataset, **kwd ): + Text.set_meta(self, dataset, **kwd ) + data_dir = dataset.extra_files_path + ## search data_dir/genome_version for files + regulation_pattern = 'regulation_(.+).bin' + # annotation files that are included in snpEff by a flag + annotations_dict = {'nextProt.bin' : '-nextprot','motif.bin': '-motif'} + regulations = [] + annotations = [] + if data_dir and os.path.isdir(data_dir): + for root, dirs, files in os.walk(data_dir): + for fname in files: + if fname.startswith('snpEffectPredictor'): + # if snpEffectPredictor.bin download succeeded + genome_version = os.path.basename(root) + dataset.metadata.genome_version = genome_version + else: + m = re.match(regulation_pattern,fname) + if m: + name = m.groups()[0] + regulations.append(name) + elif fname in annotations_dict: + value = annotations_dict[fname] + name = value.lstrip('-') + annotations.append(name) + dataset.metadata.regulation = regulations + dataset.metadata.annotation = annotations + try: + fh = file(dataset.file_name,'w') + fh.write("%s\n" % genome_version) + if annotations: + fh.write("annotations: %s\n" % ','.join(annotations)) + if regulations: + fh.write("regulations: %s\n" % ','.join(regulations)) + fh.close() + except: + pass + From b75445e64d346685cbef71ad0112c61a6d7cf3ab Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 16 Jan 2015 21:42:08 +0100 Subject: [PATCH 2/6] activate snpeffdb and snpeffdbv4 --- config/datatypes_conf.xml.sample | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 50b55b28cca..786c2fbabc7 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -176,6 +176,8 @@ + + From b3f4cd74ea238cd29e55b3c6d6874cf2f755507d Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 16 Jan 2015 22:07:41 +0100 Subject: [PATCH 3/6] add SnpSiftDbNSFP datatype --- lib/galaxy/datatypes/text.py | 71 +++++++++++++++++++++++++++++++++++- 1 file changed, 70 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 2291434b279..32a899feec9 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -2,7 +2,6 @@ """ Clearing house for generic text datatypes that are not XML or tabular. """ - from galaxy.datatypes.data import Text from galaxy.datatypes.data import get_file_peek from galaxy.datatypes.data import nice_size @@ -12,6 +11,7 @@ from galaxy import util import tempfile import subprocess import json +import gzip import os import re @@ -307,3 +307,72 @@ class SnpEffDb( Text ): except: pass + +class SnpSiftDbNSFP( Text ): + """Class describing a dbNSFP database prepared fpr use by SnpSift dbnsfp """ + MetadataElement( name='reference_name', default='dbSNFP' , desc='Reference Name', readonly=True, visible=True, set_in_upload=True, no_value='dbSNFP' ) + MetadataElement( name="bgzip", default=None, desc="dbNSFP bgzip", readonly=True, visible=True, no_value=None ) + MetadataElement( name="index", default=None, desc="Tabix Index File", readonly=True, visible=True, no_value=None) + MetadataElement( name="annotation", default=[], desc="Annotation Names", readonly=True, visible=True, no_value=[] ) + file_ext = "snpsiftdbnsfp" + composite_type = 'auto_primary_file' + allow_datatype_change = False + """ + ## The dbNSFP file is a tabular file with 1 header line + ## The first 4 columns are required to be: chrom pos ref alt + ## These match columns 1,2,4,5 of the VCF file + ## SnpSift requires the file to be block-gzipped and the indexed with samtools tabix + ## Example: + ## Compress using block-gzip algorithm + bgzip dbNSFP2.3.txt + ## Create tabix index + tabix -s 1 -b 2 -e 2 dbNSFP2.3.txt.gz + """ + def __init__( self, **kwd ): + Text.__init__( self, **kwd ) + self.add_composite_file( '%s.grp', description = 'Group File', substitute_name_with_metadata = 'reference_name', is_binary = False ) + self.add_composite_file( '%s.ti', description = '', substitute_name_with_metadata = 'reference_name', is_binary = False ) + def init_meta( self, dataset, copy_from=None ): + Text.init_meta( self, dataset, copy_from=copy_from ) + def generate_primary_file( self, dataset = None ): + """ + This is called only at upload to write the html file + cannot rename the datasets here - they come with the default unfortunately + """ + regenerate_primary_file( self, dataset) + def regenerate_primary_file(self,dataset): + """ + cannot do this until we are setting metadata + """ + annotations = "dbNSFP Annotations: %s\n" % ','.join(dataset.metadata.annotation) + f = open(dataset.file_name,'a') + if dataset.metadata.bgzip: + bn = dataset.metadata.bgzip + f.write(bn) + f.write('\n') + f.write(annotations) + f.close() + def set_meta( self, dataset, overwrite=True, **kwd ): + try: + efp = dataset.extra_files_path + if os.path.exists(efp): + flist = os.listdir(efp) + for i,fname in enumerate(flist): + if fname.endswith('.gz'): + dataset.metadata.bgzip = fname + try: + fh = gzip.open(os.path.join(efp,fname),'r') + buf = fh.read(5000) + lines = buf.splitlines() + headers = lines[0].split('\t') + dataset.metadata.annotation = headers[4:] + except Exception,e: + log.warn("set_meta fname: %s %s" % (fname,str(e))) + finally: + fh.close() + if fname.endswith('.tbi'): + dataset.metadata.index = fname + self.regenerate_primary_file(dataset) + except Exception,e: + log.warn("set_meta fname: %s %s" % (dataset.file_name if dataset and dataset.file_name else 'Unkwown',str(e))) + From 0f0364e9fab0c2942395aba21cb20e48b3ae84c7 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 16 Jan 2015 22:08:29 +0100 Subject: [PATCH 4/6] activate SnpSiftDbNSFP and dbnsfp.tabular --- config/datatypes_conf.xml.sample | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 786c2fbabc7..44ace07191b 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -113,7 +113,7 @@ - + @@ -151,11 +151,11 @@ - - - - - + + + + + @@ -171,13 +171,17 @@ - - - + + + - - + + + + + + From 5e6e2c5617b1ae678dc87e98dea0eef2d073bee4 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Fri, 16 Jan 2015 22:59:04 +0100 Subject: [PATCH 5/6] add converters --- .../datatypes/converters/tabular_to_dbnsfp.xml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml diff --git a/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml new file mode 100644 index 00000000000..84803d593e2 --- /dev/null +++ b/lib/galaxy/datatypes/converters/tabular_to_dbnsfp.xml @@ -0,0 +1,12 @@ + + + tabular_to_dbnsfp.py $input $dbnsfp.extra_files_path/dbNSFP.gz + + + + + + + + + From b0d92cadd7af2327940af70beaed09ba4efe3bf5 Mon Sep 17 00:00:00 2001 From: Bjoern Gruening Date: Sat, 17 Jan 2015 14:05:44 +0100 Subject: [PATCH 6/6] Remove not needed datatype, we now can filter datatypes by metadata. --- config/datatypes_conf.xml.sample | 1 - 1 file changed, 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 44ace07191b..c5783d442c8 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -177,7 +177,6 @@ -