From ff7ecaa58df6426a56a2909aa135a89d6f7e280e Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 1 Oct 2007 21:06:58 +0000 Subject: [PATCH] Eliminated non-text datatypes from sniff order since galaxy hinders non-text file uploads (e.g., gmaj.zip files are sniffed as txt data types since Galaxy automaticaally changes EOL characters upon upload. Fixed the guess_ext() function in sniff.py to eliminate the call to sep2tabs since this function changes the data in the file (the guess_ext() function probably shouldn't be doing that). Added a sniff function to the Gmaj class as an example for zip data types. --- lib/galaxy/datatypes/images.py | 29 +++++++++++++++++++--------- lib/galaxy/datatypes/registry.py | 2 -- lib/galaxy/datatypes/sniff.py | 31 +++++++++++++++++------------- lib/galaxy/tools/actions/upload.py | 4 ++++ universe_wsgi.ini.sample | 24 +++++++++++------------ 5 files changed, 53 insertions(+), 37 deletions(-) diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index 4885fb554fe..6699c6b34aa 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -6,6 +6,7 @@ import data import logging from galaxy.datatypes.sniff import * from urllib import urlencode +import zipfile log = logging.getLogger(__name__) @@ -15,7 +16,6 @@ class Image( data.Data ): dataset.peek = 'Image in %s format (%s)' % ( dataset.extension, data.nice_size( dataset.get_size() ) ) dataset.blurb = 'image' - class Gmaj( data.Data ): """Class describing a GMAJ Applet""" file_ext = "gmaj.zip" @@ -32,10 +32,25 @@ class Gmaj( data.Data ): def get_mime(self): """Returns the mime type of the datatype""" return 'application/zip' - def sniff( self, filename ): - #TODO: fix me - return False - + def sniff(self, filename): + """ + NOTE: the sniff.convert_newlines() call in the upload utility will keep Gmaj data types from being + correctly sniffed, but the files can be uploaded (they'll be sniffed as 'txt'). This sniff function + is here to provide an example of a sniffer for a zip file. + """ + if not zipfile.is_zipfile( filename ): + return False + contains_gmaj_file = False + zip_file = zipfile.ZipFile(filename, "r") + for name in zip_file.namelist(): + if name.split(".")[1].strip().lower() == 'gmaj': + contains_gmaj_file = True + break + zip_file.close() + if not contains_gmaj_file: + return False + return True + class Html( data.Text ): """Class describing an html file""" file_ext = "html" @@ -60,7 +75,6 @@ class Html( data.Text ): True """ headers = get_headers( filename, None ) - try: for i, hdr in enumerate(headers): if hdr and hdr[0].lower().find( '' ) >=0: @@ -82,7 +96,4 @@ class Laj( data.Text ): return dataset.peek except: return "peek unavailable" - def sniff( self, filename ): - #TODO: fix me... - return False diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index ddbd9198d49..0b4666175ab 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -102,8 +102,6 @@ class Registry( object ): #default values if len(self.sniff_order) < 1: self.sniff_order = [ - images.Gmaj(), - images.Laj(), sequence.Maf(), sequence.Lav(), sequence.Fasta(), diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index f7b6e07dd0c..508e3ab43dd 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -161,16 +161,19 @@ def guess_ext( fname ): >>> guess_ext(fname) 'gff3' >>> fname = get_test_fname('temp.txt') - >>> file(fname, 'wt').write("a 2\\nc 1\\nd 0") + >>> file(fname, 'wt').write("a\\t2\\nc\\t1\\nd\\t0") >>> guess_ext(fname) 'tabular' >>> fname = get_test_fname('temp.txt') >>> file(fname, 'wt').write("a 1 2 x\\nb 3 4 y\\nc 5 6 z") >>> guess_ext(fname) - 'tabular' + 'txt' >>> fname = get_test_fname('test_tab1.tabular') >>> guess_ext(fname) 'tabular' + >>> fname = get_test_fname('alignment.lav') + >>> guess_ext(fname) + 'lav' """ datatypes_registry = registry.Registry() for datatype in datatypes_registry.sniff_order: @@ -188,19 +191,21 @@ def guess_ext( fname ): except: pass - """Default binary file extension""" - for line in file( fname ): - for char in line: - if ord(char) > 128: - return 'data' - else: + headers = get_headers( fname, None ) + is_binary = True + for hdr in headers: + for char in hdr: + try: + if not ord(char) > 128: + is_binary = False + except: + is_binary = False break - break - if is_column_based( fname, ' ', 1 ): - sep2tabs(fname) + if is_binary: + return 'data' #default binary data type file extension if is_column_based( fname, '\t', 1): - return 'tabular' - return 'txt' + return 'tabular' #default tabular data type file extension + return 'txt' #default text data type file extension if __name__ == '__main__': import doctest, sys diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index bd31dd5728e..c3d5d468db4 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -87,6 +87,10 @@ class UploadToolAction( object ): if self.empty: raise BadFileException( "attempted to upload an empty or inappropriate file" ) + """ + NOTE: the following will keep binary and zip files (e.g., gmaj.zip) from being correctly sniffed, but + the files can be uploaded (they'll be sniffed as 'txt'). This should restrict some unwanted behavior. + """ sniff.convert_newlines(temp_name) if space_to_tab: diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 6fe6aa43e26..b114e94a016 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -174,16 +174,14 @@ nametable = galaxy.datatypes.data:Text [galaxy:sniff_order] -05 = galaxy.datatypes.images:Gmaj -10 = galaxy.datatypes.images:Laj -15 = galaxy.datatypes.sequence:Maf -20 = galaxy.datatypes.sequence:Lav -25 = galaxy.datatypes.sequence:Fasta -30 = galaxy.datatypes.interval:Wiggle -35 = galaxy.datatypes.images:Html -40 = galaxy.datatypes.sequence:Axt -45 = galaxy.datatypes.interval:Bed -50 = galaxy.datatypes.interval:CustomTrack -55 = galaxy.datatypes.interval:Gff -60 = galaxy.datatypes.interval:Gff3 -65 = galaxy.datatypes.interval:Interval +05 = galaxy.datatypes.sequence:Maf +10 = galaxy.datatypes.sequence:Lav +15 = galaxy.datatypes.sequence:Fasta +20 = galaxy.datatypes.interval:Wiggle +25 = galaxy.datatypes.images:Html +30 = galaxy.datatypes.sequence:Axt +35 = galaxy.datatypes.interval:Bed +40 = galaxy.datatypes.interval:CustomTrack +45 = galaxy.datatypes.interval:Gff +50 = galaxy.datatypes.interval:Gff3 +55 = galaxy.datatypes.interval:Interval