diff --git a/lib/galaxy/datatypes/images.py b/lib/galaxy/datatypes/images.py index 6699c6b34aa..b14278e5c7d 100644 --- a/lib/galaxy/datatypes/images.py +++ b/lib/galaxy/datatypes/images.py @@ -10,6 +10,66 @@ import zipfile log = logging.getLogger(__name__) +class Ab1( data.Data ): + """Class describing an ab1 binary sequence file""" + file_ext = "ab1" + def set_peek( self, dataset ): + export_url = "/history_add_to?"+urlencode({'history_id':dataset.history_id,'ext':'ab1','name':'ab1 sequence','info':'Sequence file','dbkey':dataset.dbkey}) + dataset.peek = "Binary ab1 sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) + dataset.blurb = "Binary ab1 sequence file" + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Binary ab1 sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) + +class Scf( data.Data ): + """Class describing an scf binary sequence file""" + file_ext = "scf" + def set_peek( self, dataset ): + export_url = "/history_add_to?"+urlencode({'history_id':dataset.history_id,'ext':'scf','name':'scf sequence','info':'Sequence file','dbkey':dataset.dbkey}) + dataset.peek = "Binary scf sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) + dataset.blurb = "Binary scf sequence file" + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Binary scf sequence file (%s)" % ( data.nice_size( dataset.get_size() ) ) + +class Binseq( data.Data ): + """Class describing a zip archive of binary sequence files""" + file_ext = "binseq.zip" + def set_peek( self, dataset ): + zip_file = zipfile.ZipFile( dataset.file_name, "r" ) + num_files = len( zip_file.namelist() ) + dataset.peek = "Binary sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) + dataset.blurb = 'Zip archive of %s binary sequence files' % ( str( num_files ) ) + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Binary sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) + def get_mime(self): + """Returns the mime type of the datatype""" + return 'application/zip' + +class Txtseq( data.Data ): + """Class describing a zip archive of text sequence files""" + file_ext = "txtseq.zip" + def set_peek( self, dataset ): + zip_file = zipfile.ZipFile( dataset.file_name, "r" ) + num_files = len( zip_file.namelist() ) + dataset.peek = "Text sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) + dataset.blurb = 'Zip archive of %s text sequence files' % ( str( num_files ) ) + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Text sequence file archive (%s)" % ( data.nice_size( dataset.get_size() ) ) + def get_mime(self): + """Returns the mime type of the datatype""" + return 'application/zip' + class Image( data.Data ): """Class describing an image""" def set_peek( self, dataset ): @@ -19,11 +79,9 @@ class Image( data.Data ): class Gmaj( data.Data ): """Class describing a GMAJ Applet""" file_ext = "gmaj.zip" - def set_peek( self, dataset ): dataset.peek = "
" dataset.blurb = 'GMAJ Multiple Alignment Viewer' - def display_peek(self, dataset): try: return dataset.peek @@ -54,15 +112,12 @@ class Gmaj( data.Data ): class Html( data.Text ): """Class describing an html file""" file_ext = "html" - def set_peek( self, dataset ): dataset.peek = "HTML file (%s)" % ( data.nice_size( dataset.get_size() ) ) dataset.blurb = data.nice_size( dataset.get_size() ) - def get_mime(self): """Returns the mime type of the datatype""" return 'text/html' - def sniff( self, filename ): """ Determines wether the file is in html format @@ -86,7 +141,6 @@ class Html( data.Text ): class Laj( data.Text ): """Class describing a LAJ Applet""" file_ext = "laj" - def set_peek( self, dataset ): export_url = "/history_add_to?"+urlencode({'history_id':dataset.history_id,'ext':'lav','name':'LAJ Output','info':'Added by LAJ','dbkey':dataset.dbkey}) dataset.peek = "" @@ -96,4 +150,3 @@ class Laj( data.Text ): return dataset.peek except: return "peek unavailable" - diff --git a/lib/galaxy/datatypes/registry.py b/lib/galaxy/datatypes/registry.py index 31f793825b2..12f2cd32d06 100644 --- a/lib/galaxy/datatypes/registry.py +++ b/lib/galaxy/datatypes/registry.py @@ -37,44 +37,42 @@ class Registry( object ): #default values if len(self.datatypes_by_extension) < 1: self.datatypes_by_extension = { - 'data' : data.Data(), - 'bed' : interval.Bed(), - 'txt' : data.Text(), - 'interval' : interval.Interval(), - 'tabular' : tabular.Tabular(), - 'png' : images.Image(), - 'pdf' : images.Image(), - 'fasta' : sequence.Fasta(), - 'maf' : sequence.Maf(), - 'axt' : sequence.Axt(), - 'gff' : interval.Gff(), - 'gff3' : interval.Gff3(), - 'wig' : interval.Wiggle(), - 'gmaj.zip' : images.Gmaj(), - 'laj' : images.Laj(), - 'lav' : sequence.Lav(), - 'html' : images.Html(), - 'customtrack' : interval.CustomTrack() + 'ab1' : images.Ab1(), + 'axt' : sequence.Axt(), + 'bed' : interval.Bed(), + 'binseq.zip' : images.Binseq(), + 'customtrack' : interval.CustomTrack(), + 'fasta' : sequence.Fasta(), + 'gff' : interval.Gff(), + 'gff3' : interval.Gff3(), + 'interval' : interval.Interval(), + 'laj' : images.Laj(), + 'lav' : sequence.Lav(), + 'maf' : sequence.Maf(), + 'scf' : images.Scf(), + 'tabular' : tabular.Tabular(), + 'txt' : data.Text(), + 'txtseq.zip' : images.Txtseq(), + 'wig' : interval.Wiggle() } self.mimetypes_by_extension = { - 'data' : 'application/octet-stream', - 'bed' : 'text/plain', - 'txt' : 'text/plain', - 'interval' : 'text/plain', - 'tabular' : 'text/plain', - 'png' : 'image/png', - 'pdf' : 'application/pdf', - 'fasta' : 'text/plain', - 'maf' : 'text/plain', - 'axt' : 'text/plain', - 'gff' : 'text/plain', - 'gff3' : 'text/plain', - 'wig' : 'text/plain', - 'gmaj.zip' : 'application/zip', - 'laj' : 'text/plain', - 'lav' : 'text/plain', - 'html' : 'text/html', - 'customtrack' : 'text/plain' + 'ab1' : 'application/octet-stream', + 'axt' : 'text/plain', + 'bed' : 'text/plain', + 'binseq.zip' : 'application/zip', + 'customtrack' : 'text/plain', + 'fasta' : 'text/plain', + 'gff' : 'text/plain', + 'gff3' : 'text/plain', + 'interval' : 'text/plain', + 'laj' : 'text/plain', + 'lav' : 'text/plain', + 'maf' : 'text/plain', + 'scf' : 'application/octet-stream', + 'tabular' : 'text/plain', + 'txt' : 'text/plain', + 'txtseq.zip' : 'application/zip', + 'wig' : 'text/plain' } """ The order in which we attempt to determine data types is critical diff --git a/lib/galaxy/tools/actions/upload.py b/lib/galaxy/tools/actions/upload.py index c235e8c705d..a3d86f71e99 100644 --- a/lib/galaxy/tools/actions/upload.py +++ b/lib/galaxy/tools/actions/upload.py @@ -1,4 +1,4 @@ -import os, shutil, urllib, StringIO, re +import os, shutil, urllib, StringIO, re, gzip, tempfile, shutil, zipfile from galaxy import datatypes, jobs from galaxy.datatypes import sniff from galaxy import model, util @@ -9,10 +9,9 @@ import logging log = logging.getLogger( __name__ ) class UploadToolAction( object ): - """ - Action for uploading files - """ - empty = False + # Action for uploading files + def __init__( self ): + self.empty = False def execute( self, tool, trans, incoming={} ): data_file = incoming['file_data'] @@ -21,44 +20,36 @@ class UploadToolAction( object ): url_paste = incoming['url_paste'] space_to_tab = False if 'space_to_tab' in incoming: - if incoming['space_to_tab'] not in ["None", None]: + if incoming['space_to_tab'] not in ["None", None]: space_to_tab = True - info = "uploaded file" temp_name = "" data_list = [] - self.empty = False - if 'filename' in dir(data_file): + + if 'filename' in dir( data_file ): try: file_name = data_file.filename - file_name = file_name.split('\\')[-1] - file_name = file_name.split('/')[-1] - data_list.append( self.add_file(trans, data_file.file, file_name, file_type, dbkey, "uploaded file",space_to_tab=space_to_tab) ) - except BadFileException: - return self.upload_empty( trans, "Error", "attempted to upload an empty or inappropriate file") - except: - pass - + file_name = file_name.split( '\\' )[-1] + file_name = file_name.split( '/' )[-1] + data_list.append( self.add_file( trans, data_file.file, file_name, file_type, dbkey, space_to_tab=space_to_tab ) ) + except BadFileException, e: + return self.upload_empty( trans, "Error:", str( e ) ) if url_paste not in [None, ""]: - if url_paste[0:7].lower() == "http://" or url_paste[0:6].lower() == "ftp://" : - url_paste = url_paste.replace("\r","").split("\n") + if url_paste[0:7].lower() == "http://" or url_paste[0:6].lower() == "ftp://": + url_paste = url_paste.replace( "\r","" ).split("\n") for line in url_paste: try: - data_list.append( self.add_file(trans, urllib.urlopen(line), line, file_type, dbkey, "uploaded url",space_to_tab=space_to_tab) ) - except BadFileException: - return self.upload_empty( trans, "Error", "attempted to upload an empty or inappropriate file") - except: - pass + data_list.append( self.add_file( trans, urllib.urlopen( line ), line, file_type, dbkey, info="uploaded url", space_to_tab=space_to_tab ) ) + except BadFileException, e: + return self.upload_empty( trans, "Error:", str( e ) ) else: try: - data_list.append( self.add_file(trans, StringIO.StringIO(url_paste), 'Pasted Entry', file_type, dbkey, "pasted entry",space_to_tab=space_to_tab) ) - except BadFileException: - return self.upload_empty( trans, "Error", "attempted to upload an empty or inappropriate file" ) - except: - pass + data_list.append( self.add_file( trans, StringIO.StringIO( url_paste ), 'Pasted Entry', file_type, dbkey, info="pasted entry", space_to_tab=space_to_tab ) ) + except BadFileException, e: + return self.upload_empty( trans, "Error:", str( e ) ) if self.empty: - return self.upload_empty(trans, "Empty file error:", "attempted to upload an empty file") - elif len(data_list)<1: - return self.upload_empty(trans, "No data error:","either you pasted no data, the url you specified is invalid, or you have not specified a file") + return self.upload_empty( trans, "Empty file error:", "you attempted to upload an empty file." ) + elif len( data_list ) < 1: + return self.upload_empty( trans, "No data error:","either you pasted no data, the url you specified is invalid, or you have not specified a file." ) return dict( output=data_list[0] ) def upload_empty(self, trans, err_code, err_msg): @@ -73,33 +64,84 @@ class UploadToolAction( object ): trans.app.model.flush() return dict( output=data ) - def add_file(self, trans, file_obj, file_name, file_type, dbkey, info, space_to_tab = False ): - temp_name = sniff.stream_to_file(file_obj) + def add_file( self, trans, file_obj, file_name, file_type, dbkey, info=None, space_to_tab=False ): + data_type = None + temp_name = sniff.stream_to_file( file_obj ) + + # See if we have an empty file + if not os.path.getsize( temp_name ) > 0: + raise BadFileException( "you attempted to upload an empty file." ) + + # See if we have a gzipped file, which, if it passes our restrictions, we'll decompress on the fly. + is_gzipped, is_valid = self.check_gzip( temp_name ) + if is_gzipped and not is_valid: + raise BadFileException( "you attempted to upload an inappropriate file." ) + elif is_gzipped and is_valid: + #We need to decompress the temp_name file + CHUNK_SIZE = 2**20 # 1Mb + fd, uncompressed = tempfile.mkstemp() + gzipped_file = gzip.GzipFile( temp_name ) + while 1: + try: + chunk = gzipped_file.read( CHUNK_SIZE ) + except IOError: + uncompressed.close() + raise BadFileException( 'problem decompressing gzipped data.' ) + if not chunk: + break + os.write( fd, chunk ) + os.close( fd ) + gzipped_file.close() + # Replace the gzipped file with the decompressed file + shutil.move( uncompressed, temp_name ) + file_name = file_name.rstrip( '.gz' ) + data_type = 'gzip' - try: - # Check against undesireable file data: - if self.check_html( temp_name ) or self.check_binary( temp_name ): - self.empty = True - except: - #User is attempting to upload a non-text file, but for some reason check_binary didn't work... - self.empty = True + if not data_type: + # See if we have a zip archive + is_zipped, is_valid, test_ext = self.check_zip( temp_name ) + if is_zipped and not is_valid: + raise BadFileException( "you attempted to upload an inappropriate file." ) + elif is_zipped and is_valid: + # Currently, we force specific tools to handle this case. We also require the user + # to manually set the incoming file_type + if ( test_ext == 'ab1' or test_ext == 'scf' ) and file_type != 'binseq.zip': + raise BadFileException( "Invalid 'File Format' for archive consisting of binary files - use 'Binseq.zip'." ) + elif test_ext == 'txt' and file_type != 'txtseq.zip': + raise BadFileException( "Invalid 'File Format' for archive consisting of text files - use 'Txtseq.zip'." ) + if not ( file_type == 'binseq.zip' or file_type == 'txtseq.zip' ): + raise BadFileException( "you must manually set the 'File Format' to either 'Binseq.zip' or 'Txtseq.zip' when uploading zip files." ) + data_type = 'zip' + ext = file_type + + if not data_type: + if self.check_binary( temp_name ): + ext = file_name.split( "." )[1].strip().lower() + if not( ext == 'ab1' or ext == 'scf' ): + raise BadFileException( "you attempted to upload an inappropriate file." ) + if ext == 'ab1' and file_type != 'ab1': + raise BadFileException( "you must manually set the 'File Format' to 'Ab1' when uploading ab1 files." ) + elif ext == 'scf' and file_type != 'scf': + raise BadFileException( "you must manually set the 'File Format' to 'Scf' when uploading scf files." ) + data_type = 'binary' - if self.empty: - raise BadFileException( "attempted to upload an empty or inappropriate file" ) - - """ - NOTE: the following will keep binary and zip files (e.g., gmaj.zip) from being correctly sniffed, but - the files can be uploaded (they'll be sniffed as 'txt'). This should restrict some unwanted behavior. - """ - sniff.convert_newlines(temp_name) - - if space_to_tab: - sniff.sep2tabs(temp_name) - - if file_type == 'auto': - ext = sniff.guess_ext(temp_name) - else: - ext = file_type + if not data_type: + # We must have a text file + if self.check_html( temp_name ): + raise BadFileException( "you attempted to upload an inappropriate file." ) + + if data_type != 'binary' and data_type != 'zip': + sniff.convert_newlines( temp_name ) + if space_to_tab: + sniff.sep2tabs( temp_name ) + if file_type == 'auto': + ext = sniff.guess_ext( temp_name ) + else: + ext = file_type + data_type = ext + + if info is None: + info = 'uploaded %s file' %data_type data = trans.app.model.Dataset() data.name = file_name @@ -119,37 +161,72 @@ class UploadToolAction( object ): data.add_validation_error( model.ValidationError( message=str( error ), err_type=error.__class__.__name__, attributes=util.object_to_string( error.__dict__ ) ) ) """ - if data.has_data(): - if data.missing_meta(): - data.datatype.set_meta(data) - dbkey_to_store = dbkey - if type(dbkey_to_store) == type([]): - dbkey_to_store = dbkey[0] - trans.history.add_dataset( data, genome_build=dbkey_to_store ) - trans.app.model.flush() - trans.log_event("Added dataset %d to history %d" %(data.id, trans.history.id), tool_id="upload") - else: - self.empty = True + if data.missing_meta(): + data.datatype.set_meta( data ) + dbkey_to_store = dbkey + if type( dbkey_to_store ) == type( [] ): + dbkey_to_store = dbkey[0] + trans.history.add_dataset( data, genome_build=dbkey_to_store ) + trans.app.model.flush() + trans.log_event( "Added dataset %d to history %d" %( data.id, trans.history.id ), tool_id="upload" ) return data - def check_html( self, temp_name ): - temp = open(temp_name, "U") + def check_gzip( self, temp_name ): + temp = open( temp_name, "U" ) + magic_check = temp.read( 2 ) + temp.close() + if magic_check != datatypes.data.gzip_magic: + return ( False, False ) + CHUNK_SIZE = 2**15 # 32Kb + gzipped_file = gzip.GzipFile( temp_name ) + chunk = gzipped_file.read( CHUNK_SIZE ) + gzipped_file.close() + if self.check_html( temp_name, chunk=chunk ) or self.check_binary( temp_name, chunk=chunk ): + return( True, False ) + return ( True, True ) + + def check_zip( self, temp_name ): + if not zipfile.is_zipfile( temp_name ): + return ( False, False, None ) + zip_file = zipfile.ZipFile( temp_name, "r" ) + # Make sure the archive consists of valid files. The current rules are: + # 1. Archives can only include .ab1, .scf or .txt files + # 2. All file extensions within an archive must be the same + name = zip_file.namelist()[0] + test_ext = name.split( "." )[1].strip().lower() + if not ( test_ext == 'scf' or test_ext == 'ab1' or test_ext == 'txt' ): + return ( True, False, test_ext ) + for name in zip_file.namelist(): + ext = name.split( "." )[1].strip().lower() + if ext != test_ext: + return ( True, False, test_ext ) + return ( True, True, test_ext ) + + def check_html( self, temp_name, chunk=None ): + if chunk is None: + temp = open(temp_name, "U") + else: + temp = chunk regexp = re.compile( "<([A-Z][A-Z0-9]*)[^>]*>", re.I ) lineno = 0 for line in temp: lineno += 1 matches = regexp.search( line ) if matches: - temp.close() + if chunk is None: + temp.close() return True if lineno > 100: - # We should be able to detmine an HTML file within 100 lines. break - temp.close() + if chunk is None: + temp.close() return False - def check_binary( self, temp_name ): - temp = open( temp_name, "U" ) + def check_binary( self, temp_name, chunk=None ): + if chunk is None: + temp = open( temp_name, "U" ) + else: + temp = chunk lineno = 0 for line in temp: lineno += 1 @@ -157,11 +234,13 @@ class UploadToolAction( object ): if line: for char in line: if ord( char ) > 128: - temp.close() + if chunk is None: + temp.close() return True if lineno > 100: break - temp.close() + if chunk is None: + temp.close() return False class BadFileException( Exception ): diff --git a/tool_conf.xml.main b/tool_conf.xml.main index 143fe0f8c53..9ea4e1dbdcc 100644 --- a/tool_conf.xml.main +++ b/tool_conf.xml.main @@ -113,6 +113,14 @@