Add support for uploading BAM files.

datatypes_conf.xml.sample

@@ -3,7 +3,7 @@
     <registration converters_path="lib/galaxy/datatypes/converters">
         <datatype extension="ab1" type="galaxy.datatypes.binary:Ab1" mimetype="application/octet-stream" display_in_upload="true"/>
         <datatype extension="axt" type="galaxy.datatypes.sequence:Axt" display_in_upload="true"/>
-        <datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream"/>
+        <datatype extension="bam" type="galaxy.datatypes.binary:Bam" mimetype="application/octet-stream" display_in_upload="true"/>
         <datatype extension="bed" type="galaxy.datatypes.interval:Bed" display_in_upload="true">
             <converter file="bed_to_gff_converter.xml" target_datatype="gff"/>
             <converter file="interval_to_coverage.xml" target_datatype="coverage"/>
@@ -203,6 +203,7 @@
           defined format first, followed by next-most rigidly defined, 
           and so on.
         -->
+        <sniffer type="galaxy.datatypes.binary:Bam"/>
         <sniffer type="galaxy.datatypes.binary:Sff"/>
         <sniffer type="galaxy.datatypes.xml:BlastXml"/>
         <sniffer type="galaxy.datatypes.sequence:Maf"/>

lib/galaxy/datatypes/binary.py

@@ -12,7 +12,7 @@ import os, subprocess, tempfile
 
 log = logging.getLogger(__name__)
 
-sniffable_binary_formats = [ 'sff' ]
+sniffable_binary_formats = [ 'sff', 'bam' ]
 # Currently these supported binary data types must be manually set on upload
 unsniffable_binary_formats = [ 'ab1', 'scf' ]
 
@@ -26,6 +26,9 @@ class Binary( data.Data ):
         else:
             dataset.peek = 'file does not exist'
             dataset.blurb = 'file purged from disk'
+    def get_mime( self ):
+        """Returns the mime type of the datatype"""
+        return 'application/octet-stream'
 
 class Ab1( Binary ):
     """Class describing an ab1 binary sequence file"""
@@ -48,29 +51,40 @@ class Bam( Binary ):
     """Class describing a BAM binary file"""
     file_ext = "bam"
     MetadataElement( name="bam_index", desc="BAM Index File", param=metadata.FileParameter, readonly=True, no_value=None, visible=False, optional=True )    
+
     def init_meta( self, dataset, copy_from=None ):
         Binary.init_meta( self, dataset, copy_from=copy_from )
-    def set_meta( self, dataset, overwrite = True, **kwd ):
         """
-        Sets index for BAM file.
+        GVK 12/2/09: just noticed this - not good and doesn't work, so commenting out for now.
+        def set_meta( self, dataset, overwrite = True, **kwd ):
+            # Sets index for BAM file.
+            index_file = dataset.metadata.bam_index
+            if not index_file:
+                index_file = dataset.metadata.spec['bam_index'].param.new_file( dataset = dataset )
+            tmp_dir = tempfile.gettempdir()
+            tmpf1 = tempfile.NamedTemporaryFile( dir=tmp_dir )
+            tmpf1bai = '%s.bai' % tmpf1.name
+            try:
+                os.system( 'cd %s' % tmp_dir )
+                os.system( 'cp %s %s' % ( dataset.file_name, tmpf1.name ) )
+                os.system( 'samtools index %s' % tmpf1.name )
+                os.system( 'cp %s %s' % ( tmpf1bai, index_file.file_name ) )
+            except Exception, ex:
+                sys.stderr.write( 'There was a problem creating the index for the BAM file\n%s\n' + str( ex ) )
+            tmpf1.close()
+            if os.path.exists( tmpf1bai ):
+                os.remove( tmpf1bai )
+            dataset.metadata.bam_index = index_file
         """
-        index_file = dataset.metadata.bam_index
-        if not index_file:
-            index_file = dataset.metadata.spec['bam_index'].param.new_file( dataset = dataset )
-        tmp_dir = tempfile.gettempdir()
-        tmpf1 = tempfile.NamedTemporaryFile( dir=tmp_dir )
-        tmpf1bai = '%s.bai' % tmpf1.name
+    def sniff( self, filename ):
+        # The first 4 bytes of any bam file is 'BAM\1', and the file is binary. 
         try:
-            os.system( 'cd %s' % tmp_dir )
-            os.system( 'cp %s %s' % ( dataset.file_name, tmpf1.name ) )
-            os.system( 'samtools index %s' % tmpf1.name )
-            os.system( 'cp %s %s' % ( tmpf1bai, index_file.file_name ) )
-        except Exception, ex:
-            sys.stderr.write( 'There was a problem creating the index for the BAM file\n%s\n' + str( ex ) )
-        tmpf1.close()
-        if os.path.exists( tmpf1bai ):
-            os.remove( tmpf1bai )
-        dataset.metadata.bam_index = index_file
+            header = open( filename ).read(4)
+            if binascii.b2a_hex( header ) == binascii.hexlify( 'BAM\1' ):
+                return True
+            return False
+        except:
+            return False
     def set_peek( self, dataset, is_multi_byte=False ):
         if not dataset.dataset.purged:
             export_url = "/history_add_to?" + urlencode( {'history_id':dataset.history_id,'ext':'bam','name':'bam alignments','info':'Alignments file','dbkey':dataset.dbkey} )
@@ -84,9 +98,6 @@ class Bam( Binary ):
             return dataset.peek
         except:
             return "Binary bam alignments file (%s)" % ( data.nice_size( dataset.get_size() ) )
-    def get_mime( self ):
-        """Returns the mime type of the datatype"""
-        return 'application/octet-stream'
 
 class Binseq( Binary ):
     """Class describing a zip archive of binary sequence files"""

lib/galaxy/datatypes/registry.py

@@ -174,6 +174,7 @@ class Registry( object ):
         # because some formats are much more flexibly defined than others.
         if len(self.sniff_order) < 1:
             self.sniff_order = [
+                binary.Bam(),
                 binary.Sff(),
                 xml.BlastXml(),
                 sequence.Maf(),

lib/galaxy/datatypes/sniff.py

@@ -252,6 +252,9 @@ def guess_ext( fname, sniff_order=None, is_multi_byte=False ):
     >>> fname = get_test_fname('1.sff')
     >>> guess_ext(fname)
     'sff'
+    >>> fname = get_test_fname('1.bam')
+    >>> guess_ext(fname)
+    'bam'
     """
     if sniff_order is None:
         datatypes_registry = registry.Registry()

test/functional/test_get_data.py

@@ -520,7 +520,23 @@ class UploadData( TwillTestCase ):
         self.check_history_for_string( '1.axt format: <span class="axt">axt</span>, database: \? Info: uploaded file' )
         self.check_metadata_for_string( 'value="1.axt" value="\?" Change data type selected value="axt" selected="yes"' )
         self.delete_history( id=self.security.encode_id( history.id ) )
-    def test_0150_url_paste( self ):
+    def test_0150_upload_file( self ):
+        """Test uploading 1.bam, NOT setting the file format"""
+        self.check_history_for_string( 'Your history is empty' )
+        history = sa_session.query( galaxy.model.History ) \
+                            .filter( and_( galaxy.model.History.table.c.deleted==False,
+                                           galaxy.model.History.table.c.user_id==admin_user.id ) ) \
+                            .order_by( desc( galaxy.model.History.table.c.create_time ) ) \
+                            .first()
+        self.upload_file( '1.bam' )
+        hda = sa_session.query( galaxy.model.HistoryDatasetAssociation ) \
+                        .order_by( desc( galaxy.model.HistoryDatasetAssociation.table.c.create_time ) ) \
+                        .first()
+        assert hda is not None, "Problem retrieving hda from database"
+        self.verify_dataset_correctness( '1.bam', hid=str( hda.hid ) )
+        self.check_history_for_string( '<span class="bam">bam</span>' )
+        self.delete_history( id=self.security.encode_id( history.id ) )
+    def test_0155_url_paste( self ):
         """Test url paste behavior"""
         # Logged in as admin_user
         # Deleting the current history should have created a new history

tools/data_source/upload.xml

@@ -57,6 +57,12 @@ blastz pairwise alignment format.  Each alignment block in an axt file contains
 
 -----
 
+**Bam**
+
+A binary file compressed in the BGZF format with a '.bam' file extension.
+
+-----
+
 **Binseq.zip**
 
 A zipped archive consisting of binary sequence files in either 'ab1' or 'scf' format.  All files in this archive must have the same file extension which is one of '.ab1' or '.scf'.  You must manually select this 'File Format' when uploading the file.