Merge pull request #4519 from shiltemann/biom2_datatype

add biom2 datatype

config/datatypes_conf.xml.sample

@@ -505,9 +505,13 @@
     <datatype extension="plybinary" type="galaxy.datatypes.constructive_solid_geometry:PlyBinary" display_in_upload="true" />
     <datatype extension="vtkascii" type="galaxy.datatypes.constructive_solid_geometry:VtkAscii" display_in_upload="true" />
     <datatype extension="vtkbinary" type="galaxy.datatypes.constructive_solid_geometry:VtkBinary" display_in_upload="true" />
-    <!-- Metagenomic Datatype -->
+    <!-- Metagenomic Datatypes -->
     <datatype extension="biom1" type="galaxy.datatypes.text:Biom1" display_in_upload="true" subclass="true" mimetype="application/json">
         <display file="biom/biom_simple.xml" />
+        <converter file="biom1_to_biom2.xml" target_datatype="biom2"/>
+    </datatype>
+    <datatype extension="biom2" type="galaxy.datatypes.binary:Biom2" mimetype="application/octet-stream" display_in_upload="true">
+        <converter file="biom2_to_biom1.xml" target_datatype="biom1"/>
     </datatype>
     <!-- Strand-specific Coordinate Count Datatype used by the Center for Eukaryotic Gene Regulation labs at Penn State -->
     <datatype extension="scidx" type="galaxy.datatypes.interval:ScIdx" display_in_upload="true" />
@@ -662,6 +666,7 @@
     <sniffer type="galaxy.datatypes.binary:MzSQlite"/>
     <sniffer type="galaxy.datatypes.binary:IdpDB"/>
     <sniffer type="galaxy.datatypes.binary:SQlite"/>
+    <sniffer type="galaxy.datatypes.binary:Biom2"/>
     <sniffer type="galaxy.datatypes.binary:H5"/>
     <sniffer type="galaxy.datatypes.binary:Bam"/>
     <sniffer type="galaxy.datatypes.binary:CRAM"/>

lib/galaxy/datatypes/binary.py

@@ -8,10 +8,12 @@ import os
 import shutil
 import struct
 import subprocess
+import sys
 import tempfile
 import zipfile
 from json import dumps
 
+import h5py
 import pysam
 from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE
 
@@ -810,6 +812,90 @@ class H5(Binary):
             return "Binary HDF5 file (%s)" % (nice_size(dataset.get_size()))
 
 
+class Biom2(H5):
+    """
+    Class describing a biom2 file (http://biom-format.org/documentation/biom_format.html)
+    """
+    MetadataElement(name="id", default=None, desc="table id", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="format_url", default=None, desc="format-url", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="format_version", default=None, desc="format-version", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="format", default=None, desc="format", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="type", default=None, desc="table type", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="generated_by", default=None, desc="generated by", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="creation_date", default=None, desc="creation date", readonly=True, visible=True, no_value=None)
+    MetadataElement(name="nnz", default=-1, desc="nnz: The number of non-zero elements in the table", readonly=True, visible=True, no_value=-1)
+    MetadataElement(name="shape", default=(), desc="shape: The number of rows and columns in the dataset", readonly=True, visible=True, no_value=())
+
+    file_ext = "biom2"
+    edam_format = "format_3746"
+
+    def sniff(self, filename):
+        """
+        >>> from galaxy.datatypes.sniff import get_test_fname
+        >>> fname = get_test_fname( 'biom2_sparse_otu_table_hdf5.biom' )
+        >>> Biom2().sniff( fname )
+        True
+        >>> fname = get_test_fname( 'test.mz5' )
+        >>> Biom2().sniff( fname )
+        False
+        >>> fname = get_test_fname( 'wiggle.wig' )
+        >>> Biom2().sniff( fname )
+        False
+        """
+        if super(Biom2, self).sniff(filename):
+            try:
+                f = h5py.File(filename)
+                attributes = list(dict(f.attrs.items()))
+                required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape']
+                return set(required_fields).issubset(attributes)
+            except Exception:
+                return False
+        return False
+
+    def set_meta(self, dataset, overwrite=True, **kwd):
+        super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd)
+        try:
+            f = h5py.File(dataset.file_name)
+            attributes = dict(f.attrs.items())
+
+            dataset.metadata.id = attributes['id']
+            dataset.metadata.format_url = attributes['format-url']
+            if 'format-version' in attributes:  # biom 2.1
+                dataset.metadata.format_version = '.'.join(map(str, list(attributes['format-version'])))
+            elif 'format' in attributes:  # biom 2.0
+                dataset.metadata.format = attributes['format']
+            dataset.metadata.type = attributes['type']
+            dataset.metadata.shape = tuple(attributes['shape'])
+            dataset.metadata.generated_by = attributes['generated-by']
+            dataset.metadata.creation_date = attributes['creation-date']
+            dataset.metadata.nnz = int(attributes['nnz'])
+
+        except Exception as e:
+            log.warning('%s, set_meta Exception: %s', self, e)
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            lines = ['Biom2 (HDF5) file']
+            try:
+                f = h5py.File(dataset.file_name)
+                for k, v in dict(f.attrs).items():
+                    lines.append('%s:  %s' % (k, v))
+            except Exception as e:
+                log.warning('%s, set_peek Exception: %s', self, e)
+            dataset.peek = '\n'.join(lines)
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except:
+            return "Biom2 (HDF5) file (%s)" % (nice_size(dataset.get_size()))
+
+
+Binary.register_sniffable_binary_format("biom2", "biom2", Biom2)
 Binary.register_sniffable_binary_format("h5", "h5", H5)
 
 
@@ -1607,3 +1693,8 @@ class DMND(Binary):
 
 
 Binary.register_sniffable_binary_format("dmnd", "dmnd", DMND)
+
+
+if __name__ == '__main__':
+    import doctest
+    doctest.testmod(sys.modules[__name__])

lib/galaxy/datatypes/converters/biom1_to_biom2.xml

@@ -0,0 +1,14 @@
+<tool id="CONVERTER_biom1_to_biom2" name="Convert Biom1 to Biom2" version="2.1.5">
+    <requirements>
+        <requirement type="package" version="2.1.5">biom-format</requirement>
+    </requirements>
+    <command>biom convert -i '$input' -o '$output' --to-hdf5 </command>
+    <inputs>
+        <param name="input" type="data" format="biom1" label="Biom1 file"/>
+    </inputs>
+    <outputs>
+        <data name="output" format="biom2"/>
+    </outputs>
+    <help>
+    </help>
+</tool>

lib/galaxy/datatypes/converters/biom2_to_biom1.xml

@@ -0,0 +1,14 @@
+<tool id="CONVERTER_biom2_to_biom1" name="Convert Biom2 to Biom1" version="2.1.5">
+    <requirements>
+        <requirement type="package" version="2.1.5">biom-format</requirement>
+    </requirements>
+    <command>biom convert -i '$input' -o '$output' --to-json </command>
+    <inputs>
+        <param name="input" type="data" format="biom2" label="Biom2 file"/>
+    </inputs>
+    <outputs>
+        <data name="output" format="biom1"/>
+    </outputs>
+    <help>
+    </help>
+</tool>

lib/galaxy/datatypes/sniff.py

@@ -390,6 +390,9 @@ def guess_ext(fname, sniff_order, is_multi_byte=False):
     >>> fname = get_test_fname('1.xls')
     >>> guess_ext(fname, sniff_order)
     'excel.xls'
+    >>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom')
+    >>> guess_ext(fname, sniff_order)
+    'biom2'
     """
     file_ext = None
     for datatype in sniff_order:

lib/galaxy/datatypes/text.py

@@ -170,6 +170,7 @@ class Biom1(Json):
         http://biom-format.org/documentation/format_versions/biom-1.0.html
     """
     file_ext = "biom1"
+    edam_format = "format_3746"
 
     MetadataElement(name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[])
     MetadataElement(name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="")

lib/galaxy/dependencies/pinned-requirements.txt

@@ -33,6 +33,7 @@ six==1.10.0
 Whoosh==2.7.4
 testfixtures==4.10.0
 galaxy_sequence_utils==1.0.2
+h5py==2.7.1
 
 # pykwalify and dependencies
 pykwalify==1.5.1