Merge pull request #7993 from msauria/mcooler

Add mcool datatypeand extend h5 tool output testing

config/datatypes_conf.xml.sample

@@ -162,6 +162,7 @@
     <datatype extension="mz5" type="galaxy.datatypes.binary:H5" subclass="true" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="hivtrace" type="galaxy.datatypes.text:Json" mimetype="application/json" subclass="true" display_in_upload="false"/>
     <datatype extension="cool" type="galaxy.datatypes.binary:Cool" mimetype="application/octet-stream" display_in_upload="true"/>
+    <datatype extension="mcool" type="galaxy.datatypes.binary:MCool" mimetype="application/octet-stream" display_in_upload="true"/>
     <datatype extension="html" type="galaxy.datatypes.text:Html" mimetype="text/html"/>
     <datatype extension="interval" type="galaxy.datatypes.interval:Interval" display_in_upload="true" description="File must start with definition line in the following format (columns may be in any order).">
       <converter file="interval_to_bed_converter.xml" target_datatype="bed"/>
@@ -775,6 +776,7 @@
     <sniffer type="galaxy.datatypes.binary:GAFASQLite"/>
     <sniffer type="galaxy.datatypes.binary:SQlite"/>
     <sniffer type="galaxy.datatypes.binary:Cool"/>
+    <sniffer type="galaxy.datatypes.binary:MCool"/>
     <sniffer type="galaxy.datatypes.binary:Loom"/>
     <sniffer type="galaxy.datatypes.binary:Anndata"/>
     <sniffer type="galaxy.datatypes.binary:Biom2"/>

lib/galaxy/datatypes/binary.py

@@ -1107,6 +1107,66 @@ class Cool(H5):
             return "Cool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
 
 
+class MCool(H5):
+    """
+    Class describing the multi-resolution cool format (https://github.com/mirnylab/cooler)
+    """
+
+    file_ext = "mcool"
+
+    def sniff(self, filename):
+        """
+        >>> from galaxy.datatypes.sniff import get_test_fname
+        >>> fname = get_test_fname('matrix.mcool')
+        >>> MCool().sniff(fname)
+        True
+        >>> fname = get_test_fname('matrix.cool')
+        >>> MCool().sniff(fname)
+        False
+        >>> fname = get_test_fname('test.mz5')
+        >>> MCool().sniff(fname)
+        False
+        >>> fname = get_test_fname('wiggle.wig')
+        >>> MCool().sniff(fname)
+        False
+        >>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom2')
+        >>> MCool().sniff(fname)
+        False
+        """
+
+        MAGIC = "HDF5::Cooler"
+        URL = "https://github.com/mirnylab/cooler"
+
+        if super(MCool, self).sniff(filename):
+            keys0 = ['resolutions']
+            with h5py.File(filename, 'r') as handle:
+                if not all(name in handle.keys() for name in keys0):
+                    return False
+                res0 = list(handle['resolutions'].keys())[0]
+                keys = ['chroms', 'bins', 'pixels', 'indexes']
+                fmt = handle['resolutions'][res0].attrs.get('format', None)
+                url = handle['resolutions'][res0].attrs.get('format-url', None)
+                if fmt == MAGIC or url == URL:
+                    if not all(name in handle['resolutions'][res0].keys() for name in keys):
+                        return False
+                    return True
+        return False
+
+    def set_peek(self, dataset, is_multi_byte=False):
+        if not dataset.dataset.purged:
+            dataset.peek = "Multi-resolution Cool (HDF5) file for storing genomic interaction data."
+            dataset.blurb = nice_size(dataset.get_size())
+        else:
+            dataset.peek = 'file does not exist'
+            dataset.blurb = 'file purged from disk'
+
+    def display_peek(self, dataset):
+        try:
+            return dataset.peek
+        except Exception:
+            return "MCool (HDF5) file (%s)." % (nice_size(dataset.get_size()))
+
+
 class Scf(Binary):
     """Class describing an scf binary sequence file"""
     edam_format = "format_1632"

lib/galaxy/tools/verify/asserts/hdf5.py

@@ -24,10 +24,19 @@ def assert_has_h5_attribute(output_bytes, key, value):
 
 
 def assert_has_h5_keys(output_bytes, keys):
-    """ Asserts the specified HDF5 output has exactly the given keys."""
+    """ Asserts the specified HDF5 output has the given keys."""
     _assert_h5py()
     keys = [k.strip() for k in keys.strip().split(',')]
     h5_keys = sorted(keys)
     output_temp = io.BytesIO(output_bytes)
-    local_keys = sorted(list(h5py.File(output_temp, 'r').keys()))
-    assert local_keys == h5_keys, "Not a HDF5 file or H5 keys do not match:\n\t%s\n\t%s" % (local_keys, h5_keys)
+    local_keys = []
+
+    def append_keys(key):
+        local_keys.append(key)
+        return None
+    h5py.File(output_temp, 'r').visit(append_keys)
+    missing = 0
+    for key in h5_keys:
+        if key not in local_keys:
+            missing += 1
+    assert missing == 0, "Not a HDF5 file or H5 keys missing:\n\t%s\n\t%s" % (local_keys, h5_keys)

lib/galaxy/tools/xsd/galaxy.xsd

@@ -1669,7 +1669,7 @@ module.
   </xs:group>
   <xs:complexType name="AssertHasH5Keys">
     <xs:annotation>
-      <xs:documentation xml:lang="en"><![CDATA[Asserts HDF5 output has a set of attributes (``keys``), specified as a comma-separated list (e.g. ``<has_h5_keys keys="bins,chroms,indexes,pixels" />``).]]></xs:documentation>
+      <xs:documentation xml:lang="en"><![CDATA[Asserts HDF5 output has a set of attributes (``keys``), specified as a comma-separated list (e.g. ``<has_h5_keys keys="bins,chroms,indexes,pixels,chroms/lengths" />``).]]></xs:documentation>
     </xs:annotation>
     <xs:attribute name="keys" type="xs:string">
       <xs:annotation>

test/functional/tools/validation_hdf5.xml

@@ -13,7 +13,7 @@
             <param name="input" value="matrix.cool" />
             <output name="output">
                 <assert_contents>
-                    <has_h5_keys keys="bins, chroms,indexes,pixels" />
+                    <has_h5_keys keys="bins, chroms,indexes,pixels,chroms/length" />
                     <has_h5_attribute key="nbins" value="33754" />
                     <has_h5_attribute key="nchroms" value="15" />
                 </assert_contents>