diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index edfca39889b..d781b0f3471 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -505,9 +505,13 @@ - + + + + + @@ -662,6 +666,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index e46eb47081b..b68df34b433 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -8,10 +8,12 @@ import os import shutil import struct import subprocess +import sys import tempfile import zipfile from json import dumps +import h5py import pysam from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE @@ -810,6 +812,90 @@ class H5(Binary): return "Binary HDF5 file (%s)" % (nice_size(dataset.get_size())) +class Biom2(H5): + """ + Class describing a biom2 file (http://biom-format.org/documentation/biom_format.html) + """ + MetadataElement(name="id", default=None, desc="table id", readonly=True, visible=True, no_value=None) + MetadataElement(name="format_url", default=None, desc="format-url", readonly=True, visible=True, no_value=None) + MetadataElement(name="format_version", default=None, desc="format-version", readonly=True, visible=True, no_value=None) + MetadataElement(name="format", default=None, desc="format", readonly=True, visible=True, no_value=None) + MetadataElement(name="type", default=None, desc="table type", readonly=True, visible=True, no_value=None) + MetadataElement(name="generated_by", default=None, desc="generated by", readonly=True, visible=True, no_value=None) + MetadataElement(name="creation_date", default=None, desc="creation date", readonly=True, visible=True, no_value=None) + MetadataElement(name="nnz", default=-1, desc="nnz: The number of non-zero elements in the table", readonly=True, visible=True, no_value=-1) + MetadataElement(name="shape", default=(), desc="shape: The number of rows and columns in the dataset", readonly=True, visible=True, no_value=()) + + file_ext = "biom2" + edam_format = "format_3746" + + def sniff(self, filename): + """ + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( 'biom2_sparse_otu_table_hdf5.biom' ) + >>> Biom2().sniff( fname ) + True + >>> fname = get_test_fname( 'test.mz5' ) + >>> Biom2().sniff( fname ) + False + >>> fname = get_test_fname( 'wiggle.wig' ) + >>> Biom2().sniff( fname ) + False + """ + if super(Biom2, self).sniff(filename): + try: + f = h5py.File(filename) + attributes = list(dict(f.attrs.items())) + required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'] + return set(required_fields).issubset(attributes) + except Exception: + return False + return False + + def set_meta(self, dataset, overwrite=True, **kwd): + super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd) + try: + f = h5py.File(dataset.file_name) + attributes = dict(f.attrs.items()) + + dataset.metadata.id = attributes['id'] + dataset.metadata.format_url = attributes['format-url'] + if 'format-version' in attributes: # biom 2.1 + dataset.metadata.format_version = '.'.join(map(str, list(attributes['format-version']))) + elif 'format' in attributes: # biom 2.0 + dataset.metadata.format = attributes['format'] + dataset.metadata.type = attributes['type'] + dataset.metadata.shape = tuple(attributes['shape']) + dataset.metadata.generated_by = attributes['generated-by'] + dataset.metadata.creation_date = attributes['creation-date'] + dataset.metadata.nnz = int(attributes['nnz']) + + except Exception as e: + log.warning('%s, set_meta Exception: %s', self, e) + + def set_peek(self, dataset, is_multi_byte=False): + if not dataset.dataset.purged: + lines = ['Biom2 (HDF5) file'] + try: + f = h5py.File(dataset.file_name) + for k, v in dict(f.attrs).items(): + lines.append('%s: %s' % (k, v)) + except Exception as e: + log.warning('%s, set_peek Exception: %s', self, e) + dataset.peek = '\n'.join(lines) + dataset.blurb = nice_size(dataset.get_size()) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Biom2 (HDF5) file (%s)" % (nice_size(dataset.get_size())) + + +Binary.register_sniffable_binary_format("biom2", "biom2", Biom2) Binary.register_sniffable_binary_format("h5", "h5", H5) @@ -1607,3 +1693,8 @@ class DMND(Binary): Binary.register_sniffable_binary_format("dmnd", "dmnd", DMND) + + +if __name__ == '__main__': + import doctest + doctest.testmod(sys.modules[__name__]) diff --git a/lib/galaxy/datatypes/converters/biom1_to_biom2.xml b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml new file mode 100644 index 00000000000..8dc76d09e26 --- /dev/null +++ b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml @@ -0,0 +1,14 @@ + + + biom-format + + biom convert -i '$input' -o '$output' --to-hdf5 + + + + + + + + + diff --git a/lib/galaxy/datatypes/converters/biom2_to_biom1.xml b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml new file mode 100644 index 00000000000..559763ff74f --- /dev/null +++ b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml @@ -0,0 +1,14 @@ + + + biom-format + + biom convert -i '$input' -o '$output' --to-json + + + + + + + + + diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 95621d6b294..18b5ad1387f 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -390,6 +390,9 @@ def guess_ext(fname, sniff_order, is_multi_byte=False): >>> fname = get_test_fname('1.xls') >>> guess_ext(fname, sniff_order) 'excel.xls' + >>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom') + >>> guess_ext(fname, sniff_order) + 'biom2' """ file_ext = None for datatype in sniff_order: diff --git a/lib/galaxy/datatypes/test/biom2_sparse_otu_table_hdf5.biom b/lib/galaxy/datatypes/test/biom2_sparse_otu_table_hdf5.biom new file mode 100644 index 00000000000..b3c019bf851 Binary files /dev/null and b/lib/galaxy/datatypes/test/biom2_sparse_otu_table_hdf5.biom differ diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 525ccb64f96..4d71302b3f0 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -170,6 +170,7 @@ class Biom1(Json): http://biom-format.org/documentation/format_versions/biom-1.0.html """ file_ext = "biom1" + edam_format = "format_3746" MetadataElement(name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) MetadataElement(name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="") diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 8448952bb75..4c4f7601c3b 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -33,6 +33,7 @@ six==1.10.0 Whoosh==2.7.4 testfixtures==4.10.0 galaxy_sequence_utils==1.0.2 +h5py==2.7.1 # pykwalify and dependencies pykwalify==1.5.1