From 80c9763757a626355a70e360fd9aedffd8f8dac3 Mon Sep 17 00:00:00 2001 From: shiltemann Date: Tue, 29 Aug 2017 17:01:43 +0200 Subject: [PATCH 01/18] add biom2 datatype --- config/datatypes_conf.xml.sample | 4 ++- lib/galaxy/datatypes/binary.py | 56 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 95235fed99f..8a8f54fb0cb 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -505,10 +505,11 @@ - + + @@ -663,6 +664,7 @@ + diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index e46eb47081b..9a7e2a0d2d8 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -8,6 +8,7 @@ import os import shutil import struct import subprocess +import sys import tempfile import zipfile from json import dumps @@ -813,6 +814,56 @@ class H5(Binary): Binary.register_sniffable_binary_format("h5", "h5", H5) +class Biom2(H5): + """ + Class describing a biom2 file + + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( 'biom2_sparse_otu_table_hdf5.biom' ) + >>> Biom2().sniff( fname ) + True + >>> fname = get_test_fname( 'test.mz5' ) + >>> Biom2().sniff( fname ) + False + >>> fname = get_test_fname( 'wiggle.wig' ) + >>> Biom2().sniff( fname ) + False + """ + file_ext = "biom2" + edam_format = "format_3746" + + def sniff(self, filename): + if super(Biom2, self).sniff(filename): + # check if HDF5 file is a biom2 file http://biom-format.org/documentation/biom_format.html + try: + header = open(filename, 'rb').read(8192) + required_fields = ['format-url', 'observation', 'sample'] + for field in required_fields: + if field not in header: + return False + return True + except Exception: + return False + return False + + def set_peek(self, dataset, is_multi_byte=False): + if not dataset.dataset.purged: + dataset.peek = "Biom2 (HDF5) file" + dataset.blurb = nice_size(dataset.get_size()) + else: + dataset.peek = 'file does not exist' + dataset.blurb = 'file purged from disk' + + def display_peek(self, dataset): + try: + return dataset.peek + except: + return "Biom2 (HDF5) file (%s)" % (nice_size(dataset.get_size())) + + +Binary.register_sniffable_binary_format("biom2", "biom2", Biom2) + + class Scf(Binary): """Class describing an scf binary sequence file""" edam_format = "format_1632" @@ -1607,3 +1658,8 @@ class DMND(Binary): Binary.register_sniffable_binary_format("dmnd", "dmnd", DMND) + + +if __name__ == '__main__': + import doctest + doctest.testmod(sys.modules[__name__]) From 86902068268903292d1caf003a2274e0a5438f7d Mon Sep 17 00:00:00 2001 From: shiltemann Date: Tue, 29 Aug 2017 17:02:11 +0200 Subject: [PATCH 02/18] add test biom2 file --- .../test/biom2_sparse_otu_table_hdf5.biom | Bin 0 -> 33800 bytes 1 file changed, 0 insertions(+), 0 deletions(-) create mode 100644 lib/galaxy/datatypes/test/biom2_sparse_otu_table_hdf5.biom diff --git a/lib/galaxy/datatypes/test/biom2_sparse_otu_table_hdf5.biom b/lib/galaxy/datatypes/test/biom2_sparse_otu_table_hdf5.biom new file mode 100644 index 0000000000000000000000000000000000000000..b3c019bf8515f8e05176fac398e0cf114281bc82 GIT binary patch literal 33800 zcmeI5O-vg{6oB8_2HZdbsj8Ixc3Y(>J(yr{g2N#pzqo2dXhS1%N$dp!mH;DTBB!E} zQL2hYtrV$J6;(Z?Kap}o>Lu!lG^vyxD&^1vm2yCB#i@rxttu*2N@sT7TjIs*B?(S2 zek0-8nR)YO=9?dew=?GB$dO~~*KJt`()s=1gF5q(J|3d;f2d+8OzBa4(9oYk|GGy# zXs`}sT0O?sp?=+~A~Gn~{qE7DAS6jY`Irr&sDSiPJp1KAAae9zmuzVdHdjag4vrrT zLedzEr_ROFiR6eGH5MBl9g545rCgW&{=Kj=})GHW9h(nYRHUs z)wfBAQN85UtN|0Z&{r#->jCwhi>Jm+v~{8lsu}HO0h*>-OWk4jiT`3f*PHbmh>yfm zv2?sIV4RnV+}d3wWT>7#@w}*SgQ>4K6*oIS&?l5GL*JH1^i|~Ot2Em^HW)KIWV!8? zI;5*qp3u=`K51fQtr5gVM&38WOZ1_9E&7@uFPYJLN8@k?3z(B2gmYUr2e&CX(xXEB zSuO!!8-;Tl97#A1<3WI(KAV`kKFJRK8LC`Rc6spR-*U@?~FvrYmIe#hYx$4yze zAPPtMuD*Vf?~B*POewd4W`qhZM%0HiR5rV ztx=81)BuRtrW{Tah-cI<0-B$4-b>fq6buELS_94dx`TV$f_vIREsZU~a7$~eN~d{D zxH>w{p6Gt>Y>4#Q;`dOPe3WmY_SoW^)y9j~QUH_>5+5q(q*?utSVNS{jo$<7^TzLm z4SC~R#HyZ8Pc#0u1IX1ML}`F9DYRnvfeA1HCcp%k025#WOn?b60Vco%m;k`Z&J#y8 z@Sv>_;0pc$PIdshJ&WvAnRgUDlDmrGnUm#7)yW=sceCPgf2;D{?1A@pDLLX?o;1t1 z)pmQJjwtHTPE2IT_LmsxOZ3LaK=`BS)S~^)vLjLk7hkOdK4?I>8=dNb=Jy9fxZ8t)5l=a#Dyhsj z=~8r~e!zNP6RL*LLo}pTh*+8K@hY{|pqmf)I!8RqbPf9A{X_DN3fWctsMqW9%0i#j35|Cn?dD`;_aQtRKBECjSkGBEmEPF`YPB6?e$T(M*5|FaajO z1eieS5>We4ueDK`-d?yZv=0q86_%s@WoBPtxwVp-Inx`3$=ZLSmk!?64&7R)>DpiQ zMfj6V*-N*6xjC2~zY>DU4{ExFn308JdVi&{N$;;Pb~U_m`=g&O-fo|$-wM8O;dQA; zIG?dspZ6JO^!X|Sv@ao$IrF=Sj?6{D+o)FtV-uaB{ZohLzx(RJuaosK5cZrlYoKkc zYWi^ot{s^DtmD?-)$NH54WBkXytxBrRpEU?Nu_bD;pZPfJ06+1yltxX>)6ocZQ$Dp zI+pD*PRX)I^m)y=pjUz?3auh(0E^f3{*K@N`sTsEcNgvrPV5Nn-(Ba0&;RiJxmLQ8 z+rb2w025#WOkmjr9^VdCC#*E$T(mH>C>xS)mbC)*rwE=-lcpRpPvx;c^_b-q>%++o z^{PXGXC3TNRs$UJaWoTP0!)AjFaah|x&++zPjv5qe6Ki}U!?uhRGY#mi?X&KOW)r; zpZycr)$Bc;we6pXlhpuw1`aa;Ccp%k025#WB~QR@|3vpZ$)|HrPWQ`Kzz%iiu%ar9 zq}PUd>`>449Mo$LJ$=QteN;He4N@1wT&6jn+5&^tz9IoeNfC-c?0k{1V{XYoui+CTUepq3YMOoXA)v|xWui0GE z(f`@@PsGV;z>*vs!33B96JP>NfC-d30k{1V-A^T--h&qLK1$Cy1yvSFX2a+H9(1Qe zN5A}LZUS+#DxRCnAtt~Cm;e)C0!&~H5O6wAYCjL>cAlTlldgfm@Dxq3pC_erboaBl mO6go?S8ntZhn&W*{TwNe-zP>~78#w!?=+r6CI~A%e*XvBCayjJ literal 0 HcmV?d00001 From b9fb935be594db1224c524bfd03c22b8b90f4bea Mon Sep 17 00:00:00 2001 From: shiltemann Date: Tue, 29 Aug 2017 17:02:46 +0200 Subject: [PATCH 03/18] add edam format to biom1 datatype --- lib/galaxy/datatypes/text.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 525ccb64f96..4d71302b3f0 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -170,6 +170,7 @@ class Biom1(Json): http://biom-format.org/documentation/format_versions/biom-1.0.html """ file_ext = "biom1" + edam_format = "format_3746" MetadataElement(name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[]) MetadataElement(name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="") From 8314867ecb57b58caceb4843c021f300dc39472e Mon Sep 17 00:00:00 2001 From: shiltemann Date: Tue, 29 Aug 2017 17:25:26 +0200 Subject: [PATCH 04/18] fix sniffer order --- config/datatypes_conf.xml.sample | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 8a8f54fb0cb..4295e3fb62c 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -663,8 +663,8 @@ - + From 6feb3e22d19c2cb162adc0b5d80655540305a805 Mon Sep 17 00:00:00 2001 From: shiltemann Date: Tue, 29 Aug 2017 17:53:15 +0200 Subject: [PATCH 05/18] order matters --- lib/galaxy/datatypes/binary.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 9a7e2a0d2d8..ceba757a100 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -811,9 +811,6 @@ class H5(Binary): return "Binary HDF5 file (%s)" % (nice_size(dataset.get_size())) -Binary.register_sniffable_binary_format("h5", "h5", H5) - - class Biom2(H5): """ Class describing a biom2 file @@ -862,6 +859,7 @@ class Biom2(H5): Binary.register_sniffable_binary_format("biom2", "biom2", Biom2) +Binary.register_sniffable_binary_format("h5", "h5", H5) class Scf(Binary): From 07ef9b4f4b2a38259ae55f382ebee36e43743a9d Mon Sep 17 00:00:00 2001 From: shiltemann Date: Tue, 29 Aug 2017 18:28:03 +0200 Subject: [PATCH 06/18] use h5py to parse hdf5 header --- lib/galaxy/datatypes/binary.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index ceba757a100..042517c3de0 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -3,6 +3,7 @@ from __future__ import print_function import binascii import gzip +import h5py import logging import os import shutil @@ -833,10 +834,11 @@ class Biom2(H5): if super(Biom2, self).sniff(filename): # check if HDF5 file is a biom2 file http://biom-format.org/documentation/biom_format.html try: - header = open(filename, 'rb').read(8192) - required_fields = ['format-url', 'observation', 'sample'] + f = h5py.File(filename) + attributes = dict(f.attrs.items()) + required_fields = ['format-url', 'format-version', 'generated-by'] for field in required_fields: - if field not in header: + if field not in attributes: return False return True except Exception: From ca1ebf97c02e7261fc36352206834de5ab74a0fd Mon Sep 17 00:00:00 2001 From: Saskia Hiltemann Date: Tue, 29 Aug 2017 22:47:08 +0200 Subject: [PATCH 07/18] set metadata --- lib/galaxy/datatypes/binary.py | 59 ++++++++++++++++++++++++++-------- 1 file changed, 45 insertions(+), 14 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 042517c3de0..5c8ad982f39 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -814,29 +814,39 @@ class H5(Binary): class Biom2(H5): """ - Class describing a biom2 file - - >>> from galaxy.datatypes.sniff import get_test_fname - >>> fname = get_test_fname( 'biom2_sparse_otu_table_hdf5.biom' ) - >>> Biom2().sniff( fname ) - True - >>> fname = get_test_fname( 'test.mz5' ) - >>> Biom2().sniff( fname ) - False - >>> fname = get_test_fname( 'wiggle.wig' ) - >>> Biom2().sniff( fname ) - False + Class describing a biom2 file (http://biom-format.org/documentation/biom_format.html) """ + MetadataElement(name="id", default=None, desc="table id", readonly=True, visible=True, no_value=None) + MetadataElement(name="format_url", default=None, desc="format-url", readonly=True, visible=True, no_value=None) + MetadataElement(name="format_version", default=None, desc="format-version", readonly=True, visible=True, no_value=None) + MetadataElement(name="format", default=None, desc="format", readonly=True, visible=True, no_value=None) + MetadataElement(name="type", default=None, desc="table type", readonly=True, visible=True, no_value=None) + MetadataElement(name="generated_by", default=None, desc="generated by", readonly=True, visible=True, no_value=None) + MetadataElement(name="creation_date", default=None, desc="creation date", readonly=True, visible=True, no_value=None) + MetadataElement(name="nnz", default=None, desc="nnz: The number of non-zero elements in the table", readonly=True, visible=True, no_value=None) + MetadataElement(name="shape", default=None, desc="shape: The number of rows and columns in the dataset", readonly=True, visible=True, no_value=None) + file_ext = "biom2" edam_format = "format_3746" def sniff(self, filename): + """ + >>> from galaxy.datatypes.sniff import get_test_fname + >>> fname = get_test_fname( 'biom2_sparse_otu_table_hdf5.biom' ) + >>> Biom2().sniff( fname ) + True + >>> fname = get_test_fname( 'test.mz5' ) + >>> Biom2().sniff( fname ) + False + >>> fname = get_test_fname( 'wiggle.wig' ) + >>> Biom2().sniff( fname ) + False + """ if super(Biom2, self).sniff(filename): - # check if HDF5 file is a biom2 file http://biom-format.org/documentation/biom_format.html try: f = h5py.File(filename) attributes = dict(f.attrs.items()) - required_fields = ['format-url', 'format-version', 'generated-by'] + required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'] for field in required_fields: if field not in attributes: return False @@ -845,6 +855,27 @@ class Biom2(H5): return False return False + def set_meta(self, dataset, overwrite=True, **kwd): + super(Biom2, self).set_meta(dataset, overwrite=overwrite, **kwd) + try: + f = h5py.File(dataset.file_name) + attributes = dict(f.attrs.items()) + + dataset.metadata.id = attributes['id'] + dataset.metadata.format_url = attributes['format-url'] + if 'format-version' in attributes: # biom 2.1 + dataset.metadata.format_version = '.'.join(map(str, list(attributes['format-version']))) + elif 'format' in attributes: # biom 2.0 + dataset.metadata.format = attributes['format'] + dataset.metadata.type = attributes['type'] + dataset.metadata.shape = str(list(attributes['shape'])) + dataset.metadata.generated_by = attributes['generated-by'] + dataset.metadata.creation_date = attributes['creation-date'] + dataset.metadata.nnz = str(attributes['nnz']) + + except Exception as e: + log.warning('%s, set_meta Exception: %s', self, e) + def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: dataset.peek = "Biom2 (HDF5) file" From ac61072452cd345d45e87d3548e6564858589029 Mon Sep 17 00:00:00 2001 From: Saskia Hiltemann Date: Tue, 29 Aug 2017 22:48:54 +0200 Subject: [PATCH 08/18] add h5py requirement for extracting metadata from biom2/hdf5 datatypes --- lib/galaxy/dependencies/pinned-requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index 9c8eab78bd0..d35a858993e 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -32,6 +32,7 @@ six==1.10.0 Whoosh==2.7.4 testfixtures==4.10.0 galaxy_sequence_utils==1.0.2 +h5py==2.7.0 # pykwalify and dependencies pykwalify==1.5.1 @@ -80,4 +81,4 @@ pysam==0.8.4+gx5 chronos-python==0.38.0 # GenomeSpace dependencies -python-genomespaceclient==0.1.8 \ No newline at end of file +python-genomespaceclient==0.1.8 From 92e25158410238bfbc80a931fedb80a55b570e2d Mon Sep 17 00:00:00 2001 From: Saskia Hiltemann Date: Wed, 30 Aug 2017 02:09:18 +0200 Subject: [PATCH 09/18] fix import order --- lib/galaxy/datatypes/binary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 5c8ad982f39..62b50f79461 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -3,7 +3,6 @@ from __future__ import print_function import binascii import gzip -import h5py import logging import os import shutil @@ -13,6 +12,7 @@ import sys import tempfile import zipfile from json import dumps +import h5py import pysam from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE From 4119caf9b7fbd1fc9c374403e7f7836b8d9397c1 Mon Sep 17 00:00:00 2001 From: Saskia Hiltemann Date: Wed, 30 Aug 2017 02:10:02 +0200 Subject: [PATCH 10/18] remove unused variable --- lib/galaxy/datatypes/data.py | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index e8aa5c6b13b..988be4272ad 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -369,7 +369,6 @@ class Data(object): if os.path.exists(file_path): if os.path.isdir(file_path): tmp_fh = tempfile.NamedTemporaryFile(delete=False) - tmp_file_name = tmp_fh.name dir_items = sorted(os.listdir(file_path)) base_path, item_name = os.path.split(file_path) tmp_fh.write('

Directory %s contents: %d items

\n' % (escape(item_name), len(dir_items))) From 062c88a3cce66df350dc0da7aed14542fe6ad65b Mon Sep 17 00:00:00 2001 From: shiltemann Date: Wed, 30 Aug 2017 11:44:49 +0200 Subject: [PATCH 11/18] add converters --- config/datatypes_conf.xml.sample | 5 ++++- lib/galaxy/datatypes/converters/biom1_to_biom2.xml | 14 ++++++++++++++ lib/galaxy/datatypes/converters/biom2_to_biom1.xml | 14 ++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/datatypes/converters/biom1_to_biom2.xml create mode 100644 lib/galaxy/datatypes/converters/biom2_to_biom1.xml diff --git a/config/datatypes_conf.xml.sample b/config/datatypes_conf.xml.sample index 4295e3fb62c..ea7d0e6219d 100644 --- a/config/datatypes_conf.xml.sample +++ b/config/datatypes_conf.xml.sample @@ -508,8 +508,11 @@ + + + + - diff --git a/lib/galaxy/datatypes/converters/biom1_to_biom2.xml b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml new file mode 100644 index 00000000000..8dc76d09e26 --- /dev/null +++ b/lib/galaxy/datatypes/converters/biom1_to_biom2.xml @@ -0,0 +1,14 @@ + + + biom-format + + biom convert -i '$input' -o '$output' --to-hdf5 + + + + + + + + + diff --git a/lib/galaxy/datatypes/converters/biom2_to_biom1.xml b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml new file mode 100644 index 00000000000..559763ff74f --- /dev/null +++ b/lib/galaxy/datatypes/converters/biom2_to_biom1.xml @@ -0,0 +1,14 @@ + + + biom-format + + biom convert -i '$input' -o '$output' --to-json + + + + + + + + + From a3b316031c5aa1565d7255eab5aa92ff4a749f4d Mon Sep 17 00:00:00 2001 From: shiltemann Date: Wed, 30 Aug 2017 14:11:22 +0200 Subject: [PATCH 12/18] simplify --- lib/galaxy/datatypes/binary.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 62b50f79461..210952a7a54 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -845,12 +845,9 @@ class Biom2(H5): if super(Biom2, self).sniff(filename): try: f = h5py.File(filename) - attributes = dict(f.attrs.items()) + attributes = list(dict(f.attrs.items())) required_fields = ['id', 'format-url', 'type', 'generated-by', 'creation-date', 'nnz', 'shape'] - for field in required_fields: - if field not in attributes: - return False - return True + return set(required_fields).issubset(attributes) except Exception: return False return False From 995f2cee4a2914c157420467829ff647a2e8c7d6 Mon Sep 17 00:00:00 2001 From: shiltemann Date: Wed, 30 Aug 2017 14:16:02 +0200 Subject: [PATCH 13/18] add guess_ext test --- lib/galaxy/datatypes/sniff.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 10222f6a7d9..40ffce165d2 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -386,6 +386,9 @@ def guess_ext(fname, sniff_order, is_multi_byte=False): >>> fname = get_test_fname('1.xls') >>> guess_ext(fname, sniff_order) 'excel.xls' + >>> fname = get_test_fname('biom2_sparse_otu_table_hdf5.biom') + >>> guess_ext(fname, sniff_order) + 'biom2' """ file_ext = None for datatype in sniff_order: From 95aacc010e730ce24e2e10557bac8395c538d2af Mon Sep 17 00:00:00 2001 From: shiltemann Date: Wed, 30 Aug 2017 16:58:57 +0200 Subject: [PATCH 14/18] add metadata to peek --- lib/galaxy/datatypes/binary.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 210952a7a54..a1bb3a91481 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -875,7 +875,14 @@ class Biom2(H5): def set_peek(self, dataset, is_multi_byte=False): if not dataset.dataset.purged: - dataset.peek = "Biom2 (HDF5) file" + lines = ['Biom2 (HDF5) file'] + try: + f = h5py.File(dataset.file_name) + for k, v in dict(f.attrs).items(): + lines.append('%s: %s' % (k, v)) + except Exception as e: + log.warning('%s, set_peek Exception: %s', self, e) + dataset.peek = '\n'.join(lines) dataset.blurb = nice_size(dataset.get_size()) else: dataset.peek = 'file does not exist' From 907e931fa8f62856fa846d2b9a9bc6392f26952a Mon Sep 17 00:00:00 2001 From: shiltemann Date: Wed, 30 Aug 2017 17:46:24 +0200 Subject: [PATCH 15/18] don't cast to string --- lib/galaxy/datatypes/binary.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index a1bb3a91481..3b3014afbbd 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -823,8 +823,8 @@ class Biom2(H5): MetadataElement(name="type", default=None, desc="table type", readonly=True, visible=True, no_value=None) MetadataElement(name="generated_by", default=None, desc="generated by", readonly=True, visible=True, no_value=None) MetadataElement(name="creation_date", default=None, desc="creation date", readonly=True, visible=True, no_value=None) - MetadataElement(name="nnz", default=None, desc="nnz: The number of non-zero elements in the table", readonly=True, visible=True, no_value=None) - MetadataElement(name="shape", default=None, desc="shape: The number of rows and columns in the dataset", readonly=True, visible=True, no_value=None) + MetadataElement(name="nnz", default=-1, desc="nnz: The number of non-zero elements in the table", readonly=True, visible=True, no_value=-1) + MetadataElement(name="shape", default=(), desc="shape: The number of rows and columns in the dataset", readonly=True, visible=True, no_value=()) file_ext = "biom2" edam_format = "format_3746" @@ -865,10 +865,10 @@ class Biom2(H5): elif 'format' in attributes: # biom 2.0 dataset.metadata.format = attributes['format'] dataset.metadata.type = attributes['type'] - dataset.metadata.shape = str(list(attributes['shape'])) + dataset.metadata.shape = tuple(attributes['shape']) dataset.metadata.generated_by = attributes['generated-by'] dataset.metadata.creation_date = attributes['creation-date'] - dataset.metadata.nnz = str(attributes['nnz']) + dataset.metadata.nnz = int(attributes['nnz']) except Exception as e: log.warning('%s, set_meta Exception: %s', self, e) From 8f3c65c59efe623c325f54e2407c383688ffeedb Mon Sep 17 00:00:00 2001 From: Saskia Hiltemann Date: Thu, 7 Sep 2017 17:39:01 +0200 Subject: [PATCH 16/18] use version 2.7.1 of h5py --- lib/galaxy/dependencies/pinned-requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt index d35a858993e..4fa62f7a8dd 100644 --- a/lib/galaxy/dependencies/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -32,7 +32,7 @@ six==1.10.0 Whoosh==2.7.4 testfixtures==4.10.0 galaxy_sequence_utils==1.0.2 -h5py==2.7.0 +h5py==2.7.1 # pykwalify and dependencies pykwalify==1.5.1 From d4cf4bff4f249c5e100fde8a31315c92acbe4fc8 Mon Sep 17 00:00:00 2001 From: shiltemann Date: Thu, 7 Sep 2017 18:11:00 +0200 Subject: [PATCH 17/18] PEP8 fix --- lib/galaxy/datatypes/binary.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/binary.py b/lib/galaxy/datatypes/binary.py index 3b3014afbbd..b68df34b433 100644 --- a/lib/galaxy/datatypes/binary.py +++ b/lib/galaxy/datatypes/binary.py @@ -12,8 +12,8 @@ import sys import tempfile import zipfile from json import dumps -import h5py +import h5py import pysam from bx.seq.twobit import TWOBIT_MAGIC_NUMBER, TWOBIT_MAGIC_NUMBER_SWAP, TWOBIT_MAGIC_SIZE From 2615094597ff6bdb0b03e496ea075c2a16f561b9 Mon Sep 17 00:00:00 2001 From: shiltemann Date: Thu, 7 Sep 2017 18:22:58 +0200 Subject: [PATCH 18/18] Revert "remove unused variable" This reverts commit 4119caf9b7fbd1fc9c374403e7f7836b8d9397c1. --- lib/galaxy/datatypes/data.py | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 988be4272ad..e8aa5c6b13b 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -369,6 +369,7 @@ class Data(object): if os.path.exists(file_path): if os.path.isdir(file_path): tmp_fh = tempfile.NamedTemporaryFile(delete=False) + tmp_file_name = tmp_fh.name dir_items = sorted(os.listdir(file_path)) base_path, item_name = os.path.split(file_path) tmp_fh.write('

Directory %s contents: %d items

\n' % (escape(item_name), len(dir_items)))