From bb23701fa80c74a6621b1826b050d9e42c12455f Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 1 Dec 2015 11:05:29 -0500 Subject: [PATCH 1/7] White space in BIOM1 datatype. --- lib/galaxy/datatypes/text.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 7a99d6f2ea5..d1187fa2aa4 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -122,41 +122,45 @@ class Ipynb( Json ): pass -class Biom1(Json): +class Biom1( Json ): + """ + BIOM version 1.0 file format description + http://biom-format.org/documentation/format_versions/biom-1.0.html + """ file_ext = "biom1" - def set_peek(self, dataset, is_multi_byte=False): - super(Biom1, self).set_peek(dataset, is_multi_byte) + def set_peek( self, dataset, is_multi_byte=False ): + super ( Biom1, self ).set_peek( dataset, is_multi_byte ) if not dataset.dataset.purged: dataset.blurb = "Biological Observation Matrix v1" - def sniff(self, filename): + def sniff( self, filename ): is_biom = False if self._looks_like_json( filename ): - is_biom = self._looks_like_biom(filename) + is_biom = self._looks_like_biom( filename ) return is_biom - def _looks_like_biom(self, filepath, load_size=50000): + def _looks_like_biom( self, filepath, load_size=50000 ): """ @param filepath: [str] The path to the evaluated file. @param load_size: [int] The size of the file block load in RAM (in bytes). """ is_biom = False - segment_size = int(load_size / 2) + segment_size = int( load_size / 2 ) try: - with open(filepath, "r") as fh: + with open( filepath, "r" ) as fh: prev_str = "" - segment_str = fh.read(segment_size) - if segment_str.strip().startswith('{'): + segment_str = fh.read( segment_size ) + if segment_str.strip().startswith( '{' ): while segment_str and not is_biom: current_str = prev_str + segment_str if '"format"' in current_str: - current_str = re.sub(r'\s', '', current_str) + current_str = re.sub( r'\s', '', current_str ) if '"format":"BiologicalObservationMatrix' in current_str: is_biom = True prev_str = segment_str - segment_str = fh.read(segment_size) + segment_str = fh.read( segment_size ) except: pass return is_biom From 64506bbd43e21b299a8b42ee3cfd2ef8e40c06d2 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 1 Dec 2015 11:07:11 -0500 Subject: [PATCH 2/7] Prevent unneccessary extra read in BIOM1 sniffer. --- lib/galaxy/datatypes/text.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index d1187fa2aa4..e3d25f3a952 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -153,12 +153,13 @@ class Biom1( Json ): prev_str = "" segment_str = fh.read( segment_size ) if segment_str.strip().startswith( '{' ): - while segment_str and not is_biom: + while segment_str: current_str = prev_str + segment_str if '"format"' in current_str: current_str = re.sub( r'\s', '', current_str ) if '"format":"BiologicalObservationMatrix' in current_str: is_biom = True + break prev_str = segment_str segment_str = fh.read( segment_size ) except: From b5c77a8e8c0255f61ca62a7ff2311d443e224bec Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 1 Dec 2015 13:15:19 -0500 Subject: [PATCH 3/7] Add metadata for BIOM1 datatype. I am not yet sure that we should store table_rows and table_columns, as these could grow quite large, but could be useful in tools. --- lib/galaxy/datatypes/text.py | 42 ++++++++++++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index e3d25f3a952..621f2efcb63 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -11,7 +11,7 @@ import subprocess import tempfile from galaxy.datatypes.data import get_file_peek, Text -from galaxy.datatypes.metadata import MetadataElement +from galaxy.datatypes.metadata import MetadataElement, MetadataParameter from galaxy.util import nice_size, string_as_bool log = logging.getLogger(__name__) @@ -129,6 +129,18 @@ class Biom1( Json ): """ file_ext = "biom1" + MetadataElement( name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=[] ) + MetadataElement( name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_format", default="", desc="table_format", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_generated_by", default="", desc="table_generated_by", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_matrix_type", default="", desc="table_matrix_type", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_shape", default=[], desc="table_shape", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=[] ) + MetadataElement( name="table_format_url", default="", desc="table_format_url", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_date", default="", desc="table_date", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_type", default="", desc="table_type", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_id", default=None, desc="table_id", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=None ) + MetadataElement( name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=[] ) + def set_peek( self, dataset, is_multi_byte=False ): super ( Biom1, self ).set_peek( dataset, is_multi_byte ) if not dataset.dataset.purged: @@ -162,10 +174,36 @@ class Biom1( Json ): break prev_str = segment_str segment_str = fh.read( segment_size ) - except: + except Exception: pass return is_biom + def set_meta( self, dataset, **kwd ): + """ + Store metadata information from the BIOM file. + """ + if dataset.has_data(): + with open( dataset.file_name ) as fh: + try: + json_dict = json.load( fh ) + except Exception: + return + for ( m_name, b_name ) in [ ('table_rows', 'rows'), + ('table_matrix_element_type', 'matrix_element_type'), + ('table_format', 'format'), + ('table_generated_by', 'generated_by'), + ('table_matrix_type', 'matrix_type'), + ('table_shape', 'shape'), + ('table_format_url', 'format_url'), + ('table_date', 'date'), + ('table_type', 'type'), + ('table_id', 'id'), + ('table_columns', 'columns') ]: + try: + setattr( dataset.metadata, m_name, json_dict.get( b_name, None ) ) + except Exception: + pass + class Obo( Text ): """ From f41540f6137a2f63a6dcb27327e33bf73752f954 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 1 Dec 2015 13:21:16 -0500 Subject: [PATCH 4/7] Most of these should probably not be user visible. --- lib/galaxy/datatypes/text.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index 621f2efcb63..e2ab4410963 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -129,17 +129,17 @@ class Biom1( Json ): """ file_ext = "biom1" - MetadataElement( name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=[] ) - MetadataElement( name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) - MetadataElement( name="table_format", default="", desc="table_format", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_rows", default=[], desc="table_rows", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[] ) + MetadataElement( name="table_matrix_element_type", default="", desc="table_matrix_element_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="" ) + MetadataElement( name="table_format", default="", desc="table_format", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="" ) MetadataElement( name="table_generated_by", default="", desc="table_generated_by", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) - MetadataElement( name="table_matrix_type", default="", desc="table_matrix_type", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) - MetadataElement( name="table_shape", default=[], desc="table_shape", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=[] ) - MetadataElement( name="table_format_url", default="", desc="table_format_url", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) + MetadataElement( name="table_matrix_type", default="", desc="table_matrix_type", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="" ) + MetadataElement( name="table_shape", default=[], desc="table_shape", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[] ) + MetadataElement( name="table_format_url", default="", desc="table_format_url", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value="" ) MetadataElement( name="table_date", default="", desc="table_date", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) MetadataElement( name="table_type", default="", desc="table_type", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value="" ) MetadataElement( name="table_id", default=None, desc="table_id", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=None ) - MetadataElement( name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=True, optional=True, no_value=[] ) + MetadataElement( name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[] ) def set_peek( self, dataset, is_multi_byte=False ): super ( Biom1, self ).set_peek( dataset, is_multi_byte ) From b20fb71e33c1779e42dd67bb68aa9b31b69ffca2 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Tue, 1 Dec 2015 14:22:43 -0500 Subject: [PATCH 5/7] Fix extraneous white space. --- lib/galaxy/datatypes/text.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index e2ab4410963..f42d181ceba 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -142,7 +142,7 @@ class Biom1( Json ): MetadataElement( name="table_columns", default=[], desc="table_columns", param=MetadataParameter, readonly=True, visible=False, optional=True, no_value=[] ) def set_peek( self, dataset, is_multi_byte=False ): - super ( Biom1, self ).set_peek( dataset, is_multi_byte ) + super( Biom1, self ).set_peek( dataset, is_multi_byte ) if not dataset.dataset.purged: dataset.blurb = "Biological Observation Matrix v1" From 2c6c5236323dbea196cfd2e8fe3915f0c7e4c6df Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 2 Dec 2015 11:53:17 -0500 Subject: [PATCH 6/7] Only store ids for row and column in BIOM1 datatype. --- lib/galaxy/datatypes/text.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index f42d181ceba..c90a6774546 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -188,6 +188,11 @@ class Biom1( Json ): json_dict = json.load( fh ) except Exception: return + def _transform_dict_list_ids( dict_list ): + if dict_list: + return [ x.get( 'id', None ) for x in dict_list ] + return [] + b_transform = { 'rows': _transform_dict_list_ids, 'columns': _transform_dict_list_ids } for ( m_name, b_name ) in [ ('table_rows', 'rows'), ('table_matrix_element_type', 'matrix_element_type'), ('table_format', 'format'), @@ -200,7 +205,10 @@ class Biom1( Json ): ('table_id', 'id'), ('table_columns', 'columns') ]: try: - setattr( dataset.metadata, m_name, json_dict.get( b_name, None ) ) + metadata_value = json_dict.get( b_name, None ) + if b_name in b_transform: + metadata_value = b_transform[ b_name ]( metadata_value ) + setattr( dataset.metadata, m_name, metadata_value ) except Exception: pass From 3acfa9c786e1fa5567cf313b294b68ab4c508c1b Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 2 Dec 2015 11:58:23 -0500 Subject: [PATCH 7/7] Add blank lines around internal function. --- lib/galaxy/datatypes/text.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/datatypes/text.py b/lib/galaxy/datatypes/text.py index c90a6774546..2ec75d23fe3 100644 --- a/lib/galaxy/datatypes/text.py +++ b/lib/galaxy/datatypes/text.py @@ -188,10 +188,12 @@ class Biom1( Json ): json_dict = json.load( fh ) except Exception: return + def _transform_dict_list_ids( dict_list ): if dict_list: return [ x.get( 'id', None ) for x in dict_list ] return [] + b_transform = { 'rows': _transform_dict_list_ids, 'columns': _transform_dict_list_ids } for ( m_name, b_name ) in [ ('table_rows', 'rows'), ('table_matrix_element_type', 'matrix_element_type'),