From cbcee585b5cde16ef356a0907d52a493fd1b9f53 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Thu, 29 Jun 2017 18:18:42 +0200 Subject: [PATCH] Use contextmanager when using compression_util.get_fileobj() --- lib/galaxy/datatypes/data.py | 5 +-- lib/galaxy/datatypes/sequence.py | 28 ++++++-------- lib/galaxy/datatypes/sniff.py | 5 +-- lib/galaxy/datatypes/tabular.py | 66 +++++++++++++++----------------- 4 files changed, 43 insertions(+), 61 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 17d0e1a7d64..dd0340128b6 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -1054,8 +1054,7 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip count = 0 file_type = None data_checked = False - temp = compression_utils.get_fileobj( file_name, "U" ) - try: + with compression_utils.get_fileobj( file_name, "U" ) as temp: while count < LINE_COUNT: line = temp.readline( WIDTH ) if line and not is_multi_byte and not data_checked: @@ -1083,8 +1082,6 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip if not skip_line: lines.append( line ) count += 1 - finally: - temp.close() if file_type == 'binary': text = "%s file" % file_type else: diff --git a/lib/galaxy/datatypes/sequence.py b/lib/galaxy/datatypes/sequence.py index 0c4cde56a13..b3c4f65f2ba 100644 --- a/lib/galaxy/datatypes/sequence.py +++ b/lib/galaxy/datatypes/sequence.py @@ -143,13 +143,10 @@ class Sequence( data.Text ): if input_datasets[0].metadata is not None and input_datasets[0].metadata.sequences is not None: total_sequences = input_datasets[0].metadata.sequences else: - in_file = compression_utils.get_fileobj(input_datasets[0].file_name) - try: + with compression_utils.get_fileobj(input_datasets[0].file_name) as in_file: total_sequences = long(0) for i, line in enumerate(in_file): total_sequences += 1 - finally: - in_file.close() total_sequences /= 4 sequences_per_file = cls.get_sequences_per_file(total_sequences, split_params) @@ -570,8 +567,7 @@ class BaseFastq ( Sequence ): data_lines = 0 sequences = 0 seq_counter = 0 # blocks should be 4 lines long - in_file = compression_utils.get_fileobj(dataset.file_name) - try: + with compression_utils.get_fileobj(dataset.file_name) as in_file: for line in in_file: line = line.strip() if line and line.startswith( '#' ) and not data_lines: @@ -590,8 +586,6 @@ class BaseFastq ( Sequence ): sequences += 1 dataset.metadata.data_lines = data_lines dataset.metadata.sequences = sequences - finally: - in_file.close() def sniff( self, filename ): """ @@ -638,15 +632,15 @@ class BaseFastq ( Sequence ): def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd): if preview: - fh = compression_utils.get_fileobj(dataset.file_name) - max_peek_size = 1000000 # 1 MB - if os.stat( dataset.file_name ).st_size < max_peek_size: - mime = "text/plain" - self._clean_and_set_mime_type( trans, mime ) - return fh.read() - return trans.stream_template_mako( "/dataset/large_file.mako", - truncated_data=fh.read(max_peek_size), - data=dataset) + with compression_utils.get_fileobj(dataset.file_name) as fh: + max_peek_size = 1000000 # 1 MB + if os.stat( dataset.file_name ).st_size < max_peek_size: + mime = "text/plain" + self._clean_and_set_mime_type( trans, mime ) + return fh.read() + return trans.stream_template_mako( "/dataset/large_file.mako", + truncated_data=fh.read(max_peek_size), + data=dataset) else: return Sequence.display_data(self, trans, dataset, preview, filename, to_ext, **kwd) diff --git a/lib/galaxy/datatypes/sniff.py b/lib/galaxy/datatypes/sniff.py index 2fa4f42c7c8..d218b1fdba9 100644 --- a/lib/galaxy/datatypes/sniff.py +++ b/lib/galaxy/datatypes/sniff.py @@ -211,8 +211,7 @@ def get_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=N [[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']] """ headers = [] - in_file = compression_utils.get_fileobj(fname) - try: + with compression_utils.get_fileobj(fname) as in_file: idx = 0 for line in in_file: line = line.rstrip('\n\r') @@ -228,8 +227,6 @@ def get_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=N idx += 1 if idx == count: break - finally: - in_file.close() return headers diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index cc28fba8ab9..16cde383c38 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -788,45 +788,41 @@ class Eland( Tabular ): - We will only check that up to the first 5 alignments are correctly formatted. """ try: - fh = compression_utils.get_fileobj(filename, gzip_only=True) - count = 0 - while True: - line = fh.readline() - line = line.strip() - if not line: - break # EOF - if line: - line_pieces = line.split('\t') - if len(line_pieces) != 22: - return False - try: - if long(line_pieces[1]) < 0: - raise Exception('Out of range') - if long(line_pieces[2]) < 0: - raise Exception('Out of range') - if long(line_pieces[3]) < 0: - raise Exception('Out of range') - int(line_pieces[4]) - int(line_pieces[5]) - # can get a lot more specific - except ValueError: - fh.close() - return False - count += 1 - if count == 5: - break - if count > 0: - fh.close() - return True - except: + with compression_utils.get_fileobj(filename, gzip_only=True) as fh: + count = 0 + while True: + line = fh.readline() + line = line.strip() + if not line: + break # EOF + if line: + line_pieces = line.split('\t') + if len(line_pieces) != 22: + return False + try: + if long(line_pieces[1]) < 0: + raise Exception('Out of range') + if long(line_pieces[2]) < 0: + raise Exception('Out of range') + if long(line_pieces[3]) < 0: + raise Exception('Out of range') + int(line_pieces[4]) + int(line_pieces[5]) + # can get a lot more specific + except ValueError: + return False + count += 1 + if count == 5: + break + if count > 0: + return True + except Exception: pass - fh.close() return False def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd ): if dataset.has_data(): - dataset_fh = compression_utils.get_fileobj(dataset.file_name, gzip_only=True) - try: + with compression_utils.get_fileobj(dataset.file_name, gzip_only=True) as dataset_fh: lanes = {} tiles = {} barcodes = {} @@ -848,8 +844,6 @@ class Eland( Tabular ): reads[line_pieces[7]] = 1 pass dataset.metadata.data_lines = i + 1 - finally: - dataset_fh.close() dataset.metadata.comment_lines = 0 dataset.metadata.columns = 21 dataset.metadata.column_types = ['str', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str']