opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-10-24 02:04:04 +08:00
Code Issues Packages Projects Releases Wiki Activity

Use contextmanager when using compression_util.get_fileobj()

Browse Source
This commit is contained in:
mvdbeek
2017-06-30 11:56:59 +02:00
parent 8169129981
commit cbcee585b5
4 changed files with 43 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/data.py
+1 -4
View File
@@ -1054,8 +1054,7 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip
count = 0
file_type = None
data_checked = False
temp = compression_utils.get_fileobj( file_name, "U" )
try:
with compression_utils.get_fileobj( file_name, "U" ) as temp:
while count < LINE_COUNT:
line = temp.readline( WIDTH )
if line and not is_multi_byte and not data_checked:
@@ -1083,8 +1082,6 @@ def get_file_peek( file_name, is_multi_byte=False, WIDTH=256, LINE_COUNT=5, skip
if not skip_line:
lines.append( line )
count += 1
finally:
temp.close()
if file_type == 'binary':
text = "%s file" % file_type
else:
lib/galaxy/datatypes/sequence.py
+11 -17
View File
@@ -143,13 +143,10 @@ class Sequence( data.Text ):
if input_datasets[0].metadata is not None and input_datasets[0].metadata.sequences is not None:
total_sequences = input_datasets[0].metadata.sequences
else:
in_file = compression_utils.get_fileobj(input_datasets[0].file_name)
try:
with compression_utils.get_fileobj(input_datasets[0].file_name) as in_file:
total_sequences = long(0)
for i, line in enumerate(in_file):
total_sequences += 1
finally:
in_file.close()
total_sequences /= 4
sequences_per_file = cls.get_sequences_per_file(total_sequences, split_params)
@@ -570,8 +567,7 @@ class BaseFastq ( Sequence ):
data_lines = 0
sequences = 0
seq_counter = 0 # blocks should be 4 lines long
in_file = compression_utils.get_fileobj(dataset.file_name)
try:
with compression_utils.get_fileobj(dataset.file_name) as in_file:
for line in in_file:
line = line.strip()
if line and line.startswith( '#' ) and not data_lines:
@@ -590,8 +586,6 @@ class BaseFastq ( Sequence ):
sequences += 1
dataset.metadata.data_lines = data_lines
dataset.metadata.sequences = sequences
finally:
in_file.close()
def sniff( self, filename ):
"""
@@ -638,15 +632,15 @@ class BaseFastq ( Sequence ):
def display_data(self, trans, dataset, preview=False, filename=None, to_ext=None, **kwd):
if preview:
fh = compression_utils.get_fileobj(dataset.file_name)
max_peek_size = 1000000 # 1 MB
if os.stat( dataset.file_name ).st_size < max_peek_size:
mime = "text/plain"
self._clean_and_set_mime_type( trans, mime )
return fh.read()
return trans.stream_template_mako( "/dataset/large_file.mako",
truncated_data=fh.read(max_peek_size),
data=dataset)
with compression_utils.get_fileobj(dataset.file_name) as fh:
max_peek_size = 1000000 # 1 MB
if os.stat( dataset.file_name ).st_size < max_peek_size:
mime = "text/plain"
self._clean_and_set_mime_type( trans, mime )
return fh.read()
return trans.stream_template_mako( "/dataset/large_file.mako",
truncated_data=fh.read(max_peek_size),
data=dataset)
else:
return Sequence.display_data(self, trans, dataset, preview, filename, to_ext, **kwd)
lib/galaxy/datatypes/sniff.py
+1 -4
View File
@@ -211,8 +211,7 @@ def get_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=N
[[''], ['chr7', 'bed2gff', 'AR', '26731313', '26731437', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731491', '26731536', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731541', '26731649', '.', '+', '.', 'score'], ['chr7', 'bed2gff', 'AR', '26731659', '26731841', '.', '+', '.', 'score']]
"""
headers = []
in_file = compression_utils.get_fileobj(fname)
try:
with compression_utils.get_fileobj(fname) as in_file:
idx = 0
for line in in_file:
line = line.rstrip('\n\r')
@@ -228,8 +227,6 @@ def get_headers( fname, sep, count=60, is_multi_byte=False, comment_designator=N
idx += 1
if idx == count:
break
finally:
in_file.close()
return headers
lib/galaxy/datatypes/tabular.py
+30 -36
View File
@@ -788,45 +788,41 @@ class Eland( Tabular ):
- We will only check that up to the first 5 alignments are correctly formatted.
"""
try:
fh = compression_utils.get_fileobj(filename, gzip_only=True)
count = 0
while True:
line = fh.readline()
line = line.strip()
if not line:
break # EOF
if line:
line_pieces = line.split('\t')
if len(line_pieces) != 22:
return False
try:
if long(line_pieces[1]) < 0:
raise Exception('Out of range')
if long(line_pieces[2]) < 0:
raise Exception('Out of range')
if long(line_pieces[3]) < 0:
raise Exception('Out of range')
int(line_pieces[4])
int(line_pieces[5])
# can get a lot more specific
except ValueError:
fh.close()
return False
count += 1
if count == 5:
break
if count > 0:
fh.close()
return True
except:
with compression_utils.get_fileobj(filename, gzip_only=True) as fh:
count = 0
while True:
line = fh.readline()
line = line.strip()
if not line:
break # EOF
if line:
line_pieces = line.split('\t')
if len(line_pieces) != 22:
return False
try:
if long(line_pieces[1]) < 0:
raise Exception('Out of range')
if long(line_pieces[2]) < 0:
raise Exception('Out of range')
if long(line_pieces[3]) < 0:
raise Exception('Out of range')
int(line_pieces[4])
int(line_pieces[5])
# can get a lot more specific
except ValueError:
return False
count += 1
if count == 5:
break
if count > 0:
return True
except Exception:
pass
fh.close()
return False
def set_meta( self, dataset, overwrite=True, skip=None, max_data_lines=5, **kwd ):
if dataset.has_data():
dataset_fh = compression_utils.get_fileobj(dataset.file_name, gzip_only=True)
try:
with compression_utils.get_fileobj(dataset.file_name, gzip_only=True) as dataset_fh:
lanes = {}
tiles = {}
barcodes = {}
@@ -848,8 +844,6 @@ class Eland( Tabular ):
reads[line_pieces[7]] = 1
pass
dataset.metadata.data_lines = i + 1
finally:
dataset_fh.close()
dataset.metadata.comment_lines = 0
dataset.metadata.columns = 21
dataset.metadata.column_types = ['str', 'int', 'int', 'int', 'int', 'int', 'str', 'int', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str', 'str']