opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 05:45:37 +08:00
Code Issues Packages Projects Releases Wiki Activity

Bugfix: Fix shutil.move for converted files in upload.py.

Browse Source
This commit is contained in:
John Chilton
2014-06-10 09:36:28 -05:00
parent a25cfad335
commit 6d4d6dd91a
2 changed files with 21 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/sniff.py
+4 -4
View File
@@ -90,7 +90,7 @@ def check_newlines( fname, bytes_to_read=52428800 ):
f.close()
return False
def convert_newlines( fname, in_place=True ):
def convert_newlines( fname, in_place=True, tmp_dir=None, tmp_prefix=None ):
"""
Converts in place a file from universal line endings
to Posix line endings.
@@ -102,7 +102,7 @@ def convert_newlines( fname, in_place=True ):
>>> file(fname).read()
'1 2\\n3 4\\n'
"""
fd, temp_name = tempfile.mkstemp()
fd, temp_name = tempfile.mkstemp( prefix=tmp_prefix, dir=tmp_dir )
fp = os.fdopen( fd, "wt" )
i = None
for i, line in enumerate( file( fname, "U" ) ):
@@ -150,7 +150,7 @@ def sep2tabs( fname, in_place=True, patt="\\s+" ):
else:
return ( i, temp_name )
def convert_newlines_sep2tabs( fname, in_place=True, patt="\\s+" ):
def convert_newlines_sep2tabs( fname, in_place=True, patt="\\s+", tmp_dir=None, tmp_prefix=None ):
"""
Combines above methods: convert_newlines() and sep2tabs()
so that files do not need to be read twice
@@ -163,7 +163,7 @@ def convert_newlines_sep2tabs( fname, in_place=True, patt="\\s+" ):
'1\\t2\\n3\\t4\\n'
"""
regexp = re.compile( patt )
fd, temp_name = tempfile.mkstemp()
fd, temp_name = tempfile.mkstemp( prefix=tmp_prefix, dir=tmp_dir )
fp = os.fdopen( fd, "wt" )
for i, line in enumerate( file( fname, "U" ) ):
line = line.rstrip( '\r\n' )
tools/data_source/upload.py
+17 -4
View File
@@ -272,10 +272,12 @@ def add_file( dataset, registry, json_file, output_path ):
# so that is becomes possible to upload gzip, bz2 or zip files with binary data without
# corrupting the content of those files.
if dataset.to_posix_lines:
tmpdir = output_adjacent_tmpdir( output_path )
tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id
if dataset.space_to_tab:
line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place )
line_count, converted_path = sniff.convert_newlines_sep2tabs( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix )
else:
line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place )
line_count, converted_path = sniff.convert_newlines( dataset.path, in_place=in_place, tmp_dir=tmpdir, tmp_prefix=tmp_prefix )
if dataset.file_type == 'auto':
ext = sniff.guess_ext( dataset.path, registry.sniff_order )
else:
@@ -343,10 +345,12 @@ def add_composite_file( dataset, registry, json_file, output_path, files_path ):
dataset.path = temp_name
dp = temp_name
if not value.is_binary:
tmpdir = output_adjacent_tmpdir( output_path )
tmp_prefix = 'data_id_%s_convert_' % dataset.dataset_id
if dataset.composite_file_paths[ value.name ].get( 'space_to_tab', value.space_to_tab ):
sniff.convert_newlines_sep2tabs( dp )
sniff.convert_newlines_sep2tabs( dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix )
else:
sniff.convert_newlines( dp )
sniff.convert_newlines( dp, tmp_dir=tmpdir, tmp_prefix=tmp_prefix )
shutil.move( dp, os.path.join( files_path, name ) )
# Move the dataset to its "real" path
shutil.move( dataset.primary_file, output_path )
@@ -356,6 +360,15 @@ def add_composite_file( dataset, registry, json_file, output_path, files_path ):
stdout = 'uploaded %s file' % dataset.file_type )
json_file.write( to_json_string( info ) + "\n" )
def output_adjacent_tmpdir( output_path ):
""" For temp files that will ultimately be moved to output_path anyway
just create the file directly in output_path's directory so shutil.move
will work optimially.
"""
return os.path.dirname( output_path )
def __main__():
if len( sys.argv ) < 4: