From 06db29413e838fd88a2ec84ede347f097be161e2 Mon Sep 17 00:00:00 2001 From: mvdbeek Date: Sun, 3 Sep 2017 12:03:36 +0200 Subject: [PATCH] Move instead of copying converted datasets when possible shutil.move tries to move files by renaming them. If that fails with OSError (due to permission or cross-filesystem rename) it falls back to copying files followed by removing them (https://github.com/python/cpython/blob/2.7/Lib/shutil.py#L279). By using shutil.move and catching permission problems we avoid an unnecessary copy if source and destination are on the same filesystem. Also avoids shutil.move if the upload tool is run as real-user which should fix https://github.com/galaxyproject/galaxy/issues/4300. --- tools/data_source/upload.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 159ff63c401..8b1543d7072 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -6,6 +6,7 @@ from __future__ import print_function import codecs +import errno import gzip import os import shutil @@ -79,16 +80,12 @@ def add_file(dataset, registry, json_file, output_path): converted_path = None stdout = None link_data_only = dataset.get('link_data_only', 'copy_files') - in_place = dataset.get('in_place', True) + run_as_real_user = in_place = dataset.get('in_place', True) purge_source = dataset.get('purge_source', True) # in_place is True if there is no external chmod in place, # however there are other instances where modifications should not occur in_place: - # in-place unpacking or editing of line-ending when linking in data or when - # importing data from the FTP folder while purge_source is set to false - if not purge_source and dataset.get('type') == 'ftp_import': - # If we do not purge the source we should not modify it in place. - in_place = False - if dataset.type in ('server_dir', 'path_paste'): + # when a file is added from a directory on the local file system (ftp import folder or any other path). + if dataset.type in ('server_dir', 'path_paste', 'ftp_import'): in_place = False check_content = dataset.get('check_content' , True) auto_decompress = dataset.get('auto_decompress', True) @@ -321,13 +318,16 @@ def add_file(dataset, registry, json_file, output_path): return if link_data_only == 'copy_files' and converted_path: # Move the dataset to its "real" path - shutil.copy(converted_path, output_path) try: - os.remove(converted_path) - except Exception: - pass + shutil.move(converted_path, output_path) + except OSError as e: + # We may not have permission to remove converted_path + if e.errno != errno.EACCES: + raise elif link_data_only == 'copy_files': - if purge_source: + if purge_source and not run_as_real_user: + # if the upload tool runs as a real user the real user + # can't move dataset.path as this path is owned by galaxy. shutil.move(dataset.path, output_path) else: shutil.copy(dataset.path, output_path)