diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index f7d7059681a..1ed4033cb03 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -1020,13 +1020,27 @@ use_interactive = True # database for authentication) and set the following two options. # This should point to a directory containing subdirectories matching users' -# email addresses, where Galaxy will look for files. +# identifier (defaults to e-mail), where Galaxy will look for files. #ftp_upload_dir = None # This should be the hostname of your FTP server, which will be provided to # users in the help text. #ftp_upload_site = None +# User attribute to use as subdirectory in calculating default ftp_upload_dir +# pattern. By default this will be email so a user's FTP upload directory will be +# ${ftp_upload_dir}/${user.email}. Can set this to other attributes such as id or +# username though. +#ftp_upload_dir_identifier = email + +# Python string template used to determine an FTP upload directory for a +# particular user. +#ftp_upload_dir_template = ${ftp_upload_dir}/${ftp_upload_dir_identifier} + +# This should be set to False to prevent Galaxy from deleting uploaded FTP files +# as it imports them. +#ftp_upload_purge = True + # Enable enforcement of quotas. Quotas can be set from the Admin interface. #enable_quotas = False diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 0a2840f7463..c62d8501269 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -293,6 +293,8 @@ class Configuration( object ): self.whoosh_index_dir = resolve_path( kwargs.get( "whoosh_index_dir", "database/whoosh_indexes" ), self.root ) self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) self.ftp_upload_dir_identifier = kwargs.get( 'ftp_upload_dir_identifier', 'email' ) # attribute on user - email, username, id, etc... + self.ftp_upload_dir_template = kwargs.get( 'ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep ) + self.ftp_upload_purge = string_as_bool( kwargs.get( 'ftp_upload_purge', 'True' ) ) self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 822dbee4475..9dde7a241a5 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -2,8 +2,8 @@ Mixins for transaction-like objects. """ -import os from json import dumps +import string from galaxy.util import bunch @@ -139,12 +139,17 @@ class ProvidesUserContext( object ): @property def user_ftp_dir( self ): - identifier = self.app.config.ftp_upload_dir_identifier base_dir = self.app.config.ftp_upload_dir if base_dir is None: return None else: - return os.path.join( base_dir, getattr( self.user, identifier ) ) + identifier = self.app.config.ftp_upload_dir_identifier + template = self.app.config.ftp_upload_dir_template + path = string.Template(template).safe_substitute(dict( + ftp_upload_dir=base_dir, + ftp_upload_dir_identifier=identifier, + )) + return path class ProvidesHistoryContext( object ): diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 5de9e45e560..8c7beb994b3 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -325,6 +325,10 @@ def create_paramfile( trans, uploaded_datasets ): uuid_str = uploaded_dataset.uuid except: uuid_str = None + try: + purge_source = uploaded_dataset.purge_source + except: + purge_source = True json = dict( file_type=uploaded_dataset.file_type, ext=uploaded_dataset.ext, name=uploaded_dataset.name, @@ -335,6 +339,7 @@ def create_paramfile( trans, uploaded_datasets ): link_data_only=link_data_only, uuid=uuid_str, to_posix_lines=getattr(uploaded_dataset, "to_posix_lines", True), + purge_source=purge_source, space_to_tab=uploaded_dataset.space_to_tab, in_place=trans.app.config.external_chown_script is None, path=uploaded_dataset.path ) diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index ebfddf42333..653decf7fe6 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -283,7 +283,7 @@ class UploadDataset( Group ): return rval def get_uploaded_datasets( self, trans, context, override_name=None, override_info=None ): - def get_data_file_filename( data_file, override_name=None, override_info=None ): + def get_data_file_filename( data_file, override_name=None, override_info=None, purge=True ): dataset_name = override_name dataset_info = override_info @@ -297,7 +297,7 @@ class UploadDataset( Group ): dataset_name = get_file_name( data_file['filename'] ) if not dataset_info: dataset_info = 'uploaded file' - return Bunch( type='file', path=data_file['local_filename'], name=dataset_name ) + return Bunch( type='file', path=data_file['local_filename'], name=dataset_name, purge_source=purge ) except: # The uploaded file should've been persisted by the upload tool action return Bunch( type=None, path=None, name=None ) @@ -364,7 +364,13 @@ class UploadDataset( Group ): if not os.path.islink( os.path.join( dirpath, filename ) ): ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, path ) ), 'filename' : os.path.basename( path ) } - file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info ) + purge = getattr(trans.app.config, 'ftp_upload_purge', True) + file_bunch = get_data_file_filename( + ftp_data_file, + override_name=name, + override_info=info, + purge=purge, + ) if file_bunch.path: break if file_bunch.path: @@ -432,7 +438,8 @@ class UploadDataset( Group ): # TODO: warning to the user (could happen if file is already imported) ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, ftp_file ) ), 'filename' : os.path.basename( ftp_file ) } - file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info ) + purge = getattr(trans.app.config, 'ftp_upload_purge', True) + file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info, purge=purge ) if file_bunch.path: file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab diff --git a/lib/galaxy/webapps/galaxy/api/remote_files.py b/lib/galaxy/webapps/galaxy/api/remote_files.py index 12118d3d0fa..c0c6be8d78d 100644 --- a/lib/galaxy/webapps/galaxy/api/remote_files.py +++ b/lib/galaxy/webapps/galaxy/api/remote_files.py @@ -90,9 +90,7 @@ class RemoteFilesAPIController( BaseAPIController ): if user_ftp_base_dir is None: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from FTP directories.' ) try: - user_ftp_dir = None - identifier = trans.app.config.ftp_upload_dir_identifier - user_ftp_dir = os.path.join( user_ftp_base_dir, getattr(trans.user, identifier) ) + user_ftp_dir = trans.user_ftp_dir if user_ftp_dir is not None: response = self.__load_all_filenames( user_ftp_dir ) else: diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 59fa187c7ca..008b2da0ee2 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -85,7 +85,7 @@ def add_file( dataset, registry, json_file, output_path ): stdout = None link_data_only = dataset.get( 'link_data_only', 'copy_files' ) in_place = dataset.get( 'in_place', True ) - + purge_source = dataset.get( 'purge_source', True ) try: ext = dataset.file_type except AttributeError: @@ -329,7 +329,10 @@ def add_file( dataset, registry, json_file, output_path ): # This should not happen, but it's here just in case shutil.copy( dataset.path, output_path ) elif link_data_only == 'copy_files': - shutil.move( dataset.path, output_path ) + if purge_source: + shutil.move( dataset.path, output_path ) + else: + shutil.copy( dataset.path, output_path ) # Write the job info stdout = stdout or 'uploaded %s file' % data_type info = dict( type='dataset',