From 54173807e56e8ae4bef4cc0511249d6b87380de2 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 14 Apr 2016 10:32:04 -0400 Subject: [PATCH 1/4] Re-use ``trans`` logic for calculating user_ftp_dir. --- lib/galaxy/webapps/galaxy/api/remote_files.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/remote_files.py b/lib/galaxy/webapps/galaxy/api/remote_files.py index 12118d3d0fa..c0c6be8d78d 100644 --- a/lib/galaxy/webapps/galaxy/api/remote_files.py +++ b/lib/galaxy/webapps/galaxy/api/remote_files.py @@ -90,9 +90,7 @@ class RemoteFilesAPIController( BaseAPIController ): if user_ftp_base_dir is None: raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from FTP directories.' ) try: - user_ftp_dir = None - identifier = trans.app.config.ftp_upload_dir_identifier - user_ftp_dir = os.path.join( user_ftp_base_dir, getattr(trans.user, identifier) ) + user_ftp_dir = trans.user_ftp_dir if user_ftp_dir is not None: response = self.__load_all_filenames( user_ftp_dir ) else: From f95cbf8f38801f03ba1ea9c065bc1796253d6eaf Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 14 Apr 2016 10:47:38 -0400 Subject: [PATCH 2/4] More documentation and flexibility for configuring FTP imports. - Add missing documentation for the ftp_upload_dir_identifier option. - Add ftp_upload_dir_template option to configure how to combine identifier and base directory to generate an ftp upload directory. --- config/galaxy.ini.sample | 12 +++++++++++- lib/galaxy/config.py | 1 + lib/galaxy/managers/context.py | 11 ++++++++--- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index dbc13b0ca32..d0d9438808d 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -1013,13 +1013,23 @@ use_interactive = True # database for authentication) and set the following two options. # This should point to a directory containing subdirectories matching users' -# email addresses, where Galaxy will look for files. +# identifer (defaults to e-mail), where Galaxy will look for files. #ftp_upload_dir = None # This should be the hostname of your FTP server, which will be provided to # users in the help text. #ftp_upload_site = None +# User attribute to use as subdirectory in calculating default ftp_upload_dir +# pattern. By default this will be email so a user's FTP upload directory will be +# ${ftp_upload_dir}/${user.email}. Can set this to other attributes such as id or +# username though. +#ftp_upload_dir_identifier = email + +# Python string tempalte used to determine an FTP upload directory for a +# particular user. +#ftp_upload_dir_template = ${ftp_upload_dir}/${ftp_upload_dir_identifier} + # Enable enforcement of quotas. Quotas can be set from the Admin interface. #enable_quotas = False diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index bfae71b72db..a85efd03f2b 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -292,6 +292,7 @@ class Configuration( object ): self.whoosh_index_dir = resolve_path( kwargs.get( "whoosh_index_dir", "database/whoosh_indexes" ), self.root ) self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) self.ftp_upload_dir_identifier = kwargs.get( 'ftp_upload_dir_identifier', 'email' ) # attribute on user - email, username, id, etc... + self.ftp_upload_dir_template = kwargs.get( 'ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep ) self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) diff --git a/lib/galaxy/managers/context.py b/lib/galaxy/managers/context.py index 822dbee4475..9dde7a241a5 100644 --- a/lib/galaxy/managers/context.py +++ b/lib/galaxy/managers/context.py @@ -2,8 +2,8 @@ Mixins for transaction-like objects. """ -import os from json import dumps +import string from galaxy.util import bunch @@ -139,12 +139,17 @@ class ProvidesUserContext( object ): @property def user_ftp_dir( self ): - identifier = self.app.config.ftp_upload_dir_identifier base_dir = self.app.config.ftp_upload_dir if base_dir is None: return None else: - return os.path.join( base_dir, getattr( self.user, identifier ) ) + identifier = self.app.config.ftp_upload_dir_identifier + template = self.app.config.ftp_upload_dir_template + path = string.Template(template).safe_substitute(dict( + ftp_upload_dir=base_dir, + ftp_upload_dir_identifier=identifier, + )) + return path class ProvidesHistoryContext( object ): From 997b53f3ae7586d4a2e90f6cb03f00b844bb4d42 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 14 Apr 2016 11:23:25 -0400 Subject: [PATCH 3/4] Implement option to prevent FTP uploads from being deleted on import. - Simply set `ftp_upload_purge` to `False` in galaxy.ini to enable this. - xref http://dev.list.galaxyproject.org/Data-removal-after-ftp-import-td4667111.html - slight xref http://dev.list.galaxyproject.org/Uploading-files-to-galaxy-from-a-folder-td4664614.html --- config/galaxy.ini.sample | 4 ++++ lib/galaxy/config.py | 1 + lib/galaxy/tools/actions/upload_common.py | 5 +++++ lib/galaxy/tools/parameters/grouping.py | 15 +++++++++++---- tools/data_source/upload.py | 7 +++++-- 5 files changed, 26 insertions(+), 6 deletions(-) diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index d0d9438808d..96f6908826a 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -1030,6 +1030,10 @@ use_interactive = True # particular user. #ftp_upload_dir_template = ${ftp_upload_dir}/${ftp_upload_dir_identifier} +# This should be set to False to prevent Galaxy from deleting uploaded FTP files +# as it imports them. +#ftp_upload_purge = True + # Enable enforcement of quotas. Quotas can be set from the Admin interface. #enable_quotas = False diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index a85efd03f2b..7385fd7ad4e 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -293,6 +293,7 @@ class Configuration( object ): self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None ) self.ftp_upload_dir_identifier = kwargs.get( 'ftp_upload_dir_identifier', 'email' ) # attribute on user - email, username, id, etc... self.ftp_upload_dir_template = kwargs.get( 'ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep ) + self.ftp_upload_purge = string_as_bool( kwargs.get( 'ftp_upload_purge', 'True' ) ) self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None ) self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' ) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index 5de9e45e560..8c7beb994b3 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -325,6 +325,10 @@ def create_paramfile( trans, uploaded_datasets ): uuid_str = uploaded_dataset.uuid except: uuid_str = None + try: + purge_source = uploaded_dataset.purge_source + except: + purge_source = True json = dict( file_type=uploaded_dataset.file_type, ext=uploaded_dataset.ext, name=uploaded_dataset.name, @@ -335,6 +339,7 @@ def create_paramfile( trans, uploaded_datasets ): link_data_only=link_data_only, uuid=uuid_str, to_posix_lines=getattr(uploaded_dataset, "to_posix_lines", True), + purge_source=purge_source, space_to_tab=uploaded_dataset.space_to_tab, in_place=trans.app.config.external_chown_script is None, path=uploaded_dataset.path ) diff --git a/lib/galaxy/tools/parameters/grouping.py b/lib/galaxy/tools/parameters/grouping.py index ebfddf42333..653decf7fe6 100644 --- a/lib/galaxy/tools/parameters/grouping.py +++ b/lib/galaxy/tools/parameters/grouping.py @@ -283,7 +283,7 @@ class UploadDataset( Group ): return rval def get_uploaded_datasets( self, trans, context, override_name=None, override_info=None ): - def get_data_file_filename( data_file, override_name=None, override_info=None ): + def get_data_file_filename( data_file, override_name=None, override_info=None, purge=True ): dataset_name = override_name dataset_info = override_info @@ -297,7 +297,7 @@ class UploadDataset( Group ): dataset_name = get_file_name( data_file['filename'] ) if not dataset_info: dataset_info = 'uploaded file' - return Bunch( type='file', path=data_file['local_filename'], name=dataset_name ) + return Bunch( type='file', path=data_file['local_filename'], name=dataset_name, purge_source=purge ) except: # The uploaded file should've been persisted by the upload tool action return Bunch( type=None, path=None, name=None ) @@ -364,7 +364,13 @@ class UploadDataset( Group ): if not os.path.islink( os.path.join( dirpath, filename ) ): ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, path ) ), 'filename' : os.path.basename( path ) } - file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info ) + purge = getattr(trans.app.config, 'ftp_upload_purge', True) + file_bunch = get_data_file_filename( + ftp_data_file, + override_name=name, + override_info=info, + purge=purge, + ) if file_bunch.path: break if file_bunch.path: @@ -432,7 +438,8 @@ class UploadDataset( Group ): # TODO: warning to the user (could happen if file is already imported) ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, ftp_file ) ), 'filename' : os.path.basename( ftp_file ) } - file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info ) + purge = getattr(trans.app.config, 'ftp_upload_purge', True) + file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info, purge=purge ) if file_bunch.path: file_bunch.to_posix_lines = to_posix_lines file_bunch.space_to_tab = space_to_tab diff --git a/tools/data_source/upload.py b/tools/data_source/upload.py index 59fa187c7ca..008b2da0ee2 100644 --- a/tools/data_source/upload.py +++ b/tools/data_source/upload.py @@ -85,7 +85,7 @@ def add_file( dataset, registry, json_file, output_path ): stdout = None link_data_only = dataset.get( 'link_data_only', 'copy_files' ) in_place = dataset.get( 'in_place', True ) - + purge_source = dataset.get( 'purge_source', True ) try: ext = dataset.file_type except AttributeError: @@ -329,7 +329,10 @@ def add_file( dataset, registry, json_file, output_path ): # This should not happen, but it's here just in case shutil.copy( dataset.path, output_path ) elif link_data_only == 'copy_files': - shutil.move( dataset.path, output_path ) + if purge_source: + shutil.move( dataset.path, output_path ) + else: + shutil.copy( dataset.path, output_path ) # Write the job info stdout = stdout or 'uploaded %s file' % data_type info = dict( type='dataset', From 3ce6fca52c1fce1027566d5dbc50811be614ca28 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Thu, 14 Apr 2016 12:23:31 -0400 Subject: [PATCH 4/4] config doc fixes spotted by @nsoranzo --- config/galaxy.ini.sample | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/galaxy.ini.sample b/config/galaxy.ini.sample index 96f6908826a..e5dd1ee4537 100644 --- a/config/galaxy.ini.sample +++ b/config/galaxy.ini.sample @@ -1013,7 +1013,7 @@ use_interactive = True # database for authentication) and set the following two options. # This should point to a directory containing subdirectories matching users' -# identifer (defaults to e-mail), where Galaxy will look for files. +# identifier (defaults to e-mail), where Galaxy will look for files. #ftp_upload_dir = None # This should be the hostname of your FTP server, which will be provided to @@ -1026,7 +1026,7 @@ use_interactive = True # username though. #ftp_upload_dir_identifier = email -# Python string tempalte used to determine an FTP upload directory for a +# Python string template used to determine an FTP upload directory for a # particular user. #ftp_upload_dir_template = ${ftp_upload_dir}/${ftp_upload_dir_identifier}