Merge pull request #2168 from jmchilton/ftp_upload

More documentation and flexibility for FTP imports.

config/galaxy.ini.sample

@@ -1020,13 +1020,27 @@ use_interactive = True
 # database for authentication) and set the following two options.
 
 # This should point to a directory containing subdirectories matching users'
-# email addresses, where Galaxy will look for files.
+# identifier (defaults to e-mail), where Galaxy will look for files.
 #ftp_upload_dir = None
 
 # This should be the hostname of your FTP server, which will be provided to
 # users in the help text.
 #ftp_upload_site = None
 
+# User attribute to use as subdirectory in calculating default ftp_upload_dir 
+# pattern. By default this will be email so a user's FTP upload directory will be
+# ${ftp_upload_dir}/${user.email}. Can set this to other attributes such as id or
+# username though.
+#ftp_upload_dir_identifier = email
+
+# Python string template used to determine an FTP upload directory for a 
+# particular user.
+#ftp_upload_dir_template = ${ftp_upload_dir}/${ftp_upload_dir_identifier}
+
+# This should be set to False to prevent Galaxy from deleting uploaded FTP files
+# as it imports them.
+#ftp_upload_purge = True
+
 # Enable enforcement of quotas.  Quotas can be set from the Admin interface.
 #enable_quotas = False
 

lib/galaxy/config.py

@@ -293,6 +293,8 @@ class Configuration( object ):
         self.whoosh_index_dir = resolve_path( kwargs.get( "whoosh_index_dir", "database/whoosh_indexes" ), self.root )
         self.ftp_upload_dir = kwargs.get( 'ftp_upload_dir', None )
         self.ftp_upload_dir_identifier = kwargs.get( 'ftp_upload_dir_identifier', 'email' )  # attribute on user - email, username, id, etc...
+        self.ftp_upload_dir_template = kwargs.get( 'ftp_upload_dir_template', '${ftp_upload_dir}%s${ftp_upload_dir_identifier}' % os.path.sep )
+        self.ftp_upload_purge = string_as_bool(  kwargs.get( 'ftp_upload_purge', 'True' ) )
         self.ftp_upload_site = kwargs.get( 'ftp_upload_site', None )
         self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False )
         self.disable_library_comptypes = kwargs.get( 'disable_library_comptypes', '' ).lower().split( ',' )

lib/galaxy/managers/context.py

@@ -2,8 +2,8 @@
 Mixins for transaction-like objects.
 """
 
-import os
 from json import dumps
+import string
 
 from galaxy.util import bunch
 
@@ -139,12 +139,17 @@ class ProvidesUserContext( object ):
 
     @property
     def user_ftp_dir( self ):
-        identifier = self.app.config.ftp_upload_dir_identifier
         base_dir = self.app.config.ftp_upload_dir
         if base_dir is None:
             return None
         else:
-            return os.path.join( base_dir, getattr( self.user, identifier ) )
+            identifier = self.app.config.ftp_upload_dir_identifier
+            template = self.app.config.ftp_upload_dir_template
+            path = string.Template(template).safe_substitute(dict(
+                ftp_upload_dir=base_dir,
+                ftp_upload_dir_identifier=identifier,
+            ))
+            return path
 
 
 class ProvidesHistoryContext( object ):

lib/galaxy/tools/actions/upload_common.py

@@ -325,6 +325,10 @@ def create_paramfile( trans, uploaded_datasets ):
                 uuid_str = uploaded_dataset.uuid
             except:
                 uuid_str = None
+            try:
+                purge_source = uploaded_dataset.purge_source
+            except:
+                purge_source = True
             json = dict( file_type=uploaded_dataset.file_type,
                          ext=uploaded_dataset.ext,
                          name=uploaded_dataset.name,
@@ -335,6 +339,7 @@ def create_paramfile( trans, uploaded_datasets ):
                          link_data_only=link_data_only,
                          uuid=uuid_str,
                          to_posix_lines=getattr(uploaded_dataset, "to_posix_lines", True),
+                         purge_source=purge_source,
                          space_to_tab=uploaded_dataset.space_to_tab,
                          in_place=trans.app.config.external_chown_script is None,
                          path=uploaded_dataset.path )

lib/galaxy/tools/parameters/grouping.py

@@ -283,7 +283,7 @@ class UploadDataset( Group ):
         return rval
 
     def get_uploaded_datasets( self, trans, context, override_name=None, override_info=None ):
-        def get_data_file_filename( data_file, override_name=None, override_info=None ):
+        def get_data_file_filename( data_file, override_name=None, override_info=None, purge=True ):
             dataset_name = override_name
             dataset_info = override_info
 
@@ -297,7 +297,7 @@ class UploadDataset( Group ):
                     dataset_name = get_file_name( data_file['filename'] )
                 if not dataset_info:
                     dataset_info = 'uploaded file'
-                return Bunch( type='file', path=data_file['local_filename'], name=dataset_name )
+                return Bunch( type='file', path=data_file['local_filename'], name=dataset_name, purge_source=purge )
             except:
                 # The uploaded file should've been persisted by the upload tool action
                 return Bunch( type=None, path=None, name=None )
@@ -364,7 +364,13 @@ class UploadDataset( Group ):
                                 if not os.path.islink( os.path.join( dirpath, filename ) ):
                                     ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, path ) ),
                                                       'filename' : os.path.basename( path ) }
-                                    file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info )
+                                    purge = getattr(trans.app.config, 'ftp_upload_purge', True)
+                                    file_bunch = get_data_file_filename(
+                                        ftp_data_file,
+                                        override_name=name,
+                                        override_info=info,
+                                        purge=purge,
+                                    )
                                     if file_bunch.path:
                                         break
                         if file_bunch.path:
@@ -432,7 +438,8 @@ class UploadDataset( Group ):
                     # TODO: warning to the user (could happen if file is already imported)
                 ftp_data_file = { 'local_filename' : os.path.abspath( os.path.join( user_ftp_dir, ftp_file ) ),
                                   'filename' : os.path.basename( ftp_file ) }
-                file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info )
+                purge = getattr(trans.app.config, 'ftp_upload_purge', True)
+                file_bunch = get_data_file_filename( ftp_data_file, override_name=name, override_info=info, purge=purge )
                 if file_bunch.path:
                     file_bunch.to_posix_lines = to_posix_lines
                     file_bunch.space_to_tab = space_to_tab

lib/galaxy/webapps/galaxy/api/remote_files.py

@@ -90,9 +90,7 @@ class RemoteFilesAPIController( BaseAPIController ):
             if user_ftp_base_dir is None:
                 raise exceptions.ConfigDoesNotAllowException( 'The configuration of this Galaxy instance does not allow upload from FTP directories.' )
             try:
-                user_ftp_dir = None
-                identifier = trans.app.config.ftp_upload_dir_identifier
-                user_ftp_dir = os.path.join( user_ftp_base_dir, getattr(trans.user, identifier) )
+                user_ftp_dir = trans.user_ftp_dir
                 if user_ftp_dir is not None:
                     response = self.__load_all_filenames( user_ftp_dir )
                 else:

tools/data_source/upload.py

@@ -85,7 +85,7 @@ def add_file( dataset, registry, json_file, output_path ):
     stdout = None
     link_data_only = dataset.get( 'link_data_only', 'copy_files' )
     in_place = dataset.get( 'in_place', True )
-
+    purge_source = dataset.get( 'purge_source', True )
     try:
         ext = dataset.file_type
     except AttributeError:
@@ -329,7 +329,10 @@ def add_file( dataset, registry, json_file, output_path ):
             # This should not happen, but it's here just in case
             shutil.copy( dataset.path, output_path )
     elif link_data_only == 'copy_files':
-        shutil.move( dataset.path, output_path )
+        if purge_source:
+            shutil.move( dataset.path, output_path )
+        else:
+            shutil.copy( dataset.path, output_path )
     # Write the job info
     stdout = stdout or 'uploaded %s file' % data_type
     info = dict( type='dataset',