diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index 1be2811b9e1..cfb3f937fac 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -87,6 +87,7 @@ class Configuration( object ): self.user_library_import_dir = kwargs.get( 'user_library_import_dir', None ) if self.user_library_import_dir is not None and not os.path.exists( self.user_library_import_dir ): raise ConfigurationError( "user_library_import_dir specified in config (%s) does not exist" % self.user_library_import_dir ) + self.allow_library_path_paste = kwargs.get( 'allow_library_path_paste', False ) # Configuration options for taking advantage of nginx features self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) self.nginx_upload_store = kwargs.get( 'nginx_upload_store', False ) diff --git a/lib/galaxy/tools/actions/upload_common.py b/lib/galaxy/tools/actions/upload_common.py index dbf3f950c42..af7af4c194c 100644 --- a/lib/galaxy/tools/actions/upload_common.py +++ b/lib/galaxy/tools/actions/upload_common.py @@ -3,6 +3,7 @@ from cgi import FieldStorage from galaxy import datatypes, util from galaxy.datatypes import sniff from galaxy.util.json import to_json_string +from galaxy.model.orm import eagerload_all import logging log = logging.getLogger( __name__ ) @@ -127,12 +128,29 @@ def new_library_upload( trans, uploaded_dataset, library_bunch, state=None ): or trans.user.email in trans.app.config.get( "admin_users", "" ).split( "," ) ): # This doesn't have to be pretty - the only time this should happen is if someone's being malicious. raise Exception( "User is not authorized to add datasets to this library." ) + folder = library_bunch.folder + if uploaded_dataset.get( 'in_folder', False ): + # Create subfolders if desired + for name in uploaded_dataset.in_folder.split( os.path.sep ): + folder.refresh() + matches = filter( lambda x: x.name == name, active_folders( trans, folder ) ) + if matches: + log.debug( 'DEBUGDEBUG: In %s, found a folder name match: %s:%s' % ( folder.name, matches[0].id, matches[0].name ) ) + folder = matches[0] + else: + new_folder = trans.app.model.LibraryFolder( name=name, description='Automatically created by upload tool' ) + new_folder.genome_build = util.dbnames.default_value + folder.add_folder( new_folder ) + new_folder.flush() + trans.app.security_agent.copy_library_permissions( folder, new_folder ) + log.debug( 'DEBUGDEBUG: In %s, created a new folder: %s:%s' % ( folder.name, new_folder.id, new_folder.name ) ) + folder = new_folder if library_bunch.replace_dataset: ld = library_bunch.replace_dataset else: - ld = trans.app.model.LibraryDataset( folder=library_bunch.folder, name=uploaded_dataset.name ) + ld = trans.app.model.LibraryDataset( folder=folder, name=uploaded_dataset.name ) ld.flush() - trans.app.security_agent.copy_library_permissions( library_bunch.folder, ld ) + trans.app.security_agent.copy_library_permissions( folder, ld ) ldda = trans.app.model.LibraryDatasetDatasetAssociation( name = uploaded_dataset.name, extension = uploaded_dataset.file_type, dbkey = uploaded_dataset.dbkey, @@ -153,8 +171,8 @@ def new_library_upload( trans, uploaded_dataset, library_bunch, state=None ): else: # Copy the current user's DefaultUserPermissions to the new LibraryDatasetDatasetAssociation.dataset trans.app.security_agent.set_all_dataset_permissions( ldda.dataset, trans.app.security_agent.user_get_default_permissions( trans.user ) ) - library_bunch.folder.add_library_dataset( ld, genome_build=uploaded_dataset.dbkey ) - library_bunch.folder.flush() + folder.add_library_dataset( ld, genome_build=uploaded_dataset.dbkey ) + folder.flush() ld.library_dataset_dataset_association_id = ldda.id ld.flush() # Handle template included in the upload form, if any @@ -230,6 +248,10 @@ def create_paramfile( uploaded_datasets ): is_binary = uploaded_dataset.datatype.is_binary except: is_binary = None + try: + link_data_only = uploaded_dataset.link_data_only + except: + link_data_only = False json = dict( file_type = uploaded_dataset.file_type, ext = uploaded_dataset.ext, name = uploaded_dataset.name, @@ -237,6 +259,7 @@ def create_paramfile( uploaded_datasets ): dbkey = uploaded_dataset.dbkey, type = uploaded_dataset.type, is_binary = is_binary, + link_data_only = link_data_only, space_to_tab = uploaded_dataset.space_to_tab, path = uploaded_dataset.path ) json_file.write( to_json_string( json ) + '\n' ) @@ -276,3 +299,13 @@ def create_job( trans, params, tool, json_file_path, data_list, folder=None ): trans.app.job_queue.put( job.id, tool ) trans.log_event( "Added job to the job queue, id: %s" % str(job.id), tool_id=job.tool_id ) return dict( [ ( 'output%i' % i, v ) for i, v in enumerate( data_list ) ] ) + +def active_folders( trans, folder ): + # Stolen from galaxy.web.controllers.library_common (importing from which causes a circular issues). + # Much faster way of retrieving all active sub-folders within a given folder than the + # performance of the mapper. This query also eagerloads the permissions on each folder. + return trans.sa_session.query( trans.app.model.LibraryFolder ) \ + .filter_by( parent=folder, deleted=False ) \ + .options( eagerload_all( "actions" ) ) \ + .order_by( trans.app.model.LibraryFolder.table.c.name ) \ + .all() diff --git a/lib/galaxy/util/__init__.py b/lib/galaxy/util/__init__.py index 9090630c753..acf4cdd7754 100644 --- a/lib/galaxy/util/__init__.py +++ b/lib/galaxy/util/__init__.py @@ -178,7 +178,7 @@ class Params: # better solution I think is to more responsibility for # sanitizing into the tool parameters themselves so that # different parameters can be sanitized in different ways. - NEVER_SANITIZE = ['file_data', 'url_paste', 'URL'] + NEVER_SANITIZE = ['file_data', 'url_paste', 'URL', 'filesystem_paths'] def __init__( self, params, safe=True, sanitize=True, tool=None ): if safe: diff --git a/lib/galaxy/web/controllers/library_common.py b/lib/galaxy/web/controllers/library_common.py index aa9e56322bd..edcab8e0ab3 100644 --- a/lib/galaxy/web/controllers/library_common.py +++ b/lib/galaxy/web/controllers/library_common.py @@ -81,7 +81,9 @@ class LibraryCommon( BaseController ): tool_params = upload_common.persist_uploads( tool_params ) uploaded_datasets = upload_common.get_uploaded_datasets( trans, tool_params, precreated_datasets, dataset_upload_inputs, library_bunch=library_bunch ) elif upload_option == 'upload_directory': - uploaded_datasets = self.get_server_dir_uploaded_datasets( trans, params, full_dir, import_dir_desc, library_bunch, err_redirect, msg ) + uploaded_datasets, err_redirect, msg = self.get_server_dir_uploaded_datasets( trans, params, full_dir, import_dir_desc, library_bunch, err_redirect, msg ) + elif upload_option == 'upload_paths': + uploaded_datasets, err_redirect, msg = self.get_path_paste_uploaded_datasets( trans, params, library_bunch, err_redirect, msg ) upload_common.cleanup_unused_precreated_datasets( precreated_datasets ) if upload_option == 'upload_file' and not uploaded_datasets: msg = 'Select a file, enter a URL or enter text' @@ -98,37 +100,86 @@ class LibraryCommon( BaseController ): json_file_path = upload_common.create_paramfile( uploaded_datasets ) data_list = [ ud.data for ud in uploaded_datasets ] return upload_common.create_job( trans, tool_params, tool, json_file_path, data_list, folder=library_bunch.folder ) + def make_library_uploaded_dataset( self, trans, params, name, path, type, library_bunch, in_folder=None ): + library_bunch.replace_dataset = None # not valid for these types of upload + uploaded_dataset = util.bunch.Bunch() + uploaded_dataset.name = name + uploaded_dataset.path = path + uploaded_dataset.type = type + uploaded_dataset.ext = None + uploaded_dataset.file_type = params.file_type + uploaded_dataset.dbkey = params.dbkey + uploaded_dataset.space_to_tab = params.space_to_tab + if in_folder: + uploaded_dataset.in_folder = in_folder + uploaded_dataset.data = upload_common.new_upload( trans, uploaded_dataset, library_bunch ) + if params.get( 'link_data_only', False ): + uploaded_dataset.link_data_only = True + uploaded_dataset.data.file_name = os.path.abspath( path ) + uploaded_dataset.data.flush() + return uploaded_dataset def get_server_dir_uploaded_datasets( self, trans, params, full_dir, import_dir_desc, library_bunch, err_redirect, msg ): files = [] try: for entry in os.listdir( full_dir ): # Only import regular files - if os.path.isfile( os.path.join( full_dir, entry ) ): - files.append( entry ) + path = os.path.join( full_dir, entry ) + if os.path.islink( path ) and os.path.isfile( path ) and params.get( 'link_data_only', False ): + # If we're linking instead of copying, link the file the link points to, not the link itself. + link_path = os.readlink( path ) + if os.path.isabs( link_path ): + path = link_path + else: + path = os.path.abspath( os.path.join( os.path.dirname( path ), link_path ) ) + if os.path.isfile( path ): + files.append( path ) except Exception, e: msg = "Unable to get file list for configured %s, error: %s" % ( import_dir_desc, str( e ) ) err_redirect = True - return None + return None, err_redirect, msg if not files: msg = "The directory '%s' contains no valid files" % full_dir err_redirect = True - return None + return None, err_redirect, msg uploaded_datasets = [] for file in files: - library_bunch.replace_dataset = None - uploaded_dataset = util.bunch.Bunch() - uploaded_dataset.path = os.path.join( full_dir, file ) - if not os.path.isfile( uploaded_dataset.path ): + name = os.path.basename( file ) + uploaded_datasets.append( self.make_library_uploaded_dataset( trans, params, name, file, 'server_dir', library_bunch ) ) + return uploaded_datasets, None, None + def get_path_paste_uploaded_datasets( self, trans, params, library_bunch, err_redirect, msg ): + if params.get( 'filesystem_paths', '' ) == '': + msg = "No paths entered in the upload form" + err_redirect = True + return None, err_redirect, msg + preserve_dirs = True + if params.get( 'dont_preserve_dirs', False ): + preserve_dirs = False + # locate files + bad_paths = [] + uploaded_datasets = [] + for line in [ l.strip() for l in params.filesystem_paths.splitlines() if l.strip() ]: + path = os.path.abspath( line ) + if not os.path.exists( path ): + bad_paths.append( path ) continue - uploaded_dataset.type = 'server_dir' - uploaded_dataset.name = file - uploaded_dataset.ext = None - uploaded_dataset.file_type = params.file_type - uploaded_dataset.dbkey = params.dbkey - uploaded_dataset.space_to_tab = params.space_to_tab - uploaded_dataset.data = upload_common.new_upload( trans, uploaded_dataset, library_bunch ) - uploaded_datasets.append( uploaded_dataset ) - return uploaded_datasets + # don't bother processing if we're just going to return an error + if not bad_paths: + if os.path.isfile( path ): + name = os.path.basename( path ) + uploaded_datasets.append( self.make_library_uploaded_dataset( trans, params, name, path, 'path_paste', library_bunch ) ) + for basedir, dirs, files in os.walk( line ): + for file in files: + file_path = os.path.abspath( os.path.join( basedir, file ) ) + if preserve_dirs: + in_folder = os.path.dirname( file_path.replace( path, '', 1 ).lstrip( '/' ) ) + else: + in_folder = None + uploaded_datasets.append( self.make_library_uploaded_dataset( trans, params, file, file_path, 'path_paste', library_bunch, in_folder ) ) + if bad_paths: + msg = "Invalid paths:
" % "
  • ".join( bad_paths ) + err_redirect = True + return None, err_redirect, msg + return uploaded_datasets, None, None @web.expose def info_template( self, trans, cntrller, library_id, response_action='library', obj_id=None, folder_id=None, ldda_id=None, **kwd ): # Only adding a new templAte to a library or folder is currently allowed. Editing an existing template is diff --git a/templates/admin/library/browse_library.mako b/templates/admin/library/browse_library.mako index c7c33e7548b..4d7ce6a1377 100644 --- a/templates/admin/library/browse_library.mako +++ b/templates/admin/library/browse_library.mako @@ -73,7 +73,7 @@ // Make ajax call $.ajax( { type: "POST", - url: "${h.url_for( controller='library_dataset', action='library_item_updates' )}", + url: "${h.url_for( controller='library_common', action='library_item_updates' )}", dataType: "json", data: { ids: ids.join( "," ), states: states.join( "," ) }, success : function ( data ) { diff --git a/templates/admin/library/upload.mako b/templates/admin/library/upload.mako index 08718b442ce..463ac399de7 100644 --- a/templates/admin/library/upload.mako +++ b/templates/admin/library/upload.mako @@ -18,6 +18,9 @@ %if trans.app.config.library_import_dir and os.path.exists( trans.app.config.library_import_dir ): Upload directory of files %endif + %if trans.app.config.allow_library_path_paste: + Upload files from filesystem paths + %endif Import datasets from your current history

    diff --git a/templates/library/browse_library.mako b/templates/library/browse_library.mako index adb9bba69d3..ee0de92ad8d 100644 --- a/templates/library/browse_library.mako +++ b/templates/library/browse_library.mako @@ -105,7 +105,7 @@ class RowCounter( object ): // Make ajax call $.ajax( { type: "POST", - url: "${h.url_for( controller='library_dataset', action='library_item_updates' )}", + url: "${h.url_for( controller='library_common', action='library_item_updates' )}", dataType: "json", data: { ids: ids.join( "," ), states: states.join( "," ) }, success : function ( data ) { diff --git a/templates/library/library_dataset_common.mako b/templates/library/library_dataset_common.mako index acad9355441..94a2a8decac 100644 --- a/templates/library/library_dataset_common.mako +++ b/templates/library/library_dataset_common.mako @@ -1,11 +1,13 @@ <%def name="render_upload_form( controller, upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, roles, history )"> <% import os, os.path %> - %if upload_option in [ 'upload_file', 'upload_directory' ]: + %if upload_option in [ 'upload_file', 'upload_directory', 'upload_paths' ]:
    - %if upload_option == 'upload_file': -
    Upload files
    - %else: + %if upload_option == 'upload_directory':
    Upload a directory of files
    + %elif upload_option == 'upload_paths': +
    Upload files from filesystem paths
    + %else: +
    Upload files
    %endif
    @@ -104,6 +106,44 @@
    + %elif upload_option == 'upload_paths': +
    + +
    + +
    +
    + Upload all files pasted in the box. The (recursive) contents of any pasted directories will be added as well. +
    +
    +
    + +
    + No +
    +
    + If checked, all files in subdirectories on the filesystem will be placed at the top level of the folder, instead of into subfolders. +
    +
    + %endif + %if upload_option in ( 'upload_directory', 'upload_paths' ): +
    + +
    + No +
    +
    + Normally data uploaded with this tool is copied into Galaxy's "files" directory + so any later changes to the data will not affect Galaxy. However, this may not + be desired (especially for large NGS datasets), so use of this option will + force Galaxy to always read the data from its original path. + %if upload_option == 'upload_directory': + Any symlinks encountered in the upload directory will be dereferenced once - + that is, Galaxy will point directly to the file that is linked, but no other + symlinks further down the line will be dereferenced. + %endif +
    +
    %endif