From 5f9f45a068ea9af1644d5f4f4dd89d2b15aa167e Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 31 Jan 2011 16:41:03 -0500 Subject: [PATCH] =?UTF-8?q?Several=20data=20library=20improvements:=201.?= =?UTF-8?q?=20Enhance=20importing=20library=20dataset=20into=20the=20curre?= =?UTF-8?q?nt=20history=20by=20now=20allowing=20the=20user=20to=20select?= =?UTF-8?q?=20from=20a=20list=20of=20existing=20histories=20or=20create=20?= =?UTF-8?q?a=20new=20named=20history=20(=20similar=20to=20copying=20datase?= =?UTF-8?q?ts=20).=20=C2=AC=E2=80=A0This=20feature=20is=20available=20for?= =?UTF-8?q?=20a=20specific=20library=20dataset,=20a=20folder=20(=20or=20ro?= =?UTF-8?q?ot=20folder=20),=20when=20selecting=20multiple=20datasets=20fro?= =?UTF-8?q?m=20in=20a=20library,=20=C2=AC=E2=80=A0and=20on=20the=20results?= =?UTF-8?q?=20page=20for=20searched=20library=20datasets.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 2. Add Brad Chapman's latest patch for fixing the lucene search implementation 3. Eliminate searches on dataset state for both whoosh and lucene searches 4. Clarify the objects being searched in the regular search and advanced search boxes in the data libraries grids. 5. Add the ability to perform certain actions on 1 or more selected library datasets on the results page for searched library datasets. 6. Add a "select all" checkbox to the browse_library.mako template. 7. Fixed library functional tests accordingly. --- lib/galaxy/web/controllers/library.py | 18 +- lib/galaxy/web/controllers/library_admin.py | 12 +- lib/galaxy/web/controllers/library_common.py | 567 +++++++++++------- scripts/data_libraries/build_lucene_index.py | 40 +- scripts/data_libraries/build_whoosh_index.py | 10 +- templates/library/common/browse_library.mako | 32 +- templates/library/common/common.mako | 76 ++- .../common/import_datasets_to_histories.mako | 101 ++++ templates/library/common/ldda_info.mako | 2 +- .../library_dataset_search_results.mako | 89 ++- test/base/twilltestcase.py | 7 + test/functional/test_library_security.py | 22 +- tools/data_source/access_libraries.xml | 2 +- universe_wsgi.ini.sample | 9 +- 14 files changed, 662 insertions(+), 325 deletions(-) create mode 100644 templates/library/common/import_datasets_to_histories.mako diff --git a/lib/galaxy/web/controllers/library.py b/lib/galaxy/web/controllers/library.py index 50d3e2863e2..93a9ef9a320 100644 --- a/lib/galaxy/web/controllers/library.py +++ b/lib/galaxy/web/controllers/library.py @@ -6,7 +6,7 @@ from galaxy.model.orm import * from galaxy.datatypes import sniff from galaxy import model, util from galaxy.util.odict import odict -from library_common import lucene_search, whoosh_search +from library_common import get_comptypes, lucene_search, whoosh_search log = logging.getLogger( __name__ ) @@ -25,17 +25,17 @@ class LibraryListGrid( grids.Grid ): template='/library/grid.mako' default_sort_key = "name" columns = [ - NameColumn( "Name", + NameColumn( "Data library name", key="name", link=( lambda library: dict( operation="browse", id=library.id ) ), attach_popup=False, filterable="advanced" ), - DescriptionColumn( "Description", + DescriptionColumn( "Data library description", key="description", attach_popup=False, filterable="advanced" ), ] - columns.append( grids.MulticolFilterColumn( "Search", + columns.append( grids.MulticolFilterColumn( "search library dataset name, info, message, dbkey", cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, @@ -74,8 +74,9 @@ class Library( BaseController ): params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) + default_action = params.get( 'default_action', None ) return trans.fill_template( "/library/index.mako", - default_action=params.get( 'default_action', None ), + default_action=default_action, message=message, status=status ) @web.expose @@ -91,8 +92,9 @@ class Library( BaseController ): search_term = kwd[ "f-free-text-search" ] if trans.app.config.enable_lucene_library_search: indexed_search_enabled = True - search_url = trans.app.config.config_dict.get( "fulltext_find_url", "" ) + search_url = trans.app.config.config_dict.get( "fulltext_url", "" ) if search_url: + indexed_search_enabled = True status, message, lddas = lucene_search( trans, 'library', search_term, search_url, **kwd ) elif trans.app.config.enable_whoosh_library_search: indexed_search_enabled = True @@ -100,11 +102,15 @@ class Library( BaseController ): else: indexed_search_enabled = False if indexed_search_enabled: + comptypes = get_comptypes( trans ) + show_deleted = util.string_as_bool( kwd.get( 'show_deleted', False ) ) use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) ) return trans.fill_template( '/library/common/library_dataset_search_results.mako', cntrller='library', search_term=search_term, + comptypes=comptypes, lddas=lddas, + show_deleted=show_deleted, use_panels=use_panels, message=message, status=status ) diff --git a/lib/galaxy/web/controllers/library_admin.py b/lib/galaxy/web/controllers/library_admin.py index 3daab155771..8012c12e796 100644 --- a/lib/galaxy/web/controllers/library_admin.py +++ b/lib/galaxy/web/controllers/library_admin.py @@ -3,7 +3,7 @@ from galaxy import model, util from galaxy.web.base.controller import * from galaxy.web.framework.helpers import time_ago, iff, grids from galaxy.model.orm import * -from library_common import lucene_search, whoosh_search +from library_common import get_comptypes, lucene_search, whoosh_search # Older py compatibility try: set() @@ -35,12 +35,12 @@ class LibraryListGrid( grids.Grid ): template='/admin/library/grid.mako' default_sort_key = "name" columns = [ - NameColumn( "Name", + NameColumn( "Data library name", key="name", link=( lambda library: dict( operation="browse", id=library.id ) ), attach_popup=False, filterable="advanced" ), - DescriptionColumn( "Description", + DescriptionColumn( "Data library description", key="description", attach_popup=False, filterable="advanced" ), @@ -50,7 +50,7 @@ class LibraryListGrid( grids.Grid ): # Columns that are valid for filtering but are not visible. grids.DeletedColumn( "Deleted", key="deleted", visible=False, filterable="advanced" ) ] - columns.append( grids.MulticolFilterColumn( "Search", + columns.append( grids.MulticolFilterColumn( "search library dataset name, info, message, dbkey", cols_to_filter=[ columns[0], columns[1] ], key="free-text-search", visible=False, @@ -133,11 +133,15 @@ class LibraryAdmin( BaseController ): else: indexed_search_enabled = False if indexed_search_enabled: + comptypes = get_comptypes( trans ) + show_deleted = util.string_as_bool( kwd.get( 'show_deleted', False ) ) use_panels = util.string_as_bool( kwd.get( 'use_panels', False ) ) return trans.fill_template( '/library/common/library_dataset_search_results.mako', cntrller='library_admin', search_term=search_term, + comptypes=comptypes, lddas=lddas, + show_deleted=show_deleted, use_panels=use_panels, message=message, status=status ) diff --git a/lib/galaxy/web/controllers/library_common.py b/lib/galaxy/web/controllers/library_common.py index 7469fa73eab..211241a5900 100644 --- a/lib/galaxy/web/controllers/library_common.py +++ b/lib/galaxy/web/controllers/library_common.py @@ -1,4 +1,4 @@ -import os, os.path, shutil, urllib, StringIO, re, gzip, tempfile, shutil, zipfile, copy, glob, string +import os, os.path, shutil, urllib, StringIO, re, gzip, tempfile, shutil, zipfile, copy, glob, string, urllib2 from galaxy.web.base.controller import * from galaxy import util, jobs from galaxy.datatypes import sniff @@ -7,6 +7,7 @@ from galaxy.util.json import to_json_string from galaxy.tools.actions import upload_common from galaxy.model.orm import * from galaxy.util.streamball import StreamBall +from galaxy.util import inflector from galaxy.web.form_builder import AddressField, CheckboxField, SelectField, TextArea, TextField, WorkflowField, WorkflowMappingField, HistoryField import logging, tempfile, zipfile, tarfile, os, sys, operator from galaxy.eggs import require @@ -20,7 +21,7 @@ try: whoosh_search_enabled = True # The following must be defined exactly like the # schema in ~/scripts/data_libraries/build_whoosh_index.py - schema = Schema( id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT, state=TEXT ) + schema = Schema( id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT ) except ImportError, e: whoosh_search_enabled = False schema = None @@ -125,16 +126,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): message += "Don't navigate away from Galaxy or use the browser's \"stop\" or \"reload\" buttons (on this tab) until the " message += "message \"This job is running\" is cleared from the \"Information\" column below for each selected dataset." status = "info" - comptypes_t = comptypes - if trans.app.config.nginx_x_archive_files_base: - comptypes_t = ['ngxzip'] - for comptype in trans.app.config.disable_library_comptypes: - # TODO: do this once, not every time (we're gonna raise an - # exception every time after the first time) - try: - comptypes_t.remove( comptype ) - except: - pass + comptypes = get_comptypes( trans ) try: return trans.fill_template( '/library/common/browse_library.mako', cntrller=cntrller, @@ -143,17 +135,18 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): created_ldda_ids=created_ldda_ids, hidden_folder_ids=hidden_folder_ids, show_deleted=show_deleted, - comptypes=comptypes_t, + comptypes=comptypes, current_user_roles=current_user_roles, message=message, status=status ) except Exception, e: message = 'Error attempting to display contents of library (%s): %s.' % ( str( library.name ), str( e ) ) status = 'error' + default_action = params.get( 'default_action', None ) return trans.response.send_redirect( web.url_for( use_panels=use_panels, controller=cntrller, action='browse_libraries', - default_action=params.get( 'default_action', None ), + default_action=default_action, message=util.sanitize_text( message ), status=status ) ) @web.expose @@ -871,7 +864,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): message += "Click the Go button at the bottom of this page to edit the permissions on these datasets if necessary." default_action = 'manage_permissions' else: - default_action = 'add' + default_action = 'import_to_histories' trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -1247,7 +1240,7 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): message += "Click the Go button at the bottom of this page to edit the permissions on these datasets if necessary." default_action = 'manage_permissions' else: - default_action = 'add' + default_action = 'import_to_histories' return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -1512,7 +1505,12 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): message=util.sanitize_text( message ), status=status ) ) @web.expose - def act_on_multiple_datasets( self, trans, cntrller, library_id, ldda_ids='', **kwd ): + def act_on_multiple_datasets( self, trans, cntrller, library_id=None, ldda_ids='', **kwd ): + # This method is called from 1 of 3 places: + # - this controller's download_dataset_from_folder() method + # - he browse_library.mako template + # - the library_dataset_search_results.mako template + # In the last case above, we will not have a library_id class NgxZip( object ): def __init__( self, url_base ): self.files = {} @@ -1534,6 +1532,16 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): show_deleted = util.string_as_bool( params.get( 'show_deleted', False ) ) use_panels = util.string_as_bool( params.get( 'use_panels', False ) ) action = params.get( 'do_action', None ) + if action == 'import_to_histories': + return trans.response.send_redirect( web.url_for( controller='library_common', + action='import_datasets_to_histories', + cntrller=cntrller, + library_id=library_id, + ldda_ids=ldda_ids, + use_panels=use_panels, + show_deleted=show_deleted, + message=message, + status=status ) ) lddas = [] error = False is_admin = trans.user_is_admin() and cntrller == 'library_admin' @@ -1557,27 +1565,177 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): message = "Invalid library dataset id ( %s ) specified." % str( ldda_id ) break lddas.append( ldda ) - if action == 'import_to_history' or action == 'add': - if trans.get_history() is None: - # Must be a bot sending a request without having a history. - error = True - message = "You do not have a current history" - elif action == 'manage_permissions': - if not is_admin: + if not is_admin: + if action == 'manage_permissions': for ldda in lddas: if not ( trans.app.security_agent.can_manage_library_item( current_user_roles, ldda ) and \ trans.app.security_agent.can_manage_dataset( current_user_roles, ldda.dataset ) ): error = True message = "You are not authorized to manage permissions on library dataset '%s'." % ldda.name break - elif action == 'delete': - if not is_admin: + elif action == 'delete': for ldda in lddas: if not trans.app.security_agent.can_modify_library_item( current_user_roles, ldda ): error = True message = "You are not authorized to modify library dataset '%s'." % ldda.name break - if error: + if not error: + if action == 'manage_permissions': + trans.response.send_redirect( web.url_for( controller='library_common', + action='ldda_permissions', + cntrller=cntrller, + use_panels=use_panels, + library_id=library_id, + folder_id=trans.security.encode_id( lddas[0].library_dataset.folder.id ), + id=",".join( ldda_ids ), + show_deleted=show_deleted, + message=util.sanitize_text( message ), + status=status ) ) + elif action == 'delete': + for ldda in lddas: + # Do not delete the association, just delete the library_dataset. The + # cleanup_datasets.py script handles everything else. + ld = ldda.library_dataset + ld.deleted = True + trans.sa_session.add( ld ) + trans.sa_session.flush() + message = "The selected datasets have been deleted." + elif action in ['zip','tgz','tbz','ngxzip']: + error = False + killme = string.punctuation + string.whitespace + trantab = string.maketrans(killme,'_'*len(killme)) + try: + outext = 'zip' + if action == 'zip': + # Can't use mkstemp - the file must not exist first + tmpd = tempfile.mkdtemp() + tmpf = os.path.join( tmpd, 'library_download.' + action ) + if ziptype == '64' and trans.app.config.upstream_gzip: + archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_STORED, True ) + elif ziptype == '64': + archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True ) + elif trans.app.config.upstream_gzip: + archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_STORED ) + else: + archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED ) + archive.add = lambda x, y: archive.write( x, y.encode('CP437') ) + elif action == 'tgz': + if trans.app.config.upstream_gzip: + archive = util.streamball.StreamBall( 'w|' ) + outext = 'tar' + else: + archive = util.streamball.StreamBall( 'w|gz' ) + outext = 'tgz' + elif action == 'tbz': + archive = util.streamball.StreamBall( 'w|bz2' ) + outext = 'tbz2' + elif action == 'ngxzip': + archive = NgxZip( trans.app.config.nginx_x_archive_files_base ) + except ( OSError, zipfile.BadZipfile ): + error = True + log.exception( "Unable to create archive for download" ) + message = "Unable to create archive for download, please report this error" + status = 'error' + except: + error = True + log.exception( "Unexpected error %s in create archive for download" % sys.exc_info()[0] ) + message = "Unable to create archive for download, please report - %s" % sys.exc_info()[0] + status = 'error' + if not error: + composite_extensions = trans.app.datatypes_registry.get_composite_extensions() + seen = [] + for ldda in lddas: + if ldda.dataset.state in [ 'new', 'upload', 'queued', 'running', 'empty', 'discarded' ]: + continue + ext = ldda.extension + is_composite = ext in composite_extensions + path = "" + parent_folder = ldda.library_dataset.folder + while parent_folder is not None: + # Exclude the now-hidden "root folder" + if parent_folder.parent is None: + path = os.path.join( parent_folder.library_root[0].name, path ) + break + path = os.path.join( parent_folder.name, path ) + parent_folder = parent_folder.parent + path += ldda.name + while path in seen: + path += '_' + seen.append( path ) + zpath = os.path.split(path)[-1] # comes as base_name/fname + outfname,zpathext = os.path.splitext(zpath) + if is_composite: + # need to add all the components from the extra_files_path to the zip + if zpathext == '': + zpath = '%s.html' % zpath # fake the real nature of the html file + try: + archive.add(ldda.dataset.file_name,zpath) # add the primary of a composite set + except IOError: + error = True + log.exception( "Unable to add composite parent %s to temporary library download archive" % ldda.dataset.file_name) + message = "Unable to create archive for download, please report this error" + status = 'error' + continue + flist = glob.glob(os.path.join(ldda.dataset.extra_files_path,'*.*')) # glob returns full paths + for fpath in flist: + efp,fname = os.path.split(fpath) + if fname > '': + fname = fname.translate(trantab) + try: + archive.add( fpath,fname ) + except IOError: + error = True + log.exception( "Unable to add %s to temporary library download archive %s" % (fname,outfname)) + message = "Unable to create archive for download, please report this error" + status = 'error' + continue + else: # simple case + try: + archive.add( ldda.dataset.file_name, path ) + except IOError: + error = True + log.exception( "Unable to write %s to temporary library download archive" % ldda.dataset.file_name) + message = "Unable to create archive for download, please report this error" + status = 'error' + if not error: + if library_id: + lname = trans.sa_session.query( trans.app.model.Library ).get( trans.security.decode_id( library_id ) ).name + else: + # Request must have coe from the library_dataset_search_results page. + lname = 'selected_dataset' + fname = lname.replace( ' ', '_' ) + '_files' + if action == 'zip': + archive.close() + tmpfh = open( tmpf ) + # clean up now + try: + os.unlink( tmpf ) + os.rmdir( tmpd ) + except OSError: + error = True + log.exception( "Unable to remove temporary library download archive and directory" ) + message = "Unable to create archive for download, please report this error" + status = 'error' + if not error: + trans.response.set_content_type( "application/x-zip-compressed" ) + trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.%s" % (fname,outext) + return tmpfh + elif action == 'ngxzip': + trans.response.set_content_type( "application/zip" ) + trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.%s" % (fname,outext) + trans.response.headers[ "X-Archive-Files" ] = "zip" + return archive + else: + trans.response.set_content_type( "application/x-tar" ) + trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.%s" % (fname,outext) + archive.wsgi_status = trans.response.wsgi_status() + archive.wsgi_headeritems = trans.response.wsgi_headeritems() + return archive.stream + else: + status = 'error' + message = 'Invalid action ( %s ) specified.' % action + if library_id: + # If we have a library_id, browse the associated library return trans.response.send_redirect( web.url_for( controller='library_common', action='browse_library', cntrller=cntrller, @@ -1585,181 +1743,129 @@ class LibraryCommon( BaseController, UsesFormDefinitions ): id=library_id, show_deleted=show_deleted, message=util.sanitize_text( message ), - status='error' ) ) - if action == 'import_to_history' or action == 'add': - history = trans.get_history() - total_imported_lddas = 0 - message = '' - status = 'done' - for ldda in lddas: - if ldda.dataset.state in [ 'new', 'upload', 'queued', 'running', 'empty', 'discarded' ]: - message += "Cannot import dataset '%s' since its state is '%s'. " % ( ldda.name, ldda.dataset.state ) - status = 'error' - elif ldda.dataset.state in [ 'ok', 'error' ]: - hda = ldda.to_history_dataset_association( target_history=history, add_to_history=True ) - total_imported_lddas += 1 - if total_imported_lddas: - trans.sa_session.add( history ) - trans.sa_session.flush() - message += "%i dataset(s) have been imported into your history. " % total_imported_lddas - elif action == 'manage_permissions': - trans.response.send_redirect( web.url_for( controller='library_common', - action='ldda_permissions', - cntrller=cntrller, - use_panels=use_panels, - library_id=library_id, - folder_id=trans.security.encode_id( lddas[0].library_dataset.folder.id ), - id=",".join( ldda_ids ), - show_deleted=show_deleted, - message=util.sanitize_text( message ), - status=status ) ) - elif action == 'delete': - for ldda in lddas: - # Do not delete the association, just delete the library_dataset. The - # cleanup_datasets.py script handles everything else. - ld = ldda.library_dataset - ld.deleted = True - trans.sa_session.add( ld ) - trans.sa_session.flush() - message = "The selected datasets have been removed from this data library" - elif action in ['zip','tgz','tbz','ngxzip']: - error = False - killme = string.punctuation + string.whitespace - trantab = string.maketrans(killme,'_'*len(killme)) - try: - outext = 'zip' - if action == 'zip': - # Can't use mkstemp - the file must not exist first - tmpd = tempfile.mkdtemp() - tmpf = os.path.join( tmpd, 'library_download.' + action ) - if ziptype == '64' and trans.app.config.upstream_gzip: - archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_STORED, True ) - elif ziptype == '64': - archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED, True ) - elif trans.app.config.upstream_gzip: - archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_STORED ) - else: - archive = zipfile.ZipFile( tmpf, 'w', zipfile.ZIP_DEFLATED ) - archive.add = lambda x, y: archive.write( x, y.encode('CP437') ) - elif action == 'tgz': - if trans.app.config.upstream_gzip: - archive = util.streamball.StreamBall( 'w|' ) - outext = 'tar' - else: - archive = util.streamball.StreamBall( 'w|gz' ) - outext = 'tgz' - elif action == 'tbz': - archive = util.streamball.StreamBall( 'w|bz2' ) - outext = 'tbz2' - elif action == 'ngxzip': - archive = NgxZip( trans.app.config.nginx_x_archive_files_base ) - except (OSError, zipfile.BadZipfile): - error = True - log.exception( "Unable to create archive for download" ) - message = "Unable to create archive for download, please report this error" - status = 'error' - except: - error = True - log.exception( "Unexpected error %s in create archive for download" % sys.exc_info()[0]) - message = "Unable to create archive for download, please report - %s" % sys.exc_info()[0] - status = 'error' - if not error: - composite_extensions = trans.app.datatypes_registry.get_composite_extensions( ) - seen = [] - for ldda in lddas: - if ldda.dataset.state in [ 'new', 'upload', 'queued', 'running', 'empty', 'discarded' ]: - continue - ext = ldda.extension - is_composite = ext in composite_extensions - path = "" - parent_folder = ldda.library_dataset.folder - while parent_folder is not None: - # Exclude the now-hidden "root folder" - if parent_folder.parent is None: - path = os.path.join( parent_folder.library_root[0].name, path ) - break - path = os.path.join( parent_folder.name, path ) - parent_folder = parent_folder.parent - path += ldda.name - while path in seen: - path += '_' - seen.append( path ) - zpath = os.path.split(path)[-1] # comes as base_name/fname - outfname,zpathext = os.path.splitext(zpath) - if is_composite: - # need to add all the components from the extra_files_path to the zip - if zpathext == '': - zpath = '%s.html' % zpath # fake the real nature of the html file - try: - archive.add(ldda.dataset.file_name,zpath) # add the primary of a composite set - except IOError: - error = True - log.exception( "Unable to add composite parent %s to temporary library download archive" % ldda.dataset.file_name) - message = "Unable to create archive for download, please report this error" - status = 'error' - continue - flist = glob.glob(os.path.join(ldda.dataset.extra_files_path,'*.*')) # glob returns full paths - for fpath in flist: - efp,fname = os.path.split(fpath) - if fname > '': - fname = fname.translate(trantab) - try: - archive.add( fpath,fname ) - except IOError: - error = True - log.exception( "Unable to add %s to temporary library download archive %s" % (fname,outfname)) - message = "Unable to create archive for download, please report this error" - status = 'error' - continue - else: # simple case - try: - archive.add( ldda.dataset.file_name, path ) - except IOError: - error = True - log.exception( "Unable to write %s to temporary library download archive" % ldda.dataset.file_name) - message = "Unable to create archive for download, please report this error" - status = 'error' - if not error: - lname = trans.sa_session.query( trans.app.model.Library ).get( trans.security.decode_id( library_id ) ).name - fname = lname.replace( ' ', '_' ) + '_files' - if action == 'zip': - archive.close() - tmpfh = open( tmpf ) - # clean up now - try: - os.unlink( tmpf ) - os.rmdir( tmpd ) - except OSError: - error = True - log.exception( "Unable to remove temporary library download archive and directory" ) - message = "Unable to create archive for download, please report this error" - status = 'error' - if not error: - trans.response.set_content_type( "application/x-zip-compressed" ) - trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.%s" % (fname,outext) - return tmpfh - elif action == 'ngxzip': - trans.response.set_content_type( "application/zip" ) - trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.%s" % (fname,outext) - trans.response.headers[ "X-Archive-Files" ] = "zip" - return archive - else: - trans.response.set_content_type( "application/x-tar" ) - trans.response.headers[ "Content-Disposition" ] = "attachment; filename=%s.%s" % (fname,outext) - archive.wsgi_status = trans.response.wsgi_status() - archive.wsgi_headeritems = trans.response.wsgi_headeritems() - return archive.stream + status=status ) ) else: - status = 'error' - message = 'Invalid action ( %s ) specified.' % action - return trans.response.send_redirect( web.url_for( controller='library_common', - action='browse_library', - cntrller=cntrller, - use_panels=use_panels, - id=library_id, - show_deleted=show_deleted, - message=util.sanitize_text( message ), - status=status ) ) + # We must have arrived here from the library_dataset_search_results page, so reddirect there. + search_term = params.get( 'search_term', '' ) + comptypes = get_comptypes( trans ) + return trans.fill_template( '/library/common/library_dataset_search_results.mako', + cntrller=cntrller, + search_term=search_term, + comptypes=comptypes, + lddas=lddas, + show_deleted=show_deleted, + use_panels=use_panels, + message=message, + status=status ) + + @web.expose + def import_datasets_to_histories( self, trans, cntrller, library_id='', folder_id='', ldda_ids='', target_history_ids='', new_history_name='', **kwd ): + # This method is called from one of the following places: + # - a menu option for a library dataset ( ldda_ids will be a singel dataset id ) + # - a menu option for a library folder ( folder_id will have a value ) + # - a menu option for a library dataset search result set ( ldda_ids will be a comma separated string of dataset ids ) + params = util.Params( kwd ) + message = util.restore_text( params.get( 'message', '' ) ) + status = params.get( 'status', 'done' ) + show_deleted = util.string_as_bool( params.get( 'show_deleted', False ) ) + use_panels = util.string_as_bool( params.get( 'use_panels', False ) ) + user = trans.get_user() + current_history = trans.get_history() + if library_id: + library = trans.sa_session.query( trans.model.Library ).get( trans.security.decode_id( library_id ) ) + else: + library = None + if folder_id: + folder = trans.sa_session.query( trans.model.LibraryFolder ).get( trans.security.decode_id( folder_id ) ) + else: + folder = None + ldda_ids = util.listify( ldda_ids ) + if ldda_ids: + # Check boxes cause 2 copies of each id to be included in the request + ldda_ids = map( trans.security.decode_id, ldda_ids ) + unique_ldda_ids = [] + for ldda_id in ldda_ids: + if ldda_id not in unique_ldda_ids: + unique_ldda_ids.append( ldda_id ) + ldda_ids = unique_ldda_ids + target_history_ids = util.listify( target_history_ids ) + if target_history_ids: + target_history_ids = [ trans.security.decode_id( target_history_id ) for target_history_id in target_history_ids if target_history_id ] + if params.get( 'import_datasets_to_histories_button', False ): + invalid_datasets = 0 + if not ldda_ids or not ( target_history_ids or new_history_name ): + message = "You must provide one or more source library datasets and one or more target histories." + status = 'error' + else: + if new_history_name: + new_history = trans.app.model.History() + new_history.name = new_history_name + new_history.user = user + trans.sa_session.add( new_history ) + trans.sa_session.flush() + target_history_ids.append( new_history.id ) + if user: + target_histories = [ hist for hist in map( trans.sa_session.query( trans.app.model.History ).get, target_history_ids ) if ( hist is not None and hist.user == user )] + else: + target_histories = [ current_history ] + if len( target_histories ) != len( target_history_ids ): + message += "You do not have permission to add datasets to %i requested histories. " % ( len( target_history_ids ) - len( target_histories ) ) + status = 'error' + for ldda in map( trans.sa_session.query( trans.app.model.LibraryDatasetDatasetAssociation ).get, ldda_ids ): + if ldda is None: + message += "You tried to import a library dataset that does not exist. " + status = 'error' + invalid_datasets += 1 + elif ldda.dataset.state not in [ trans.model.Dataset.states.OK, trans.model.Dataset.states.ERROR ]: + message += "Cannot import dataset '%s' since its state is '%s'. " % ( ldda.name, ldda.dataset.state ) + status = 'error' + invalid_datasets += 1 + elif not ldda.has_data(): + message += "Cannot import empty dataset '%s'. " % ldda.name + status = 'error' + invalid_datasets += 1 + else: + for target_history in target_histories: + hda = ldda.to_history_dataset_association( target_history=target_history, add_to_history=True ) + trans.sa_session.flush() + hist_names_str = ", ".join( [ target_history.name for target_history in target_histories ] ) + num_source = len( ldda_ids ) - invalid_datasets + num_target = len( target_histories ) + message = "%i %s have been imported into %i %s: %s" % ( num_source, + inflector.cond_plural( num_source, "dataset" ), + num_target, + inflector.cond_plural( num_target, "history" ), + hist_names_str ) + trans.sa_session.refresh( current_history ) + current_user_roles = trans.get_current_user_roles() + source_lddas = [] + if folder: + for library_dataset in folder.datasets: + ldda = library_dataset.library_dataset_dataset_association + if not ldda.deleted and trans.app.security_agent.can_access_library_item( current_user_roles, ldda, trans.user ): + source_lddas.append( ldda ) + elif ldda_ids: + for ldda_id in ldda_ids: + # Secuirty access permiision chcck is not needed here since the current user had access + # to the lddas in order for the menu optin to be available. + ldda = trans.sa_session.query( trans.model.LibraryDatasetDatasetAssociation ).get( ldda_id ) + source_lddas.append( ldda ) + target_histories = [ current_history ] + if user: + target_histories = user.active_histories + return trans.fill_template( "/library/common/import_datasets_to_histories.mako", + cntrller=cntrller, + library=library, + current_history=trans.get_history(), + ldda_ids=ldda_ids, + target_history_ids=target_history_ids, + source_lddas=source_lddas, + target_histories=target_histories, + new_history_name=new_history_name, + show_deleted=show_deleted, + use_panels=use_panels, + message=message, + status=status ) @web.expose def manage_template_inheritance( self, trans, cntrller, item_type, library_id, folder_id=None, ldda_id=None, **kwd ): params = util.Params( kwd ) @@ -2064,6 +2170,31 @@ def get_containing_library_from_library_dataset( trans, library_dataset ): if library.root_folder == folder: return library return None +def get_comptypes( trans ): + comptypes_t = comptypes + if trans.app.config.nginx_x_archive_files_base: + comptypes_t = ['ngxzip'] + for comptype in trans.app.config.disable_library_comptypes: + # TODO: do this once, not every time (we're gonna raise an + # exception every time after the first time) + try: + comptypes_t.remove( comptype ) + except: + pass + return comptypes_t +def get_sorted_accessible_library_items( trans, cntrller, items, sort_attr ): + is_admin = trans.user_is_admin() and cntrller == 'library_admin' + if is_admin: + accessible_items = items + else: + # Enforce access permission settings + current_user_roles = trans.get_current_user_roles() + accessible_items = [] + for item in items: + if trans.app.security_agent.can_access_library_item( current_user_roles, item, trans.user ): + accessible_items.append( item ) + # Sort by name + return sort_by_attr( [ item for item in accessible_items ], sort_attr ) def sort_by_attr( seq, attr ): """ Sort the sequence of objects by object's attribute @@ -2079,25 +2210,12 @@ def sort_by_attr( seq, attr ): intermed = map( None, map( getattr, seq, ( attr, ) * len( seq ) ), xrange( len( seq ) ), seq ) intermed.sort() return map( operator.getitem, intermed, ( -1, ) * len( intermed ) ) -def get_sorted_accessible_library_items( trans, cntrller, items, sort_attr ): - is_admin = trans.user_is_admin() and cntrller == 'library_admin' - if is_admin: - accessible_items = items - else: - # Enforce access permission settings - current_user_roles = trans.get_current_user_roles() - accessible_items = [] - for item in items: - if trans.app.security_agent.can_access_library_item( current_user_roles, item, trans.user ): - accessible_items.append( item ) - # Sort by name - return sort_by_attr( [ item for item in accessible_items ], sort_attr ) def lucene_search( trans, cntrller, search_term, search_url, **kwd ): """Return display of results from a full-text lucene search of data libraries.""" params = util.Params( kwd ) message = util.restore_text( params.get( 'message', '' ) ) status = params.get( 'status', 'done' ) - full_url = "%s?%s" % ( search_url, urllib.urlencode( { "kwd" : search_term } ) ) + full_url = "%s/find?%s" % ( search_url, urllib.urlencode( { "kwd" : search_term } ) ) response = urllib2.urlopen( full_url ) ldda_ids = util.json.from_json_string( response.read() )[ "ids" ] response.close() @@ -2115,17 +2233,20 @@ def whoosh_search( trans, cntrller, search_term, **kwd ): if index_exists: index = whoosh.index.open_dir( whoosh_index_dir ) # Set field boosts for searcher to place equal weight on all search fields. - searcher = index.searcher( weighting=BM25F( field_B={ 'name_B' : 3.5, - 'info_B' : 2.3, - 'dbkey_B' : 3.1, - 'message_B' : 2.1, - 'state_B' : 1.2 } ) ) + searcher = index.searcher( weighting=BM25F( field_B={ 'name_B' : 3.4, + 'info_B' : 3.2, + 'dbkey_B' : 3.3, + 'message_B' : 3.5 } ) ) # Perform search - parser = MultifieldParser( [ 'name', 'info', 'dbkey', 'message', 'state' ], schema=schema ) + parser = MultifieldParser( [ 'name', 'info', 'dbkey', 'message' ], schema=schema ) # Search term with wildcards may be slow... - results = searcher.search( parser.parse( '*' + search_term + '*' ), minscore=1.0 ) + results = searcher.search( parser.parse( '*' + search_term + '*' ), minscore=0.1 ) ldda_ids = [ result[ 'id' ] for result in results ] - lddas = [ trans.app.model.LibraryDatasetDatasetAssociation.get( ldda_id ) for ldda_id in ldda_ids ] + lddas = [] + for ldda_id in ldda_ids: + ldda = trans.app.model.LibraryDatasetDatasetAssociation.get( ldda_id ) + if ldda: + lddas.append( ldda ) lddas = get_sorted_accessible_library_items( trans, cntrller, lddas, 'name' ) else: message = "Tell your Galaxy administrator that the directory %s does not contain valid whoosh indexes" % str( whoosh_index_dir ) diff --git a/scripts/data_libraries/build_lucene_index.py b/scripts/data_libraries/build_lucene_index.py index 10f503be8a3..5489c18bf93 100644 --- a/scripts/data_libraries/build_lucene_index.py +++ b/scripts/data_libraries/build_lucene_index.py @@ -30,7 +30,7 @@ def main( ini_file ): sa_session, gconfig = get_sa_session( ini_file ) max_size = float( gconfig.get( "fulltext_max_size", 100 ) ) * 1048576 ignore_exts = gconfig.get( "fulltext_noindex_filetypes", "" ).split( "," ) - search_url = gconfig.get( "fulltext_index_url", None ) + search_url = gconfig.get( "fulltext_url", None ) if not search_url: raise ValueError( "Need to specify search functionality in universe_wsgi.ini" ) dataset_file = create_dataset_file( get_lddas( sa_session, max_size, ignore_exts ) ) @@ -41,7 +41,7 @@ def main( ini_file ): os.remove( dataset_file ) def build_index( search_url, dataset_file ): - url = "%s?%s" % ( search_url, urllib.urlencode( { "docfile" : dataset_file } ) ) + url = "%s/index?%s" % ( search_url, urllib.urlencode( { "docfile" : dataset_file } ) ) request = urllib2.Request( url ) request.get_method = lambda: "PUT" response = urllib2.urlopen( request ) @@ -57,21 +57,35 @@ def create_dataset_file( dataset_iter ): def get_lddas( sa_session, max_size, ignore_exts ): for ldda in sa_session.query( model.LibraryDatasetDatasetAssociation ).filter_by( deleted=False ): - id = ldda.id - name = ldda.name - info = ldda.library_dataset.get_info() - if info and not info.startswith( 'upload' ): - info = info.replace( 'no info', '' ) - else: - info = '' - dbkey = ldda.metadata.dbkey - state = ldda.state - message = ldda.message if ( float( ldda.dataset.get_size() ) > max_size or ldda.extension in ignore_exts ): fname = "" else: fname = ldda.dataset.get_file_name() - yield id, name, info, dbkey, state, message, fname + yield ldda.id, fname, _get_dataset_metadata(ldda) + +def _get_dataset_metadata(ldda): + """Retrieve descriptions and information associated with a dataset. + """ + lds = ldda.library_dataset + folder_info = _get_folder_info(lds.folder) + lds_info = lds.get_info() + if lds_info and not lds_info.startswith("upload"): + lds_info = lds_info.replace("no info", "") + else: + lds_info = "" + return "%s %s %s %s %s" % (lds.name or "", lds_info, ldda.metadata.dbkey, + ldda.message, folder_info) + +def _get_folder_info(folder): + """Get names and descriptions for all parent folders except top level. + """ + folder_info = "" + if folder and folder.parent: + folder_info = _get_folder_info(folder.parent) + folder_info += " %s %s" % ( + folder.name.replace("Unnamed folder", ""), + folder.description or "") + return folder_info def get_sa_session( ini_file ): conf_parser = ConfigParser.ConfigParser( { 'here':os.getcwd() } ) diff --git a/scripts/data_libraries/build_whoosh_index.py b/scripts/data_libraries/build_whoosh_index.py index dc17f1bb08c..32f54e62678 100644 --- a/scripts/data_libraries/build_whoosh_index.py +++ b/scripts/data_libraries/build_whoosh_index.py @@ -24,7 +24,7 @@ try: from whoosh.fields import Schema, STORED, ID, KEYWORD, TEXT from whoosh.index import Index whoosh_search_enabled = True - schema = Schema( id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT, state=TEXT ) + schema = Schema( id=STORED, name=TEXT, info=TEXT, dbkey=TEXT, message=TEXT ) import galaxy.model.mapping from galaxy import config, model import pkg_resources @@ -43,13 +43,12 @@ def build_index( sa_session, whoosh_index_dir ): else: return a_basestr lddas_indexed = 0 - for id, name, info, dbkey, state, message in get_lddas( sa_session ): + for id, name, info, dbkey, message in get_lddas( sa_session ): writer.add_document( id=id, name=to_unicode( name ), info=to_unicode( info ), dbkey=to_unicode( dbkey ), - message=to_unicode( message ), - state=to_unicode( state ) ) + message=to_unicode( message ) ) lddas_indexed += 1 writer.commit() print "Number of active library datasets indexed: ", lddas_indexed @@ -64,9 +63,8 @@ def get_lddas( sa_session ): else: info = '' dbkey = ldda.metadata.dbkey - state = ldda.state message = ldda.message - yield id, name, info, dbkey, state, message + yield id, name, info, dbkey, message def get_sa_session_and_needed_config_settings( ini_file ): conf_parser = ConfigParser.ConfigParser( { 'here' : os.getcwd() } ) diff --git a/templates/library/common/browse_library.mako b/templates/library/common/browse_library.mako index d598f7dad9f..f72fc831ad6 100644 --- a/templates/library/common/browse_library.mako +++ b/templates/library/common/browse_library.mako @@ -1,6 +1,7 @@ <%namespace file="/message.mako" import="render_msg" /> <%namespace file="/library/common/library_item_info.mako" import="render_library_item_info" /> <%namespace file="/library/common/common.mako" import="render_actions_on_multiple_items" /> +<%namespace file="/library/common/common.mako" import="common_javascripts" /> <%! def inherit(context): @@ -46,6 +47,7 @@ <%def name="javascripts()"> ${parent.javascripts()} ${h.js("class", "jquery.jstore")} + ${common_javascripts()} ${self.grid_javascripts()} @@ -152,15 +154,6 @@ }); }); - function checkForm() { - if ( $("select#action_on_datasets_select option:selected").text() == "delete" ) { - if ( confirm( "Click OK to delete these datasets?" ) ) { - return true; - } else { - return false; - } - } - } // Looks for changes in dataset state using an async request. Keeps // calling itself (via setTimeout) until all datasets are in a terminal // state. @@ -212,7 +205,7 @@ -<%def name="render_dataset( cntrller, ldda, library_dataset, selected, library, folder, pad, parent, row_counter, tracked_datasets, show_deleted=False, render_checkboxes=True )"> +<%def name="render_dataset( cntrller, ldda, library_dataset, selected, library, folder, pad, parent, row_counter, tracked_datasets, show_deleted=False )"> <% ## The received ldda must always be a LibraryDatasetDatasetAssociation object. The object id passed to methods ## from the drop down menu should be the ldda id to prevent id collision ( which could happen when displaying @@ -248,13 +241,11 @@ %endif id="libraryItem-${ldda.id}"> - %if render_checkboxes: - + %if ldda.library_dataset.deleted: %endif @@ -288,7 +279,7 @@ Upload a new version of this dataset %endif %if not branch_deleted( folder ) and not ldda.library_dataset.deleted and ldda.has_data: - Import this dataset into your current history + Import this dataset into selected histories Download this dataset %endif %if can_modify: @@ -379,6 +370,7 @@ Add sub-folder %endif %if not branch_deleted( folder ): + Select folder datasets for import into selected histories %if can_modify: Edit information %else: @@ -515,6 +507,7 @@ %endif Edit permissions %endif + Select datasets for import into selected histories %elif can_modify and not library.purged: Undelete this data library %elif library.purged: @@ -539,7 +532,10 @@ - + diff --git a/templates/library/common/common.mako b/templates/library/common/common.mako index a51cf5d37db..29c3abb70e0 100644 --- a/templates/library/common/common.mako +++ b/templates/library/common/common.mako @@ -1,5 +1,48 @@ <%namespace file="/common/template_common.mako" import="render_template_field" /> +<%def name="common_javascripts()"> + + + <%def name="render_upload_form( cntrller, upload_option, action, library_id, folder_id, replace_dataset, file_formats, dbkeys, space_to_tab, link_data_only, widgets, roles_select_list, history, show_deleted )"> <% import os, os.path @@ -319,26 +362,34 @@ %endif -<%def name="render_actions_on_multiple_items()"> +<%def name="render_actions_on_multiple_items( actions_to_exclude=[] )"> + <% + is_admin = trans.user_is_admin() and cntrller=='library_admin' + can_delete = 'delete' not in actions_to_exclude and is_admin + can_download = 'download' not in actions_to_exclude + can_import_to_histories = 'import_to_histories' not in actions_to_exclude + can_manage_permissions = 'manage_permissions' not in actions_to_exclude + %> diff --git a/templates/library/common/import_datasets_to_histories.mako b/templates/library/common/import_datasets_to_histories.mako new file mode 100644 index 00000000000..ad7522b7bf8 --- /dev/null +++ b/templates/library/common/import_datasets_to_histories.mako @@ -0,0 +1,101 @@ +<%namespace file="/message.mako" import="render_msg" /> +<%inherit file="/base.mako"/> +<%namespace file="/message.mako" import="javascripts" /> +<%def name="title()">Import library datasets to histories + +<%def name="javascripts()"> + +${parent.javascripts()} +${h.js( "jquery", "galaxy.base" )} + + + + +%if message: + ${render_msg( message, status )} +%endif + +

+

Import library datasets into histories.
+
+

+

+
+

+
+ + %if source_lddas: + %for source_ldda in source_lddas: + <% + checked = "" + encoded_id = trans.security.encode_id( source_ldda.id ) + if source_ldda.id in ldda_ids: + checked = " checked='checked'" + %> +
+ + +
+ %endfor + %else: +
This folder has no accessible library datasets.
+ %endif +
+
+
→
+
+
Destination Histories:
+
+
+ +

+ Choose multiple histories +
+ + %if trans.get_user(): + <% + checked = "" + if "create_new_history" in target_history_ids: + checked = " checked='checked'" + %> +
+
— OR —
+
+ + +
+ %endif +
+
+
+
+ +
+ + +

diff --git a/templates/library/common/ldda_info.mako b/templates/library/common/ldda_info.mako index 4b23cddf864..82ac6107f26 100644 --- a/templates/library/common/ldda_info.mako +++ b/templates/library/common/ldda_info.mako @@ -66,7 +66,7 @@ Upload a new version of this dataset %endif %if cntrller=='library' and ldda.has_data(): - Import this dataset into your current history + Select histories to import this datasetDownload this dataset %endif %if show_associated_hdas_and_lddas: diff --git a/templates/library/common/library_dataset_search_results.mako b/templates/library/common/library_dataset_search_results.mako index 6a930471a28..b0ac151a5b7 100644 --- a/templates/library/common/library_dataset_search_results.mako +++ b/templates/library/common/library_dataset_search_results.mako @@ -1,5 +1,7 @@ <%namespace file="/message.mako" import="render_msg" /> <%namespace file="/library/common/browse_library.mako" import="render_dataset" /> +<%namespace file="/library/common/common.mako" import="render_actions_on_multiple_items" /> +<%namespace file="/library/common/common.mako" import="common_javascripts" /> <%! def inherit(context): @@ -44,6 +46,7 @@ <%def name="javascripts()"> ${parent.javascripts()} ${h.js("class", "jquery.jstore")} + ${common_javascripts()} <%def name="render_searched_components()"> @@ -52,7 +55,6 @@
  • info
  • dbkey (genome build)
  • message
    • -
  • state
    • %if trans.app.config.enable_lucene_library_search:
  • disk file content
    • %endif @@ -91,34 +93,69 @@ %if lddas:

    The string "${search_term}" was found in at least one of the following information components of the displayed library datasets.

    ${render_searched_components()} -
    Name + + Name + Message Uploaded By Date
    For selected items:
    - - - - - - - - - - <% - tracked_datasets = {} - row_counter = RowCounter() - my_row = row_counter.count - %> - %for ldda in lddas: - <% - library_dataset = ldda.library_dataset - folder = library_dataset.folder - library = folder.parent_library - %> - ${render_dataset( cntrller, ldda, library_dataset, False, library, folder, 0, my_row, row_counter, tracked_datasets, show_deleted=False, render_checkboxes=False )} + +
    NameMessageUploaded ByDateFile Size
    + + + + + + + + + <% + tracked_datasets = {} + row_counter = RowCounter() my_row = row_counter.count - row_counter.increment() %> - %endfor -
    + + Name + MessageUploaded ByDateFile Size
    + %for ldda in lddas: + <% + library_dataset = ldda.library_dataset + folder = library_dataset.folder + library = folder.parent_library + %> + ${render_dataset( cntrller, ldda, library_dataset, False, library, folder, 0, my_row, row_counter, tracked_datasets, show_deleted=False )} + <% + my_row = row_counter.count + row_counter.increment() + %> + %endfor + ${render_actions_on_multiple_items( actions_to_exclude=[ 'manage_permissions' ] )} + + + ## Help about compression types + +
    +

    + TIP: You can download individual library files by selecting "Download this dataset" from the context menu (triangle) next to the dataset's name. +

    +
    + %if len( comptypes ) > 1: +
    +

    + TIP: Multiple compression options are available for downloading library datasets: +

    +
      + %if 'gz' in comptypes: +
    • gzip: Recommended for fast network connections + %if trans.app.config.upstream_gzip: + NOTE: The file you receive will be an uncompressed .tar file - this is because the Galaxy server compresses it and your browser decompresses it on the fly. + %endif +
      • + %endif + %if 'bz2' in comptypes: +
    • bzip2: Recommended for slower network connections (smaller size but takes longer to compress)
      • + %endif + %if 'zip' in comptypes: +
    • zip: Not recommended but is provided as an option for those who cannot open the above formats
      • + %endif +
    +
    + %endif %elif status != 'error':

    The string "${search_term}" was not found in any of the following information components for any library datasets that you can access.

    ${render_searched_components()} diff --git a/test/base/twilltestcase.py b/test/base/twilltestcase.py index dd83dd27669..47211875421 100644 --- a/test/base/twilltestcase.py +++ b/test/base/twilltestcase.py @@ -2106,6 +2106,13 @@ class TwillTestCase( unittest.TestCase ): % ( self.url, cntrller, library_id, ldda_ids, do_action ) ) for check_str in strings_displayed: self.check_page_for_string( check_str ) + def import_datasets_to_histories( self, cntrller, library_id, ldda_ids='', new_history_name='Unnamed history', strings_displayed=[] ): + # Can't use the ~/library_admin/libraries form as twill barfs on it so we'll simulate the form submission + # by going directly to the form action + self.visit_url( '%s/library_common/import_datasets_to_histories?cntrller=%s&library_id=%s&ldda_ids=%s&new_history_name=%s&import_datasets_to_histories_button=Import+library+datasets' \ + % ( self.url, cntrller, library_id, ldda_ids, new_history_name ) ) + for check_str in strings_displayed: + self.check_page_for_string( check_str ) def download_archive_of_library_files( self, cntrller, library_id, ldda_ids, format ): self.home() # Here it would be ideal to have twill set form values and submit the form, but diff --git a/test/functional/test_library_security.py b/test/functional/test_library_security.py index 00cd7e997b4..09ad5b37c3b 100644 --- a/test/functional/test_library_security.py +++ b/test/functional/test_library_security.py @@ -296,11 +296,11 @@ class TestLibrarySecurity( TwillTestCase ): strings_displayed=[ '2.bed', 'This is the latest version of this library dataset', 'Edit attributes of 2.bed' ] ) - self.act_on_multiple_datasets( 'library', - self.security.encode_id( library1.id ), - 'import_to_history', - ldda_ids=self.security.encode_id( ldda2.id ), - strings_displayed=[ '1 dataset(s) have been imported into your history' ] ) + self.import_datasets_to_histories( cntrller='library', + library_id=self.security.encode_id( library1.id ), + ldda_ids=self.security.encode_id( ldda2.id ), + new_history_name='goodbye', + strings_displayed=[ '1 dataset have been imported into 1 history' ] ) self.logout() # regular_user2 should not be able to see ldda2 self.login( email=regular_user2.email ) @@ -382,10 +382,10 @@ class TestLibrarySecurity( TwillTestCase ): def check_edit_page( lddas, strings_displayed=[], strings_not_displayed=[] ): for ldda in lddas: # Import each library dataset into our history - self.act_on_multiple_datasets( 'library', - self.security.encode_id( library1.id ), - 'import_to_history', - ldda_ids=self.security.encode_id( ldda.id ) ) + self.import_datasets_to_histories( cntrller='library', + library_id=self.security.encode_id( library1.id ), + ldda_ids=self.security.encode_id( ldda.id ), + new_history_name='hello' ) # Determine the new HistoryDatasetAssociation id created when the library dataset was imported into our history last_hda_created = get_latest_hda() self.edit_hda_attribute_info( str( last_hda_created.id ), @@ -393,8 +393,8 @@ class TestLibrarySecurity( TwillTestCase ): # admin_user is associated with role1, so should have all permissions on imported datasets check_edit_page( latest_3_lddas, strings_displayed=[ 'Manage dataset permissions on', - 'can manage the roles associated with permissions on this dataset', - 'can import this dataset into their history for analysis' ] ) + 'can manage the roles associated with permissions on this dataset', + 'can import this dataset into their history for analysis' ] ) self.logout() # regular_user1 is associated with role1, so should have all permissions on imported datasets self.login( email=regular_user1.email ) diff --git a/tools/data_source/access_libraries.xml b/tools/data_source/access_libraries.xml index 31195582f3e..374cbe1bcd8 100644 --- a/tools/data_source/access_libraries.xml +++ b/tools/data_source/access_libraries.xml @@ -2,7 +2,7 @@ stored locally - + diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 65b96e312b9..eba12617ea8 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -337,18 +337,17 @@ use_interactive = True #transfer_manager_port = 8163 # Search data libraries with whoosh -enable_whoosh_library_search = True +#enable_whoosh_library_search = True # Whoosh indexes are stored in this directory. -whoosh_index_dir = database/whoosh_indexes +#whoosh_index_dir = database/whoosh_indexes # Search data libraries with lucene #enable_lucene_library_search = False # maxiumum file size to index for searching, in MB #fulltext_max_size = 500 #fulltext_noindex_filetypes=bam,sam,wig,bigwig,fasta,fastq,fastqsolexa,fastqillumina,fastqsanger -# base URL of server providing search functionality using whoosh. -#fulltext_index_url = http://localhost:8081/index -#fulltext_find_url = http://localhost:8081/find +# base URL of server providing search functionality using lucene +#fulltext_url = http://localhost:8081 # -- Users and Security