Performance improvements in the GalaxyRBACAgent for checking access permissions on libraries. Also added information to main config regarding mysql timeouts ( resolves ticket # 132 ).

lib/galaxy/model/__init__.py

@@ -723,15 +723,19 @@ class LibraryFolder( object ):
         return None
     @property
     def active_components( self ):
-        return list( self.active_folders ) + list( self.active_datasets )
+        return list( self.active_folders ) + list( self.active_library_datasets )
     @property
-    def active_datasets( self ):
+    def active_library_datasets( self ):
          # This needs to be a list
         return [ ld for ld in self.datasets if not ld.library_dataset_dataset_association.deleted ]
     @property
-    def activatable_datasets( self ):
+    def activatable_library_datasets( self ):
          # This needs to be a list
         return [ ld for ld in self.datasets if not ld.library_dataset_dataset_association.dataset.deleted ]
+    @property
+    def active_datasets( self ):
+         # This needs to be a list
+        return [ ld.library_dataset_dataset_association.dataset for ld in self.datasets if not ld.library_dataset_dataset_association.deleted ]
     @property #make this a relation
     def activatable_folders( self ):
         # This needs to be a list

lib/galaxy/model/mapping.py

@@ -740,7 +740,7 @@ assign_mapper( context, GroupRoleAssociation, GroupRoleAssociation.table,
 assign_mapper( context, DatasetPermissions, DatasetPermissions.table,
     properties=dict(
         dataset=relation( Dataset, backref="actions" ),
-        role=relation( Role, backref="actions" )
+        role=relation( Role, backref="dataset_actions" )
     )
 )
 

lib/galaxy/security/__init__.py

@@ -33,8 +33,10 @@ class RBACAgent:
     def get_actions( self ):
         """Get all permitted actions as a list of Action objects"""
         return self.permitted_actions.__dict__.values()
-    def allow_action( self, user, action, **kwd ):
+    def allow_action( self, user, roles, action, **kwd ):
         raise 'No valid method of checking action (%s) on %s for user %s.' % ( action, kwd, user )
+    def get_item_action( self, action, item ):
+        raise 'No valid method of retrieving action (%s) for item %s.' % ( action, item )
     def guess_derived_permissions_for_datasets( self, datasets = [] ):
         raise "Unimplemented Method"
     def associate_components( self, **kwd ):
@@ -79,40 +81,48 @@ class GalaxyRBACAgent( RBACAgent ):
             ( self.model.LibraryFolder, self.model.LibraryFolderPermissions ),
             ( self.model.LibraryDataset, self.model.LibraryDatasetPermissions ),
             ( self.model.LibraryDatasetDatasetAssociation, self.model.LibraryDatasetDatasetAssociationPermissions ) )
-    def allow_action( self, user, action, **kwd ):
+    @property
+    def sa_session( self ):
+        """
+        Returns a SQLAlchemy session -- currently just gets the current
+        session from the threadlocal session context, but this is provided
+        to allow migration toward a more SQLAlchemy 0.4 style of use.
+        """
+        return self.model.context.current
+    def allow_action( self, user, roles, action, **kwd ):
         if 'dataset' in kwd:
-            return self.allow_dataset_action( user, action, kwd[ 'dataset' ] )
+            return self.allow_dataset_action( user, roles, action, kwd[ 'dataset' ] )
         elif 'library_item' in kwd:
-            return self.allow_library_item_action( user, action, kwd[ 'library_item' ] )
+            return self.allow_library_item_action( user, roles, action, kwd[ 'library_item' ] )
         raise 'No valid method of checking action (%s) for user %s using kwd %s' % ( action, str( user ), str( kwd ) )
-    def allow_dataset_action( self, user, action, dataset ):
+    def allow_dataset_action( self, user, roles, action, dataset ):
         """Returns true when user has permission to perform an action"""
-        if not isinstance( dataset, self.model.Dataset ):
-            dataset = dataset.dataset
         if not user:
             if action == self.permitted_actions.DATASET_ACCESS and action.action not in [ dp.action for dp in dataset.actions ]:
-                return True # anons only get access, and only if there are no roles required for the access action
-            # other actions (or if the dataset has roles defined for the access action) fall through to the false below
+                # anons only get access, and only if there are no roles required for the access action
+                # Other actions (or if the dataset has roles defined for the access action) fall through
+                # to the false below
+                return True
         elif action.action not in [ dp.action for dp in dataset.actions ]:
             if action.model == 'restrict':
-                return True # implicit access to restrict-style actions if the dataset does not have the action
-            # grant-style actions fall through to the false below
+                # Implicit access to restrict-style actions if the dataset does not have the action
+                # Grant style actions fall through to the false below
+                return True
         else:
-            user_role_ids = sorted( [ r.id for r in user.all_roles() ] )
             perms = self.get_dataset_permissions( dataset )
             if action in perms.keys():
                 # The filter() returns a list of the dataset's role ids of which the user is not a member,
                 # so an empty list means the user has all of the required roles.
-                if not filter( lambda x: x not in user_role_ids, [ r.id for r in perms[ action ] ] ):
-                    return True # user has all of the roles required to perform the action
-                # Fall through to the false because the user is missing at least one required role
-        return False # default is to reject
-    def allow_library_item_action( self, user, action, library_item ):
+                if not filter( lambda x: x not in roles, [ r for r in perms[ action ] ] ):
+                    # User has all of the roles required to perform the action
+                    return True
+        # The user is missing at least one required role
+        return False
+    def allow_library_item_action( self, user, roles, action, library_item ):
         if user is None:
             # All permissions are granted, so non-users cannot have permissions
             return False
         if action.model == 'grant':
-            user_role_ids = [ r.id for r in user.all_roles() ]
             # Check to see if user has access to any of the roles
             allowed_role_assocs = []
             for item_class, permission_class in self.library_item_assocs:
@@ -126,11 +136,17 @@ class GalaxyRBACAgent( RBACAgent ):
                     elif permission_class == self.model.LibraryDatasetDatasetAssociationPermissions:
                         allowed_role_assocs = permission_class.filter_by( action=action.action, library_dataset_dataset_association_id=library_item.id ).all()
             for allowed_role_assoc in allowed_role_assocs:
-                if allowed_role_assoc.role_id in user_role_ids:
+                if allowed_role_assoc.role in roles:
                     return True
             return False
         else:
             raise 'Unimplemented model (%s) specified for action (%s)' % ( action.model, action.action )
+    def get_item_action( self, action, item ):
+        # item must be one of: Dataset, Library, LibraryFolder, LibraryDataset, LibraryDatasetDatasetAssociation
+        for permission in item.actions:
+            if permission.action == action:
+                return permission
+        return None
     def guess_derived_permissions_for_datasets( self, datasets=[] ):
         """Returns a dict of { action : [ role, role, ... ] } for the output dataset based upon provided datasets"""
         perms = {}
@@ -260,7 +276,10 @@ class GalaxyRBACAgent( RBACAgent ):
                 if [ assoc for assoc in dataset.history_associations if assoc.history not in user.histories ]:
                     # Don't change permissions on a dataset associated with a history not owned by the user
                     continue
-                if bypass_manage_permission or self.allow_action( user, self.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=dataset ):
+                if bypass_manage_permission or self.allow_action( user,
+                                                                  user.all_roles(),
+                                                                  self.permitted_actions.DATASET_MANAGE_PERMISSIONS,
+                                                                  dataset=dataset ):
                     self.set_all_dataset_permissions( dataset, permissions )
     def history_get_default_permissions( self, history ):
         permissions = {}
@@ -402,17 +421,18 @@ class GalaxyRBACAgent( RBACAgent ):
                         lp = permission_class( action.action, target_library_item, private_role )
                         lp.flush()
             else:
-                raise 'Invalid class (%s) specified for target_library_item (%s)' % ( target_library_item.__class__, target_library_item.__class__.__name__ )
-    def show_library_item( self, user, library_item ):
-        if self.allow_action( user, self.permitted_actions.LIBRARY_MODIFY, library_item=library_item ) or \
-            self.allow_action( user, self.permitted_actions.LIBRARY_MANAGE, library_item=library_item ) or \
-            self.allow_action( user, self.permitted_actions.LIBRARY_ADD, library_item=library_item ):
+                raise 'Invalid class (%s) specified for target_library_item (%s)' % \
+                    ( target_library_item.__class__, target_library_item.__class__.__name__ )
+    def show_library_item( self, user, roles, library_item ):
+        if self.allow_action( user, roles, self.permitted_actions.LIBRARY_MODIFY, library_item=library_item ) or \
+            self.allow_action( user, roles, self.permitted_actions.LIBRARY_MANAGE, library_item=library_item ) or \
+            self.allow_action( user, roles, self.permitted_actions.LIBRARY_ADD, library_item=library_item ):
             return True
         if isinstance( library_item, self.model.Library ):
-            return self.show_library_item( user, library_item.root_folder )
+            return self.show_library_item( user, roles, library_item.root_folder )
         elif isinstance( library_item, self.model.LibraryFolder ):
             for folder in library_item.folders:
-                if self.show_library_item( user, folder ):
+                if self.show_library_item( user, roles, folder ):
                     return True
         return False
     def set_entity_user_associations( self, users=[], roles=[], groups=[], delete_existing_assocs=True ):
@@ -462,26 +482,30 @@ class GalaxyRBACAgent( RBACAgent ):
             if 'role' in kwd:
                 return self.model.GroupRoleAssociation.filter_by( role_id = kwd['role'].id, group_id = kwd['group'].id ).first()
         raise 'No valid method of associating provided components: %s' % kwd
-    def check_folder_contents( self, user, entry ):
+    def check_folder_contents( self, user, roles, folder ):
         """
-        Return true if there are any datasets under 'folder' that the
-        user has access permission on.  We do this a lot and it's a
-        pretty inefficient method, optimizations are welcomed.
+        Return true if there are any datasets under 'folder' that are public or that the
+        user has access permission on.
         """
-        if isinstance( entry, self.model.Library ):
-            return self.check_folder_contents( user, entry.root_folder )
-        elif isinstance( entry, self.model.LibraryFolder ):
-            for library_dataset in entry.active_datasets:
-                if self.allow_action( user, self.permitted_actions.DATASET_ACCESS, dataset=library_dataset.library_dataset_dataset_association.dataset ):
-                    return True
-            for folder in entry.active_folders:
-                if self.check_folder_contents( user, folder ):
-                    return True
-            return False
-        elif isinstance( entry, self.model.LibraryDatasetDatasetAssociation ):
-            return self.allow_action( user, self.permitted_actions.DATASET_ACCESS, dataset=entry.dataset )
-        else:
-            raise 'Passed an illegal object to check_folder_contents: %s' % type( entry )
+        action = self.permitted_actions.DATASET_ACCESS.action
+        lddas = self.sa_session.query( self.model.LibraryDatasetDatasetAssociation ) \
+            .join( "library_dataset" ) \
+            .filter( self.model.LibraryDataset.folder == folder ) \
+            .join( "dataset" ) \
+            .options( eagerload_all( "dataset.actions" ) ) \
+            .all()
+        for ldda in lddas:
+            ldda_access = self.get_item_action( action, ldda.dataset )
+            if ldda_access is None:
+                # Dataset is public
+                return True
+            if ldda_access.role in roles:
+                # The current user has access permission on the dataset
+                return True
+        for sub_folder in folder.active_folders:
+            if self.check_folder_contents( user, roles, sub_folder ):
+                return True
+        return False
 
 class HostAgent( RBACAgent ):
     """

lib/galaxy/tools/actions/__init__.py

@@ -47,8 +47,15 @@ class DefaultToolAction( object ):
                             assoc.dataset = new_data
                             assoc.flush()
                             data = new_data
-                # TODO, Nate: Make sure the permitted actions here are appropriate.
-                if data and not trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset=data ):
+                user = trans.user
+                if user:
+                    roles = user.all_roles()
+                else:
+                    roles = None
+                if data and not trans.app.security_agent.allow_action( user,
+                                                                       roles,
+                                                                       data.permitted_actions.DATASET_ACCESS,
+                                                                       dataset=data.dataset ):
                     raise "User does not have permission to use a dataset (%s) provided for input." % data.id
                 return data
             if isinstance( input, DataToolParameter ):
@@ -261,10 +268,17 @@ class DefaultToolAction( object ):
         #        parameters to the command as a special case.
         for name, value in tool.params_to_strings( incoming, trans.app ).iteritems():
             job.add_parameter( name, value )
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
         for name, dataset in inp_data.iteritems():
             if dataset:
-                # TODO, Nate: Make sure the permitted actions here are appropriate.
-                if not trans.app.security_agent.allow_action( trans.user, dataset.permitted_actions.DATASET_ACCESS, dataset=dataset ):
+                if not trans.app.security_agent.allow_action( user,
+                                                              roles,
+                                                              dataset.permitted_actions.DATASET_ACCESS,
+                                                              dataset=dataset.dataset ):
                     raise "User does not have permission to use a dataset (%s) provided for input." % data.id
                 job.add_input_dataset( name, dataset )
             else:

lib/galaxy/tools/parameters/basic.py

@@ -1137,6 +1137,11 @@ class DataToolParameter( ToolParameter ):
         field = form_builder.SelectField( self.name, self.multiple, None, self.refresh_on_change, refresh_on_change_values = self.refresh_on_change_values )
         # CRUCIAL: the dataset_collector function needs to be local to DataToolParameter.get_html_field()
         def dataset_collector( hdas, parent_hid ):
+            user = trans.user
+            if user:
+                roles = user.all_roles()
+            else:
+                roles = None
             for i, hda in enumerate( hdas ):
                 if len( hda.name ) > 30:
                     hda_name = '%s..%s' % ( hda.name[:17], hda.name[-11:] )
@@ -1148,12 +1153,18 @@ class DataToolParameter( ToolParameter ):
                     hid = str( hda.hid )
                 if not hda.dataset.state in [galaxy.model.Dataset.states.ERROR, galaxy.model.Dataset.states.DISCARDED] and \
                     hda.visible and \
-                    trans.app.security_agent.allow_action( trans.user, hda.permitted_actions.DATASET_ACCESS, dataset=hda ):
+                    trans.app.security_agent.allow_action( user,
+                                                           roles,
+                                                           hda.permitted_actions.DATASET_ACCESS,
+                                                           dataset=hda.dataset ):
                     # If we are sending data to an external application, then we need to make sure there are no roles
                     # associated with the dataset that restrict it's access from "public".  We determine this by sending
                     # None as the user to the allow_action method.
                     if self.tool and self.tool.tool_type == 'data_destination':
-                        if not trans.app.security_agent.allow_action( None, hda.permitted_actions.DATASET_ACCESS, dataset=hda ):
+                        if not trans.app.security_agent.allow_action( None,
+                                                                      None,
+                                                                      hda.permitted_actions.DATASET_ACCESS,
+                                                                      dataset=hda.dataset ):
                             continue
                     if self.options and hda.get_dbkey() != filter_value:
                         continue
@@ -1165,7 +1176,10 @@ class DataToolParameter( ToolParameter ):
                         if target_ext:
                             if converted_dataset:
                                 hda = converted_dataset
-                            if not trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.DATASET_ACCESS, dataset=hda.dataset ):
+                            if not trans.app.security_agent.allow_action( user,
+                                                                          roles,
+                                                                          trans.app.security_agent.permitted_actions.DATASET_ACCESS,
+                                                                          dataset=hda.dataset ):
                                 continue
                             selected = ( value and ( hda in value ) )
                             field.add_option( "%s: (as %s) %s" % ( hid, target_ext, hda_name ), hda.id, selected )

lib/galaxy/web/controllers/admin.py

@@ -269,7 +269,7 @@ class Admin( BaseController ):
             gra.delete()
             gra.flush()
         # Delete DatasetPermissionss
-        for dp in role.actions:
+        for dp in role.dataset_actions:
             dp.delete()
             dp.flush()
         msg = "The following have been purged from the database for role '%s': " % role.name
@@ -997,16 +997,8 @@ class Admin( BaseController ):
                                         msg=msg,
                                         messagetype=messagetype )
         elif action == 'delete':
-            def delete_folder( folder ):
-                folder.refresh()
-                for subfolder in folder.active_folders:
-                    delete_folder( subfolder )
-                for ldda in folder.active_datasets:
-                    ldda.deleted = True
-                    ldda.flush()
-                folder.deleted = True
-                folder.flush()
-            delete_folder( folder )
+            folder.deleted = True
+            folder.flush()
             msg = "Folder '%s' and all of its contents have been marked deleted" % folder.name
             return trans.response.send_redirect( web.url_for( action='browse_library',
                                                               id=library_id,

lib/galaxy/web/controllers/dataset.py

@@ -108,7 +108,15 @@ class DatasetInterface( BaseController ):
         data = trans.app.model.HistoryDatasetAssociation.get( dataset_id )
         if not data:
             raise paste.httpexceptions.HTTPRequestRangeNotSatisfiable( "Invalid reference dataset id: %s." % str( dataset_id ) )
-        if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
+        if trans.app.security_agent.allow_action( user,
+                                                  roles,
+                                                  data.permitted_actions.DATASET_ACCESS,
+                                                  dataset=data.dataset ):
             if data.state == trans.model.Dataset.states.UPLOAD:
                 return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to view it." )
             if filename is None or filename.lower() == "index":
@@ -142,9 +150,17 @@ class DatasetInterface( BaseController ):
         if 'display_url' not in kwd or 'redirect_url' not in kwd:
             return trans.show_error_message( 'Invalid parameters specified for "display at" link, please contact a Galaxy administrator' )
         redirect_url = kwd['redirect_url'] % urllib.quote_plus( kwd['display_url'] )
-        if trans.app.security_agent.allow_action( None, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
+        if trans.app.security_agent.allow_action( None, None, data.permitted_actions.DATASET_ACCESS, dataset=data.dataset ):
             return trans.response.send_redirect( redirect_url ) # anon access already permitted by rbac
-        if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+        if trans.app.security_agent.allow_action( user,
+                                                  roles,
+                                                  data.permitted_actions.DATASET_ACCESS,
+                                                  dataset=data.dataset ):
             trans.app.host_security_agent.set_dataset_permissions( data, trans.user, site )
             return trans.response.send_redirect( redirect_url )
         else:

lib/galaxy/web/controllers/history.py

@@ -411,6 +411,7 @@ class HistoryController( BaseController ):
                                                           err_msg=err_msg,
                                                           share_button=True ) )
         user = trans.get_user()
+        user_roles = user.all_roles()
         histories, send_to_users, send_to_err = self._get_histories_and_users( trans, user, id, email )
         send_to_err = ''
         # The user has made a choice, so dictionaries will be built for sharing
@@ -442,13 +443,15 @@ class HistoryController( BaseController ):
                     for hda in history.activatable_datasets:
                         # If the current dataset is not public, we may need to perform an action on it to
                         # make it accessible by the other user.
-                        if not trans.app.security_agent.allow_action( send_to_user, 
+                        if not trans.app.security_agent.allow_action( send_to_user,
+                                                                      send_to_user.all_roles(),
                                                                       trans.app.security_agent.permitted_actions.DATASET_ACCESS, 
-                                                                      dataset=hda ):
+                                                                      dataset=hda.dataset ):
                             # The user with which we are sharing the history does not have access permission on the current dataset
-                            if trans.app.security_agent.allow_action( user, 
+                            if trans.app.security_agent.allow_action( user,
+                                                                      user_roles,
                                                                       trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, 
-                                                                      dataset=hda ) and not hda.dataset.library_associations:
+                                                                      dataset=hda.dataset ) and not hda.dataset.library_associations:
                                 # The current user has authority to change permissions on the current dataset because
                                 # they have permission to manage permissions on the dataset and the dataset is not associated 
                                 # with a library.
@@ -525,6 +528,7 @@ class HistoryController( BaseController ):
         cannot_change = {}
         no_change_needed = {}
         unique_no_change_needed = {}
+        user_roles = user.all_roles()
         for history in histories:
             for send_to_user in send_to_users:
                 # Make sure the current history has not already been shared with the current send_to_user
@@ -552,13 +556,15 @@ class HistoryController( BaseController ):
                                 no_change_needed[ send_to_user ][ history ] = [ hda ]
                             else:
                                 no_change_needed[ send_to_user ][ history ].append( hda )
-                        elif not trans.app.security_agent.allow_action( send_to_user, 
+                        elif not trans.app.security_agent.allow_action( send_to_user,
+                                                                        send_to_user.all_roles(),
                                                                         trans.app.security_agent.permitted_actions.DATASET_ACCESS, 
-                                                                        dataset=hda ):
+                                                                        dataset=hda.dataset ):
                             # The user with which we are sharing the history does not have access permission on the current dataset
-                            if trans.app.security_agent.allow_action( user, 
+                            if trans.app.security_agent.allow_action( user,
+                                                                      user_roles,
                                                                       trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, 
-                                                                      dataset=hda ) and not hda.dataset.library_associations:
+                                                                      dataset=hda.dataset ) and not hda.dataset.library_associations:
                                 # The current user has authority to change permissions on the current dataset because
                                 # they have permission to manage permissions on the dataset and the dataset is not associated 
                                 # with a library.

lib/galaxy/web/controllers/library.py

@@ -62,14 +62,29 @@ class Library( BaseController ):
         params = util.Params( kwd )
         msg = util.restore_text( params.get( 'msg', ''  ) )
         messagetype = params.get( 'messagetype', 'done' )
-        all_libraries = trans.app.model.Library.filter( trans.app.model.Library.table.c.deleted==False ).order_by( trans.app.model.Library.name ).all()
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
+        all_libraries = trans.app.model.Library.filter( trans.app.model.Library.table.c.deleted==False ) \
+                                               .order_by( trans.app.model.Library.name ).all()
         authorized_libraries = []
         for library in all_libraries:
-            if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library ) or \
-               trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library ) or \
-               trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=library ) or \
-               trans.app.security_agent.check_folder_contents( trans.user, library ) or \
-               trans.app.security_agent.show_library_item( trans.user, library ):
+            if trans.app.security_agent.allow_action( user,
+                                                      roles,
+                                                      trans.app.security_agent.permitted_actions.LIBRARY_ADD,
+                                                      library_item=library ) or \
+               trans.app.security_agent.allow_action( user,
+                                                      roles,
+                                                      trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
+                                                      library_item=library ) or \
+               trans.app.security_agent.allow_action( user,
+                                                      roles,
+                                                      trans.app.security_agent.permitted_actions.LIBRARY_MANAGE,
+                                                      library_item=library ) or \
+               trans.app.security_agent.check_folder_contents( user, roles, library.root_folder ) or \
+               trans.app.security_agent.show_library_item( user, roles, library ):
                 authorized_libraries.append( library )
         return trans.fill_template( '/library/browse_libraries.mako', 
                                     libraries=authorized_libraries,
@@ -264,9 +279,15 @@ class Library( BaseController ):
                                                                   msg=util.sanitize_text( msg ),
                                                                   messagetype='error' ) )
             seen = []
+            user = trans.user
+            if user:
+                roles = user.all_roles()
+            else:
+                roles = None
             for id in ldda_ids:
                 ldda = trans.app.model.LibraryDatasetDatasetAssociation.get( id )
-                if not ldda or not trans.app.security_agent.allow_action( trans.user,
+                if not ldda or not trans.app.security_agent.allow_action( user,
+                                                                          roles,
                                                                           trans.app.security_agent.permitted_actions.DATASET_ACCESS,
                                                                           dataset = ldda.dataset ):
                     continue
@@ -363,9 +384,15 @@ class Library( BaseController ):
                                                               id=library_id,
                                                               msg=util.sanitize_text( msg ),
                                                               messagetype='error' ) )
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
         if action == 'information':
             if params.get( 'edit_attributes_button', False ):
-                if trans.app.security_agent.allow_action( trans.user,
+                if trans.app.security_agent.allow_action( user,
+                                                          roles,
                                                           trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
                                                           library_item=library_dataset ):
                     if params.get( 'edit_attributes_button', False ):
@@ -391,7 +418,8 @@ class Library( BaseController ):
                                         messagetype=messagetype )
         elif action == 'permissions':
             if params.get( 'update_roles_button', False ):
-                if trans.app.security_agent.allow_action( trans.user,
+                if trans.app.security_agent.allow_action( user,
+                                                          roles,
                                                           trans.app.security_agent.permitted_actions.LIBRARY_MANAGE,
                                                           library_item=library_dataset ):
                     # The user clicked the Save button on the 'Associate With Roles' form
@@ -436,6 +464,11 @@ class Library( BaseController ):
                 last_used_build = replace_dataset.library_dataset_dataset_association.dbkey
         else:
             replace_dataset = None
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
         # Let's not overwrite the imported datatypes module with the variable datatypes?
         # The built-in 'id' is overwritten in lots of places as well
         ldatatypes = [ dtype_name for dtype_name, dtype_value in trans.app.datatypes_registry.datatypes_by_extension.iteritems() if dtype_value.allow_datatype_change ]
@@ -479,10 +512,12 @@ class Library( BaseController ):
             if action == 'permissions':
                 if params.get( 'update_roles_button', False ):
                     # The user clicked the Save button on the 'Associate With Roles' form
-                    if trans.app.security_agent.allow_action( trans.user,
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MANAGE,
                                                               library_item=ldda ) and \
-                        trans.app.security_agent.allow_action( trans.user,
+                        trans.app.security_agent.allow_action( user,
+                                                               roles,
                                                                trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS,
                                                                dataset=ldda.dataset ):
                         permissions = {}
@@ -523,7 +558,8 @@ class Library( BaseController ):
             elif action == 'edit_info':
                 if params.get( 'change', False ):
                     # The user clicked the Save button on the 'Change data type' form
-                    if trans.app.security_agent.allow_action( trans.user,
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
                                                               library_item=ldda ):
                         if ldda.datatype.allow_datatype_change and trans.app.datatypes_registry.get_datatype_by_extension( params.datatype ).allow_datatype_change:
@@ -546,7 +582,8 @@ class Library( BaseController ):
                                                 messagetype=messagetype )
                 elif params.get( 'save', False ):
                     # The user clicked the Save button on the 'Edit Attributes' form
-                    if trans.app.security_agent.allow_action( trans.user,
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
                                                               library_item=ldda ):
                         old_name = ldda.name
@@ -587,7 +624,8 @@ class Library( BaseController ):
                                                 messagetype=messagetype )
                 elif params.get( 'detect', False ):
                     # The user clicked the Auto-detect button on the 'Edit Attributes' form
-                    if trans.app.security_agent.allow_action( trans.user,
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
                                                               library_item=ldda ):
                         for name, spec in ldda.datatype.metadata_spec.items():
@@ -611,7 +649,8 @@ class Library( BaseController ):
                                                 msg=msg,
                                                 messagetype=messagetype )
                 elif params.get( 'delete', False ):
-                    if trans.app.security_agent.allow_action( trans.user,
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
                                                               library_item=folder ):
                         ldda.deleted = True
@@ -628,7 +667,8 @@ class Library( BaseController ):
                                                 widgets=widgets,
                                                 msg=msg,
                                                 messagetype=messagetype )
-                if trans.app.security_agent.allow_action( trans.user,
+                if trans.app.security_agent.allow_action( user,
+                                                          roles,
                                                           trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
                                                           library_item=ldda ):
                     ldda.datatype.before_edit( ldda )
@@ -668,10 +708,12 @@ class Library( BaseController ):
                                                            messagetype='error' ) )
             if action == 'permissions':
                 if params.get( 'update_roles_button', False ):
-                    if trans.app.security_agent.allow_action( trans.user,
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MANAGE,
                                                               library_item=ldda ) and \
-                        trans.app.security_agent.allow_action( trans.user,
+                        trans.app.security_agent.allow_action( user,
+                                                               roles,
                                                                trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS,
                                                                dataset=ldda.dataset ):
                         permissions = {}
@@ -704,10 +746,12 @@ class Library( BaseController ):
                                                 library_id=library_id,
                                                 msg=msg,
                                                 messagetype=messagetype )
-                if trans.app.security_agent.allow_action( trans.user,
+                if trans.app.security_agent.allow_action( user,
+                                                          roles,
                                                           trans.app.security_agent.permitted_actions.LIBRARY_MANAGE,
                                                           library_item=ldda ) and \
-                    trans.app.security_agent.allow_action( trans.user,
+                    trans.app.security_agent.allow_action( user,
+                                                           roles,
                                                            trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS,
                                                            dataset=ldda.dataset ):
                     # Ensure that the permissions across all library items are identical, otherwise we can't update them together.
@@ -741,10 +785,12 @@ class Library( BaseController ):
                                             library_id=library_id,
                                             msg=msg,
                                             messagetype=messagetype )
-        if trans.app.security_agent.allow_action( trans.user,
+        if trans.app.security_agent.allow_action( user,
+                                                  roles,
                                                   trans.app.security_agent.permitted_actions.LIBRARY_ADD,
                                                   library_item=folder ) or \
-             ( replace_dataset and trans.app.security_agent.allow_action( trans.user,
+             ( replace_dataset and trans.app.security_agent.allow_action( user,
+                                                                          roles,
                                                                           trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
                                                                           library_item=replace_dataset ) ):
             if params.get( 'new_dataset_button', False ):
@@ -769,7 +815,8 @@ class Library( BaseController ):
                     # Since permissions on all LibraryDatasetDatasetAssociations must be the same at this point, we only need
                     # to check one of them to see if the current user can manage permissions on them.
                     check_ldda = trans.app.model.LibraryDatasetDatasetAssociation.get( ldda_id_list[0] )
-                    if trans.app.security_agent.allow_action( trans.user,
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MANAGE,
                                                               library_item=check_ldda ):
                         if replace_dataset:
@@ -892,7 +939,13 @@ class Library( BaseController ):
                     # Since permissions on all LibraryDatasetDatasetAssociations must be the same at this point, we only need
                     # to check one of them to see if the current user can manage permissions on them.
                     check_ldda = trans.app.model.LibraryDatasetDatasetAssociation.get( ldda_id_list[0] )
-                    if trans.app.security_agent.allow_action( trans.user,
+                    user = trans.user
+                    if user:
+                        roles = user.all_roles()
+                    else:
+                        roles = None
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
                                                               trans.app.security_agent.permitted_actions.LIBRARY_MANAGE,
                                                               library_item=check_ldda ):
                         if replace_dataset:
@@ -957,6 +1010,11 @@ class Library( BaseController ):
                                                               id=library_id,
                                                               msg=util.sanitize_text( msg ),
                                                               messagetype='error' ) )
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
         if action == 'new':
             if params.new == 'submitted':
                 new_folder = trans.app.model.LibraryFolder( name=util.restore_text( params.name ),
@@ -994,7 +1052,8 @@ class Library( BaseController ):
             else:
                 widgets = []
             if params.get( 'rename_folder_button', False ):
-                if trans.app.security_agent.allow_action( trans.user, 
+                if trans.app.security_agent.allow_action( user,
+                                                          roles,
                                                           trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, 
                                                           library_item=folder ):
                     old_name = folder.name
@@ -1037,7 +1096,8 @@ class Library( BaseController ):
         elif action == 'permissions':
             if params.get( 'update_roles_button', False ):
                 # The user clicked the Save button on the 'Associate With Roles' form
-                if trans.app.security_agent.allow_action( trans.user, 
+                if trans.app.security_agent.allow_action( user,
+                                                          roles,
                                                           trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, 
                                                           library_item=folder ):
                     permissions = {}
@@ -1162,12 +1222,24 @@ class Library( BaseController ):
                                                           msg=util.sanitize_text( msg ),
                                                           messagetype='done' ) )
 
-
-def get_authorized_libs(trans, user):
-    all_libraries = trans.app.model.Library.filter(trans.app.model.Library.table.c.deleted == False).order_by(trans.app.model.Library.name).all()
+def get_authorized_libs( trans, user ):
+    # TODO: this is a mis-named function - the name should reflect the authorization policy
+    # If user is not authenticated, this method should not even be called.  Also, it looks
+    # like all that is using this is the new request stuff, so it should be placed there.
+    if not user:
+        return []
+    roles = user.all_roles()
+    all_libraries = trans.app.model.Library.filter( trans.app.model.Library.table.c.deleted == False ) \
+                                           .order_by( trans.app.model.Library.name ).all()
     authorized_libraries = []
     for library in all_libraries:
-        if trans.app.security_agent.allow_action(user, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library) \
-            or trans.app.security_agent.allow_action(user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library):
-            authorized_libraries.append(library)
-    return authorized_libraries
+        if trans.app.security_agent.allow_action( user,
+                                                  roles,
+                                                  trans.app.security_agent.permitted_actions.LIBRARY_ADD,
+                                                  library_item=library ) \
+            or trans.app.security_agent.allow_action( user,
+                                                      roles,
+                                                      trans.app.security_agent.permitted_actions.LIBRARY_MODIFY,
+                                                      library_item=library ):
+            authorized_libraries.append( library )
+    return authorized_libraries

lib/galaxy/web/controllers/root.py

@@ -152,7 +152,15 @@ class RootController( BaseController ):
             except:
                 return "Dataset id '%s' is invalid" %str( id )
         if data:
-            if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+            user = trans.user
+            if user:
+                roles = user.all_roles
+            else:
+                roles = None
+            if trans.app.security_agent.allow_action( user,
+                                                      roles,
+                                                      data.permitted_actions.DATASET_ACCESS,
+                                                      dataset = data.dataset ):
                 mime = trans.app.datatypes_registry.get_mimetype_by_extension( data.extension.lower() )
                 trans.response.set_content_type(mime)
                 if tofile:
@@ -184,7 +192,15 @@ class RootController( BaseController ):
             if data:
                 child = data.get_child_by_designation( designation )
                 if child:
-                    if trans.app.security_agent.allow_action( trans.user, child.permitted_actions.DATASET_ACCESS, dataset = child ):
+                    user = trans.user
+                    if user:
+                        roles = user.all_roles
+                    else:
+                        roles = None
+                    if trans.app.security_agent.allow_action( user,
+                                                              roles,
+                                                              child.permitted_actions.DATASET_ACCESS,
+                                                              dataset = child ):
                         return self.display( trans, id=child.id, tofile=tofile, toext=toext )
                     else:
                         return "You are not privileged to access this dataset."
@@ -200,11 +216,21 @@ class RootController( BaseController ):
         if 'authz_method' in kwd:
             authz_method = kwd['authz_method']
         if data:
-            if authz_method == 'rbac' and trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+            user = trans.user
+            if user:
+                roles = user.all_roles
+            else:
+                roles = None
+            if authz_method == 'rbac' and trans.app.security_agent.allow_action( user,
+                                                                                 roles,
+                                                                                 data.permitted_actions.DATASET_ACCESS,
+                                                                                 dataset = data ):
                 trans.response.set_content_type( data.get_mime() )
                 trans.log_event( "Formatted dataset id %s for display at %s" % ( str( id ), display_app ) )
                 return data.as_display_type( display_app, **kwd )
-            elif authz_method == 'display_at' and trans.app.host_security_agent.allow_action( trans.request.remote_addr, data.permitted_actions.DATASET_ACCESS, dataset = data ):
+            elif authz_method == 'display_at' and trans.app.host_security_agent.allow_action( trans.request.remote_addr,
+                                                                                              data.permitted_actions.DATASET_ACCESS,
+                                                                                              dataset = data ):
                 trans.response.set_content_type( data.get_mime() )
                 return data.as_display_type( display_app, **kwd )
             else:
@@ -247,7 +273,15 @@ class RootController( BaseController ):
             return trans.show_error_message( "Problem retrieving dataset." )
         if id is not None and data.history.user is not None and data.history.user != trans.user:
             return trans.show_error_message( "This instance of a dataset (%s) in a history does not belong to you." % ( data.id ) )
-        if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset=data ):
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
+        if trans.app.security_agent.allow_action( user,
+                                                  roles,
+                                                  data.permitted_actions.DATASET_ACCESS,
+                                                  dataset=data.dataset ):
             if data.state == trans.model.Dataset.states.UPLOAD:
                 return trans.show_error_message( "Please wait until this dataset finishes uploading before attempting to edit its metadata." )
             params = util.Params( kwd, safe=False )
@@ -313,7 +347,10 @@ class RootController( BaseController ):
             elif params.update_roles_button:
                 if not trans.user:
                     return trans.show_error_message( "You must be logged in if you want to change permissions." )
-                if trans.app.security_agent.allow_action( trans.user, data.dataset.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset = data.dataset ):
+                if trans.app.security_agent.allow_action( user,
+                                                          roles,
+                                                          data.dataset.permitted_actions.DATASET_MANAGE_PERMISSIONS,
+                                                          dataset = data.dataset ):
                     permissions = {}
                     for k, v in trans.app.model.Dataset.permitted_actions.items():
                         in_roles = params.get( k + '_in', [] )

templates/admin/library/browse_library.mako

@@ -138,12 +138,12 @@ def name_sorted( l ):
         %if show_deleted:
             <%
                 parent_folders = folder.activatable_folders
-                parent_datasets = folder.activatable_datasets
+                parent_datasets = folder.activatable_library_datasets
             %>
         %else:
             <%
                 parent_folders = folder.active_folders
-                parent_datasets = folder.active_datasets
+                parent_datasets = folder.active_library_datasets
             %>
         %endif
         %for folder in name_sorted( parent_folders ):

templates/dataset/edit_attributes.mako

@@ -3,6 +3,13 @@
 
 <%def name="title()">${_('Edit Dataset Attributes')}</%def>
 
+<%
+    user = trans.user
+    if user:
+        user_roles = user.all_roles()
+    else:
+        user_roles = None
+%>
 
 <%def name="datatype( dataset, datatypes )">
     <select name="datatype">
@@ -134,9 +141,9 @@
 </div>
 <p />
 
-%if trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset = data ):
+%if trans.app.security_agent.allow_action( user, user_roles, data.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=data.dataset ):
     <%namespace file="/dataset/security_common.mako" import="render_permission_form" />
-    ${render_permission_form( data.dataset, data.name, h.url_for( controller='root', action='edit', id=data.id ), trans.user.all_roles() )}
+    ${render_permission_form( data.dataset, data.name, h.url_for( controller='root', action='edit', id=data.id ), user_roles )}
 %elif trans.user:
     <div class="toolForm">
         <div class="toolFormTitle">View Permissions</div>

templates/library/browse_library.mako

@@ -1,6 +1,15 @@
 <%inherit file="/base.mako"/>
 <%namespace file="/message.mako" import="render_msg" />
-<% from galaxy import util %>
+<% 
+    from galaxy import util
+    from time import strftime
+    
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
+%>
 
 <%def name="title()">Browse data library</%def>
 <%def name="stylesheets()">
@@ -110,14 +119,14 @@ class RowCounter( object ):
             <a href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, info=True )}"><b>${ldda.name[:60]}</b></a>
             <a id="dataset-${ldda.id}-popup" class="popup-arrow" style="display: none;">&#9660;</a>
             <div popupmenu="dataset-${ldda.id}-popup">
-                %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
+                %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
                     <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, edit_info=True )}">Edit this dataset's information</a>
                 %else:
                     <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, information=True )}">View this dataset's information</a>
                 %endif
-                %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=ldda.dataset ) and trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=ldda.library_dataset ):
+                %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=ldda.dataset ) and trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=ldda.library_dataset ):
                     <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, permissions=True )}">Edit this dataset's permissions</a>
-                %if current_version and trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
+                %if current_version and trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
                     <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, replace_id=library_dataset.id )}">Upload a new version of this dataset</a>
                 %endif
                 %endif
@@ -126,24 +135,26 @@ class RowCounter( object ):
                     <a class="action-button" href="${h.url_for( controller='library', action='download_dataset_from_folder', id=ldda.id, library_id=library.id )}">Download this dataset</a>
                 %endif
             </div>
-            
         </td>
         <td>${ldda.message}</td>
         <td>${uploaded_by}</td>
         <td>${ldda.create_time.strftime( "%Y-%m-%d" )}</td>
-    </tr>
-                
+    </tr>     
     <%
         my_row = row_counter.count
         row_counter.increment()
     %>
 </%def>
 
-
 <%def name="render_folder( folder, folder_pad, created_ldda_ids, library_id, parent=None, row_counter=None )">
     <%
         def show_folder():
-            if trans.app.security_agent.check_folder_contents( trans.user, folder ) or trans.app.security_agent.show_library_item( trans.user, folder ):
+            ## TODO: instead of calling check_folder_contents(), which we've already done prior to getting here,
+            ## add a new method that will itself call check_folder_contents() and build a list of accessible folders
+            ## for each library - this should improve performance dor large libraries where the current user can only
+            ## access a small number of folders.
+            if trans.app.security_agent.check_folder_contents( user, roles, folder ) or \
+                trans.app.security_agent.show_library_item( user, roles, folder ):
                 return True
             return False
         if not show_folder:
@@ -179,21 +190,21 @@ class RowCounter( object ):
                 %endif
                 <a id="folder_img-${folder.id}-popup" class="popup-arrow" style="display: none;">&#9660;</a>
                 <div popupmenu="folder_img-${folder.id}-popup">
-                    %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=folder ):
+                    %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=folder ):
                         <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library_id, folder_id=folder.id )}">Add datasets to this folder</a>
                         <a class="action-button" href="${h.url_for( controller='library', action='folder', new=True, id=folder.id, library_id=library_id )}">Create a new sub-folder in this folder</a>
                     %endif
-                    %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=folder ):
+                    %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=folder ):
                         <a class="action-button" href="${h.url_for( controller='library', action='folder', information=True, id=folder.id, library_id=library_id )}">Edit this folder's information</a>
                     %else:
                         <a class="action-button" href="${h.url_for( controller='library', action='folder', information=True, id=folder.id, library_id=library_id )}">View this folder's information</a>
                     %endif
                     %if forms and not folder.info_association:
-                        %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library ):
+                        %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library ):
                             <a class="action-button" href="${h.url_for( controller='library', action='info_template', library_id=library.id, add=True )}">Add an information template to this folder</a>
                         %endif
                     %endif
-                    %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=folder ):
+                    %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=folder ):
                         <a class="action-button" href="${h.url_for( controller='library', action='folder', permissions=True, id=folder.id, library_id=library_id )}">Edit this folder's permissions</a>
                     %endif
                 </div>
@@ -208,11 +219,11 @@ class RowCounter( object ):
     %for child_folder in name_sorted( folder.active_folders ):
         ${render_folder( child_folder, pad, created_ldda_ids, library_id, my_row, row_counter )}
     %endfor
-    %for library_dataset in name_sorted( folder.active_datasets ):
+    %for library_dataset in name_sorted( folder.active_library_datasets ):
         <%
             selected = created_ldda_ids and library_dataset.library_dataset_dataset_association.id in created_ldda_ids
         %>
-        %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.DATASET_ACCESS, dataset=library_dataset.library_dataset_dataset_association.dataset ):
+        %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.DATASET_ACCESS, dataset=library_dataset.library_dataset_dataset_association.dataset ):
             ${render_dataset( library_dataset, selected, library, pad, my_row, row_counter )}
         %endif
     %endfor
@@ -221,7 +232,7 @@ class RowCounter( object ):
 <h2>Data Library &ldquo;${library.name}&rdquo;</h2>
 
 <ul class="manage-table-actions">
-    %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library ):
+    %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library ):
             %if not deleted:
                 <li>
                     <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library.root_folder.id )}"><span>Add datasets to this library</span></a>
@@ -231,17 +242,17 @@ class RowCounter( object ):
                 </li>
             %endif
     %endif
-    %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library ):
+    %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library ):
         <li><a class="action-button" href="${h.url_for( controller='library', action='library', information=True, id=library.id )}">Edit this library's information</a></li>
     %else:
         <li><a class="action-button" href="${h.url_for( controller='library', action='library', information=True, id=library.id )}">View this library's information</a></li>
     %endif
     %if forms and not library.info_association:
-        %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library ):
+        %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=library ):
             <a class="action-button" href="${h.url_for( controller='library', action='info_template', library_id=library.id, add=True )}">Add an information template to this library</a>
         %endif
     %endif
-    %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=library ):
+    %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=library ):
         <li><a class="action-button" href="${h.url_for( controller='library', action='library', permissions=True, id=library.id )}">Edit this library's permissions</a></li>
     %endif 
 </ul>
@@ -265,7 +276,7 @@ class RowCounter( object ):
             </thead>
         </tr>
         <% row_counter = RowCounter() %>
-        ${render_folder( library.root_folder, 0, created_ldda_ids, library.id, Nonw, row_counter )}
+        ${render_folder( library.root_folder, 0, created_ldda_ids, library.id, None, row_counter )}
         <tfoot>
             <tr>
                 <td colspan="4" style="padding-left: 42px;">

templates/library/common.mako

@@ -1,65 +1,3 @@
-<% from time import strftime %>
-
-<%def name="render_dataset( library_dataset, selected, library )">
-    <%
-        ## The received data must always be a LibraryDataset object, but the object id passed to methods from the drop down menu
-        ## should be the underlying ldda id to prevent id collision ( which could happen when displaying children, which are always
-        ## lddas ).  We also need to make sure we're displaying the latest version of this library_dataset, so we display the attributes
-        ## from the ldda.
-        ldda = library_dataset.library_dataset_dataset_association
-        if ldda.user:
-            uploaded_by = ldda.user.email
-        else:
-            uploaded_by = 'anonymous'
-        if ldda == ldda.library_dataset.library_dataset_dataset_association:
-            current_version = True
-        else:
-            current_version = False
-    %>
-    <div class="historyItemWrapper historyItem historyItem-${ldda.state}" id="libraryItem-${ldda.id}">
-        ## Header row for library items (name, state, action buttons)
-        <div class="historyItemTitleBar">     
-            <table cellspacing="0" cellpadding="0" border="0" width="100%">
-                <tr>
-                    <td width="*">
-                        %if selected:
-                            <input type="checkbox" name="ldda_ids" value="${ldda.id}" checked/>
-                        %else:
-                            <input type="checkbox" name="ldda_ids" value="${ldda.id}"/>
-                        %endif
-                        <a href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, info=True )}"><b>${ldda.name[:60]}</b></a>
-                        <a id="dataset-${ldda.id}-popup" class="popup-arrow" style="display: none;">&#9660;</a>
-                        <div popupmenu="dataset-${ldda.id}-popup">
-                            %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
-                                <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, edit_info=True )}">Edit this dataset's information</a>
-                            %else:
-                                <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, information=True )}">View this dataset's information</a>
-                            %endif
-                            ## We're disabling the ability to add templates at the LDDA and LibraryDataset level, but will leave this here for possible future use
-                            ##%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_ADD, library_item=ldda.library_dataset ):
-                            ##    <a class="action-button" href="${h.url_for( controller='library', action='info_template', library_id=library.id, library_dataset_id=library_dataset.id, new_template=True )}">Add an information template to this dataset</a>
-                            ##%endif
-                            %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=ldda.dataset ) and trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=ldda.library_dataset ):
-                                <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, id=ldda.id, permissions=True )}">Edit this dataset's permissions</a>
-                            %if current_version and trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
-                                <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library.id, folder_id=library_dataset.folder.id, replace_id=library_dataset.id )}">Upload a new version of this dataset</a>
-                            %endif
-                            %endif
-                            %if ldda.has_data:
-                                <a class="action-button" href="${h.url_for( controller='library', action='datasets', library_id=library.id, ldda_ids=str( ldda.id ), do_action='add' )}">Import this dataset into your current history</a>
-                                <a class="action-button" href="${h.url_for( controller='library', action='download_dataset_from_folder', id=ldda.id, library_id=library.id )}">Download this dataset</a>
-                            %endif
-                        </div>
-                    </td>
-                    <td width="500">${ldda.message}</td>
-                    <td width="150">${uploaded_by}</td>
-                    <td width="60">${ldda.create_time.strftime( "%Y-%m-%d" )}</td>
-                </tr>
-            </table>
-        </div>
-    </div>
-</%def>
-
 <%def name="render_template_info( library_item, library_id, widgets, editable=True )">
     <%
         library_item_type = 'unknown type'
@@ -76,13 +14,18 @@
         elif isinstance( library_item, trans.app.model.LibraryDatasetDatasetAssociation ):
             library_item_type = 'library_dataset_dataset_association'
             library_item_desc = 'library dataset'
+        user = trans.user
+        if user:
+            roles = user.all_roles()
+        else:
+            roles = None
     %>
     %if widgets:
         <p/>
         <div class="toolForm">
             <div class="toolFormTitle">Other information about ${library_item_desc} ${library_item.name}</div>
             <div class="toolFormBody">
-                %if editable and trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library_item ):
+                %if editable and trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library_item ):
                     <form name="edit_info" action="${h.url_for( controller='library', action='edit_template_info', library_id=library_id, num_widgets=len( widgets ) )}" method="post">
                         <input type="hidden" name="library_item_id" value="${library_item.id}"/>
                         <input type="hidden" name="library_item_type" value="${library_item_type}"/>

templates/library/folder_info.mako

@@ -2,6 +2,14 @@
 <%namespace file="/message.mako" import="render_msg" />
 <%namespace file="/library/common.mako" import="render_template_info" />
 
+<%
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
+%>
+
 <br/><br/>
 <ul class="manage-table-actions">
     <li>
@@ -16,7 +24,7 @@
 <div class="toolForm">
     <div class="toolFormTitle">Edit folder name and description</div>
     <div class="toolFormBody">
-        %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=folder ):
+        %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=folder ):
             <form name="folder" action="${h.url_for( controller='library', action='folder', rename=True, id=folder.id, library_id=library_id )}" method="post" >
                 <div class="form-row">
                     <label>Name:</label>

templates/library/folder_permissions.mako

@@ -2,6 +2,14 @@
 <%namespace file="/message.mako" import="render_msg" />
 <%namespace file="/dataset/security_common.mako" import="render_permission_form" />
 
+<%
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
+%>
+
 <br/><br/>
 <ul class="manage-table-actions">
     <li>
@@ -13,6 +21,6 @@
     ${render_msg( msg, messagetype )}
 %endif
 
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=folder ):
+%if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=folder ):
     ${render_permission_form( folder, folder.name, h.url_for( controller='library', action='folder', id=folder.id, library_id=library_id, permissions=True ), trans.user.all_roles() )}
 %endif

templates/library/ldda_edit_info.mako

@@ -3,6 +3,14 @@
 <%namespace file="/library/common.mako" import="render_template_info" />
 <% from galaxy import util %>
 
+<%
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
+%>
+
 %if ldda == ldda.library_dataset.library_dataset_dataset_association:
     <b><i>This is the latest version of this library dataset</i></b>
 %else:
@@ -32,7 +40,7 @@
     </select>
 </%def>
 
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
+%if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
     <div class="toolForm">
         <div class="toolFormTitle">Edit attributes of ${ldda.name}</div>
         <div class="toolFormBody">

templates/library/ldda_info.mako

@@ -8,6 +8,11 @@
         current_version = True
     else:
         current_version = False
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
 %>
 
 %if current_version:
@@ -39,15 +44,15 @@
         Information about ${ldda.name}
         <a id="dataset-${ldda.id}-popup" class="popup-arrow" style="display: none;">&#9660;</a>
         <div popupmenu="dataset-${ldda.id}-popup">
-            %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
+            %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
                 <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library_id, folder_id=ldda.library_dataset.folder.id, id=ldda.id, edit_info=True )}">Edit this dataset's information</a>
             %else:
                 <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library_id, folder_id=ldda.library_dataset.folder.id, id=ldda.id, information=True )}">View this dataset's information</a>
             %endif
-            %if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=ldda.dataset ) and trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=ldda.library_dataset ):
+            %if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.DATASET_MANAGE_PERMISSIONS, dataset=ldda.dataset ) and trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=ldda.library_dataset ):
                 <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library_id, folder_id=ldda.library_dataset.folder.id, id=ldda.id, permissions=True )}">Edit this dataset's permissions</a>
             %endif
-            %if current_version and trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
+            %if current_version and trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=ldda.library_dataset ):
                 <a class="action-button" href="${h.url_for( controller='library', action='library_dataset_dataset_association', library_id=library_id, folder_id=ldda.library_dataset.folder.id, replace_id=ldda.library_dataset.id )}">Upload a new version of this dataset</a>
             %endif
             %if ldda.has_data:
@@ -92,6 +97,8 @@
                     <div><pre id="peek${ldda.id}" class="peek">${ldda.display_peek()}</pre></div>
                 %endif
                 ## Recurse for child datasets
+                ## TODO: eliminate this - child datasets are deprecated, and where does
+                ## render_dataset() come from anyway - it's not imported!
                 %if len( ldda.visible_children ) > 0:
                     <div>
                         There are ${len( ldda.visible_children )} secondary datasets.

templates/library/library_dataset_info.mako

@@ -2,6 +2,14 @@
 <%namespace file="/message.mako" import="render_msg" />
 <%namespace file="/library/common.mako" import="render_template_info" />
 
+<%
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
+%>
+
 %if library_dataset == library_dataset.library_dataset_dataset_association.library_dataset:
     <b><i>This is the latest version of this library dataset</i></b>
 %else:
@@ -19,7 +27,7 @@
     ${render_msg( msg, messagetype )}
 %endif
 
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library_dataset ):
+%if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library_dataset ):
     <div class="toolForm">
         <div class="toolFormTitle">Edit attributes of ${library_dataset.name}</div>
         <div class="toolFormBody">

templates/library/library_dataset_permissions.mako

@@ -2,6 +2,14 @@
 <%namespace file="/message.mako" import="render_msg" />
 <%namespace file="/dataset/security_common.mako" import="render_permission_form" />>
 
+<%
+    user = trans.user
+    if user:
+        user_roles = user.all_roles()
+    else:
+        user_roles = None
+%>
+
 %if library_dataset == library_dataset.library_dataset_dataset_association.library_dataset:
     <b><i>This is the latest version of this library dataset</i></b>
 %else:
@@ -19,7 +27,7 @@
     ${render_msg( msg, messagetype )}
 %endif
 
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_manage, library_item=library_dataset ):
+%if trans.app.security_agent.allow_action( user, user_roles, trans.app.security_agent.permitted_actions.LIBRARY_manage, library_item=library_dataset ):
     <%
         roles = trans.app.model.Role.filter( trans.app.model.Role.table.c.deleted==False ).order_by( trans.app.model.Role.table.c.name ).all()
     %>

templates/library/library_info.mako

@@ -2,6 +2,14 @@
 <%namespace file="/message.mako" import="render_msg" />
 <%namespace file="/library/common.mako" import="render_template_info" />
 
+<%
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
+%>
+
 <br/><br/>
 <ul class="manage-table-actions">
     <li>
@@ -13,7 +21,7 @@
     ${render_msg( msg, messagetype )}
 %endif
 
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library ):
+%if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library ):
     <div class="toolForm">
         <div class="toolFormTitle">Change library name and description</div>
         <div class="toolFormBody">

templates/library/library_permissions.mako

@@ -2,6 +2,14 @@
 <%namespace file="/message.mako" import="render_msg" />
 <%namespace file="/dataset/security_common.mako" import="render_permission_form" />
 
+<%
+    user = trans.user
+    if user:
+        user_roles = user.all_roles()
+    else:
+        user_roles = None
+%>
+
 <br/><br/>
 <ul class="manage-table-actions">
     <li>
@@ -13,7 +21,7 @@
     ${render_msg( msg, messagetype )}
 %endif
 
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=library ):
+%if trans.app.security_agent.allow_action( user, user_roles, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=library ):
     <%
         roles = trans.app.model.Role.filter( trans.app.model.Role.table.c.deleted==False ).order_by( trans.app.model.Role.table.c.name ).all()
     %>

templates/mobile/history/detail.mako

@@ -36,8 +36,14 @@
         
         <div class="secondary">
         ## Body for history items, extra info and actions, data "peek"
-        	
-            %if not trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data.dataset ):
+        	<%
+        	   user = trans.user
+        	   if user:
+        	       roles = user.all_roles()
+        	   else:
+        	       roles = None
+        	%>
+            %if not trans.app.security_agent.allow_action( user, roles, data.permitted_actions.DATASET_ACCESS, dataset = data.dataset ):
                 <div>You do not have permission to view this dataset.</div>
             %elif data_state == "queued":
                 <div>Job is waiting to run</div>

templates/mobile/manage_library.mako

@@ -3,11 +3,19 @@
 <%namespace file="/dataset/security_common.mako" import="render_permission_form" />
 <%namespace file="/library/common.mako" import="render_template_info" />
 
+<%
+    user = trans.user
+    if user:
+        roles = user.all_roles()
+    else:
+        roles = None
+%>
+
 %if msg:
     ${render_msg( msg, messagetype )}
 %endif
 
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library ):
+%if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MODIFY, library_item=library ):
     <div class="toolForm">
         <div class="toolFormTitle">Change library name and description</div>
         <div class="toolFormBody">
@@ -53,7 +61,7 @@
         </div>
     </div>
 %endif
-%if trans.app.security_agent.allow_action( trans.user, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=library ):
+%if trans.app.security_agent.allow_action( user, roles, trans.app.security_agent.permitted_actions.LIBRARY_MANAGE, library_item=library ):
     <%
         roles = trans.app.model.Role.filter( trans.app.model.Role.table.c.deleted==False ).order_by( trans.app.model.Role.table.c.name ).all()
     %>

templates/root/history_common.mako

@@ -2,12 +2,17 @@
 ## Render the dataset `data` as history item, using `hid` as the displayed id
 <%def name="render_dataset( data, hid, show_deleted_on_refresh = False )">
     <%
-	if data.state in ['no state','',None]:
-	    data_state = "queued"
-	else:
-	    data_state = data.state
+    	if data.state in ['no state','',None]:
+    	    data_state = "queued"
+    	else:
+    	    data_state = data.state
+    	user = trans.user
+    	if user:
+    	   roles = user.all_roles()
+    	else:
+    	   roles = None
     %>
-    %if not trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data.dataset ):
+    %if not trans.app.security_agent.allow_action( user, roles, data.permitted_actions.DATASET_ACCESS, dataset = data.dataset ):
         <div class="historyItemWrapper historyItem historyItem-${data_state} historyItem-noPermission" id="historyItem-${data.id}">
     %else:
         <div class="historyItemWrapper historyItem historyItem-${data_state}" id="historyItem-${data.id}">
@@ -41,7 +46,7 @@
         ## Body for history items, extra info and actions, data "peek"
         
         <div id="info${data.id}" class="historyItemBody">
-            %if not trans.app.security_agent.allow_action( trans.user, data.permitted_actions.DATASET_ACCESS, dataset = data.dataset ):
+            %if not trans.app.security_agent.allow_action( user, roles, data.permitted_actions.DATASET_ACCESS, dataset = data.dataset ):
                 <div>You do not have permission to view this dataset.</div>
             %elif data_state == "upload":
                 <div>Dataset is uploading</div>

test/base/twilltestcase.py

@@ -1229,7 +1229,7 @@ class TwillTestCase( unittest.TestCase ):
         tc.fv( '1', 'new_element_description_1', ele_help_1.replace( '+', ' ' ) )
         tc.submit( 'new_info_template_button' )
         self.home()
-    def add_folder( self, library_id, folder_id, name='Folder One', description='NThis is Folder One' ):
+    def add_folder( self, library_id, folder_id, name='Folder One', description='This is Folder One' ):
         """Create a new folder"""
         self.home()
         self.visit_url( "%s/admin/folder?library_id=%s&id=%s&new=True" % ( self.url, library_id, folder_id ) )

universe_wsgi.ini.sample

@@ -26,6 +26,11 @@ database_file = database/universe.sqlite
 #database_engine_option_pool_size = 10
 #database_engine_option_max_overflow = 20
 
+# If using MySQL, see:
+# http://rapd.wordpress.com/2008/03/02/sqlalchemy-sqlerror-operationalerror-2006-mysql-server-has-gone-away/
+# To handle this issue, try the following setting:
+#database_engine_option_pool_recycle = 7200
+
 # Where dataset files are saved
 file_path = database/files
 # Temporary storage for additional datasets, this should be shared through the cluster