diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index a28deb61c48..df571eb1120 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -480,7 +480,7 @@ class Sam( Tabular ): if result != 0: raise Exception('Result %s from %s' % (result, cmd)) if len(split_files) > 1: - cmd = 'egrep -v "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file ) + cmd = 'egrep -v -h "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file ) result = os.system(cmd) if result != 0: raise Exception('Result %s from %s' % (result, cmd)) diff --git a/lib/galaxy/eggs/scramble.py b/lib/galaxy/eggs/scramble.py index 0a7970db2b9..528bb60dd49 100644 --- a/lib/galaxy/eggs/scramble.py +++ b/lib/galaxy/eggs/scramble.py @@ -134,7 +134,7 @@ class ScrambleEgg( Egg ): elif zipfile.is_zipfile( self.source_path ): self.unpack_zip() else: - raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, source_path ) ) + raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, self.source_path ) ) log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name ) log.warning( " %s" % self.buildpath ) def unpack_zip( self ): diff --git a/lib/galaxy/managers/folders.py b/lib/galaxy/managers/folders.py index ca5b5cf3894..7968048bdcd 100644 --- a/lib/galaxy/managers/folders.py +++ b/lib/galaxy/managers/folders.py @@ -3,6 +3,8 @@ Manager and Serializer for Library Folders. """ import galaxy.exceptions +from sqlalchemy.orm.exc import MultipleResultsFound +from sqlalchemy.orm.exc import NoResultFound import logging log = logging.getLogger( __name__ ) @@ -19,19 +21,21 @@ class FolderManager( object ): :param decoded_folder_id: decoded folder id :type decoded_folder_id: int - :param check_manageable: flag whether the check that user can manage item - :type check_manageable: bool + :param check_manageable: flag whether the check that user can manage item + :type check_manageable: bool :param check_accessible: flag whether to check that user can access item :type check_accessible: bool :returns: the requested folder :rtype: LibraryFolder + + :raises: InconsistentDatabase, RequestParameterInvalidException, InternalServerError """ try: folder = trans.sa_session.query( trans.app.model.LibraryFolder ).filter( trans.app.model.LibraryFolder.table.c.id == decoded_folder_id ).one() - except galaxy.exceptions.MultipleResultsFound: + except MultipleResultsFound: raise galaxy.exceptions.InconsistentDatabase( 'Multiple folders found with the same id.' ) - except galaxy.exceptions.NoResultFound: + except NoResultFound: raise galaxy.exceptions.RequestParameterInvalidException( 'No folder found with the id provided.' ) except Exception, e: raise galaxy.exceptions.InternalServerError( 'Error loading from the database.' + str( e ) ) @@ -139,6 +143,14 @@ class FolderManager( object ): """ Mark given folder deleted/undeleted based on the flag. + :param folder: the model object + :type folder: LibraryFolder + :param undelete: flag whether to delete (when False) or undelete + :type undelete: Bool + + :returns: the folder + :rtype: LibraryFolder + :raises: ItemAccessibilityException """ if not trans.user_is_admin(): @@ -190,6 +202,14 @@ class FolderManager( object ): def cut_the_prefix( self, encoded_folder_id ): """ Remove the prefix from the encoded folder id. + + :param encoded_folder_id: encoded id of the Folder object with 'F' prepended + :type encoded_folder_id: string + + :returns: encoded Folder id without the 'F' prefix + :rtype: string + + :raises: MalformedId """ if ( ( len( encoded_folder_id ) % 16 == 1 ) and encoded_folder_id.startswith( 'F' ) ): cut_id = encoded_folder_id[ 1: ] @@ -200,6 +220,14 @@ class FolderManager( object ): def decode_folder_id( self, trans, encoded_folder_id ): """ Decode the folder id given that it has already lost the prefixed 'F'. + + :param encoded_folder_id: encoded id of the Folder object + :type encoded_folder_id: string + + :returns: decoded Folder id + :rtype: int + + :raises: MalformedId """ try: decoded_id = trans.security.decode_id( encoded_folder_id ) @@ -210,5 +238,11 @@ class FolderManager( object ): def cut_and_decode( self, trans, encoded_folder_id ): """ Cuts the folder prefix (the prepended 'F') and returns the decoded id. + + :param encoded_folder_id: encoded id of the Folder object + :type encoded_folder_id: string + + :returns: decoded Folder id + :rtype: int """ return self.decode_folder_id( trans, self.cut_the_prefix( encoded_folder_id ) ) diff --git a/lib/galaxy/model/util.py b/lib/galaxy/model/util.py new file mode 100644 index 00000000000..76128609496 --- /dev/null +++ b/lib/galaxy/model/util.py @@ -0,0 +1,34 @@ +""" +Utility helpers related to the model +""" + + +def pgcalc( sa_session, id, dryrun=False ): + """ + Utility method for quickly recalculating user disk usage in postgres. + + TODO: Check against the recently updated versions of sqlalchemy if this + 'special' postgresql version is even necessary. + """ + sql = """ + UPDATE galaxy_user + SET disk_usage = (SELECT COALESCE(SUM(total_size), 0) + FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id + FROM history_dataset_association hda + JOIN history h ON h.id = hda.history_id + JOIN dataset d ON hda.dataset_id = d.id + WHERE h.user_id = :id + AND h.purged = false + AND hda.purged = false + AND d.purged = false + AND d.id NOT IN (SELECT dataset_id + FROM library_dataset_dataset_association) + ) sizes) + WHERE id = :id + RETURNING disk_usage; + """ + r = sa_session.execute(sql, {'id': id}) + new = r.fetchone()[0] + if dryrun: + sa_session.rollback() + return new diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 1b571565b3e..590cf9a37eb 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -2,7 +2,7 @@ A simple WSGI application/framework. """ -import cgi # For FieldStorage +import cgi # For FieldStorage import logging import os.path import socket @@ -67,7 +67,7 @@ class WebApplication( object ): # FIXME: The following two options are deprecated and should be # removed. Consult the Routes documentation. self.mapper.minimization = True - #self.mapper.explicit = False + # self.mapper.explicit = False self.transaction_factory = DefaultWebTransaction # Set if trace logging is enabled self.trace_logger = None @@ -79,12 +79,12 @@ class WebApplication( object ): method use `add_route`. """ log.debug( "Enabling '%s' controller, class: %s", - controller_name, controller.__class__.__name__ ) + controller_name, controller.__class__.__name__ ) self.controllers[ controller_name ] = controller def add_api_controller( self, controller_name, controller ): log.debug( "Enabling '%s' API controller, class: %s", - controller_name, controller.__class__.__name__ ) + controller_name, controller.__class__.__name__ ) self.api_controllers[ controller_name ] = controller def add_route( self, route, **kwargs ): @@ -148,7 +148,7 @@ class WebApplication( object ): else: environ[ 'is_api_request' ] = False controllers = self.controllers - if map == None: + if map is None: raise httpexceptions.HTTPNotFound( "No route for " + path_info ) self.trace( path_info=path_info, map=map ) # Setup routes @@ -239,11 +239,13 @@ class WSGIEnvironmentProperty( object ): associated object (provides property style access to keys in the WSGI environment) """ - def __init__( self, key, default = '' ): + def __init__( self, key, default='' ): self.key = key self.default = default - def __get__( self, obj, type = None ): - if obj is None: return self + + def __get__( self, obj, type=None ): + if obj is None: + return self return obj.environ.get( self.key, self.default ) @@ -254,8 +256,10 @@ class LazyProperty( object ): """ def __init__( self, func ): self.func = func - def __get__(self, obj, type = None ): - if obj is None: return self + + def __get__(self, obj, type=None ): + if obj is None: + return self value = self.func( obj ) setattr( obj, self.func.func_name, value ) return value @@ -273,6 +277,7 @@ class DefaultWebTransaction( object ): self.environ = environ self.request = Request( environ ) self.response = Response() + @lazy_property def session( self ): """ @@ -291,11 +296,14 @@ class DefaultWebTransaction( object ): # but for performance reasons it's way better to use Paste's tempfile than to # create a new one and copy. import tempfile + + class FieldStorage( cgi.FieldStorage ): def make_file(self, binary=None): return tempfile.NamedTemporaryFile() + def read_lines(self): - # Always make a new file + # Always make a new file self.file = self.make_file() self.__file = None if self.outerboundary: @@ -313,45 +321,58 @@ class Request( webob.Request ): """ Create a new request wrapping the WSGI environment `environ` """ - ## self.environ = environ + # self.environ = environ webob.Request.__init__( self, environ, charset='utf-8', decode_param_names=False ) # Properties that are computed and cached on first use + @lazy_property def remote_host( self ): try: return socket.gethostbyname( self.remote_addr ) except socket.error: return self.remote_addr + @lazy_property def remote_hostname( self ): try: return socket.gethostbyaddr( self.remote_addr )[0] except socket.error: return self.remote_addr + @lazy_property def cookies( self ): return get_cookies( self.environ ) + @lazy_property def base( self ): return ( self.scheme + "://" + self.host ) - ## @lazy_property - ## def params( self ): - ## return parse_formvars( self.environ ) + + # @lazy_property + # def params( self ): + # return parse_formvars( self.environ ) + @lazy_property def path( self ): return self.environ['SCRIPT_NAME'] + self.environ['PATH_INFO'] + @lazy_property def browser_url( self ): return self.base + self.path + # Descriptors that map properties to the associated environment - ## scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' ) - ## remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' ) + + # scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' ) + # remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' ) + remote_port = WSGIEnvironmentProperty( 'REMOTE_PORT' ) - ## method = WSGIEnvironmentProperty( 'REQUEST_METHOD' ) - ## script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' ) + + # method = WSGIEnvironmentProperty( 'REQUEST_METHOD' ) + # script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' ) + protocol = WSGIEnvironmentProperty( 'SERVER_PROTOCOL' ) - ## query_string = WSGIEnvironmentProperty( 'QUERY_STRING' ) - ## path_info = WSGIEnvironmentProperty( 'PATH_INFO' ) + + # query_string = WSGIEnvironmentProperty( 'QUERY_STRING' ) + # path_info = WSGIEnvironmentProperty( 'PATH_INFO' ) class Response( object ): @@ -406,7 +427,8 @@ class Response( object ): # ---- Utilities ------------------------------------------------------------ -CHUNK_SIZE = 2**16 +CHUNK_SIZE = 2 ** 16 + def send_file( start_response, trans, body ): # If configured use X-Accel-Redirect header for nginx @@ -426,6 +448,7 @@ def send_file( start_response, trans, body ): trans.response.wsgi_headeritems() ) return body + def iterate_file( file ): """ Progressively return chunks from `file`. @@ -436,13 +459,14 @@ def iterate_file( file ): break yield chunk + def flatten( seq ): """ Flatten a possible nested set of iterables """ for x in seq: if isinstance( x, ( types.GeneratorType, list, tuple ) ): - for y in flatten( x, encoding ): + for y in flatten( x ): yield y else: yield x diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index cc980c7febb..a5149e5848d 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -1,15 +1,17 @@ import imp import logging import os +from sqlalchemy.sql import expression import galaxy.queue_worker import galaxy.util from galaxy import model -from galaxy.model import tool_shed_install as install_model from galaxy import web from galaxy.actions.admin import AdminActions from galaxy.exceptions import MessageException -from galaxy.util import sanitize_text +from galaxy.model import tool_shed_install as install_model +from galaxy.model.util import pgcalc +from galaxy.util import nice_size, sanitize_text from galaxy.util.odict import odict from galaxy.web import url_for from galaxy.web.base.controller import BaseUIController, UsesQuotaMixin @@ -25,23 +27,17 @@ log = logging.getLogger( __name__ ) class UserListGrid( grids.Grid ): - class EmailColumn( grids.TextColumn ): - def get_value( self, trans, grid, user ): return user.email - class UserNameColumn( grids.TextColumn ): - def get_value( self, trans, grid, user ): if user.username: return user.username return 'not set' - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.purged: return "purged" @@ -49,47 +45,35 @@ class UserListGrid( grids.Grid ): return "deleted" return "" - class GroupsColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.groups: return len( user.groups ) return 0 - class RolesColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.roles: return len( user.roles ) return 0 - class ExternalColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.external: return 'yes' return 'no' - class LastLoginColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.galaxy_sessions: return self.format( user.galaxy_sessions[ 0 ].update_time ) return 'never' - class TimeCreatedColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): return user.create_time.strftime('%x') - class ActivatedColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.active: return 'Y' @@ -99,7 +83,7 @@ class UserListGrid( grids.Grid ): # Grid definition title = "Users" model_class = model.User - template='/admin/user/grid.mako' + template = '/admin/user/grid.mako' default_sort_key = "email" columns = [ EmailColumn( "Email", @@ -140,7 +124,11 @@ class UserListGrid( grids.Grid ): condition=( lambda item: not item.deleted ), allow_multiple=True, allow_popup=False, - url_args=dict( webapp="galaxy", action="reset_user_password" ) ) + url_args=dict( webapp="galaxy", action="reset_user_password" ) ), + grids.GridOperation( "Recalculate Disk Usage", + condition=( lambda item: not item.deleted ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="recalculate_user_disk_usage" ) ) ] standard_filters = [ grids.GridColumnFilter( "Active", args=dict( deleted=False ) ), @@ -158,46 +146,33 @@ class UserListGrid( grids.Grid ): class RoleListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, role ): return role.name - class DescriptionColumn( grids.TextColumn ): - def get_value( self, trans, grid, role ): if role.description: return role.description return '' - class TypeColumn( grids.TextColumn ): - def get_value( self, trans, grid, role ): return role.type - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, role ): if role.deleted: return "deleted" return "" - class GroupsColumn( grids.GridColumn ): - def get_value( self, trans, grid, role ): if role.groups: return len( role.groups ) return 0 - class UsersColumn( grids.GridColumn ): - - def get_value( self, trans, grid, role ): if role.users: return len( role.users ) @@ -206,7 +181,7 @@ class RoleListGrid( grids.Grid ): # Grid definition title = "Roles" model_class = model.Role - template='/admin/dataset_security/role/grid.mako' + template = '/admin/dataset_security/role/grid.mako' default_sort_key = "name" columns = [ NameColumn( "Name", @@ -270,31 +245,23 @@ class RoleListGrid( grids.Grid ): class GroupListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, group ): return group.name - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, group ): if group.deleted: return "deleted" return "" - class RolesColumn( grids.GridColumn ): - def get_value( self, trans, grid, group ): if group.roles: return len( group.roles ) return 0 - class UsersColumn( grids.GridColumn ): - def get_value( self, trans, grid, group ): if group.members: return len( group.members ) @@ -303,7 +270,7 @@ class GroupListGrid( grids.Grid ): # Grid definition title = "Groups" model_class = model.Group - template='/admin/dataset_security/group/grid.mako' + template = '/admin/dataset_security/group/grid.mako' default_sort_key = "name" columns = [ NameColumn( "Name", @@ -351,31 +318,24 @@ class GroupListGrid( grids.Grid ): preserve_state = False use_paging = True + class QuotaListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, quota ): return quota.name - class DescriptionColumn( grids.TextColumn ): - def get_value( self, trans, grid, quota ): if quota.description: return quota.description return '' - class AmountColumn( grids.TextColumn ): - def get_value( self, trans, grid, quota ): return quota.operation + quota.display_amount - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, quota ): if quota.deleted: return "deleted" @@ -383,17 +343,13 @@ class QuotaListGrid( grids.Grid ): return "default for %s users" % quota.default[0].type return "" - class UsersColumn( grids.GridColumn ): - def get_value( self, trans, grid, quota ): if quota.users: return len( quota.users ) return 0 - class GroupsColumn( grids.GridColumn ): - def get_value( self, trans, grid, quota ): if quota.groups: return len( quota.groups ) @@ -402,7 +358,7 @@ class QuotaListGrid( grids.Grid ): # Grid definition title = "Quotas" model_class = model.Quota - template='/admin/quota/grid.mako' + template = '/admin/quota/grid.mako' default_sort_key = "name" columns = [ NameColumn( "Name", @@ -417,10 +373,10 @@ class QuotaListGrid( grids.Grid ): attach_popup=False, filterable="advanced" ), AmountColumn( "Amount", - key='amount', - model_class=model.Quota, - attach_popup=False, - filterable="advanced" ), + key='amount', + model_class=model.Quota, + attach_popup=False, + filterable="advanced" ), UsersColumn( "Users", attach_popup=False ), GroupsColumn( "Groups", attach_popup=False ), StatusColumn( "Status", attach_popup=False ), @@ -483,9 +439,7 @@ class QuotaListGrid( grids.Grid ): class ToolVersionListGrid( grids.Grid ): - class ToolIdColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_version ): toolbox = trans.app.toolbox if toolbox.has_tool( tool_version.tool_id, exact=True ): @@ -494,9 +448,7 @@ class ToolVersionListGrid( grids.Grid ): return '
%s%s
' % ( link_str, tool_version.tool_id ) return tool_version.tool_id - class ToolVersionsColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_version ): tool_ids_str = '' toolbox = trans.app.toolbox @@ -508,10 +460,11 @@ class ToolVersionListGrid( grids.Grid ): else: tool_ids_str += '%s
' % tool_id return tool_ids_str + # Grid definition title = "Tool versions" model_class = install_model.ToolVersion - template='/admin/tool_version/grid.mako' + template = '/admin/tool_version/grid.mako' default_sort_key = "tool_id" columns = [ ToolIdColumn( "Tool id", @@ -531,7 +484,7 @@ class ToolVersionListGrid( grids.Grid ): num_rows_per_page = 50 preserve_state = False use_paging = True - + def build_initial_query( self, trans, **kwd ): return trans.install_model.context.query( self.model_class ) @@ -593,14 +546,14 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP new_in_users = [] new_in_groups = [] for user in trans.sa_session.query( trans.app.model.User ) \ - .filter( trans.app.model.User.table.c.deleted==False ) \ + .filter( trans.app.model.User.table.c.deleted == expression.false() ) \ .order_by( trans.app.model.User.table.c.email ): if user.id in in_users: new_in_users.append( ( user.id, user.email ) ) else: params.out_users.append( ( user.id, user.email ) ) for group in trans.sa_session.query( trans.app.model.Group ) \ - .filter( trans.app.model.Group.table.c.deleted==False ) \ + .filter( trans.app.model.Group.table.c.deleted == expression.false() ) \ .order_by( trans.app.model.Group.table.c.name ): if group.id in in_groups: new_in_groups.append( ( group.id, group.name ) ) @@ -645,14 +598,14 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP in_groups = [] out_groups = [] for user in trans.sa_session.query( trans.app.model.User ) \ - .filter( trans.app.model.User.table.c.deleted==False ) \ + .filter( trans.app.model.User.table.c.deleted == expression.false() ) \ .order_by( trans.app.model.User.table.c.email ): if user in [ x.user for x in quota.users ]: in_users.append( ( user.id, user.email ) ) else: out_users.append( ( user.id, user.email ) ) for group in trans.sa_session.query( trans.app.model.Group ) \ - .filter( trans.app.model.Group.table.c.deleted==False ) \ + .filter( trans.app.model.Group.table.c.deleted == expression.false()) \ .order_by( trans.app.model.Group.table.c.name ): if group in [ x.group for x in quota.groups ]: in_groups.append( ( group.id, group.name ) ) @@ -775,7 +728,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP webapp=params.webapp, message=sanitize_text( str( e ) ), status='error' ) ) - if do_op == True or ( do_op != False and params.get( do_op, False ) ): + if do_op is True or ( do_op is not False and params.get( do_op, False ) ): try: message = op_method( quota, params ) return None, trans.response.send_redirect( web.url_for( controller='admin', @@ -899,10 +852,48 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP kwargs={'display_application_ids': kwd.get( 'id' )} ) reloaded, failed = trans.app.datatypes_registry.reload_display_applications( kwd.get( 'id' ) ) if not reloaded and failed: - return trans.show_error_message( 'Unable to reload any of the %i requested display applications ("%s").' % ( len( failed ), '", "'.join( failed ) ) ) + return trans.show_error_message( 'Unable to reload any of the %i requested display applications ("%s").' + % ( len( failed ), '", "'.join( failed ) ) ) if failed: return trans.show_warn_message( 'Reloaded %i display applications ("%s"), but failed to reload %i display applications ("%s").' - % ( len( reloaded ), '", "'.join( reloaded ), len( failed ), '", "'.join( failed ) ) ) + % ( len( reloaded ), '", "'.join( reloaded ), len( failed ), '", "'.join( failed ) ) ) if not reloaded: return trans.show_warn_message( 'You need to request at least one display application to reload.' ) return trans.show_ok_message( 'Reloaded %i requested display applications ("%s").' % ( len( reloaded ), '", "'.join( reloaded ) ) ) + + @web.expose + @web.require_admin + def recalculate_user_disk_usage( self, trans, **kwd ): + user_id = kwd.get( 'id', None ) + user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) ) + if not user: + return trans.show_error_message( "User not found for id (%s)" % sanitize_text( str( user_id ) ) ) + engine = None + if trans.app.config.database_connection: + engine = trans.app.config.database_connection.split(':')[0] + if engine not in ( 'postgres', 'postgresql' ): + done = False + while not done: + current = user.get_disk_usage() + new = user.calculate_disk_usage() + trans.sa_session.refresh( user ) + # make sure usage didn't change while calculating, set done + if user.get_disk_usage() == current: + done = True + if new not in (current, None): + user.set_disk_usage( new ) + trans.sa_session.add( user ) + trans.sa_session.flush() + else: + # We can use the lightning fast pgcalc! + current = user.get_disk_usage() + new = pgcalc( self.sa_session, user.id ) + # yes, still a small race condition between here and the flush + if new in ( current, None ): + message = 'Usage is unchanged at %s.' % nice_size( current ) + else: + message = 'Usage has changed by %s to %s.' % ( nice_size( new - current ), nice_size( current ) ) + return trans.response.send_redirect( web.url_for( controller='admin', + action='users', + message=sanitize_text( message ), + status='info' ) ) diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index 24dc7fc0d0f..05fdceab3ac 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -136,6 +136,10 @@ class Configuration( object ): self.citation_cache_data_dir = resolve_path( kwargs.get( "citation_cache_data_dir", "database/tool_shed_citations/data" ), self.root ) self.citation_cache_lock_dir = resolve_path( kwargs.get( "citation_cache_lock_dir", "database/tool_shed_citations/locks" ), self.root ) + @property + def shed_tool_data_path( self ): + return self.tool_data_path + @property def sentry_dsn_public( self ): """ diff --git a/lib/tool_shed/galaxy_install/installed_repository_manager.py b/lib/tool_shed/galaxy_install/installed_repository_manager.py index 9437c983753..7bd50214b5e 100644 --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -103,12 +103,12 @@ class InstalledRepositoryManager( object ): # Hack to add repository.name here, which is actually the root of the installed repository data_manager_relative_install_dir = os.path.join( data_manager_relative_install_dir, repository.name ) dmh = data_manager.DataManagerHandler( self.app ) - new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, - repository.metadata, - repository.get_shed_config_dict( self.app ), - data_manager_relative_install_dir, - repository, - repository_tools_tups ) + dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + repository.metadata, + repository.get_shed_config_dict( self.app ), + data_manager_relative_install_dir, + repository, + repository_tools_tups ) self.install_model.context.add( repository ) self.install_model.context.flush() if repository.includes_datatypes: @@ -400,14 +400,8 @@ class InstalledRepositoryManager( object ): installed_repository_dependencies = {} missing_rd_tups = [] installed_rd_tups = [] - description, \ - repository_clone_url, \ - changeset_revision, \ - ctx_rev, \ - repository_owner, \ - repository_dependencies, \ - tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + ( description, repository_clone_url, changeset_revision, ctx_rev, + repository_owner, repository_dependencies, tool_dependencies ) = suc.get_repo_info_tuple_contents( repo_info_tuple ) if repository_dependencies: description = repository_dependencies[ 'description' ] root_key = repository_dependencies[ 'root_key' ] @@ -460,7 +454,7 @@ class InstalledRepositoryManager( object ): prior_installation_required, only_if_compiling_contained_td, None, - 'Never installed' ] + 'Never installed' ] if not util.asbool( only_if_compiling_contained_td ): # A repository dependency that is not installed will not be considered missing if its value for # only_if_compiling_contained_td is True - see above... @@ -564,10 +558,10 @@ class InstalledRepositoryManager( object ): continue if required_repository_tup not in dependency_tups: dependency_tups.append( required_repository_tup ) - return get_repository_dependency_tups_for_installed_repository( required_repository, - dependency_tups=dependency_tups ) + return self.get_repository_dependency_tups_for_installed_repository( required_repository, + dependency_tups=dependency_tups ) return dependency_tups - + def get_repository_tuple_for_installed_repository_manager( self, repository ): return ( str( repository.tool_shed ), str( repository.name ), @@ -611,10 +605,10 @@ class InstalledRepositoryManager( object ): contents = open( env_shell_file_path, 'r' ).read() except Exception, e: contents = None - log.debug( 'Error reading file %s, so cannot determine if package %s requires package %s at run time: %s' % \ - ( str( env_shell_file_path ), str( td.name ), str( tool_dependency.name ), str( e ) ) ) + log.debug( 'Error reading file %s, so cannot determine if package %s requires package %s at run time: %s' % + ( str( env_shell_file_path ), str( td.name ), str( tool_dependency.name ), str( e ) ) ) if contents is not None and contents.find( required_env_shell_file_path ) >= 0: - td_tuple = get_tool_dependency_tuple_for_installed_repository_manager( td ) + td_tuple = self.get_tool_dependency_tuple_for_installed_repository_manager( td ) runtime_dependent_tool_dependency_tups.append( td_tuple ) return runtime_dependent_tool_dependency_tups @@ -672,8 +666,7 @@ class InstalledRepositoryManager( object ): # Remove this tool_dependency from all values in # self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = {} - for td_tup, installed_runtime_dependent_tool_dependency_tups in \ - self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.items(): + for ( td_tup, installed_runtime_dependent_tool_dependency_tups ) in self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.items(): if tool_dependency_tup in installed_runtime_dependent_tool_dependency_tups: # Remove the tool_dependency from the list. installed_runtime_dependent_tool_dependency_tups.remove( tool_dependency_tup ) @@ -684,7 +677,7 @@ class InstalledRepositoryManager( object ): altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies # Remove the entry for this tool_dependency from self.runtime_tool_dependencies_of_installed_tool_dependencies. self.remove_entry_from_runtime_tool_dependencies_of_installed_tool_dependencies( tool_dependency ) - # Remove the entry for this tool_dependency from + # Remove the entry for this tool_dependency from # self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies. self.remove_entry_from_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies( tool_dependency ) # Remove this repository's entry from self.installed_tool_dependencies_of_installed_repositories. @@ -864,7 +857,7 @@ class InstalledRepositoryManager( object ): else: status = 'error' message = 'A repository must have the status New in order to be purged. This repository has ' - message += ' the status %s.' % str( repository.status ) + message += ' the status %s.' % str( repository.status ) return status, message def remove_entry_from_installed_repository_dependencies_of_installed_repositories( self, repository ): @@ -884,7 +877,7 @@ class InstalledRepositoryManager( object ): debug_msg += "from the dependent list for revision %s of repository %s owned by %s " % \ ( r_installed_changeset_revision, r_name, r_owner ) debug_msg += "in installed_repository_dependencies_of_installed_repositories." - log.debug( debug_msg ) + log.debug( debug_msg ) v_tups.remove( repository_tup ) altered_installed_dependent_repositories_of_installed_repositories[ r_tup ] = v_tups self.installed_dependent_repositories_of_installed_repositories = \ @@ -957,7 +950,7 @@ class InstalledRepositoryManager( object ): if cleaned_repository_dependency_tool_shed == cleaned_tool_shed and \ repository_dependency.name == name and \ repository_dependency.owner == owner and \ - ( repository_dependency.installed_changeset_revision == changeset_revision or \ + ( repository_dependency.installed_changeset_revision == changeset_revision or repository_dependency.changeset_revision == changeset_revision ): return True return False @@ -970,9 +963,9 @@ class InstalledRepositoryManager( object ): # This method is called only from Galaxy when rendering repository dependencies # for an installed tool shed repository. # TODO: Do we need to check more than changeset_revision here? - required_repository_tup = [ required_repository.tool_shed, \ - required_repository.name, \ - required_repository.owner, \ + required_repository_tup = [ required_repository.tool_shed, + required_repository.name, + required_repository.owner, required_repository.changeset_revision ] for tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td: partial_tup = tup[ 0:4 ] @@ -1009,12 +1002,12 @@ class InstalledRepositoryManager( object ): pulled by a Galaxy administrator when getting updates to an installed tool shed repository. The original_dependency_dict is a single tool dependency definition, an example of which is:: - + {"name": "bwa", "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", "type": "package", "version": "0.6.2"} - + The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method. """ new_tool_dependency = None @@ -1046,12 +1039,12 @@ class InstalledRepositoryManager( object ): break if new_dependency_name and new_dependency_type and new_dependency_version: # Update all attributes of the tool_dependency record in the database. - log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \ - ( str( tool_dependency.version ), - str( tool_dependency.type ), - str( tool_dependency.name ), - str( new_dependency_version ), - str( new_dependency_type ) ) ) + log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." + % ( str( tool_dependency.version ), + str( tool_dependency.type ), + str( tool_dependency.name ), + str( new_dependency_version ), + str( new_dependency_type ) ) ) tool_dependency.type = new_dependency_type tool_dependency.version = new_dependency_version tool_dependency.status = self.app.install_model.ToolDependency.installation_status.UNINSTALLED @@ -1062,8 +1055,8 @@ class InstalledRepositoryManager( object ): else: # We have no new tool dependency definition based on a matching dependency name, so remove # the existing tool dependency record from the database. - log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \ - ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) + log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." + % ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) context.delete( tool_dependency ) context.flush() return new_tool_dependency diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py index 282ad589aa4..b4ae53aaae9 100755 --- a/scripts/set_user_disk_usage.py +++ b/scripts/set_user_disk_usage.py @@ -1,9 +1,22 @@ #!/usr/bin/env python -import os, sys +import os +import sys + from ConfigParser import ConfigParser from optparse import OptionParser +sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) ) + +from galaxy import eggs +import pkg_resources + +import galaxy.config +from galaxy.model.util import pgcalc +from galaxy.util import nice_size +from galaxy.objectstore import build_object_store_from_config + + default_config = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'config/galaxy.ini') ) parser = OptionParser() @@ -13,6 +26,7 @@ parser.add_option( '-e', '--email', dest='email', help='Email address of user to parser.add_option( '--dry-run', dest='dryrun', help='Dry run (show changes but do not save to database)', action='store_true', default=False ) ( options, args ) = parser.parse_args() + def init(): options.config = os.path.abspath( options.config ) @@ -21,19 +35,8 @@ def init(): if options.email == 'all': options.email = None - sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) ) - - from galaxy import eggs - import pkg_resources - - import galaxy.config - from galaxy.objectstore import build_object_store_from_config - - # lazy - globals()['nice_size'] = __import__( 'galaxy.util', globals(), locals(), ( 'nice_size', ) ).nice_size - - config_parser = ConfigParser( dict( here = os.getcwd(), - database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) ) + config_parser = ConfigParser( dict( here=os.getcwd(), + database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) ) config_parser.read( options.config ) config_dict = {} @@ -45,31 +48,10 @@ def init(): from galaxy.model import mapping - return mapping.init( config.file_path, config.database_connection, create_tables = False, object_store = object_store ), object_store, config.database_connection.split(':')[0] + return (mapping.init( config.file_path, config.database_connection, create_tables=False, object_store=object_store ), + object_store, + config.database_connection.split(':')[0]) -def pgcalc( sa_session, id ): - sql = """ - UPDATE galaxy_user - SET disk_usage = (SELECT COALESCE(SUM(total_size), 0) - FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id - FROM history_dataset_association hda - JOIN history h ON h.id = hda.history_id - JOIN dataset d ON hda.dataset_id = d.id - WHERE h.user_id = :id - AND h.purged = false - AND hda.purged = false - AND d.purged = false - AND d.id NOT IN (SELECT dataset_id - FROM library_dataset_dataset_association) - ) sizes) - WHERE id = :id - RETURNING disk_usage; - """ - r = sa_session.execute(sql, {'id':id}) - new = r.fetchone()[0] - if options.dryrun: - sa_session.rollback() - return new def quotacheck( sa_session, users, engine ): sa_session.refresh( user ) @@ -83,7 +65,7 @@ def quotacheck( sa_session, users, engine ): print 'usage changed while calculating, trying again...' return quotacheck( sa_session, user, engine ) else: - new = pgcalc( sa_session, user.id ) + new = pgcalc( sa_session, user.id, dryrun=options.dryrun ) # yes, still a small race condition between here and the flush print 'old usage:', nice_size( current ), 'change:', if new in ( current, None ): diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 0294fb2f5a3..f812b6c1100 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -27,6 +27,7 @@ + diff --git a/test/functional/tools/tol_level_data.xml b/test/functional/tools/tol_level_data.xml new file mode 100644 index 00000000000..37d2ba7e00e --- /dev/null +++ b/test/functional/tools/tol_level_data.xml @@ -0,0 +1,34 @@ + + + cat '${f1}' >> $out1; + echo '${library.f2}' >> $out2; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +