opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'dev' of https://github.com/galaxyproject/galaxy into dev

Browse Source
This commit is contained in:
guerler
2015-02-26 01:09:21 -05:00
parent a7bffef846 cf8fd5c597
commit 85b0b66004
11 changed files with 280 additions and 183 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/datatypes/tabular.py
+1 -1
View File
@@ -480,7 +480,7 @@ class Sam( Tabular ):
if result != 0:
raise Exception('Result %s from %s' % (result, cmd))
if len(split_files) > 1:
cmd = 'egrep -v "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file )
cmd = 'egrep -v -h "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file )
result = os.system(cmd)
if result != 0:
raise Exception('Result %s from %s' % (result, cmd))
lib/galaxy/eggs/scramble.py
+1 -1
View File
@@ -134,7 +134,7 @@ class ScrambleEgg( Egg ):
elif zipfile.is_zipfile( self.source_path ):
self.unpack_zip()
else:
raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, source_path ) )
raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, self.source_path ) )
log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name )
log.warning( " %s" % self.buildpath )
def unpack_zip( self ):
lib/galaxy/managers/folders.py
+38 -4
View File
@@ -3,6 +3,8 @@ Manager and Serializer for Library Folders.
"""
import galaxy.exceptions
from sqlalchemy.orm.exc import MultipleResultsFound
from sqlalchemy.orm.exc import NoResultFound
import logging
log = logging.getLogger( __name__ )
@@ -19,19 +21,21 @@ class FolderManager( object ):
:param decoded_folder_id: decoded folder id
:type decoded_folder_id: int
:param check_manageable: flag whether the check that user can manage item
:type check_manageable: bool
:param check_manageable: flag whether the check that user can manage item
:type check_manageable: bool
:param check_accessible: flag whether to check that user can access item
:type check_accessible: bool
:returns: the requested folder
:rtype: LibraryFolder
:raises: InconsistentDatabase, RequestParameterInvalidException, InternalServerError
"""
try:
folder = trans.sa_session.query( trans.app.model.LibraryFolder ).filter( trans.app.model.LibraryFolder.table.c.id == decoded_folder_id ).one()
except galaxy.exceptions.MultipleResultsFound:
except MultipleResultsFound:
raise galaxy.exceptions.InconsistentDatabase( 'Multiple folders found with the same id.' )
except galaxy.exceptions.NoResultFound:
except NoResultFound:
raise galaxy.exceptions.RequestParameterInvalidException( 'No folder found with the id provided.' )
except Exception, e:
raise galaxy.exceptions.InternalServerError( 'Error loading from the database.' + str( e ) )
@@ -139,6 +143,14 @@ class FolderManager( object ):
"""
Mark given folder deleted/undeleted based on the flag.
:param folder: the model object
:type folder: LibraryFolder
:param undelete: flag whether to delete (when False) or undelete
:type undelete: Bool
:returns: the folder
:rtype: LibraryFolder
:raises: ItemAccessibilityException
"""
if not trans.user_is_admin():
@@ -190,6 +202,14 @@ class FolderManager( object ):
def cut_the_prefix( self, encoded_folder_id ):
"""
Remove the prefix from the encoded folder id.
:param encoded_folder_id: encoded id of the Folder object with 'F' prepended
:type encoded_folder_id: string
:returns: encoded Folder id without the 'F' prefix
:rtype: string
:raises: MalformedId
"""
if ( ( len( encoded_folder_id ) % 16 == 1 ) and encoded_folder_id.startswith( 'F' ) ):
cut_id = encoded_folder_id[ 1: ]
@@ -200,6 +220,14 @@ class FolderManager( object ):
def decode_folder_id( self, trans, encoded_folder_id ):
"""
Decode the folder id given that it has already lost the prefixed 'F'.
:param encoded_folder_id: encoded id of the Folder object
:type encoded_folder_id: string
:returns: decoded Folder id
:rtype: int
:raises: MalformedId
"""
try:
decoded_id = trans.security.decode_id( encoded_folder_id )
@@ -210,5 +238,11 @@ class FolderManager( object ):
def cut_and_decode( self, trans, encoded_folder_id ):
"""
Cuts the folder prefix (the prepended 'F') and returns the decoded id.
:param encoded_folder_id: encoded id of the Folder object
:type encoded_folder_id: string
:returns: decoded Folder id
:rtype: int
"""
return self.decode_folder_id( trans, self.cut_the_prefix( encoded_folder_id ) )
lib/galaxy/model/util.py
+34
View File
@@ -0,0 +1,34 @@
"""
Utility helpers related to the model
"""
def pgcalc( sa_session, id, dryrun=False ):
"""
Utility method for quickly recalculating user disk usage in postgres.
TODO: Check against the recently updated versions of sqlalchemy if this
'special' postgresql version is even necessary.
"""
sql = """
UPDATE galaxy_user
SET disk_usage = (SELECT COALESCE(SUM(total_size), 0)
FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id
FROM history_dataset_association hda
JOIN history h ON h.id = hda.history_id
JOIN dataset d ON hda.dataset_id = d.id
WHERE h.user_id = :id
AND h.purged = false
AND hda.purged = false
AND d.purged = false
AND d.id NOT IN (SELECT dataset_id
FROM library_dataset_dataset_association)
) sizes)
WHERE id = :id
RETURNING disk_usage;
"""
r = sa_session.execute(sql, {'id': id})
new = r.fetchone()[0]
if dryrun:
sa_session.rollback()
return new
lib/galaxy/web/framework/base.py
+47 -23
View File
@@ -2,7 +2,7 @@
A simple WSGI application/framework.
"""
import cgi # For FieldStorage
import cgi # For FieldStorage
import logging
import os.path
import socket
@@ -67,7 +67,7 @@ class WebApplication( object ):
# FIXME: The following two options are deprecated and should be
# removed. Consult the Routes documentation.
self.mapper.minimization = True
#self.mapper.explicit = False
# self.mapper.explicit = False
self.transaction_factory = DefaultWebTransaction
# Set if trace logging is enabled
self.trace_logger = None
@@ -79,12 +79,12 @@ class WebApplication( object ):
method use `add_route`.
"""
log.debug( "Enabling '%s' controller, class: %s",
controller_name, controller.__class__.__name__ )
controller_name, controller.__class__.__name__ )
self.controllers[ controller_name ] = controller
def add_api_controller( self, controller_name, controller ):
log.debug( "Enabling '%s' API controller, class: %s",
controller_name, controller.__class__.__name__ )
controller_name, controller.__class__.__name__ )
self.api_controllers[ controller_name ] = controller
def add_route( self, route, **kwargs ):
@@ -148,7 +148,7 @@ class WebApplication( object ):
else:
environ[ 'is_api_request' ] = False
controllers = self.controllers
if map == None:
if map is None:
raise httpexceptions.HTTPNotFound( "No route for " + path_info )
self.trace( path_info=path_info, map=map )
# Setup routes
@@ -239,11 +239,13 @@ class WSGIEnvironmentProperty( object ):
associated object (provides property style access to keys in the WSGI
environment)
"""
def __init__( self, key, default = '' ):
def __init__( self, key, default='' ):
self.key = key
self.default = default
def __get__( self, obj, type = None ):
if obj is None: return self
def __get__( self, obj, type=None ):
if obj is None:
return self
return obj.environ.get( self.key, self.default )
@@ -254,8 +256,10 @@ class LazyProperty( object ):
"""
def __init__( self, func ):
self.func = func
def __get__(self, obj, type = None ):
if obj is None: return self
def __get__(self, obj, type=None ):
if obj is None:
return self
value = self.func( obj )
setattr( obj, self.func.func_name, value )
return value
@@ -273,6 +277,7 @@ class DefaultWebTransaction( object ):
self.environ = environ
self.request = Request( environ )
self.response = Response()
@lazy_property
def session( self ):
"""
@@ -291,11 +296,14 @@ class DefaultWebTransaction( object ):
# but for performance reasons it's way better to use Paste's tempfile than to
# create a new one and copy.
import tempfile
class FieldStorage( cgi.FieldStorage ):
def make_file(self, binary=None):
return tempfile.NamedTemporaryFile()
def read_lines(self):
# Always make a new file
# Always make a new file
self.file = self.make_file()
self.__file = None
if self.outerboundary:
@@ -313,45 +321,58 @@ class Request( webob.Request ):
"""
Create a new request wrapping the WSGI environment `environ`
"""
## self.environ = environ
# self.environ = environ
webob.Request.__init__( self, environ, charset='utf-8', decode_param_names=False )
# Properties that are computed and cached on first use
@lazy_property
def remote_host( self ):
try:
return socket.gethostbyname( self.remote_addr )
except socket.error:
return self.remote_addr
@lazy_property
def remote_hostname( self ):
try:
return socket.gethostbyaddr( self.remote_addr )[0]
except socket.error:
return self.remote_addr
@lazy_property
def cookies( self ):
return get_cookies( self.environ )
@lazy_property
def base( self ):
return ( self.scheme + "://" + self.host )
## @lazy_property
## def params( self ):
## return parse_formvars( self.environ )
# @lazy_property
# def params( self ):
# return parse_formvars( self.environ )
@lazy_property
def path( self ):
return self.environ['SCRIPT_NAME'] + self.environ['PATH_INFO']
@lazy_property
def browser_url( self ):
return self.base + self.path
# Descriptors that map properties to the associated environment
## scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' )
## remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' )
# scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' )
# remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' )
remote_port = WSGIEnvironmentProperty( 'REMOTE_PORT' )
## method = WSGIEnvironmentProperty( 'REQUEST_METHOD' )
## script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' )
# method = WSGIEnvironmentProperty( 'REQUEST_METHOD' )
# script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' )
protocol = WSGIEnvironmentProperty( 'SERVER_PROTOCOL' )
## query_string = WSGIEnvironmentProperty( 'QUERY_STRING' )
## path_info = WSGIEnvironmentProperty( 'PATH_INFO' )
# query_string = WSGIEnvironmentProperty( 'QUERY_STRING' )
# path_info = WSGIEnvironmentProperty( 'PATH_INFO' )
class Response( object ):
@@ -406,7 +427,8 @@ class Response( object ):
# ---- Utilities ------------------------------------------------------------
CHUNK_SIZE = 2**16
CHUNK_SIZE = 2 ** 16
def send_file( start_response, trans, body ):
# If configured use X-Accel-Redirect header for nginx
@@ -426,6 +448,7 @@ def send_file( start_response, trans, body ):
trans.response.wsgi_headeritems() )
return body
def iterate_file( file ):
"""
Progressively return chunks from `file`.
@@ -436,13 +459,14 @@ def iterate_file( file ):
break
yield chunk
def flatten( seq ):
"""
Flatten a possible nested set of iterables
"""
for x in seq:
if isinstance( x, ( types.GeneratorType, list, tuple ) ):
for y in flatten( x, encoding ):
for y in flatten( x ):
yield y
else:
yield x
lib/galaxy/webapps/galaxy/controllers/admin.py
+66 -75
View File
@@ -1,15 +1,17 @@
import imp
import logging
import os
from sqlalchemy.sql import expression
import galaxy.queue_worker
import galaxy.util
from galaxy import model
from galaxy.model import tool_shed_install as install_model
from galaxy import web
from galaxy.actions.admin import AdminActions
from galaxy.exceptions import MessageException
from galaxy.util import sanitize_text
from galaxy.model import tool_shed_install as install_model
from galaxy.model.util import pgcalc
from galaxy.util import nice_size, sanitize_text
from galaxy.util.odict import odict
from galaxy.web import url_for
from galaxy.web.base.controller import BaseUIController, UsesQuotaMixin
@@ -25,23 +27,17 @@ log = logging.getLogger( __name__ )
class UserListGrid( grids.Grid ):
class EmailColumn( grids.TextColumn ):
def get_value( self, trans, grid, user ):
return user.email
class UserNameColumn( grids.TextColumn ):
def get_value( self, trans, grid, user ):
if user.username:
return user.username
return 'not set'
class StatusColumn( grids.GridColumn ):
def get_value( self, trans, grid, user ):
if user.purged:
return "purged"
@@ -49,47 +45,35 @@ class UserListGrid( grids.Grid ):
return "deleted"
return ""
class GroupsColumn( grids.GridColumn ):
def get_value( self, trans, grid, user ):
if user.groups:
return len( user.groups )
return 0
class RolesColumn( grids.GridColumn ):
def get_value( self, trans, grid, user ):
if user.roles:
return len( user.roles )
return 0
class ExternalColumn( grids.GridColumn ):
def get_value( self, trans, grid, user ):
if user.external:
return 'yes'
return 'no'
class LastLoginColumn( grids.GridColumn ):
def get_value( self, trans, grid, user ):
if user.galaxy_sessions:
return self.format( user.galaxy_sessions[ 0 ].update_time )
return 'never'
class TimeCreatedColumn( grids.GridColumn ):
def get_value( self, trans, grid, user ):
return user.create_time.strftime('%x')
class ActivatedColumn( grids.GridColumn ):
def get_value( self, trans, grid, user ):
if user.active:
return 'Y'
@@ -99,7 +83,7 @@ class UserListGrid( grids.Grid ):
# Grid definition
title = "Users"
model_class = model.User
template='/admin/user/grid.mako'
template = '/admin/user/grid.mako'
default_sort_key = "email"
columns = [
EmailColumn( "Email",
@@ -140,7 +124,11 @@ class UserListGrid( grids.Grid ):
condition=( lambda item: not item.deleted ),
allow_multiple=True,
allow_popup=False,
url_args=dict( webapp="galaxy", action="reset_user_password" ) )
url_args=dict( webapp="galaxy", action="reset_user_password" ) ),
grids.GridOperation( "Recalculate Disk Usage",
condition=( lambda item: not item.deleted ),
allow_multiple=False,
url_args=dict( webapp="galaxy", action="recalculate_user_disk_usage" ) )
]
standard_filters = [
grids.GridColumnFilter( "Active", args=dict( deleted=False ) ),
@@ -158,46 +146,33 @@ class UserListGrid( grids.Grid ):
class RoleListGrid( grids.Grid ):
class NameColumn( grids.TextColumn ):
def get_value( self, trans, grid, role ):
return role.name
class DescriptionColumn( grids.TextColumn ):
def get_value( self, trans, grid, role ):
if role.description:
return role.description
return ''
class TypeColumn( grids.TextColumn ):
def get_value( self, trans, grid, role ):
return role.type
class StatusColumn( grids.GridColumn ):
def get_value( self, trans, grid, role ):
if role.deleted:
return "deleted"
return ""
class GroupsColumn( grids.GridColumn ):
def get_value( self, trans, grid, role ):
if role.groups:
return len( role.groups )
return 0
class UsersColumn( grids.GridColumn ):
def get_value( self, trans, grid, role ):
if role.users:
return len( role.users )
@@ -206,7 +181,7 @@ class RoleListGrid( grids.Grid ):
# Grid definition
title = "Roles"
model_class = model.Role
template='/admin/dataset_security/role/grid.mako'
template = '/admin/dataset_security/role/grid.mako'
default_sort_key = "name"
columns = [
NameColumn( "Name",
@@ -270,31 +245,23 @@ class RoleListGrid( grids.Grid ):
class GroupListGrid( grids.Grid ):
class NameColumn( grids.TextColumn ):
def get_value( self, trans, grid, group ):
return group.name
class StatusColumn( grids.GridColumn ):
def get_value( self, trans, grid, group ):
if group.deleted:
return "deleted"
return ""
class RolesColumn( grids.GridColumn ):
def get_value( self, trans, grid, group ):
if group.roles:
return len( group.roles )
return 0
class UsersColumn( grids.GridColumn ):
def get_value( self, trans, grid, group ):
if group.members:
return len( group.members )
@@ -303,7 +270,7 @@ class GroupListGrid( grids.Grid ):
# Grid definition
title = "Groups"
model_class = model.Group
template='/admin/dataset_security/group/grid.mako'
template = '/admin/dataset_security/group/grid.mako'
default_sort_key = "name"
columns = [
NameColumn( "Name",
@@ -351,31 +318,24 @@ class GroupListGrid( grids.Grid ):
preserve_state = False
use_paging = True
class QuotaListGrid( grids.Grid ):
class NameColumn( grids.TextColumn ):
def get_value( self, trans, grid, quota ):
return quota.name
class DescriptionColumn( grids.TextColumn ):
def get_value( self, trans, grid, quota ):
if quota.description:
return quota.description
return ''
class AmountColumn( grids.TextColumn ):
def get_value( self, trans, grid, quota ):
return quota.operation + quota.display_amount
class StatusColumn( grids.GridColumn ):
def get_value( self, trans, grid, quota ):
if quota.deleted:
return "deleted"
@@ -383,17 +343,13 @@ class QuotaListGrid( grids.Grid ):
return "<strong>default for %s users</strong>" % quota.default[0].type
return ""
class UsersColumn( grids.GridColumn ):
def get_value( self, trans, grid, quota ):
if quota.users:
return len( quota.users )
return 0
class GroupsColumn( grids.GridColumn ):
def get_value( self, trans, grid, quota ):
if quota.groups:
return len( quota.groups )
@@ -402,7 +358,7 @@ class QuotaListGrid( grids.Grid ):
# Grid definition
title = "Quotas"
model_class = model.Quota
template='/admin/quota/grid.mako'
template = '/admin/quota/grid.mako'
default_sort_key = "name"
columns = [
NameColumn( "Name",
@@ -417,10 +373,10 @@ class QuotaListGrid( grids.Grid ):
attach_popup=False,
filterable="advanced" ),
AmountColumn( "Amount",
key='amount',
model_class=model.Quota,
attach_popup=False,
filterable="advanced" ),
key='amount',
model_class=model.Quota,
attach_popup=False,
filterable="advanced" ),
UsersColumn( "Users", attach_popup=False ),
GroupsColumn( "Groups", attach_popup=False ),
StatusColumn( "Status", attach_popup=False ),
@@ -483,9 +439,7 @@ class QuotaListGrid( grids.Grid ):
class ToolVersionListGrid( grids.Grid ):
class ToolIdColumn( grids.TextColumn ):
def get_value( self, trans, grid, tool_version ):
toolbox = trans.app.toolbox
if toolbox.has_tool( tool_version.tool_id, exact=True ):
@@ -494,9 +448,7 @@ class ToolVersionListGrid( grids.Grid ):
return '<div class="count-box state-color-ok">%s%s</a></div>' % ( link_str, tool_version.tool_id )
return tool_version.tool_id
class ToolVersionsColumn( grids.TextColumn ):
def get_value( self, trans, grid, tool_version ):
tool_ids_str = ''
toolbox = trans.app.toolbox
@@ -508,10 +460,11 @@ class ToolVersionListGrid( grids.Grid ):
else:
tool_ids_str += '%s<br/>' % tool_id
return tool_ids_str
# Grid definition
title = "Tool versions"
model_class = install_model.ToolVersion
template='/admin/tool_version/grid.mako'
template = '/admin/tool_version/grid.mako'
default_sort_key = "tool_id"
columns = [
ToolIdColumn( "Tool id",
@@ -531,7 +484,7 @@ class ToolVersionListGrid( grids.Grid ):
num_rows_per_page = 50
preserve_state = False
use_paging = True
def build_initial_query( self, trans, **kwd ):
return trans.install_model.context.query( self.model_class )
@@ -593,14 +546,14 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
new_in_users = []
new_in_groups = []
for user in trans.sa_session.query( trans.app.model.User ) \
.filter( trans.app.model.User.table.c.deleted==False ) \
.filter( trans.app.model.User.table.c.deleted == expression.false() ) \
.order_by( trans.app.model.User.table.c.email ):
if user.id in in_users:
new_in_users.append( ( user.id, user.email ) )
else:
params.out_users.append( ( user.id, user.email ) )
for group in trans.sa_session.query( trans.app.model.Group ) \
.filter( trans.app.model.Group.table.c.deleted==False ) \
.filter( trans.app.model.Group.table.c.deleted == expression.false() ) \
.order_by( trans.app.model.Group.table.c.name ):
if group.id in in_groups:
new_in_groups.append( ( group.id, group.name ) )
@@ -645,14 +598,14 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
in_groups = []
out_groups = []
for user in trans.sa_session.query( trans.app.model.User ) \
.filter( trans.app.model.User.table.c.deleted==False ) \
.filter( trans.app.model.User.table.c.deleted == expression.false() ) \
.order_by( trans.app.model.User.table.c.email ):
if user in [ x.user for x in quota.users ]:
in_users.append( ( user.id, user.email ) )
else:
out_users.append( ( user.id, user.email ) )
for group in trans.sa_session.query( trans.app.model.Group ) \
.filter( trans.app.model.Group.table.c.deleted==False ) \
.filter( trans.app.model.Group.table.c.deleted == expression.false()) \
.order_by( trans.app.model.Group.table.c.name ):
if group in [ x.group for x in quota.groups ]:
in_groups.append( ( group.id, group.name ) )
@@ -775,7 +728,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
webapp=params.webapp,
message=sanitize_text( str( e ) ),
status='error' ) )
if do_op == True or ( do_op != False and params.get( do_op, False ) ):
if do_op is True or ( do_op is not False and params.get( do_op, False ) ):
try:
message = op_method( quota, params )
return None, trans.response.send_redirect( web.url_for( controller='admin',
@@ -899,10 +852,48 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP
kwargs={'display_application_ids': kwd.get( 'id' )} )
reloaded, failed = trans.app.datatypes_registry.reload_display_applications( kwd.get( 'id' ) )
if not reloaded and failed:
return trans.show_error_message( 'Unable to reload any of the %i requested display applications ("%s").' % ( len( failed ), '", "'.join( failed ) ) )
return trans.show_error_message( 'Unable to reload any of the %i requested display applications ("%s").'
% ( len( failed ), '", "'.join( failed ) ) )
if failed:
return trans.show_warn_message( 'Reloaded %i display applications ("%s"), but failed to reload %i display applications ("%s").'
% ( len( reloaded ), '", "'.join( reloaded ), len( failed ), '", "'.join( failed ) ) )
% ( len( reloaded ), '", "'.join( reloaded ), len( failed ), '", "'.join( failed ) ) )
if not reloaded:
return trans.show_warn_message( 'You need to request at least one display application to reload.' )
return trans.show_ok_message( 'Reloaded %i requested display applications ("%s").' % ( len( reloaded ), '", "'.join( reloaded ) ) )
@web.expose
@web.require_admin
def recalculate_user_disk_usage( self, trans, **kwd ):
user_id = kwd.get( 'id', None )
user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) )
if not user:
return trans.show_error_message( "User not found for id (%s)" % sanitize_text( str( user_id ) ) )
engine = None
if trans.app.config.database_connection:
engine = trans.app.config.database_connection.split(':')[0]
if engine not in ( 'postgres', 'postgresql' ):
done = False
while not done:
current = user.get_disk_usage()
new = user.calculate_disk_usage()
trans.sa_session.refresh( user )
# make sure usage didn't change while calculating, set done
if user.get_disk_usage() == current:
done = True
if new not in (current, None):
user.set_disk_usage( new )
trans.sa_session.add( user )
trans.sa_session.flush()
else:
# We can use the lightning fast pgcalc!
current = user.get_disk_usage()
new = pgcalc( self.sa_session, user.id )
# yes, still a small race condition between here and the flush
if new in ( current, None ):
message = 'Usage is unchanged at %s.' % nice_size( current )
else:
message = 'Usage has changed by %s to %s.' % ( nice_size( new - current ), nice_size( current ) )
return trans.response.send_redirect( web.url_for( controller='admin',
action='users',
message=sanitize_text( message ),
status='info' ) )
lib/galaxy/webapps/tool_shed/config.py
+4
View File
@@ -136,6 +136,10 @@ class Configuration( object ):
self.citation_cache_data_dir = resolve_path( kwargs.get( "citation_cache_data_dir", "database/tool_shed_citations/data" ), self.root )
self.citation_cache_lock_dir = resolve_path( kwargs.get( "citation_cache_lock_dir", "database/tool_shed_citations/locks" ), self.root )
@property
def shed_tool_data_path( self ):
return self.tool_data_path
@property
def sentry_dsn_public( self ):
"""
lib/tool_shed/galaxy_install/installed_repository_manager.py
+33 -40
View File
@@ -103,12 +103,12 @@ class InstalledRepositoryManager( object ):
# Hack to add repository.name here, which is actually the root of the installed repository
data_manager_relative_install_dir = os.path.join( data_manager_relative_install_dir, repository.name )
dmh = data_manager.DataManagerHandler( self.app )
new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file,
repository.metadata,
repository.get_shed_config_dict( self.app ),
data_manager_relative_install_dir,
repository,
repository_tools_tups )
dmh.install_data_managers( self.app.config.shed_data_manager_config_file,
repository.metadata,
repository.get_shed_config_dict( self.app ),
data_manager_relative_install_dir,
repository,
repository_tools_tups )
self.install_model.context.add( repository )
self.install_model.context.flush()
if repository.includes_datatypes:
@@ -400,14 +400,8 @@ class InstalledRepositoryManager( object ):
installed_repository_dependencies = {}
missing_rd_tups = []
installed_rd_tups = []
description, \
repository_clone_url, \
changeset_revision, \
ctx_rev, \
repository_owner, \
repository_dependencies, \
tool_dependencies = \
suc.get_repo_info_tuple_contents( repo_info_tuple )
( description, repository_clone_url, changeset_revision, ctx_rev,
repository_owner, repository_dependencies, tool_dependencies ) = suc.get_repo_info_tuple_contents( repo_info_tuple )
if repository_dependencies:
description = repository_dependencies[ 'description' ]
root_key = repository_dependencies[ 'root_key' ]
@@ -460,7 +454,7 @@ class InstalledRepositoryManager( object ):
prior_installation_required,
only_if_compiling_contained_td,
None,
'Never installed' ]
'Never installed' ]
if not util.asbool( only_if_compiling_contained_td ):
# A repository dependency that is not installed will not be considered missing if its value for
# only_if_compiling_contained_td is True - see above...
@@ -564,10 +558,10 @@ class InstalledRepositoryManager( object ):
continue
if required_repository_tup not in dependency_tups:
dependency_tups.append( required_repository_tup )
return get_repository_dependency_tups_for_installed_repository( required_repository,
dependency_tups=dependency_tups )
return self.get_repository_dependency_tups_for_installed_repository( required_repository,
dependency_tups=dependency_tups )
return dependency_tups
def get_repository_tuple_for_installed_repository_manager( self, repository ):
return ( str( repository.tool_shed ),
str( repository.name ),
@@ -611,10 +605,10 @@ class InstalledRepositoryManager( object ):
contents = open( env_shell_file_path, 'r' ).read()
except Exception, e:
contents = None
log.debug( 'Error reading file %s, so cannot determine if package %s requires package %s at run time: %s' % \
( str( env_shell_file_path ), str( td.name ), str( tool_dependency.name ), str( e ) ) )
log.debug( 'Error reading file %s, so cannot determine if package %s requires package %s at run time: %s' %
( str( env_shell_file_path ), str( td.name ), str( tool_dependency.name ), str( e ) ) )
if contents is not None and contents.find( required_env_shell_file_path ) >= 0:
td_tuple = get_tool_dependency_tuple_for_installed_repository_manager( td )
td_tuple = self.get_tool_dependency_tuple_for_installed_repository_manager( td )
runtime_dependent_tool_dependency_tups.append( td_tuple )
return runtime_dependent_tool_dependency_tups
@@ -672,8 +666,7 @@ class InstalledRepositoryManager( object ):
# Remove this tool_dependency from all values in
# self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies
altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = {}
for td_tup, installed_runtime_dependent_tool_dependency_tups in \
self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.items():
for ( td_tup, installed_runtime_dependent_tool_dependency_tups ) in self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.items():
if tool_dependency_tup in installed_runtime_dependent_tool_dependency_tups:
# Remove the tool_dependency from the list.
installed_runtime_dependent_tool_dependency_tups.remove( tool_dependency_tup )
@@ -684,7 +677,7 @@ class InstalledRepositoryManager( object ):
altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies
# Remove the entry for this tool_dependency from self.runtime_tool_dependencies_of_installed_tool_dependencies.
self.remove_entry_from_runtime_tool_dependencies_of_installed_tool_dependencies( tool_dependency )
# Remove the entry for this tool_dependency from
# Remove the entry for this tool_dependency from
# self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.
self.remove_entry_from_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies( tool_dependency )
# Remove this repository's entry from self.installed_tool_dependencies_of_installed_repositories.
@@ -864,7 +857,7 @@ class InstalledRepositoryManager( object ):
else:
status = 'error'
message = 'A repository must have the status <b>New</b> in order to be purged. This repository has '
message += ' the status %s.' % str( repository.status )
message += ' the status %s.' % str( repository.status )
return status, message
def remove_entry_from_installed_repository_dependencies_of_installed_repositories( self, repository ):
@@ -884,7 +877,7 @@ class InstalledRepositoryManager( object ):
debug_msg += "from the dependent list for revision %s of repository %s owned by %s " % \
( r_installed_changeset_revision, r_name, r_owner )
debug_msg += "in installed_repository_dependencies_of_installed_repositories."
log.debug( debug_msg )
log.debug( debug_msg )
v_tups.remove( repository_tup )
altered_installed_dependent_repositories_of_installed_repositories[ r_tup ] = v_tups
self.installed_dependent_repositories_of_installed_repositories = \
@@ -957,7 +950,7 @@ class InstalledRepositoryManager( object ):
if cleaned_repository_dependency_tool_shed == cleaned_tool_shed and \
repository_dependency.name == name and \
repository_dependency.owner == owner and \
( repository_dependency.installed_changeset_revision == changeset_revision or \
( repository_dependency.installed_changeset_revision == changeset_revision or
repository_dependency.changeset_revision == changeset_revision ):
return True
return False
@@ -970,9 +963,9 @@ class InstalledRepositoryManager( object ):
# This method is called only from Galaxy when rendering repository dependencies
# for an installed tool shed repository.
# TODO: Do we need to check more than changeset_revision here?
required_repository_tup = [ required_repository.tool_shed, \
required_repository.name, \
required_repository.owner, \
required_repository_tup = [ required_repository.tool_shed,
required_repository.name,
required_repository.owner,
required_repository.changeset_revision ]
for tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td:
partial_tup = tup[ 0:4 ]
@@ -1009,12 +1002,12 @@ class InstalledRepositoryManager( object ):
pulled by a Galaxy administrator when getting updates to an installed tool shed
repository. The original_dependency_dict is a single tool dependency definition,
an example of which is::
{"name": "bwa",
"readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ",
"type": "package",
"version": "0.6.2"}
The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method.
"""
new_tool_dependency = None
@@ -1046,12 +1039,12 @@ class InstalledRepositoryManager( object ):
break
if new_dependency_name and new_dependency_type and new_dependency_version:
# Update all attributes of the tool_dependency record in the database.
log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \
( str( tool_dependency.version ),
str( tool_dependency.type ),
str( tool_dependency.name ),
str( new_dependency_version ),
str( new_dependency_type ) ) )
log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s."
% ( str( tool_dependency.version ),
str( tool_dependency.type ),
str( tool_dependency.name ),
str( new_dependency_version ),
str( new_dependency_type ) ) )
tool_dependency.type = new_dependency_type
tool_dependency.version = new_dependency_version
tool_dependency.status = self.app.install_model.ToolDependency.installation_status.UNINSTALLED
@@ -1062,8 +1055,8 @@ class InstalledRepositoryManager( object ):
else:
# We have no new tool dependency definition based on a matching dependency name, so remove
# the existing tool dependency record from the database.
log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \
( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) )
log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined."
% ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) )
context.delete( tool_dependency )
context.flush()
return new_tool_dependency
scripts/set_user_disk_usage.py
+21 -39
View File
@@ -1,9 +1,22 @@
#!/usr/bin/env python
import os, sys
import os
import sys
from ConfigParser import ConfigParser
from optparse import OptionParser
sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) )
from galaxy import eggs
import pkg_resources
import galaxy.config
from galaxy.model.util import pgcalc
from galaxy.util import nice_size
from galaxy.objectstore import build_object_store_from_config
default_config = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'config/galaxy.ini') )
parser = OptionParser()
@@ -13,6 +26,7 @@ parser.add_option( '-e', '--email', dest='email', help='Email address of user to
parser.add_option( '--dry-run', dest='dryrun', help='Dry run (show changes but do not save to database)', action='store_true', default=False )
( options, args ) = parser.parse_args()
def init():
options.config = os.path.abspath( options.config )
@@ -21,19 +35,8 @@ def init():
if options.email == 'all':
options.email = None
sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) )
from galaxy import eggs
import pkg_resources
import galaxy.config
from galaxy.objectstore import build_object_store_from_config
# lazy
globals()['nice_size'] = __import__( 'galaxy.util', globals(), locals(), ( 'nice_size', ) ).nice_size
config_parser = ConfigParser( dict( here = os.getcwd(),
database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) )
config_parser = ConfigParser( dict( here=os.getcwd(),
database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) )
config_parser.read( options.config )
config_dict = {}
@@ -45,31 +48,10 @@ def init():
from galaxy.model import mapping
return mapping.init( config.file_path, config.database_connection, create_tables = False, object_store = object_store ), object_store, config.database_connection.split(':')[0]
return (mapping.init( config.file_path, config.database_connection, create_tables=False, object_store=object_store ),
object_store,
config.database_connection.split(':')[0])
def pgcalc( sa_session, id ):
sql = """
UPDATE galaxy_user
SET disk_usage = (SELECT COALESCE(SUM(total_size), 0)
FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id
FROM history_dataset_association hda
JOIN history h ON h.id = hda.history_id
JOIN dataset d ON hda.dataset_id = d.id
WHERE h.user_id = :id
AND h.purged = false
AND hda.purged = false
AND d.purged = false
AND d.id NOT IN (SELECT dataset_id
FROM library_dataset_dataset_association)
) sizes)
WHERE id = :id
RETURNING disk_usage;
"""
r = sa_session.execute(sql, {'id':id})
new = r.fetchone()[0]
if options.dryrun:
sa_session.rollback()
return new
def quotacheck( sa_session, users, engine ):
sa_session.refresh( user )
@@ -83,7 +65,7 @@ def quotacheck( sa_session, users, engine ):
print 'usage changed while calculating, trying again...'
return quotacheck( sa_session, user, engine )
else:
new = pgcalc( sa_session, user.id )
new = pgcalc( sa_session, user.id, dryrun=options.dryrun )
# yes, still a small race condition between here and the flush
print 'old usage:', nice_size( current ), 'change:',
if new in ( current, None ):
test/functional/tools/samples_tool_conf.xml
+1
View File
@@ -27,6 +27,7 @@
<tool file="column_param.xml" />
<tool file="column_multi_param.xml" />
<tool file="special_params.xml" />
<tool file="top_level_data.xml" />
<tool file="validation_default.xml" />
<tool file="validation_sanitizer.xml" />
<tool file="validation_repeat.xml" />
test/functional/tools/tol_level_data.xml
+34
View File
@@ -0,0 +1,34 @@
<tool id="top_level_data" name="top_level_data" version="0.1.0">
<command>
cat '${f1}' >> $out1;
echo '${library.f2}' >> $out2; <!-- cannot use just f2 here -->
</command>
<inputs>
<conditional name="library">
<param name="type" type="select" label="Parameter Settings">
<option value="no">Use defaults</option>
<option value="yes">Full parameter list</option>
</param>
<when value="yes">
<param name="f1" type="data" format="txt" label="Data 1" />
<param name="f2" type="text" label="Text 1" />
</when>
<when value="no">
<param name="f1" type="data" format="txt" label="Data 1" />
<param name="f2" type="text" label="TExt 1" />
</when>
</conditional>
</inputs>
<outputs>
<data format="txt" name="out1" />
<data format="txt" name="out2" />
</outputs>
<tests>
<test>
<param name="f1" value="simple_line.txt" />
<param name="f2" value="This is a line of text." />
<output name="out1" file="simple_line.txt" />
<output name="out2" file="simple_line.txt" />
</test>
</tests>
</tool>