From 0ca4068dbca2511e214bdcf0c3c184c6b635f1b9 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 18 Feb 2015 11:34:23 -0500 Subject: [PATCH 01/17] Tool diagnostic API endpoints. It is easy to get Galaxy's database and shed tool files out of sync and the results are bad. I don't think this is new but I may have made it worse this cycle by trying to slowly walk back Galaxy's use of the database to track lineages (e.g. https://bitbucket.org/galaxy/galaxy-central/commits/08f8850853d004bf8a456a147436a6694d0a1a11). New API endpoints include: /api/configuration/dynamic_tool_confs /api/configuration/tool_lineages /api//diagnostics --- .../tools/toolbox/lineages/interface.py | 6 +++ lib/galaxy/tools/toolbox/lineages/stock.py | 7 ++++ .../tools/toolbox/lineages/tool_shed.py | 14 ++++++- .../webapps/galaxy/api/configuration.py | 39 +++++++++++++++++++ lib/galaxy/webapps/galaxy/api/tools.py | 30 ++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 13 +++++++ 6 files changed, 108 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/tools/toolbox/lineages/interface.py b/lib/galaxy/tools/toolbox/lineages/interface.py index 46a49ecb87d..6e7fe290185 100644 --- a/lib/galaxy/tools/toolbox/lineages/interface.py +++ b/lib/galaxy/tools/toolbox/lineages/interface.py @@ -40,3 +40,9 @@ class ToolLineageVersion(object): externally). """ return self.version is None + + def to_dict(self): + return dict( + id=self.id, + version=self.version, + ) diff --git a/lib/galaxy/tools/toolbox/lineages/stock.py b/lib/galaxy/tools/toolbox/lineages/stock.py index c2db57bee7f..da0c1a22607 100644 --- a/lib/galaxy/tools/toolbox/lineages/stock.py +++ b/lib/galaxy/tools/toolbox/lineages/stock.py @@ -38,6 +38,13 @@ class StockLineage(ToolLineage): # method for comparing tool versions. return sorted( versions, key=_to_loose_version ) + def to_dict(self): + return dict( + tool_id=self.tool_id, + tool_versions=list(self.tool_versions), + lineage_type='stock', + ) + def _to_loose_version( tool_lineage_version ): version = str( tool_lineage_version.version ) diff --git a/lib/galaxy/tools/toolbox/lineages/tool_shed.py b/lib/galaxy/tools/toolbox/lineages/tool_shed.py index 7e261c367fb..9576ab875ba 100644 --- a/lib/galaxy/tools/toolbox/lineages/tool_shed.py +++ b/lib/galaxy/tools/toolbox/lineages/tool_shed.py @@ -8,9 +8,11 @@ class ToolShedLineage(ToolLineage): """ Representation of tool lineage derived from tool shed repository installations. """ - def __init__(self, app, tool_version): + def __init__(self, app, tool_version, tool_shed_repository=None): self.app = app self.tool_version_id = tool_version.id + # Only used for logging + self._tool_shed_repository = tool_shed_repository @staticmethod def from_tool( app, tool, tool_shed_repository ): @@ -36,6 +38,16 @@ class ToolShedLineage(ToolLineage): def get_versions( self, reverse=False ): return map( ToolLineageVersion.from_guid, self.get_version_ids( reverse=reverse ) ) + def to_dict(self): + tool_shed_repository = self._tool_shed_repository + rval = dict( + tool_version_id=self.tool_version_id, + tool_versions=map(lambda v: v.to_dict(), self.get_versions()), + tool_shed_repository=tool_shed_repository if tool_shed_repository is not None else None, + lineage_type='tool_shed', + ) + return rval + def get_install_tool_version( app, tool_id ): return app.install_model.context.query( diff --git a/lib/galaxy/webapps/galaxy/api/configuration.py b/lib/galaxy/webapps/galaxy/api/configuration.py index e9f722a2a01..0dca89b526d 100644 --- a/lib/galaxy/webapps/galaxy/api/configuration.py +++ b/lib/galaxy/webapps/galaxy/api/configuration.py @@ -4,6 +4,8 @@ and configuration settings. """ from galaxy.web import _future_expose_api_anonymous as expose_api_anonymous +from galaxy.web import _future_expose_api as expose_api +from galaxy.web import require_admin from galaxy.web.base.controller import BaseAPIController from galaxy.managers import base @@ -33,6 +35,13 @@ class ConfigurationController( BaseAPIController ): @expose_api_anonymous def version( self, trans, **kwds ): + """ + GET /api/version + Return a description of the major version of Galaxy (e.g. 15.03). + + :rtype: dict + :returns: dictionary with major version keyed on 'version_major' + """ return {"version_major": self.app.config.version_major } def get_config_dict( self, trans, return_admin=False, view=None, keys=None, default_view='all' ): @@ -52,6 +61,29 @@ class ConfigurationController( BaseAPIController ): view=view, keys=keys, default_view=default_view ) return serialized + @expose_api + @require_admin + def dynamic_tool_confs(self, trans): + confs = self.app.toolbox.dynamic_confs(include_migrated_tool_conf=True) + return map(_tool_conf_to_dict, confs) + + @expose_api + @require_admin + def tool_lineages(self, trans): + rval = [] + for id, tool in self.app.toolbox.tools(): + if hasattr( tool, 'lineage' ): + lineage_dict = tool.lineage.to_dict() + else: + lineage_dict = None + + entry = dict( + id=id, + lineage=lineage_dict + ) + rval.append(entry) + return rval + #TODO: for lack of a manager file for the config. May well be better in config.py? Circ imports? class ConfigSerializer( base.ModelSerializer ): @@ -122,3 +154,10 @@ class AdminConfigSerializer( ConfigSerializer ): 'allow_user_creation' : _defaults_to( False ), 'allow_user_deletion' : _defaults_to( False ), }) + + +def _tool_conf_to_dict(conf): + return dict( + config_filename=conf['config_filename'], + tool_path=conf['tool_path'], + ) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index c8e8eddff2a..f1657928172 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -94,6 +94,36 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): message, status = trans.app.toolbox.reload_tool_by_id( tool_id ) return { status: message } + @_future_expose_api + @web.require_admin + def diagnostics( self, trans, id, **kwd ): + """ + GET /api/tools/{tool_id}/diagnostics + Return diagnostic information to help debug panel + and dependency related problems. + """ + to_dict = lambda x: x.to_dict() + tool = self._get_tool( id, user=trans.user ) + if hasattr( tool, 'lineage' ): + lineage_dict = tool.lineage.to_dict() + else: + lineage_dict = None + tool_shed_dependencies = tool.installed_tool_dependencies + if tool_shed_dependencies: + tool_shed_dependencies_dict = map(to_dict, tool_shed_dependencies) + else: + tool_shed_dependencies_dict = None + tool = self._get_tool( id, user=trans.user ) + return { + "tool_id": tool.id, + "tool_version": tool.version, + "dependency_shell_commands": tool.build_dependency_shell_commands(), + "lineage": lineage_dict, + "requirements": map(to_dict, tool.requirements), + "installed_tool_shed_dependencies": tool_shed_dependencies_dict, + "tool_dir": tool.tool_dir, + } + @_future_expose_api_anonymous def citations( self, trans, id, **kwds ): tool = self._get_tool( id, user=trans.user ) diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 3605de8bfb3..9c3de5c5adf 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -210,6 +210,7 @@ def populate_api_routes( webapp, app ): webapp.mapper.resource_with_deleted( 'quota', 'quotas', path_prefix='/api' ) webapp.mapper.connect( '/api/tools/{id:.+?}/build', action='build', controller="tools" ) webapp.mapper.connect( '/api/tools/{id:.+?}/reload', action='reload', controller="tools" ) + webapp.mapper.connect( '/api/tools/{id:.+?}/diagnostics', action='diagnostics', controller="tools" ) webapp.mapper.connect( '/api/tools/{id:.+?}/citations', action='citations', controller="tools" ) webapp.mapper.connect( '/api/tools/{id:.+?}/download', action='download', controller="tools" ) webapp.mapper.connect( '/api/tools/{id:.+?}', action='show', controller="tools" ) @@ -221,6 +222,18 @@ def populate_api_routes( webapp, app ): webapp.mapper.resource( 'workflow', 'workflows', path_prefix='/api' ) webapp.mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' ) webapp.mapper.connect( '/api/histories/{history_id}/citations', action='citations', controller="histories" ) + webapp.mapper.connect( + 'dynamic_tool_confs', + '/api/configuration/dynamic_tool_confs', + controller="configuration", + action="dynamic_tool_confs" + ) + webapp.mapper.connect( + 'tool_lineages', + '/api/configuration/tool_lineages', + controller="configuration", + action="tool_lineages" + ) webapp.mapper.resource( 'configuration', 'configuration', path_prefix='/api' ) webapp.mapper.connect( "configuration_version", "/api/version", controller="configuration", From 0b2a8b7b0faed6cd7e3da8721dfecd3bec8c6ebd Mon Sep 17 00:00:00 2001 From: John Chilton Date: Mon, 23 Feb 2015 19:52:17 -0500 Subject: [PATCH 02/17] Improved tool diagnostics API. --- lib/galaxy/webapps/galaxy/api/tools.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/api/tools.py b/lib/galaxy/webapps/galaxy/api/tools.py index f1657928172..736986b062a 100644 --- a/lib/galaxy/webapps/galaxy/api/tools.py +++ b/lib/galaxy/webapps/galaxy/api/tools.py @@ -102,6 +102,7 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): Return diagnostic information to help debug panel and dependency related problems. """ + # TODO: Move this into tool. to_dict = lambda x: x.to_dict() tool = self._get_tool( id, user=trans.user ) if hasattr( tool, 'lineage' ): @@ -113,7 +114,6 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): tool_shed_dependencies_dict = map(to_dict, tool_shed_dependencies) else: tool_shed_dependencies_dict = None - tool = self._get_tool( id, user=trans.user ) return { "tool_id": tool.id, "tool_version": tool.version, @@ -122,6 +122,11 @@ class ToolsController( BaseAPIController, UsesVisualizationMixin ): "requirements": map(to_dict, tool.requirements), "installed_tool_shed_dependencies": tool_shed_dependencies_dict, "tool_dir": tool.tool_dir, + "tool_shed": tool.tool_shed, + "repository_name": tool.repository_name, + "repository_owner": tool.repository_owner, + "installed_changeset_revision": None, + "guid": tool.guid, } @_future_expose_api_anonymous From 09a65cfcd4f1a96adcdb99443818764ba21aa07b Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 25 Feb 2015 09:10:17 -0500 Subject: [PATCH 03/17] Expand out meta parameters when validating for tool.to_json(). Bring this code closer inline with what handle_input() does and should solve the problem with column validation of parameters that are being 'batched'. See https://trello.com/c/8NweTvLh and comment on https://github.com/galaxyproject/galaxy/commit/d57088409c7b23e55ead4c9f2fad38b2600eb8b7. --- lib/galaxy/tools/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/tools/__init__.py b/lib/galaxy/tools/__init__.py index cf6d0c67016..16b8d687d55 100755 --- a/lib/galaxy/tools/__init__.py +++ b/lib/galaxy/tools/__init__.py @@ -2299,6 +2299,8 @@ class Tool( object, Dictifiable ): # create parameter object params = galaxy.util.Params( kwd, sanitize = False ) + expanded_incomings, _ = expand_meta_parameters( trans, self, params.__dict__ ) + params.__dict__ = expanded_incomings[ 0 ] # convert value to jsonifiable value def jsonify(v): From 62129c7a28a5f4b36534cd6c094b4adbcfa2f9e0 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 09:29:23 -0500 Subject: [PATCH 04/17] Cleanup and pep8 in galaxy admin controller prior to new dev. --- .../webapps/galaxy/controllers/admin.py | 82 ++++--------------- 1 file changed, 15 insertions(+), 67 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index cc980c7febb..96f3ef8aa04 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -25,23 +25,17 @@ log = logging.getLogger( __name__ ) class UserListGrid( grids.Grid ): - class EmailColumn( grids.TextColumn ): - def get_value( self, trans, grid, user ): return user.email - class UserNameColumn( grids.TextColumn ): - def get_value( self, trans, grid, user ): if user.username: return user.username return 'not set' - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.purged: return "purged" @@ -49,47 +43,35 @@ class UserListGrid( grids.Grid ): return "deleted" return "" - class GroupsColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.groups: return len( user.groups ) return 0 - class RolesColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.roles: return len( user.roles ) return 0 - class ExternalColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.external: return 'yes' return 'no' - class LastLoginColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.galaxy_sessions: return self.format( user.galaxy_sessions[ 0 ].update_time ) return 'never' - class TimeCreatedColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): return user.create_time.strftime('%x') - class ActivatedColumn( grids.GridColumn ): - def get_value( self, trans, grid, user ): if user.active: return 'Y' @@ -99,7 +81,7 @@ class UserListGrid( grids.Grid ): # Grid definition title = "Users" model_class = model.User - template='/admin/user/grid.mako' + template = '/admin/user/grid.mako' default_sort_key = "email" columns = [ EmailColumn( "Email", @@ -158,46 +140,33 @@ class UserListGrid( grids.Grid ): class RoleListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, role ): return role.name - class DescriptionColumn( grids.TextColumn ): - def get_value( self, trans, grid, role ): if role.description: return role.description return '' - class TypeColumn( grids.TextColumn ): - def get_value( self, trans, grid, role ): return role.type - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, role ): if role.deleted: return "deleted" return "" - class GroupsColumn( grids.GridColumn ): - def get_value( self, trans, grid, role ): if role.groups: return len( role.groups ) return 0 - class UsersColumn( grids.GridColumn ): - - def get_value( self, trans, grid, role ): if role.users: return len( role.users ) @@ -206,7 +175,7 @@ class RoleListGrid( grids.Grid ): # Grid definition title = "Roles" model_class = model.Role - template='/admin/dataset_security/role/grid.mako' + template = '/admin/dataset_security/role/grid.mako' default_sort_key = "name" columns = [ NameColumn( "Name", @@ -270,31 +239,23 @@ class RoleListGrid( grids.Grid ): class GroupListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, group ): return group.name - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, group ): if group.deleted: return "deleted" return "" - class RolesColumn( grids.GridColumn ): - def get_value( self, trans, grid, group ): if group.roles: return len( group.roles ) return 0 - class UsersColumn( grids.GridColumn ): - def get_value( self, trans, grid, group ): if group.members: return len( group.members ) @@ -303,7 +264,7 @@ class GroupListGrid( grids.Grid ): # Grid definition title = "Groups" model_class = model.Group - template='/admin/dataset_security/group/grid.mako' + template = '/admin/dataset_security/group/grid.mako' default_sort_key = "name" columns = [ NameColumn( "Name", @@ -351,31 +312,24 @@ class GroupListGrid( grids.Grid ): preserve_state = False use_paging = True + class QuotaListGrid( grids.Grid ): - class NameColumn( grids.TextColumn ): - def get_value( self, trans, grid, quota ): return quota.name - class DescriptionColumn( grids.TextColumn ): - def get_value( self, trans, grid, quota ): if quota.description: return quota.description return '' - class AmountColumn( grids.TextColumn ): - def get_value( self, trans, grid, quota ): return quota.operation + quota.display_amount - class StatusColumn( grids.GridColumn ): - def get_value( self, trans, grid, quota ): if quota.deleted: return "deleted" @@ -383,17 +337,13 @@ class QuotaListGrid( grids.Grid ): return "default for %s users" % quota.default[0].type return "" - class UsersColumn( grids.GridColumn ): - def get_value( self, trans, grid, quota ): if quota.users: return len( quota.users ) return 0 - class GroupsColumn( grids.GridColumn ): - def get_value( self, trans, grid, quota ): if quota.groups: return len( quota.groups ) @@ -402,7 +352,7 @@ class QuotaListGrid( grids.Grid ): # Grid definition title = "Quotas" model_class = model.Quota - template='/admin/quota/grid.mako' + template = '/admin/quota/grid.mako' default_sort_key = "name" columns = [ NameColumn( "Name", @@ -417,10 +367,10 @@ class QuotaListGrid( grids.Grid ): attach_popup=False, filterable="advanced" ), AmountColumn( "Amount", - key='amount', - model_class=model.Quota, - attach_popup=False, - filterable="advanced" ), + key='amount', + model_class=model.Quota, + attach_popup=False, + filterable="advanced" ), UsersColumn( "Users", attach_popup=False ), GroupsColumn( "Groups", attach_popup=False ), StatusColumn( "Status", attach_popup=False ), @@ -483,9 +433,7 @@ class QuotaListGrid( grids.Grid ): class ToolVersionListGrid( grids.Grid ): - class ToolIdColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_version ): toolbox = trans.app.toolbox if toolbox.has_tool( tool_version.tool_id, exact=True ): @@ -494,9 +442,7 @@ class ToolVersionListGrid( grids.Grid ): return '
%s%s
' % ( link_str, tool_version.tool_id ) return tool_version.tool_id - class ToolVersionsColumn( grids.TextColumn ): - def get_value( self, trans, grid, tool_version ): tool_ids_str = '' toolbox = trans.app.toolbox @@ -508,10 +454,11 @@ class ToolVersionListGrid( grids.Grid ): else: tool_ids_str += '%s
' % tool_id return tool_ids_str + # Grid definition title = "Tool versions" model_class = install_model.ToolVersion - template='/admin/tool_version/grid.mako' + template = '/admin/tool_version/grid.mako' default_sort_key = "tool_id" columns = [ ToolIdColumn( "Tool id", @@ -531,7 +478,7 @@ class ToolVersionListGrid( grids.Grid ): num_rows_per_page = 50 preserve_state = False use_paging = True - + def build_initial_query( self, trans, **kwd ): return trans.install_model.context.query( self.model_class ) @@ -899,10 +846,11 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP kwargs={'display_application_ids': kwd.get( 'id' )} ) reloaded, failed = trans.app.datatypes_registry.reload_display_applications( kwd.get( 'id' ) ) if not reloaded and failed: - return trans.show_error_message( 'Unable to reload any of the %i requested display applications ("%s").' % ( len( failed ), '", "'.join( failed ) ) ) + return trans.show_error_message( 'Unable to reload any of the %i requested display applications ("%s").' + % ( len( failed ), '", "'.join( failed ) ) ) if failed: return trans.show_warn_message( 'Reloaded %i display applications ("%s"), but failed to reload %i display applications ("%s").' - % ( len( reloaded ), '", "'.join( reloaded ), len( failed ), '", "'.join( failed ) ) ) + % ( len( reloaded ), '", "'.join( reloaded ), len( failed ), '", "'.join( failed ) ) ) if not reloaded: return trans.show_warn_message( 'You need to request at least one display application to reload.' ) return trans.show_ok_message( 'Reloaded %i requested display applications ("%s").' % ( len( reloaded ), '", "'.join( reloaded ) ) ) From bf56b93c7f8fda3c6d579872780c0d56e7e9f114 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 09:34:42 -0500 Subject: [PATCH 05/17] Update queries/filters in galaxy admin controller to use sqlalchemy constructs for True, False, testing for None. --- lib/galaxy/webapps/galaxy/controllers/admin.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 96f3ef8aa04..e7060bdb7c8 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -2,6 +2,8 @@ import imp import logging import os +from sqlalchemy.sql import expression + import galaxy.queue_worker import galaxy.util from galaxy import model @@ -540,14 +542,14 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP new_in_users = [] new_in_groups = [] for user in trans.sa_session.query( trans.app.model.User ) \ - .filter( trans.app.model.User.table.c.deleted==False ) \ + .filter( trans.app.model.User.table.c.deleted == expression.false() ) \ .order_by( trans.app.model.User.table.c.email ): if user.id in in_users: new_in_users.append( ( user.id, user.email ) ) else: params.out_users.append( ( user.id, user.email ) ) for group in trans.sa_session.query( trans.app.model.Group ) \ - .filter( trans.app.model.Group.table.c.deleted==False ) \ + .filter( trans.app.model.Group.table.c.deleted == expression.false() ) \ .order_by( trans.app.model.Group.table.c.name ): if group.id in in_groups: new_in_groups.append( ( group.id, group.name ) ) @@ -592,14 +594,14 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP in_groups = [] out_groups = [] for user in trans.sa_session.query( trans.app.model.User ) \ - .filter( trans.app.model.User.table.c.deleted==False ) \ + .filter( trans.app.model.User.table.c.deleted == expression.false() ) \ .order_by( trans.app.model.User.table.c.email ): if user in [ x.user for x in quota.users ]: in_users.append( ( user.id, user.email ) ) else: out_users.append( ( user.id, user.email ) ) for group in trans.sa_session.query( trans.app.model.Group ) \ - .filter( trans.app.model.Group.table.c.deleted==False ) \ + .filter( trans.app.model.Group.table.c.deleted == expression.false()) \ .order_by( trans.app.model.Group.table.c.name ): if group in [ x.group for x in quota.groups ]: in_groups.append( ( group.id, group.name ) ) @@ -722,7 +724,7 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP webapp=params.webapp, message=sanitize_text( str( e ) ), status='error' ) ) - if do_op == True or ( do_op != False and params.get( do_op, False ) ): + if do_op is True or ( do_op is not False and params.get( do_op, False ) ): try: message = op_method( quota, params ) return None, trans.response.send_redirect( web.url_for( controller='admin', From 3de762e3e07f61a5c51917f89b003866ecaed206 Mon Sep 17 00:00:00 2001 From: Peter Cock Date: Wed, 25 Feb 2015 14:37:56 +0000 Subject: [PATCH 06/17] Suppress filenames in SAM merge using egrep Based on fix suggested by Roberto Alonso (at CIPF) on the mailing list, see: https://lists.galaxyproject.org/pipermail/galaxy-dev/2015-February/021589.html --- lib/galaxy/datatypes/tabular.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/datatypes/tabular.py b/lib/galaxy/datatypes/tabular.py index a28deb61c48..df571eb1120 100644 --- a/lib/galaxy/datatypes/tabular.py +++ b/lib/galaxy/datatypes/tabular.py @@ -480,7 +480,7 @@ class Sam( Tabular ): if result != 0: raise Exception('Result %s from %s' % (result, cmd)) if len(split_files) > 1: - cmd = 'egrep -v "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file ) + cmd = 'egrep -v -h "^@" %s >> %s' % ( ' '.join(split_files[1:]), output_file ) result = os.system(cmd) if result != 0: raise Exception('Result %s from %s' % (result, cmd)) From 1f1bb29a9023d62dc8d5fad915cfd808ec85abab Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 25 Feb 2015 09:56:37 -0500 Subject: [PATCH 07/17] fix missing sqlalchemy imports and add documentation to folder manager --- lib/galaxy/managers/folders.py | 42 ++++++++++++++++++++++++++++++---- 1 file changed, 38 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/managers/folders.py b/lib/galaxy/managers/folders.py index ca5b5cf3894..7968048bdcd 100644 --- a/lib/galaxy/managers/folders.py +++ b/lib/galaxy/managers/folders.py @@ -3,6 +3,8 @@ Manager and Serializer for Library Folders. """ import galaxy.exceptions +from sqlalchemy.orm.exc import MultipleResultsFound +from sqlalchemy.orm.exc import NoResultFound import logging log = logging.getLogger( __name__ ) @@ -19,19 +21,21 @@ class FolderManager( object ): :param decoded_folder_id: decoded folder id :type decoded_folder_id: int - :param check_manageable: flag whether the check that user can manage item - :type check_manageable: bool + :param check_manageable: flag whether the check that user can manage item + :type check_manageable: bool :param check_accessible: flag whether to check that user can access item :type check_accessible: bool :returns: the requested folder :rtype: LibraryFolder + + :raises: InconsistentDatabase, RequestParameterInvalidException, InternalServerError """ try: folder = trans.sa_session.query( trans.app.model.LibraryFolder ).filter( trans.app.model.LibraryFolder.table.c.id == decoded_folder_id ).one() - except galaxy.exceptions.MultipleResultsFound: + except MultipleResultsFound: raise galaxy.exceptions.InconsistentDatabase( 'Multiple folders found with the same id.' ) - except galaxy.exceptions.NoResultFound: + except NoResultFound: raise galaxy.exceptions.RequestParameterInvalidException( 'No folder found with the id provided.' ) except Exception, e: raise galaxy.exceptions.InternalServerError( 'Error loading from the database.' + str( e ) ) @@ -139,6 +143,14 @@ class FolderManager( object ): """ Mark given folder deleted/undeleted based on the flag. + :param folder: the model object + :type folder: LibraryFolder + :param undelete: flag whether to delete (when False) or undelete + :type undelete: Bool + + :returns: the folder + :rtype: LibraryFolder + :raises: ItemAccessibilityException """ if not trans.user_is_admin(): @@ -190,6 +202,14 @@ class FolderManager( object ): def cut_the_prefix( self, encoded_folder_id ): """ Remove the prefix from the encoded folder id. + + :param encoded_folder_id: encoded id of the Folder object with 'F' prepended + :type encoded_folder_id: string + + :returns: encoded Folder id without the 'F' prefix + :rtype: string + + :raises: MalformedId """ if ( ( len( encoded_folder_id ) % 16 == 1 ) and encoded_folder_id.startswith( 'F' ) ): cut_id = encoded_folder_id[ 1: ] @@ -200,6 +220,14 @@ class FolderManager( object ): def decode_folder_id( self, trans, encoded_folder_id ): """ Decode the folder id given that it has already lost the prefixed 'F'. + + :param encoded_folder_id: encoded id of the Folder object + :type encoded_folder_id: string + + :returns: decoded Folder id + :rtype: int + + :raises: MalformedId """ try: decoded_id = trans.security.decode_id( encoded_folder_id ) @@ -210,5 +238,11 @@ class FolderManager( object ): def cut_and_decode( self, trans, encoded_folder_id ): """ Cuts the folder prefix (the prepended 'F') and returns the decoded id. + + :param encoded_folder_id: encoded id of the Folder object + :type encoded_folder_id: string + + :returns: decoded Folder id + :rtype: int """ return self.decode_folder_id( trans, self.cut_the_prefix( encoded_folder_id ) ) From 9dfa83bf0f2abd1c2182da9edef1bb0996d7d757 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 25 Feb 2015 10:52:35 -0500 Subject: [PATCH 08/17] create library root folder with '/' as default name fix for showing actual name of root folder when calling GET /api/libraries/{library_id}/contents --- lib/galaxy/managers/libraries.py | 2 +- lib/galaxy/webapps/galaxy/api/library_contents.py | 9 +++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/managers/libraries.py b/lib/galaxy/managers/libraries.py index c3949608bb5..67a7aeb17fb 100644 --- a/lib/galaxy/managers/libraries.py +++ b/lib/galaxy/managers/libraries.py @@ -54,7 +54,7 @@ class LibraryManager( object ): raise exceptions.ItemAccessibilityException( 'Only administrators can create libraries.' ) else: library = trans.app.model.Library( name=name, description=description, synopsis=synopsis ) - root_folder = trans.app.model.LibraryFolder( name=name, description='' ) + root_folder = trans.app.model.LibraryFolder( name='/', description='' ) library.root_folder = root_folder trans.sa_session.add_all( ( library, root_folder ) ) trans.sa_session.flush() diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py index 55dff652056..3f7c1a72895 100644 --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -85,12 +85,13 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar raise exceptions.InternalServerError( 'Error loading from the database.' + str(e)) if not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( current_user_roles, library ) ): raise exceptions.RequestParameterInvalidException( 'No library found with the id provided.' ) - encoded_id = 'F' + trans.security.encode_id( library.root_folder.id ) + root_encoded_id = 'F' + trans.security.encode_id( library.root_folder.id ) + root_name = library.root_folder.name # appending root folder - rval.append( dict( id=encoded_id, + rval.append( dict( id=root_encoded_id, type='folder', - name='/', - url=url_for( 'library_content', library_id=library_id, id=encoded_id ) ) ) + name=root_name, + url=url_for( 'library_content', library_id=library_id, id=root_encoded_id ) ) ) library.root_folder.api_path = '' # appending all other items in the library recursively for content in traverse( library.root_folder ): From 964e081038565bfb5b5911b210aef1610340d142 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 11:48:01 -0500 Subject: [PATCH 09/17] Initial commit of recalculating user disk usage from the admin interface (user grid). --- .../webapps/galaxy/controllers/admin.py | 77 ++++++++++++++++++- 1 file changed, 73 insertions(+), 4 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index e7060bdb7c8..0631d13d181 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -1,17 +1,16 @@ import imp import logging import os - from sqlalchemy.sql import expression import galaxy.queue_worker import galaxy.util from galaxy import model -from galaxy.model import tool_shed_install as install_model from galaxy import web from galaxy.actions.admin import AdminActions from galaxy.exceptions import MessageException -from galaxy.util import sanitize_text +from galaxy.model import tool_shed_install as install_model +from galaxy.util import nice_size, sanitize_text from galaxy.util.odict import odict from galaxy.web import url_for from galaxy.web.base.controller import BaseUIController, UsesQuotaMixin @@ -124,7 +123,11 @@ class UserListGrid( grids.Grid ): condition=( lambda item: not item.deleted ), allow_multiple=True, allow_popup=False, - url_args=dict( webapp="galaxy", action="reset_user_password" ) ) + url_args=dict( webapp="galaxy", action="reset_user_password" ) ), + grids.GridOperation( "Recalculate Disk Usage", + condition=( lambda item: not item.deleted ), + allow_multiple=False, + url_args=dict( webapp="galaxy", action="recalculate_user_disk_usage" ) ) ] standard_filters = [ grids.GridColumnFilter( "Active", args=dict( deleted=False ) ), @@ -856,3 +859,69 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP if not reloaded: return trans.show_warn_message( 'You need to request at least one display application to reload.' ) return trans.show_ok_message( 'Reloaded %i requested display applications ("%s").' % ( len( reloaded ), '", "'.join( reloaded ) ) ) + + @web.expose + @web.require_admin + def recalculate_user_disk_usage( self, trans, **kwd ): + user_id = kwd.get( 'id', None ) + user = trans.sa_session.query( trans.model.User ).get( trans.security.decode_id( user_id ) ) + if not user: + return trans.show_error_message( "User not found for id (%s)" % sanitize_text( str( user_id ) ) ) + engine = None + if trans.app.config.database_connection: + engine = trans.app.config.database_connection.split(':')[0] + if engine not in ( 'postgres', 'postgresql' ): + done = False + while not done: + current = user.get_disk_usage() + new = user.calculate_disk_usage() + trans.sa_session.refresh( user ) + # make sure usage didn't change while calculating, set done + if user.get_disk_usage() == current: + done = True + if new not in (current, None): + user.set_disk_usage( new ) + trans.sa_session.add( user ) + trans.sa_session.flush() + else: + # We can use the lightning fast pgcalc! + current = user.get_disk_usage() + new = pgcalc( self.sa_session, user.id ) + # yes, still a small race condition between here and the flush + if new in ( current, None ): + message = 'Usage is unchanged at %s.' % nice_size( current ) + else: + message = 'Usage has changed by %s to %s.' % ( nice_size( new - current ), nice_size( current ) ) + return trans.response.send_redirect( web.url_for( controller='admin', + action='users', + message=sanitize_text( message ), + status='info' ) ) + + +def pgcalc( sa_session, id, dryrun=False ): + """ + Utility method for quickly recalculating user disk usage in postgres. + TODO: Move this elsewhere and update scripts/set_user_disk_usage.py + """ + sql = """ + UPDATE galaxy_user + SET disk_usage = (SELECT COALESCE(SUM(total_size), 0) + FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id + FROM history_dataset_association hda + JOIN history h ON h.id = hda.history_id + JOIN dataset d ON hda.dataset_id = d.id + WHERE h.user_id = :id + AND h.purged = false + AND hda.purged = false + AND d.purged = false + AND d.id NOT IN (SELECT dataset_id + FROM library_dataset_dataset_association) + ) sizes) + WHERE id = :id + RETURNING disk_usage; + """ + r = sa_session.execute(sql, {'id': id}) + new = r.fetchone()[0] + if dryrun: + sa_session.rollback() + return new From b89a5be3193b8609a046dd738b0dc4e6eb0281fd Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Wed, 25 Feb 2015 12:07:39 -0500 Subject: [PATCH 10/17] Fix for ToolShed not having config.shed_tool_data_path --- lib/galaxy/webapps/tool_shed/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index 24dc7fc0d0f..05fdceab3ac 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -136,6 +136,10 @@ class Configuration( object ): self.citation_cache_data_dir = resolve_path( kwargs.get( "citation_cache_data_dir", "database/tool_shed_citations/data" ), self.root ) self.citation_cache_lock_dir = resolve_path( kwargs.get( "citation_cache_lock_dir", "database/tool_shed_citations/locks" ), self.root ) + @property + def shed_tool_data_path( self ): + return self.tool_data_path + @property def sentry_dsn_public( self ): """ From fe226e1a9bf478720b31570ce44aa78ce3f5041b Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 12:52:47 -0500 Subject: [PATCH 11/17] Refactoring of set_user_disk_usage and admin controller disk usage to use the same pgcalc --- lib/galaxy/model/util.py | 36 +++++++++++ .../webapps/galaxy/controllers/admin.py | 30 +--------- scripts/set_user_disk_usage.py | 60 +++++++------------ 3 files changed, 58 insertions(+), 68 deletions(-) create mode 100644 lib/galaxy/model/util.py diff --git a/lib/galaxy/model/util.py b/lib/galaxy/model/util.py new file mode 100644 index 00000000000..4faf32e999a --- /dev/null +++ b/lib/galaxy/model/util.py @@ -0,0 +1,36 @@ +""" +Utility helpers related to the model +""" + + +def pgcalc( sa_session, id, dryrun=False ): + """ + Utility method for quickly recalculating user disk usage in postgres. + + TODO: Move this elsewhere and update scripts/set_user_disk_usage.py + + TODO: Check against the recently updated versions of sqlalchemy if this + 'special' postgresql version is even necessary. + """ + sql = """ + UPDATE galaxy_user + SET disk_usage = (SELECT COALESCE(SUM(total_size), 0) + FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id + FROM history_dataset_association hda + JOIN history h ON h.id = hda.history_id + JOIN dataset d ON hda.dataset_id = d.id + WHERE h.user_id = :id + AND h.purged = false + AND hda.purged = false + AND d.purged = false + AND d.id NOT IN (SELECT dataset_id + FROM library_dataset_dataset_association) + ) sizes) + WHERE id = :id + RETURNING disk_usage; + """ + r = sa_session.execute(sql, {'id': id}) + new = r.fetchone()[0] + if dryrun: + sa_session.rollback() + return new diff --git a/lib/galaxy/webapps/galaxy/controllers/admin.py b/lib/galaxy/webapps/galaxy/controllers/admin.py index 0631d13d181..a5149e5848d 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin.py @@ -10,6 +10,7 @@ from galaxy import web from galaxy.actions.admin import AdminActions from galaxy.exceptions import MessageException from galaxy.model import tool_shed_install as install_model +from galaxy.model.util import pgcalc from galaxy.util import nice_size, sanitize_text from galaxy.util.odict import odict from galaxy.web import url_for @@ -896,32 +897,3 @@ class AdminGalaxy( BaseUIController, Admin, AdminActions, UsesQuotaMixin, QuotaP action='users', message=sanitize_text( message ), status='info' ) ) - - -def pgcalc( sa_session, id, dryrun=False ): - """ - Utility method for quickly recalculating user disk usage in postgres. - TODO: Move this elsewhere and update scripts/set_user_disk_usage.py - """ - sql = """ - UPDATE galaxy_user - SET disk_usage = (SELECT COALESCE(SUM(total_size), 0) - FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id - FROM history_dataset_association hda - JOIN history h ON h.id = hda.history_id - JOIN dataset d ON hda.dataset_id = d.id - WHERE h.user_id = :id - AND h.purged = false - AND hda.purged = false - AND d.purged = false - AND d.id NOT IN (SELECT dataset_id - FROM library_dataset_dataset_association) - ) sizes) - WHERE id = :id - RETURNING disk_usage; - """ - r = sa_session.execute(sql, {'id': id}) - new = r.fetchone()[0] - if dryrun: - sa_session.rollback() - return new diff --git a/scripts/set_user_disk_usage.py b/scripts/set_user_disk_usage.py index 282ad589aa4..b4ae53aaae9 100755 --- a/scripts/set_user_disk_usage.py +++ b/scripts/set_user_disk_usage.py @@ -1,9 +1,22 @@ #!/usr/bin/env python -import os, sys +import os +import sys + from ConfigParser import ConfigParser from optparse import OptionParser +sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) ) + +from galaxy import eggs +import pkg_resources + +import galaxy.config +from galaxy.model.util import pgcalc +from galaxy.util import nice_size +from galaxy.objectstore import build_object_store_from_config + + default_config = os.path.abspath( os.path.join( os.path.dirname( __file__ ), '..', 'config/galaxy.ini') ) parser = OptionParser() @@ -13,6 +26,7 @@ parser.add_option( '-e', '--email', dest='email', help='Email address of user to parser.add_option( '--dry-run', dest='dryrun', help='Dry run (show changes but do not save to database)', action='store_true', default=False ) ( options, args ) = parser.parse_args() + def init(): options.config = os.path.abspath( options.config ) @@ -21,19 +35,8 @@ def init(): if options.email == 'all': options.email = None - sys.path.insert( 1, os.path.join( os.path.dirname( __file__ ), '..', 'lib' ) ) - - from galaxy import eggs - import pkg_resources - - import galaxy.config - from galaxy.objectstore import build_object_store_from_config - - # lazy - globals()['nice_size'] = __import__( 'galaxy.util', globals(), locals(), ( 'nice_size', ) ).nice_size - - config_parser = ConfigParser( dict( here = os.getcwd(), - database_connection = 'sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) ) + config_parser = ConfigParser( dict( here=os.getcwd(), + database_connection='sqlite:///database/universe.sqlite?isolation_level=IMMEDIATE' ) ) config_parser.read( options.config ) config_dict = {} @@ -45,31 +48,10 @@ def init(): from galaxy.model import mapping - return mapping.init( config.file_path, config.database_connection, create_tables = False, object_store = object_store ), object_store, config.database_connection.split(':')[0] + return (mapping.init( config.file_path, config.database_connection, create_tables=False, object_store=object_store ), + object_store, + config.database_connection.split(':')[0]) -def pgcalc( sa_session, id ): - sql = """ - UPDATE galaxy_user - SET disk_usage = (SELECT COALESCE(SUM(total_size), 0) - FROM ( SELECT DISTINCT ON (d.id) d.total_size, d.id - FROM history_dataset_association hda - JOIN history h ON h.id = hda.history_id - JOIN dataset d ON hda.dataset_id = d.id - WHERE h.user_id = :id - AND h.purged = false - AND hda.purged = false - AND d.purged = false - AND d.id NOT IN (SELECT dataset_id - FROM library_dataset_dataset_association) - ) sizes) - WHERE id = :id - RETURNING disk_usage; - """ - r = sa_session.execute(sql, {'id':id}) - new = r.fetchone()[0] - if options.dryrun: - sa_session.rollback() - return new def quotacheck( sa_session, users, engine ): sa_session.refresh( user ) @@ -83,7 +65,7 @@ def quotacheck( sa_session, users, engine ): print 'usage changed while calculating, trying again...' return quotacheck( sa_session, user, engine ) else: - new = pgcalc( sa_session, user.id ) + new = pgcalc( sa_session, user.id, dryrun=options.dryrun ) # yes, still a small race condition between here and the flush print 'old usage:', nice_size( current ), 'change:', if new in ( current, None ): From d901bc52231c8959322c096667aa240acce6ff9e Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 12:54:43 -0500 Subject: [PATCH 12/17] Remove finished TODO. --- lib/galaxy/model/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/galaxy/model/util.py b/lib/galaxy/model/util.py index 4faf32e999a..76128609496 100644 --- a/lib/galaxy/model/util.py +++ b/lib/galaxy/model/util.py @@ -7,8 +7,6 @@ def pgcalc( sa_session, id, dryrun=False ): """ Utility method for quickly recalculating user disk usage in postgres. - TODO: Move this elsewhere and update scripts/set_user_disk_usage.py - TODO: Check against the recently updated versions of sqlalchemy if this 'special' postgresql version is even necessary. """ From 713c52a47d4e3c89928213342c69f5bd8b918542 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 13:34:38 -0500 Subject: [PATCH 13/17] Fix two incorrect (undefined) references and misc cleanup in installed_repository_manager.py --- .../installed_repository_manager.py | 73 +++++++++---------- 1 file changed, 33 insertions(+), 40 deletions(-) diff --git a/lib/tool_shed/galaxy_install/installed_repository_manager.py b/lib/tool_shed/galaxy_install/installed_repository_manager.py index 9437c983753..7bd50214b5e 100644 --- a/lib/tool_shed/galaxy_install/installed_repository_manager.py +++ b/lib/tool_shed/galaxy_install/installed_repository_manager.py @@ -103,12 +103,12 @@ class InstalledRepositoryManager( object ): # Hack to add repository.name here, which is actually the root of the installed repository data_manager_relative_install_dir = os.path.join( data_manager_relative_install_dir, repository.name ) dmh = data_manager.DataManagerHandler( self.app ) - new_data_managers = dmh.install_data_managers( self.app.config.shed_data_manager_config_file, - repository.metadata, - repository.get_shed_config_dict( self.app ), - data_manager_relative_install_dir, - repository, - repository_tools_tups ) + dmh.install_data_managers( self.app.config.shed_data_manager_config_file, + repository.metadata, + repository.get_shed_config_dict( self.app ), + data_manager_relative_install_dir, + repository, + repository_tools_tups ) self.install_model.context.add( repository ) self.install_model.context.flush() if repository.includes_datatypes: @@ -400,14 +400,8 @@ class InstalledRepositoryManager( object ): installed_repository_dependencies = {} missing_rd_tups = [] installed_rd_tups = [] - description, \ - repository_clone_url, \ - changeset_revision, \ - ctx_rev, \ - repository_owner, \ - repository_dependencies, \ - tool_dependencies = \ - suc.get_repo_info_tuple_contents( repo_info_tuple ) + ( description, repository_clone_url, changeset_revision, ctx_rev, + repository_owner, repository_dependencies, tool_dependencies ) = suc.get_repo_info_tuple_contents( repo_info_tuple ) if repository_dependencies: description = repository_dependencies[ 'description' ] root_key = repository_dependencies[ 'root_key' ] @@ -460,7 +454,7 @@ class InstalledRepositoryManager( object ): prior_installation_required, only_if_compiling_contained_td, None, - 'Never installed' ] + 'Never installed' ] if not util.asbool( only_if_compiling_contained_td ): # A repository dependency that is not installed will not be considered missing if its value for # only_if_compiling_contained_td is True - see above... @@ -564,10 +558,10 @@ class InstalledRepositoryManager( object ): continue if required_repository_tup not in dependency_tups: dependency_tups.append( required_repository_tup ) - return get_repository_dependency_tups_for_installed_repository( required_repository, - dependency_tups=dependency_tups ) + return self.get_repository_dependency_tups_for_installed_repository( required_repository, + dependency_tups=dependency_tups ) return dependency_tups - + def get_repository_tuple_for_installed_repository_manager( self, repository ): return ( str( repository.tool_shed ), str( repository.name ), @@ -611,10 +605,10 @@ class InstalledRepositoryManager( object ): contents = open( env_shell_file_path, 'r' ).read() except Exception, e: contents = None - log.debug( 'Error reading file %s, so cannot determine if package %s requires package %s at run time: %s' % \ - ( str( env_shell_file_path ), str( td.name ), str( tool_dependency.name ), str( e ) ) ) + log.debug( 'Error reading file %s, so cannot determine if package %s requires package %s at run time: %s' % + ( str( env_shell_file_path ), str( td.name ), str( tool_dependency.name ), str( e ) ) ) if contents is not None and contents.find( required_env_shell_file_path ) >= 0: - td_tuple = get_tool_dependency_tuple_for_installed_repository_manager( td ) + td_tuple = self.get_tool_dependency_tuple_for_installed_repository_manager( td ) runtime_dependent_tool_dependency_tups.append( td_tuple ) return runtime_dependent_tool_dependency_tups @@ -672,8 +666,7 @@ class InstalledRepositoryManager( object ): # Remove this tool_dependency from all values in # self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies = {} - for td_tup, installed_runtime_dependent_tool_dependency_tups in \ - self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.items(): + for ( td_tup, installed_runtime_dependent_tool_dependency_tups ) in self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies.items(): if tool_dependency_tup in installed_runtime_dependent_tool_dependency_tups: # Remove the tool_dependency from the list. installed_runtime_dependent_tool_dependency_tups.remove( tool_dependency_tup ) @@ -684,7 +677,7 @@ class InstalledRepositoryManager( object ): altered_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies # Remove the entry for this tool_dependency from self.runtime_tool_dependencies_of_installed_tool_dependencies. self.remove_entry_from_runtime_tool_dependencies_of_installed_tool_dependencies( tool_dependency ) - # Remove the entry for this tool_dependency from + # Remove the entry for this tool_dependency from # self.installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies. self.remove_entry_from_installed_runtime_dependent_tool_dependencies_of_installed_tool_dependencies( tool_dependency ) # Remove this repository's entry from self.installed_tool_dependencies_of_installed_repositories. @@ -864,7 +857,7 @@ class InstalledRepositoryManager( object ): else: status = 'error' message = 'A repository must have the status New in order to be purged. This repository has ' - message += ' the status %s.' % str( repository.status ) + message += ' the status %s.' % str( repository.status ) return status, message def remove_entry_from_installed_repository_dependencies_of_installed_repositories( self, repository ): @@ -884,7 +877,7 @@ class InstalledRepositoryManager( object ): debug_msg += "from the dependent list for revision %s of repository %s owned by %s " % \ ( r_installed_changeset_revision, r_name, r_owner ) debug_msg += "in installed_repository_dependencies_of_installed_repositories." - log.debug( debug_msg ) + log.debug( debug_msg ) v_tups.remove( repository_tup ) altered_installed_dependent_repositories_of_installed_repositories[ r_tup ] = v_tups self.installed_dependent_repositories_of_installed_repositories = \ @@ -957,7 +950,7 @@ class InstalledRepositoryManager( object ): if cleaned_repository_dependency_tool_shed == cleaned_tool_shed and \ repository_dependency.name == name and \ repository_dependency.owner == owner and \ - ( repository_dependency.installed_changeset_revision == changeset_revision or \ + ( repository_dependency.installed_changeset_revision == changeset_revision or repository_dependency.changeset_revision == changeset_revision ): return True return False @@ -970,9 +963,9 @@ class InstalledRepositoryManager( object ): # This method is called only from Galaxy when rendering repository dependencies # for an installed tool shed repository. # TODO: Do we need to check more than changeset_revision here? - required_repository_tup = [ required_repository.tool_shed, \ - required_repository.name, \ - required_repository.owner, \ + required_repository_tup = [ required_repository.tool_shed, + required_repository.name, + required_repository.owner, required_repository.changeset_revision ] for tup in repository.tuples_of_repository_dependencies_needed_for_compiling_td: partial_tup = tup[ 0:4 ] @@ -1009,12 +1002,12 @@ class InstalledRepositoryManager( object ): pulled by a Galaxy administrator when getting updates to an installed tool shed repository. The original_dependency_dict is a single tool dependency definition, an example of which is:: - + {"name": "bwa", "readme": "\\nCompiling BWA requires zlib and libpthread to be present on your system.\\n ", "type": "package", "version": "0.6.2"} - + The new_dependencies_dict is the dictionary generated by the metadata_util.generate_tool_dependency_metadata method. """ new_tool_dependency = None @@ -1046,12 +1039,12 @@ class InstalledRepositoryManager( object ): break if new_dependency_name and new_dependency_type and new_dependency_version: # Update all attributes of the tool_dependency record in the database. - log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." % \ - ( str( tool_dependency.version ), - str( tool_dependency.type ), - str( tool_dependency.name ), - str( new_dependency_version ), - str( new_dependency_type ) ) ) + log.debug( "Updating version %s of tool dependency %s %s to have new version %s and type %s." + % ( str( tool_dependency.version ), + str( tool_dependency.type ), + str( tool_dependency.name ), + str( new_dependency_version ), + str( new_dependency_type ) ) ) tool_dependency.type = new_dependency_type tool_dependency.version = new_dependency_version tool_dependency.status = self.app.install_model.ToolDependency.installation_status.UNINSTALLED @@ -1062,8 +1055,8 @@ class InstalledRepositoryManager( object ): else: # We have no new tool dependency definition based on a matching dependency name, so remove # the existing tool dependency record from the database. - log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." % \ - ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) + log.debug( "Deleting version %s of tool dependency %s %s from the database since it is no longer defined." + % ( str( tool_dependency.version ), str( tool_dependency.type ), str( tool_dependency.name ) ) ) context.delete( tool_dependency ) context.flush() return new_tool_dependency From 42cbd8bea6e6004bf999548112a9733805411cfa Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 13:50:15 -0500 Subject: [PATCH 14/17] Cleanup in galaxy.web.framework.base, correct bug where encoding didn't exist and would blow up if (not that this ever happened afaik) we attempted to stream back a nested iterable. --- lib/galaxy/web/framework/base.py | 70 +++++++++++++++++++++----------- 1 file changed, 47 insertions(+), 23 deletions(-) diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 1b571565b3e..590cf9a37eb 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -2,7 +2,7 @@ A simple WSGI application/framework. """ -import cgi # For FieldStorage +import cgi # For FieldStorage import logging import os.path import socket @@ -67,7 +67,7 @@ class WebApplication( object ): # FIXME: The following two options are deprecated and should be # removed. Consult the Routes documentation. self.mapper.minimization = True - #self.mapper.explicit = False + # self.mapper.explicit = False self.transaction_factory = DefaultWebTransaction # Set if trace logging is enabled self.trace_logger = None @@ -79,12 +79,12 @@ class WebApplication( object ): method use `add_route`. """ log.debug( "Enabling '%s' controller, class: %s", - controller_name, controller.__class__.__name__ ) + controller_name, controller.__class__.__name__ ) self.controllers[ controller_name ] = controller def add_api_controller( self, controller_name, controller ): log.debug( "Enabling '%s' API controller, class: %s", - controller_name, controller.__class__.__name__ ) + controller_name, controller.__class__.__name__ ) self.api_controllers[ controller_name ] = controller def add_route( self, route, **kwargs ): @@ -148,7 +148,7 @@ class WebApplication( object ): else: environ[ 'is_api_request' ] = False controllers = self.controllers - if map == None: + if map is None: raise httpexceptions.HTTPNotFound( "No route for " + path_info ) self.trace( path_info=path_info, map=map ) # Setup routes @@ -239,11 +239,13 @@ class WSGIEnvironmentProperty( object ): associated object (provides property style access to keys in the WSGI environment) """ - def __init__( self, key, default = '' ): + def __init__( self, key, default='' ): self.key = key self.default = default - def __get__( self, obj, type = None ): - if obj is None: return self + + def __get__( self, obj, type=None ): + if obj is None: + return self return obj.environ.get( self.key, self.default ) @@ -254,8 +256,10 @@ class LazyProperty( object ): """ def __init__( self, func ): self.func = func - def __get__(self, obj, type = None ): - if obj is None: return self + + def __get__(self, obj, type=None ): + if obj is None: + return self value = self.func( obj ) setattr( obj, self.func.func_name, value ) return value @@ -273,6 +277,7 @@ class DefaultWebTransaction( object ): self.environ = environ self.request = Request( environ ) self.response = Response() + @lazy_property def session( self ): """ @@ -291,11 +296,14 @@ class DefaultWebTransaction( object ): # but for performance reasons it's way better to use Paste's tempfile than to # create a new one and copy. import tempfile + + class FieldStorage( cgi.FieldStorage ): def make_file(self, binary=None): return tempfile.NamedTemporaryFile() + def read_lines(self): - # Always make a new file + # Always make a new file self.file = self.make_file() self.__file = None if self.outerboundary: @@ -313,45 +321,58 @@ class Request( webob.Request ): """ Create a new request wrapping the WSGI environment `environ` """ - ## self.environ = environ + # self.environ = environ webob.Request.__init__( self, environ, charset='utf-8', decode_param_names=False ) # Properties that are computed and cached on first use + @lazy_property def remote_host( self ): try: return socket.gethostbyname( self.remote_addr ) except socket.error: return self.remote_addr + @lazy_property def remote_hostname( self ): try: return socket.gethostbyaddr( self.remote_addr )[0] except socket.error: return self.remote_addr + @lazy_property def cookies( self ): return get_cookies( self.environ ) + @lazy_property def base( self ): return ( self.scheme + "://" + self.host ) - ## @lazy_property - ## def params( self ): - ## return parse_formvars( self.environ ) + + # @lazy_property + # def params( self ): + # return parse_formvars( self.environ ) + @lazy_property def path( self ): return self.environ['SCRIPT_NAME'] + self.environ['PATH_INFO'] + @lazy_property def browser_url( self ): return self.base + self.path + # Descriptors that map properties to the associated environment - ## scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' ) - ## remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' ) + + # scheme = WSGIEnvironmentProperty( 'wsgi.url_scheme' ) + # remote_addr = WSGIEnvironmentProperty( 'REMOTE_ADDR' ) + remote_port = WSGIEnvironmentProperty( 'REMOTE_PORT' ) - ## method = WSGIEnvironmentProperty( 'REQUEST_METHOD' ) - ## script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' ) + + # method = WSGIEnvironmentProperty( 'REQUEST_METHOD' ) + # script_name = WSGIEnvironmentProperty( 'SCRIPT_NAME' ) + protocol = WSGIEnvironmentProperty( 'SERVER_PROTOCOL' ) - ## query_string = WSGIEnvironmentProperty( 'QUERY_STRING' ) - ## path_info = WSGIEnvironmentProperty( 'PATH_INFO' ) + + # query_string = WSGIEnvironmentProperty( 'QUERY_STRING' ) + # path_info = WSGIEnvironmentProperty( 'PATH_INFO' ) class Response( object ): @@ -406,7 +427,8 @@ class Response( object ): # ---- Utilities ------------------------------------------------------------ -CHUNK_SIZE = 2**16 +CHUNK_SIZE = 2 ** 16 + def send_file( start_response, trans, body ): # If configured use X-Accel-Redirect header for nginx @@ -426,6 +448,7 @@ def send_file( start_response, trans, body ): trans.response.wsgi_headeritems() ) return body + def iterate_file( file ): """ Progressively return chunks from `file`. @@ -436,13 +459,14 @@ def iterate_file( file ): break yield chunk + def flatten( seq ): """ Flatten a possible nested set of iterables """ for x in seq: if isinstance( x, ( types.GeneratorType, list, tuple ) ): - for y in flatten( x, encoding ): + for y in flatten( x ): yield y else: yield x From 92433a498626b0d61cd808c50ab911145c46e406 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Wed, 25 Feb 2015 13:53:01 -0500 Subject: [PATCH 15/17] Revert "create library root folder with '/' as default name" This reverts commit 9dfa83bf0f2abd1c2182da9edef1bb0996d7d757. --- lib/galaxy/managers/libraries.py | 2 +- lib/galaxy/webapps/galaxy/api/library_contents.py | 9 ++++----- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/galaxy/managers/libraries.py b/lib/galaxy/managers/libraries.py index 67a7aeb17fb..c3949608bb5 100644 --- a/lib/galaxy/managers/libraries.py +++ b/lib/galaxy/managers/libraries.py @@ -54,7 +54,7 @@ class LibraryManager( object ): raise exceptions.ItemAccessibilityException( 'Only administrators can create libraries.' ) else: library = trans.app.model.Library( name=name, description=description, synopsis=synopsis ) - root_folder = trans.app.model.LibraryFolder( name='/', description='' ) + root_folder = trans.app.model.LibraryFolder( name=name, description='' ) library.root_folder = root_folder trans.sa_session.add_all( ( library, root_folder ) ) trans.sa_session.flush() diff --git a/lib/galaxy/webapps/galaxy/api/library_contents.py b/lib/galaxy/webapps/galaxy/api/library_contents.py index 3f7c1a72895..55dff652056 100644 --- a/lib/galaxy/webapps/galaxy/api/library_contents.py +++ b/lib/galaxy/webapps/galaxy/api/library_contents.py @@ -85,13 +85,12 @@ class LibraryContentsController( BaseAPIController, UsesLibraryMixin, UsesLibrar raise exceptions.InternalServerError( 'Error loading from the database.' + str(e)) if not ( trans.user_is_admin() or trans.app.security_agent.can_access_library( current_user_roles, library ) ): raise exceptions.RequestParameterInvalidException( 'No library found with the id provided.' ) - root_encoded_id = 'F' + trans.security.encode_id( library.root_folder.id ) - root_name = library.root_folder.name + encoded_id = 'F' + trans.security.encode_id( library.root_folder.id ) # appending root folder - rval.append( dict( id=root_encoded_id, + rval.append( dict( id=encoded_id, type='folder', - name=root_name, - url=url_for( 'library_content', library_id=library_id, id=root_encoded_id ) ) ) + name='/', + url=url_for( 'library_content', library_id=library_id, id=encoded_id ) ) ) library.root_folder.api_path = '' # appending all other items in the library recursively for content in traverse( library.root_folder ): From 79d50d831c09553b1ea5d4a3ec1d7957e85649d7 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Wed, 25 Feb 2015 13:57:54 -0500 Subject: [PATCH 16/17] Fix bug in scramble exception, incorrect reference to source_path --- lib/galaxy/eggs/scramble.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/eggs/scramble.py b/lib/galaxy/eggs/scramble.py index 0a7970db2b9..528bb60dd49 100644 --- a/lib/galaxy/eggs/scramble.py +++ b/lib/galaxy/eggs/scramble.py @@ -134,7 +134,7 @@ class ScrambleEgg( Egg ): elif zipfile.is_zipfile( self.source_path ): self.unpack_zip() else: - raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, source_path ) ) + raise Exception( "%s(): Unknown archive file type for %s" % ( sys._getframe().f_code.co_name, self.source_path ) ) log.warning( "%s(): Unpacked to:" % sys._getframe().f_code.co_name ) log.warning( " %s" % self.buildpath ) def unpack_zip( self ): From cf8fd5c5970b0bc5f6dd4f8a9af710a29d90f2e4 Mon Sep 17 00:00:00 2001 From: John Chilton Date: Wed, 25 Feb 2015 20:06:12 -0500 Subject: [PATCH 17/17] Add test case demonstrating data params don't need namespace... ... unlike other parameters. Is this a bug? Has been this way for at least three releases it seems. https://github.com/galaxyproject/tools-devteam/commit/c01b354711d982f61f1fd086148692cb084aa8a5#commitcomment-9922496 --- test/functional/tools/samples_tool_conf.xml | 1 + test/functional/tools/tol_level_data.xml | 34 +++++++++++++++++++++ 2 files changed, 35 insertions(+) create mode 100644 test/functional/tools/tol_level_data.xml diff --git a/test/functional/tools/samples_tool_conf.xml b/test/functional/tools/samples_tool_conf.xml index 0294fb2f5a3..f812b6c1100 100644 --- a/test/functional/tools/samples_tool_conf.xml +++ b/test/functional/tools/samples_tool_conf.xml @@ -27,6 +27,7 @@ + diff --git a/test/functional/tools/tol_level_data.xml b/test/functional/tools/tol_level_data.xml new file mode 100644 index 00000000000..37d2ba7e00e --- /dev/null +++ b/test/functional/tools/tol_level_data.xml @@ -0,0 +1,34 @@ + + + cat '${f1}' >> $out1; + echo '${library.f2}' >> $out2; + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +