diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index be76b48d2dd..8ca695e9f55 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -1,6 +1,7 @@ import logging from paste.httpexceptions import HTTPBadRequest, HTTPForbidden +from time import strftime from galaxy import util from galaxy import web @@ -10,6 +11,7 @@ from galaxy.web.base.controller import BaseAPIController from tool_shed.galaxy_install import repository_util from tool_shed.util import common_util from tool_shed.util import encoding_util +from tool_shed.util import metadata_util from tool_shed.util import workflow_util import tool_shed.util.shed_util_common as suc @@ -487,6 +489,49 @@ class ToolShedRepositoriesController( BaseAPIController ): # Display the list of repaired repositories. return tool_shed_repositories + @web.expose_api + def reset_metadata_on_installed_repositories( self, trans, payload, **kwd ): + """ + PUT /api/tool_shed_repositories/reset_metadata_on_installed_repositories + + Resets all metadata on all repositories installed into Galaxy in an "orderly fashion". + + :param key: the API key of the Galaxy admin user. + """ + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + successful_count=0, + unsuccessful_count=0, + repository_status=[] ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to reset metadata on repositories installed into this Galaxy instance.' ) + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=False ) + # Now reset metadata on all remaining repositories. + for repository in query: + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_installed_repository( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Galaxy tool_shed_repositories API in reset_metadata_on_installed_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + @web.expose_api def show( self, trans, id, **kwd ): """ diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 28b0812ae90..3cdb0ba0939 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -187,6 +187,7 @@ def app_factory( global_conf, **kwargs ): 'exported_workflows' : 'GET', 'import_workflow' : 'POST', 'import_workflows' : 'POST' }, + collection={ 'reset_metadata_on_installed_repositories' : 'POST' }, controller='tool_shed_repositories', name_prefix='tool_shed_repository_', path_prefix='/api', diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index dc3d9af7bac..665e19e6ba8 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -1,11 +1,16 @@ import logging +from time import strftime from galaxy.web.framework.helpers import time_ago from galaxy import eggs from galaxy import web from galaxy import util +from galaxy.util import json from galaxy.web.base.controller import BaseAPIController +import tool_shed.repository_types.util as rt_util import tool_shed.util.shed_util_common as suc from tool_shed.galaxy_install import repository_util +from tool_shed.util import metadata_util +from tool_shed.util import tool_util eggs.require( 'mercurial' ) @@ -171,6 +176,69 @@ class RepositoriesController( BaseAPIController ): trans.response.status = 500 return message + @web.expose_api + def reset_metadata_on_repositories( self, trans, payload, **kwd ): + """ + PUT /api/repositories/reset_metadata_on_repositories + + Resets all metadata on all repositories in the Tool Shed in an "orderly fashion". Since there are currently only two + repository types (tool_dependecy_definition and unrestricted), the order in which metadata is reset is repositories of + type tool_dependecy_definition first followed by repositories of type unrestricted, and only one pass is necessary. If + a new repository type is introduced, the process will undoubtedly need to be revisited. To facilitate this order, an + in-memory list of repository ids that have been processed is maintained. + + :param key: the API key of the Tool Shed user. + + The following parameters can optionally be included in the payload. + :param my_writable (optional): if the API key is associated with an admin user in the Tool Shed, setting this param value + to True will restrict resetting metadata to only repositories that are writable by the user + in addition to those repositories of type tool_dependency_definition. This param is ignored + if the current user is not an admin user, in which case this same restriction is automatic. + """ + def handle_repository( trans, repository, results ): + repository_id = trans.security.encode_id( repository.id ) + try: + invalid_file_tups, metadata_dict = metadata_util.reset_all_metadata_on_repository_in_tool_shed( trans, repository_id ) + if invalid_file_tups: + message = tool_util.generate_message_for_invalid_tools( trans, invalid_file_tups, repository, None, as_html=False ) + results[ 'unsuccessful_count' ] += 1 + else: + message = "Successfully reset metadata on repository %s" % str( repository.name ) + results[ 'successful_count' ] += 1 + except Exception, e: + message = "Error resetting metadata on repository %s: %s" % ( str( repository.name ), str( e ) ) + results[ 'unsuccessful_count' ] += 1 + results[ 'repository_status' ].append( message ) + return results + try: + start_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results = dict( start_time=start_time, + repository_status=[], + successful_count=0, + unsuccessful_count=0 ) + handled_repository_ids = [] + if trans.user_is_admin(): + my_writable = util.asbool( payload.get( 'my_writable', False ) ) + else: + my_writable = True + query = suc.get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=False ) + # First reset metadata on all repositories of type repository_dependency_definition. + for repository in query: + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + # Now reset metadata on all remaining repositories. + for repository in query: + if repository.type != rt_util.TOOL_DEPENDENCY_DEFINITION and repository.id not in handled_repository_ids: + results = handle_repository( trans, repository, results ) + stop_time = strftime( "%Y-%m-%d %H:%M:%S" ) + results[ 'stop_time' ] = stop_time + return json.to_json_string( results, sort_keys=True, indent=4 * ' ' ) + except Exception, e: + message = "Error in the Tool Shed repositories API in reset_metadata_on_repositories: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 6c915ab54d6..34d03c54fb7 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -84,7 +84,8 @@ def app_factory( global_conf, **kwargs ): 'repositories', controller='repositories', collection={ 'get_repository_revision_install_info' : 'GET', - 'get_ordered_installable_revisions' : 'GET' }, + 'get_ordered_installable_revisions' : 'GET', + 'reset_metadata_on_repositories' : 'POST' }, name_prefix='repository_', path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) diff --git a/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py new file mode 100644 index 00000000000..ff21bd57719 --- /dev/null +++ b/lib/tool_shed/scripts/api/reset_metadata_on_repositories.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Script to reset metadata on certain repositories in the Tool Shed. If the received API key is associated +with an admin user in the Tool Shed, setting the my_writable param value to True will restrict resetting +metadata to only repositories that are writable by the user in addition to those repositories of type +tool_dependency_definition. The my_writable param is ignored if the current user is not an admin user, +in which case this same restriction is automatic. + +usage: reset_metadata_on_repositories.py key + +Here is a working example of how to use this script to reset metadata on certain repositories in a specified Tool Shed. +python ./reset_metadata_on_repositories.py -a 22be3b -m True -u http://localhost:9009/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + my_writable = options.my_writable + base_tool_shed_url = options.tool_shed_url.rstrip( '/' ) + data = dict( my_writable=my_writable ) + url = '%s/api/repositories/reset_metadata_on_repositories' % base_tool_shed_url + submit( url, data, options.api ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on certain repositories in the Tool Shed via the Tool Shed API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-m", "--my_writable", dest="my_writable", required=False, default='False', help="Restrict to my writable repositories" ) + parser.add_argument( "-u", "--url", dest="tool_shed_url", required=True, help="Tool Shed URL" ) + options = parser.parse_args() + main( options ) diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index 7fbb51edeee..a641abc9c6b 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1782,10 +1782,10 @@ def reset_metadata_on_selected_repositories( trans, **kwd ): log.debug( message ) unsuccessful_count += 1 else: - log.debug( "Successfully reset metadata on repository %s" % repository.name ) + log.debug( "Successfully reset metadata on repository %s" % str( repository.name ) ) successful_count += 1 except: - log.exception( "Error attempting to reset metadata on repository %s", repository.name ) + log.exception( "Error attempting to reset metadata on repository %s", str( repository.name ) ) unsuccessful_count += 1 message = "Successfully reset metadata on %d %s. " % ( successful_count, inflector.cond_plural( successful_count, "repository" ) ) if unsuccessful_count: diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index f084643df00..09ce477cfba 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -20,6 +20,7 @@ import sqlalchemy.orm.exc from tool_shed.util import common_util from tool_shed.util import encoding_util from tool_shed.util import xml_util +import tool_shed.repository_types.util as rt_util from xml.etree import ElementTree as XmlET from urllib2 import HTTPError @@ -104,45 +105,15 @@ This message was sent from the Galaxy Tool Shed instance hosted on the server def build_repository_ids_select_field( trans, name='repository_ids', multiple=True, display='checkboxes', my_writable=False ): """Method called from both Galaxy and the Tool Shed to generate the current list of repositories for resetting metadata.""" repositories_select_field = SelectField( name=name, multiple=multiple, display=display ) - if trans.webapp.name == 'tool_shed': - # We're in the tool shed. - if my_writable: - username = trans.user.username - clause_list = [] - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ): - allow_push = repository.allow_push( trans.app ) - if allow_push: - allow_push_usernames = allow_push.split( ',' ) - if username in allow_push_usernames: - clause_list.append( trans.model.Repository.table.c.id == repository.id ) - if clause_list: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( or_( *clause_list ) ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + query = get_query_for_setting_metadata_on_repositories( trans, my_writable=my_writable, order=True ) + for repository in query: + if trans.webapp.name == 'tool_shed': + owner = str( repository.user.username ) else: - for repository in trans.sa_session.query( trans.model.Repository ) \ - .filter( trans.model.Repository.table.c.deleted == False ) \ - .order_by( trans.model.Repository.table.c.name, - trans.model.Repository.table.c.user_id ): - owner = repository.user.username - option_label = '%s (%s)' % ( repository.name, owner ) - option_value = '%s' % trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) - else: - # We're in Galaxy. - for repository in trans.sa_session.query( trans.model.ToolShedRepository ) \ - .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ - .order_by( trans.model.ToolShedRepository.table.c.name, - trans.model.ToolShedRepository.table.c.owner ): - option_label = '%s (%s)' % ( repository.name, repository.owner ) - option_value = trans.security.encode_id( repository.id ) - repositories_select_field.add_option( option_label, option_value ) + owner = str( repository.owner ) + option_label = '%s (%s)' % ( str( repository.name ), owner ) + option_value = '%s' % trans.security.encode_id( repository.id ) + repositories_select_field.add_option( option_label, option_value ) return repositories_select_field def build_tool_dependencies_select_field( trans, tool_shed_repository, name, multiple=True, display='checkboxes', uninstalled_only=False ): @@ -836,6 +807,63 @@ def get_prior_import_or_install_required_dict( trans, tsr_ids, repo_info_dicts ) prior_import_or_install_required_dict[ encoded_repository_id ] = prior_import_or_install_ids return prior_import_or_install_required_dict +def get_query_for_setting_metadata_on_repositories( trans, my_writable=False, order=True ): + """ + Return a query containing repositories for resetting metadata. This method is called from both the Tool Shed and Galaxy. The + my_writable parameter is ignored unless called from the Tool Shed, and the order parameter is used for displaying the list of + repositories ordered alphabetically for display on a page. When called from wither the Tool Shed or Galaxy API, order is False. + """ + if trans.webapp.name == 'tool_shed': + # When called from the Tool Shed API, the metadata is reset on all repositories of type tool_dependency_definition in addition + # to other selected repositories. + if my_writable: + username = trans.user.username + clause_list = [] + for repository in trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ): + allow_push = repository.allow_push( trans.app ) + if not order: + # We've been called from the Tool Shed API, so reset metadata on all repositories of type tool_dependency_definition. + if repository.type == rt_util.TOOL_DEPENDENCY_DEFINITION: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + elif allow_push: + # Include all repositories that are writable by the current user. + allow_push_usernames = allow_push.split( ',' ) + if username in allow_push_usernames: + clause_list.append( trans.model.Repository.table.c.id == repository.id ) + if clause_list: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( or_( *clause_list ) ) + else: + # Return an empty query. + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.id == -1 ) + else: + if order: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) \ + .order_by( trans.model.Repository.table.c.name, + trans.model.Repository.table.c.user_id ) + else: + return trans.sa_session.query( trans.model.Repository ) \ + .filter( trans.model.Repository.table.c.deleted == False ) + else: + # We're in Galaxy. + if order: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) \ + .order_by( trans.model.ToolShedRepository.table.c.name, + trans.model.ToolShedRepository.table.c.owner ) + else: + return trans.sa_session.query( trans.model.ToolShedRepository ) \ + .filter( trans.model.ToolShedRepository.table.c.uninstalled == False ) + def get_repo_info_tuple_contents( repo_info_tuple ): """Take care in handling the repo_info_tuple as it evolves over time as new tool shed features are introduced.""" if len( repo_info_tuple ) == 6: diff --git a/scripts/api/import_workflows_from_installed_tool_shed_repository.py b/scripts/api/import_workflows_from_installed_tool_shed_repository.py index dadfd418f3e..3f02d198109 100644 --- a/scripts/api/import_workflows_from_installed_tool_shed_repository.py +++ b/scripts/api/import_workflows_from_installed_tool_shed_repository.py @@ -45,7 +45,7 @@ def main( options ): data = {} # NOTE: to import a single workflow, add an index to data (e.g., # data[ 'index' ] = 0 - # and change the url to be ~/import_workflow (simgular). For example, + # and change the url to be ~/import_workflow (singular). For example, # url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflow' % str( tool_shed_repository_id ) ) url = '%s%s' % ( base_galaxy_url, '/api/tool_shed_repositories/%s/import_workflows' % str( tool_shed_repository_id ) ) submit( options.api, url, data ) diff --git a/scripts/api/reset_metadata_on_installed_repositories.py b/scripts/api/reset_metadata_on_installed_repositories.py new file mode 100644 index 00000000000..0260d37410d --- /dev/null +++ b/scripts/api/reset_metadata_on_installed_repositories.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python +""" +Script to reset metadata on all Tool Shed repositories installed into a Galaxy instance. The received +API key must be associated with a Galaxy admin user. + +usage: reset_metadata_on_installed_repositories.py key + +Here is a working example of how to use this script. +python ./reset_metadata_on_installed_repositories.py -a 22be3b -u http://localhost:8763/ +""" + +import argparse +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +def main( options ): + api_key = options.api + base_galaxy_url = options.galaxy_url.rstrip( '/' ) + url = '%s/api/tool_shed_repositories/reset_metadata_on_installed_repositories' % base_galaxy_url + submit( options.api, url, {} ) + +if __name__ == '__main__': + parser = argparse.ArgumentParser( description='Reset metadata on all Tool Shed repositories installed into Galaxy via the Galaxy API.' ) + parser.add_argument( "-a", "--api", dest="api", required=True, help="API Key" ) + parser.add_argument( "-u", "--url", dest="galaxy_url", required=True, help="Galaxy URL" ) + options = parser.parse_args() + main( options ) diff --git a/test/tool_shed/base/twilltestcase.py b/test/tool_shed/base/twilltestcase.py index 75c3ad57b43..8bdfad7d6b7 100644 --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -969,7 +969,7 @@ class ShedTwillTestCase( TwillTestCase ): self.check_for_strings( [ 'Metadata has been reset' ] ) def reset_metadata_on_selected_repositories( self, repository_ids ): - self.visit_url( '/admin/reset_metadata_on_selected_repositories_in_tool_shed' ) + self.visit_url( '/admin/reset_metadata_on_selected_repositories' ) kwd = dict( repository_ids=repository_ids ) self.submit_form( form_no=1, button="reset_metadata_on_selected_repositories_button", **kwd )