From f3868d143c4d14fd69485eca208c42c8ac696bc5 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Thu, 21 Mar 2013 19:23:43 -0400 Subject: [PATCH 01/21] Update install and test script to correctly load tool data tables. --- .../functional_tests.py | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index 46a9d386d37..58397d7252a 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -107,12 +107,21 @@ if os.path.exists( 'tool_data_table_conf.test.xml' ): else: additional_tool_data_tables = None additional_tool_data_path = None -# And set up a blank tool_data_table_conf.xml and shed_tool_data_table_conf.xml. + +# Also set up default tool data tables. +if os.path.exists( 'tool_data_table_conf.xml' ): + tool_data_table_conf = 'tool_data_table_conf.xml' +elif os.path.exists( 'tool_data_table_conf.xml.sample' ): + tool_data_table_conf = 'tool_data_table_conf.xml.sample' +else: + tool_data_table_conf = None + +# And set up a blank shed_tool_data_table_conf.xml. tool_data_table_conf_xml_template = ''' ''' - + # The tool shed url and api key must be set for this script to work correctly. Additionally, if the tool shed url does not # point to one of the defaults, the GALAXY_INSTALL_TEST_TOOL_SHEDS_CONF needs to point to a tool sheds configuration file # that contains a definition for that tool shed. @@ -328,7 +337,7 @@ def main(): galaxy_test_proxy_port = None # Set up the configuration files for the Galaxy instance. shed_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF', os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_data_table_conf.xml' ) ) - galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_data_table_conf.xml' ) ) + galaxy_tool_data_table_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF', tool_data_table_conf ) galaxy_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_tool_conf.xml' ) ) galaxy_shed_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_SHED_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_shed_tool_conf.xml' ) ) galaxy_migrated_tool_conf_file = os.environ.get( 'GALAXY_INSTALL_TEST_MIGRATED_TOOL_CONF', os.path.join( galaxy_test_tmp_dir, 'test_migrated_tool_conf.xml' ) ) @@ -350,7 +359,7 @@ def main(): galaxy_file_path = os.path.join( galaxy_db_path, 'files' ) new_repos_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) galaxy_tempfiles = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) - galaxy_shed_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) + galaxy_shed_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir, prefix='shed_tools' ) galaxy_migrated_tool_path = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) galaxy_tool_dependency_dir = tempfile.mkdtemp( dir=galaxy_test_tmp_dir ) os.environ[ 'GALAXY_INSTALL_TEST_TOOL_DEPENDENCY_DIR' ] = galaxy_tool_dependency_dir @@ -367,9 +376,6 @@ def main(): print "Database connection: ", database_connection - # Generate the tool_data_table_conf.xml file. - file( galaxy_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template ) - os.environ[ 'GALAXY_INSTALL_TEST_TOOL_DATA_TABLE_CONF' ] = galaxy_tool_data_table_conf_file # Generate the shed_tool_data_table_conf.xml file. file( shed_tool_data_table_conf_file, 'w' ).write( tool_data_table_conf_xml_template ) os.environ[ 'GALAXY_INSTALL_TEST_SHED_TOOL_DATA_TABLE_CONF' ] = shed_tool_data_table_conf_file From 9dee845318acf2faf57bd731315d04f198e80aab Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Fri, 22 Mar 2013 10:43:58 -0400 Subject: [PATCH 02/21] Eliminate unnecessary resource mappers and api classes in the tool shed api, clean up the remaining tool shed api classes, and enhance the tool shed api by adding the ability to retrieve information about a specified repository and repository revision. --- .../webapps/tool_shed/api/repositories.py | 95 ++++++++++++++++--- .../tool_shed/api/repository_contents.py | 33 ------- .../api/repository_revision_contents.py | 39 -------- .../tool_shed/api/repository_revisions.py | 26 ++--- lib/galaxy/webapps/tool_shed/buildapp.py | 28 +++--- .../tool_shed_repository_revision_update.py | 37 -------- 6 files changed, 110 insertions(+), 148 deletions(-) delete mode 100644 lib/galaxy/webapps/tool_shed/api/repository_contents.py delete mode 100644 lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py delete mode 100755 scripts/api/tool_shed_repository_revision_update.py diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index efd31a98640..e8d4acc64f8 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -1,10 +1,30 @@ import logging +from galaxy.web.framework.helpers import time_ago import tool_shed.util.shed_util_common as suc -from galaxy import web, util +from galaxy import web +from galaxy import util from galaxy.web.base.controller import BaseAPIController +from galaxy import eggs +import pkg_resources + +pkg_resources.require( 'mercurial' ) +from mercurial import hg, ui, commands + log = logging.getLogger( __name__ ) +def default_repository_value_mapper( trans, repository ): + value_mapper={ 'id' : trans.security.encode_id( repository.id ), + 'user_id' : trans.security.encode_id( repository.user_id ) } + return value_mapper + +def default_repository_metadata_value_mapper( trans, repository_metadata ): + value_mapper = { 'id' : trans.security.encode_id( repository_metadata.id ), + 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) } + if repository_metadata.time_last_tested: + value_mapper[ 'time_last_tested' ] = time_ago( repository_metadata.time_last_tested ) + return value_mapper + class RepositoriesController( BaseAPIController ): """RESTful controller for interactions with repositories in the Tool Shed.""" @@ -15,6 +35,7 @@ class RepositoriesController( BaseAPIController ): GET /api/repositories Displays a collection (list) of repositories. """ + # Example URL: http://localhost:9009/api/repositories repository_dicts = [] deleted = util.string_as_bool( deleted ) try: @@ -23,12 +44,10 @@ class RepositoriesController( BaseAPIController ): .order_by( trans.app.model.Repository.table.c.name ) \ .all() for repository in query: - value_mapper={ 'id' : trans.security.encode_id( repository.id ), - 'user_id' : trans.security.encode_id( repository.user_id ) } - repository_dict = repository.get_api_value( view='collection', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repository_contents', - action='index', - repository_id=trans.security.encode_id( repository.id ) ) + repository_dict = repository.get_api_value( view='collection', value_mapper=default_repository_value_mapper( trans, repository ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) repository_dicts.append( repository_dict ) return repository_dicts except Exception, e: @@ -41,21 +60,67 @@ class RepositoriesController( BaseAPIController ): def show( self, trans, id, **kwd ): """ GET /api/repositories/{encoded_repository_id} - Displays information about a repository in the Tool Shed. + Returns information about a repository in the Tool Shed. - :param id: the encoded id of the `Repository` object + :param id: the encoded id of the Repository object """ + # Example URL: http://localhost:9009/api/repositories/f9cad7b01a472135 try: repository = suc.get_repository_in_tool_shed( trans, id ) - value_mapper={ 'id' : trans.security.encode_id( repository.id ), - 'user_id' : trans.security.encode_id( repository.user_id ) } - repository_dict = repository.get_api_value( view='element', value_mapper=value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repository_contents', - action='index', - repository_id=trans.security.encode_id( repository.id ) ) + repository_dict = repository.get_api_value( view='element', value_mapper=default_repository_value_mapper( trans, repository ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=trans.security.encode_id( repository.id ) ) return repository_dict except Exception, e: message = "Error in the Tool Shed repositories API in show: %s" % str( e ) log.error( message, exc_info=True ) trans.response.status = 500 return message + + @web.expose_api + def get_repository_and_revision( self, trans, name, owner, changeset_revision, **kwd ): + """ + GET /api/repository/get_repository_and_revision + Returns information about a repository revision in the Tool Shed. + + :param name: the name of the Repository object + :param owner: the owner of the Repository object + :param changset_revision: the changset_revision of the RepositoryMetadata object associated with the Repository object + """ + # Example URL: http://localhost:9009/api/repositories/get_repository_and_revision?name=add_column&owner=test&changeset_revision=3a08cc21466f + try: + # Get the repository information. + repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) + encoded_repository_id = trans.security.encode_id( repository.id ) + repository_dict = repository.get_api_value( view='element', value_mapper=default_repository_value_mapper( trans, repository ) ) + repository_dict[ 'url' ] = web.url_for( controller='repositories', + action='show', + id=encoded_repository_id ) + # Get the repository_metadata information. + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, changeset_revision ) + if not repository_metadata: + # The changeset_revision column in the repository_metadata table has been updated with a new value value, so find the + # changeset_revision to which we need to update. + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, new_changeset_revision ) + if repository_metadata: + encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) + repository_metadata_dict = repository_metadata.get_api_value( view='collection', + value_mapper=default_repository_metadata_value_mapper( trans, repository_metadata ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=encoded_repository_metadata_id ) + return repository_dict, repository_metadata_dict + else: + message = "Unable to locate repository_metadata record for repository id %d and changeset_revision %s" % ( repository.id, changeset_revision ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return repository_dict, {} + except Exception, e: + message = "Error in the Tool Shed repositories API in get_repository_and_revision: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message diff --git a/lib/galaxy/webapps/tool_shed/api/repository_contents.py b/lib/galaxy/webapps/tool_shed/api/repository_contents.py deleted file mode 100644 index 339a44fa43c..00000000000 --- a/lib/galaxy/webapps/tool_shed/api/repository_contents.py +++ /dev/null @@ -1,33 +0,0 @@ -import logging -import tool_shed.util.shed_util_common as suc -from galaxy import web -from galaxy.web.base.controller import BaseAPIController - -log = logging.getLogger( __name__ ) - - -class RepositoryContentsController( BaseAPIController ): - - @web.expose_api - def index( self, trans, **kwd ): - """ - GET /api/repositories/{encoded_repository_id} - Displays a collection (dictionary) of repository contents. - - :param repository_id: the encoded id of the `Repository` object - """ - try: - repository_id = kwd[ 'repository_id' ] - repository = suc.get_repository_in_tool_shed( trans, repository_id ) - value_mapper={ 'id' : trans.security.encode_id( repository.id ), - 'user_id' : trans.security.encode_id( repository.user_id ) } - repository_dict = repository.as_dict( value_mapper ) - repository_dict[ 'url' ] = web.url_for( controller='repository_contents', - action='index', - repository_id=repository_id ) - return repository_dict - except Exception, e: - message = "Error in the Tool Shed repository_contents API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message diff --git a/lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py b/lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py deleted file mode 100644 index 1249d32859d..00000000000 --- a/lib/galaxy/webapps/tool_shed/api/repository_revision_contents.py +++ /dev/null @@ -1,39 +0,0 @@ -import logging -from galaxy import web -from galaxy.web.framework.helpers import time_ago -from tool_shed.util import metadata_util -from galaxy.web.base.controller import BaseAPIController - -log = logging.getLogger( __name__ ) - -def default_value_mapper( trans, repository_metadata ): - value_mapper = { 'id' : trans.security.encode_id( repository_metadata.id ), - 'repository_id' : trans.security.encode_id( repository_metadata.repository_id ) } - if repository_metadata.time_last_tested: - value_mapper[ 'time_last_tested' ] = time_ago( repository_metadata.time_last_tested ) - return value_mapper - - -class RepositoryRevisionContentsController( BaseAPIController ): - - @web.expose_api - def index( self, trans, **kwd ): - """ - GET /api/repository_revisions/{encoded_repository_metadata_id} - Displays a collection (dictionary) of repository_metadata contents. - - :param repository_metadata_id: the encoded id of the `RepositoryMetadata` object - """ - try: - repository_metadata_id = kwd.get( 'repository_metadata_id', None ) - repository_metadata = metadata_util.get_repository_metadata_by_id( trans, repository_metadata_id ) - repository_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) ) - repository_dict[ 'url' ] = web.url_for( controller='repository_revision_contents', - action='index', - repository_metadata_id=repository_metadata_id ) - return repository_dict - except Exception, e: - message = "Error in the Tool Shed repository_revision_contents API in index: %s" % str( e ) - log.error( message, exc_info=True ) - trans.response.status = 500 - return message diff --git a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py index b4a2758f914..fb751446a7a 100644 --- a/lib/galaxy/webapps/tool_shed/api/repository_revisions.py +++ b/lib/galaxy/webapps/tool_shed/api/repository_revisions.py @@ -1,7 +1,9 @@ -import datetime, logging +import datetime +import logging from galaxy.web.framework.helpers import time_ago from tool_shed.util import metadata_util -from galaxy import web, util +from galaxy import web +from galaxy import util from galaxy.model.orm import and_ from galaxy.web.base.controller import BaseAPIController @@ -24,6 +26,7 @@ class RepositoryRevisionsController( BaseAPIController ): GET /api/repository_revisions Displays a collection (list) of repository revisions. """ + # Example URL: http://localhost:9009/api/repository_revisions repository_metadata_dicts = [] # Build up an anded clause list of filters. clause_list = [] @@ -55,9 +58,9 @@ class RepositoryRevisionsController( BaseAPIController ): for repository_metadata in query: repository_metadata_dict = repository_metadata.get_api_value( view='collection', value_mapper=default_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revision_contents', - action='index', - repository_metadata_id=trans.security.encode_id( repository_metadata.id ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) repository_metadata_dicts.append( repository_metadata_dict ) return repository_metadata_dicts except Exception, e: @@ -74,12 +77,13 @@ class RepositoryRevisionsController( BaseAPIController ): :param id: the encoded id of the `RepositoryMetadata` object """ + # Example URL: http://localhost:9009/api/repository_revisions/bb125606ff9ea620 try: repository_metadata = metadata_util.get_repository_metadata_by_id( trans, id ) repository_metadata_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revision_contents', - action='index', - repository_metadata_id=trans.security.encode_id( repository_metadata.id ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) return repository_metadata_dict except Exception, e: message = "Error in the Tool Shed repository_revisions API in show: %s" % str( e ) @@ -114,7 +118,7 @@ class RepositoryRevisionsController( BaseAPIController ): trans.response.status = 500 return message repository_metadata_dict = repository_metadata.as_dict( value_mapper=default_value_mapper( trans, repository_metadata ) ) - repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revision_contents', - action='index', - repository_metadata_id=trans.security.encode_id( repository_metadata.id ) ) + repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', + action='show', + id=trans.security.encode_id( repository_metadata.id ) ) return repository_metadata_dict diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index b3241fe234d..a7f5e86131b 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -1,8 +1,11 @@ """ Provides factory methods to assemble the Galaxy web application """ - -import logging, atexit, os, os.path, sys, config +import atexit +import config +import logging +import os +import sys from inspect import isclass @@ -69,22 +72,21 @@ def app_factory( global_conf, **kwargs ): webapp.add_route( '/:controller/:action', action='index' ) webapp.add_route( '/:action', controller='repository', action='index' ) webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' ) - # Add the web API. + # Add the web API. # A good resource for RESTful services - http://routes.readthedocs.org/en/latest/restful.html webapp.add_api_controllers( 'galaxy.webapps.tool_shed.api', app ) - webapp.api_mapper.resource( 'content', - 'contents', - controller='repository_contents', + webapp.api_mapper.resource( 'repository', + 'repositories', + controller='repositories', + collection={ 'get_repository_and_revision' : 'GET' }, name_prefix='repository_', - path_prefix='/api/repositories/:repository_id', + path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) - webapp.api_mapper.resource( 'content', - 'contents', - controller='repository_revision_contents', + webapp.api_mapper.resource( 'repository_revision', + 'repository_revisions', + controller='repository_revisions', name_prefix='repository_revision_', - path_prefix='/api/repository_revisions/:repository_metadata_id', + path_prefix='/api', parent_resources=dict( member_name='repository_revision', collection_name='repository_revisions' ) ) - webapp.api_mapper.resource( 'repository', 'repositories', path_prefix='/api' ) - webapp.api_mapper.resource( 'repository_revision', 'repository_revisions', path_prefix='/api' ) webapp.finalize_config() # Wrap the webapp in some useful middleware if kwargs.get( 'middleware', True ): diff --git a/scripts/api/tool_shed_repository_revision_update.py b/scripts/api/tool_shed_repository_revision_update.py deleted file mode 100755 index a073de02533..00000000000 --- a/scripts/api/tool_shed_repository_revision_update.py +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env python -""" -PUT/update script to update appropriate values in a repository_metadata table record in the Tool Shed. - -usage: tool_shed_repository_revision_update.py key url key1=value1 key2=value2 ... -""" - -import os, sys -sys.path.insert( 0, os.path.dirname( __file__ ) ) -from common import update - -import pkg_resources -pkg_resources.require( "simplejson" ) - -import simplejson - -to_json_string = simplejson.dumps -from_json_string = simplejson.loads - -data = {} -for key, value in [ kwarg.split( '=', 1 ) for kwarg in sys.argv[ 3: ] ]: - """ - This example script will properly handle updating the value of one or more of the following RepositoryMetadata attributes: - tools_functionally_correct, do_not_test, tool_test_errors - """ - if key in [ 'tools_functionally_correct', 'do_not_test' ]: - if str( value ).lower() in [ 'true', 'yes', 'on' ]: - new_value = True - else: - new_value = False - elif key in [ 'tool_test_errors' ]: - new_value = from_json_string( value ) - else: - new_value = str( value ) - data[ key ] = new_value - -update( sys.argv[ 1 ], sys.argv[ 2 ], data ) From 98a51470f23a2dcabade036ffc3e6c7f71ccaa72 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 22 Mar 2013 11:43:42 -0400 Subject: [PATCH 03/21] Allow old-style display applications to be disabled. --- lib/galaxy/config.py | 1 + lib/galaxy/webapps/galaxy/controllers/dataset.py | 2 ++ universe_wsgi.ini.sample | 10 ++++++++++ 3 files changed, 13 insertions(+) diff --git a/lib/galaxy/config.py b/lib/galaxy/config.py index bc810670d5a..c5f35983246 100644 --- a/lib/galaxy/config.py +++ b/lib/galaxy/config.py @@ -151,6 +151,7 @@ class Configuration( object ): self.log_actions = string_as_bool( kwargs.get( 'log_actions', 'False' ) ) self.log_events = string_as_bool( kwargs.get( 'log_events', 'False' ) ) self.sanitize_all_html = string_as_bool( kwargs.get( 'sanitize_all_html', True ) ) + self.enable_old_display_applications = string_as_bool( kwargs.get( "enable_old_display_applications", "True" ) ) self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea,ucla" ).lower().split(",") self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "modencode,sgd_yeast,tair,wormbase,wormbase_ws120,wormbase_ws140,wormbase_ws170,wormbase_ws180,wormbase_ws190,wormbase_ws200,wormbase_ws204,wormbase_ws210,wormbase_ws220,wormbase_ws225" ).lower().split(",") self.brand = kwargs.get( 'brand', None ) diff --git a/lib/galaxy/webapps/galaxy/controllers/dataset.py b/lib/galaxy/webapps/galaxy/controllers/dataset.py index feed4a4c643..5f6c9235792 100644 --- a/lib/galaxy/webapps/galaxy/controllers/dataset.py +++ b/lib/galaxy/webapps/galaxy/controllers/dataset.py @@ -697,6 +697,8 @@ class DatasetInterface( BaseUIController, UsesAnnotations, UsesHistoryMixin, Use @web.expose def display_at( self, trans, dataset_id, filename=None, **kwd ): """Sets up a dataset permissions so it is viewable at an external site""" + if not trans.app.config.enable_old_display_applications: + return trans.show_error_message( "This method of accessing external display applications has been disabled by a Galaxy administrator." ) site = filename data = trans.sa_session.query( trans.app.model.HistoryDatasetAssociation ).get( dataset_id ) if not data: diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 186db377787..d474d526c5f 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -247,6 +247,16 @@ paste.app_factory = galaxy.web.buildapp:app_factory # them). #display_servers = hgw1.cse.ucsc.edu,hgw2.cse.ucsc.edu,hgw3.cse.ucsc.edu,hgw4.cse.ucsc.edu,hgw5.cse.ucsc.edu,hgw6.cse.ucsc.edu,hgw7.cse.ucsc.edu,hgw8.cse.ucsc.edu,lowepub.cse.ucsc.edu +# To disable the old-style display applications that are hardcoded into datatype classes, +# set enable_old_display_applications = False. +# This may be desirable due to using the new-style, XML-defined, display applications that +# have been defined for many of the datatypes that have the old-style. +# There is also a potential security concern with the old-style applications, +# where a malicious party could provide a link that appears to reference the Galaxy server, +# but contains a redirect to a third-party server, tricking a Galaxy user to access said +# site. +#enable_old_display_applications = True + # -- Next gen LIMS interface on top of existing Galaxy Sample/Request management code. use_nglims = False From 09c768293bf90196a21d2db97b28e158fcebdb30 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 22 Mar 2013 12:23:05 -0400 Subject: [PATCH 04/21] Do not display old-style display links in server-side history when they are disabled. --- lib/galaxy/datatypes/data.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/data.py b/lib/galaxy/datatypes/data.py index 3f9c4d74b09..acdb438173d 100644 --- a/lib/galaxy/datatypes/data.py +++ b/lib/galaxy/datatypes/data.py @@ -448,12 +448,12 @@ class Data( object ): apply anyway. """ try: - if type in self.get_display_types(): + if app.config.enable_old_display_applications and type in self.get_display_types(): return target_frame, getattr ( self, self.supported_display_apps[type]['links_function'] ) ( dataset, type, app, base_url, **kwd ) except: log.exception( 'Function %s is referred to in datatype %s for generating links for type %s, but is not accessible' \ % ( self.supported_display_apps[type]['links_function'], self.__class__.__name__, type ) ) - return [] + return target_frame, [] def get_converter_types(self, original_dataset, datatypes_registry): """Returns available converters by type for this dataset""" return datatypes_registry.get_converters_by_datatype(original_dataset.ext) From 81a246eebcdc1ba94f883e5921305c0cd913429d Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 22 Mar 2013 12:26:03 -0400 Subject: [PATCH 05/21] Do not display old-style display links in client-side history when they are disabled. --- lib/galaxy/web/base/controller.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/galaxy/web/base/controller.py b/lib/galaxy/web/base/controller.py index f95d7677c0b..253560deb2b 100644 --- a/lib/galaxy/web/base/controller.py +++ b/lib/galaxy/web/base/controller.py @@ -394,6 +394,8 @@ class UsesHistoryDatasetAssociationMixin: def get_old_display_applications( self, trans, hda ): display_apps = [] + if not trans.app.config.enable_old_display_applications: + return display_apps for display_app_name in hda.datatype.get_display_types(): link_generator = get_display_app_link_generator( display_app_name ) display_links = link_generator.generate_links( trans, hda ) From 0224858da8666ccca0612fa7d2b5268383cdb3b6 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 22 Mar 2013 12:33:10 -0400 Subject: [PATCH 06/21] Add some comments to the new-style display applications module. --- lib/galaxy/datatypes/display_applications/__init__.py | 4 +++- .../datatypes/display_applications/link_generator.py | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/datatypes/display_applications/__init__.py b/lib/galaxy/datatypes/display_applications/__init__.py index 8b137891791..61eaed1ef56 100644 --- a/lib/galaxy/datatypes/display_applications/__init__.py +++ b/lib/galaxy/datatypes/display_applications/__init__.py @@ -1 +1,3 @@ - +""" +Contains functionality of the newer XML defined external display applications (not hardcoded into datatype classes). +""" diff --git a/lib/galaxy/datatypes/display_applications/link_generator.py b/lib/galaxy/datatypes/display_applications/link_generator.py index 3641a59af53..1499042f756 100644 --- a/lib/galaxy/datatypes/display_applications/link_generator.py +++ b/lib/galaxy/datatypes/display_applications/link_generator.py @@ -1,8 +1,11 @@ -"""Classes to generate links for display applications. +"""Classes to generate links for old-style display applications. Separating Transaction based elements of display applications from datatypes. """ +#FIXME: The code contained within this file is for old-style display applications, but +#this module namespace is intended to only handle the new-style display applications. + import urllib # for the url_for hack @@ -19,6 +22,8 @@ from galaxy.datatypes.interval import Interval, Gff, Wiggle, CustomTrack #TODO: these could be extended to handle file_function and parse/contain the builds.txt files +#HACK: these duplicate functionality from the individual datatype classes themselves + def get_display_app_link_generator( display_app_name ): """Returns an instance of the proper link generator class based on the display_app_name or DisplayAppLinkGenerator From 9a48465e01d8a48080a516cece17f74d56b82995 Mon Sep 17 00:00:00 2001 From: Daniel Blankenberg Date: Fri, 22 Mar 2013 14:23:24 -0400 Subject: [PATCH 07/21] Fix for average_rating in repository_review. --- lib/galaxy/webapps/tool_shed/controllers/repository_review.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py index 2e0702b373b..a9b1e633e3a 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository_review.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository_review.py @@ -320,7 +320,8 @@ class RepositoryReviewController( BaseUIController, common_util.ItemRatings ): trans.model.ComponentReview.table.c.deleted == False, trans.model.ComponentReview.table.c.approved != trans.model.ComponentReview.approved_states.NA ) ) \ .scalar() - review.rating = int( average_rating ) + if average_rating is not None: + review.rating = int( average_rating ) trans.sa_session.add( review ) trans.sa_session.flush() # Update the information in components_dict. From fe410b4e093b5a8254dd0e43bf741c9c0a950fee Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Fri, 22 Mar 2013 14:38:01 -0400 Subject: [PATCH 08/21] Extract several methods from Galaxy's admin_toolshed controller and the tool shed's repository controller and move them to ~/tool_shed/galaxy_install/repository_util in preparation for enhancing the Galaxy api to enable installing a repository from the tool shed. --- lib/galaxy/model/__init__.py | 12 +- .../galaxy/controllers/admin_toolshed.py | 197 +++----------- .../tool_shed/controllers/repository.py | 77 +++--- .../webapps/tool_shed/model/__init__.py | 4 +- .../galaxy_install/repository_util.py | 240 +++++++++++++++++- lib/tool_shed/util/shed_util_common.py | 18 +- 6 files changed, 332 insertions(+), 216 deletions(-) diff --git a/lib/galaxy/model/__init__.py b/lib/galaxy/model/__init__.py index 9dd57bf5347..f411af4e8cd 100644 --- a/lib/galaxy/model/__init__.py +++ b/lib/galaxy/model/__init__.py @@ -3040,9 +3040,9 @@ class APIKeys( object ): pass class ToolShedRepository( object ): - api_collection_visible_keys = ( 'id', 'name', 'tool_shed', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', + api_collection_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', 'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) - api_element_visible_keys = ( 'id', 'name', 'tool_shed', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', + api_element_visible_keys = ( 'id', 'tool_shed', 'name', 'owner', 'installed_changeset_revision', 'changeset_revision', 'ctx_rev', 'includes_datatypes', 'update_available', 'deleted', 'uninstalled', 'dist_to_shed', 'status', 'error_message' ) installation_status = Bunch( NEW='New', CLONING='Cloning', @@ -3079,10 +3079,8 @@ class ToolShedRepository( object ): self.dist_to_shed = dist_to_shed self.status = status self.error_message = error_message - def as_dict( self, trans ): - tsr_dict = self.get_api_value( view='element' ) - tsr_dict[ 'id' ] = trans.security.encode_id( self.id ) - return tsr_dict + def as_dict( self, value_mapper=None ): + return self.get_api_value( view='element', value_mapper=value_mapper ) def repo_files_directory( self, app ): repo_path = self.repo_path( app ) if repo_path: @@ -3182,7 +3180,7 @@ class ToolShedRepository( object ): try: rval[ key ] = self.__getattribute__( key ) if key in value_mapper: - rval[ key ] = value_mapper.get( key )( rval[ key ] ) + rval[ key ] = value_mapper.get( key, rval[ key ] ) except AttributeError: rval[ key ] = None return rval diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 14729b82072..3035ef238fa 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -373,91 +373,6 @@ class AdminToolshed( AdminGalaxy ): tool_version = tool_util.get_tool_version( app, guid ) return tool_version.get_version_ids( app, reverse=True ) - def handle_repository_contents( self, trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, - tool_section=None, shed_tool_conf=None, reinstalling=False ): - """ - Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy (never the tool shed) - when an admin is installing a new repository or reinstalling an uninstalled repository. - """ - shed_config_dict = trans.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) - metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, - repository=tool_shed_repository, - changeset_revision=tool_shed_repository.changeset_revision, - repository_clone_url=repository_clone_url, - shed_config_dict=shed_config_dict, - relative_install_dir=relative_install_dir, - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=True ) - tool_shed_repository.metadata = metadata_dict - trans.sa_session.add( tool_shed_repository ) - trans.sa_session.flush() - if 'tool_dependencies' in metadata_dict and not reinstalling: - tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True ) - if 'tools' in metadata_dict: - tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) - sample_files = metadata_dict.get( 'sample_files', [] ) - tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) - tool_util.copy_sample_files( trans.app, tool_index_sample_files, tool_path=tool_path ) - sample_files_copied = [ str( s ) for s in tool_index_sample_files ] - repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) - if repository_tools_tups: - # Handle missing data table entries for tool parameters that are dynamically generated select lists. - repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups ) - # Handle missing index files for tool parameters that are dynamically generated select lists. - repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( trans.app, - tool_path, - sample_files, - repository_tools_tups, - sample_files_copied ) - # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. - tool_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) - tool_util.add_to_tool_panel( app=trans.app, - repository_name=tool_shed_repository.name, - repository_clone_url=repository_clone_url, - changeset_revision=tool_shed_repository.installed_changeset_revision, - repository_tools_tups=repository_tools_tups, - owner=tool_shed_repository.owner, - shed_tool_conf=shed_tool_conf, - tool_panel_dict=tool_panel_dict, - new_install=True ) - if 'data_manager' in metadata_dict: - new_data_managers = data_manager_util.install_data_managers( trans.app, - trans.app.config.shed_data_manager_config_file, - metadata_dict, - shed_config_dict, - relative_install_dir, - tool_shed_repository, - repository_tools_tups ) - if 'datatypes' in metadata_dict: - tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES - if not tool_shed_repository.includes_datatypes: - tool_shed_repository.includes_datatypes = True - trans.sa_session.add( tool_shed_repository ) - trans.sa_session.flush() - files_dir = relative_install_dir - if shed_config_dict.get( 'tool_path' ): - files_dir = os.path.join( shed_config_dict['tool_path'], files_dir ) - datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir ) - # Load data types required by tools. - converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) - if converter_path or display_path: - # Create a dictionary of tool shed repository related information. - repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, - name=tool_shed_repository.name, - owner=tool_shed_repository.owner, - installed_changeset_revision=tool_shed_repository.installed_changeset_revision, - tool_dicts=metadata_dict.get( 'tools', [] ), - converter_path=converter_path, - display_path=display_path ) - if converter_path: - # Load proprietary datatype converters - trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) - if display_path: - # Load proprietary datatype display applications - trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) - @web.expose @web.require_admin def import_workflow( self, trans, workflow_name, repository_id, **kwd ): @@ -638,15 +553,15 @@ class AdminToolshed( AdminGalaxy ): repo = hg.repository( suc.get_configured_ui(), path=os.path.abspath( install_dir ) ) repository_util.pull_repository( repo, repository_clone_url, current_changeset_revision ) suc.update_repository( repo, ctx_rev=current_ctx_rev ) - self.handle_repository_contents( trans, - tool_shed_repository=tool_shed_repository, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - tool_shed=tool_shed_repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - reinstalling=reinstalling ) + repository_util.handle_repository_contents( trans, + tool_shed_repository=tool_shed_repository, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + tool_shed=tool_shed_repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + reinstalling=reinstalling ) trans.sa_session.refresh( tool_shed_repository ) metadata = tool_shed_repository.metadata if 'tools' in metadata: @@ -956,7 +871,7 @@ class AdminToolshed( AdminGalaxy ): install_tool_dependencies = kwd.get( 'install_tool_dependencies', '' ) encoded_repo_info_dicts = util.listify( kwd.get( 'encoded_repo_info_dicts', None ) ) if not encoded_repo_info_dicts: - # The request originated in the tool shed. + # The request originated in the tool shed via a tool search. repository_ids = kwd.get( 'repository_ids', None ) changeset_revisions = kwd.get( 'changeset_revisions', None ) # Get the information necessary to install each repository. @@ -981,78 +896,40 @@ class AdminToolshed( AdminGalaxy ): else: install_tool_dependencies = False tool_path = suc.get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf ) + installation_dict = dict( install_repository_dependencies=install_repository_dependencies, + new_tool_panel_section=new_tool_panel_section, + no_changes_checked=False, + reinstalling=False, + repo_info_dicts=repo_info_dicts, + tool_panel_section=tool_panel_section, + tool_path=tool_path, + tool_shed_url=tool_shed_url ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \ - repository_dependency_util.create_repository_dependency_objects( trans, - tool_path, - tool_shed_url, - repo_info_dicts, - reinstalling=False, - install_repository_dependencies=install_repository_dependencies, - no_changes_checked=False, - tool_panel_section=tool_panel_section, - new_tool_panel_section=new_tool_panel_section ) + repository_util.handle_tool_shed_repositories( trans, installation_dict ) if message and len( repo_info_dicts ) == 1: - installed_tool_shed_repository = created_or_updated_tool_shed_repositories[ 0 ] - message+= 'Click here to manage the repository. ' % \ - ( web.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( installed_tool_shed_repository.id ) ) ) return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='browse_repositories', message=message, status='error' ) ) + if created_or_updated_tool_shed_repositories: - # Handle contained tools. - if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section or tool_panel_section ): - if new_tool_panel_section: - section_id = new_tool_panel_section.lower().replace( ' ', '_' ) - tool_panel_section_key = 'section_%s' % str( section_id ) - if tool_panel_section_key in trans.app.toolbox.tool_panel: - # Appending a tool to an existing section in trans.app.toolbox.tool_panel - log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) - tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] - else: - # Appending a new section to trans.app.toolbox.tool_panel - log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) - elem = Element( 'section' ) - elem.attrib[ 'name' ] = new_tool_panel_section - elem.attrib[ 'id' ] = section_id - elem.attrib[ 'version' ] = '' - tool_section = tools.ToolSection( elem ) - trans.app.toolbox.tool_panel[ tool_panel_section_key ] = tool_section - else: - tool_panel_section_key = 'section_%s' % tool_panel_section - tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] - else: - tool_panel_section_key = None - tool_section = None - encoded_repository_ids = [ trans.security.encode_id( tsr.id ) for tsr in created_or_updated_tool_shed_repositories ] - # Create a one-to-one mapping of tool shed repository id and tool panel section key. All tools contained in the repositories - # being installed will be loaded into the same section in the tool panel. - for tsr in created_or_updated_tool_shed_repositories: - tool_panel_section_keys.append( tool_panel_section_key ) - new_kwd = dict( includes_tools=includes_tools, - includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, - has_repository_dependencies=has_repository_dependencies, - install_repository_dependencies=install_repository_dependencies, - includes_tool_dependencies=includes_tool_dependencies, - install_tool_dependencies=install_tool_dependencies, - message=message, - repo_info_dicts=filtered_repo_info_dicts, - shed_tool_conf=shed_tool_conf, - status=status, - tool_path=tool_path, - tool_panel_section_keys=tool_panel_section_keys, - tool_shed_repository_ids=encoded_repository_ids, - tool_shed_url=tool_shed_url ) - encoded_kwd = encoding_util.tool_shed_encode( new_kwd ) - tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] - tool_shed_repositories = [] - for tsr_id in tsr_ids: - tsr = trans.sa_session.query( trans.model.ToolShedRepository ).get( tsr_id ) - tool_shed_repositories.append( tsr ) - clause_list = [] - for tsr_id in tsr_ids: - clause_list.append( trans.model.ToolShedRepository.table.c.id == tsr_id ) - query = trans.sa_session.query( trans.model.ToolShedRepository ).filter( or_( *clause_list ) ) + installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, + filtered_repo_info_dicts=filtered_repo_info_dicts, + has_repository_dependencies=has_repository_dependencies, + includes_tool_dependencies=includes_tool_dependencies, + includes_tools=includes_tools, + includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, + install_repository_dependencies=install_repository_dependencies, + install_tool_dependencies=install_tool_dependencies, + message=message, + new_tool_panel_section=new_tool_panel_section, + shed_tool_conf=shed_tool_conf, + status=status, + tool_panel_section=tool_panel_section, + tool_panel_section_keys=tool_panel_section_keys, + tool_path=tool_path, + tool_shed_url=tool_shed_url ) + encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = repository_util.initiate_repository_installation( trans, installation_dict ) return trans.fill_template( 'admin/tool_shed_repository/initiate_repository_installation.mako', encoded_kwd=encoded_kwd, query=query, diff --git a/lib/galaxy/webapps/tool_shed/controllers/repository.py b/lib/galaxy/webapps/tool_shed/controllers/repository.py index 79174183e0c..6c86ba1c90c 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/repository.py +++ b/lib/galaxy/webapps/tool_shed/controllers/repository.py @@ -1,26 +1,47 @@ -import os, logging, re, tempfile, ConfigParser, string -from time import gmtime, strftime +import ConfigParser +import logging +import os +import re +import string +import tempfile +from time import gmtime +from time import strftime from datetime import date, datetime -from galaxy import util, web +from galaxy import util +from galaxy import web from galaxy.util.odict import odict from galaxy.web.base.controller import BaseUIController -from galaxy.web.form_builder import CheckboxField, build_select_field +from galaxy.web.form_builder import CheckboxField +from galaxy.web.form_builder import build_select_field from galaxy.webapps.tool_shed import model from galaxy.webapps.tool_shed.model import directory_hash_id from galaxy.web.framework.helpers import grids from galaxy.util import json -from galaxy.model.orm import and_, or_ +from galaxy.model.orm import and_ +from galaxy.model.orm import or_ import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, metadata_util, readme_util, repository_dependency_util, review_util, tool_dependency_util, tool_util, workflow_util +from tool_shed.util import encoding_util +from tool_shed.util import metadata_util +from tool_shed.util import readme_util +from tool_shed.util import repository_dependency_util +from tool_shed.util import review_util +from tool_shed.util import tool_dependency_util +from tool_shed.util import tool_util +from tool_shed.util import workflow_util from tool_shed.galaxy_install import repository_util -from galaxy.webapps.tool_shed.util import common_util, container_util +from galaxy.webapps.tool_shed.util import common_util +from galaxy.webapps.tool_shed.util import container_util import galaxy.tools import tool_shed.grids.repository_grids as repository_grids import tool_shed.grids.util as grids_util from galaxy import eggs eggs.require('mercurial') -from mercurial import hg, ui, patch, commands + +from mercurial import commands +from mercurial import hg +from mercurial import patch +from mercurial import ui log = logging.getLogger( __name__ ) @@ -1124,35 +1145,17 @@ class RepositoryController( BaseUIController, common_util.ItemRatings ): includes_tool_dependencies = False repo_info_dicts = [] for tup in zip( util.listify( repository_ids ), util.listify( changeset_revisions ) ): - repository_id, changeset_revision = tup - repository = suc.get_repository_in_tool_shed( trans, repository_id ) - repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ) - repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) - metadata = repository_metadata.metadata - if not includes_tools: - if 'tools' in metadata: - includes_tools = True - if not includes_tools_for_display_in_tool_panel: - includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel - if not has_repository_dependencies: - if 'repository_dependencies' in metadata: - has_repository_dependencies = True - if not includes_tool_dependencies: - if 'tool_dependencies' in metadata: - includes_tool_dependencies = True - repo_dir = repository.repo_path( trans.app ) - repo = hg.repository( suc.get_configured_ui(), repo_dir ) - ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) - repo_info_dict = repository_util.create_repo_info_dict( trans=trans, - repository_clone_url=repository_clone_url, - changeset_revision=changeset_revision, - ctx_rev=str( ctx.rev() ), - repository_owner=repository.user.username, - repository_name=repository.name, - repository=repository, - repository_metadata=repository_metadata, - tool_dependencies=None, - repository_dependencies=None ) + repository_id, changeset_revision = tup + repo_info_dict, cur_includes_tools, cur_includes_tool_dependencies, cur_includes_tools_for_display_in_tool_panel, cur_has_repository_dependencies = \ + repository_util.get_repo_info_dict( trans, repository_id, changeset_revision ) + if cur_has_repository_dependencies and not has_repository_dependencies: + has_repository_dependencies = True + if cur_includes_tools and not includes_tools: + includes_tools = True + if cur_includes_tool_dependencies and not includes_tool_dependencies: + includes_tool_dependencies = True + if cur_includes_tools_for_display_in_tool_panel and not includes_tools_for_display_in_tool_panel: + includes_tools_for_display_in_tool_panel = True repo_info_dicts.append( encoding_util.tool_shed_encode( repo_info_dict ) ) return dict( includes_tools=includes_tools, includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, diff --git a/lib/galaxy/webapps/tool_shed/model/__init__.py b/lib/galaxy/webapps/tool_shed/model/__init__.py index 1295506ea94..2680f9c54c7 100644 --- a/lib/galaxy/webapps/tool_shed/model/__init__.py +++ b/lib/galaxy/webapps/tool_shed/model/__init__.py @@ -192,10 +192,10 @@ class Repository( object, APIItem ): class RepositoryMetadata( object, APIItem ): api_collection_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'has_repository_dependencies', 'includes_datatypes', - 'includes_tools', 'includes_tool_dependencies', 'includes_workflows' ) + 'includes_tools', 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) api_element_visible_keys = ( 'id', 'repository_id', 'changeset_revision', 'malicious', 'downloadable', 'tools_functionally_correct', 'do_not_test', 'time_last_tested', 'tool_test_errors', 'has_repository_dependencies', 'includes_datatypes', 'includes_tools', - 'includes_tool_dependencies', 'includes_workflows' ) + 'includes_tool_dependencies', 'includes_tools_for_display_in_tool_panel', 'includes_workflows' ) def __init__( self, id=None, repository_id=None, changeset_revision=None, metadata=None, tool_versions=None, malicious=False, downloadable=False, tools_functionally_correct=False, do_not_test=False, time_last_tested=None, tool_test_errors=None, has_repository_dependencies=False, includes_datatypes=False, includes_tools=False, includes_tool_dependencies=False, includes_workflows=False ): diff --git a/lib/tool_shed/galaxy_install/repository_util.py b/lib/tool_shed/galaxy_install/repository_util.py index 8b9e832fc63..12f4784a186 100644 --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -1,14 +1,31 @@ -import os, logging, threading, urllib2 -from galaxy.web import url_for +import logging +import os +import threading +import urllib2 +from galaxy import tools +from galaxy import web +from galaxy.model.orm import or_ from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, repository_dependency_util, tool_dependency_util, tool_util +from tool_shed.util import data_manager_util +from tool_shed.util import datatype_util +from tool_shed.util import encoding_util +from tool_shed.util import repository_dependency_util +from tool_shed.util import metadata_util +from tool_shed.util import tool_dependency_util +from tool_shed.util import tool_util from galaxy import eggs import pkg_resources pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands +from mercurial import commands +from mercurial import hg +from mercurial import ui + +pkg_resources.require( 'elementtree' ) +from elementtree import ElementTree +from elementtree.ElementTree import Element log = logging.getLogger( __name__ ) @@ -41,7 +58,7 @@ def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_ repository_dependency_util.get_repository_dependencies_for_changeset_revision( trans=trans, repository=repository, repository_metadata=repository_metadata, - toolshed_base_url=str( url_for( '/', qualified=True ) ).rstrip( '/' ), + toolshed_base_url=str( web.url_for( '/', qualified=True ) ).rstrip( '/' ), key_rd_dicts_to_be_processed=None, all_repository_dependencies=None, handled_key_rd_dicts=None, @@ -74,6 +91,39 @@ def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_ tool_dependencies ) return repo_info_dict +def get_repo_info_dict( trans, repository_id, changeset_revision ): + repository = suc.get_repository_in_tool_shed( trans, repository_id ) + repository_clone_url = suc.generate_clone_url_for_repository_in_tool_shed( trans, repository ) + repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, repository_id, changeset_revision ) + metadata = repository_metadata.metadata + if 'tools' in metadata: + includes_tools = True + else: + includes_tools = False + includes_tools_for_display_in_tool_panel = repository_metadata.includes_tools_for_display_in_tool_panel + if 'repository_dependencies' in metadata: + has_repository_dependencies = True + else: + has_repository_dependencies = False + if 'tool_dependencies' in metadata: + includes_tool_dependencies = True + else: + includes_tool_dependencies = False + repo_dir = repository.repo_path( trans.app ) + repo = hg.repository( suc.get_configured_ui(), repo_dir ) + ctx = suc.get_changectx_for_changeset( repo, changeset_revision ) + repo_info_dict = create_repo_info_dict( trans=trans, + repository_clone_url=repository_clone_url, + changeset_revision=changeset_revision, + ctx_rev=str( ctx.rev() ), + repository_owner=repository.user.username, + repository_name=repository.name, + repository=repository, + repository_metadata=repository_metadata, + tool_dependencies=None, + repository_dependencies=None ) + return repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies + def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): """Return the changeset revision hash to which the repository can be updated.""" changeset_revision_dict = {} @@ -117,6 +167,186 @@ def get_update_to_changeset_revision_and_ctx_rev( trans, repository ): changeset_revision_dict[ 'ctx_rev' ] = None return changeset_revision_dict +def handle_repository_contents( trans, tool_shed_repository, tool_path, repository_clone_url, relative_install_dir, tool_shed=None, tool_section=None, shed_tool_conf=None, + reinstalling=False ): + """ + Generate the metadata for the installed tool shed repository, among other things. This method is called from Galaxy (never the tool shed) + when an administrator is installing a new repository or reinstalling an uninstalled repository. + """ + shed_config_dict = trans.app.toolbox.get_shed_config_dict_by_filename( shed_tool_conf ) + metadata_dict, invalid_file_tups = metadata_util.generate_metadata_for_changeset_revision( app=trans.app, + repository=tool_shed_repository, + changeset_revision=tool_shed_repository.changeset_revision, + repository_clone_url=repository_clone_url, + shed_config_dict=shed_config_dict, + relative_install_dir=relative_install_dir, + repository_files_dir=None, + resetting_all_metadata_on_repository=False, + updating_installed_repository=False, + persist=True ) + tool_shed_repository.metadata = metadata_dict + trans.sa_session.add( tool_shed_repository ) + trans.sa_session.flush() + if 'tool_dependencies' in metadata_dict and not reinstalling: + tool_dependencies = tool_dependency_util.create_tool_dependency_objects( trans.app, tool_shed_repository, relative_install_dir, set_status=True ) + if 'tools' in metadata_dict: + tool_panel_dict = tool_util.generate_tool_panel_dict_for_new_install( metadata_dict[ 'tools' ], tool_section ) + sample_files = metadata_dict.get( 'sample_files', [] ) + tool_index_sample_files = tool_util.get_tool_index_sample_files( sample_files ) + tool_util.copy_sample_files( trans.app, tool_index_sample_files, tool_path=tool_path ) + sample_files_copied = [ str( s ) for s in tool_index_sample_files ] + repository_tools_tups = suc.get_repository_tools_tups( trans.app, metadata_dict ) + if repository_tools_tups: + # Handle missing data table entries for tool parameters that are dynamically generated select lists. + repository_tools_tups = tool_util.handle_missing_data_table_entry( trans.app, relative_install_dir, tool_path, repository_tools_tups ) + # Handle missing index files for tool parameters that are dynamically generated select lists. + repository_tools_tups, sample_files_copied = tool_util.handle_missing_index_file( trans.app, tool_path, sample_files, repository_tools_tups, sample_files_copied ) + # Copy remaining sample files included in the repository to the ~/tool-data directory of the local Galaxy instance. + tool_util.copy_sample_files( trans.app, sample_files, tool_path=tool_path, sample_files_copied=sample_files_copied ) + tool_util.add_to_tool_panel( app=trans.app, + repository_name=tool_shed_repository.name, + repository_clone_url=repository_clone_url, + changeset_revision=tool_shed_repository.installed_changeset_revision, + repository_tools_tups=repository_tools_tups, + owner=tool_shed_repository.owner, + shed_tool_conf=shed_tool_conf, + tool_panel_dict=tool_panel_dict, + new_install=True ) + if 'data_manager' in metadata_dict: + new_data_managers = data_manager_util.install_data_managers( trans.app, + trans.app.config.shed_data_manager_config_file, + metadata_dict, + shed_config_dict, + relative_install_dir, + tool_shed_repository, + repository_tools_tups ) + if 'datatypes' in metadata_dict: + tool_shed_repository.status = trans.model.ToolShedRepository.installation_status.LOADING_PROPRIETARY_DATATYPES + if not tool_shed_repository.includes_datatypes: + tool_shed_repository.includes_datatypes = True + trans.sa_session.add( tool_shed_repository ) + trans.sa_session.flush() + files_dir = relative_install_dir + if shed_config_dict.get( 'tool_path' ): + files_dir = os.path.join( shed_config_dict[ 'tool_path' ], files_dir ) + datatypes_config = suc.get_config_from_disk( 'datatypes_conf.xml', files_dir ) + # Load data types required by tools. + converter_path, display_path = datatype_util.alter_config_and_load_prorietary_datatypes( trans.app, datatypes_config, files_dir, override=False ) + if converter_path or display_path: + # Create a dictionary of tool shed repository related information. + repository_dict = datatype_util.create_repository_dict_for_proprietary_datatypes( tool_shed=tool_shed, + name=tool_shed_repository.name, + owner=tool_shed_repository.owner, + installed_changeset_revision=tool_shed_repository.installed_changeset_revision, + tool_dicts=metadata_dict.get( 'tools', [] ), + converter_path=converter_path, + display_path=display_path ) + if converter_path: + # Load proprietary datatype converters + trans.app.datatypes_registry.load_datatype_converters( trans.app.toolbox, installed_repository_dict=repository_dict ) + if display_path: + # Load proprietary datatype display applications + trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) + +def handle_tool_shed_repositories( trans, installation_dict ): + # The following installation_dict entries are all required. + install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] + new_tool_panel_section = installation_dict[ 'new_tool_panel_section' ] + no_changes_checked = installation_dict[ 'no_changes_checked' ] + reinstalling = installation_dict[ 'reinstalling' ] + repo_info_dicts = installation_dict[ 'repo_info_dicts' ] + tool_panel_section = installation_dict[ 'tool_panel_section' ] + tool_path = installation_dict[ 'tool_path' ] + tool_shed_url = installation_dict[ 'tool_shed_url' ] + created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \ + repository_dependency_util.create_repository_dependency_objects( trans=trans, + tool_path=tool_path, + tool_shed_url=tool_shed_url, + repo_info_dicts=repo_info_dicts, + reinstalling=reinstalling, + install_repository_dependencies=install_repository_dependencies, + no_changes_checked=no_changes_checked, + tool_panel_section=tool_panel_section, + new_tool_panel_section=new_tool_panel_section ) + if message and len( repo_info_dicts ) == 1: + installed_tool_shed_repository = created_or_updated_tool_shed_repositories[ 0 ] + message += 'Click here to manage the repository. ' % \ + ( web.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( installed_tool_shed_repository.id ) ) ) + return created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message + +def initiate_repository_installation( trans, installation_dict ): + # The following installation_dict entries are all required. + created_or_updated_tool_shed_repositories = installation_dict[ 'created_or_updated_tool_shed_repositories' ] + filtered_repo_info_dicts = installation_dict[ 'filtered_repo_info_dicts' ] + has_repository_dependencies = installation_dict[ 'has_repository_dependencies' ] + includes_tool_dependencies = installation_dict[ 'includes_tool_dependencies' ] + includes_tools = installation_dict[ 'includes_tools' ] + includes_tools_for_display_in_tool_panel = installation_dict[ 'includes_tools_for_display_in_tool_panel' ] + install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] + install_tool_dependencies = installation_dict[ 'install_tool_dependencies' ] + message = installation_dict[ 'message' ] + new_tool_panel_section = installation_dict[ 'new_tool_panel_section' ] + shed_tool_conf = installation_dict[ 'shed_tool_conf' ] + status = installation_dict[ 'status' ] + tool_panel_section = installation_dict[ 'tool_panel_section' ] + tool_panel_section_keys = installation_dict[ 'tool_panel_section_keys' ] + tool_path = installation_dict[ 'tool_path' ] + tool_shed_url = installation_dict[ 'tool_shed_url' ] + # Handle contained tools. + if includes_tools_for_display_in_tool_panel and ( new_tool_panel_section or tool_panel_section ): + if new_tool_panel_section: + section_id = new_tool_panel_section.lower().replace( ' ', '_' ) + tool_panel_section_key = 'section_%s' % str( section_id ) + if tool_panel_section_key in trans.app.toolbox.tool_panel: + # Appending a tool to an existing section in trans.app.toolbox.tool_panel + log.debug( "Appending to tool panel section: %s" % new_tool_panel_section ) + tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] + else: + # Appending a new section to trans.app.toolbox.tool_panel + log.debug( "Loading new tool panel section: %s" % new_tool_panel_section ) + elem = Element( 'section' ) + elem.attrib[ 'name' ] = new_tool_panel_section + elem.attrib[ 'id' ] = section_id + elem.attrib[ 'version' ] = '' + tool_section = tools.ToolSection( elem ) + trans.app.toolbox.tool_panel[ tool_panel_section_key ] = tool_section + else: + tool_panel_section_key = 'section_%s' % tool_panel_section + tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] + else: + tool_panel_section_key = None + tool_section = None + encoded_repository_ids = [ trans.security.encode_id( tsr.id ) for tsr in created_or_updated_tool_shed_repositories ] + # Create a one-to-one mapping of tool shed repository id and tool panel section key. All tools contained in the repositories being installed will be loaded + # into the same section in the tool panel. + for tsr in created_or_updated_tool_shed_repositories: + tool_panel_section_keys.append( tool_panel_section_key ) + new_kwd = dict( includes_tools=includes_tools, + includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, + has_repository_dependencies=has_repository_dependencies, + install_repository_dependencies=install_repository_dependencies, + includes_tool_dependencies=includes_tool_dependencies, + install_tool_dependencies=install_tool_dependencies, + message=message, + repo_info_dicts=filtered_repo_info_dicts, + shed_tool_conf=shed_tool_conf, + status=status, + tool_path=tool_path, + tool_panel_section_keys=tool_panel_section_keys, + tool_shed_repository_ids=encoded_repository_ids, + tool_shed_url=tool_shed_url ) + encoded_kwd = encoding_util.tool_shed_encode( new_kwd ) + tsr_ids = [ r.id for r in created_or_updated_tool_shed_repositories ] + tool_shed_repositories = [] + for tsr_id in tsr_ids: + tsr = trans.sa_session.query( trans.model.ToolShedRepository ).get( tsr_id ) + tool_shed_repositories.append( tsr ) + clause_list = [] + for tsr_id in tsr_ids: + clause_list.append( trans.model.ToolShedRepository.table.c.id == tsr_id ) + query = trans.sa_session.query( trans.model.ToolShedRepository ).filter( or_( *clause_list ) ) + return encoded_kwd, query, tool_shed_repositories, encoded_repository_ids + def merge_containers_dicts_for_new_install( containers_dicts ): """ When installing one or more tool shed repositories for the first time, the received list of containers_dicts contains a containers_dict for diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 70e53bc2ecb..2bc7c92935e 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -1,6 +1,12 @@ -import os, shutil, tempfile, logging, string, urllib2 +import logging +import os +import shutil +import string +import tempfile +import urllib2 from datetime import datetime -from time import gmtime, strftime +from time import gmtime +from time import strftime from galaxy import util from galaxy.util import json from galaxy.web import url_for @@ -16,8 +22,10 @@ pkg_resources.require( 'mercurial' ) from mercurial import hg, ui, commands pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement +from elementtree import ElementTree +from elementtree import ElementInclude +from elementtree.ElementTree import Element +from elementtree.ElementTree import SubElement eggs.require( 'markupsafe' ) import markupsafe @@ -522,7 +530,7 @@ def get_next_downloadable_changeset_revision( repository, repo, after_changeset_ if changeset_revision in changeset_revisions: return changeset_revision elif not found_after_changeset_revision and changeset_revision == after_changeset_revision: - # We've found the changeset in the changelog for which we need to get the next downloadable changset. + # We've found the changeset in the changelog for which we need to get the next downloadable changeset. found_after_changeset_revision = True return None From f762868aca09156e26c7b7111287f047a3b32a9f Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Fri, 22 Mar 2013 16:26:07 -0400 Subject: [PATCH 09/21] Add an expose_api_anonymous decorator to the Galaxy web framework to enable exposing API methods without requiring an API key or a Galaxy or Tool Shed user session. --- lib/galaxy/web/__init__.py | 15 ++++++++++++--- lib/galaxy/web/framework/__init__.py | 10 ++++++++-- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/web/__init__.py b/lib/galaxy/web/__init__.py index 838bafa490d..1b46484b112 100644 --- a/lib/galaxy/web/__init__.py +++ b/lib/galaxy/web/__init__.py @@ -1,7 +1,16 @@ """ The Galaxy web application framework """ - -from framework import expose, json, json_pretty, require_login, require_admin, url_for, error, form, FormBuilder, expose_api, expose_api_raw +from framework import expose +from framework import json +from framework import json_pretty +from framework import require_login +from framework import require_admin +from framework import url_for +from framework import error +from framework import form +from framework import FormBuilder +from framework import expose_api +from framework import expose_api_anonymous +from framework import expose_api_raw from framework.base import httpexceptions - diff --git a/lib/galaxy/web/framework/__init__.py b/lib/galaxy/web/framework/__init__.py index fd70ba3d90d..ae41a3e0fe1 100644 --- a/lib/galaxy/web/framework/__init__.py +++ b/lib/galaxy/web/framework/__init__.py @@ -106,7 +106,13 @@ def expose_api_raw( func ): """ return expose_api( func, to_json=False ) -def expose_api( func, to_json=True ): +def expose_api_anonymous( func, to_json=True ): + """ + Expose this function via the API but don't require an API key. + """ + return expose_api( func, to_json=to_json, key_required=False ) + +def expose_api( func, to_json=True, key_required=True ): @wraps(func) def decorator( self, trans, *args, **kwargs ): def error( environ, start_response ): @@ -114,7 +120,7 @@ def expose_api( func, to_json=True ): return error_message error_status = '403 Forbidden' ## If there is a user, we've authenticated a session. - if not trans.user and isinstance(trans.galaxy_session, Bunch): + if key_required and not trans.user and isinstance( trans.galaxy_session, Bunch ): # If trans.user is already set, don't check for a key. # This happens when we're authenticating using session instead of an API key. # The Bunch clause is used to prevent the case where there's no user, but there is a real session. From 6267972180669ca929e612e932031c4fb76f2505 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Sat, 23 Mar 2013 10:44:46 -0400 Subject: [PATCH 10/21] Refactor several methods in Galaxy's admin_toolshed controller in preparation for supporting installing tool shed repositories via the tool shed and Galaxy APIs. --- .../galaxy/controllers/admin_toolshed.py | 204 ++++-------------- .../galaxy_install/repository_util.py | 91 +++++++- lib/tool_shed/scripts/api/common.py | 165 ++++++++++++++ lib/tool_shed/util/shed_util_common.py | 25 +++ lib/tool_shed/util/tool_util.py | 42 +++- 5 files changed, 361 insertions(+), 166 deletions(-) create mode 100644 lib/tool_shed/scripts/api/common.py diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 3035ef238fa..1c2f9e3d476 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -1,19 +1,35 @@ -import logging, os, shutil, tempfile, urllib2 +import logging +import os +import shutil +import urllib2 from admin import AdminGalaxy -from galaxy import web, util, eggs, tools -from galaxy.web.form_builder import SelectField, CheckboxField -from galaxy.web.framework.helpers import iff, grids +from galaxy import eggs +from galaxy import web +from galaxy import util +from galaxy.web.form_builder import CheckboxField +from galaxy.web.framework.helpers import grids +from galaxy.web.framework.helpers import iff from galaxy.util import json from galaxy.model.orm import or_ import tool_shed.util.shed_util_common as suc -from tool_shed.util import common_install_util, data_manager_util, datatype_util, encoding_util, metadata_util -from tool_shed.util import readme_util, repository_dependency_util, tool_dependency_util, tool_util, workflow_util +from tool_shed.util import common_install_util +from tool_shed.util import data_manager_util +from tool_shed.util import datatype_util +from tool_shed.util import encoding_util +from tool_shed.util import metadata_util +from tool_shed.util import readme_util +from tool_shed.util import repository_dependency_util +from tool_shed.util import tool_dependency_util +from tool_shed.util import tool_util +from tool_shed.util import workflow_util from tool_shed.galaxy_install import repository_util import tool_shed.galaxy_install.grids.admin_toolshed_grids as admin_toolshed_grids import pkg_resources eggs.require( 'mercurial' ) -from mercurial import hg, ui, commands +from mercurial import commands +from mercurial import hg +from mercurial import ui pkg_resources.require( 'elementtree' ) from elementtree import ElementTree @@ -520,96 +536,21 @@ class AdminToolshed( AdminGalaxy ): """Install specified tool shed repositories.""" shed_tool_conf = kwd.get( 'shed_tool_conf', '' ) tool_path = kwd[ 'tool_path' ] - includes_tool_dependencies = util.string_as_bool( kwd[ 'includes_tool_dependencies' ] ) install_tool_dependencies = CheckboxField.is_checked( kwd.get( 'install_tool_dependencies', '' ) ) - # There must be a one-to-one mapping between items in the 3 lists:tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. + # There must be a one-to-one mapping between items in the 3 lists: tool_shed_repositories, tool_panel_section_keys, repo_info_dicts. tool_panel_section_keys = util.listify( kwd[ 'tool_panel_section_keys' ] ) repo_info_dicts = util.listify( kwd[ 'repo_info_dicts' ] ) for index, tool_shed_repository in enumerate( tool_shed_repositories ): repo_info_dict = repo_info_dicts[ index ] tool_panel_section_key = tool_panel_section_keys[ index ] - if tool_panel_section_key: - tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] - else: - tool_section = None - if isinstance( repo_info_dict, basestring ): - repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) - # Clone each repository to the configured location. - suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) - repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] - description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple - relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision ) - clone_dir = os.path.join( tool_path, relative_clone_dir ) - relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) - install_dir = os.path.join( tool_path, relative_install_dir ) - cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) - if cloned_ok: - if reinstalling: - # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated. - changeset_revision_dict = repository_util.get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository ) - current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) - current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) - if current_ctx_rev != ctx_rev: - repo = hg.repository( suc.get_configured_ui(), path=os.path.abspath( install_dir ) ) - repository_util.pull_repository( repo, repository_clone_url, current_changeset_revision ) - suc.update_repository( repo, ctx_rev=current_ctx_rev ) - repository_util.handle_repository_contents( trans, - tool_shed_repository=tool_shed_repository, - tool_path=tool_path, - repository_clone_url=repository_clone_url, - relative_install_dir=relative_install_dir, - tool_shed=tool_shed_repository.tool_shed, - tool_section=tool_section, - shed_tool_conf=shed_tool_conf, - reinstalling=reinstalling ) - trans.sa_session.refresh( tool_shed_repository ) - metadata = tool_shed_repository.metadata - if 'tools' in metadata: - # Get the tool_versions from the tool shed for each tool in the installed change set. - suc.update_tool_shed_repository_status( trans.app, - tool_shed_repository, - trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) - tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed ) - url = suc.url_join( tool_shed_url, - '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ - ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) - response = urllib2.urlopen( url ) - text = response.read() - response.close() - if text: - tool_version_dicts = json.from_json_string( text ) - tool_util.handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) - else: - message += "Version information for the tools included in the %s repository is missing. " % name - message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " - message += "from the installed repository's Repository Actions menu. " - status = 'error' - if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: - work_dir = tempfile.mkdtemp() - # Install tool dependencies. - suc.update_tool_shed_repository_status( trans.app, - tool_shed_repository, - trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) - # Get the tool_dependencies.xml file from the repository. - tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir )#relative_install_dir ) - installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, - tool_shed_repository=tool_shed_repository, - tool_dependencies_config=tool_dependencies_config, - tool_dependencies=tool_shed_repository.tool_dependencies ) - try: - shutil.rmtree( work_dir ) - except: - pass - suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) - else: - # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. - self.set_repository_attributes( trans, - tool_shed_repository, - status=trans.model.ToolShedRepository.installation_status.ERROR, - error_message=error_message, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + repository_util.install_tool_shed_repository( trans, + tool_shed_repository, + repo_info_dict, + tool_panel_section_key, + shed_tool_conf, + tool_path, + install_tool_dependencies, + reinstalling=reinstalling ) tsr_ids_for_monitoring = [ trans.security.encode_id( tsr.id ) for tsr in tool_shed_repositories ] return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='monitor_repository_installation', @@ -844,7 +785,7 @@ class AdminToolshed( AdminGalaxy ): @web.expose @web.require_admin def prepare_for_install( self, trans, **kwd ): - if not have_shed_tool_conf_for_install( trans ): + if not suc.have_shed_tool_conf_for_install( trans ): message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a ' message += '<toolbox> tag that includes a tool_path attribute value which is a directory relative to the Galaxy installation ' message += 'directory in order to automatically install tools from a Galaxy tool shed (e.g., the file name shed_tool_conf.xml whose ' @@ -942,9 +883,9 @@ class AdminToolshed( AdminGalaxy ): return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='manage_repositories', **kwd ) ) - shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) + shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans ) tool_path = suc.get_tool_path_by_shed_tool_conf_filename( trans, shed_tool_conf ) - tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) + tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field( trans ) if len( repo_info_dicts ) == 1: # If we're installing a single repository, see if it contains a readme or dependencies that we can display. repo_info_dict = repo_info_dicts[ 0 ] @@ -1278,7 +1219,7 @@ class AdminToolshed( AdminGalaxy ): original_section_name = '' else: original_section_name = '' - tool_panel_section_select_field = build_tool_panel_section_select_field( trans ) + tool_panel_section_select_field = tool_util.build_tool_panel_section_select_field( trans ) no_changes_check_box = CheckboxField( 'no_changes', checked=True ) if original_section_name: message += "The tools contained in your %s repository were last loaded into the tool panel section %s. " \ @@ -1294,7 +1235,7 @@ class AdminToolshed( AdminGalaxy ): no_changes_check_box = None original_section_name = '' tool_panel_section_select_field = None - shed_tool_conf_select_field = build_shed_tool_conf_select_field( trans ) + shed_tool_conf_select_field = tool_util.build_shed_tool_conf_select_field( trans ) containers_dict = repository_util.populate_containers_dict_for_new_install( trans=trans, tool_shed_url=tool_shed_url, tool_path=tool_path, @@ -1398,13 +1339,13 @@ class AdminToolshed( AdminGalaxy ): """An error occurred while cloning the repository, so reset everything necessary to enable another attempt.""" repository = suc.get_installed_tool_shed_repository( trans, kwd[ 'id' ] ) if kwd.get( 'reset_repository', False ): - self.set_repository_attributes( trans, - repository, - status=trans.model.ToolShedRepository.installation_status.NEW, - error_message=None, - deleted=False, - uninstalled=False, - remove_from_disk=True ) + suc.set_repository_attributes( trans, + repository, + status=trans.model.ToolShedRepository.installation_status.NEW, + error_message=None, + deleted=False, + uninstalled=False, + remove_from_disk=True ) new_kwd = {} new_kwd[ 'message' ] = "You can now attempt to install the repository named %s again." % repository.name new_kwd[ 'status' ] = "done" @@ -1415,20 +1356,6 @@ class AdminToolshed( AdminGalaxy ): action='manage_repository', **kwd ) ) - def set_repository_attributes( self, trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): - if remove_from_disk: - relative_install_dir = repository.repo_path( trans.app ) - if relative_install_dir: - clone_dir = os.path.abspath( relative_install_dir ) - shutil.rmtree( clone_dir ) - log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) - repository.error_message = error_message - repository.status = status - repository.deleted = deleted - repository.uninstalled = uninstalled - trans.sa_session.add( repository ) - trans.sa_session.flush() - @web.expose @web.require_admin def set_tool_versions( self, trans, **kwd ): @@ -1680,46 +1607,3 @@ class AdminToolshed( AdminGalaxy ): metadata=metadata, message=message, status=status ) - -## ---- Utility methods ------------------------------------------------------- - -def build_shed_tool_conf_select_field( trans ): - """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs.""" - options = [] - for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: - shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ] - if shed_tool_conf_filename != trans.app.config.migrated_tools_config: - if shed_tool_conf_filename.startswith( './' ): - option_label = shed_tool_conf_filename.replace( './', '', 1 ) - else: - option_label = shed_tool_conf_filename - options.append( ( option_label, shed_tool_conf_filename ) ) - select_field = SelectField( name='shed_tool_conf' ) - for option_tup in options: - select_field.add_option( option_tup[0], option_tup[1] ) - return select_field - -def build_tool_panel_section_select_field( trans ): - """Build a SelectField whose options are the sections of the current in-memory toolbox.""" - options = [] - for k, v in trans.app.toolbox.tool_panel.items(): - if isinstance( v, tools.ToolSection ): - options.append( ( v.name, v.id ) ) - select_field = SelectField( name='tool_panel_section', display='radio' ) - for option_tup in options: - select_field.add_option( option_tup[0], option_tup[1] ) - return select_field - -def can_select_tool_panel_section(): - pass - -def have_shed_tool_conf_for_install( trans ): - if not trans.app.toolbox.shed_tool_confs: - return False - migrated_tools_conf_path, migrated_tools_conf_name = os.path.split( trans.app.config.migrated_tools_config ) - for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: - shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] - shed_tool_conf_path, shed_tool_conf_name = os.path.split( shed_tool_conf ) - if shed_tool_conf_name != migrated_tools_conf_name: - return True - return False diff --git a/lib/tool_shed/galaxy_install/repository_util.py b/lib/tool_shed/galaxy_install/repository_util.py index 12f4784a186..0322bbf53f8 100644 --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -1,12 +1,16 @@ import logging import os +import shutil +import tempfile import threading import urllib2 from galaxy import tools +from galaxy.util import json from galaxy import web from galaxy.model.orm import or_ from galaxy.webapps.tool_shed.util import container_util import tool_shed.util.shed_util_common as suc +from tool_shed.util import common_install_util from tool_shed.util import data_manager_util from tool_shed.util import datatype_util from tool_shed.util import encoding_util @@ -63,7 +67,7 @@ def create_repo_info_dict( trans, repository_clone_url, changeset_revision, ctx_ all_repository_dependencies=None, handled_key_rd_dicts=None, circular_repository_dependencies=None ) - tool_dependencies = metadata.get( 'tool_dependencies', None ) + tool_dependencies = metadata.get( 'tool_dependencies', {} ) if tool_dependencies: new_tool_dependencies = {} for dependency_key, requirements_dict in tool_dependencies.items(): @@ -347,6 +351,91 @@ def initiate_repository_installation( trans, installation_dict ): query = trans.sa_session.query( trans.model.ToolShedRepository ).filter( or_( *clause_list ) ) return encoded_kwd, query, tool_shed_repositories, encoded_repository_ids +def install_tool_shed_repository( trans, tool_shed_repository, repo_info_dict, tool_panel_section_key, shed_tool_conf, tool_path, install_tool_dependencies, + reinstalling=False ): + if tool_panel_section_key: + tool_section = trans.app.toolbox.tool_panel[ tool_panel_section_key ] + else: + tool_section = None + if isinstance( repo_info_dict, basestring ): + repo_info_dict = encoding_util.tool_shed_decode( repo_info_dict ) + # Clone each repository to the configured location. + suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.CLONING ) + repo_info_tuple = repo_info_dict[ tool_shed_repository.name ] + description, repository_clone_url, changeset_revision, ctx_rev, repository_owner, repository_dependencies, tool_dependencies = repo_info_tuple + relative_clone_dir = suc.generate_tool_shed_repository_install_dir( repository_clone_url, tool_shed_repository.installed_changeset_revision ) + clone_dir = os.path.join( tool_path, relative_clone_dir ) + relative_install_dir = os.path.join( relative_clone_dir, tool_shed_repository.name ) + install_dir = os.path.join( tool_path, relative_install_dir ) + cloned_ok, error_message = suc.clone_repository( repository_clone_url, os.path.abspath( install_dir ), ctx_rev ) + if cloned_ok: + if reinstalling: + # Since we're reinstalling the repository we need to find the latest changeset revision to which is can be updated. + changeset_revision_dict = get_update_to_changeset_revision_and_ctx_rev( trans, tool_shed_repository ) + current_changeset_revision = changeset_revision_dict.get( 'changeset_revision', None ) + current_ctx_rev = changeset_revision_dict.get( 'ctx_rev', None ) + if current_ctx_rev != ctx_rev: + repo = hg.repository( suc.get_configured_ui(), path=os.path.abspath( install_dir ) ) + pull_repository( repo, repository_clone_url, current_changeset_revision ) + suc.update_repository( repo, ctx_rev=current_ctx_rev ) + handle_repository_contents( trans, + tool_shed_repository=tool_shed_repository, + tool_path=tool_path, + repository_clone_url=repository_clone_url, + relative_install_dir=relative_install_dir, + tool_shed=tool_shed_repository.tool_shed, + tool_section=tool_section, + shed_tool_conf=shed_tool_conf, + reinstalling=reinstalling ) + trans.sa_session.refresh( tool_shed_repository ) + metadata = tool_shed_repository.metadata + if 'tools' in metadata: + # Get the tool_versions from the tool shed for each tool in the installed change set. + suc.update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.SETTING_TOOL_VERSIONS ) + tool_shed_url = suc.get_url_from_tool_shed( trans.app, tool_shed_repository.tool_shed ) + url = suc.url_join( tool_shed_url, + '/repository/get_tool_versions?name=%s&owner=%s&changeset_revision=%s' % \ + ( tool_shed_repository.name, tool_shed_repository.owner, tool_shed_repository.changeset_revision ) ) + response = urllib2.urlopen( url ) + text = response.read() + response.close() + if text: + tool_version_dicts = json.from_json_string( text ) + tool_util.handle_tool_versions( trans.app, tool_version_dicts, tool_shed_repository ) + else: + message += "Version information for the tools included in the %s repository is missing. " % name + message += "Reset all of this repository's metadata in the tool shed, then set the installed tool versions " + message += "from the installed repository's Repository Actions menu. " + status = 'error' + if install_tool_dependencies and tool_shed_repository.tool_dependencies and 'tool_dependencies' in metadata: + work_dir = tempfile.mkdtemp() + # Install tool dependencies. + suc.update_tool_shed_repository_status( trans.app, + tool_shed_repository, + trans.model.ToolShedRepository.installation_status.INSTALLING_TOOL_DEPENDENCIES ) + # Get the tool_dependencies.xml file from the repository. + tool_dependencies_config = suc.get_config_from_disk( 'tool_dependencies.xml', install_dir )#relative_install_dir ) + installed_tool_dependencies = common_install_util.handle_tool_dependencies( app=trans.app, + tool_shed_repository=tool_shed_repository, + tool_dependencies_config=tool_dependencies_config, + tool_dependencies=tool_shed_repository.tool_dependencies ) + try: + shutil.rmtree( work_dir ) + except: + pass + suc.update_tool_shed_repository_status( trans.app, tool_shed_repository, trans.model.ToolShedRepository.installation_status.INSTALLED ) + else: + # An error occurred while cloning the repository, so reset everything necessary to enable another attempt. + suc.set_repository_attributes( trans, + tool_shed_repository, + status=trans.model.ToolShedRepository.installation_status.ERROR, + error_message=error_message, + deleted=False, + uninstalled=False, + remove_from_disk=True ) + def merge_containers_dicts_for_new_install( containers_dicts ): """ When installing one or more tool shed repositories for the first time, the received list of containers_dicts contains a containers_dict for diff --git a/lib/tool_shed/scripts/api/common.py b/lib/tool_shed/scripts/api/common.py new file mode 100644 index 00000000000..f16df756002 --- /dev/null +++ b/lib/tool_shed/scripts/api/common.py @@ -0,0 +1,165 @@ +import os, sys, urllib, urllib2 + +new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', '..', 'lib' ) ] +new_path.extend( sys.path[ 1: ] ) +sys.path = new_path + +from galaxy import eggs +import pkg_resources + +pkg_resources.require( "simplejson" ) +import simplejson + +pkg_resources.require( "pycrypto" ) +from Crypto.Cipher import Blowfish +from Crypto.Util.randpool import RandomPool +from Crypto.Util import number + +def encode_id( config_id_secret, obj_id ): + # Utility method to encode ID's + id_cipher = Blowfish.new( config_id_secret ) + # Convert to string + s = str( obj_id ) + # Pad to a multiple of 8 with leading "!" + s = ( "!" * ( 8 - len(s) % 8 ) ) + s + # Encrypt + return id_cipher.encrypt( s ).encode( 'hex' ) + +def delete( api_key, url, data, return_formatted=True ): + # Sends an API DELETE request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + try: + url = make_url( api_key, url ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req.get_method = lambda: 'DELETE' + r = simplejson.loads( urllib2.urlopen( req ).read() ) + except urllib2.HTTPError, e: + if return_formatted: + print e + print e.read( 1024 ) + sys.exit( 1 ) + else: + return 'Error. '+ str( e.read( 1024 ) ) + if not return_formatted: + return r + print 'Response' + print '--------' + print r + +def display( api_key, url, return_formatted=True ): + # Sends an API GET request and acts as a generic formatter for the JSON response. + try: + r = get( api_key, url ) + except urllib2.HTTPError, e: + print e + print e.read( 1024 ) # Only return the first 1K of errors. + sys.exit( 1 ) + if type( r ) == unicode: + print 'error: %s' % r + return None + if not return_formatted: + return r + elif type( r ) == list: + # Response is a collection as defined in the REST style. + print 'Collection Members' + print '------------------' + for n, i in enumerate(r): + # All collection members should have a name and url in the response. + print '#%d: %s' % (n+1, i.pop( 'url' ) ) + if 'name' in i: + print ' name: %s' % i.pop( 'name' ) + for k, v in i.items(): + print ' %s: %s' % ( k, v ) + print '' + print '%d element(s) in collection' % len( r ) + elif type( r ) == dict: + # Response is an element as defined in the REST style. + print 'Member Information' + print '------------------' + for k, v in r.items(): + print '%s: %s' % ( k, v ) + elif type( r ) == str: + print r + else: + print 'response is unknown type: %s' % type( r ) + +def get( api_key, url ): + # Do the actual GET. + url = make_url( api_key, url ) + try: + return simplejson.loads( urllib2.urlopen( url ).read() ) + except simplejson.decoder.JSONDecodeError, e: + print "URL did not return JSON data" + sys.exit(1) + +def make_url( api_key, url, args=None ): + # Adds the API Key to the URL if it's not already there. + if args is None: + args = [] + argsep = '&' + if '?' not in url: + argsep = '?' + if '?key=' not in url and '&key=' not in url: + args.insert( 0, ( 'key', api_key ) ) + return url + argsep + '&'.join( [ '='.join( t ) for t in args ] ) + +def post( api_key, url, data ): + # Do the actual POST. + url = make_url( api_key, url ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data ) ) + return simplejson.loads( urllib2.urlopen( req ).read() ) + +def put( api_key, url, data ): + # Do the actual PUT. + url = make_url( api_key, url ) + req = urllib2.Request( url, headers = { 'Content-Type': 'application/json' }, data = simplejson.dumps( data )) + req.get_method = lambda: 'PUT' + return simplejson.loads( urllib2.urlopen( req ).read() ) + +def submit( api_key, url, data, return_formatted=True ): + # Sends an API POST request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + try: + r = post( api_key, url, data ) + except urllib2.HTTPError, e: + if return_formatted: + print e + print e.read( 1024 ) + sys.exit( 1 ) + else: + return 'Error. '+ str( e.read( 1024 ) ) + if not return_formatted: + return r + print 'Response' + print '--------' + if type( r ) == list: + # Currently the only implemented responses are lists of dicts, because submission creates some number of collection elements. + for i in r: + if type( i ) == dict: + if 'url' in i: + print i.pop( 'url' ) + else: + print '----' + if 'name' in i: + print ' name: %s' % i.pop( 'name' ) + for k, v in i.items(): + print ' %s: %s' % ( k, v ) + else: + print i + else: + print r + +def update( api_key, url, data, return_formatted=True ): + # Sends an API PUT request and acts as a generic formatter for the JSON response - 'data' will become the JSON payload read by Galaxy. + try: + r = put( api_key, url, data ) + except urllib2.HTTPError, e: + if return_formatted: + print e + print e.read( 1024 ) + sys.exit( 1 ) + else: + return 'Error. '+ str( e.read( 1024 ) ) + if not return_formatted: + return r + print 'Response' + print '--------' + print r diff --git a/lib/tool_shed/util/shed_util_common.py b/lib/tool_shed/util/shed_util_common.py index 2bc7c92935e..29b8608f942 100644 --- a/lib/tool_shed/util/shed_util_common.py +++ b/lib/tool_shed/util/shed_util_common.py @@ -985,6 +985,17 @@ def handle_galaxy_url( trans, **kwd ): galaxy_url = trans.get_cookie( name='toolshedgalaxyurl' ) return galaxy_url +def have_shed_tool_conf_for_install( trans ): + if not trans.app.toolbox.shed_tool_confs: + return False + migrated_tools_conf_path, migrated_tools_conf_name = os.path.split( trans.app.config.migrated_tools_config ) + for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: + shed_tool_conf = shed_tool_conf_dict[ 'config_filename' ] + shed_tool_conf_path, shed_tool_conf_name = os.path.split( shed_tool_conf ) + if shed_tool_conf_name != migrated_tools_conf_name: + return True + return False + def open_repository_files_folder( trans, folder_path ): """Return a list of dictionaries, each of which contains information for a file or directory contained within a directory in a repository file hierarchy.""" try: @@ -1072,6 +1083,20 @@ def reversed_upper_bounded_changelog( repo, included_upper_bounds_changeset_revi """Return a reversed list of changesets in the repository changelog up to and including the included_upper_bounds_changeset_revision.""" return reversed_lower_upper_bounded_changelog( repo, INITIAL_CHANGELOG_HASH, included_upper_bounds_changeset_revision ) +def set_repository_attributes( trans, repository, status, error_message, deleted, uninstalled, remove_from_disk=False ): + if remove_from_disk: + relative_install_dir = repository.repo_path( trans.app ) + if relative_install_dir: + clone_dir = os.path.abspath( relative_install_dir ) + shutil.rmtree( clone_dir ) + log.debug( "Removed repository installation directory: %s" % str( clone_dir ) ) + repository.error_message = error_message + repository.status = status + repository.deleted = deleted + repository.uninstalled = uninstalled + trans.sa_session.add( repository ) + trans.sa_session.flush() + def strip_path( fpath ): """Attempt to strip the path from a file name.""" if not fpath: diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 5a9cff653fa..7412f191f5d 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -1,12 +1,17 @@ -import filecmp, logging, os, shutil, tempfile -import tool_shed.util.shed_util_common as suc -from galaxy import util +import filecmp +import logging +import os +import shutil +import tempfile import galaxy.tools -from galaxy.tools.search import ToolBoxSearch -from galaxy.model.orm import and_ +from galaxy import util from galaxy.datatypes import checkers +from galaxy.model.orm import and_ from galaxy.tools import parameters from galaxy.tools.parameters import dynamic_options +from galaxy.tools.search import ToolBoxSearch +from galaxy.web.form_builder import SelectField +import tool_shed.util.shed_util_common as suc from galaxy import eggs import pkg_resources @@ -79,6 +84,33 @@ def add_to_tool_panel( app, repository_name, repository_clone_url, changeset_rev app.toolbox.write_integrated_tool_panel_config_file() app.toolbox_search = ToolBoxSearch( app.toolbox ) +def build_shed_tool_conf_select_field( trans ): + """Build a SelectField whose options are the keys in trans.app.toolbox.shed_tool_confs.""" + options = [] + for shed_tool_conf_dict in trans.app.toolbox.shed_tool_confs: + shed_tool_conf_filename = shed_tool_conf_dict[ 'config_filename' ] + if shed_tool_conf_filename != trans.app.config.migrated_tools_config: + if shed_tool_conf_filename.startswith( './' ): + option_label = shed_tool_conf_filename.replace( './', '', 1 ) + else: + option_label = shed_tool_conf_filename + options.append( ( option_label, shed_tool_conf_filename ) ) + select_field = SelectField( name='shed_tool_conf' ) + for option_tup in options: + select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) + return select_field + +def build_tool_panel_section_select_field( trans ): + """Build a SelectField whose options are the sections of the current in-memory toolbox.""" + options = [] + for k, v in trans.app.toolbox.tool_panel.items(): + if isinstance( v, galaxy.tools.ToolSection ): + options.append( ( v.name, v.id ) ) + select_field = SelectField( name='tool_panel_section', display='radio' ) + for option_tup in options: + select_field.add_option( option_tup[ 0 ], option_tup[ 1 ] ) + return select_field + def can_use_tool_config_disk_file( trans, repository, repo, file_path, changeset_revision ): """ Determine if repository's tool config file on disk can be used. This method is restricted to tool config files since, with the From d297d284989873eb49992d69fd8ace1b515ff782 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Mon, 25 Mar 2013 09:36:02 -0400 Subject: [PATCH 11/21] Update tests to reflect new default for component review approval status. --- test/tool_shed/base/twilltestcase.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test/tool_shed/base/twilltestcase.py b/test/tool_shed/base/twilltestcase.py index 513ee76520c..d178cda8bc5 100644 --- a/test/tool_shed/base/twilltestcase.py +++ b/test/tool_shed/base/twilltestcase.py @@ -454,9 +454,10 @@ class ShedTwillTestCase( TwillTestCase ): def fill_review_form( self, review_contents_dict, strings_displayed=[], strings_not_displayed=[] ): kwd = dict() + changed = False for label, contents in review_contents_dict.items(): - strings_displayed.append( label ) if contents: + changed = True kwd[ '%s__ESEP__comment' % label ] = contents[ 'comment' ] kwd[ '%s__ESEP__rating' % label ] = contents[ 'rating' ] if 'private' in contents: @@ -464,8 +465,10 @@ class ShedTwillTestCase( TwillTestCase ): kwd[ '%s__ESEP__approved' % label ] = contents[ 'approved' ] else: kwd[ '%s__ESEP__approved' % label ] = 'not_applicable' + self.check_for_strings( strings_displayed, strings_not_displayed ) self.submit_form( 1, 'Workflows__ESEP__review_button', **kwd ) - strings_displayed.append( 'Reviews were saved' ) + if changed: + strings_displayed.append( 'Reviews were saved' ) self.check_for_strings( strings_displayed, strings_not_displayed ) def galaxy_login( self, email='test@bx.psu.edu', password='testuser', username='admin-user', redirect='' ): From ade0bcad0601f6808e6b267b92b61d0430bed847 Mon Sep 17 00:00:00 2001 From: Jeremy Goecks Date: Mon, 25 Mar 2013 09:37:17 -0400 Subject: [PATCH 12/21] Trackster: more flexible parsing of location string. --- static/scripts/viz/trackster/tracks.js | 55 ++++++++++++++++---------- 1 file changed, 35 insertions(+), 20 deletions(-) diff --git a/static/scripts/viz/trackster/tracks.js b/static/scripts/viz/trackster/tracks.js index 38d785f90df..5abbd3846a6 100644 --- a/static/scripts/viz/trackster/tracks.js +++ b/static/scripts/viz/trackster/tracks.js @@ -1276,7 +1276,7 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { view.reference_track.init(); } } - if (low !== undefined && high !== undefined) { + if (low && high) { view.low = Math.max(low, 0); view.high = Math.min(high, view.max_high); } @@ -1289,34 +1289,49 @@ extend( TracksterView.prototype, DrawableCollection.prototype, { view.request_redraw(); } }, + + /** + * Change viewing region to that denoted by string. General format of string is: + * + * [ {separator}[-] ] + * + * where separator can be whitespace or a colon. Examples: + * + * chr22 + * chr1:100-200 + * chr7 89999 + * chr8 90000 990000 + */ go_to: function(str) { - // Preprocess str to remove spaces and commas. - str = str.replace(/ |,/g, ""); - - // Go to new location. - var view = this, - new_low, - new_high, - chrom_pos = str.split(":"), + // Remove commas. + str = str.replace(/,/g, ''); + + // Replace colons and hyphens with space for easy parsing. + str = str.replace(/:|\-/g, ' '); + + // Parse new location. + var chrom_pos = str.split(/\s+/), chrom = chrom_pos[0], - pos = chrom_pos[1]; - - if (pos !== undefined) { - try { - var pos_split = pos.split("-"); - new_low = parseInt(pos_split[0], 10); - new_high = parseInt(pos_split[1], 10); - } catch (e) { - return false; - } + new_low = (chrom_pos[1] ? parseInt(chrom_pos[1], 10) : null), + new_high = (chrom_pos[2] ? parseInt(chrom_pos[2], 10) : null); + + // If no new high, new_low is the position of focus, so adjust low, high + // accordingly. + if (!new_high) { + // HACK: max resolution is 30 bases,so adjust low, high accordingly. + new_low = new_low - 15; + new_high = new_low + 15; } - view.change_chrom(chrom, new_low, new_high); + + this.change_chrom(chrom, new_low, new_high); }, + move_fraction: function(fraction) { var view = this; var span = view.high - view.low; this.move_delta(fraction * span); }, + move_delta: function(delta_chrom) { // Update low, high. var view = this; From 13fec82eed2deff483c9402d88807308e2c962a6 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Mon, 25 Mar 2013 12:05:17 -0400 Subject: [PATCH 13/21] Specify return codes so the automated framework will more correctly report success/failure. --- .../functional_tests.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index 58397d7252a..f4c300dc077 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -734,7 +734,7 @@ def main(): ( changeset_revision, name ) ) # Run the cleanup method. This removes tool functional test methods from the test_toolbox module and uninstalls the # repository using Twill. - execute_uninstall_method( repository_info_dict ) + success = execute_uninstall_method( repository_info_dict ) # Set the test_toolbox.toolbox module-level variable to the new app.toolbox. test_toolbox.toolbox = app.toolbox else: @@ -782,20 +782,23 @@ def main(): print "# %d repositories failed:" % len( repositories_failed ) show_summary_output( repositories_failed ) if repositories_failed_install: + # Set success to False so that the return code will not be 0. + success = False print '# ----------------------------------------------------------------------------------' print "# %d repositories not installed correctly:" % len( repositories_failed_install ) show_summary_output( repositories_failed_install ) print "####################################################################################" - - if success: - return 0 + if repositories_tested > 0: + if success: + return 0 + else: + return 1 else: - return 1 + return 0 if __name__ == "__main__": now = strftime( "%Y-%m-%d %H:%M:%S" ) print "####################################################################################" print "# %s - running repository installation and testing script." % now print "####################################################################################" - return_code = main() - sys.exit( return_code ) + sys.exit( main() ) From 7621c977f012355eb673c15e83de1577091b2ba8 Mon Sep 17 00:00:00 2001 From: James Taylor Date: Mon, 25 Mar 2013 16:30:16 -0400 Subject: [PATCH 14/21] biostar: bugfix for encoding data --- lib/galaxy/webapps/galaxy/controllers/biostar.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/biostar.py b/lib/galaxy/webapps/galaxy/controllers/biostar.py index c038bdbc7ae..c8c92500c9c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/biostar.py +++ b/lib/galaxy/webapps/galaxy/controllers/biostar.py @@ -22,7 +22,7 @@ def encode_data( key, data ): """ Encode data to send a question to Biostar """ - text = json.dumps(data) + text = json.to_json_string(data) text = base64.urlsafe_b64encode(text) digest = hmac.new(key, text).hexdigest() return text, digest From b9c6df9d9173717ffe38a2302cb5f613c63d6009 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 25 Mar 2013 17:05:14 -0400 Subject: [PATCH 15/21] Add baseline cupport for installing tool shed repositories from a specified tool shed into Galaxy via a combination of the tool shed API and the now enhanced Galaxy API. There are 3 new example scripts in ~/scripts/api that demonstrate how this works. The are named like: install_repository... In this implementation, all tools contained in repositories will be loaded into the Galaxy tool panel. The user can specify the following when sending a POST to the new Galaxy API URL: /api/tool_shed_repositories/new/install_repository_revision 1) an existing tool panel section id for loading tools or a new tool panel section label in which case a new ToolPanelSection will be created for containing the tools. 2) if they want tool dependencies installed along with the repository. 3) if they want repository dependencies installed along with the repository 4) which defined shed-related tool panel config file to use for installed tools --- .../galaxy/api/tool_shed_repositories.py | 324 ++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 16 +- .../galaxy/controllers/admin_toolshed.py | 2 +- .../webapps/tool_shed/api/repositories.py | 75 +++- lib/galaxy/webapps/tool_shed/buildapp.py | 2 +- .../galaxy_install/repository_util.py | 4 +- scripts/api/common.py | 7 +- ..._tools_into_existing_tool_panel_section.py | 40 +++ ...itory_tools_into_new_tool_panel_section.py | 35 ++ ...repository_with_repository_dependencies.py | 35 ++ 10 files changed, 513 insertions(+), 27 deletions(-) create mode 100644 lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py create mode 100755 scripts/api/install_repository_tools_into_existing_tool_panel_section.py create mode 100644 scripts/api/install_repository_tools_into_new_tool_panel_section.py create mode 100644 scripts/api/install_repository_with_repository_dependencies.py diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py new file mode 100644 index 00000000000..9de96018a2c --- /dev/null +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -0,0 +1,324 @@ +import logging +import urllib2 +from galaxy.util import json +from galaxy import util +from galaxy import web +from galaxy.web.base.controller import BaseAPIController +from tool_shed.galaxy_install import repository_util +import tool_shed.util.shed_util_common as suc + +log = logging.getLogger( __name__ ) + +def default_tool_shed_repository_value_mapper( trans, tool_shed_repository ): + value_mapper={ 'id' : trans.security.encode_id( tool_shed_repository.id ), + 'error_message' : tool_shed_repository.error_message or '' } + return value_mapper + +def get_message_for_no_shed_tool_config(): + # This Galaxy instance is not configured with a shed-related tool panel configuration file. + message = 'The tool_config_file setting in universe_wsgi.ini must include at least one shed tool configuration file name with a ' + message += 'tag that includes a tool_path attribute value which is a directory relative to the Galaxy installation directory in order to ' + message += 'automatically install tools from a tool shed into Galaxy (e.g., the file name shed_tool_conf.xml whose tag is ' + message += '). For details, see the "Installation of Galaxy tool shed repository tools into a local ' + message += 'Galaxy instance" section of the Galaxy tool shed wiki at http://wiki.galaxyproject.org/InstallingRepositoriesToGalaxy#' + message += 'Installing_Galaxy_tool_shed_repository_tools_into_a_local_Galaxy_instance.' + return message + +class ToolShedRepositoriesController( BaseAPIController ): + """RESTful controller for interactions with tool shed repositories.""" + + @web.expose_api + def index( self, trans, **kwd ): + """ + GET /api/tool_shed_repositories + Display a list of dictionaries containing information about installed tool shed repositories. + """ + # Example URL: http://localhost:8763/api/tool_shed_repositories + tool_shed_repository_dicts = [] + try: + query = trans.sa_session.query( trans.app.model.ToolShedRepository ) \ + .order_by( trans.app.model.ToolShedRepository.table.c.name ) \ + .all() + for tool_shed_repository in query: + tool_shed_repository_dict = tool_shed_repository.get_api_value( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + tool_shed_repository_dicts.append( tool_shed_repository_dict ) + return tool_shed_repository_dicts + except Exception, e: + message = "Error in the tool_shed_repositories API in index: %s" % str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + + @web.expose_api + def show( self, trans, id, **kwd ): + """ + GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} + Display a dictionary containing information about a specified tool_shed_repository. + + :param tool_shed_repository_id: the encoded id of the `ToolShedRepository` object + """ + # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e + try: + tool_shed_repository = suc.get_tool_shed_repository_by_id( trans, id ) + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + return tool_shed_repository_dict + except Exception, e: + message = "Error in tool_shed_repositories API in index: " + str( e ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return message + + @web.expose_api + def install_repository_revision( self, trans, payload, **kwd ): + """ + POST /api/tool_shed_repositories/install_repository_revision + Install a specified repository revision from a specified tool shed into Galaxy. + + :param tool_shed_url: the base URL of the Tool Shed from which to install the Repository + :param name: the name of the Repository + :param owner: the owner of the Repository + :param changset_revision: the changset_revision of the RepositoryMetadata object associated with the Repository + :param key: the current Galaxy admin user's API key + :param new_tool_panel_section_label: optional label of a new section to be added to the Galaxy tool panel in which to load + tools contained in the Repository. Either this parameter must be an empty string or + the tool_panel_section_id parameter must be an empty string, as both cannot be used. + :param tool_panel_section_id: optional id of the Galaxy tool panel section in which to load tools contained in the Repository. + If not set, tools will be loaded outside of any sections in the tool panel. Either this + parameter must be an empty string or the tool_panel_section_id parameter must be an empty string, + as both cannot be used. + """ + # Get the information about the repository to be installed from the payload. + tool_shed_url = payload.get( 'tool_shed_url', '' ) + if not tool_shed_url: + raise HTTPBadRequest( detail="Missing required parameter 'tool_shed_url'." ) + name = payload.get( 'name', '' ) + if not name: + raise HTTPBadRequest( detail="Missing required parameter 'name'." ) + owner = payload.get( 'owner', '' ) + if not owner: + raise HTTPBadRequest( detail="Missing required parameter 'owner'." ) + changeset_revision = payload.get( 'changeset_revision', '' ) + if not changeset_revision: + raise HTTPBadRequest( detail="Missing required parameter 'changeset_revision'." ) + # Make sure this Galaxy instance is configured with a shed-related tool panel configuration file. + if not suc.have_shed_tool_conf_for_install( trans ): + message = get_message_for_no_shed_tool_config() + log.error( message, exc_info=True ) + trans.response.status = 500 + return dict( status='error', error=message ) + # Make sure the current user's API key proves he is an admin user in this Galaxy instance. + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' ) + # Keep track of all repositories that are installed - there may be more than one if repository dependencies are installed. + installed_tool_shed_repositories = [] + # Get all of the information necessary for installing the repository from the specified tool shed. + url = suc.url_join( tool_shed_url, + 'api/repositories/get_repository_revision_install_info?name=%s&owner=%s&changeset_revision=%s' % \ + ( name, owner, changeset_revision ) ) + try: + response = urllib2.urlopen( url ) + raw_text = response.read() + response.close() + except Exception, e: + message = "Error attempting to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ + ( str( tool_shed_url ), str( name ), str( owner ), str( changeset_revision ), str( e ) ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return dict( status='error', error=message ) + if raw_text: + items = json.from_json_string( raw_text ) + repository_dict = items[ 0 ] + repository_revision_dict = items[ 1 ] + repo_info_dict = items[ 2 ] + else: + message = "Unable to retrieve installation information from tool shed %s for revision %s of repository %s owned by %s: %s" % \ + ( str( tool_shed_url ), str( name ), str( owner ), str( changeset_revision ) ) + log.error( message, exc_info=True ) + trans.response.status = 500 + return dict( status='error', error=message ) + repo_info_dicts = [ repo_info_dict ] + # Make sure the tool shed returned everything we need for installing the repository. + try: + has_repository_dependencies = repository_revision_dict[ 'has_repository_dependencies' ] + except: + raise HTTPBadRequest( detail="Missing required parameter 'has_repository_dependencies'." ) + try: + includes_tools = repository_revision_dict[ 'includes_tools' ] + except: + raise HTTPBadRequest( detail="Missing required parameter 'includes_tools'." ) + try: + includes_tool_dependencies = repository_revision_dict[ 'includes_tool_dependencies' ] + except: + raise HTTPBadRequest( detail="Missing required parameter 'includes_tool_dependencies'." ) + try: + includes_tools_for_display_in_tool_panel = repository_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] + except: + raise HTTPBadRequest( detail="Missing required parameter 'includes_tools_for_display_in_tool_panel'." ) + # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository + # information. + install_repository_dependencies = payload.get( 'install_repository_dependencies', False ) + install_tool_dependencies = payload.get( 'install_tool_dependencies', False ) + new_tool_panel_section = payload.get( 'new_tool_panel_section_label', '' ) + shed_tool_conf = payload.get( 'shed_tool_conf', None ) + tool_path = payload.get( 'tool_path', None ) + tool_panel_section_id = payload.get( 'tool_panel_section_id', '' ) + if tool_panel_section_id is not None: + tool_panel_section = trans.app.toolbox.tool_panel[ tool_panel_section_id ] + else: + tool_panel_section = '' + if not shed_tool_conf or not tool_path: + # Pick a semi-random shed-related tool panel configuration file. + for shed_config_dict in trans.app.toolbox.shed_tool_confs: + if shed_config_dict[ 'config_filename' ] != trans.app.config.migrated_tools_config: + break + shed_tool_conf = shed_config_dict[ 'config_filename' ] + tool_path = shed_config_dict[ 'tool_path' ] + if not shed_tool_conf: + raise HTTPBadRequest( detail="Missing required parameter 'shed_tool_conf'." ) + if not tool_path: + raise HTTPBadRequest( detail="Missing required parameter 'tool_path'." ) + # Build the dictionary of information necessary for creating tool_shed_repository database records for each repository being installed. + installation_dict = dict( install_repository_dependencies=install_repository_dependencies, + new_tool_panel_section=new_tool_panel_section, + no_changes_checked=False, + reinstalling=False, + repo_info_dicts=repo_info_dicts, + tool_panel_section=tool_panel_section, + tool_path=tool_path, + tool_shed_url=tool_shed_url ) + # Create the tool_shed_repository database records and gather additional information for repository installation. + created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \ + repository_util.handle_tool_shed_repositories( trans, installation_dict, using_api=True ) + if message and len( repo_info_dicts ) == 1: + # We're attempting to install a single repository that has already been installed into this Galaxy instance. + log.error( message, exc_info=True ) + trans.response.status = 500 + return dict( status='error', error=message ) + if created_or_updated_tool_shed_repositories: + # Build the dictionary of information necessary for installing the repositories. + installation_dict = dict( created_or_updated_tool_shed_repositories=created_or_updated_tool_shed_repositories, + filtered_repo_info_dicts=filtered_repo_info_dicts, + has_repository_dependencies=has_repository_dependencies, + includes_tool_dependencies=includes_tool_dependencies, + includes_tools=includes_tools, + includes_tools_for_display_in_tool_panel=includes_tools_for_display_in_tool_panel, + install_repository_dependencies=install_repository_dependencies, + install_tool_dependencies=install_tool_dependencies, + message='', + new_tool_panel_section=new_tool_panel_section, + shed_tool_conf=shed_tool_conf, + status='done', + tool_panel_section=tool_panel_section, + tool_panel_section_keys=tool_panel_section_keys, + tool_path=tool_path, + tool_shed_url=tool_shed_url ) + # Prepare the repositories for installation. Even though this method receives a single combination of tool_shed_url, name, owner and + # changeset_revision, there may be multiple repositories for installation at this point because repository dependencies may have added + # additional repositories for installation along with the single specified repository. + encoded_kwd, query, tool_shed_repositories, encoded_repository_ids = repository_util.initiate_repository_installation( trans, installation_dict ) + # Install the repositories, keeping track of each one for later display. + for index, tool_shed_repository in enumerate( tool_shed_repositories ): + repo_info_dict = repo_info_dicts[ index ] + tool_panel_section_key = tool_panel_section_keys[ index ] + repository_util.install_tool_shed_repository( trans, + tool_shed_repository, + repo_info_dict, + tool_panel_section_key, + shed_tool_conf, + tool_path, + install_tool_dependencies, + reinstalling=False ) + tool_shed_repository_dict = tool_shed_repository.as_dict( value_mapper=default_tool_shed_repository_value_mapper( trans, tool_shed_repository ) ) + tool_shed_repository_dict[ 'url' ] = web.url_for( controller='tool_shed_repositories', + action='show', + id=trans.security.encode_id( tool_shed_repository.id ) ) + installed_tool_shed_repositories.append( tool_shed_repository_dict ) + else: + log.error( message, exc_info=True ) + trans.response.status = 500 + return dict( status='error', error=message ) + # Display the list of installed repositories. + return installed_tool_shed_repositories + + @web.expose_api + def install_repository_revisions( self, trans, payload, **kwd ): + """ + POST /api/tool_shed_repositories/install_repository_revisions + Install one or more specified repository revisions from one or more specified tool sheds into Galaxy. The received parameters + must be ordered lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated. + + It's questionable whether this method is needed as the above method for installing a single repository can probably cover all + desired scenarios. We'll keep this one around just in case... + + :param tool_shed_urls: the base URLs of the Tool Sheds from which to install a specified Repository + :param names: the names of the Repositories to be installed + :param owners: the owners of the Repositories to be installed + :param changset_revisions: the changset_revisions of each RepositoryMetadata object associated with each Repository to be installed + :param key: the current Galaxy admin user's API key + :param new_tool_panel_section_label: optional label of a new section to be added to the Galaxy tool panel in which to load + tools contained in the Repository. Either this parameter must be an empty string or + the tool_panel_section_id parameter must be an empty string, as both cannot be used. + :param tool_panel_section_id: optional id of the Galaxy tool panel section in which to load tools contained in the Repository. + If not set, tools will be loaded outside of any sections in the tool panel. Either this + parameter must be an empty string or the tool_panel_section_id parameter must be an empty string, + as both cannot be used. + """ + if not suc.have_shed_tool_conf_for_install( trans ): + # This Galaxy instance is not configured with a shed-related tool panel configuration file. + message = get_message_for_no_shed_tool_config() + log.error( message, exc_info=True ) + trans.response.status = 500 + return dict( status='error', error=message ) + if not trans.user_is_admin(): + raise HTTPForbidden( detail='You are not authorized to install a tool shed repository into this Galaxy instance.' ) + # Get the information about all of the repositories to be installed. + tool_shed_urls = util.listify( payload.get( 'tool_shed_urls', '' ) ) + names = util.listify( payload.get( 'names', '' ) ) + owners = util.listify( payload.get( 'owners', '' ) ) + changeset_revisions = util.listify( payload.get( 'changeset_revisions', '' ) ) + num_specified_repositories = len( tool_shed_urls ) + if len( names ) != num_specified_repositories or \ + len( owners ) != num_specified_repositories or \ + len( changeset_revisions ) != num_specified_repositories: + message = 'Error in tool_shed_repositories API in install_repository_revisions: the received parameters must be ordered ' + message += 'lists so that positional values in tool_shed_urls, names, owners and changeset_revisions are associated.' + log.error( message, exc_info=True ) + trans.response.status = 500 + return dict( status='error', error=message ) + # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain information + # about each of the repositories being installed. + # TODO: we may want to enhance this method to allow for each of the following to be associated with each repository instead of + # forcing all repositories to use the same settings. + install_repository_dependencies = payload.get( 'install_repository_dependencies', False ) + install_tool_dependencies = payload.get( 'install_tool_dependencies', False ) + new_tool_panel_section = payload.get( 'new_tool_panel_section_label', '' ) + shed_tool_conf = payload.get( 'shed_tool_conf', None ) + tool_path = payload.get( 'tool_path', None ) + tool_panel_section_id = payload.get( 'tool_panel_section_id', '' ) + all_installed_tool_shed_repositories = [] + for index, tool_shed_url in enumerate( tool_shed_urls ): + current_payload = {} + current_payload[ 'tool_shed_url' ] = tool_shed_url + current_payload[ 'name' ] = names[ index ] + current_payload[ 'owner' ] = owners[ index ] + current_payload[ 'changeset_revision' ] = changeset_revisions[ index ] + current_payload[ 'install_repository_dependencies' ] = install_repository_dependencies + current_payload[ 'install_tool_dependencies' ] = install_tool_dependencies + current_payload[ 'new_tool_panel_section' ] = new_tool_panel_section + current_payload[ 'shed_tool_conf' ] = shed_tool_conf + current_payload[ 'tool_path' ] = tool_path + current_payload[ 'tool_panel_section_id' ] = tool_panel_section_id + installed_tool_shed_repositories = self.install_repository_revision( trans, **current_payload ) + if isinstance( installed_tool_shed_repositories, dict ): + # We encountered an error. + return installed_tool_shed_repositories + elif isinstance( installed_tool_shed_repositories, list ): + all_installed_tool_shed_repositories.extend( installed_tool_shed_repositories ) + return all_installed_tool_shed_repositories + \ No newline at end of file diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index 6435e08369f..800d3bca8b9 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -104,12 +104,6 @@ def app_factory( global_conf, **kwargs ): name_prefix='group_', path_prefix='/api/groups/:group_id', parent_resources=dict( member_name='group', collection_name='groups' ) ) - webapp.api_mapper.resource( 'content', - 'contents', - controller='tool_shed_repository_contents', - name_prefix='tool_shed_repository_', - path_prefix='/api/tool_shed_repositories/:tool_shed_repository_id', - parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) ) _add_item_tags_controller( webapp, name_prefix="history_content_", path_prefix='/api/histories/:history_id/contents/:history_content_id' ) @@ -142,7 +136,6 @@ def app_factory( global_conf, **kwargs ): webapp.api_mapper.resource( 'tool', 'tools', path_prefix='/api' ) webapp.api_mapper.resource_with_deleted( 'user', 'users', path_prefix='/api' ) webapp.api_mapper.resource( 'genome', 'genomes', path_prefix='/api' ) - webapp.api_mapper.resource( 'tool_shed_repository', 'tool_shed_repositories', path_prefix='/api' ) webapp.api_mapper.resource( 'visualization', 'visualizations', path_prefix='/api' ) webapp.api_mapper.resource( 'workflow', 'workflows', path_prefix='/api' ) webapp.api_mapper.resource_with_deleted( 'history', 'histories', path_prefix='/api' ) @@ -155,7 +148,14 @@ def app_factory( global_conf, **kwargs ): webapp.api_mapper.connect("workflow_dict", '/api/workflows/{workflow_id}/download', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) # Preserve the following download route for now for dependent applications -- deprecate at some point webapp.api_mapper.connect("workflow_dict", '/api/workflows/download/{workflow_id}', controller='workflows', action='workflow_dict', conditions=dict(method=['GET'])) - + # Galaxy API for tool shed features. + webapp.api_mapper.resource( 'tool_shed_repository', + 'tool_shed_repositories', + controller='tool_shed_repositories', + name_prefix='tool_shed_repository_', + path_prefix='/api', + new={ 'install_repository_revision' : 'POST' }, + parent_resources=dict( member_name='tool_shed_repository', collection_name='tool_shed_repositories' ) ) # Connect logger from app if app.trace_logger: webapp.trace_logger = app.trace_logger diff --git a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py index 1c2f9e3d476..dc4cc69cf84 100644 --- a/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py +++ b/lib/galaxy/webapps/galaxy/controllers/admin_toolshed.py @@ -846,7 +846,7 @@ class AdminToolshed( AdminGalaxy ): tool_path=tool_path, tool_shed_url=tool_shed_url ) created_or_updated_tool_shed_repositories, tool_panel_section_keys, repo_info_dicts, filtered_repo_info_dicts, message = \ - repository_util.handle_tool_shed_repositories( trans, installation_dict ) + repository_util.handle_tool_shed_repositories( trans, installation_dict, using_api=False ) if message and len( repo_info_dicts ) == 1: return trans.response.send_redirect( web.url_for( controller='admin_toolshed', action='browse_repositories', diff --git a/lib/galaxy/webapps/tool_shed/api/repositories.py b/lib/galaxy/webapps/tool_shed/api/repositories.py index e8d4acc64f8..e9084765f61 100644 --- a/lib/galaxy/webapps/tool_shed/api/repositories.py +++ b/lib/galaxy/webapps/tool_shed/api/repositories.py @@ -1,9 +1,10 @@ import logging from galaxy.web.framework.helpers import time_ago -import tool_shed.util.shed_util_common as suc from galaxy import web from galaxy import util from galaxy.web.base.controller import BaseAPIController +import tool_shed.util.shed_util_common as suc +from tool_shed.galaxy_install import repository_util from galaxy import eggs import pkg_resources @@ -29,7 +30,7 @@ def default_repository_metadata_value_mapper( trans, repository_metadata ): class RepositoriesController( BaseAPIController ): """RESTful controller for interactions with repositories in the Tool Shed.""" - @web.expose_api + @web.expose_api_anonymous def index( self, trans, deleted=False, **kwd ): """ GET /api/repositories @@ -56,7 +57,7 @@ class RepositoriesController( BaseAPIController ): trans.response.status = 500 return message - @web.expose_api + @web.expose_api_anonymous def show( self, trans, id, **kwd ): """ GET /api/repositories/{encoded_repository_id} @@ -78,17 +79,59 @@ class RepositoriesController( BaseAPIController ): trans.response.status = 500 return message - @web.expose_api - def get_repository_and_revision( self, trans, name, owner, changeset_revision, **kwd ): + @web.expose_api_anonymous + def get_repository_revision_install_info( self, trans, name, owner, changeset_revision, **kwd ): """ - GET /api/repository/get_repository_and_revision - Returns information about a repository revision in the Tool Shed. + GET /api/repository/get_repository_revision_install_info - :param name: the name of the Repository object - :param owner: the owner of the Repository object - :param changset_revision: the changset_revision of the RepositoryMetadata object associated with the Repository object + :param name: the name of the Repository + :param owner: the owner of the Repository + :param changset_revision: the changset_revision of the RepositoryMetadata object associated with the Repository + + Returns a list of the following dictionaries:: + - a dictionary defining the Repository. For example: + { + "deleted": false, + "deprecated": false, + "description": "add_column hello", + "id": "f9cad7b01a472135", + "long_description": "add_column hello", + "name": "add_column", + "owner": "test", + "private": false, + "times_downloaded": 6, + "url": "/api/repositories/f9cad7b01a472135", + "user_id": "f9cad7b01a472135" + } + - a dictionary defining the Repsoitory revision (RepositoryMetadata). For example: + { + "changeset_revision": "3a08cc21466f", + "downloadable": true, + "has_repository_dependencies": false, + "id": "f9cad7b01a472135", + "includes_datatypes": false, + "includes_tool_dependencies": false, + "includes_tools": true, + "includes_tools_for_display_in_tool_panel": true, + "includes_workflows": false, + "malicious": false, + "repository_id": "f9cad7b01a472135", + "url": "/api/repository_revisions/f9cad7b01a472135" + } + - a dictionary including the additional information required to install the repository. For example: + { + "add_column": [ + "add_column hello", + "http://test@localhost:9009/repos/test/add_column", + "3a08cc21466f", + "1", + "test", + {}, + {} + ] + } """ - # Example URL: http://localhost:9009/api/repositories/get_repository_and_revision?name=add_column&owner=test&changeset_revision=3a08cc21466f + # Example URL: http://localhost:9009/api/repositories/get_repository_revision_install_info?name=add_column&owner=test&changeset_revision=3a08cc21466f try: # Get the repository information. repository = suc.get_repository_by_name_and_owner( trans.app, name, owner ) @@ -106,6 +149,7 @@ class RepositoriesController( BaseAPIController ): repo = hg.repository( suc.get_configured_ui(), repo_dir ) new_changeset_revision = suc.get_next_downloadable_changeset_revision( repository, repo, changeset_revision ) repository_metadata = suc.get_repository_metadata_by_changeset_revision( trans, encoded_repository_id, new_changeset_revision ) + changeset_revision = new_changeset_revision if repository_metadata: encoded_repository_metadata_id = trans.security.encode_id( repository_metadata.id ) repository_metadata_dict = repository_metadata.get_api_value( view='collection', @@ -113,14 +157,17 @@ class RepositoriesController( BaseAPIController ): repository_metadata_dict[ 'url' ] = web.url_for( controller='repository_revisions', action='show', id=encoded_repository_metadata_id ) - return repository_dict, repository_metadata_dict + # Get the repo_info_dict for installing the repository. + repo_info_dict, includes_tools, includes_tool_dependencies, includes_tools_for_display_in_tool_panel, has_repository_dependencies = \ + repository_util.get_repo_info_dict( trans, encoded_repository_id, changeset_revision ) + return repository_dict, repository_metadata_dict, repo_info_dict else: message = "Unable to locate repository_metadata record for repository id %d and changeset_revision %s" % ( repository.id, changeset_revision ) log.error( message, exc_info=True ) trans.response.status = 500 - return repository_dict, {} + return repository_dict, {}, {} except Exception, e: - message = "Error in the Tool Shed repositories API in get_repository_and_revision: %s" % str( e ) + message = "Error in the Tool Shed repositories API in get_repository_revision_install_info: %s" % str( e ) log.error( message, exc_info=True ) trans.response.status = 500 return message diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index a7f5e86131b..37982573fb7 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -77,7 +77,7 @@ def app_factory( global_conf, **kwargs ): webapp.api_mapper.resource( 'repository', 'repositories', controller='repositories', - collection={ 'get_repository_and_revision' : 'GET' }, + collection={ 'get_repository_revision_install_info' : 'GET' }, name_prefix='repository_', path_prefix='/api', parent_resources=dict( member_name='repository', collection_name='repositories' ) ) diff --git a/lib/tool_shed/galaxy_install/repository_util.py b/lib/tool_shed/galaxy_install/repository_util.py index 0322bbf53f8..53854311b8c 100644 --- a/lib/tool_shed/galaxy_install/repository_util.py +++ b/lib/tool_shed/galaxy_install/repository_util.py @@ -252,7 +252,7 @@ def handle_repository_contents( trans, tool_shed_repository, tool_path, reposito # Load proprietary datatype display applications trans.app.datatypes_registry.load_display_applications( installed_repository_dict=repository_dict ) -def handle_tool_shed_repositories( trans, installation_dict ): +def handle_tool_shed_repositories( trans, installation_dict, using_api=False ): # The following installation_dict entries are all required. install_repository_dependencies = installation_dict[ 'install_repository_dependencies' ] new_tool_panel_section = installation_dict[ 'new_tool_panel_section' ] @@ -272,7 +272,7 @@ def handle_tool_shed_repositories( trans, installation_dict ): no_changes_checked=no_changes_checked, tool_panel_section=tool_panel_section, new_tool_panel_section=new_tool_panel_section ) - if message and len( repo_info_dicts ) == 1: + if message and len( repo_info_dicts ) == 1 and not using_api: installed_tool_shed_repository = created_or_updated_tool_shed_repositories[ 0 ] message += 'Click here to manage the repository. ' % \ ( web.url_for( controller='admin_toolshed', action='manage_repository', id=trans.security.encode_id( installed_tool_shed_repository.id ) ) ) diff --git a/scripts/api/common.py b/scripts/api/common.py index 541fcad0710..b39a0b136a2 100644 --- a/scripts/api/common.py +++ b/scripts/api/common.py @@ -1,4 +1,7 @@ -import os, sys, urllib, urllib2 +import logging +import os +import sys +import urllib2 new_path = [ os.path.join( os.path.dirname( __file__ ), '..', '..', 'lib' ) ] new_path.extend( sys.path[1:] ) @@ -15,6 +18,8 @@ from Crypto.Cipher import Blowfish from Crypto.Util.randpool import RandomPool from Crypto.Util import number +log = logging.getLogger( __name__ ) + def make_url( api_key, url, args=None ): # Adds the API Key to the URL if it's not already there. if args is None: diff --git a/scripts/api/install_repository_tools_into_existing_tool_panel_section.py b/scripts/api/install_repository_tools_into_existing_tool_panel_section.py new file mode 100755 index 00000000000..a326933e65e --- /dev/null +++ b/scripts/api/install_repository_tools_into_existing_tool_panel_section.py @@ -0,0 +1,40 @@ +#!/usr/bin/env python +""" +Install a specified repository revision from a specified tool shed into Galaxy. This example demonstrates installation of a repository that contains +valid tools, loading them into an existing section of the Galaxy tool panel. The repository has no tool dependencies or repository dependencies, so only +a single repository will be installed. + +This example requires a tool panel config file (e.g., tool_conf.xml, shed_tool_conf.xml, etc) to contain a tool panel section like the following: + +
+
+ +usage: ./install_repository_tools_into_existing_tool_panel_section tool_shed_url name owner changeset_revision tool_panel_section_id + +Here is a working example of how to use this script to install a repository from the test tool shed. +./install_repository_tools_into_existing_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb from_test_tool_shed +""" + +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +try: + assert sys.argv[ 7 ] +except IndexError: + print 'usage: %s key url tool_shed_url name owner changeset_revision tool_panel_section_id' % os.path.basename( sys.argv[ 0 ] ) + sys.exit( 1 ) + +try: + data = {} + data[ 'tool_shed_url' ] = sys.argv[ 3 ] + data[ 'name' ] = sys.argv[ 4 ] + data[ 'owner' ] = sys.argv[ 5 ] + data[ 'changeset_revision' ] = sys.argv[ 6 ] + data[ 'tool_panel_section_id' ] = sys.argv[ 7 ] +except IndexError: + print 'usage: %s key url tool_shed_url name owner changeset_revision tool_panel_section_id' % os.path.basename( sys.argv[ 0 ] ) + sys.exit( 1 ) + +submit( sys.argv[ 1 ], sys.argv[ 2 ], data ) diff --git a/scripts/api/install_repository_tools_into_new_tool_panel_section.py b/scripts/api/install_repository_tools_into_new_tool_panel_section.py new file mode 100644 index 00000000000..4c47def9c15 --- /dev/null +++ b/scripts/api/install_repository_tools_into_new_tool_panel_section.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Install a specified repository revision from a specified tool shed into Galaxy. This example demonstrates installation of a repository that contains +valid tools, loading them into a new section of the Galaxy tool panel. The repository has no tool dependencies or repository dependencies, so only +a single repository will be installed. + +usage: ./install_repository_tools_into_existing_tool_panel_section tool_shed_url name owner changeset_revision new_tool_panel_section_label + +Here is a working example of how to use this script to install a repository from the test tool shed. +./install_repository_tools_into_existing_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb From%20Test%20Tool%20Shed +""" + +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +try: + assert sys.argv[ 7 ] +except IndexError: + print 'usage: %s key url tool_shed_url name owner changeset_revision tool_panel_section_id' % os.path.basename( sys.argv[ 0 ] ) + sys.exit( 1 ) + +try: + data = {} + data[ 'tool_shed_url' ] = sys.argv[ 3 ] + data[ 'name' ] = sys.argv[ 4 ] + data[ 'owner' ] = sys.argv[ 5 ] + data[ 'changeset_revision' ] = sys.argv[ 6 ] + data[ 'new_tool_panel_section_label' ] = sys.argv[ 7 ] +except IndexError: + print 'usage: %s key url tool_shed_url name owner changeset_revision new_tool_panel_section_label' % os.path.basename( sys.argv[ 0 ] ) + sys.exit( 1 ) + +submit( sys.argv[ 1 ], sys.argv[ 2 ], data ) diff --git a/scripts/api/install_repository_with_repository_dependencies.py b/scripts/api/install_repository_with_repository_dependencies.py new file mode 100644 index 00000000000..1724034bd20 --- /dev/null +++ b/scripts/api/install_repository_with_repository_dependencies.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python +""" +Install a specified repository revision from a specified tool shed into Galaxy. This example demonstrates installation of a repository that has +repository dependencies, so multiple repositories will ultimately be installed. Since no Galaxy tool panel section information is used, all tools +contained in the installed repositories will be loaded into the Galaxy tool panel outside of any sections. + +usage: ./install_repository_tools_into_existing_tool_panel_section tool_shed_url name owner changeset_revision True + +Here is a working example of how to use this script to install a repository from the test tool shed. +./install_repository_tools_into_existing_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu emboss_5 devteam 8ddad0c9a75a True +""" + +import os +import sys +sys.path.insert( 0, os.path.dirname( __file__ ) ) +from common import submit + +try: + assert sys.argv[ 7 ] +except IndexError: + print 'usage: %s key url tool_shed_url name owner changeset_revision install_repository_dependencies' % os.path.basename( sys.argv[ 0 ] ) + sys.exit( 1 ) + +try: + data = {} + data[ 'tool_shed_url' ] = sys.argv[ 3 ] + data[ 'name' ] = sys.argv[ 4 ] + data[ 'owner' ] = sys.argv[ 5 ] + data[ 'changeset_revision' ] = sys.argv[ 6 ] + data[ 'install_repository_dependencies' ] = sys.argv[ 7 ] +except IndexError: + print 'usage: %s key url tool_shed_url name owner changeset_revision install_repository_dependencies' % os.path.basename( sys.argv[ 0 ] ) + sys.exit( 1 ) + +submit( sys.argv[ 1 ], sys.argv[ 2 ], data ) \ No newline at end of file From ed7248a819ecdde5096e0f48e417a253feb38f1d Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Mon, 25 Mar 2013 17:18:17 -0400 Subject: [PATCH 16/21] Fix a few typos in my last commit. --- lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py | 2 +- .../install_repository_tools_into_new_tool_panel_section.py | 4 ++-- .../api/install_repository_with_repository_dependencies.py | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) mode change 100644 => 100755 scripts/api/install_repository_tools_into_new_tool_panel_section.py mode change 100644 => 100755 scripts/api/install_repository_with_repository_dependencies.py diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 9de96018a2c..93282d9e40f 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -168,7 +168,7 @@ class ToolShedRepositoriesController( BaseAPIController ): shed_tool_conf = payload.get( 'shed_tool_conf', None ) tool_path = payload.get( 'tool_path', None ) tool_panel_section_id = payload.get( 'tool_panel_section_id', '' ) - if tool_panel_section_id is not None: + if tool_panel_section_id not in [ None, '' ]: tool_panel_section = trans.app.toolbox.tool_panel[ tool_panel_section_id ] else: tool_panel_section = '' diff --git a/scripts/api/install_repository_tools_into_new_tool_panel_section.py b/scripts/api/install_repository_tools_into_new_tool_panel_section.py old mode 100644 new mode 100755 index 4c47def9c15..6c946fe2dfb --- a/scripts/api/install_repository_tools_into_new_tool_panel_section.py +++ b/scripts/api/install_repository_tools_into_new_tool_panel_section.py @@ -4,10 +4,10 @@ Install a specified repository revision from a specified tool shed into Galaxy. valid tools, loading them into a new section of the Galaxy tool panel. The repository has no tool dependencies or repository dependencies, so only a single repository will be installed. -usage: ./install_repository_tools_into_existing_tool_panel_section tool_shed_url name owner changeset_revision new_tool_panel_section_label +usage: ./install_repository_tools_into_new_tool_panel_section tool_shed_url name owner changeset_revision new_tool_panel_section_label Here is a working example of how to use this script to install a repository from the test tool shed. -./install_repository_tools_into_existing_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb From%20Test%20Tool%20Shed +./install_repository_tools_into_new_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb From%20Test%20Tool%20Shed """ import os diff --git a/scripts/api/install_repository_with_repository_dependencies.py b/scripts/api/install_repository_with_repository_dependencies.py old mode 100644 new mode 100755 index 1724034bd20..e9c5c0d4a17 --- a/scripts/api/install_repository_with_repository_dependencies.py +++ b/scripts/api/install_repository_with_repository_dependencies.py @@ -4,10 +4,10 @@ Install a specified repository revision from a specified tool shed into Galaxy. repository dependencies, so multiple repositories will ultimately be installed. Since no Galaxy tool panel section information is used, all tools contained in the installed repositories will be loaded into the Galaxy tool panel outside of any sections. -usage: ./install_repository_tools_into_existing_tool_panel_section tool_shed_url name owner changeset_revision True +usage: ./install_repository_with_repository_dependencies.py tool_shed_url name owner changeset_revision True Here is a working example of how to use this script to install a repository from the test tool shed. -./install_repository_tools_into_existing_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu emboss_5 devteam 8ddad0c9a75a True +./install_repository_with_repository_dependencies.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu emboss_5 devteam 8ddad0c9a75a True """ import os @@ -32,4 +32,4 @@ except IndexError: print 'usage: %s key url tool_shed_url name owner changeset_revision install_repository_dependencies' % os.path.basename( sys.argv[ 0 ] ) sys.exit( 1 ) -submit( sys.argv[ 1 ], sys.argv[ 2 ], data ) \ No newline at end of file +submit( sys.argv[ 1 ], sys.argv[ 2 ], data ) From 86faef99b43ef3ddde6b7de2eefad98fa16683fb Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 26 Mar 2013 09:27:39 -0400 Subject: [PATCH 17/21] More explicit specification of return codes, documentation of same. --- .../functional_tests.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index f4c300dc077..2c592e0f48d 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -787,14 +787,19 @@ def main(): print '# ----------------------------------------------------------------------------------' print "# %d repositories not installed correctly:" % len( repositories_failed_install ) show_summary_output( repositories_failed_install ) - print "####################################################################################" - if repositories_tested > 0: - if success: - return 0 else: - return 1 + success = True else: + success = True + print "####################################################################################" + # Normally, the value of 'success' would determine whether this test suite is marked as passed or failed + # in the automated buildbot framework. However, due to the procedure used here, we only want to report + # failure if a repository fails to install correctly. Therefore, we have overriden the value of 'success' + # here based on what actions the script has executed. + if success: return 0 + else: + return 1 if __name__ == "__main__": now = strftime( "%Y-%m-%d %H:%M:%S" ) From 5d502906e51498644250193ab6a21ef492910462 Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Tue, 26 Mar 2013 09:29:02 -0400 Subject: [PATCH 18/21] Enhance and correct documentation in the new tool_shed_repositories Galaxy API controller as well as the documentation in the example API install scripts. Fix discovery of the tool_path setting in the selected shed-related tool panel config file in the tool_shed_repositories controller's install_repository_revision method. --- .../galaxy/api/tool_shed_repositories.py | 61 ++++++++++++------- ..._tools_into_existing_tool_panel_section.py | 2 +- ...itory_tools_into_new_tool_panel_section.py | 4 +- 3 files changed, 41 insertions(+), 26 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py index 93282d9e40f..9541045c5cf 100644 --- a/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py +++ b/lib/galaxy/webapps/galaxy/api/tool_shed_repositories.py @@ -58,7 +58,7 @@ class ToolShedRepositoriesController( BaseAPIController ): GET /api/tool_shed_repositories/{encoded_tool_shed_repsository_id} Display a dictionary containing information about a specified tool_shed_repository. - :param tool_shed_repository_id: the encoded id of the `ToolShedRepository` object + :param id: the encoded id of the ToolShedRepository object """ # Example URL: http://localhost:8763/api/tool_shed_repositories/df7a1f0c02a5b08e try: @@ -80,18 +80,32 @@ class ToolShedRepositoriesController( BaseAPIController ): POST /api/tool_shed_repositories/install_repository_revision Install a specified repository revision from a specified tool shed into Galaxy. - :param tool_shed_url: the base URL of the Tool Shed from which to install the Repository - :param name: the name of the Repository - :param owner: the owner of the Repository - :param changset_revision: the changset_revision of the RepositoryMetadata object associated with the Repository :param key: the current Galaxy admin user's API key - :param new_tool_panel_section_label: optional label of a new section to be added to the Galaxy tool panel in which to load - tools contained in the Repository. Either this parameter must be an empty string or - the tool_panel_section_id parameter must be an empty string, as both cannot be used. - :param tool_panel_section_id: optional id of the Galaxy tool panel section in which to load tools contained in the Repository. - If not set, tools will be loaded outside of any sections in the tool panel. Either this - parameter must be an empty string or the tool_panel_section_id parameter must be an empty string, - as both cannot be used. + + The following parameters are included in the payload. + :param tool_shed_url (required): the base URL of the Tool Shed from which to install the Repository + :param name (required): the name of the Repository + :param owner (required): the owner of the Repository + :param changset_revision (required): the changset_revision of the RepositoryMetadata object associated with the Repository + :param new_tool_panel_section_label (optional): label of a new section to be added to the Galaxy tool panel in which to load + tools contained in the Repository. Either this parameter must be an empty string or + the tool_panel_section_id parameter must be an empty string or both must be an empty + string (both cannot be used simultaneously). + :param tool_panel_section_id (optional): id of the Galaxy tool panel section in which to load tools contained in the Repository. + If this parameter is an empty string and the above new_tool_panel_section_label parameter is an + empty string, tools will be loaded outside of any sections in the tool panel. Either this + parameter must be an empty string or the tool_panel_section_id parameter must be an empty string + of both must be an empty string (both cannot be used simultaneously). + :param install_repository_dependencies (optional): Set to True if you want to install repository dependencies defined for the specified + repository being installed. The default setting is False. + :param install_tool_dependencies (optional): Set to True if you want to install tool dependencies defined for the specified repository being + installed. The default setting is False. + :param shed_tool_conf (optional): The shed-related tool panel configuration file configured in the "tool_config_file" setting in the Galaxy config file + (e.g., universe_wsgi.ini). At least one shed-related tool panel config file is required to be configured. Setting + this parameter to a specific file enables you to choose where the specified repository will be installed because + the tool_path attribute of the from the specified file is used as the installation location + (e.g., ). If this parameter is not set, a shed-related tool panel configuration + file will be selected automatically. """ # Get the information about the repository to be installed from the payload. tool_shed_url = payload.get( 'tool_shed_url', '' ) @@ -160,29 +174,30 @@ class ToolShedRepositoriesController( BaseAPIController ): includes_tools_for_display_in_tool_panel = repository_revision_dict[ 'includes_tools_for_display_in_tool_panel' ] except: raise HTTPBadRequest( detail="Missing required parameter 'includes_tools_for_display_in_tool_panel'." ) - # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository - # information. + # Get the information about the Galaxy components (e.g., tool pane section, tool config file, etc) that will contain the repository information. install_repository_dependencies = payload.get( 'install_repository_dependencies', False ) install_tool_dependencies = payload.get( 'install_tool_dependencies', False ) new_tool_panel_section = payload.get( 'new_tool_panel_section_label', '' ) shed_tool_conf = payload.get( 'shed_tool_conf', None ) - tool_path = payload.get( 'tool_path', None ) - tool_panel_section_id = payload.get( 'tool_panel_section_id', '' ) - if tool_panel_section_id not in [ None, '' ]: - tool_panel_section = trans.app.toolbox.tool_panel[ tool_panel_section_id ] + if shed_tool_conf: + # Get the tool_path setting. + index, shed_conf_dict = suc.get_shed_tool_conf_dict( trans.app, shed_tool_conf ) + tool_path = shed_config_dict[ 'tool_path' ] else: - tool_panel_section = '' - if not shed_tool_conf or not tool_path: - # Pick a semi-random shed-related tool panel configuration file. + # Pick a semi-random shed-related tool panel configuration file and get the tool_path setting. for shed_config_dict in trans.app.toolbox.shed_tool_confs: + # Don't use migrated_tools_conf.xml. if shed_config_dict[ 'config_filename' ] != trans.app.config.migrated_tools_config: break shed_tool_conf = shed_config_dict[ 'config_filename' ] tool_path = shed_config_dict[ 'tool_path' ] if not shed_tool_conf: raise HTTPBadRequest( detail="Missing required parameter 'shed_tool_conf'." ) - if not tool_path: - raise HTTPBadRequest( detail="Missing required parameter 'tool_path'." ) + tool_panel_section_id = payload.get( 'tool_panel_section_id', '' ) + if tool_panel_section_id not in [ None, '' ]: + tool_panel_section = trans.app.toolbox.tool_panel[ tool_panel_section_id ] + else: + tool_panel_section = '' # Build the dictionary of information necessary for creating tool_shed_repository database records for each repository being installed. installation_dict = dict( install_repository_dependencies=install_repository_dependencies, new_tool_panel_section=new_tool_panel_section, diff --git a/scripts/api/install_repository_tools_into_existing_tool_panel_section.py b/scripts/api/install_repository_tools_into_existing_tool_panel_section.py index a326933e65e..b7caae89e1f 100755 --- a/scripts/api/install_repository_tools_into_existing_tool_panel_section.py +++ b/scripts/api/install_repository_tools_into_existing_tool_panel_section.py @@ -9,7 +9,7 @@ This example requires a tool panel config file (e.g., tool_conf.xml, shed_tool_c
-usage: ./install_repository_tools_into_existing_tool_panel_section tool_shed_url name owner changeset_revision tool_panel_section_id +usage: ./install_repository_tools_into_existing_tool_panel_section.py tool_shed_url name owner changeset_revision tool_panel_section_id Here is a working example of how to use this script to install a repository from the test tool shed. ./install_repository_tools_into_existing_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb from_test_tool_shed diff --git a/scripts/api/install_repository_tools_into_new_tool_panel_section.py b/scripts/api/install_repository_tools_into_new_tool_panel_section.py index 6c946fe2dfb..d3e451cf222 100755 --- a/scripts/api/install_repository_tools_into_new_tool_panel_section.py +++ b/scripts/api/install_repository_tools_into_new_tool_panel_section.py @@ -4,10 +4,10 @@ Install a specified repository revision from a specified tool shed into Galaxy. valid tools, loading them into a new section of the Galaxy tool panel. The repository has no tool dependencies or repository dependencies, so only a single repository will be installed. -usage: ./install_repository_tools_into_new_tool_panel_section tool_shed_url name owner changeset_revision new_tool_panel_section_label +usage: ./install_repository_tools_into_new_tool_panel_section.py tool_shed_url name owner changeset_revision new_tool_panel_section_label Here is a working example of how to use this script to install a repository from the test tool shed. -./install_repository_tools_into_new_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb From%20Test%20Tool%20Shed +./install_repository_tools_into_new_tool_panel_section.py /api/tool_shed_repositories/new/install_repository_revision http://testtoolshed.g2.bx.psu.edu gregs_filter greg f28d5018f9cb 'From Test Tool Shed' """ import os From 701c0dacb03a1183aed86be6d8235c50443e26b5 Mon Sep 17 00:00:00 2001 From: Dave Bouvier Date: Tue, 26 Mar 2013 10:02:17 -0400 Subject: [PATCH 19/21] Set do_not_test = True if the repository fails functional tests. Set do_not_test = False if the repository passes functional tests, so that the repository will always be re-tested against the most recent code. --- .../functional_tests.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/install_and_test_tool_shed_repositories/functional_tests.py b/test/install_and_test_tool_shed_repositories/functional_tests.py index 2c592e0f48d..3b11eed2226 100644 --- a/test/install_and_test_tool_shed_repositories/functional_tests.py +++ b/test/install_and_test_tool_shed_repositories/functional_tests.py @@ -241,8 +241,7 @@ def get_repositories_to_install( location, source='file', format='json' ): listing = file( location, 'r' ).read() elif source == 'url': assert tool_shed_api_key is not None, 'Cannot proceed without tool shed API key.' - params = urllib.urlencode( dict( tools_functionally_correct='false', - do_not_test='false', + params = urllib.urlencode( dict( do_not_test='false', downloadable='true', malicious='false', includes_tools='true' ) ) @@ -281,10 +280,15 @@ def json_from_url( url ): return from_json_string( url_contents ) def register_test_result( url, metadata_id, test_results_dict, tests_passed=False ): + ''' + Set do_not_test = True if the repository fails functional tests. Set do_not_test = False + if the repository passes functional tests, so that the repository will always be re-tested + against the most recent code. + ''' params = {} if tests_passed: params[ 'tools_functionally_correct' ] = 'true' - params[ 'do_not_test' ] = 'true' + params[ 'do_not_test' ] = 'false' else: params[ 'tools_functionally_correct' ] = 'false' params[ 'do_not_test' ] = 'true' From fa3b6691a6d4b70d829d5974f622d69462c4a5ea Mon Sep 17 00:00:00 2001 From: Greg Von Kuster Date: Tue, 26 Mar 2013 11:41:49 -0400 Subject: [PATCH 20/21] Fix tool shed utility file imports and withe space to follow new Galaxy dev team coding standards. --- lib/tool_shed/util/common_install_util.py | 22 ++++++---- lib/tool_shed/util/common_util.py | 6 ++- lib/tool_shed/util/data_manager_util.py | 3 +- lib/tool_shed/util/datatype_util.py | 16 ++++--- lib/tool_shed/util/encoding_util.py | 6 ++- lib/tool_shed/util/metadata_util.py | 35 ++++++++++------ lib/tool_shed/util/readme_util.py | 6 ++- .../util/repository_dependency_util.py | 20 +++++---- lib/tool_shed/util/review_util.py | 11 +++-- lib/tool_shed/util/tool_dependency_util.py | 9 ++-- lib/tool_shed/util/tool_util.py | 12 ++++-- lib/tool_shed/util/workflow_util.py | 42 +++++++++++++++---- 12 files changed, 128 insertions(+), 60 deletions(-) diff --git a/lib/tool_shed/util/common_install_util.py b/lib/tool_shed/util/common_install_util.py index 44815c9ffa2..d3c86ffe70a 100644 --- a/lib/tool_shed/util/common_install_util.py +++ b/lib/tool_shed/util/common_install_util.py @@ -1,17 +1,25 @@ -import os, logging, urllib2 -import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, data_manager_util, datatype_util, tool_util -from tool_shed.galaxy_install.tool_dependencies.install_util import install_package, set_environment +import logging +import os +import urllib2 +from galaxy import eggs from galaxy import util from galaxy.util import json from galaxy.webapps.tool_shed.util import container_util +import tool_shed.util.shed_util_common as suc +from tool_shed.util import encoding_util +from tool_shed.util import data_manager_util +from tool_shed.util import datatype_util +from tool_shed.util import tool_util +from tool_shed.galaxy_install.tool_dependencies.install_util import install_package +from tool_shed.galaxy_install.tool_dependencies.install_util import set_environment -from galaxy import eggs import pkg_resources pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement +from elementtree import ElementTree +from elementtree import ElementInclude +from elementtree.ElementTree import Element +from elementtree.ElementTree import SubElement log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/common_util.py b/lib/tool_shed/util/common_util.py index 4d4d07affbe..72ccfebeb9f 100644 --- a/lib/tool_shed/util/common_util.py +++ b/lib/tool_shed/util/common_util.py @@ -1,4 +1,5 @@ -import os, urllib2 +import os +import urllib2 from galaxy import util from galaxy.util.odict import odict from tool_shed.util import encoding_util @@ -63,6 +64,7 @@ def check_for_missing_tools( app, tool_panel_configs, latest_tool_migration_scri exception_msg += 'The entry for this tool shed must always be available in this file, so re-add it before attempting to start your Galaxy server.\n' raise Exception( exception_msg ) return tool_shed_accessible, missing_tool_configs_dict + def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_dict ): file_path = elem.get( 'file', None ) if file_path: @@ -71,6 +73,7 @@ def check_tool_tag_set( elem, migrated_tool_configs_dict, missing_tool_configs_d tool_dependencies = migrated_tool_configs_dict[ name ] missing_tool_configs_dict[ name ] = tool_dependencies return missing_tool_configs_dict + def get_non_shed_tool_panel_configs( app ): # Get the non-shed related tool panel configs - there can be more than one, and the default is tool_conf.xml. config_filenames = [] @@ -83,6 +86,7 @@ def get_non_shed_tool_panel_configs( app ): if tool_path is None: config_filenames.append( config_filename ) return config_filenames + def get_tool_shed_url_from_tools_xml_file_path( app, tool_shed ): search_str = '://%s' % tool_shed for shed_name, shed_url in app.tool_shed_registry.tool_sheds.items(): diff --git a/lib/tool_shed/util/data_manager_util.py b/lib/tool_shed/util/data_manager_util.py index 349d4d72b8f..8f4ec2e6f21 100644 --- a/lib/tool_shed/util/data_manager_util.py +++ b/lib/tool_shed/util/data_manager_util.py @@ -1,4 +1,5 @@ -import os, logging +import logging +import os from galaxy import util import tool_shed.util.shed_util_common as suc diff --git a/lib/tool_shed/util/datatype_util.py b/lib/tool_shed/util/datatype_util.py index f5224aef6b9..98267bdef95 100644 --- a/lib/tool_shed/util/datatype_util.py +++ b/lib/tool_shed/util/datatype_util.py @@ -1,13 +1,17 @@ -import os, logging, tempfile -import tool_shed.util.shed_util_common as suc -from galaxy import util - +import logging +import os +import tempfile from galaxy import eggs +from galaxy import util +import tool_shed.util.shed_util_common as suc + import pkg_resources pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement +from elementtree import ElementTree +from elementtree import ElementInclude +from elementtree.ElementTree import Element +from elementtree.ElementTree import SubElement log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/encoding_util.py b/lib/tool_shed/util/encoding_util.py index 58b3a387baa..026549bb5ed 100644 --- a/lib/tool_shed/util/encoding_util.py +++ b/lib/tool_shed/util/encoding_util.py @@ -1,8 +1,9 @@ -import binascii, logging +import binascii +import logging +from galaxy import eggs from galaxy.util.hash_util import hmac_new from galaxy.util.json import json_fix -from galaxy import eggs import pkg_resources pkg_resources.require( "simplejson" ) @@ -34,6 +35,7 @@ def tool_shed_decode( value ): if values is None: values = value return values + def tool_shed_encode( val ): if isinstance( val, dict ): value = simplejson.dumps( val ) diff --git a/lib/tool_shed/util/metadata_util.py b/lib/tool_shed/util/metadata_util.py index 7eaa73a80fc..84ffd62efaf 100644 --- a/lib/tool_shed/util/metadata_util.py +++ b/lib/tool_shed/util/metadata_util.py @@ -1,21 +1,32 @@ -import os, tempfile, logging, urllib2 -from galaxy import web, util -from galaxy.util import inflector, json -from galaxy.webapps.tool_shed.util import container_util -from galaxy.datatypes import checkers -from galaxy.tools.data_manager.manager import DataManager -import tool_shed.util.shed_util_common as suc -from tool_shed.util import common_install_util, readme_util, tool_dependency_util, tool_util -from galaxy.model.orm import and_ - +import logging +import os +import tempfile +import urllib2 from galaxy import eggs +from galaxy import util +from galaxy import web +from galaxy.datatypes import checkers +from galaxy.model.orm import and_ +from galaxy.tools.data_manager.manager import DataManager +from galaxy.util import inflector +from galaxy.util import json +from galaxy.webapps.tool_shed.util import container_util +import tool_shed.util.shed_util_common as suc +from tool_shed.util import common_install_util +from tool_shed.util import readme_util +from tool_shed.util import tool_dependency_util +from tool_shed.util import tool_util + import pkg_resources pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands +from mercurial import commands +from mercurial import hg +from mercurial import ui pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude +from elementtree import ElementTree +from elementtree import ElementInclude log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/readme_util.py b/lib/tool_shed/util/readme_util.py index 6b5530c7f32..8a302ffa117 100644 --- a/lib/tool_shed/util/readme_util.py +++ b/lib/tool_shed/util/readme_util.py @@ -1,6 +1,8 @@ -import os, logging, urllib2 -import tool_shed.util.shed_util_common as suc +import logging +import os +import urllib2 from galaxy.util import json +import tool_shed.util.shed_util_common as suc log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/repository_dependency_util.py b/lib/tool_shed/util/repository_dependency_util.py index cd1687bdcd7..4c1b37b118b 100644 --- a/lib/tool_shed/util/repository_dependency_util.py +++ b/lib/tool_shed/util/repository_dependency_util.py @@ -1,14 +1,20 @@ -import os, logging -from galaxy.util import json -import tool_shed.util.shed_util_common as suc -from tool_shed.util import common_install_util, encoding_util, metadata_util, tool_util -from galaxy.webapps.tool_shed.util import container_util - +import logging +import os from galaxy import eggs +from galaxy.util import json +from galaxy.webapps.tool_shed.util import container_util +import tool_shed.util.shed_util_common as suc +from tool_shed.util import common_install_util +from tool_shed.util import encoding_util +from tool_shed.util import metadata_util +from tool_shed.util import tool_util + import pkg_resources pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands +from mercurial import commands +from mercurial import hg +from mercurial import ui log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/review_util.py b/lib/tool_shed/util/review_util.py index 35e7f8c1a3c..62e82da9a2c 100644 --- a/lib/tool_shed/util/review_util.py +++ b/lib/tool_shed/util/review_util.py @@ -1,13 +1,16 @@ -import os, logging +import logging +import os +from galaxy import eggs +from galaxy.model.orm import and_ from galaxy.util.odict import odict import tool_shed.util.shed_util_common as suc -from galaxy.model.orm import and_ -from galaxy import eggs import pkg_resources pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands +from mercurial import commands +from mercurial import hg +from mercurial import ui log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/tool_dependency_util.py b/lib/tool_shed/util/tool_dependency_util.py index b246f518f54..7a800fe3350 100644 --- a/lib/tool_shed/util/tool_dependency_util.py +++ b/lib/tool_shed/util/tool_dependency_util.py @@ -1,15 +1,14 @@ import logging import os import shutil - from galaxy import eggs -eggs.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude - +from galaxy import util from galaxy.model.orm import and_ import tool_shed.util.shed_util_common as suc -from galaxy import util +eggs.require( 'elementtree' ) +from elementtree import ElementTree +from elementtree import ElementInclude log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/tool_util.py b/lib/tool_shed/util/tool_util.py index 7412f191f5d..f21a6c2f65a 100644 --- a/lib/tool_shed/util/tool_util.py +++ b/lib/tool_shed/util/tool_util.py @@ -4,6 +4,7 @@ import os import shutil import tempfile import galaxy.tools +from galaxy import eggs from galaxy import util from galaxy.datatypes import checkers from galaxy.model.orm import and_ @@ -13,15 +14,18 @@ from galaxy.tools.search import ToolBoxSearch from galaxy.web.form_builder import SelectField import tool_shed.util.shed_util_common as suc -from galaxy import eggs import pkg_resources pkg_resources.require( 'mercurial' ) -from mercurial import hg, ui, commands +from mercurial import commands +from mercurial import hg +from mercurial import ui pkg_resources.require( 'elementtree' ) -from elementtree import ElementTree, ElementInclude -from elementtree.ElementTree import Element, SubElement +from elementtree import ElementTree +from elementtree import ElementInclude +from elementtree.ElementTree import Element +from elementtree.ElementTree import SubElement log = logging.getLogger( __name__ ) diff --git a/lib/tool_shed/util/workflow_util.py b/lib/tool_shed/util/workflow_util.py index c52eb97ce26..f8faaa44c11 100644 --- a/lib/tool_shed/util/workflow_util.py +++ b/lib/tool_shed/util/workflow_util.py @@ -1,19 +1,25 @@ +import logging +import galaxy.tools +import galaxy.tools.parameters +import galaxy.webapps.galaxy.controllers.workflow from galaxy import eggs +from galaxy.util import json +from galaxy.workflow.modules import InputDataModule +from galaxy.workflow.modules import ToolModule +from galaxy.workflow.modules import WorkflowModuleFactory +import tool_shed.util.shed_util_common as suc +from tool_shed.util import encoding_util +from tool_shed.util import metadata_util +from tool_shed.util import tool_util + import pkg_resources pkg_resources.require( "SVGFig" ) - -import logging, svgfig -from galaxy.util import json -import tool_shed.util.shed_util_common as suc -from tool_shed.util import encoding_util, metadata_util, tool_util -from galaxy.workflow.modules import InputDataModule, ToolModule, WorkflowModuleFactory -import galaxy.webapps.galaxy.controllers.workflow -import galaxy.tools -import galaxy.tools.parameters +import svgfig log = logging.getLogger( __name__ ) + class RepoInputDataModule( InputDataModule ): type = "data_input" @@ -24,12 +30,14 @@ class RepoInputDataModule( InputDataModule ): module = Class( trans ) module.state = dict( name="Input Dataset" ) return module + @classmethod def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata=None, secure=True ): module = Class( trans ) state = json.from_json_string( step_dict[ "tool_state" ] ) module.state = dict( name=state.get( "name", "Input Dataset" ) ) return module + @classmethod def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ): module = Class( trans ) @@ -38,6 +46,7 @@ class RepoInputDataModule( InputDataModule ): module.state[ 'name' ] = step.tool_inputs[ 'name' ] return module + class RepoToolModule( ToolModule ): type = "tool" @@ -64,11 +73,13 @@ class RepoToolModule( ToolModule ): self.post_job_actions = {} self.workflow_outputs = [] self.state = None + @classmethod def new( Class, trans, repository_id, changeset_revision, tools_metadata, tool_id=None ): module = Class( trans, repository_id, changeset_revision, tools_metadata, tool_id ) module.state = module.tool.new_state( trans, all_pages=True ) return module + @classmethod def from_dict( Class, trans, repository_id, changeset_revision, step_dict, tools_metadata, secure=True ): tool_id = step_dict[ 'tool_id' ] @@ -78,6 +89,7 @@ class RepoToolModule( ToolModule ): module.state.decode( step_dict[ "tool_state" ], module.tool, module.trans.app, secure=secure ) module.errors = step_dict.get( "tool_errors", None ) return module + @classmethod def from_workflow_step( Class, trans, repository_id, changeset_revision, tools_metadata, step ): module = Class( trans, repository_id, changeset_revision, tools_metadata, step.tool_id ) @@ -88,6 +100,7 @@ class RepoToolModule( ToolModule ): module.state.inputs = {} module.errors = step.tool_errors return module + def get_data_inputs( self ): data_inputs = [] def callback( input, value, prefixed_name, prefixed_label ): @@ -98,6 +111,7 @@ class RepoToolModule( ToolModule ): if self.tool: galaxy.tools.parameters.visit_input_values( self.tool.inputs, self.state.inputs, callback ) return data_inputs + def get_data_outputs( self ): data_outputs = [] if self.tool: @@ -124,18 +138,23 @@ class RepoToolModule( ToolModule ): data_outputs.append( dict( name=name, extensions=formats ) ) return data_outputs + class RepoWorkflowModuleFactory( WorkflowModuleFactory ): + def __init__( self, module_types ): self.module_types = module_types + def new( self, trans, type, tools_metadata=None, tool_id=None ): """Return module for type and (optional) tool_id initialized with new / default state.""" assert type in self.module_types return self.module_types[type].new( trans, tool_id ) + def from_dict( self, trans, repository_id, changeset_revision, step_dict, **kwd ): """Return module initialized from the data in dictionary `step_dict`.""" type = step_dict[ 'type' ] assert type in self.module_types return self.module_types[ type ].from_dict( trans, repository_id, changeset_revision, step_dict, **kwd ) + def from_workflow_step( self, trans, repository_id, changeset_revision, tools_metadata, step ): """Return module initialized from the WorkflowStep object `step`.""" type = step.type @@ -292,6 +311,7 @@ def generate_workflow_image( trans, workflow_name, repository_metadata_id=None, canvas[ 'viewBox' ] = "0 0 %s %s" % ( width, height ) trans.response.set_content_type( "image/svg+xml" ) return canvas.standalone_xml() + def get_workflow_data_inputs( step, module ): if module.type == 'tool': if module.tool: @@ -306,6 +326,7 @@ def get_workflow_data_inputs( step, module ): data_inputs.append( data_inputs_dict ) return data_inputs return module.get_data_inputs() + def get_workflow_data_outputs( step, module, steps ): if module.type == 'tool': if module.tool: @@ -329,6 +350,7 @@ def get_workflow_data_outputs( step, module, steps ): data_outputs.append( data_outputs_dict ) return data_outputs return module.get_data_outputs() + def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id, changeset_revision ): """ Return an in-memory Workflow object from the dictionary object created when it was exported. This method is called from @@ -383,6 +405,7 @@ def get_workflow_from_dict( trans, workflow_dict, tools_metadata, repository_id, galaxy.webapps.galaxy.controllers.workflow.attach_ordered_steps( workflow, steps ) # Return the in-memory Workflow object for display or later persistence to the Galaxy database. return workflow, missing_tool_tups + def get_workflow_module_name( module, missing_tool_tups ): module_name = module.get_name() if module.type == 'tool' and module_name == 'unavailable': @@ -392,6 +415,7 @@ def get_workflow_module_name( module, missing_tool_tups ): module_name = '%s' % missing_tool_name break return module_name + def save_workflow( trans, workflow ): """Use the received in-memory Workflow object for saving to the Galaxy database.""" stored = trans.model.StoredWorkflow() From 620263d96c2fabf9a32309155505d2b6e39141ed Mon Sep 17 00:00:00 2001 From: James Taylor Date: Tue, 26 Mar 2013 13:04:36 -0400 Subject: [PATCH 21/21] biostar: pass slugified version of tool name as tag when sending to question form --- .../webapps/galaxy/controllers/biostar.py | 40 ++++++++++++++++--- 1 file changed, 35 insertions(+), 5 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/biostar.py b/lib/galaxy/webapps/galaxy/controllers/biostar.py index c8c92500c9c..f625e49a968 100644 --- a/lib/galaxy/webapps/galaxy/controllers/biostar.py +++ b/lib/galaxy/webapps/galaxy/controllers/biostar.py @@ -8,16 +8,35 @@ import base64 from galaxy.util import json import hmac +# Slugifying from Armin Ronacher (http://flask.pocoo.org/snippets/5/) + +import re +from unicodedata import normalize + +_punct_re = re.compile(r'[\t !"#$%&\'()*\-/<=>?@\[\\\]^_`{|},.]+') + + +def slugify(text, delim=u'-'): + """Generates an slightly worse ASCII-only slug.""" + result = [] + for word in _punct_re.split(text.lower()): + word = normalize('NFKD', word).encode('ascii', 'ignore') + if word: + result.append(word) + return unicode(delim.join(result)) + + # Biostar requires all keys to be present, so we start with a template DEFAULT_PAYLOAD = { - 'email': "", - 'title': "Question about Galaxy", + 'email': "", + 'title': "Question about Galaxy", 'tags': 'galaxy', - 'tool_name': '', - 'tool_version': '', + 'tool_name': '', + 'tool_version': '', 'tool_id': '' } + def encode_data( key, data ): """ Encode data to send a question to Biostar @@ -28,6 +47,13 @@ def encode_data( key, data ): return text, digest +def tag_for_tool( tool ): + """ + Generate a reasonavle biostar tag for a tool. + """ + return slugify( unicode( tool.name ) ) + + class BiostarController( BaseUIController ): """ Provides integration with Biostar through external authentication, see: http://liondb.com/help/x/ @@ -81,6 +107,10 @@ class BiostarController( BaseUIController ): if not tool: return error( "No tool found matching '%s'" % tool_id ) # Tool specific information for payload - payload = { 'title': "Question about Galaxy tool '%s'" % tool.name, 'tool_name': tool.name, 'tool_version': tool.version, 'tool_id': tool.id } + payload = { 'title': "Question about Galaxy tool '%s'" % tool.name, + 'tool_name': tool.name, + 'tool_version': tool.version, + 'tool_id': tool.id, + 'tags': 'galaxy ' + tag_for_tool( tool ) } # Pass on to regular question method return self.biostar_question_redirect( trans, payload ) \ No newline at end of file