From 34f9a6b5c26a0d8b0d61e48cd34bc072c82e1e0d Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 8 Aug 2016 12:11:44 -0400 Subject: [PATCH 1/4] disable hg middleware --- lib/galaxy/webapps/tool_shed/buildapp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 50659327f42..9b14b3ec0b6 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -205,8 +205,8 @@ def wrap_in_middleware( app, global_conf, **local_conf ): app = httpexceptions.make_middleware( app, conf ) log.debug( "Enabling 'httpexceptions' middleware" ) # Then load the Hg middleware. - app = hg.Hg( app, conf ) - log.debug( "Enabling 'hg' middleware" ) + # app = hg.Hg( app, conf ) + # log.debug( "Enabling 'hg' middleware" ) # If we're using remote_user authentication, add middleware that # protects Galaxy from improperly configured authentication in the # upstream server From af4fab16f58c36ea66328725e5bdcfaa3acbacda Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 8 Aug 2016 13:04:03 -0400 Subject: [PATCH 2/4] remove unwanted hg middleware remove obsolete test --- lib/galaxy/webapps/tool_shed/buildapp.py | 4 - .../tool_shed/framework/middleware/hg.py | 325 ------------------ .../functional/test_0310_hg_api_features.py | 22 -- 3 files changed, 351 deletions(-) delete mode 100644 lib/galaxy/webapps/tool_shed/framework/middleware/hg.py diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 9b14b3ec0b6..2d855a2148d 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -13,7 +13,6 @@ import galaxy.webapps.tool_shed.model import galaxy.webapps.tool_shed.model.mapping import galaxy.web.framework.webapp from galaxy.webapps.util import build_template_error_formatters -from galaxy.webapps.tool_shed.framework.middleware import hg from galaxy import util from galaxy.config import process_is_uwsgi from galaxy.util.properties import load_app_properties @@ -204,9 +203,6 @@ def wrap_in_middleware( app, global_conf, **local_conf ): # other middleware): app = httpexceptions.make_middleware( app, conf ) log.debug( "Enabling 'httpexceptions' middleware" ) - # Then load the Hg middleware. - # app = hg.Hg( app, conf ) - # log.debug( "Enabling 'hg' middleware" ) # If we're using remote_user authentication, add middleware that # protects Galaxy from improperly configured authentication in the # upstream server diff --git a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py b/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py deleted file mode 100644 index 4f743bd9ac8..00000000000 --- a/lib/galaxy/webapps/tool_shed/framework/middleware/hg.py +++ /dev/null @@ -1,325 +0,0 @@ -"""Middle-ware for handling hg authentication for users pushing change sets to local repositories.""" -import json -import logging -import os -import sqlalchemy -import sys -import tempfile -import urlparse -from paste.auth.basic import AuthBasicAuthenticator -from paste.httpheaders import AUTH_TYPE -from paste.httpheaders import REMOTE_USER -from six import string_types - -from galaxy.util import asbool, safe_relpath -from galaxy.util.hash_util import new_secure_hash -from tool_shed.util import hg_util -from tool_shed.util import commit_util -import tool_shed.repository_types.util as rt_util - -import mercurial.__version__ - -log = logging.getLogger(__name__) - -CHUNK_SIZE = 65536 - - -class Hg( object ): - - def __init__( self, app, config ): - log.debug( "mercurial version is: %s", mercurial.__version__.version ) - self.app = app - self.config = config - # Authenticate this mercurial request using basic authentication - self.authentication = AuthBasicAuthenticator( 'hgweb in the tool shed', self.__basic_authentication ) - # Determine the database url - if 'database_connection' in self.config: - self.db_url = self.config[ 'database_connection' ] - else: - self.db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config[ 'database_file' ] - # Keep track of whether we're setting repository metadata so that we do not increment the times_downloaded - # count for the repository. - self.setting_repository_metadata = False - - def __call__( self, environ, start_response ): - if 'PATH_INFO' in environ: - path_info = environ[ 'PATH_INFO' ].lstrip( '/' ) - if path_info == 'repository/reset_all_metadata': - self.setting_repository_metadata = True - cmd = self.__get_hg_command( **environ ) - # The 'getbundle' command indicates that a mercurial client is getting a bundle of one or more changesets, indicating - # a clone or a pull. However, we do not want to increment the times_downloaded count if we're only setting repository - # metadata. - if cmd == 'getbundle' and not self.setting_repository_metadata: - hg_args = urlparse.parse_qs( environ[ 'HTTP_X_HGARG_1' ] ) - # The 'common' parameter indicates the full sha-1 hash of the changeset the client currently has checked out. If - # this is 0000000000000000000000000000000000000000, then the client is performing a fresh checkout. If it has any - # other value, the client is getting updates to an existing checkout. - if 'common' in hg_args and hg_args[ 'common' ][-1] == '0000000000000000000000000000000000000000': - # Increment the value of the times_downloaded column in the repository table for the cloned repository. - if 'PATH_INFO' in environ: - # Instantiate a database connection - engine = sqlalchemy.create_engine( self.db_url ) - connection = engine.connect() - path_info = environ[ 'PATH_INFO' ].lstrip( '/' ) - user_id, repository_name = self.__get_user_id_repository_name_from_path_info( connection, path_info ) - sql_cmd = "SELECT times_downloaded FROM repository WHERE user_id = %d AND name = '%s'" % \ - ( user_id, repository_name.lower() ) - result_set = connection.execute( sql_cmd ) - for row in result_set: - # Should only be 1 row... - times_downloaded = row[ 'times_downloaded' ] - times_downloaded += 1 - sql_cmd = "UPDATE repository SET times_downloaded = %d WHERE user_id = %d AND name = '%s'" % \ - ( times_downloaded, user_id, repository_name.lower() ) - connection.execute( sql_cmd ) - connection.close() - elif cmd in [ 'unbundle', 'pushkey' ]: - if self.config.get('disable_push', True): - msg = 'Pushing to Tool Shed is disabled. Please use Galaxy Planemo to upload your changes.' - return self.__display_exception_remotely( start_response, msg ) - # This is an hg push from the command line. When doing this, the following commands, in order, - # will be retrieved from environ (see the docs at http://mercurial.selenic.com/wiki/WireProtocol): - # # If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey -> listkeys - # - # The mercurial API unbundle() ( i.e., hg push ) and pushkey() methods ultimately require authorization. - # We'll force password entry every time a change set is pushed. - # - # When a user executes hg commit, it is not guaranteed to succeed. Mercurial records your name - # and address with each change that you commit, so that you and others will later be able to - # tell who made each change. Mercurial tries to automatically figure out a sensible username - # to commit the change with. It will attempt each of the following methods, in order: - # - # 1) If you specify a -u option to the hg commit command on the command line, followed by a username, - # this is always given the highest precedence. - # 2) If you have set the HGUSER environment variable, this is checked next. - # 3) If you create a file in your home directory called .hgrc with a username entry, that - # will be used next. - # 4) If you have set the EMAIL environment variable, this will be used next. - # 5) Mercurial will query your system to find out your local user name and host name, and construct - # a username from these components. Since this often results in a username that is not very useful, - # it will print a warning if it has to do this. - # - # If all of these mechanisms fail, Mercurial will fail, printing an error message. In this case, it - # will not let you commit until you set up a username. - result = self.authentication( environ ) - if not isinstance( result, string_types ) and cmd == 'unbundle' and 'wsgi.input' in environ: - bundle_data_stream = environ[ 'wsgi.input' ] - # Convert the incoming mercurial bundle into a json object and persit it to a temporary file for inspection. - fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-hg-bundle" ) - tmp_filename = fh.name - fh.close() - fh = open( tmp_filename, 'wb' ) - while 1: - chunk = bundle_data_stream.read( CHUNK_SIZE ) - if not chunk: - break - fh.write( chunk ) - fh.close() - fh = open( tmp_filename, 'rb' ) - try: - changeset_groups = json.loads( hg_util.bundle_to_json( fh ) ) - except AttributeError: - msg = 'Your version of Mercurial is not supported. Please use a version < 3.5' - return self.__display_exception_remotely( start_response, msg ) - fh.close() - try: - os.unlink( tmp_filename ) - except: - pass - if changeset_groups: - # Check the repository type to make sure inappropriate files are not being pushed. - if 'PATH_INFO' in environ: - # Ensure there are no symlinks with targets outside the repo - for entry in changeset_groups: - if len( entry ) == 2: - filename, change_list = entry - if not isinstance(change_list, list): - change_list = [change_list] - for change in change_list: - for patch in change['data']: - target = patch['block'].strip() - if ( ( patch['end'] - patch['start'] == 0 ) and not safe_relpath( target ) ): - msg = "Changes include a symlink outside of the repository: %s -> %s" % ( filename, target ) - log.warning( msg ) - return self.__display_exception_remotely( start_response, msg ) - # Instantiate a database connection - engine = sqlalchemy.create_engine( self.db_url ) - connection = engine.connect() - path_info = environ[ 'PATH_INFO' ].lstrip( '/' ) - user_id, repository_name = self.__get_user_id_repository_name_from_path_info( connection, path_info ) - sql_cmd = "SELECT type FROM repository WHERE user_id = %d AND name = '%s'" % ( user_id, repository_name.lower() ) - result_set = connection.execute( sql_cmd ) - for row in result_set: - # Should only be 1 row... - repository_type = str( row[ 'type' ] ) - if repository_type == rt_util.REPOSITORY_SUITE_DEFINITION: - # Handle repositories of type repository_suite_definition, which can only contain a single - # file named repository_dependencies.xml. - for entry in changeset_groups: - if len( entry ) == 2: - # We possibly found an altered file entry. - filename, change_list = entry - if filename and isinstance( filename, string_types ): - if filename == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME: - # Make sure the any complex repository dependency definitions contain valid tags. - is_valid, error_msg = self.repository_tags_are_valid( filename, change_list ) - if not is_valid: - log.debug( error_msg ) - return self.__display_exception_remotely( start_response, error_msg ) - else: - msg = "Only a single file named repository_dependencies.xml can be pushed to a repository " - msg += "of type 'Repository suite definition'." - log.debug( msg ) - return self.__display_exception_remotely( start_response, msg ) - elif repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION: - # Handle repositories of type tool_dependency_definition, which can only contain a single - # file named tool_dependencies.xml. - for entry in changeset_groups: - if len( entry ) == 2: - # We possibly found an altered file entry. - filename, change_list = entry - if filename and isinstance( filename, string_types ): - if filename == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME: - # Make sure the any complex repository dependency definitions contain valid tags. - is_valid, error_msg = self.repository_tags_are_valid( filename, change_list ) - if not is_valid: - log.debug( error_msg ) - return self.__display_exception_remotely( start_response, error_msg ) - else: - msg = "Only a single file named tool_dependencies.xml can be pushed to a repository " - msg += "of type 'Tool dependency definition'." - log.debug( msg ) - return self.__display_exception_remotely( start_response, msg ) - else: - # If the changeset includes changes to dependency definition files, make sure tag sets - # are not missing "toolshed" or "changeset_revision" attributes since automatically populating - # them is not supported when pushing from the command line. These attributes are automatically - # populated only when using the tool shed upload utility. - for entry in changeset_groups: - if len( entry ) == 2: - # We possibly found an altered file entry. - filename, change_list = entry - if filename and isinstance( filename, string_types ): - if filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME, - rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]: - # We check both files since tool dependency definitions files can contain complex - # repository dependency definitions. - is_valid, error_msg = self.repository_tags_are_valid( filename, change_list ) - if not is_valid: - log.debug( error_msg ) - return self.__display_exception_remotely( start_response, error_msg ) - if isinstance( result, string_types ): - # Authentication was successful - AUTH_TYPE.update( environ, 'basic' ) - REMOTE_USER.update( environ, result ) - else: - return result.wsgi_application( environ, start_response ) - return self.app( environ, start_response ) - - def __authenticate( self, username, password ): - db_password = None - # Instantiate a database connection - engine = sqlalchemy.create_engine( self.db_url ) - connection = engine.connect() - result_set = connection.execute( "select email, password from galaxy_user where username = '%s'" % username.lower() ) - for row in result_set: - # Should only be 1 row... - db_password = row[ 'password' ] - connection.close() - if db_password: - # Check if password matches db_password when hashed. - return new_secure_hash( text_type=password ) == db_password - return False - - def __authenticate_remote_user( self, environ, username, password ): - """ - Look after a remote user and "authenticate" - upstream server should already have achieved - this for us, but we check that the user exists at least. Hg allow_push = must include username - - some versions of mercurial blow up with 500 errors. - """ - db_username = None - ru_email = environ[ 'HTTP_REMOTE_USER' ].lower() - # Instantiate a database connection... - engine = sqlalchemy.create_engine( self.db_url ) - connection = engine.connect() - result_set = connection.execute( "select email, username, password from galaxy_user where email = '%s'" % ru_email ) - for row in result_set: - # Should only be 1 row... - db_username = row[ 'username' ] - connection.close() - if db_username: - # We could check the password here except that the function galaxy.web.framework.get_or_create_remote_user() - # does some random generation of a password - so that no-one knows the password and only the hash is stored... - return db_username == username - return False - - def __basic_authentication( self, environ, username, password ): - """The environ parameter is needed in basic authentication. We also check it if use_remote_user is true.""" - if asbool( self.config.get( 'use_remote_user', False ) ): - assert "HTTP_REMOTE_USER" in environ, "use_remote_user is set but no HTTP_REMOTE_USER variable" - return self.__authenticate_remote_user( environ, username, password ) - else: - return self.__authenticate( username, password ) - - def __display_exception_remotely( self, start_response, msg ): - # Display the exception to the remote user's command line. - status = "500 %s" % msg - response_headers = [ ("content-type", "text/plain") ] - start_response( status, response_headers, sys.exc_info() ) - return [ msg ] - - def __get_hg_command( self, **kwd ): - """Pulls mercurial commands from environ[ 'QUERY_STRING" ] and returns them.""" - if 'QUERY_STRING' in kwd: - for qry in kwd[ 'QUERY_STRING' ].split( '&' ): - if qry.startswith( 'cmd' ): - return qry.split( '=' )[ -1 ] - return None - - def __get_user_id_repository_name_from_path_info( self, db_connection, path_info ): - # An example of path_info is: '/repos/test/column1' - path_info_components = path_info.split( '/' ) - username = path_info_components[ 1 ] - repository_name = path_info_components[ 2 ] - # Get the id of the current user using hg from the command line. - result_set = db_connection.execute( "select id from galaxy_user where username = '%s'" % username.lower() ) - for row in result_set: - # Should only be 1 row... - user_id = row[ 'id' ] - return user_id, repository_name - - def repository_tag_is_valid( self, filename, line ): - """ - Checks changes made to tags in a dependency definition file being pushed to the - Tool Shed from the command line to ensure that all required attributes exist. - """ - required_attributes = [ 'toolshed', 'name', 'owner', 'changeset_revision' ] - defined_attributes = line.split() - for required_attribute in required_attributes: - defined = False - for defined_attribute in defined_attributes: - if defined_attribute.startswith( required_attribute ): - defined = True - break - if not defined: - error_msg = 'The %s file contains a tag that is missing the required attribute %s. ' % \ - ( filename, required_attribute ) - error_msg += 'Automatically populating dependency definition attributes occurs only when using ' - error_msg += 'the Tool Shed upload utility. ' - return False, error_msg - return True, '' - - def repository_tags_are_valid( self, filename, change_list ): - """ - Make sure the any complex repository dependency definitions contain valid tags when pushing - changes to the tool shed on the command line. - """ - tag = ' Date: Mon, 8 Aug 2016 13:06:09 -0400 Subject: [PATCH 3/4] remove obsolete config option --- config/tool_shed.ini.sample | 3 --- lib/galaxy/webapps/tool_shed/config.py | 1 - 2 files changed, 4 deletions(-) diff --git a/config/tool_shed.ini.sample b/config/tool_shed.ini.sample index 07d2f5e47db..cf017220d36 100644 --- a/config/tool_shed.ini.sample +++ b/config/tool_shed.ini.sample @@ -35,9 +35,6 @@ database_file = database/community.sqlite # The default is the Galaxy installation directory. #hgweb_config_dir = None -# Disable Mercurial pushing to repositories. -#disable_push = True - # Where tool shed repositories are stored. file_path = database/community_files # Temporary storage for additional datasets, diff --git a/lib/galaxy/webapps/tool_shed/config.py b/lib/galaxy/webapps/tool_shed/config.py index f2da55b9abe..9d75231ebcf 100644 --- a/lib/galaxy/webapps/tool_shed/config.py +++ b/lib/galaxy/webapps/tool_shed/config.py @@ -131,7 +131,6 @@ class Configuration( object ): self.sentry_dsn = kwargs.get( 'sentry_dsn', None ) # Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory. self.hgweb_config_dir = resolve_path( kwargs.get( 'hgweb_config_dir', '' ), self.root ) - self.disable_push = string_as_bool( kwargs.get( "disable_push", "True" ) ) # Proxy features self.apache_xsendfile = kwargs.get( 'apache_xsendfile', False ) self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False ) From 9618386fd0b7511aead50b14dea763b32bfdd4a8 Mon Sep 17 00:00:00 2001 From: Martin Cech Date: Mon, 8 Aug 2016 14:40:43 -0400 Subject: [PATCH 4/4] remove code related to hg push --- lib/galaxy/webapps/tool_shed/buildapp.py | 1 + .../webapps/tool_shed/controllers/hg.py | 54 ++----------------- 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/lib/galaxy/webapps/tool_shed/buildapp.py b/lib/galaxy/webapps/tool_shed/buildapp.py index 2d855a2148d..f8dcfd56eea 100644 --- a/lib/galaxy/webapps/tool_shed/buildapp.py +++ b/lib/galaxy/webapps/tool_shed/buildapp.py @@ -79,6 +79,7 @@ def app_factory( global_conf, **kwargs ): image_file=None ) webapp.add_route( '/{controller}/{action}', action='index' ) webapp.add_route( '/{action}', controller='repository', action='index' ) + # Enable 'hg clone' functionality on repos by letting hgwebapp handle the request webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' ) # Add the web API. # A good resource for RESTful services - http://routes.readthedocs.org/en/latest/restful.html webapp.add_api_controllers( 'galaxy.webapps.tool_shed.api', app ) diff --git a/lib/galaxy/webapps/tool_shed/controllers/hg.py b/lib/galaxy/webapps/tool_shed/controllers/hg.py index 7ac550d66ff..42e53195bd8 100644 --- a/lib/galaxy/webapps/tool_shed/controllers/hg.py +++ b/lib/galaxy/webapps/tool_shed/controllers/hg.py @@ -1,17 +1,10 @@ import logging -from galaxy import web -from galaxy.web.base.controller import BaseUIController -from tool_shed.util.common_util import generate_clone_url_for_repository_in_tool_shed -from tool_shed.util.repository_util import get_repository_by_name_and_owner -from tool_shed.util.hg_util import update_repository -from tool_shed.metadata import repository_metadata_manager - -import mercurial.__version__ from mercurial.hgweb.hgwebdir_mod import hgwebdir from mercurial.hgweb.request import wsgiapplication -from mercurial import hg -from mercurial import ui + +from galaxy import web +from galaxy.web.base.controller import BaseUIController log = logging.getLogger(__name__) @@ -21,51 +14,10 @@ class HgController( BaseUIController ): def handle_request( self, trans, **kwd ): # The os command that results in this method being called will look something like: # hg clone http://test@127.0.0.1:9009/repos/test/convert_characters1 - hg_version = mercurial.__version__.version - cmd = kwd.get( 'cmd', None ) hgweb_config = trans.app.hgweb_config_manager.hgweb_config def make_web_app(): hgwebapp = hgwebdir( hgweb_config ) return hgwebapp wsgi_app = wsgiapplication( make_web_app ) - if hg_version >= '2.2.3' and cmd == 'pushkey': - # When doing an "hg push" from the command line, the following commands, in order, will be - # retrieved from environ, depending upon the mercurial version being used. In mercurial - # version 2.2.3, section 15.2. Command changes includes a new feature: - # pushkey: add hooks for pushkey/listkeys - # (see http://mercurial.selenic.com/wiki/WhatsNew#Mercurial_2.2.3_.282012-07-01.29). - # We require version 2.2.3 since the pushkey hook was added in that version. - # If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey - path_info = kwd.get( 'path_info', None ) - if path_info: - owner, name = path_info.split( '/' ) - repository = get_repository_by_name_and_owner( trans.app, name, owner ) - if repository: - if hg_version >= '2.2.3': - # Update the repository on disk to the tip revision, because the web upload - # form uses the on-disk working directory. If the repository is not updated - # on disk, pushing from the command line and then uploading via the web - # interface will result in a new head being created. - repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) ) - update_repository( repo, ctx_rev=None ) - repository_clone_url = generate_clone_url_for_repository_in_tool_shed( trans.user, repository ) - # Set metadata using the repository files on disk. - rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app, - user=trans.user, - repository=repository, - changeset_revision=repository.tip( trans.app ), - repository_clone_url=repository_clone_url, - relative_install_dir=repository.repo_path( trans.app ), - repository_files_dir=None, - resetting_all_metadata_on_repository=False, - updating_installed_repository=False, - persist=False ) - error_message, status = rmm.set_repository_metadata( trans.request.host ) - if status == 'ok' and error_message: - log.debug( "Successfully reset metadata on repository %s owned by %s, but encountered problem: %s" % - ( str( repository.name ), str( repository.user.username ), error_message ) ) - elif status != 'ok' and error_message: - log.debug( "Error resetting metadata on repository %s owned by %s: %s" % - ( str( repository.name ), str( repository.user.username ), error_message ) ) return wsgi_app