opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2753 from martenson/disable_hg_middleware

Browse Source 
remove TS hg middleware
This commit is contained in:
Dannon Baker
2016-08-08 16:02:41 -04:00
committed by GitHub
parent 0e9c044256 9618386fd0
commit 580ace125a
6 changed files with 4 additions and 406 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/tool_shed.ini.sample
-3
View File
@@ -35,9 +35,6 @@ database_file = database/community.sqlite
# The default is the Galaxy installation directory.
#hgweb_config_dir = None
# Disable Mercurial pushing to repositories.
#disable_push = True
# Where tool shed repositories are stored.
file_path = database/community_files
# Temporary storage for additional datasets,
lib/galaxy/webapps/tool_shed/buildapp.py
+1 -4
View File
@@ -13,7 +13,6 @@ import galaxy.webapps.tool_shed.model
import galaxy.webapps.tool_shed.model.mapping
import galaxy.web.framework.webapp
from galaxy.webapps.util import build_template_error_formatters
from galaxy.webapps.tool_shed.framework.middleware import hg
from galaxy import util
from galaxy.config import process_is_uwsgi
from galaxy.util.properties import load_app_properties
@@ -80,6 +79,7 @@ def app_factory( global_conf, **kwargs ):
image_file=None )
webapp.add_route( '/{controller}/{action}', action='index' )
webapp.add_route( '/{action}', controller='repository', action='index' )
# Enable 'hg clone' functionality on repos by letting hgwebapp handle the request
webapp.add_route( '/repos/*path_info', controller='hg', action='handle_request', path_info='/' )
# Add the web API. # A good resource for RESTful services - http://routes.readthedocs.org/en/latest/restful.html
webapp.add_api_controllers( 'galaxy.webapps.tool_shed.api', app )
@@ -204,9 +204,6 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
# other middleware):
app = httpexceptions.make_middleware( app, conf )
log.debug( "Enabling 'httpexceptions' middleware" )
# Then load the Hg middleware.
app = hg.Hg( app, conf )
log.debug( "Enabling 'hg' middleware" )
# If we're using remote_user authentication, add middleware that
# protects Galaxy from improperly configured authentication in the
# upstream server
lib/galaxy/webapps/tool_shed/config.py
-1
View File
@@ -131,7 +131,6 @@ class Configuration( object ):
self.sentry_dsn = kwargs.get( 'sentry_dsn', None )
# Where the tool shed hgweb.config file is stored - the default is the Galaxy installation directory.
self.hgweb_config_dir = resolve_path( kwargs.get( 'hgweb_config_dir', '' ), self.root )
self.disable_push = string_as_bool( kwargs.get( "disable_push", "True" ) )
# Proxy features
self.apache_xsendfile = kwargs.get( 'apache_xsendfile', False )
self.nginx_x_accel_redirect_base = kwargs.get( 'nginx_x_accel_redirect_base', False )
lib/galaxy/webapps/tool_shed/controllers/hg.py
+3 -51
View File
@@ -1,17 +1,10 @@
import logging
from galaxy import web
from galaxy.web.base.controller import BaseUIController
from tool_shed.util.common_util import generate_clone_url_for_repository_in_tool_shed
from tool_shed.util.repository_util import get_repository_by_name_and_owner
from tool_shed.util.hg_util import update_repository
from tool_shed.metadata import repository_metadata_manager
import mercurial.__version__
from mercurial.hgweb.hgwebdir_mod import hgwebdir
from mercurial.hgweb.request import wsgiapplication
from mercurial import hg
from mercurial import ui
from galaxy import web
from galaxy.web.base.controller import BaseUIController
log = logging.getLogger(__name__)
@@ -21,51 +14,10 @@ class HgController( BaseUIController ):
def handle_request( self, trans, **kwd ):
# The os command that results in this method being called will look something like:
# hg clone http://test@127.0.0.1:9009/repos/test/convert_characters1
hg_version = mercurial.__version__.version
cmd = kwd.get( 'cmd', None )
hgweb_config = trans.app.hgweb_config_manager.hgweb_config
def make_web_app():
hgwebapp = hgwebdir( hgweb_config )
return hgwebapp
wsgi_app = wsgiapplication( make_web_app )
if hg_version >= '2.2.3' and cmd == 'pushkey':
# When doing an "hg push" from the command line, the following commands, in order, will be
# retrieved from environ, depending upon the mercurial version being used. In mercurial
# version 2.2.3, section 15.2. Command changes includes a new feature:
# pushkey: add hooks for pushkey/listkeys
# (see http://mercurial.selenic.com/wiki/WhatsNew#Mercurial_2.2.3_.282012-07-01.29).
# We require version 2.2.3 since the pushkey hook was added in that version.
# If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey
path_info = kwd.get( 'path_info', None )
if path_info:
owner, name = path_info.split( '/' )
repository = get_repository_by_name_and_owner( trans.app, name, owner )
if repository:
if hg_version >= '2.2.3':
# Update the repository on disk to the tip revision, because the web upload
# form uses the on-disk working directory. If the repository is not updated
# on disk, pushing from the command line and then uploading via the web
# interface will result in a new head being created.
repo = hg.repository( ui.ui(), repository.repo_path( trans.app ) )
update_repository( repo, ctx_rev=None )
repository_clone_url = generate_clone_url_for_repository_in_tool_shed( trans.user, repository )
# Set metadata using the repository files on disk.
rmm = repository_metadata_manager.RepositoryMetadataManager( app=trans.app,
user=trans.user,
repository=repository,
changeset_revision=repository.tip( trans.app ),
repository_clone_url=repository_clone_url,
relative_install_dir=repository.repo_path( trans.app ),
repository_files_dir=None,
resetting_all_metadata_on_repository=False,
updating_installed_repository=False,
persist=False )
error_message, status = rmm.set_repository_metadata( trans.request.host )
if status == 'ok' and error_message:
log.debug( "Successfully reset metadata on repository %s owned by %s, but encountered problem: %s" %
( str( repository.name ), str( repository.user.username ), error_message ) )
elif status != 'ok' and error_message:
log.debug( "Error resetting metadata on repository %s owned by %s: %s" %
( str( repository.name ), str( repository.user.username ), error_message ) )
return wsgi_app
lib/galaxy/webapps/tool_shed/framework/middleware/hg.py
-325
View File
@@ -1,325 +0,0 @@
"""Middle-ware for handling hg authentication for users pushing change sets to local repositories."""
import json
import logging
import os
import sqlalchemy
import sys
import tempfile
import urlparse
from paste.auth.basic import AuthBasicAuthenticator
from paste.httpheaders import AUTH_TYPE
from paste.httpheaders import REMOTE_USER
from six import string_types
from galaxy.util import asbool, safe_relpath
from galaxy.util.hash_util import new_secure_hash
from tool_shed.util import hg_util
from tool_shed.util import commit_util
import tool_shed.repository_types.util as rt_util
import mercurial.__version__
log = logging.getLogger(__name__)
CHUNK_SIZE = 65536
class Hg( object ):
def __init__( self, app, config ):
log.debug( "mercurial version is: %s", mercurial.__version__.version )
self.app = app
self.config = config
# Authenticate this mercurial request using basic authentication
self.authentication = AuthBasicAuthenticator( 'hgweb in the tool shed', self.__basic_authentication )
# Determine the database url
if 'database_connection' in self.config:
self.db_url = self.config[ 'database_connection' ]
else:
self.db_url = "sqlite:///%s?isolation_level=IMMEDIATE" % self.config[ 'database_file' ]
# Keep track of whether we're setting repository metadata so that we do not increment the times_downloaded
# count for the repository.
self.setting_repository_metadata = False
def __call__( self, environ, start_response ):
if 'PATH_INFO' in environ:
path_info = environ[ 'PATH_INFO' ].lstrip( '/' )
if path_info == 'repository/reset_all_metadata':
self.setting_repository_metadata = True
cmd = self.__get_hg_command( **environ )
# The 'getbundle' command indicates that a mercurial client is getting a bundle of one or more changesets, indicating
# a clone or a pull. However, we do not want to increment the times_downloaded count if we're only setting repository
# metadata.
if cmd == 'getbundle' and not self.setting_repository_metadata:
hg_args = urlparse.parse_qs( environ[ 'HTTP_X_HGARG_1' ] )
# The 'common' parameter indicates the full sha-1 hash of the changeset the client currently has checked out. If
# this is 0000000000000000000000000000000000000000, then the client is performing a fresh checkout. If it has any
# other value, the client is getting updates to an existing checkout.
if 'common' in hg_args and hg_args[ 'common' ][-1] == '0000000000000000000000000000000000000000':
# Increment the value of the times_downloaded column in the repository table for the cloned repository.
if 'PATH_INFO' in environ:
# Instantiate a database connection
engine = sqlalchemy.create_engine( self.db_url )
connection = engine.connect()
path_info = environ[ 'PATH_INFO' ].lstrip( '/' )
user_id, repository_name = self.__get_user_id_repository_name_from_path_info( connection, path_info )
sql_cmd = "SELECT times_downloaded FROM repository WHERE user_id = %d AND name = '%s'" % \
( user_id, repository_name.lower() )
result_set = connection.execute( sql_cmd )
for row in result_set:
# Should only be 1 row...
times_downloaded = row[ 'times_downloaded' ]
times_downloaded += 1
sql_cmd = "UPDATE repository SET times_downloaded = %d WHERE user_id = %d AND name = '%s'" % \
( times_downloaded, user_id, repository_name.lower() )
connection.execute( sql_cmd )
connection.close()
elif cmd in [ 'unbundle', 'pushkey' ]:
if self.config.get('disable_push', True):
msg = 'Pushing to Tool Shed is disabled. Please use Galaxy Planemo to upload your changes.'
return self.__display_exception_remotely( start_response, msg )
# This is an hg push from the command line. When doing this, the following commands, in order,
# will be retrieved from environ (see the docs at http://mercurial.selenic.com/wiki/WireProtocol):
# # If mercurial version >= '2.2.3': capabilities -> batch -> branchmap -> unbundle -> listkeys -> pushkey -> listkeys
#
# The mercurial API unbundle() ( i.e., hg push ) and pushkey() methods ultimately require authorization.
# We'll force password entry every time a change set is pushed.
#
# When a user executes hg commit, it is not guaranteed to succeed. Mercurial records your name
# and address with each change that you commit, so that you and others will later be able to
# tell who made each change. Mercurial tries to automatically figure out a sensible username
# to commit the change with. It will attempt each of the following methods, in order:
#
# 1) If you specify a -u option to the hg commit command on the command line, followed by a username,
# this is always given the highest precedence.
# 2) If you have set the HGUSER environment variable, this is checked next.
# 3) If you create a file in your home directory called .hgrc with a username entry, that
# will be used next.
# 4) If you have set the EMAIL environment variable, this will be used next.
# 5) Mercurial will query your system to find out your local user name and host name, and construct
# a username from these components. Since this often results in a username that is not very useful,
# it will print a warning if it has to do this.
#
# If all of these mechanisms fail, Mercurial will fail, printing an error message. In this case, it
# will not let you commit until you set up a username.
result = self.authentication( environ )
if not isinstance( result, string_types ) and cmd == 'unbundle' and 'wsgi.input' in environ:
bundle_data_stream = environ[ 'wsgi.input' ]
# Convert the incoming mercurial bundle into a json object and persit it to a temporary file for inspection.
fh = tempfile.NamedTemporaryFile( 'wb', prefix="tmp-hg-bundle" )
tmp_filename = fh.name
fh.close()
fh = open( tmp_filename, 'wb' )
while 1:
chunk = bundle_data_stream.read( CHUNK_SIZE )
if not chunk:
break
fh.write( chunk )
fh.close()
fh = open( tmp_filename, 'rb' )
try:
changeset_groups = json.loads( hg_util.bundle_to_json( fh ) )
except AttributeError:
msg = 'Your version of Mercurial is not supported. Please use a version < 3.5'
return self.__display_exception_remotely( start_response, msg )
fh.close()
try:
os.unlink( tmp_filename )
except:
pass
if changeset_groups:
# Check the repository type to make sure inappropriate files are not being pushed.
if 'PATH_INFO' in environ:
# Ensure there are no symlinks with targets outside the repo
for entry in changeset_groups:
if len( entry ) == 2:
filename, change_list = entry
if not isinstance(change_list, list):
change_list = [change_list]
for change in change_list:
for patch in change['data']:
target = patch['block'].strip()
if ( ( patch['end'] - patch['start'] == 0 ) and not safe_relpath( target ) ):
msg = "Changes include a symlink outside of the repository: %s -> %s" % ( filename, target )
log.warning( msg )
return self.__display_exception_remotely( start_response, msg )
# Instantiate a database connection
engine = sqlalchemy.create_engine( self.db_url )
connection = engine.connect()
path_info = environ[ 'PATH_INFO' ].lstrip( '/' )
user_id, repository_name = self.__get_user_id_repository_name_from_path_info( connection, path_info )
sql_cmd = "SELECT type FROM repository WHERE user_id = %d AND name = '%s'" % ( user_id, repository_name.lower() )
result_set = connection.execute( sql_cmd )
for row in result_set:
# Should only be 1 row...
repository_type = str( row[ 'type' ] )
if repository_type == rt_util.REPOSITORY_SUITE_DEFINITION:
# Handle repositories of type repository_suite_definition, which can only contain a single
# file named repository_dependencies.xml.
for entry in changeset_groups:
if len( entry ) == 2:
# We possibly found an altered file entry.
filename, change_list = entry
if filename and isinstance( filename, string_types ):
if filename == rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME:
# Make sure the any complex repository dependency definitions contain valid <repository> tags.
is_valid, error_msg = self.repository_tags_are_valid( filename, change_list )
if not is_valid:
log.debug( error_msg )
return self.__display_exception_remotely( start_response, error_msg )
else:
msg = "Only a single file named repository_dependencies.xml can be pushed to a repository "
msg += "of type 'Repository suite definition'."
log.debug( msg )
return self.__display_exception_remotely( start_response, msg )
elif repository_type == rt_util.TOOL_DEPENDENCY_DEFINITION:
# Handle repositories of type tool_dependency_definition, which can only contain a single
# file named tool_dependencies.xml.
for entry in changeset_groups:
if len( entry ) == 2:
# We possibly found an altered file entry.
filename, change_list = entry
if filename and isinstance( filename, string_types ):
if filename == rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME:
# Make sure the any complex repository dependency definitions contain valid <repository> tags.
is_valid, error_msg = self.repository_tags_are_valid( filename, change_list )
if not is_valid:
log.debug( error_msg )
return self.__display_exception_remotely( start_response, error_msg )
else:
msg = "Only a single file named tool_dependencies.xml can be pushed to a repository "
msg += "of type 'Tool dependency definition'."
log.debug( msg )
return self.__display_exception_remotely( start_response, msg )
else:
# If the changeset includes changes to dependency definition files, make sure tag sets
# are not missing "toolshed" or "changeset_revision" attributes since automatically populating
# them is not supported when pushing from the command line. These attributes are automatically
# populated only when using the tool shed upload utility.
for entry in changeset_groups:
if len( entry ) == 2:
# We possibly found an altered file entry.
filename, change_list = entry
if filename and isinstance( filename, string_types ):
if filename in [ rt_util.REPOSITORY_DEPENDENCY_DEFINITION_FILENAME,
rt_util.TOOL_DEPENDENCY_DEFINITION_FILENAME ]:
# We check both files since tool dependency definitions files can contain complex
# repository dependency definitions.
is_valid, error_msg = self.repository_tags_are_valid( filename, change_list )
if not is_valid:
log.debug( error_msg )
return self.__display_exception_remotely( start_response, error_msg )
if isinstance( result, string_types ):
# Authentication was successful
AUTH_TYPE.update( environ, 'basic' )
REMOTE_USER.update( environ, result )
else:
return result.wsgi_application( environ, start_response )
return self.app( environ, start_response )
def __authenticate( self, username, password ):
db_password = None
# Instantiate a database connection
engine = sqlalchemy.create_engine( self.db_url )
connection = engine.connect()
result_set = connection.execute( "select email, password from galaxy_user where username = '%s'" % username.lower() )
for row in result_set:
# Should only be 1 row...
db_password = row[ 'password' ]
connection.close()
if db_password:
# Check if password matches db_password when hashed.
return new_secure_hash( text_type=password ) == db_password
return False
def __authenticate_remote_user( self, environ, username, password ):
"""
Look after a remote user and "authenticate" - upstream server should already have achieved
this for us, but we check that the user exists at least. Hg allow_push = must include username
- some versions of mercurial blow up with 500 errors.
"""
db_username = None
ru_email = environ[ 'HTTP_REMOTE_USER' ].lower()
# Instantiate a database connection...
engine = sqlalchemy.create_engine( self.db_url )
connection = engine.connect()
result_set = connection.execute( "select email, username, password from galaxy_user where email = '%s'" % ru_email )
for row in result_set:
# Should only be 1 row...
db_username = row[ 'username' ]
connection.close()
if db_username:
# We could check the password here except that the function galaxy.web.framework.get_or_create_remote_user()
# does some random generation of a password - so that no-one knows the password and only the hash is stored...
return db_username == username
return False
def __basic_authentication( self, environ, username, password ):
"""The environ parameter is needed in basic authentication. We also check it if use_remote_user is true."""
if asbool( self.config.get( 'use_remote_user', False ) ):
assert "HTTP_REMOTE_USER" in environ, "use_remote_user is set but no HTTP_REMOTE_USER variable"
return self.__authenticate_remote_user( environ, username, password )
else:
return self.__authenticate( username, password )
def __display_exception_remotely( self, start_response, msg ):
# Display the exception to the remote user's command line.
status = "500 %s" % msg
response_headers = [ ("content-type", "text/plain") ]
start_response( status, response_headers, sys.exc_info() )
return [ msg ]
def __get_hg_command( self, **kwd ):
"""Pulls mercurial commands from environ[ 'QUERY_STRING" ] and returns them."""
if 'QUERY_STRING' in kwd:
for qry in kwd[ 'QUERY_STRING' ].split( '&' ):
if qry.startswith( 'cmd' ):
return qry.split( '=' )[ -1 ]
return None
def __get_user_id_repository_name_from_path_info( self, db_connection, path_info ):
# An example of path_info is: '/repos/test/column1'
path_info_components = path_info.split( '/' )
username = path_info_components[ 1 ]
repository_name = path_info_components[ 2 ]
# Get the id of the current user using hg from the command line.
result_set = db_connection.execute( "select id from galaxy_user where username = '%s'" % username.lower() )
for row in result_set:
# Should only be 1 row...
user_id = row[ 'id' ]
return user_id, repository_name
def repository_tag_is_valid( self, filename, line ):
"""
Checks changes made to <repository> tags in a dependency definition file being pushed to the
Tool Shed from the command line to ensure that all required attributes exist.
"""
required_attributes = [ 'toolshed', 'name', 'owner', 'changeset_revision' ]
defined_attributes = line.split()
for required_attribute in required_attributes:
defined = False
for defined_attribute in defined_attributes:
if defined_attribute.startswith( required_attribute ):
defined = True
break
if not defined:
error_msg = 'The %s file contains a <repository> tag that is missing the required attribute %s. ' % \
( filename, required_attribute )
error_msg += 'Automatically populating dependency definition attributes occurs only when using '
error_msg += 'the Tool Shed upload utility. '
return False, error_msg
return True, ''
def repository_tags_are_valid( self, filename, change_list ):
"""
Make sure the any complex repository dependency definitions contain valid <repository> tags when pushing
changes to the tool shed on the command line.
"""
tag = '<repository'
for change_dict in change_list:
lines = commit_util.get_change_lines_in_file_for_tag( tag, change_dict )
for line in lines:
is_valid, error_msg = self.repository_tag_is_valid( filename, line )
if not is_valid:
return False, error_msg
return True, ''
test/shed_functional/functional/test_0310_hg_api_features.py
-22
View File
@@ -15,7 +15,6 @@ category_description = 'Test 0310 for verifying the tool shed http interface to
'''
1. Create a repository.
2. Clone the repository to a local path.
3. Check Mercurial push is disabled.
'''
@@ -93,24 +92,3 @@ class TestHgWebFeatures( ShedTwillTestCase ):
self.clone_repository( repository, clone_path )
files_in_repository = os.listdir( clone_path )
assert 'filtering.py' in files_in_repository, 'File not found in repository: filtering.py'
def test_0015_commit_and_push( self ):
'''
Edit a file and attempt a push as a user that does have write access.
We expect this to fail as pushing is disabled by default via disable_push config option.
We are at step 3 - Check Mercurial push is disabled.
'''
repository = self.test_db_util.get_repository_by_name_and_owner( repository_name, common.test_user_1_name )
clone_path = self.generate_temp_path( 'test_0310', additional_paths=[ 'filtering_0310', 'user1' ] )
self.clone_repository( repository, clone_path )
hgrepo = self.get_hg_repo( clone_path )
files_in_repository = os.listdir( clone_path )
assert 'filtering.py' in files_in_repository, 'File not found in repository: filtering.py'
filepath = os.path.join( clone_path, 'filtering.py' )
file_contents = [ '# This is another dummy comment to generate a new changeset.' ]
file_contents.extend( open( filepath, 'r' ).readlines() )
open( filepath, 'w' ).write( '\n'.join( file_contents ) )
commit_options = dict( user=common.test_user_1_name, message='Added another line to filtering.py.' )
success = self.commit_and_push( repository, hgrepo, commit_options, username=common.test_user_1_name, password='testuser' )
assert success is False, 'Test user 1 (repo owner) was able to commit and push to the remote repository.'