From 35b7c7788ac439f6ff3ae383aa910545c7ee85ae Mon Sep 17 00:00:00 2001 From: Carl Eberhard Date: Tue, 21 Apr 2015 15:09:28 -0400 Subject: [PATCH] API, batch: initial commit --- lib/galaxy/web/framework/base.py | 6 +- lib/galaxy/web/framework/middleware/batch.py | 126 +++++++++++++++++++ lib/galaxy/webapps/galaxy/buildapp.py | 8 ++ test/api/test_api_batch.py | 35 ++++++ test/casperjs/api-batch-tests.js | 57 +++++++++ 5 files changed, 231 insertions(+), 1 deletion(-) create mode 100644 lib/galaxy/web/framework/middleware/batch.py create mode 100644 test/api/test_api_batch.py create mode 100644 test/casperjs/api-batch-tests.js diff --git a/lib/galaxy/web/framework/base.py b/lib/galaxy/web/framework/base.py index 590cf9a37eb..6d7ee5706ad 100644 --- a/lib/galaxy/web/framework/base.py +++ b/lib/galaxy/web/framework/base.py @@ -136,7 +136,7 @@ class WebApplication( object ): if self.trace_logger: self.trace_logger.context_remove( "request_id" ) - def handle_request( self, environ, start_response ): + def handle_request( self, environ, start_response, body_renderer=None ): # Grab the request_id (should have been set by middleware) request_id = environ.get( 'request_id', 'unknown' ) # Map url using routes @@ -193,6 +193,10 @@ class WebApplication( object ): body = self.handle_controller_exception( e, trans, **kwargs ) if not body: raise + body_renderer = body_renderer or self._render_body + return body_renderer( trans, body, environ, start_response ) + + def _render_body( self, trans, body, environ, start_response ): # Now figure out what we got back and try to get it to the browser in # a smart way if callable( body ): diff --git a/lib/galaxy/web/framework/middleware/batch.py b/lib/galaxy/web/framework/middleware/batch.py new file mode 100644 index 00000000000..b40b5ae2202 --- /dev/null +++ b/lib/galaxy/web/framework/middleware/batch.py @@ -0,0 +1,126 @@ +""" +""" + +import pkg_resources +pkg_resources.require( "Routes" ) +import routes + +import StringIO +from urlparse import urlparse +import simplejson as json + +import pprint + +import logging +log = logging.getLogger( __name__ ) + + +class BatchMiddleware(object): + """ + """ + DEFAULT_CONFIG = { + 'route' : '/api/batch' + } + # TODO: whitelist or blacklisted urls + + def __init__( self, galaxy, application, config=None ): + #: the original galaxy webapp + self.galaxy = galaxy + #: the wrapped webapp + self.application = application + self.config = config or self.DEFAULT_CONFIG + self.handle_request = self.galaxy.handle_request + + def __call__( self, environ, start_response ): + if environ[ 'PATH_INFO' ] == self.config[ 'route' ]: + return self.process_batch_requests( environ, start_response ) + return self.application( environ, start_response ) + + def process_batch_requests( self, batch_environ, start_response ): + # log.debug( ( '=' * 40 ) ) + # log.debug( 'BATCH_REQUEST' ) + + payload = self._read_post_payload( batch_environ ) + requests = payload.get( 'batch', [] ) + responses = [] + for request in requests: + if not self._valid( request ): + continue + + # log.debug( ( '-' * 40 ) + 'REQUEST BEGIN' ) + request_environ = self._build_request_environ( batch_environ, request ) + # log.debug( 'request: %s', request ) + responses.append( self._proccess_batch_request( request, request_environ, start_response ) ) + # log.debug( ( '-' * 40 ) + 'END' ) + + # log.debug( 'batch_response_body: %s', pprint.pformat( responses ) ) + batch_response_body = json.dumps( responses ) + start_response( '200 OK', [ + ( 'Content-Length', len( batch_response_body ) ), + ( 'Content-Type', 'application/json' ), + ]) + # log.debug( ( '=' * 40 ) ) + return batch_response_body + + def _valid( self, request ): + return True + + def _read_post_payload( self, environ ): + request_body_size = int( environ.get( 'CONTENT_LENGTH', 0 ) ) + request_body = environ[ 'wsgi.input' ].read( request_body_size ) or '{}' + # TODO: json decode error handling + # log.debug( 'request_body: (%s)\n%s', type( request_body ), request_body ) + payload = json.loads( request_body ) + return payload + + def _build_request_environ( self, original_environ, request ): + """ + Given a request and the original environ used to call the batch, return + a new environ parsable/suitable for the individual api call. + """ + # TODO: use a dict of defaults/config + # copy the original environ and reconstruct a fake version for each batched request + request_environ = original_environ.copy() + # TODO: for now, do not overwrite the other headers used in the main api/batch request + request_environ[ 'CONTENT_TYPE' ] = request.get( 'contentType', 'application/json' ) + request_environ[ 'REQUEST_METHOD' ] = request.get( 'method', request.get( 'type', 'GET' ) ) + url = '{0}://{1}{2}'.format( request_environ.get( 'wsgi.url_scheme' ), + request_environ.get( 'HTTP_HOST' ), + request[ 'url' ] ) + parsed = urlparse( url ) + request_environ[ 'PATH_INFO' ] = parsed.path + request_environ[ 'QUERY_STRING' ] = parsed.query + request_environ[ 'wsgi.input' ] = StringIO.StringIO( request.get( 'body', '' ) ) + + # log.debug( 'request_environ:\n%s', pprint.pformat( request_environ ) ) + return request_environ + + def _proccess_batch_request( self, request, environ, start_response ): + # log.debug( ( '.' * 40 ) ) + + # We have to re-create the handle request method here in order to bypass reusing the 'api/batch' request + # because reuse will cause the paste error: + # File "/Users/carleberhard/galaxy/api-v2/eggs/Paste-1.7.5.1-py2.7.egg/paste/httpserver.py", line 166, in wsgi_start_response + # assert 0, "Attempt to set headers a second time w/o an exc_info" + status, headers, body = self.galaxy.handle_request( environ, start_response, body_renderer=self.body_renderer ) + + # We may need to include middleware to record various reponses, but this way of doing that won't work: + # status, headers, body = self.application( environ, start_response, body_renderer=self.body_renderer ) + + # log.debug( ( '.' * 40 ) ) + return dict( + status=status, + headers=headers, + body=body, + ) + + def body_renderer( self, trans, body, environ, start_response ): + # this is a dummy renderer that does not call start_response + return ( + trans.response.status, + trans.response.headers, + json.loads( self.galaxy.make_body_iterable( trans, body )[0] ) + ) + + def handle_exception( self, environ ): + return False diff --git a/lib/galaxy/webapps/galaxy/buildapp.py b/lib/galaxy/webapps/galaxy/buildapp.py index f6148079644..8fac89de1d3 100644 --- a/lib/galaxy/webapps/galaxy/buildapp.py +++ b/lib/galaxy/webapps/galaxy/buildapp.py @@ -586,6 +586,8 @@ def wrap_in_middleware( app, global_conf, **local_conf ): Based on the configuration wrap `app` in a set of common and useful middleware. """ + webapp = app + # Merge the global and local configurations conf = global_conf.copy() conf.update(local_conf) @@ -658,6 +660,12 @@ def wrap_in_middleware( app, global_conf, **local_conf ): from galaxy.web.framework.middleware.request_id import RequestIDMiddleware app = RequestIDMiddleware( app ) log.debug( "Enabling 'Request ID' middleware" ) + + # batch call processing middleware + from galaxy.web.framework.middleware.batch import BatchMiddleware + app = BatchMiddleware( webapp, app, {}) + log.debug( "Enabling 'Batch' middleware" ); + return app diff --git a/test/api/test_api_batch.py b/test/api/test_api_batch.py new file mode 100644 index 00000000000..f419326d9ad --- /dev/null +++ b/test/api/test_api_batch.py @@ -0,0 +1,35 @@ +import simplejson +from requests import post + +from base import api +# from .helpers import DatasetPopulator + +import logging +log = logging.getLogger( "functional_tests.py" ) + + +class ApiBatchTestCase( api.ApiTestCase ): + + def _get_api_key( self, admin=False ): + return self.galaxy_interactor.api_key if not admin else self.galaxy_interactor.master_api_key + + def _with_key( self, url, admin=False ): + return url + '?key=' + self._get_api_key( admin=admin ) + + def _post_batch( self, batch ): + data = simplejson.dumps({ "batch" : batch }) + return post( "%s/batch" % ( self.galaxy_interactor.api_url ), data=data ) + + def test_simple_array( self ): + # post_body = dict( name='wert' ) + batch = [ + dict( url=self._with_key( '/api/histories' ) ), + dict( url=self._with_key( '/api/histories' ), + method='POST', body=simplejson.dumps( dict( name='Wat' ) ) ), + dict( url=self._with_key( '/api/histories' ) ), + ] + response = self._post_batch( batch ) + response = response.json() + # log.debug( 'RESPONSE %s\n%s', ( '-' * 40 ), pprint.pformat( response ) ) + self.assertIsInstance( response, list ) + self.assertEquals( len( response ), 3 ) diff --git a/test/casperjs/api-batch-tests.js b/test/casperjs/api-batch-tests.js new file mode 100644 index 00000000000..e4f0c09fbf2 --- /dev/null +++ b/test/casperjs/api-batch-tests.js @@ -0,0 +1,57 @@ +var require = patchRequire( require ), + spaceghost = require( 'spaceghost' ).fromCasper( casper ), + xpath = require( 'casper' ).selectXPath, + utils = require( 'utils' ), + format = utils.format; + +spaceghost.test.begin( 'Test the API batch system', 0, function suite( test ){ + spaceghost.start(); + + // ======================================================================== SET UP + var email = spaceghost.user.getRandomEmail(), + password = '123456'; + if( spaceghost.fixtureData.testUser ){ + email = spaceghost.fixtureData.testUser.email; + password = spaceghost.fixtureData.testUser.password; + } + spaceghost.user.registerUser( email, password ); + + var responseKeys = [ 'body', 'headers', 'status' ]; + + // ======================================================================== TESTS + spaceghost.then( function(){ + // -------------------------------------------------------------------- + this.test.comment( 'API batching should allow multiple requests and responses, executed in order' ); + var batch = [ + { url : '/api/histories' }, + { url : '/api/histories', type: 'POST', body: JSON.stringify({ name: 'wert' }) }, + { url : '/api/histories' }, + ], + responses = this.api._ajax( 'api/batch', { + type : 'POST', + contentType : 'application/json', + data : { batch : batch } + // data : JSON.stringify({ batch : batch }) + }); + this.debug( 'responses:' + this.jsonStr( responses ) ); + + this.test.assert( utils.isArray( responses ), "returned an array: length " + responses.length ); + this.test.assert( responses.length === 3, 'Has three responses' ); + + var historiesBeforeCreate = responses[0], + createdHistory = responses[1], + historiesAfterCreate = responses[2]; + this.test.assert( utils.isArray( historiesBeforeCreate.body ), + "first histories call returned an array" + historiesBeforeCreate.body.length ); + this.test.assert( utils.isObject( createdHistory.body ), 'history create returned an object' ); + this.test.assert( historiesAfterCreate.body[0].id === createdHistory.body.id, + "second histories call includes the newly created history:" + historiesAfterCreate.body[0].id ); + /* + */ + }); + //spaceghost.user.logout(); + + // =================================================================== + spaceghost.run( function(){ test.done(); }); +}); +