API, batch: initial commit

lib/galaxy/web/framework/base.py

@@ -136,7 +136,7 @@ class WebApplication( object ):
             if self.trace_logger:
                 self.trace_logger.context_remove( "request_id" )
 
-    def handle_request( self, environ, start_response ):
+    def handle_request( self, environ, start_response, body_renderer=None ):
         # Grab the request_id (should have been set by middleware)
         request_id = environ.get( 'request_id', 'unknown' )
         # Map url using routes
@@ -193,6 +193,10 @@ class WebApplication( object ):
             body = self.handle_controller_exception( e, trans, **kwargs )
             if not body:
                 raise
+        body_renderer = body_renderer or self._render_body
+        return body_renderer( trans, body, environ, start_response )
+
+    def _render_body( self, trans, body, environ, start_response ):
         # Now figure out what we got back and try to get it to the browser in
         # a smart way
         if callable( body ):

lib/galaxy/web/framework/middleware/batch.py

@@ -0,0 +1,126 @@
+"""
+"""
+
+import pkg_resources
+pkg_resources.require( "Routes" )
+import routes
+
+import StringIO
+from urlparse import urlparse
+import simplejson as json
+
+import pprint
+
+import logging
+log = logging.getLogger( __name__ )
+
+
+class BatchMiddleware(object):
+    """
+    """
+    DEFAULT_CONFIG = {
+        'route' : '/api/batch'
+    }
+    # TODO: whitelist or blacklisted urls
+
+    def __init__( self, galaxy, application, config=None ):
+        #: the original galaxy webapp
+        self.galaxy = galaxy
+        #: the wrapped webapp
+        self.application = application
+        self.config = config or self.DEFAULT_CONFIG
+        self.handle_request = self.galaxy.handle_request
+
+    def __call__( self, environ, start_response ):
+        if environ[ 'PATH_INFO' ] == self.config[ 'route' ]:
+            return self.process_batch_requests( environ, start_response )
+        return self.application( environ, start_response )
+
+    def process_batch_requests( self, batch_environ, start_response ):
+        # log.debug( ( '=' * 40 ) )
+        # log.debug( 'BATCH_REQUEST' )
+
+        payload = self._read_post_payload( batch_environ )
+        requests = payload.get( 'batch', [] )
+        responses = []
+        for request in requests:
+            if not self._valid( request ):
+                continue
+
+            # log.debug( ( '-' * 40 ) + 'REQUEST BEGIN' )
+            request_environ = self._build_request_environ( batch_environ, request )
+            # log.debug( 'request: %s', request )
+            responses.append( self._proccess_batch_request( request, request_environ, start_response ) )
+            # log.debug( ( '-' * 40 ) + 'END' )
+
+        # log.debug( 'batch_response_body: %s', pprint.pformat( responses ) )
+        batch_response_body = json.dumps( responses )
+        start_response( '200 OK', [
+            ( 'Content-Length', len( batch_response_body ) ),
+            ( 'Content-Type', 'application/json' ),
+        ])
+        # log.debug( ( '=' * 40 ) )
+        return batch_response_body
+
+    def _valid( self, request ):
+        return True
+
+    def _read_post_payload( self, environ ):
+        request_body_size = int( environ.get( 'CONTENT_LENGTH', 0 ) )
+        request_body = environ[ 'wsgi.input' ].read( request_body_size ) or '{}'
+        # TODO: json decode error handling
+        # log.debug( 'request_body: (%s)\n%s', type( request_body ), request_body )
+        payload = json.loads( request_body )
+        return payload
+
+    def _build_request_environ( self, original_environ, request ):
+        """
+        Given a request and the original environ used to call the batch, return
+        a new environ parsable/suitable for the individual api call.
+        """
+        # TODO: use a dict of defaults/config
+        # copy the original environ and reconstruct a fake version for each batched request
+        request_environ = original_environ.copy()
+        # TODO: for now, do not overwrite the other headers used in the main api/batch request
+        request_environ[ 'CONTENT_TYPE' ] = request.get( 'contentType', 'application/json' )
+        request_environ[ 'REQUEST_METHOD' ] = request.get( 'method', request.get( 'type', 'GET' ) )
+        url = '{0}://{1}{2}'.format( request_environ.get( 'wsgi.url_scheme' ),
+                                     request_environ.get( 'HTTP_HOST' ),
+                                     request[ 'url' ] )
+        parsed = urlparse( url )
+        request_environ[ 'PATH_INFO' ] = parsed.path
+        request_environ[ 'QUERY_STRING' ] = parsed.query
+        request_environ[ 'wsgi.input' ] = StringIO.StringIO( request.get( 'body', '' ) )
+
+        # log.debug( 'request_environ:\n%s', pprint.pformat( request_environ ) )
+        return request_environ
+
+    def _proccess_batch_request( self, request, environ, start_response ):
+        # log.debug( ( '.' * 40 ) )
+
+        # We have to re-create the handle request method here in order to bypass reusing the 'api/batch' request
+        #   because reuse will cause the paste error:
+        # File "/Users/carleberhard/galaxy/api-v2/eggs/Paste-1.7.5.1-py2.7.egg/paste/httpserver.py", line 166, in wsgi_start_response
+        #     assert 0, "Attempt to set headers a second time w/o an exc_info"
+        status, headers, body = self.galaxy.handle_request( environ, start_response, body_renderer=self.body_renderer )
+
+        # We may need to include middleware to record various reponses, but this way of doing that won't work:
+        # status, headers, body = self.application( environ, start_response, body_renderer=self.body_renderer )
+
+        # log.debug( ( '.' * 40 ) )
+        return dict(
+            status=status,
+            headers=headers,
+            body=body,
+        )
+
+    def body_renderer( self, trans, body, environ, start_response ):
+        # this is a dummy renderer that does not call start_response
+        return (
+            trans.response.status,
+            trans.response.headers,
+            json.loads( self.galaxy.make_body_iterable( trans, body )[0] )
+        )
+
+    def handle_exception( self, environ ):
+        return False

lib/galaxy/webapps/galaxy/buildapp.py

@@ -586,6 +586,8 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
     Based on the configuration wrap `app` in a set of common and useful
     middleware.
     """
+    webapp = app
+
     # Merge the global and local configurations
     conf = global_conf.copy()
     conf.update(local_conf)
@@ -658,6 +660,12 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
     from galaxy.web.framework.middleware.request_id import RequestIDMiddleware
     app = RequestIDMiddleware( app )
     log.debug( "Enabling 'Request ID' middleware" )
+
+    # batch call processing middleware
+    from galaxy.web.framework.middleware.batch import BatchMiddleware
+    app = BatchMiddleware( webapp, app, {})
+    log.debug( "Enabling 'Batch' middleware" );
+
     return app
 
 

test/api/test_api_batch.py

@@ -0,0 +1,35 @@
+import simplejson
+from requests import post
+
+from base import api
+# from .helpers import DatasetPopulator
+
+import logging
+log = logging.getLogger( "functional_tests.py" )
+
+
+class ApiBatchTestCase( api.ApiTestCase ):
+
+    def _get_api_key( self, admin=False ):
+        return self.galaxy_interactor.api_key if not admin else self.galaxy_interactor.master_api_key
+
+    def _with_key( self, url, admin=False ):
+        return url + '?key=' + self._get_api_key( admin=admin )
+
+    def _post_batch( self, batch ):
+        data = simplejson.dumps({ "batch" : batch })
+        return post( "%s/batch" % ( self.galaxy_interactor.api_url ), data=data )
+
+    def test_simple_array( self ):
+        # post_body = dict( name='wert' )
+        batch = [
+            dict( url=self._with_key( '/api/histories' ) ),
+            dict( url=self._with_key( '/api/histories' ),
+                  method='POST', body=simplejson.dumps( dict( name='Wat' ) ) ),
+            dict( url=self._with_key( '/api/histories' ) ),
+        ]
+        response = self._post_batch( batch )
+        response = response.json()
+        # log.debug( 'RESPONSE %s\n%s', ( '-' * 40 ), pprint.pformat( response ) )
+        self.assertIsInstance( response, list )
+        self.assertEquals( len( response ), 3 )

test/casperjs/api-batch-tests.js

@@ -0,0 +1,57 @@
+var require = patchRequire( require ),
+    spaceghost = require( 'spaceghost' ).fromCasper( casper ),
+    xpath = require( 'casper' ).selectXPath,
+    utils = require( 'utils' ),
+    format = utils.format;
+
+spaceghost.test.begin( 'Test the API batch system', 0, function suite( test ){
+    spaceghost.start();
+
+    // ======================================================================== SET UP
+    var email = spaceghost.user.getRandomEmail(),
+        password = '123456';
+    if( spaceghost.fixtureData.testUser ){
+        email = spaceghost.fixtureData.testUser.email;
+        password = spaceghost.fixtureData.testUser.password;
+    }
+    spaceghost.user.registerUser( email, password );
+
+    var responseKeys = [ 'body', 'headers', 'status' ];
+
+    // ======================================================================== TESTS
+    spaceghost.then( function(){
+        // --------------------------------------------------------------------
+        this.test.comment( 'API batching should allow multiple requests and responses, executed in order' );
+        var batch = [
+                { url : '/api/histories' },
+                { url : '/api/histories', type: 'POST', body: JSON.stringify({ name: 'wert' }) },
+                { url : '/api/histories' },
+            ],
+            responses = this.api._ajax( 'api/batch', {
+                type        : 'POST',
+                contentType : 'application/json',
+                data        : { batch : batch }
+                // data        : JSON.stringify({ batch : batch })
+            });
+        this.debug( 'responses:' + this.jsonStr( responses ) );
+
+        this.test.assert( utils.isArray( responses ), "returned an array: length " + responses.length );
+        this.test.assert( responses.length === 3, 'Has three responses' );
+
+        var historiesBeforeCreate = responses[0],
+            createdHistory = responses[1],
+            historiesAfterCreate = responses[2];
+        this.test.assert( utils.isArray( historiesBeforeCreate.body ),
+            "first histories call returned an array" + historiesBeforeCreate.body.length );
+        this.test.assert( utils.isObject( createdHistory.body ), 'history create returned an object' );
+        this.test.assert( historiesAfterCreate.body[0].id === createdHistory.body.id,
+            "second histories call includes the newly created history:" + historiesAfterCreate.body[0].id );
+    /*
+    */
+    });
+    //spaceghost.user.logout();
+
+    // ===================================================================
+    spaceghost.run( function(){ test.done(); });
+});
+