Support for using Sentry for error logging. If sentry_dsn is set in

universe_wsgi.ini, then:
 - A middleware will be installed that send thrown exceptions to sentry
 - All logging messages of level WARN and above will be sent to sentry
 - All pages extending base/base_panels will have javascript errors
   captured and sent to sentry

eggs.ini

@@ -66,6 +66,7 @@ Babel = 0.9.4
 wchartype = 0.1
 Whoosh = 0.3.18
 ; fluent_logger = 0.3.3
+raven = 3.1.8
 
 ; extra version information
 [tags]

lib/galaxy/config.py

@@ -2,7 +2,7 @@
 Universe configuration builder.
 """
 
-import sys, os, tempfile
+import sys, os, tempfile, re
 import logging, logging.config
 import ConfigParser
 from datetime import timedelta
@@ -261,11 +261,24 @@ class Configuration( object ):
         self.api_folders = string_as_bool( kwargs.get( 'api_folders', False ) )
         # This is for testing new library browsing capabilities.
         self.new_lib_browse = string_as_bool( kwargs.get( 'new_lib_browse', False ) )
+        # Error logging with sentry
+        self.sentry_dsn = kwargs.get( 'sentry_dsn', None )
         # Logging with fluentd
         self.fluent_log = string_as_bool( kwargs.get( 'fluent_log', False ) )
         self.fluent_host = kwargs.get( 'fluent_host', 'localhost' )
         self.fluent_port = int( kwargs.get( 'fluent_port', 24224 ) )
 
+    @property
+    def sentry_dsn_public( self ):
+        """
+        Sentry URL with private key removed for use in client side scripts, 
+        sentry server will need to be configured to accept events
+        """
+        if self.sentry_dsn:
+            return re.sub( r"^([^:/?#]+:)?//(\w+):(\w+)", r"\1//\2", self.sentry_dsn )
+        else:
+            return None
+
     def __read_tool_job_config( self, global_conf_parser, section, key ):
         try:
             tool_runners_config = global_conf_parser.items( section )
@@ -387,8 +400,7 @@ def get_database_engine_options( kwargs ):
 
 def configure_logging( config ):
     """
-    Allow some basic logging configuration to be read from the cherrpy
-    config.
+    Allow some basic logging configuration to be read from ini file.
     """
     # PasteScript will have already configured the logger if the appropriate
     # sections were found in the config file, so we do nothing if the
@@ -420,3 +432,10 @@ def configure_logging( config ):
     # Hook everything up
     handler.setFormatter( formatter )
     root.addHandler( handler )
+    # If sentry is configured, also log to it
+    if config.sentry_dsn:
+        pkg_resources.require( "raven" )
+        from raven.handlers.logging import SentryHandler
+        sentry_handler = SentryHandler( config.sentry_dsn )
+        sentry_handler.setLevel( logging.WARN )
+        root.addHandler( sentry_handler )

lib/galaxy/web/framework/base.py

@@ -9,7 +9,6 @@ import os.path
 import sys
 import tarfile
 import threading
-import uuid
 
 from Cookie import SimpleCookie
 
@@ -70,9 +69,6 @@ class WebApplication( object ):
         self.mapper.explicit = False
         self.api_mapper = routes.Mapper()
         self.transaction_factory = DefaultWebTransaction
-        # Each request will have a unique id. Since we are assuming
-        # a threaded model for the moment we can store that here
-        self.request_id = threading.local()
         # Set if trace logging is enabled
         self.trace_logger = None
     def add_ui_controller( self, controller_name, controller ):
@@ -124,7 +120,7 @@ class WebApplication( object ):
         and calls it.
         """
         # Immediately create request_id which we will use for logging
-        self.request_id = request_id = uuid.uuid1().hex
+        request_id = environ.get( 'request_id', 'unknown' )
         if self.trace_logger:
             self.trace_logger.context_set( "request_id", request_id )
         self.trace( message="Starting request" )
@@ -136,6 +132,8 @@ class WebApplication( object ):
                 self.trace_logger.context_remove( "request_id" )
 
     def handle_request( self, environ, start_response ):
+        # Grab the request_id (should have been set by middleware)
+        request_id = environ.get( 'request_id', 'unknown' )
         # Map url using routes
         path_info = environ.get( 'PATH_INFO', '' )
         map = self.mapper.match( path_info, environ )
@@ -157,6 +155,7 @@ class WebApplication( object ):
         rc.environ = environ
         # Setup the transaction
         trans = self.transaction_factory( environ )
+        trans.request_id = request_id
         rc.redirect = trans.response.send_redirect
         # Get the controller class
         controller_name = map.pop( 'controller', None )

lib/galaxy/webapps/galaxy/buildapp.py

@@ -240,6 +240,14 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
         from paste import recursive
         app = recursive.RecursiveMiddleware( app, conf )
         log.debug( "Enabling 'recursive' middleware" )
+    # If sentry logging is enabled, log here before propogating up to
+    # the error middleware
+    sentry_url = conf.get( 'sentry_url', None )
+    if sentry_url:
+        pkg_resources.require( "raven")
+        from raven import Client
+        from raven.middleware import Sentry
+        app = Sentry( app, Client( sentry_url ) )
     # Various debug middleware that can only be turned on if the debug
     # flag is set, either because they are insecure or greatly hurt
     # performance
@@ -254,12 +262,6 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
             from paste.debug import profile
             app = profile.ProfileMiddleware( app, conf )
             log.debug( "Enabling 'profile' middleware" )
-        # Middleware that intercepts print statements and shows them on the
-        # returned page
-        if asbool( conf.get( 'use_printdebug', True ) ):
-            from paste.debug import prints
-            app = prints.PrintDebugMiddleware( app, conf )
-            log.debug( "Enabling 'print debug' middleware" )
     if debug and asbool( conf.get( 'use_interactive', False ) ):
         # Interactive exception debugging, scary dangerous if publicly
         # accessible, if not enabled we'll use the regular error printing
@@ -288,6 +290,10 @@ def wrap_in_middleware( app, global_conf, **local_conf ):
     from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware
     app = XForwardedHostMiddleware( app )
     log.debug( "Enabling 'x-forwarded-host' middleware" )
+    # Request ID middleware
+    from galaxy.web.framework.middleware.request_id import RequestIDMiddleware
+    app = RequestIDMiddleware( app )
+    log.debug( "Enabling 'Request ID' middleware" )
     return app
     
 def wrap_in_static( app, global_conf, **local_conf ):

static/scripts/libs/raven.js

@@ -0,0 +1,438 @@
+(function(){
+
+// Raven.js
+//
+// Originally based on the Arecibo JavaScript client.
+//
+// Requires:
+//     * TraceKit (included in the full and minified distribution files)
+
+'use strict';
+
+// First, check for JSON support
+// If there is no JSON, we no-op the core features of Raven
+// since JSON is required to encode the payload
+var _Raven = window.Raven,
+    hasJSON = !isUndefined(window.JSON),
+    globalServer,
+    globalUser,
+    globalKey,
+    globalProject,
+    globalOptions = {
+        logger: 'javascript',
+        ignoreErrors: [],
+        ignoreUrls: []
+    };
+
+var TK = TraceKit.noConflict();
+
+// Disable Tracekit's remote fetching by default
+TK.remoteFetching = false;
+
+/*
+ * The core Raven object
+ */
+var Raven = {
+    VERSION: '@VERSION',
+
+    /*
+     * Raven.noConflict()
+     *
+     * Allow multiple versions of Raven to be installed.
+     */
+    noConflict: function() {
+        window.Raven = _Raven;
+        return Raven;
+    },
+
+    /*
+     * Raven.config()
+     *
+     * Configure raven with a DSN and extra options
+     */
+    config: function(dsn, options) {
+        var uri = parseUri(dsn),
+            lastSlash = uri.path.lastIndexOf('/'),
+            path = uri.path.substr(1, lastSlash);
+
+        // merge in options
+        if (options) {
+            each(options, function(key, value){
+                globalOptions[key] = value;
+            });
+        }
+
+        // "Script error." is hard coded into browsers for errors that it can't read.
+        // this is the result of a script being pulled in from an external domain and CORS.
+        globalOptions.ignoreErrors.push('Script error.');
+
+        globalKey = uri.user;
+        globalProject = ~~uri.path.substr(lastSlash + 1);
+
+        // assemble the endpoint from the uri pieces
+        globalServer = uri.protocol + '://' + uri.host +
+                      (uri.port ? ':' + uri.port : '') +
+                      '/' + path + 'api/' + globalProject + '/store/';
+
+        if (globalOptions.fetchContext) {
+            TK.remoteFetching = true;
+        }
+
+        // return for chaining
+        return Raven;
+    },
+
+    /*
+     * Raven.install()
+     *
+     * Installs a global window.onerror error handler
+     * to capture and report uncaught exceptions.
+     */
+    install: function() {
+        if (!isSetup()) return;
+
+        TK.report.subscribe(handleStackInfo);
+
+        return Raven;
+    },
+
+    /*
+     * Raven.context()
+     *
+     * Wrap code within a context so Raven can capture errors
+     * reliably across domains that is executed immediately.
+     */
+    context: function(options, func, args) {
+        if (isFunction(options)) {
+            args = func;
+            func = options;
+            options = undefined;
+        }
+
+        Raven.wrap(options, func).apply(this, args);
+    },
+
+    /* Raven.wrap()
+     *
+     * Wrap code within a context and returns back a new function to be executed
+     */
+    wrap: function(options, func) {
+        // options is optional
+        if (isFunction(options)) {
+            func = options;
+            options = undefined;
+        }
+
+        return function() {
+            try {
+                func.apply(this, arguments);
+            } catch(e) {
+                Raven.captureException(e, options);
+            }
+        };
+    },
+
+    /*
+     * Raven.uninstall()
+     *
+     * Uninstalls the global error handler.
+     */
+    uninstall: function() {
+        TK.report.unsubscribe(handleStackInfo);
+
+        return Raven;
+    },
+
+    /*
+     * Raven.captureException()
+     *
+     * Manually capture an exception and send it over to Sentry
+     */
+    captureException: function(ex, options) {
+        // TraceKit.report will re-raise any exception passed to it,
+        // which means you have to wrap it in try/catch. Instead, we
+        // can wrap it here and only re-raise if TraceKit.report
+        // raises an exception different from the one we asked to
+        // report on.
+        try {
+            TK.report(ex, options);
+        } catch(ex1) {
+            if(ex !== ex1) {
+                throw ex1;
+            }
+        }
+
+        return Raven;
+    },
+
+    /*
+     * Raven.captureMessage()
+     *
+     * Manually send a message to Sentry
+     */
+    captureMessage: function(msg, options) {
+        // Fire away!
+        send(
+            arrayMerge({
+                message: msg
+            }, options)
+        );
+
+        return Raven;
+    },
+
+    /*
+     * Raven.setUser()
+     *
+     * Set/clear a user to be sent along with the payload.
+     */
+    setUser: function(user) {
+       globalUser = user;
+
+       return Raven;
+    }
+};
+
+var uriKeys = 'source protocol authority userInfo user password host port relative path directory file query anchor'.split(' '),
+    uriPattern = /^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/;
+
+/**** Private functions ****/
+function parseUri(str) {
+    var m = uriPattern.exec(str),
+        uri = {},
+        i = 14;
+
+    while (i--) uri[uriKeys[i]] = m[i] || '';
+
+    return uri;
+}
+
+function isUndefined(what) {
+    return typeof what === 'undefined';
+}
+
+function isFunction(what) {
+    return typeof what === 'function';
+}
+
+function each(obj, callback) {
+    var i, j;
+
+    if (obj.length === undefined) {
+        for (i in obj) {
+            if (obj.hasOwnProperty(i)) {
+                callback.call(null, i, obj[i]);
+            }
+        }
+    } else {
+        for (i = 0, j = obj.length; i < j; i++) {
+            callback.call(null, i, obj[i]);
+        }
+    }
+}
+
+var cachedAuth;
+
+function getAuthQueryString() {
+    if (cachedAuth) return cachedAuth;
+
+    var qs = [
+        'sentry_version=2.0',
+        'sentry_client=raven-js/' + Raven.VERSION
+    ];
+    if (globalKey) {
+        qs.push('sentry_key=' + globalKey);
+    }
+
+    cachedAuth = '?' + qs.join('&');
+    return cachedAuth;
+}
+
+function handleStackInfo(stackInfo, options) {
+    var frames = [], i = 0, j = stackInfo.stack && stackInfo.stack.length || 0, frame;
+
+    for (; i < j; i++) {
+        frame = normalizeFrame(stackInfo.stack[i]);
+        if (frame) {
+            frames.push(frame);
+        }
+    }
+
+    processException(
+        stackInfo.name,
+        stackInfo.message,
+        stackInfo.url,
+        stackInfo.lineno,
+        frames,
+        options
+    );
+}
+
+function normalizeFrame(frame) {
+    if (!frame.url) return;
+
+    // normalize the frames data
+    var normalized = {
+        filename:   frame.url,
+        lineno:     frame.line,
+        colno:      frame.column,
+        'function': frame.func || '?'
+    }, context = extractContextFromFrame(frame);
+
+    if (context) {
+        var i = 3, keys = ['pre_context', 'context_line', 'post_context'];
+        while (i--) normalized[keys[i]] = context[i];
+    }
+
+    normalized.in_app = !/(Raven|TraceKit)\./.test(normalized['function']);
+
+    return normalized;
+}
+
+function extractContextFromFrame(frame) {
+    // immediately check if we should even attempt to parse a context
+    if (!frame.context || !globalOptions.fetchContext) return;
+
+    var context = frame.context,
+        pivot = ~~(context.length / 2),
+        i = context.length, isMinified = false;
+
+    while (i--) {
+        // We're making a guess to see if the source is minified or not.
+        // To do that, we make the assumption if *any* of the lines passed
+        // in are greater than 300 characters long, we bail.
+        // Sentry will see that there isn't a context
+        if (context[i].length > 300) {
+            isMinified = true;
+            break;
+        }
+    }
+
+    if (isMinified) {
+        // The source is minified and we don't know which column. Fuck it.
+        if (isUndefined(frame.column)) return;
+
+        // If the source is minified and has a frame column
+        // we take a chunk of the offending line to hopefully shed some light
+        return [
+            [],  // no pre_context
+            context[pivot].substr(frame.column, 50), // grab 50 characters, starting at the offending column
+            []   // no post_context
+        ];
+    }
+
+    return [
+        context.slice(0, pivot),    // pre_context
+        context[pivot],             // context_line
+        context.slice(pivot + 1)    // post_context
+    ];
+}
+
+function processException(type, message, fileurl, lineno, frames, options) {
+    var stacktrace, label, i;
+
+    // IE8 really doesn't have Array.prototype.indexOf
+    // Filter out a message that matches our ignore list
+    i = globalOptions.ignoreErrors.length;
+    while (i--) {
+        if (message === globalOptions.ignoreErrors[i]) {
+            return;
+        }
+    }
+
+    if (frames && frames.length) {
+        stacktrace = {frames: frames};
+        fileurl = fileurl || frames[0].filename;
+    } else if (fileurl) {
+        stacktrace = {
+            frames: [{
+                filename: fileurl,
+                lineno: lineno
+            }]
+        };
+    }
+
+    i = globalOptions.ignoreUrls.length;
+    while (i--) {
+        if (globalOptions.ignoreUrls[i].test(fileurl)) {
+            return;
+        }
+    }
+
+    label = lineno ? message + ' at ' + lineno : message;
+
+    // Fire away!
+    send(
+        arrayMerge({
+            'sentry.interfaces.Exception': {
+                type: type,
+                value: message
+            },
+            'sentry.interfaces.Stacktrace': stacktrace,
+            culprit: fileurl,
+            message: label
+        }, options)
+    );
+}
+
+function arrayMerge(arr1, arr2) {
+    if (!arr2) {
+        return arr1;
+    }
+    each(arr2, function(key, value){
+        arr1[key] = value;
+    });
+    return arr1;
+}
+
+function getHttpData() {
+    var http = {
+        url: window.location.href,
+        headers: {
+            'User-Agent': navigator.userAgent
+        }
+    };
+
+    if (window.document.referrer) {
+        http.headers.Referer = window.document.referrer;
+    }
+
+    return http;
+}
+
+function send(data) {
+    if (!isSetup()) return;
+
+    data = arrayMerge({
+        project: globalProject,
+        logger: globalOptions.logger,
+        site: globalOptions.site,
+        platform: 'javascript',
+        'sentry.interfaces.Http': getHttpData()
+    }, data );
+
+    if (globalUser) data['sentry.interfaces.User'] = globalUser;
+
+    if (isFunction(globalOptions.dataCallback)) {
+        data = globalOptions.dataCallback(data);
+    }
+
+    makeRequest(data);
+}
+
+function makeRequest(data) {
+    new Image().src = globalServer + getAuthQueryString() + '&sentry_data=' + encodeURIComponent(JSON.stringify(data));
+}
+
+function isSetup() {
+    if (!hasJSON) return false;  // needs JSON support
+    if (!globalServer) {
+        console.error("Error: Raven has not been configured.");
+        return false;
+    }
+    return true;
+}
+
+window.Raven = Raven;
+
+})();

static/scripts/packed/libs/raven.js

@@ -0,0 +1 @@
+(function(){var x=window.Raven,u=!h(window.JSON),e,o,p,q,f={logger:"javascript",ignoreErrors:[],ignoreUrls:[]};var z=TraceKit.noConflict();z.remoteFetching=false;var n={VERSION:"@VERSION",noConflict:function(){window.Raven=x;return n},config:function(C,B){var D=v(C),A=D.path.lastIndexOf("/"),E=D.path.substr(1,A);if(B){j(B,function(F,G){f[F]=G})}f.ignoreErrors.push("Script error.");p=D.user;q=~~D.path.substr(A+1);e=D.protocol+"://"+D.host+(D.port?":"+D.port:"")+"/"+E+"api/"+q+"/store/";if(f.fetchContext){z.remoteFetching=true}return n},install:function(){if(!s()){return}z.report.subscribe(m);return n},context:function(B,C,A){if(c(B)){A=C;C=B;B=undefined}n.wrap(B,C).apply(this,A)},wrap:function(A,B){if(c(A)){B=A;A=undefined}return function(){try{B.apply(this,arguments)}catch(C){n.captureException(C,A)}}},uninstall:function(){z.report.unsubscribe(m);return n},captureException:function(B,A){try{z.report(B,A)}catch(C){if(B!==C){throw C}}return n},captureMessage:function(B,A){l(r({message:B},A));return n},setUser:function(A){o=A;return n}};var b="source protocol authority userInfo user password host port relative path directory file query anchor".split(" "),a=/^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/;function v(D){var A=a.exec(D),C={},B=14;while(B--){C[b[B]]=A[B]||""}return C}function h(A){return typeof A==="undefined"}function c(A){return typeof A==="function"}function j(C,D){var B,A;if(C.length===undefined){for(B in C){if(C.hasOwnProperty(B)){D.call(null,B,C[B])}}}else{for(B=0,A=C.length;B<A;B++){D.call(null,B,C[B])}}}var g;function d(){if(g){return g}var A=["sentry_version=2.0","sentry_client=raven-js/"+n.VERSION];if(p){A.push("sentry_key="+p)}g="?"+A.join("&");return g}function m(D,B){var F=[],C=0,A=D.stack&&D.stack.length||0,E;for(;C<A;C++){E=k(D.stack[C]);if(E){F.push(E)}}i(D.name,D.message,D.url,D.lineno,F,B)}function k(E){if(!E.url){return}var D={filename:E.url,lineno:E.line,colno:E.column,"function":E.func||"?"},B=w(E);if(B){var A=3,C=["pre_context","context_line","post_context"];while(A--){D[C[A]]=B[A]}}D.in_app=!/(Raven|TraceKit)\./.test(D["function"]);return D}function w(E){if(!E.context||!f.fetchContext){return}var C=E.context,A=~~(C.length/2),B=C.length,D=false;while(B--){if(C[B].length>300){D=true;break}}if(D){if(h(E.column)){return}return[[],C[A].substr(E.column,50),[]]}return[C.slice(0,A),C[A],C.slice(A+1)]}function i(D,H,F,A,C,I){var G,E,B;B=f.ignoreErrors.length;while(B--){if(H===f.ignoreErrors[B]){return}}if(C&&C.length){G={frames:C};F=F||C[0].filename}else{if(F){G={frames:[{filename:F,lineno:A}]}}}B=f.ignoreUrls.length;while(B--){if(f.ignoreUrls[B].test(F)){return}}E=A?H+" at "+A:H;l(r({"sentry.interfaces.Exception":{type:D,value:H},"sentry.interfaces.Stacktrace":G,culprit:F,message:E},I))}function r(B,A){if(!A){return B}j(A,function(C,D){B[C]=D});return B}function y(){var A={url:window.location.href,headers:{"User-Agent":navigator.userAgent}};if(window.document.referrer){A.headers.Referer=window.document.referrer}return A}function l(A){if(!s()){return}A=r({project:q,logger:f.logger,site:f.site,platform:"javascript","sentry.interfaces.Http":y()},A);if(o){A["sentry.interfaces.User"]=o}if(c(f.dataCallback)){A=f.dataCallback(A)}t(A)}function t(A){new Image().src=e+d()+"&sentry_data="+encodeURIComponent(JSON.stringify(A))}function s(){if(!u){return false}if(!e){console.error("Error: Raven has not been configured.");return false}return true}window.Raven=n})();

templates/base.mako

@@ -24,6 +24,17 @@
 ## Default javascripts
 <%def name="javascripts()">
     
+    ## Send errors to Sntry server if configured
+    %if app.config.sentry_dsn:
+        ${h.js( "libs/tracekit", "libs/raven" )}
+        <script>
+            Raven.config('${app.config.sentry_dsn_public}').install();
+            %if trans.user:
+                Raven.setUser( { email: "${trans.user.email}" } );
+            %endif
+        </script>
+    %endif
+
     ${h.js(
         "libs/jquery/jquery",
         "libs/jquery/select2",
@@ -54,7 +65,7 @@
             error   : function(){},
             assert  : function(){}
         };
-      
+
         // Set up needed paths.
         var galaxy_paths = new GalaxyPaths({
             root_path: '${h.url_for( "/" )}',

templates/base/base_panels.mako

@@ -45,9 +45,18 @@
 
 ## Default javascripts
 <%def name="javascripts()">
-    <!--[if lt IE 7]>
-    ${h.js( 'libs/IE/IE7', 'libs/IE/ie7-recalc' )}
-    <![endif]-->
+
+    ## Send errors to Sntry server if configured
+    %if app.config.sentry_dsn:
+        ${h.js( "libs/tracekit", "libs/raven" )}
+        <script>
+            Raven.config('${app.config.sentry_dsn_public}').install();
+            %if trans.user:
+                Raven.setUser( { email: "${trans.user.email}" } );
+            %endif
+        </script>
+    %endif
+
     ${h.js(
         'libs/jquery/jquery',
         'libs/json2',

templates/root/tool_menu.mako

@@ -74,7 +74,7 @@
                 });
                 */
                 
-                $( '.tooltip' ).tooltip();
+                $( '.tooltip' ).tooltip().foasdasdasdoo();
                 
                 // TODO: is this necessary?
                 $( "a[minsizehint]" ).click( function() {