New reports app server and 1st report showing disk usage for dataset file system - still need to mess with styles a bit.

Also fixed update_dataset_size script and delete deprecatd random_intervals code file.
2026-09-21 13:50:20 +08:00 · 2008-01-11 21:24:21 +00:00
parent caa5c3b955
commit 01378424b4
31 changed files with 1478 additions and 224 deletions
@@ -0,0 +1,3 @@
+"""Galaxy Reports root package -- this is a namespace package."""
+
+__import__( "pkg_resources" ).declare_namespace( __name__ )
@@ -0,0 +1,3 @@
+"""The Galaxy Reports application."""
+
+from galaxy.web.framework import expose, url_for
@@ -0,0 +1,32 @@
+import sys, os, atexit
+import galaxy.model
+import config
+
+class UniverseApplication( object ):
+    """Encapsulates the state of a Universe application"""
+    def __init__( self, **kwargs ):
+        print >> sys.stderr, "python path is: " + ", ".join( sys.path )
+        # Read config file and check for errors
+        self.config = config.Configuration( **kwargs )
+        self.config.check()
+        config.configure_logging( self.config )
+        # Determine the database url
+        if self.config.database_connection:
+            db_url = self.config.database_connection
+        else:
+            db_url = "sqlite://%s?isolation_level=IMMEDIATE" % self.config.database
+        # Setup the database engine and ORM
+        self.model = galaxy.model.mapping.init( self.config.file_path,
+                                                db_url,
+                                                self.config.database_engine_options,
+                                                create_tables = True )
+        self.heartbeat = None
+        # Start the heartbeat process if configured and available
+        if self.config.use_heartbeat:
+            from galaxy.util import heartbeat
+            if heartbeat.Heartbeat:
+                self.heartbeat = heartbeat.Heartbeat()
+                self.heartbeat.start()
+    def shutdown( self ):
+        if self.heartbeat:
+            self.heartbeat.shutdown()
@@ -0,0 +1,27 @@
+"""Contains functionality needed in every webapp interface"""
+
+import os, time, logging
+
+# Pieces of Galaxy to make global in every controller
+#from galaxy import config, tools, web, model, util
+from galaxy import web
+
+from Cheetah.Template import Template
+
+log = logging.getLogger( __name__ )
+
+class BaseController( object ):
+    """Base class for Galaxy webapp application controllers."""
+    
+    beta = False
+
+    def __init__( self, app ):
+        """Initialize an interface for application 'app'"""
+        self.app = app
+
+    #def get_toolbox(self):
+    #    """Returns the application toolbox"""
+    #    return self.app.toolbox
+        
+#Root = BaseController
+"""Deprecated: `BaseController` used to be available under the name `Root`"""
@@ -0,0 +1,190 @@
+"""
+Provides factory methods to assemble the Galaxy web application
+"""
+
+import logging, atexit
+import os, os.path, sys
+
+from inspect import isclass
+
+from paste.request import parse_formvars
+from paste.util import import_string
+from paste import httpexceptions
+from paste.deploy.converters import asbool
+import flup.middleware.session as flup_session
+import pkg_resources
+
+log = logging.getLogger( __name__ )
+
+import config
+import galaxy.model
+import galaxy.model.mapping
+import galaxy.web.framework
+
+def add_controllers( webapp, app ):
+    """
+    Search for controllers in the 'galaxy.webapps.controllers' module and add 
+    them to the webapp.
+    """
+    from galaxy.webapps.reports.base.controller import BaseController
+    import galaxy.webapps.reports.controllers
+    controller_dir = galaxy.webapps.reports.controllers.__path__[0]
+    for fname in os.listdir( controller_dir ):
+        if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ):
+            name = fname[:-3]
+            module_name = "galaxy.webapps.reports.controllers." + name
+            module = __import__( module_name )
+            for comp in module_name.split( "." )[1:]:
+                module = getattr( module, comp )
+            # Look for a controller inside the modules
+            for key in dir( module ):
+                T = getattr( module, key )
+                if isclass( T ) and T is not BaseController and issubclass( T, BaseController ):
+                    #if app.config.enable_beta_features or not ( T.beta ):
+                    webapp.add_controller( name, T( app ) )
+
+def app_factory( global_conf, **kwargs ):
+    """Return a wsgi application serving the root object"""
+    # Create the Galaxy application unless passed in
+    if 'app' in kwargs:
+        app = kwargs.pop( 'app' )
+    else:
+        from galaxy.webapps.reports.app import UniverseApplication
+        app = UniverseApplication( global_conf = global_conf, **kwargs )
+    atexit.register( app.shutdown )
+    # Create the universe WSGI application
+    webapp = galaxy.web.framework.WebApplication( app )
+    add_controllers( webapp, app )
+    # These two routes handle our simple needs at the moment
+    webapp.add_route( '/:controller/:action', controller="root", action='index' )
+    webapp.add_route( '/:action', controller='root', action='index' )
+    webapp.finalize_config()
+    # Wrap the webapp in some useful middleware
+    if kwargs.get( 'middleware', True ):
+        webapp = wrap_in_middleware( webapp, global_conf, **kwargs )
+    if kwargs.get( 'static_enabled', True ):
+        webapp = wrap_in_static( webapp, global_conf, **kwargs )
+    # Close any pooled database connections before forking
+    try:
+        galaxy.model.mapping.metadata.engine.connection_provider._pool.dispose()
+    except:
+        pass
+    # Return
+    return webapp
+    
+def wrap_in_middleware( app, global_conf, **local_conf ):
+    """Based on the configuration wrap `app` in a set of common and useful middleware."""
+    # Merge the global and local configurations
+    conf = global_conf.copy()
+    conf.update(local_conf)
+    debug = asbool( conf.get( 'debug', False ) )
+    # First put into place httpexceptions, which must be most closely
+    # wrapped around the application (it can interact poorly with
+    # other middleware):
+    app = httpexceptions.make_middleware( app, conf )
+    log.debug( "Enabling 'httpexceptions' middleware" )
+    # The recursive middleware allows for including requests in other 
+    # requests or forwarding of requests, all on the server side.
+    if asbool(conf.get('use_recursive', True)):
+        from paste import recursive
+        app = recursive.RecursiveMiddleware( app, conf )
+        log.debug( "Enabling 'recursive' middleware" )
+    ## # Session middleware puts a session factory into the environment 
+    ## if asbool( conf.get( 'use_session', True ) ):
+    ##     store = flup_session.MemorySessionStore()
+    ##     app = flup_session.SessionMiddleware( store, app )
+    ##     log.debug( "Enabling 'flup session' middleware" )
+    # Beaker session middleware
+    if asbool( conf.get( 'use_beaker_session', False ) ):
+        pkg_resources.require( "Beaker" )
+        import beaker.session
+        app = beaker.session.SessionMiddleware( app, conf )
+        log.debug( "Enabling 'beaker session' middleware" )
+    # Various debug middleware that can only be turned on if the debug
+    # flag is set, either because they are insecure or greatly hurt
+    # performance
+    if debug:
+        # Middleware to check for WSGI compliance
+        if asbool( conf.get( 'use_lint', True ) ):
+            from paste import lint
+            app = lint.make_middleware( app, conf )
+            log.debug( "Enabling 'lint' middleware" )
+        # Middleware to run the python profiler on each request
+        if asbool( conf.get( 'use_profile', False ) ):
+            import profile
+            app = profile.ProfileMiddleware( app, conf )
+            log.debug( "Enabling 'profile' middleware" )
+        # Middleware that intercepts print statements and shows them on the
+        # returned page
+        if asbool( conf.get( 'use_printdebug', True ) ):
+            from paste.debug import prints
+            app = prints.PrintDebugMiddleware( app, conf )
+            log.debug( "Enabling 'print debug' middleware" )
+    if debug and asbool( conf.get( 'use_interactive', False ) ):
+        # Interactive exception debugging, scary dangerous if publicly
+        # accessible, if not enabled we'll use the regular error printing
+        # middleware.
+        pkg_resources.require( "WebError" )
+        from weberror import evalexception
+        app = evalexception.EvalException( app, conf,
+                                           templating_formatters=build_template_error_formatters() )
+        log.debug( "Enabling 'eval exceptions' middleware" )
+    else:
+        # Not in interactive debug mode, just use the regular error middleware
+        from paste.exceptions import errormiddleware
+        app = errormiddleware.ErrorMiddleware( app, conf )
+        log.debug( "Enabling 'error' middleware" )
+    # Transaction logging (apache access.log style)
+    if asbool( conf.get( 'use_translogger', True ) ):
+        from paste.translogger import TransLogger
+        app = TransLogger( app )
+        log.debug( "Enabling 'trans logger' middleware" )
+    # Config middleware just stores the paste config along with the request,
+    # not sure we need this but useful
+    from paste.deploy.config import ConfigMiddleware
+    app = ConfigMiddleware( app, conf )
+    log.debug( "Enabling 'config' middleware" )
+    # X-Forwarded-Host handling
+    from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware
+    app = XForwardedHostMiddleware( app )
+    log.debug( "Enabling 'x-forwarded-host' middleware" )
+    return app
+    
+def wrap_in_static( app, global_conf, **local_conf ):
+    from paste.urlmap import URLMap
+    from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static
+    urlmap = URLMap()
+    # Merge the global and local configurations
+    conf = global_conf.copy()
+    conf.update(local_conf)
+    # Get cache time in seconds
+    cache_time = conf.get( "static_cache_time", None )
+    if cache_time is not None:
+        cache_time = int( cache_time )
+    # Send to dynamic app by default
+    urlmap["/"] = app
+    # Define static mappings from config
+    urlmap["/static"] = Static( conf.get( "static_dir" ), cache_time )
+    urlmap["/images"] = Static( conf.get( "static_images_dir" ), cache_time )
+    urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir" ), cache_time )
+    urlmap["/static/style"] = Static( conf.get( "static_style_dir" ), cache_time )
+    urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir" ), cache_time )
+    # URL mapper becomes the root webapp
+    return urlmap
+    
+def build_template_error_formatters():
+    """
+    Build a list of template error formatters for WebError. When an error
+    occurs, WebError pass the exception to each function in this list until
+    one returns a value, which will be displayed on the error page.
+    """
+    formatters = []
+    # Formatter for mako
+    import mako.exceptions
+    def mako_html_data( exc_value ):
+        if isinstance( exc_value, ( mako.exceptions.CompileException, mako.exceptions.SyntaxException ) ):
+            return mako.exceptions.html_error_template().render( full=False, css=False )
+        if isinstance( exc_value, AttributeError ) and exc_value.args[0].startswith( "'Undefined' object has no attribute" ):
+            return mako.exceptions.html_error_template().render( full=False, css=False )
+    formatters.append( mako_html_data )
+    return formatters
@@ -0,0 +1,144 @@
+"""
+Universe configuration builder.
+"""
+
+import sys, os
+import logging, logging.config
+from optparse import OptionParser
+import ConfigParser
+from galaxy.util import string_as_bool
+
+log = logging.getLogger( __name__ )
+
+def resolve_path( path, root ):
+    """If 'path' is relative make absolute by prepending 'root'"""
+    if not( os.path.isabs( path ) ):
+        path = os.path.join( root, path )
+    return path
+      
+class ConfigurationError( Exception ):
+    pass
+
+class Configuration( object ):
+    def __init__( self, **kwargs ):
+        self.config_dict = kwargs
+        self.root = kwargs.get( 'root_dir', '.' )
+        #self.enable_beta_features = kwargs.get( "enable_beta_features", False )
+        # Database related configuration
+        self.database = resolve_path( kwargs.get( "database_file", "database/universe.d" ), self.root )
+        self.database_connection =  kwargs.get( "database_connection", False )
+        self.database_engine_options = get_database_engine_options( kwargs )                        
+        # Where dataset files are stored
+        self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root )
+        self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root )
+        #self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root )
+        self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root )
+        #self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root )
+        #self.tool_secret = kwargs.get( "tool_secret", "" )
+        self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root )
+        self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root )
+        #self.job_queue_workers = int( kwargs.get( "job_queue_workers", "10" ) )
+        #self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO")
+        #self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") )
+        #self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root )
+        self.admin_pass = kwargs.get('admin_pass',"galaxy")
+        self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail")
+        self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join@bx.psu.edu")
+        self.error_email_to = kwargs.get( 'error_email_to', None )
+        #self.smtp_server = kwargs.get( 'smtp_server', None )
+        #self.use_pbs = kwargs.get('use_pbs', False )
+        #self.pbs_server = kwargs.get('pbs_server', "" )
+        #self.pbs_instance_path = kwargs.get('pbs_instance_path', os.getcwd() )
+        #self.pbs_application_server = kwargs.get('pbs_application_server', "" )
+        #self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" )
+        #self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" )
+        self.use_heartbeat = kwargs.get( 'use_heartbeat', False )
+        #self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",")
+        #self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "volvox,wormbase,flybase" ).lower().split(",")
+        self.brand = kwargs.get( 'brand', None )
+        self.wiki_url = kwargs.get( 'wiki_url', None )
+        self.bugs_email = kwargs.get( 'bugs_email', None )
+        self.blog_url = kwargs.get( 'blog_url', None )
+        self.screencasts_url = kwargs.get( 'screencasts_url', None )
+        #Parse global_conf
+        global_conf = kwargs.get( 'global_conf', None )
+        global_conf_parser = ConfigParser.ConfigParser()
+        if global_conf and "__file__" in global_conf:
+            global_conf_parser.read(global_conf['__file__'])
+        #Store datatypes config
+        #try:
+        #    self.datatypes = global_conf_parser.items("galaxy:datatypes")
+        #except ConfigParser.NoSectionError:
+        #    self.datatypes = []
+        #Store sniff order config
+        #try:
+        #    self.sniff_order = global_conf_parser.items("galaxy:sniff_order")
+        #except ConfigParser.NoSectionError:
+        #    self.sniff_order = []
+        #self.datatype_converters_config = kwargs.get( 'datatype_converters_config_file', "datatype_converters_conf.xml" )
+        #self.datatype_converters_path = kwargs.get( 'datatype_converters_path', os.path.join(self.root,"lib/galaxy/datatypes/converters") )
+    def get( self, key, default ):
+        return self.config_dict.get( key, default )
+    def check( self ):
+        # Check that required directories exist
+        for path in self.root, self.file_path, self.template_path:
+            if not os.path.isdir( path ):
+                raise ConfigurationError("Directory does not exist: %s" % path )
+
+def get_database_engine_options( kwargs ):
+    """
+    Allow options for the SQLAlchemy database engine to be passed by using
+    the prefix "database_engine_option_".
+    """
+    conversions =  {
+        'convert_unicode': string_as_bool,
+        'pool_timeout': int,
+        'echo': string_as_bool,
+        'echo_pool': string_as_bool,
+        'pool_recycle': int,
+        'pool_size': int,
+        'max_overflow': int,
+        'pool_threadlocal': string_as_bool
+    }
+    prefix = "database_engine_option_"
+    prefix_len = len( prefix )
+    rval = {}
+    for key, value in kwargs.iteritems():
+        if key.startswith( prefix ):
+            key = key[prefix_len:]
+            if key in conversions:
+                value = conversions[key](value)
+            rval[ key  ] = value
+    return rval
+
+def configure_logging( config ):
+    """
+    Allow some basic logging configuration to be read from the cherrpy
+    config.
+    """
+    format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" )
+    level = logging._levelNames[ config.get( "log_level", "DEBUG" ) ]
+    destination = config.get( "log_destination", "stdout" )
+    log.info( "Logging at '%s' level to '%s'" % ( level, destination ) )
+    # Get root logger
+    root = logging.getLogger()
+    # Set level
+    root.setLevel( level )
+    # Turn down paste httpserver logging
+    if level <= logging.DEBUG:
+        logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN )
+    # Remove old handlers
+    for h in root.handlers[:]: 
+        root.removeHandler(h)
+    # Create handler
+    if destination == "stdout":
+        handler = logging.StreamHandler( sys.stdout )
+    else:
+        handler = logging.FileHandler( destination )
+    # Create formatter
+    formatter = logging.Formatter( format )    
+    # Hook everything up
+    handler.setFormatter( formatter )
+    root.addHandler( handler )
+    
+    
@@ -0,0 +1 @@
+"""Galaxy reports controllers."""
@@ -0,0 +1,32 @@
+import sys, sets, string, shutil
+import re, socket
+
+from galaxy import web
+
+from cgi import escape, FieldStorage
+import urllib
+
+import operator, os
+from galaxy.webapps.reports.base.controller import *
+
+import logging, sets, time
+log = logging.getLogger( __name__ )
+
+class Report( BaseController ):
+    @web.expose
+    def index( self, trans, **kwd ):
+        return trans.fill_template( 'index.tmpl' )
+
+    @web.expose
+    def masthead( self, trans ):
+        brand = trans.app.config.get( "brand", None )
+        wiki_url = trans.app.config.get( "wiki_url", None )
+        bugs_email = trans.app.config.get( "bugs_email", None )
+        blog_url = trans.app.config.get( "blog_url", None )
+        screencasts_url = trans.app.config.get( "screencasts_url", None )
+        return trans.fill_template( "masthead.tmpl", brand=brand, wiki_url=wiki_url, blog_url=blog_url,bugs_email=bugs_email, screencasts_url=screencasts_url )
+
+    @web.expose
+    def main_frame( self, trans ):
+        return trans.fill_template( "main_frame.tmpl" )
+
@@ -0,0 +1,48 @@
+import sys, sets, string, shutil
+import re, socket
+
+from galaxy import web
+
+from cgi import escape, FieldStorage
+import urllib
+
+import operator, os
+from galaxy.webapps.reports.base.controller import *
+
+import logging, sets, time
+log = logging.getLogger( __name__ )
+
+class System( BaseController ):
+
+    def get_disk_usage( self, file_path ):
+        df_cmd = 'df -h ' + file_path
+        df_file = os.popen( df_cmd )
+
+        while True:
+            df_list = df_file.readline()
+            if not df_list:
+                break #EOF
+            dflistlower = df_list.lower()
+            if 'filesystem' in dflistlower or 'proc' in dflistlower:
+                continue
+            file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount = df_list.split()
+            break
+
+        df_file.close()
+        return ( file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount  )
+
+    @web.expose
+    def disk_usage( self, trans, **kwd ):
+        file_path = trans.app.config.file_path
+        disk_usage = self.get_disk_usage( file_path )
+        min_file_size = 2**40 # 1 GB
+        file_size_str = '1 GB'
+        datasets = []
+        dt = trans.model.Dataset.table
+
+        for row in dt.select( dt.c.file_size>min_file_size ).execute():
+            datasets.append( ( row.id, str( row.create_time )[0:10], row.history_id, row.deleted, row.file_size ) )
+
+        datasets = sorted( datasets, key=operator.itemgetter(4), reverse=True )
+        return trans.fill_template('disk_usage.tmpl', file_path=file_path, disk_usage=disk_usage, datasets=datasets, file_size_str=file_size_str)
+
@@ -0,0 +1,277 @@
+@import url( "reset.css" );
+
+body 
+{
+    font: 75% verdana, "Bitstream Vera Sans", geneva, arial, helvetica, helve, sans-serif; 
+    background: $base_bg_bottom;
+    color: $base_text;
+    background-image: url(base_bg.png);
+    background-repeat: repeat-x;
+    background-position: top;
+    margin: 10px;
+}
+
+a:link, a:visited, a:active
+{
+    color: $link_text;
+}
+
+h1, h2, h3, h4 
+{
+    color: $header_text;
+    /*text-shadow: #bbb 2px 2px 1px;*/
+}
+
+hr
+{
+    border: none;
+    border-bottom: dotted $base_text 1px;
+}
+
+div.report
+{
+  border: solid $report_border 1px;
+}
+
+div.reportTitle
+{ 
+  font-weight: bold;
+  padding: 5px;
+  padding-left: 10px;
+  padding-right: 10px;
+  background: $report_title_bg_bottom;
+  background-image: url(report_title_bg.png);
+  background-repeat: repeat-x;
+  background-position: top;
+  border-bottom: solid $report_border 1px;
+}
+
+div.reportParamHelp
+{
+  color: #666;
+}
+
+div.reportParamHelp a
+{
+  color: #666;
+}
+
+div.reportBody
+{ 
+  background: $report_body_bg_bottom;
+  background-image: url(report_body_bg.png);
+  background-repeat: repeat-x;
+  background-position: top;
+  padding: 5px;
+  padding-left: 10px;
+  padding-right: 10px;
+}
+
+div.reportBody div.reportTitle
+{
+    background: transparent;
+	border: none;
+	font-weight: bold;
+	border-bottom: solid $report_border 1px;
+	margin-bottom: 5px;
+}
+
+div.reportDisabled div.reportTitle {
+    background: ${layout_bg};
+    border-color: ${layout_border};
+}
+
+div.reportDisabled {
+    border-color: ${layout_border};
+}
+
+div.reportHelp
+{ 
+}
+
+div.reportHelpBody
+{ 
+	width: 100%;
+	overflow: auto;
+}
+
+div.titleRow {
+    font-weight: bold;
+    border-bottom: dotted gray 1px;
+    margin-bottom: 0.5em;
+    padding-bottom: 0.25em;
+}
+
+div.report-row 
+{
+    margin-bottom: 1em;
+}
+
+div.report-title-row
+{
+    margin-bottom: 0.5em;
+}
+
+div.report-row-error
+{
+    background: $error_message_bg;
+    padding: 5px;
+}
+
+div.report-row label
+{
+    font-weight: bold;
+    display: block;
+    margin-bottom: .2em;
+}
+
+/* Messages */
+
+.errormessage, .warningmessage, .donemessage, .infomessage, .welcomeBlue, .welcomeRed
+{
+    padding: 10px;
+    padding-left: 52px;
+    min-height: 32px;
+    border: 1px solid $error_message_border;
+    background-color: $error_message_bg;
+    background-image: url(error_message_icon.png);
+    background-repeat: no-repeat;
+    background-position: 10px 10px;
+}
+
+.warningmessage 
+{
+    background-image: url(warn_message_icon.png);
+    border-color: $warn_message_border;
+    background-color: $warn_message_bg;
+}
+
+.donemessage 
+{
+    background-image: url(done_message_icon.png);
+    border-color: $done_message_border;
+    background-color: $done_message_bg;
+}
+
+.infomessage 
+{
+    background-image: url(info_message_icon.png);
+    border-color: $info_message_border;
+    background-color: $info_message_bg;
+}
+
+.welcomeBlue 
+{
+    padding-left: 10px;
+    border-color: $info_message_border;
+    background-color: $info_message_bg;
+    background-image: none;
+}
+
+.welcomeRed 
+{
+    padding-left: 10px;
+    border-color: $error_message_border;
+    background-color: $error_message_bg;
+    background-image: none;
+}
+
+.errormessagesmall, .warningmessagesmall, .donemessagesmall, .infomessagesmall 
+{
+    padding: 5px;
+    padding-left: 25px;
+    min-height: 25px;
+    border: 1px solid $error_message_border;
+    background-color: $error_message_bg;
+    background-image: url(error_small.png);
+    background-repeat: no-repeat;
+    background-position: 5px 5px;
+}
+
+.warningmessagesmall
+{
+    background-image: url(warn_small.png);
+    border-color: $warn_message_border;
+    background-color: $warn_message_bg;
+}
+
+.donemessagesmall
+{
+    background-image: url(ok_small.png);
+    border-color: $done_message_border;
+    background-color: $done_message_bg;
+}
+
+.infomessagesmall
+{
+    background-image: url(info_small.png);
+    border-color: $info_message_border;
+    background-color: $info_message_bg;
+}
+
+.errormark, .warningmark, .donemark, .infomark, .ok_bgr, .err_bgr
+{
+    padding-left: 20px;
+    min-height: 15px;
+    background: url(error_small.png) no-repeat;
+}
+
+.warningmark 
+{
+    background-image: url(warn_small.png);
+}
+
+.donemark 
+{
+    background-image: url(ok_small.png);
+}
+
+.infomark, .ok_bgr
+{
+  background-image: url(info_small.png);
+}
+
+table.colored
+{
+  border-top: solid $table_border 1px;
+  border-bottom: solid $table_border 1px;
+}
+
+table.colored td
+{
+    padding: 5px;
+}
+
+table.colored tr.header
+{ 
+  background: $table_header_bg;
+  background-image: url(report_title_bg.png);
+  background-repeat: repeat-x;
+  background-position: top;
+  border-bottom: solid $table_border 1px;
+  font-weight: bold;
+}
+
+table.colored tr
+{ 
+  background: $table_row_bg;
+}
+
+table.colored tr.odd_row
+{ 
+  background: $odd_row_bg;
+}
+
+div.debug
+{
+    margin: 10px;
+    padding: 5px;
+    background: #FFFF99;
+    border: solid #FFFF33 1px;
+    color: black;
+}
+    
+#footer {
+  display: none;
+}
+
@@ -0,0 +1,277 @@
+@import url( "reset.css" );
+
+body 
+{
+    font: 75% verdana, "Bitstream Vera Sans", geneva, arial, helvetica, helve, sans-serif; 
+    background: #FFFFFF;
+    color: #303030;
+    background-image: url(base_bg.png);
+    background-repeat: repeat-x;
+    background-position: top;
+    margin: 10px;
+}
+
+a:link, a:visited, a:active
+{
+    color: #303030;
+}
+
+h1, h2, h3, h4 
+{
+    color: #023858;
+    /*text-shadow: #bbb 2px 2px 1px;*/
+}
+
+hr
+{
+    border: none;
+    border-bottom: dotted #303030 1px;
+}
+
+div.report
+{
+  border: solid #d8b365 1px;
+}
+
+div.reportTitle
+{ 
+  font-weight: bold;
+  padding: 5px;
+  padding-left: 10px;
+  padding-right: 10px;
+  background: #d2c099;
+  background-image: url(report_title_bg.png);
+  background-repeat: repeat-x;
+  background-position: top;
+  border-bottom: solid #d8b365 1px;
+}
+
+div.reportParamHelp
+{
+  color: #666;
+}
+
+div.reportParamHelp a
+{
+  color: #666;
+}
+
+div.reportBody
+{ 
+  background: #FFFFFF;
+  background-image: url(report_body_bg.png);
+  background-repeat: repeat-x;
+  background-position: top;
+  padding: 5px;
+  padding-left: 10px;
+  padding-right: 10px;
+}
+
+div.reportBody div.reportTitle
+{
+    background: transparent;
+	border: none;
+	font-weight: bold;
+	border-bottom: solid #d8b365 1px;
+	margin-bottom: 5px;
+}
+
+div.reportDisabled div.reportTitle {
+    background: #eee;
+    border-color: #999;
+}
+
+div.reportDisabled {
+    border-color: #999;
+}
+
+div.reportHelp
+{ 
+}
+
+div.reportHelpBody
+{ 
+	width: 100%;
+	overflow: auto;
+}
+
+div.titleRow {
+    font-weight: bold;
+    border-bottom: dotted gray 1px;
+    margin-bottom: 0.5em;
+    padding-bottom: 0.25em;
+}
+
+div.report-row 
+{
+    margin-bottom: 1em;
+}
+
+div.report-title-row
+{
+    margin-bottom: 0.5em;
+}
+
+div.report-row-error
+{
+    background: #FFCCCC;
+    padding: 5px;
+}
+
+div.report-row label
+{
+    font-weight: bold;
+    display: block;
+    margin-bottom: .2em;
+}
+
+/* Messages */
+
+.errormessage, .warningmessage, .donemessage, .infomessage, .welcomeBlue, .welcomeRed
+{
+    padding: 10px;
+    padding-left: 52px;
+    min-height: 32px;
+    border: 1px solid #AA6666;
+    background-color: #FFCCCC;
+    background-image: url(error_message_icon.png);
+    background-repeat: no-repeat;
+    background-position: 10px 10px;
+}
+
+.warningmessage 
+{
+    background-image: url(warn_message_icon.png);
+    border-color: #AAAA66;
+    background-color: #FFFFCC;
+}
+
+.donemessage 
+{
+    background-image: url(done_message_icon.png);
+    border-color: #66AA66;
+    background-color: #CCFFCC;
+}
+
+.infomessage 
+{
+    background-image: url(info_message_icon.png);
+    border-color: #6666AA;
+    background-color: #CCCCFF;
+}
+
+.welcomeBlue 
+{
+    padding-left: 10px;
+    border-color: #6666AA;
+    background-color: #CCCCFF;
+    background-image: none;
+}
+
+.welcomeRed 
+{
+    padding-left: 10px;
+    border-color: #AA6666;
+    background-color: #FFCCCC;
+    background-image: none;
+}
+
+.errormessagesmall, .warningmessagesmall, .donemessagesmall, .infomessagesmall 
+{
+    padding: 5px;
+    padding-left: 25px;
+    min-height: 25px;
+    border: 1px solid #AA6666;
+    background-color: #FFCCCC;
+    background-image: url(error_small.png);
+    background-repeat: no-repeat;
+    background-position: 5px 5px;
+}
+
+.warningmessagesmall
+{
+    background-image: url(warn_small.png);
+    border-color: #AAAA66;
+    background-color: #FFFFCC;
+}
+
+.donemessagesmall
+{
+    background-image: url(ok_small.png);
+    border-color: #66AA66;
+    background-color: #CCFFCC;
+}
+
+.infomessagesmall
+{
+    background-image: url(info_small.png);
+    border-color: #6666AA;
+    background-color: #CCCCFF;
+}
+
+.errormark, .warningmark, .donemark, .infomark, .ok_bgr, .err_bgr
+{
+    padding-left: 20px;
+    min-height: 15px;
+    background: url(error_small.png) no-repeat;
+}
+
+.warningmark 
+{
+    background-image: url(warn_small.png);
+}
+
+.donemark 
+{
+    background-image: url(ok_small.png);
+}
+
+.infomark, .ok_bgr
+{
+  background-image: url(info_small.png);
+}
+
+table.colored
+{
+  border-top: solid #d8b365 1px;
+  border-bottom: solid #d8b365 1px;
+}
+
+table.colored td
+{
+    padding: 5px;
+}
+
+table.colored tr.header
+{ 
+  background: #ebd9b2;
+  background-image: url(report_title_bg.png);
+  background-repeat: repeat-x;
+  background-position: top;
+  border-bottom: solid #d8b365 1px;
+  font-weight: bold;
+}
+
+table.colored tr
+{ 
+  background: white;
+}
+
+table.colored tr.odd_row
+{ 
+  background: #FFFF99;
+}
+
+div.debug
+{
+    margin: 10px;
+    padding: 5px;
+    background: #FFFF99;
+    border: solid #FFFF33 1px;
+    color: black;
+}
+    
+#footer {
+  display: none;
+}
+
@@ -0,0 +1,26 @@
+body
+{
+    background: #2C3143 url(masthead_bg.png) bottom;
+    color: #eeeeee;
+    padding: 0;
+    border: 0;
+    margin: 3px;
+    margin-right: 5px;
+    margin-left: 5px;
+}
+
+div.pageTitle
+{
+    font-size: 175%;
+    font-weight: bold;
+}
+
+div.pageTitle a:link, div.pageTitle a:visited, div.pageTitle a:active, div.pageTitle a:hover
+{
+    text-decoration: none;
+}
+
+a:link, a:visited, a:active
+{
+    color: #eeeeee;
+}
@@ -0,0 +1,62 @@
+base_text=#303030
+base_bg_top=#FFFFFF
+base_bg_bottom=#FFFFFF
+link_text=#303030
+header_text=#023858
+menu_bg_top=#DADFEF
+menu_bg_over=#C1C9E5
+menu_bg_hatch=-
+#menu_bg_hatch=#AAAAFF
+# Forms
+report_title_bg_top=#ebd9b2
+report_title_bg_bottom=#d2c099
+report_title_bg_hatch=-
+report_border=#d8b365
+report_body_bg=#FFFFFF
+report_body_bg_top=#FFFFFF
+report_body_bg_bottom=#FFFFFF
+odd_row_bg=#FFFF99
+# Messages
+error_message_border=#AA6666
+error_message_bg=#FFCCCC
+warn_message_border=#AAAA66
+warn_message_bg=#FFFFCC
+done_message_border=#66AA66
+done_message_bg=#CCFFCC
+info_message_border=#6666AA
+info_message_bg=#CCCCFF
+# Tables
+table_header_bg=#ebd9b2
+table_row_bg=white
+table_border=#d8b365
+# Footers
+footer_bg=#023858
+footer_title_bg=#023858
+footer_title_hatch=#000000
+# History
+history_error_border=#AA6666
+history_error_bg=#FFCCCC
+history_running_border=#AAAA66
+history_running_bg=#FFFFCC
+history_ok_border=#66AA66
+history_ok_bg=#CCFFCC
+history_queued_border=#888888
+history_queued_bg=#EEEEEE
+peek_table_header=#023858
+# Masthead
+masthead_bg=#2C3143
+masthead_text=#eeeeee
+masthead_bg_hatch=-
+masthead_link=#eeeeee
+# ---- Layout -----------------------------------------------------------------
+# Overall background color (including space between panels)
+layout_bg=#eee
+# Line underneath masthead
+layout_masthead_border=#444
+# Borders around panels
+layout_border=#999
+# Hover color when mouse over drag bars (panel resize)
+layout_hover=#AAAAEE
+# Gradient for the panel title backgrounds
+panel_header_bg_top=#f5f5f5
+panel_header_bg_bottom=#cccccc
@@ -0,0 +1,72 @@
+#!/usr/bin/env python2.4
+
+"""
+usage: %prog width height bg_color hatch_color [color alpha stop_pos] +
+"""
+
+from __future__ import division
+
+import sys
+import cairo
+
+def parse_css_color( color ):
+    if color.startswith( '#' ):
+        color = color[1:]
+    if len( color ) == 3:
+        r = int( color[0], 16 )
+        g = int( color[1], 16 )
+        b = int( color[2], 16 )
+    elif len( color ) == 6:
+        r = int( color[0:2], 16 )
+        g = int( color[2:4], 16 )
+        b = int( color[4:6], 16 )
+    else:
+        raise Exception( "Color should be 3 hex numbers" )
+    return r/256, g/256, b/256
+        
+def gradient( width, height, args ):
+    pat = cairo.LinearGradient(0.0, 0.0, 0.0, height)
+    while len( args ) > 2:
+        col = parse_css_color( args[0] )
+        alpha = float( args[1])
+        pos = float( args[2] )
+        pat.add_color_stop_rgba( pos, col[0], col[1], col[2], alpha )
+        args = args[3:]
+    return pat
+    
+def hatch( width, height, color ):
+    im_surf = cairo.ImageSurface( cairo.FORMAT_ARGB32, width, width )
+    c = cairo.Context( im_surf )
+    c.set_source_rgb ( *color )
+    c.set_line_width( 0.75 )
+    for i in range( 0, 2*max(height,width), 3 ):
+        c.move_to ( 0-10, i+10 )
+        c.line_to ( width+10, i - width - 10 )
+    c.stroke()
+    pat = cairo.SurfacePattern( im_surf )
+    pat.set_extend (cairo.EXTEND_REPEAT)
+    return pat
+                
+width = int( sys.argv[1] )
+height = int( sys.argv[2] )
+    
+surface = cairo.ImageSurface( cairo.FORMAT_ARGB32, width, height )
+c = cairo.Context( surface )
+
+c.rectangle(0,0,width,height)
+c.set_source_rgb( *parse_css_color( sys.argv[3] ) )
+c.fill()
+
+if sys.argv[4] != "-":
+    c.rectangle (0, 0, width, height)
+    c.set_source( hatch( width, height, parse_css_color( sys.argv[4] ) ) )
+    c.fill()
+
+pat = cairo.LinearGradient(0.0, 0.0, 0.0, height)
+pat.add_color_stop_rgba( 0, 1, 1, 1, 0 )
+pat.add_color_stop_rgba( 1, 1, 1, 1, 1 )
+c.rectangle (0, 0, width, height)
+c.set_source( gradient( width, height, sys.argv[5:] ) )
+c.fill()
+
+surface.write_to_png( "/dev/stdout" )
@@ -0,0 +1,43 @@
+#!/usr/bin/env python2.4
+
+import pkg_resources; 
+pkg_resources.require( "Cheetah" )
+
+import sys
+from Cheetah.Template import Template
+import string
+from subprocess import Popen, PIPE
+import os.path
+
+def run( cmd ):
+    return Popen( cmd, stdout=PIPE).communicate()[0]
+
+templates = [ ( "masthead.css.tmpl", "masthead.css"),
+              ( "base.css.tmpl", "base.css" ) ]
+            
+images = [ 
+           ( "./gradient.py 9 1000 $base_bg_top - $base_bg_bottom 0 0 $base_bg_bottom 1 1", "base_bg.png" ),
+           ( "./gradient.py 9 50 $masthead_bg $masthead_bg_hatch", "masthead_bg.png" ),
+           ( "./gradient.py 9 30 $footer_title_bg $footer_title_hatch 000000 0 0.5 000000 1 1", "footer_title_bg.png" )
+           ]
+
+vars, out_dir = sys.argv[1:]
+
+context = dict()
+for line in open( vars ):
+    line = line.rstrip( '\r\n' )
+    if line and not line.startswith( '#' ):
+        key, value = line.split( '=' )
+        if value.startswith( '"' ) and value.endswith( '"' ):
+            value = value[1:-1]
+        context[key] = value
+
+for input, output in templates:
+    print input ,"->", output
+    open( os.path.join( out_dir, output ), "w" ).write( str( Template( file=input, searchList=[context] ) ) )
+
+for rule, output in images:
+    t = string.Template( rule ).substitute( context ) 
+    print t, "->", output
+    open( os.path.join( out_dir, output ), "w" ).write( run( t.split() ) )
+
@@ -0,0 +1,26 @@
+body
+{
+    background: $masthead_bg url(masthead_bg.png) bottom;
+    color: $masthead_text;
+    padding: 0;
+    border: 0;
+    margin: 3px;
+    margin-right: 5px;
+    margin-left: 5px;
+}
+
+div.pageTitle
+{
+    font-size: 175%;
+    font-weight: bold;
+}
+
+div.pageTitle a:link, div.pageTitle a:visited, div.pageTitle a:active, div.pageTitle a:hover
+{
+    text-decoration: none;
+}
+
+a:link, a:visited, a:active
+{
+    color: $masthead_link;
+}
@@ -0,0 +1,56 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+  <head>
+    <title>Galaxy Administration</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+    <link href="$h.url_for('/static/style/base.css')" rel="stylesheet" type="text/css" />
+  </head>
+  <body>
+    <h3 align="center">System -> Disk Usage</h3>
+    <table align="center" width="60%" class="colored">
+      <tr><td colspan="5"><div class="reportTitle">Disk Usage for $file_path</div></td></tr>
+      <tr class="header">
+        <td>File System</td>
+        <td>Disk Size</td>
+        <td>Used</td>
+        <td>Available</td>
+        <td>Percent Used</td>
+      </tr>
+      <tr class="tr">
+        <td>$disk_usage[0]</td>
+        <td>$disk_usage[1]</td>
+        <td>$disk_usage[2]</td>
+        <td>$disk_usage[3]</td>
+        <td>$disk_usage[4]</td>
+      </tr>
+      #set $dlen = len( $datasets )
+      #if $dlen == 0:
+        <tr class="header"><td colspan="5">There are no datasets larger than $file_size_str</td></tr>
+      #else:
+        <tr><td colspan="5"><div class="reportTitle">$dlen largest datasets over $file_size_str</div></td></tr>
+        <tr class="header">
+          <td>File</td>
+          <td>Created</td>
+          <td>History ID</td>
+          <td>Deleted</td>
+          <td>Size on Disk</td>
+        </tr>
+        #set ctr =  0
+        #for $dataset in $datasets
+          #if $dlen > 2 and ctr % 2 == 1:
+            <tr class="odd_row">
+          #else:
+            <tr class="tr"
+          #end if
+            <td>dataset_$dataset[0] dat</td>
+            <td>$dataset[1]</td>
+            <td>$dataset[2]</td>
+            <td>$dataset[3]</td>
+            <td>$dataset[4]</td>
+          </tr>
+          #set ctr+=1
+        #end for
+      #end if
+    </table>
+  </body>
+</html>
@@ -0,0 +1,12 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Frameset//EN" "http://www.w3.org/TR/html4/frameset.dtd">
+<html>
+  <head>
+    <title>Galaxy Reports Main</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+    <link href="$h.url_for('/static/style/base.css')" rel="stylesheet" type="text/css" />
+  </head>
+  <frameset rows="36,*" border="0" framespacing="0" frameborder="0">
+    <frame name="masthead" src="$h.url_for('masthead')" frameborder="0" border="0" framespacing="0">
+    <frame name="main_frame" src="$h.url_for('main_frame')" border="3" framespacing="3" frameborder="1">
+  </frameset>
+</html>
@@ -0,0 +1,16 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+<html>
+  <head>
+    <title>Galaxy Reports</title>
+    <link href="$h.url_for('/static/style/base.css')" rel="stylesheet" type="text/css" />
+  </head>
+  <body>
+    <div class="body">
+      <h3 align="center">Galaxy Reports</h3>
+      <table align="center" width="40%" class="colored">
+        <tr><td><div class="reportTitle">System</div></td></tr>
+        <tr><td><div class="reportBody"><a href="$h.url_for( controller='system', action='disk_usage' )">Disk Usage</a></div></td></tr>
+      </table>
+    </div>
+  </body>
+</html>
@@ -0,0 +1,49 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
+
+#if $brand is not None:
+    #set $brand = "<span class='brand'>/%s</span>" % $brand
+#end if
+
+#if $wiki_url is None:    
+    #set $wiki_url = "http://g2.trac.bx.psu.edu/"  
+#end if
+
+#if $bugs_email is None:    
+    #set $bugs_email = "mailto:galaxy-bugs@bx.psu.edu"  
+#end if
+
+#if $blog_url is None:    
+    #set $blog_url = "http://g2.trac.bx.psu.edu/blog"  
+#end if
+
+#if $screencasts_url is None:    
+    #set $screencasts_url = "http://g2.trac.bx.psu.edu/wiki/ScreenCasts"  
+#end if
+
+<html>
+  <head>
+    <title>Galaxy Reports</title>
+    <meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
+    <link href="$h.url_for('/static/style/base.css')" rel="stylesheet" type="text/css" />
+    <link href="$h.url_for('/static/style/masthead.css')" rel="stylesheet" type="text/css" />
+  </head>
+  <body>
+    <table width="100%" cellspacing="0" border="0">
+      <tr valign="middle">
+        <td width="26px"><a target="_blank" href="$wiki_url">
+          <img border="0" src="${h.url_for('/static/images/galaxyIcon_noText.png')}"></a>
+        </td>
+        <td align="left" valign="middle"><div class="pageTitle">Galaxy$brand</div></td>
+        <td align="right" valign="middle">
+          Info: <a href="$bugs_email">report bugs</a>
+          | <a target="_blank" href="$wiki_url">wiki</a>                  
+          | <a target="_blank" href="$screencasts_url">screencasts</a>
+          | <a target="_blank" href="$blog_url">blog</a>
+          &nbsp;&nbsp;&nbsp;
+          <a target="main_frame" href="$h.url_for( controller='root', action='main_frame' )">Galaxy Reports Home</a>
+          &nbsp;
+        </td>
+      </tr>
+    </table>
+  </body>
+</html>
@@ -0,0 +1,76 @@
+# ---- HTTP Server ----------------------------------------------------------
+
+[server:main]
+
+use = egg:Paste#http
+port = 8088
+host = 127.0.0.1
+use_threadpool = true
+threadpool_workers = 10
+
+# ---- Galaxy Webapps Report Interface -------------------------------------------------
+
+[app:main]
+
+# Specifies the factory for the universe WSGI application
+paste.app_factory = galaxy.webapps.reports.buildapp:app_factory
+log_level = DEBUG
+
+# Database connection
+#database_file = database/universe.sqlite
+# You may use a SQLAlchemy connection string to specify an external database instead
+database_connection = postgres:///galaxy_test
+
+# Where dataset files are saved
+file_path = database/files
+# Temporary storage for additional datasets, this should be shared through the cluster
+new_file_path = database/tmp
+
+# Where templates are stored
+template_path = lib/galaxy/webapps/reports/templates
+
+# Session support (beaker)
+use_beaker_session = True
+session_type = memory
+session_data_dir = %(here)s/database/beaker_sessions
+session_key = galaxysessions
+session_secret = changethisinproduction
+
+# Configuration for debugging middleware
+debug = true
+use_lint = false
+
+# NEVER enable this on a public site (even test or QA)
+# use_interactive = true
+
+# Admin Password
+admin_pass = "galaxy"
+
+# path to sendmail
+sendmail_path = /usr/sbin/sendmail
+
+# Address to join mailing list
+mailing_join_addr = galaxy-user-join@bx.psu.edu
+
+# Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!)
+## use_heartbeat = True
+
+# Profiling middleware (cProfile based)
+## use_profile = True
+
+# Mail
+smtp_server = coltrane.bx.psu.edu
+error_email_to = galaxy_bugs@bx.psu.edu
+
+# Use the new iframe / javascript based layout
+use_new_layout = true
+
+# Serving static files (needed if running standalone)
+static_enabled = True
+static_cache_time = 360
+static_dir = lib/galaxy/webapps/reports/static
+static_images_dir = lib/galaxy/webapps/reports/static/images
+static_favicon_dir = lib/galaxy/webapps/reports/static/favicon.ico
+static_scripts_dir = lib/galaxy/webapps/reports/static/scripts
+static_style_dir = lib/galaxy/webapps/reports/static/january_2008_style/blue
+
@@ -0,0 +1,5 @@
+#!/bin/sh
+
+source setup_paths.sh
+
+python2.4 ./scripts/paster.py serve reports_wsgi.ini $@
@@ -20,10 +20,9 @@ def main():
    #Step through Datasets, determining size on disk for each.
    print "Determining the size of each dataset..."
    for row in app.model.Dataset.table.select().execute():
-        deleted = app.model.Dataset.get( row.id ).deleted
        purged = app.model.Dataset.get( row.id ).purged
        file_size = app.model.Dataset.get( row.id ).file_size
-        if file_size is None and not deleted and not purged:
+        if file_size is None and not purged:
            size_on_disk = app.model.Dataset.get( row.id ).get_size()
            print "Updating Dataset.%d with file_size: %d" %( row.id, size_on_disk )
            app.model.Dataset.table.update( app.model.Dataset.table.c.id == row.id ).execute( file_size=size_on_disk )
@@ -1,222 +0,0 @@
-#!/usr/bin/env python2.4
-#%prog bounding_region_file mask_intervals_file intervals_to_mimic_file out_file mask_chr mask_start mask_end interval_chr interval_start interval_end
-
-
-from __future__ import division 
-
-import pkg_resources
-pkg_resources.require( "bx-python" )
-
-import sys, random
-import bisect
-from bx.bitset import *
-
-def stop_err( msg ):
-    sys.stderr.write( msg )
-    sys.exit()
-
-def throw_random_2( lengths, mask ):
-    """
-    Version of throw using gap lists (like Hiram's randomPlacement). This 
-    is not ready yet!!!
-    """
-    # Projected version for throwing
-    bits = BitSet( mask.size )
-    # Use mask to find the gaps
-    gaps = []
-    start = end = 0
-    while 1:
-        start = mask.next_clear( end )
-        if start == mask.size: break
-        end = mask.next_set( start )
-        gaps.append( ( end-start, start, end ) )
-    # Sort (long regions first )    
-    gaps.sort()
-    gaps.reverse()
-    # And throw
-    for length in lengths:
-        max_candidate = 0
-        candidate_bases = 0
-        for gap in gaps:
-            if gap[0] >= length:
-                max_candidate += 1
-                candidate_bases += ( gap[0] - length )
-            else: 
-                break
-        if max_candidate == 0:
-            raise MaxtriesException( "No gap can fit region of length %d" % length )
-        # Select start position
-        s = random.randrange( candidate_bases )
-        # Map back to region
-        chosen_index = 0
-        for gap in gaps:
-            gap_length, gap_start, gap_end = gap
-            if s > ( gap_length - length ):
-                s -= ( gap_length - length )
-                chosen_index += 1
-            else:
-                break
-        # Remove the chosen gap and split
-        assert ( gap_length, gap_start, gap_end ) == gaps.pop( chosen_index )
-        # gap_length, gap_start, gap_end =  gaps.pop( chosen_index )
-        assert s >= 0
-        assert gap_start + s + length <= gap_end, "Expected: %d + %d + %d == %d <= %d" % ( gap_start, s, length, gap_start + s + length, gap_end )
-        gaps.reverse()
-        if s > 0:
-            bisect.insort( gaps, ( s, gap_start, gap_start + s ) )
-        if s + length < gap_length:
-            bisect.insort( gaps, ( gap_length - ( s + length ), gap_start + s + length, gap_end) )
-        gaps.reverse()
-        # And finally set the bits
-        assert bits[gap_start + s] == 0 
-        assert bits.next_set( gap_start + s, gap_start+s+length ) == gap_start+s+length 
-        assert( gap_start + s >= 0 and gap_start + s + length <= bits.size ), "Bad interval %d %d %d %d %d" % ( gap_start, s, length, gap_start + s + length, bits.size )
-        bits.set_range( gap_start + s, length )   
-    assert bits.count_range( 0, bits.size ) == sum( lengths )
-    return bits
-
-def overlapping_in_bed( fname, r_chr, r_start, r_stop, chr_col, start_col, end_col ):
-    rval = []
-    for line in open( fname ):
-        if line.startswith( "#" ) or line.startswith( "track" ):
-            continue
-        fields = line.split()
-        try:
-            chr, start, stop = fields[chr_col], int( fields[start_col] ), int( fields[end_col] )
-            if chr == r_chr and start < r_stop and stop >= r_start:
-                rval.append( ( chr, max( start, r_start ), min( stop, r_stop ) ) )
-        except:
-            continue
-    return rval        
-
-def as_bits( region_start, region_length, intervals ):
-    bits = BitSet( region_length )
-    for chr, start, stop in intervals:
-        bits.set_range( start - region_start, stop - start )
-    return bits
-
-def bit_clone( bits ):
-    new = BitSet( bits.size )
-    new.ior( bits )
-    return new
-
-def count_overlap( bits1, bits2 ):
-    b = BitSet( bits1.size )
-    b |= bits1
-    b &= bits2
-    return b.count_range( 0, b.size )
-
-def interval_lengths( bits ):
-    end = 0
-    while 1:
-        start = bits.next_set( end )
-        if start == bits.size: break
-        end = bits.next_clear( start )
-        yield end - start
-
-def main():
-    #region_fname = sys.argv[1]
-    region_uid = sys.argv[1]
-    mask_fname = sys.argv[2]
-    intervals_fname = sys.argv[3]
-    out_fname = sys.argv[4]
-    try:
-        mask_chr = int( sys.argv[5] ) - 1
-    except:
-        stop_err( "'%s' is an invalid chrom column for 'Intervals to Mask' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[5] ) )
-    try:
-        mask_start = int( sys.argv[6] ) - 1
-    except:
-        stop_err( "'%s' is an invalid start column for 'Intervals to Mask' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[6] ) )
-    try:
-        mask_end = int( sys.argv[7] ) - 1
-    except:
-        stop_err( "'%s' is an invalid end column for 'Intervals to Mask' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[7] ) )
-    try:
-        interval_chr = int( sys.argv[8] ) - 1
-    except:
-        stop_err( "'%s' is an invalid chrom column for 'File to Mimick' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[8] ) )
-    try:
-        interval_start = int( sys.argv[9] ) - 1
-    except:
-        stop_err( "'%s' is an invalid start column for 'File to Mimick' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[9] ) )
-    try:
-        interval_end = int( sys.argv[10] ) - 1
-    except:
-        stop_err( "'%s' is an invalid end column for 'File to Mimick' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[10] ) )
-    use_mask = sys.argv[11]
-    available_regions = {}
-    loc_file = "/depot/data2/galaxy/regions.loc"
-
-    for i, line in enumerate( open( loc_file ) ):
-        line = line.rstrip( '\r\n' )
-        if line and not line.startswith( '#' ):
-            fields = line.split( '\t' )
-            #read each line, if not enough fields, go to next line
-            try:
-                build = fields[0]
-                uid = fields[1]
-                description = fields[2]
-                filepath = fields[3]
-                available_regions[uid] = filepath
-            except:
-                continue
-
-    if region_uid not in available_regions:
-        stop_err( "Invalid region '%s' selected." % region_uid )
-
-    region_fname = available_regions[region_uid]
-    try:
-        out_file = open ( out_fname, "w" )
-    except:
-        stop_err( "Error opening output file '%s'." % out_fname )
-
-    skipped_lines = 0
-    first_invalid_line = 0
-    invalid_line = ''
-
-    for line_count, line in enumerate( file( region_fname ) ):
-        if line and not line.startswith( '#' ):
-            try:
-                # Load lengths for all intervals overlapping region
-                fields = line.split()
-                r_chr, r_start, r_stop = fields[0], int( fields[1] ), int( fields[2] )
-                r_length = r_stop - r_start
-                # Load the mask
-                if use_mask == "no_mask":
-                    mask = []
-                else:
-                    mask = overlapping_in_bed( mask_fname, r_chr, r_start, r_stop, mask_chr, mask_start, mask_end )
-                bits_mask = as_bits( r_start, r_length, mask )
-                bits_not_masked = bit_clone( bits_mask )
-                bits_not_masked.invert()
-                # Load the first set
-                intervals1 = overlapping_in_bed( intervals_fname, r_chr, r_start, r_stop, interval_chr, interval_start, interval_end )
-                bits1 = as_bits( r_start, r_length, intervals1 )
-                # Intersect it with the mask 
-                bits1.iand( bits_not_masked )
-                # Sanity checks
-                if count_overlap( bits1, bits_mask ) != 0:
-                    stop_err( "Overlap problem, bits: %s, mask: %s" %( str( bits1 ), str( bits_mask ) ) )
-                chrom = r_chr
-                # For each data set
-                lengths1 = list( interval_lengths( bits1 ) )
-                random1 = throw_random_2( lengths1, bits_mask )
-                end =0
-                while 1:
-                    start = random1.next_set( end )
-                    if start == random1.size:
-                        break
-                    end = random1.next_clear( start )
-                    print >>out_file, "%s\t%d\t%d" % ( chrom, start, end )
-            except:
-                skipped_lines += 1
-                if not first_invalid_line:
-                    first_invalid_line = line_count
-                    invalid_line = line
-                continue
-    out_file.close()
-    if skipped_lines:
-        print "Skipped %d invalid lines starting at line # %d: %s" % ( skipped_lines, first_invalid_line, invalid_line )
-
-if __name__ == "__main__": main()