diff --git a/lib/galaxy/webapps/__init__.py b/lib/galaxy/webapps/__init__.py new file mode 100644 index 00000000000..1ec78510e11 --- /dev/null +++ b/lib/galaxy/webapps/__init__.py @@ -0,0 +1,3 @@ +"""Galaxy Reports root package -- this is a namespace package.""" + +__import__( "pkg_resources" ).declare_namespace( __name__ ) \ No newline at end of file diff --git a/lib/galaxy/webapps/reports/__init__.py b/lib/galaxy/webapps/reports/__init__.py new file mode 100644 index 00000000000..5fe689ffc4d --- /dev/null +++ b/lib/galaxy/webapps/reports/__init__.py @@ -0,0 +1,3 @@ +"""The Galaxy Reports application.""" + +from galaxy.web.framework import expose, url_for diff --git a/lib/galaxy/webapps/reports/app.py b/lib/galaxy/webapps/reports/app.py new file mode 100644 index 00000000000..e3dec32cf7d --- /dev/null +++ b/lib/galaxy/webapps/reports/app.py @@ -0,0 +1,32 @@ +import sys, os, atexit +import galaxy.model +import config + +class UniverseApplication( object ): + """Encapsulates the state of a Universe application""" + def __init__( self, **kwargs ): + print >> sys.stderr, "python path is: " + ", ".join( sys.path ) + # Read config file and check for errors + self.config = config.Configuration( **kwargs ) + self.config.check() + config.configure_logging( self.config ) + # Determine the database url + if self.config.database_connection: + db_url = self.config.database_connection + else: + db_url = "sqlite://%s?isolation_level=IMMEDIATE" % self.config.database + # Setup the database engine and ORM + self.model = galaxy.model.mapping.init( self.config.file_path, + db_url, + self.config.database_engine_options, + create_tables = True ) + self.heartbeat = None + # Start the heartbeat process if configured and available + if self.config.use_heartbeat: + from galaxy.util import heartbeat + if heartbeat.Heartbeat: + self.heartbeat = heartbeat.Heartbeat() + self.heartbeat.start() + def shutdown( self ): + if self.heartbeat: + self.heartbeat.shutdown() diff --git a/lib/galaxy/webapps/reports/base/__init__.py b/lib/galaxy/webapps/reports/base/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/lib/galaxy/webapps/reports/base/controller.py b/lib/galaxy/webapps/reports/base/controller.py new file mode 100644 index 00000000000..4b77b886ad4 --- /dev/null +++ b/lib/galaxy/webapps/reports/base/controller.py @@ -0,0 +1,27 @@ +"""Contains functionality needed in every webapp interface""" + +import os, time, logging + +# Pieces of Galaxy to make global in every controller +#from galaxy import config, tools, web, model, util +from galaxy import web + +from Cheetah.Template import Template + +log = logging.getLogger( __name__ ) + +class BaseController( object ): + """Base class for Galaxy webapp application controllers.""" + + beta = False + + def __init__( self, app ): + """Initialize an interface for application 'app'""" + self.app = app + + #def get_toolbox(self): + # """Returns the application toolbox""" + # return self.app.toolbox + +#Root = BaseController +"""Deprecated: `BaseController` used to be available under the name `Root`""" \ No newline at end of file diff --git a/lib/galaxy/webapps/reports/buildapp.py b/lib/galaxy/webapps/reports/buildapp.py new file mode 100644 index 00000000000..a9392f17848 --- /dev/null +++ b/lib/galaxy/webapps/reports/buildapp.py @@ -0,0 +1,190 @@ +""" +Provides factory methods to assemble the Galaxy web application +""" + +import logging, atexit +import os, os.path, sys + +from inspect import isclass + +from paste.request import parse_formvars +from paste.util import import_string +from paste import httpexceptions +from paste.deploy.converters import asbool +import flup.middleware.session as flup_session +import pkg_resources + +log = logging.getLogger( __name__ ) + +import config +import galaxy.model +import galaxy.model.mapping +import galaxy.web.framework + +def add_controllers( webapp, app ): + """ + Search for controllers in the 'galaxy.webapps.controllers' module and add + them to the webapp. + """ + from galaxy.webapps.reports.base.controller import BaseController + import galaxy.webapps.reports.controllers + controller_dir = galaxy.webapps.reports.controllers.__path__[0] + for fname in os.listdir( controller_dir ): + if not( fname.startswith( "_" ) ) and fname.endswith( ".py" ): + name = fname[:-3] + module_name = "galaxy.webapps.reports.controllers." + name + module = __import__( module_name ) + for comp in module_name.split( "." )[1:]: + module = getattr( module, comp ) + # Look for a controller inside the modules + for key in dir( module ): + T = getattr( module, key ) + if isclass( T ) and T is not BaseController and issubclass( T, BaseController ): + #if app.config.enable_beta_features or not ( T.beta ): + webapp.add_controller( name, T( app ) ) + +def app_factory( global_conf, **kwargs ): + """Return a wsgi application serving the root object""" + # Create the Galaxy application unless passed in + if 'app' in kwargs: + app = kwargs.pop( 'app' ) + else: + from galaxy.webapps.reports.app import UniverseApplication + app = UniverseApplication( global_conf = global_conf, **kwargs ) + atexit.register( app.shutdown ) + # Create the universe WSGI application + webapp = galaxy.web.framework.WebApplication( app ) + add_controllers( webapp, app ) + # These two routes handle our simple needs at the moment + webapp.add_route( '/:controller/:action', controller="root", action='index' ) + webapp.add_route( '/:action', controller='root', action='index' ) + webapp.finalize_config() + # Wrap the webapp in some useful middleware + if kwargs.get( 'middleware', True ): + webapp = wrap_in_middleware( webapp, global_conf, **kwargs ) + if kwargs.get( 'static_enabled', True ): + webapp = wrap_in_static( webapp, global_conf, **kwargs ) + # Close any pooled database connections before forking + try: + galaxy.model.mapping.metadata.engine.connection_provider._pool.dispose() + except: + pass + # Return + return webapp + +def wrap_in_middleware( app, global_conf, **local_conf ): + """Based on the configuration wrap `app` in a set of common and useful middleware.""" + # Merge the global and local configurations + conf = global_conf.copy() + conf.update(local_conf) + debug = asbool( conf.get( 'debug', False ) ) + # First put into place httpexceptions, which must be most closely + # wrapped around the application (it can interact poorly with + # other middleware): + app = httpexceptions.make_middleware( app, conf ) + log.debug( "Enabling 'httpexceptions' middleware" ) + # The recursive middleware allows for including requests in other + # requests or forwarding of requests, all on the server side. + if asbool(conf.get('use_recursive', True)): + from paste import recursive + app = recursive.RecursiveMiddleware( app, conf ) + log.debug( "Enabling 'recursive' middleware" ) + ## # Session middleware puts a session factory into the environment + ## if asbool( conf.get( 'use_session', True ) ): + ## store = flup_session.MemorySessionStore() + ## app = flup_session.SessionMiddleware( store, app ) + ## log.debug( "Enabling 'flup session' middleware" ) + # Beaker session middleware + if asbool( conf.get( 'use_beaker_session', False ) ): + pkg_resources.require( "Beaker" ) + import beaker.session + app = beaker.session.SessionMiddleware( app, conf ) + log.debug( "Enabling 'beaker session' middleware" ) + # Various debug middleware that can only be turned on if the debug + # flag is set, either because they are insecure or greatly hurt + # performance + if debug: + # Middleware to check for WSGI compliance + if asbool( conf.get( 'use_lint', True ) ): + from paste import lint + app = lint.make_middleware( app, conf ) + log.debug( "Enabling 'lint' middleware" ) + # Middleware to run the python profiler on each request + if asbool( conf.get( 'use_profile', False ) ): + import profile + app = profile.ProfileMiddleware( app, conf ) + log.debug( "Enabling 'profile' middleware" ) + # Middleware that intercepts print statements and shows them on the + # returned page + if asbool( conf.get( 'use_printdebug', True ) ): + from paste.debug import prints + app = prints.PrintDebugMiddleware( app, conf ) + log.debug( "Enabling 'print debug' middleware" ) + if debug and asbool( conf.get( 'use_interactive', False ) ): + # Interactive exception debugging, scary dangerous if publicly + # accessible, if not enabled we'll use the regular error printing + # middleware. + pkg_resources.require( "WebError" ) + from weberror import evalexception + app = evalexception.EvalException( app, conf, + templating_formatters=build_template_error_formatters() ) + log.debug( "Enabling 'eval exceptions' middleware" ) + else: + # Not in interactive debug mode, just use the regular error middleware + from paste.exceptions import errormiddleware + app = errormiddleware.ErrorMiddleware( app, conf ) + log.debug( "Enabling 'error' middleware" ) + # Transaction logging (apache access.log style) + if asbool( conf.get( 'use_translogger', True ) ): + from paste.translogger import TransLogger + app = TransLogger( app ) + log.debug( "Enabling 'trans logger' middleware" ) + # Config middleware just stores the paste config along with the request, + # not sure we need this but useful + from paste.deploy.config import ConfigMiddleware + app = ConfigMiddleware( app, conf ) + log.debug( "Enabling 'config' middleware" ) + # X-Forwarded-Host handling + from galaxy.web.framework.middleware.xforwardedhost import XForwardedHostMiddleware + app = XForwardedHostMiddleware( app ) + log.debug( "Enabling 'x-forwarded-host' middleware" ) + return app + +def wrap_in_static( app, global_conf, **local_conf ): + from paste.urlmap import URLMap + from galaxy.web.framework.middleware.static import CacheableStaticURLParser as Static + urlmap = URLMap() + # Merge the global and local configurations + conf = global_conf.copy() + conf.update(local_conf) + # Get cache time in seconds + cache_time = conf.get( "static_cache_time", None ) + if cache_time is not None: + cache_time = int( cache_time ) + # Send to dynamic app by default + urlmap["/"] = app + # Define static mappings from config + urlmap["/static"] = Static( conf.get( "static_dir" ), cache_time ) + urlmap["/images"] = Static( conf.get( "static_images_dir" ), cache_time ) + urlmap["/static/scripts"] = Static( conf.get( "static_scripts_dir" ), cache_time ) + urlmap["/static/style"] = Static( conf.get( "static_style_dir" ), cache_time ) + urlmap["/favicon.ico"] = Static( conf.get( "static_favicon_dir" ), cache_time ) + # URL mapper becomes the root webapp + return urlmap + +def build_template_error_formatters(): + """ + Build a list of template error formatters for WebError. When an error + occurs, WebError pass the exception to each function in this list until + one returns a value, which will be displayed on the error page. + """ + formatters = [] + # Formatter for mako + import mako.exceptions + def mako_html_data( exc_value ): + if isinstance( exc_value, ( mako.exceptions.CompileException, mako.exceptions.SyntaxException ) ): + return mako.exceptions.html_error_template().render( full=False, css=False ) + if isinstance( exc_value, AttributeError ) and exc_value.args[0].startswith( "'Undefined' object has no attribute" ): + return mako.exceptions.html_error_template().render( full=False, css=False ) + formatters.append( mako_html_data ) + return formatters \ No newline at end of file diff --git a/lib/galaxy/webapps/reports/config.py b/lib/galaxy/webapps/reports/config.py new file mode 100644 index 00000000000..cccf371b5a2 --- /dev/null +++ b/lib/galaxy/webapps/reports/config.py @@ -0,0 +1,144 @@ +""" +Universe configuration builder. +""" + +import sys, os +import logging, logging.config +from optparse import OptionParser +import ConfigParser +from galaxy.util import string_as_bool + +log = logging.getLogger( __name__ ) + +def resolve_path( path, root ): + """If 'path' is relative make absolute by prepending 'root'""" + if not( os.path.isabs( path ) ): + path = os.path.join( root, path ) + return path + +class ConfigurationError( Exception ): + pass + +class Configuration( object ): + def __init__( self, **kwargs ): + self.config_dict = kwargs + self.root = kwargs.get( 'root_dir', '.' ) + #self.enable_beta_features = kwargs.get( "enable_beta_features", False ) + # Database related configuration + self.database = resolve_path( kwargs.get( "database_file", "database/universe.d" ), self.root ) + self.database_connection = kwargs.get( "database_connection", False ) + self.database_engine_options = get_database_engine_options( kwargs ) + # Where dataset files are stored + self.file_path = resolve_path( kwargs.get( "file_path", "database/files" ), self.root ) + self.new_file_path = resolve_path( kwargs.get( "new_file_path", "database/tmp" ), self.root ) + #self.tool_path = resolve_path( kwargs.get( "tool_path", "tools" ), self.root ) + self.test_conf = resolve_path( kwargs.get( "test_conf", "" ), self.root ) + #self.tool_config = resolve_path( kwargs.get( 'tool_config_file', 'tool_conf.xml' ), self.root ) + #self.tool_secret = kwargs.get( "tool_secret", "" ) + self.template_path = resolve_path( kwargs.get( "template_path", "templates" ), self.root ) + self.template_cache = resolve_path( kwargs.get( "template_cache_path", "database/compiled_templates" ), self.root ) + #self.job_queue_workers = int( kwargs.get( "job_queue_workers", "10" ) ) + #self.job_scheduler_policy = kwargs.get("job_scheduler_policy", "FIFO") + #self.job_queue_cleanup_interval = int( kwargs.get("job_queue_cleanup_interval", "5") ) + #self.job_working_directory = resolve_path( kwargs.get( "job_working_directory", "database/job_working_directory" ), self.root ) + self.admin_pass = kwargs.get('admin_pass',"galaxy") + self.sendmail_path = kwargs.get('sendmail_path',"/usr/sbin/sendmail") + self.mailing_join_addr = kwargs.get('mailing_join_addr',"galaxy-user-join@bx.psu.edu") + self.error_email_to = kwargs.get( 'error_email_to', None ) + #self.smtp_server = kwargs.get( 'smtp_server', None ) + #self.use_pbs = kwargs.get('use_pbs', False ) + #self.pbs_server = kwargs.get('pbs_server', "" ) + #self.pbs_instance_path = kwargs.get('pbs_instance_path', os.getcwd() ) + #self.pbs_application_server = kwargs.get('pbs_application_server', "" ) + #self.pbs_dataset_server = kwargs.get('pbs_dataset_server', "" ) + #self.pbs_dataset_path = kwargs.get('pbs_dataset_path', "" ) + self.use_heartbeat = kwargs.get( 'use_heartbeat', False ) + #self.ucsc_display_sites = kwargs.get( 'ucsc_display_sites', "main,test,archaea" ).lower().split(",") + #self.gbrowse_display_sites = kwargs.get( 'gbrowse_display_sites', "volvox,wormbase,flybase" ).lower().split(",") + self.brand = kwargs.get( 'brand', None ) + self.wiki_url = kwargs.get( 'wiki_url', None ) + self.bugs_email = kwargs.get( 'bugs_email', None ) + self.blog_url = kwargs.get( 'blog_url', None ) + self.screencasts_url = kwargs.get( 'screencasts_url', None ) + #Parse global_conf + global_conf = kwargs.get( 'global_conf', None ) + global_conf_parser = ConfigParser.ConfigParser() + if global_conf and "__file__" in global_conf: + global_conf_parser.read(global_conf['__file__']) + #Store datatypes config + #try: + # self.datatypes = global_conf_parser.items("galaxy:datatypes") + #except ConfigParser.NoSectionError: + # self.datatypes = [] + #Store sniff order config + #try: + # self.sniff_order = global_conf_parser.items("galaxy:sniff_order") + #except ConfigParser.NoSectionError: + # self.sniff_order = [] + #self.datatype_converters_config = kwargs.get( 'datatype_converters_config_file', "datatype_converters_conf.xml" ) + #self.datatype_converters_path = kwargs.get( 'datatype_converters_path', os.path.join(self.root,"lib/galaxy/datatypes/converters") ) + def get( self, key, default ): + return self.config_dict.get( key, default ) + def check( self ): + # Check that required directories exist + for path in self.root, self.file_path, self.template_path: + if not os.path.isdir( path ): + raise ConfigurationError("Directory does not exist: %s" % path ) + +def get_database_engine_options( kwargs ): + """ + Allow options for the SQLAlchemy database engine to be passed by using + the prefix "database_engine_option_". + """ + conversions = { + 'convert_unicode': string_as_bool, + 'pool_timeout': int, + 'echo': string_as_bool, + 'echo_pool': string_as_bool, + 'pool_recycle': int, + 'pool_size': int, + 'max_overflow': int, + 'pool_threadlocal': string_as_bool + } + prefix = "database_engine_option_" + prefix_len = len( prefix ) + rval = {} + for key, value in kwargs.iteritems(): + if key.startswith( prefix ): + key = key[prefix_len:] + if key in conversions: + value = conversions[key](value) + rval[ key ] = value + return rval + +def configure_logging( config ): + """ + Allow some basic logging configuration to be read from the cherrpy + config. + """ + format = config.get( "log_format", "%(name)s %(levelname)s %(asctime)s %(message)s" ) + level = logging._levelNames[ config.get( "log_level", "DEBUG" ) ] + destination = config.get( "log_destination", "stdout" ) + log.info( "Logging at '%s' level to '%s'" % ( level, destination ) ) + # Get root logger + root = logging.getLogger() + # Set level + root.setLevel( level ) + # Turn down paste httpserver logging + if level <= logging.DEBUG: + logging.getLogger( "paste.httpserver.ThreadPool" ).setLevel( logging.WARN ) + # Remove old handlers + for h in root.handlers[:]: + root.removeHandler(h) + # Create handler + if destination == "stdout": + handler = logging.StreamHandler( sys.stdout ) + else: + handler = logging.FileHandler( destination ) + # Create formatter + formatter = logging.Formatter( format ) + # Hook everything up + handler.setFormatter( formatter ) + root.addHandler( handler ) + + diff --git a/lib/galaxy/webapps/reports/controllers/__init__.py b/lib/galaxy/webapps/reports/controllers/__init__.py new file mode 100644 index 00000000000..28d2c922994 --- /dev/null +++ b/lib/galaxy/webapps/reports/controllers/__init__.py @@ -0,0 +1 @@ +"""Galaxy reports controllers.""" \ No newline at end of file diff --git a/lib/galaxy/webapps/reports/controllers/root.py b/lib/galaxy/webapps/reports/controllers/root.py new file mode 100644 index 00000000000..f96163cd4b6 --- /dev/null +++ b/lib/galaxy/webapps/reports/controllers/root.py @@ -0,0 +1,32 @@ +import sys, sets, string, shutil +import re, socket + +from galaxy import web + +from cgi import escape, FieldStorage +import urllib + +import operator, os +from galaxy.webapps.reports.base.controller import * + +import logging, sets, time +log = logging.getLogger( __name__ ) + +class Report( BaseController ): + @web.expose + def index( self, trans, **kwd ): + return trans.fill_template( 'index.tmpl' ) + + @web.expose + def masthead( self, trans ): + brand = trans.app.config.get( "brand", None ) + wiki_url = trans.app.config.get( "wiki_url", None ) + bugs_email = trans.app.config.get( "bugs_email", None ) + blog_url = trans.app.config.get( "blog_url", None ) + screencasts_url = trans.app.config.get( "screencasts_url", None ) + return trans.fill_template( "masthead.tmpl", brand=brand, wiki_url=wiki_url, blog_url=blog_url,bugs_email=bugs_email, screencasts_url=screencasts_url ) + + @web.expose + def main_frame( self, trans ): + return trans.fill_template( "main_frame.tmpl" ) + diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py new file mode 100644 index 00000000000..a6a612a7b43 --- /dev/null +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -0,0 +1,48 @@ +import sys, sets, string, shutil +import re, socket + +from galaxy import web + +from cgi import escape, FieldStorage +import urllib + +import operator, os +from galaxy.webapps.reports.base.controller import * + +import logging, sets, time +log = logging.getLogger( __name__ ) + +class System( BaseController ): + + def get_disk_usage( self, file_path ): + df_cmd = 'df -h ' + file_path + df_file = os.popen( df_cmd ) + + while True: + df_list = df_file.readline() + if not df_list: + break #EOF + dflistlower = df_list.lower() + if 'filesystem' in dflistlower or 'proc' in dflistlower: + continue + file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount = df_list.split() + break + + df_file.close() + return ( file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount ) + + @web.expose + def disk_usage( self, trans, **kwd ): + file_path = trans.app.config.file_path + disk_usage = self.get_disk_usage( file_path ) + min_file_size = 2**40 # 1 GB + file_size_str = '1 GB' + datasets = [] + dt = trans.model.Dataset.table + + for row in dt.select( dt.c.file_size>min_file_size ).execute(): + datasets.append( ( row.id, str( row.create_time )[0:10], row.history_id, row.deleted, row.file_size ) ) + + datasets = sorted( datasets, key=operator.itemgetter(4), reverse=True ) + return trans.fill_template('disk_usage.tmpl', file_path=file_path, disk_usage=disk_usage, datasets=datasets, file_size_str=file_size_str) + diff --git a/lib/galaxy/webapps/reports/static/favicon.ico b/lib/galaxy/webapps/reports/static/favicon.ico new file mode 100644 index 00000000000..74049dbe65b Binary files /dev/null and b/lib/galaxy/webapps/reports/static/favicon.ico differ diff --git a/lib/galaxy/webapps/reports/static/images/galaxyIcon_noText.png b/lib/galaxy/webapps/reports/static/images/galaxyIcon_noText.png new file mode 100644 index 00000000000..df696fc543c Binary files /dev/null and b/lib/galaxy/webapps/reports/static/images/galaxyIcon_noText.png differ diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/base.css.tmpl b/lib/galaxy/webapps/reports/static/january_2008_style/base.css.tmpl new file mode 100644 index 00000000000..40f7c054342 --- /dev/null +++ b/lib/galaxy/webapps/reports/static/january_2008_style/base.css.tmpl @@ -0,0 +1,277 @@ +@import url( "reset.css" ); + +body +{ + font: 75% verdana, "Bitstream Vera Sans", geneva, arial, helvetica, helve, sans-serif; + background: $base_bg_bottom; + color: $base_text; + background-image: url(base_bg.png); + background-repeat: repeat-x; + background-position: top; + margin: 10px; +} + +a:link, a:visited, a:active +{ + color: $link_text; +} + +h1, h2, h3, h4 +{ + color: $header_text; + /*text-shadow: #bbb 2px 2px 1px;*/ +} + +hr +{ + border: none; + border-bottom: dotted $base_text 1px; +} + +div.report +{ + border: solid $report_border 1px; +} + +div.reportTitle +{ + font-weight: bold; + padding: 5px; + padding-left: 10px; + padding-right: 10px; + background: $report_title_bg_bottom; + background-image: url(report_title_bg.png); + background-repeat: repeat-x; + background-position: top; + border-bottom: solid $report_border 1px; +} + +div.reportParamHelp +{ + color: #666; +} + +div.reportParamHelp a +{ + color: #666; +} + +div.reportBody +{ + background: $report_body_bg_bottom; + background-image: url(report_body_bg.png); + background-repeat: repeat-x; + background-position: top; + padding: 5px; + padding-left: 10px; + padding-right: 10px; +} + +div.reportBody div.reportTitle +{ + background: transparent; + border: none; + font-weight: bold; + border-bottom: solid $report_border 1px; + margin-bottom: 5px; +} + +div.reportDisabled div.reportTitle { + background: ${layout_bg}; + border-color: ${layout_border}; +} + +div.reportDisabled { + border-color: ${layout_border}; +} + +div.reportHelp +{ +} + +div.reportHelpBody +{ + width: 100%; + overflow: auto; +} + +div.titleRow { + font-weight: bold; + border-bottom: dotted gray 1px; + margin-bottom: 0.5em; + padding-bottom: 0.25em; +} + +div.report-row +{ + margin-bottom: 1em; +} + +div.report-title-row +{ + margin-bottom: 0.5em; +} + +div.report-row-error +{ + background: $error_message_bg; + padding: 5px; +} + +div.report-row label +{ + font-weight: bold; + display: block; + margin-bottom: .2em; +} + +/* Messages */ + +.errormessage, .warningmessage, .donemessage, .infomessage, .welcomeBlue, .welcomeRed +{ + padding: 10px; + padding-left: 52px; + min-height: 32px; + border: 1px solid $error_message_border; + background-color: $error_message_bg; + background-image: url(error_message_icon.png); + background-repeat: no-repeat; + background-position: 10px 10px; +} + +.warningmessage +{ + background-image: url(warn_message_icon.png); + border-color: $warn_message_border; + background-color: $warn_message_bg; +} + +.donemessage +{ + background-image: url(done_message_icon.png); + border-color: $done_message_border; + background-color: $done_message_bg; +} + +.infomessage +{ + background-image: url(info_message_icon.png); + border-color: $info_message_border; + background-color: $info_message_bg; +} + +.welcomeBlue +{ + padding-left: 10px; + border-color: $info_message_border; + background-color: $info_message_bg; + background-image: none; +} + +.welcomeRed +{ + padding-left: 10px; + border-color: $error_message_border; + background-color: $error_message_bg; + background-image: none; +} + +.errormessagesmall, .warningmessagesmall, .donemessagesmall, .infomessagesmall +{ + padding: 5px; + padding-left: 25px; + min-height: 25px; + border: 1px solid $error_message_border; + background-color: $error_message_bg; + background-image: url(error_small.png); + background-repeat: no-repeat; + background-position: 5px 5px; +} + +.warningmessagesmall +{ + background-image: url(warn_small.png); + border-color: $warn_message_border; + background-color: $warn_message_bg; +} + +.donemessagesmall +{ + background-image: url(ok_small.png); + border-color: $done_message_border; + background-color: $done_message_bg; +} + +.infomessagesmall +{ + background-image: url(info_small.png); + border-color: $info_message_border; + background-color: $info_message_bg; +} + +.errormark, .warningmark, .donemark, .infomark, .ok_bgr, .err_bgr +{ + padding-left: 20px; + min-height: 15px; + background: url(error_small.png) no-repeat; +} + +.warningmark +{ + background-image: url(warn_small.png); +} + +.donemark +{ + background-image: url(ok_small.png); +} + +.infomark, .ok_bgr +{ + background-image: url(info_small.png); +} + +table.colored +{ + border-top: solid $table_border 1px; + border-bottom: solid $table_border 1px; +} + +table.colored td +{ + padding: 5px; +} + +table.colored tr.header +{ + background: $table_header_bg; + background-image: url(report_title_bg.png); + background-repeat: repeat-x; + background-position: top; + border-bottom: solid $table_border 1px; + font-weight: bold; +} + +table.colored tr +{ + background: $table_row_bg; +} + +table.colored tr.odd_row +{ + background: $odd_row_bg; +} + +div.debug +{ + margin: 10px; + padding: 5px; + background: #FFFF99; + border: solid #FFFF33 1px; + color: black; +} + +#footer { + display: none; +} + diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css b/lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css new file mode 100644 index 00000000000..a0e33b925f6 --- /dev/null +++ b/lib/galaxy/webapps/reports/static/january_2008_style/blue/base.css @@ -0,0 +1,277 @@ +@import url( "reset.css" ); + +body +{ + font: 75% verdana, "Bitstream Vera Sans", geneva, arial, helvetica, helve, sans-serif; + background: #FFFFFF; + color: #303030; + background-image: url(base_bg.png); + background-repeat: repeat-x; + background-position: top; + margin: 10px; +} + +a:link, a:visited, a:active +{ + color: #303030; +} + +h1, h2, h3, h4 +{ + color: #023858; + /*text-shadow: #bbb 2px 2px 1px;*/ +} + +hr +{ + border: none; + border-bottom: dotted #303030 1px; +} + +div.report +{ + border: solid #d8b365 1px; +} + +div.reportTitle +{ + font-weight: bold; + padding: 5px; + padding-left: 10px; + padding-right: 10px; + background: #d2c099; + background-image: url(report_title_bg.png); + background-repeat: repeat-x; + background-position: top; + border-bottom: solid #d8b365 1px; +} + +div.reportParamHelp +{ + color: #666; +} + +div.reportParamHelp a +{ + color: #666; +} + +div.reportBody +{ + background: #FFFFFF; + background-image: url(report_body_bg.png); + background-repeat: repeat-x; + background-position: top; + padding: 5px; + padding-left: 10px; + padding-right: 10px; +} + +div.reportBody div.reportTitle +{ + background: transparent; + border: none; + font-weight: bold; + border-bottom: solid #d8b365 1px; + margin-bottom: 5px; +} + +div.reportDisabled div.reportTitle { + background: #eee; + border-color: #999; +} + +div.reportDisabled { + border-color: #999; +} + +div.reportHelp +{ +} + +div.reportHelpBody +{ + width: 100%; + overflow: auto; +} + +div.titleRow { + font-weight: bold; + border-bottom: dotted gray 1px; + margin-bottom: 0.5em; + padding-bottom: 0.25em; +} + +div.report-row +{ + margin-bottom: 1em; +} + +div.report-title-row +{ + margin-bottom: 0.5em; +} + +div.report-row-error +{ + background: #FFCCCC; + padding: 5px; +} + +div.report-row label +{ + font-weight: bold; + display: block; + margin-bottom: .2em; +} + +/* Messages */ + +.errormessage, .warningmessage, .donemessage, .infomessage, .welcomeBlue, .welcomeRed +{ + padding: 10px; + padding-left: 52px; + min-height: 32px; + border: 1px solid #AA6666; + background-color: #FFCCCC; + background-image: url(error_message_icon.png); + background-repeat: no-repeat; + background-position: 10px 10px; +} + +.warningmessage +{ + background-image: url(warn_message_icon.png); + border-color: #AAAA66; + background-color: #FFFFCC; +} + +.donemessage +{ + background-image: url(done_message_icon.png); + border-color: #66AA66; + background-color: #CCFFCC; +} + +.infomessage +{ + background-image: url(info_message_icon.png); + border-color: #6666AA; + background-color: #CCCCFF; +} + +.welcomeBlue +{ + padding-left: 10px; + border-color: #6666AA; + background-color: #CCCCFF; + background-image: none; +} + +.welcomeRed +{ + padding-left: 10px; + border-color: #AA6666; + background-color: #FFCCCC; + background-image: none; +} + +.errormessagesmall, .warningmessagesmall, .donemessagesmall, .infomessagesmall +{ + padding: 5px; + padding-left: 25px; + min-height: 25px; + border: 1px solid #AA6666; + background-color: #FFCCCC; + background-image: url(error_small.png); + background-repeat: no-repeat; + background-position: 5px 5px; +} + +.warningmessagesmall +{ + background-image: url(warn_small.png); + border-color: #AAAA66; + background-color: #FFFFCC; +} + +.donemessagesmall +{ + background-image: url(ok_small.png); + border-color: #66AA66; + background-color: #CCFFCC; +} + +.infomessagesmall +{ + background-image: url(info_small.png); + border-color: #6666AA; + background-color: #CCCCFF; +} + +.errormark, .warningmark, .donemark, .infomark, .ok_bgr, .err_bgr +{ + padding-left: 20px; + min-height: 15px; + background: url(error_small.png) no-repeat; +} + +.warningmark +{ + background-image: url(warn_small.png); +} + +.donemark +{ + background-image: url(ok_small.png); +} + +.infomark, .ok_bgr +{ + background-image: url(info_small.png); +} + +table.colored +{ + border-top: solid #d8b365 1px; + border-bottom: solid #d8b365 1px; +} + +table.colored td +{ + padding: 5px; +} + +table.colored tr.header +{ + background: #ebd9b2; + background-image: url(report_title_bg.png); + background-repeat: repeat-x; + background-position: top; + border-bottom: solid #d8b365 1px; + font-weight: bold; +} + +table.colored tr +{ + background: white; +} + +table.colored tr.odd_row +{ + background: #FFFF99; +} + +div.debug +{ + margin: 10px; + padding: 5px; + background: #FFFF99; + border: solid #FFFF33 1px; + color: black; +} + +#footer { + display: none; +} + diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/blue/base_bg.png b/lib/galaxy/webapps/reports/static/january_2008_style/blue/base_bg.png new file mode 100644 index 00000000000..8ad812ef561 Binary files /dev/null and b/lib/galaxy/webapps/reports/static/january_2008_style/blue/base_bg.png differ diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/blue/footer_title_bg.png b/lib/galaxy/webapps/reports/static/january_2008_style/blue/footer_title_bg.png new file mode 100644 index 00000000000..77da839cbf1 Binary files /dev/null and b/lib/galaxy/webapps/reports/static/january_2008_style/blue/footer_title_bg.png differ diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/blue/masthead.css b/lib/galaxy/webapps/reports/static/january_2008_style/blue/masthead.css new file mode 100644 index 00000000000..f206e7f1b44 --- /dev/null +++ b/lib/galaxy/webapps/reports/static/january_2008_style/blue/masthead.css @@ -0,0 +1,26 @@ +body +{ + background: #2C3143 url(masthead_bg.png) bottom; + color: #eeeeee; + padding: 0; + border: 0; + margin: 3px; + margin-right: 5px; + margin-left: 5px; +} + +div.pageTitle +{ + font-size: 175%; + font-weight: bold; +} + +div.pageTitle a:link, div.pageTitle a:visited, div.pageTitle a:active, div.pageTitle a:hover +{ + text-decoration: none; +} + +a:link, a:visited, a:active +{ + color: #eeeeee; +} diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/blue/masthead_bg.png b/lib/galaxy/webapps/reports/static/january_2008_style/blue/masthead_bg.png new file mode 100644 index 00000000000..b6488b52277 Binary files /dev/null and b/lib/galaxy/webapps/reports/static/january_2008_style/blue/masthead_bg.png differ diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/blue/report_title_bg.png b/lib/galaxy/webapps/reports/static/january_2008_style/blue/report_title_bg.png new file mode 100644 index 00000000000..dd7599aa3b7 Binary files /dev/null and b/lib/galaxy/webapps/reports/static/january_2008_style/blue/report_title_bg.png differ diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/blue_colors.ini b/lib/galaxy/webapps/reports/static/january_2008_style/blue_colors.ini new file mode 100644 index 00000000000..bdb19687358 --- /dev/null +++ b/lib/galaxy/webapps/reports/static/january_2008_style/blue_colors.ini @@ -0,0 +1,62 @@ +base_text=#303030 +base_bg_top=#FFFFFF +base_bg_bottom=#FFFFFF +link_text=#303030 +header_text=#023858 +menu_bg_top=#DADFEF +menu_bg_over=#C1C9E5 +menu_bg_hatch=- +#menu_bg_hatch=#AAAAFF +# Forms +report_title_bg_top=#ebd9b2 +report_title_bg_bottom=#d2c099 +report_title_bg_hatch=- +report_border=#d8b365 +report_body_bg=#FFFFFF +report_body_bg_top=#FFFFFF +report_body_bg_bottom=#FFFFFF +odd_row_bg=#FFFF99 +# Messages +error_message_border=#AA6666 +error_message_bg=#FFCCCC +warn_message_border=#AAAA66 +warn_message_bg=#FFFFCC +done_message_border=#66AA66 +done_message_bg=#CCFFCC +info_message_border=#6666AA +info_message_bg=#CCCCFF +# Tables +table_header_bg=#ebd9b2 +table_row_bg=white +table_border=#d8b365 +# Footers +footer_bg=#023858 +footer_title_bg=#023858 +footer_title_hatch=#000000 +# History +history_error_border=#AA6666 +history_error_bg=#FFCCCC +history_running_border=#AAAA66 +history_running_bg=#FFFFCC +history_ok_border=#66AA66 +history_ok_bg=#CCFFCC +history_queued_border=#888888 +history_queued_bg=#EEEEEE +peek_table_header=#023858 +# Masthead +masthead_bg=#2C3143 +masthead_text=#eeeeee +masthead_bg_hatch=- +masthead_link=#eeeeee +# ---- Layout ----------------------------------------------------------------- +# Overall background color (including space between panels) +layout_bg=#eee +# Line underneath masthead +layout_masthead_border=#444 +# Borders around panels +layout_border=#999 +# Hover color when mouse over drag bars (panel resize) +layout_hover=#AAAAEE +# Gradient for the panel title backgrounds +panel_header_bg_top=#f5f5f5 +panel_header_bg_bottom=#cccccc \ No newline at end of file diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/gradient.py b/lib/galaxy/webapps/reports/static/january_2008_style/gradient.py new file mode 100755 index 00000000000..7e8403b1b30 --- /dev/null +++ b/lib/galaxy/webapps/reports/static/january_2008_style/gradient.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python2.4 + +""" +usage: %prog width height bg_color hatch_color [color alpha stop_pos] + +""" + +from __future__ import division + +import sys +import cairo + +def parse_css_color( color ): + if color.startswith( '#' ): + color = color[1:] + if len( color ) == 3: + r = int( color[0], 16 ) + g = int( color[1], 16 ) + b = int( color[2], 16 ) + elif len( color ) == 6: + r = int( color[0:2], 16 ) + g = int( color[2:4], 16 ) + b = int( color[4:6], 16 ) + else: + raise Exception( "Color should be 3 hex numbers" ) + return r/256, g/256, b/256 + +def gradient( width, height, args ): + pat = cairo.LinearGradient(0.0, 0.0, 0.0, height) + while len( args ) > 2: + col = parse_css_color( args[0] ) + alpha = float( args[1]) + pos = float( args[2] ) + pat.add_color_stop_rgba( pos, col[0], col[1], col[2], alpha ) + args = args[3:] + return pat + +def hatch( width, height, color ): + im_surf = cairo.ImageSurface( cairo.FORMAT_ARGB32, width, width ) + c = cairo.Context( im_surf ) + c.set_source_rgb ( *color ) + c.set_line_width( 0.75 ) + for i in range( 0, 2*max(height,width), 3 ): + c.move_to ( 0-10, i+10 ) + c.line_to ( width+10, i - width - 10 ) + c.stroke() + pat = cairo.SurfacePattern( im_surf ) + pat.set_extend (cairo.EXTEND_REPEAT) + return pat + +width = int( sys.argv[1] ) +height = int( sys.argv[2] ) + +surface = cairo.ImageSurface( cairo.FORMAT_ARGB32, width, height ) +c = cairo.Context( surface ) + +c.rectangle(0,0,width,height) +c.set_source_rgb( *parse_css_color( sys.argv[3] ) ) +c.fill() + +if sys.argv[4] != "-": + c.rectangle (0, 0, width, height) + c.set_source( hatch( width, height, parse_css_color( sys.argv[4] ) ) ) + c.fill() + +pat = cairo.LinearGradient(0.0, 0.0, 0.0, height) +pat.add_color_stop_rgba( 0, 1, 1, 1, 0 ) +pat.add_color_stop_rgba( 1, 1, 1, 1, 1 ) +c.rectangle (0, 0, width, height) +c.set_source( gradient( width, height, sys.argv[5:] ) ) +c.fill() + +surface.write_to_png( "/dev/stdout" ) diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/make_style.py b/lib/galaxy/webapps/reports/static/january_2008_style/make_style.py new file mode 100755 index 00000000000..0a764900162 --- /dev/null +++ b/lib/galaxy/webapps/reports/static/january_2008_style/make_style.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python2.4 + +import pkg_resources; +pkg_resources.require( "Cheetah" ) + +import sys +from Cheetah.Template import Template +import string +from subprocess import Popen, PIPE +import os.path + +def run( cmd ): + return Popen( cmd, stdout=PIPE).communicate()[0] + +templates = [ ( "masthead.css.tmpl", "masthead.css"), + ( "base.css.tmpl", "base.css" ) ] + +images = [ + ( "./gradient.py 9 1000 $base_bg_top - $base_bg_bottom 0 0 $base_bg_bottom 1 1", "base_bg.png" ), + ( "./gradient.py 9 50 $masthead_bg $masthead_bg_hatch", "masthead_bg.png" ), + ( "./gradient.py 9 30 $footer_title_bg $footer_title_hatch 000000 0 0.5 000000 1 1", "footer_title_bg.png" ) + ] + +vars, out_dir = sys.argv[1:] + +context = dict() +for line in open( vars ): + line = line.rstrip( '\r\n' ) + if line and not line.startswith( '#' ): + key, value = line.split( '=' ) + if value.startswith( '"' ) and value.endswith( '"' ): + value = value[1:-1] + context[key] = value + +for input, output in templates: + print input ,"->", output + open( os.path.join( out_dir, output ), "w" ).write( str( Template( file=input, searchList=[context] ) ) ) + +for rule, output in images: + t = string.Template( rule ).substitute( context ) + print t, "->", output + open( os.path.join( out_dir, output ), "w" ).write( run( t.split() ) ) + diff --git a/lib/galaxy/webapps/reports/static/january_2008_style/masthead.css.tmpl b/lib/galaxy/webapps/reports/static/january_2008_style/masthead.css.tmpl new file mode 100644 index 00000000000..0c759f2aba6 --- /dev/null +++ b/lib/galaxy/webapps/reports/static/january_2008_style/masthead.css.tmpl @@ -0,0 +1,26 @@ +body +{ + background: $masthead_bg url(masthead_bg.png) bottom; + color: $masthead_text; + padding: 0; + border: 0; + margin: 3px; + margin-right: 5px; + margin-left: 5px; +} + +div.pageTitle +{ + font-size: 175%; + font-weight: bold; +} + +div.pageTitle a:link, div.pageTitle a:visited, div.pageTitle a:active, div.pageTitle a:hover +{ + text-decoration: none; +} + +a:link, a:visited, a:active +{ + color: $masthead_link; +} diff --git a/lib/galaxy/webapps/reports/templates/disk_usage.tmpl b/lib/galaxy/webapps/reports/templates/disk_usage.tmpl new file mode 100644 index 00000000000..156626dc374 --- /dev/null +++ b/lib/galaxy/webapps/reports/templates/disk_usage.tmpl @@ -0,0 +1,56 @@ + + + + Galaxy Administration + + + + +

System -> Disk Usage

+ + + + + + + + + + + + + + + + + #set $dlen = len( $datasets ) + #if $dlen == 0: + + #else: + + + + + + + + + #set ctr = 0 + #for $dataset in $datasets + #if $dlen > 2 and ctr % 2 == 1: + + #else: + dataset_$dataset[0] dat + + + + + + #set ctr+=1 + #end for + #end if +
Disk Usage for $file_path
File SystemDisk SizeUsedAvailablePercent Used
$disk_usage[0]$disk_usage[1]$disk_usage[2]$disk_usage[3]$disk_usage[4]
There are no datasets larger than $file_size_str
$dlen largest datasets over $file_size_str
FileCreatedHistory IDDeletedSize on Disk
$dataset[1]$dataset[2]$dataset[3]$dataset[4]
+ + diff --git a/lib/galaxy/webapps/reports/templates/index.tmpl b/lib/galaxy/webapps/reports/templates/index.tmpl new file mode 100644 index 00000000000..81fd3765dd2 --- /dev/null +++ b/lib/galaxy/webapps/reports/templates/index.tmpl @@ -0,0 +1,12 @@ + + + + Galaxy Reports Main + + + + + + + + diff --git a/lib/galaxy/webapps/reports/templates/main_frame.tmpl b/lib/galaxy/webapps/reports/templates/main_frame.tmpl new file mode 100644 index 00000000000..cedf54fbad2 --- /dev/null +++ b/lib/galaxy/webapps/reports/templates/main_frame.tmpl @@ -0,0 +1,16 @@ + + + + Galaxy Reports + + + +
+

Galaxy Reports

+ + + +
System
+
+ + diff --git a/lib/galaxy/webapps/reports/templates/masthead.tmpl b/lib/galaxy/webapps/reports/templates/masthead.tmpl new file mode 100644 index 00000000000..0b786290f23 --- /dev/null +++ b/lib/galaxy/webapps/reports/templates/masthead.tmpl @@ -0,0 +1,49 @@ + + +#if $brand is not None: + #set $brand = "/%s" % $brand +#end if + +#if $wiki_url is None: + #set $wiki_url = "http://g2.trac.bx.psu.edu/" +#end if + +#if $bugs_email is None: + #set $bugs_email = "mailto:galaxy-bugs@bx.psu.edu" +#end if + +#if $blog_url is None: + #set $blog_url = "http://g2.trac.bx.psu.edu/blog" +#end if + +#if $screencasts_url is None: + #set $screencasts_url = "http://g2.trac.bx.psu.edu/wiki/ScreenCasts" +#end if + + + + Galaxy Reports + + + + + + + + + + + +
+ +
Galaxy$brand
+ Info: report bugs + | wiki + | screencasts + | blog +     + Galaxy Reports Home +   +
+ + diff --git a/reports_wsgi.ini.sample b/reports_wsgi.ini.sample new file mode 100644 index 00000000000..d70f8de417e --- /dev/null +++ b/reports_wsgi.ini.sample @@ -0,0 +1,76 @@ +# ---- HTTP Server ---------------------------------------------------------- + +[server:main] + +use = egg:Paste#http +port = 8088 +host = 127.0.0.1 +use_threadpool = true +threadpool_workers = 10 + +# ---- Galaxy Webapps Report Interface ------------------------------------------------- + +[app:main] + +# Specifies the factory for the universe WSGI application +paste.app_factory = galaxy.webapps.reports.buildapp:app_factory +log_level = DEBUG + +# Database connection +#database_file = database/universe.sqlite +# You may use a SQLAlchemy connection string to specify an external database instead +database_connection = postgres:///galaxy_test + +# Where dataset files are saved +file_path = database/files +# Temporary storage for additional datasets, this should be shared through the cluster +new_file_path = database/tmp + +# Where templates are stored +template_path = lib/galaxy/webapps/reports/templates + +# Session support (beaker) +use_beaker_session = True +session_type = memory +session_data_dir = %(here)s/database/beaker_sessions +session_key = galaxysessions +session_secret = changethisinproduction + +# Configuration for debugging middleware +debug = true +use_lint = false + +# NEVER enable this on a public site (even test or QA) +# use_interactive = true + +# Admin Password +admin_pass = "galaxy" + +# path to sendmail +sendmail_path = /usr/sbin/sendmail + +# Address to join mailing list +mailing_join_addr = galaxy-user-join@bx.psu.edu + +# Write thread status periodically to 'heartbeat.log' (careful, uses disk space rapidly!) +## use_heartbeat = True + +# Profiling middleware (cProfile based) +## use_profile = True + +# Mail +smtp_server = coltrane.bx.psu.edu +error_email_to = galaxy_bugs@bx.psu.edu + +# Use the new iframe / javascript based layout +use_new_layout = true + +# Serving static files (needed if running standalone) +static_enabled = True +static_cache_time = 360 +static_dir = lib/galaxy/webapps/reports/static +static_images_dir = lib/galaxy/webapps/reports/static/images +static_favicon_dir = lib/galaxy/webapps/reports/static/favicon.ico +static_scripts_dir = lib/galaxy/webapps/reports/static/scripts +static_style_dir = lib/galaxy/webapps/reports/static/january_2008_style/blue + diff --git a/run_reports.sh b/run_reports.sh new file mode 100644 index 00000000000..811765310d6 --- /dev/null +++ b/run_reports.sh @@ -0,0 +1,5 @@ +#!/bin/sh + +source setup_paths.sh + +python2.4 ./scripts/paster.py serve reports_wsgi.ini $@ diff --git a/scripts/update_dataset_size.py b/scripts/update_dataset_size.py index d2db381a5d3..6a6bd0f7a2e 100644 --- a/scripts/update_dataset_size.py +++ b/scripts/update_dataset_size.py @@ -20,10 +20,9 @@ def main(): #Step through Datasets, determining size on disk for each. print "Determining the size of each dataset..." for row in app.model.Dataset.table.select().execute(): - deleted = app.model.Dataset.get( row.id ).deleted purged = app.model.Dataset.get( row.id ).purged file_size = app.model.Dataset.get( row.id ).file_size - if file_size is None and not deleted and not purged: + if file_size is None and not purged: size_on_disk = app.model.Dataset.get( row.id ).get_size() print "Updating Dataset.%d with file_size: %d" %( row.id, size_on_disk ) app.model.Dataset.table.update( app.model.Dataset.table.c.id == row.id ).execute( file_size=size_on_disk ) diff --git a/tools/encode/random_intervals.py b/tools/encode/random_intervals.py deleted file mode 100644 index 1f4495243e0..00000000000 --- a/tools/encode/random_intervals.py +++ /dev/null @@ -1,222 +0,0 @@ -#!/usr/bin/env python2.4 -#%prog bounding_region_file mask_intervals_file intervals_to_mimic_file out_file mask_chr mask_start mask_end interval_chr interval_start interval_end - - -from __future__ import division - -import pkg_resources -pkg_resources.require( "bx-python" ) - -import sys, random -import bisect -from bx.bitset import * - -def stop_err( msg ): - sys.stderr.write( msg ) - sys.exit() - -def throw_random_2( lengths, mask ): - """ - Version of throw using gap lists (like Hiram's randomPlacement). This - is not ready yet!!! - """ - # Projected version for throwing - bits = BitSet( mask.size ) - # Use mask to find the gaps - gaps = [] - start = end = 0 - while 1: - start = mask.next_clear( end ) - if start == mask.size: break - end = mask.next_set( start ) - gaps.append( ( end-start, start, end ) ) - # Sort (long regions first ) - gaps.sort() - gaps.reverse() - # And throw - for length in lengths: - max_candidate = 0 - candidate_bases = 0 - for gap in gaps: - if gap[0] >= length: - max_candidate += 1 - candidate_bases += ( gap[0] - length ) - else: - break - if max_candidate == 0: - raise MaxtriesException( "No gap can fit region of length %d" % length ) - # Select start position - s = random.randrange( candidate_bases ) - # Map back to region - chosen_index = 0 - for gap in gaps: - gap_length, gap_start, gap_end = gap - if s > ( gap_length - length ): - s -= ( gap_length - length ) - chosen_index += 1 - else: - break - # Remove the chosen gap and split - assert ( gap_length, gap_start, gap_end ) == gaps.pop( chosen_index ) - # gap_length, gap_start, gap_end = gaps.pop( chosen_index ) - assert s >= 0 - assert gap_start + s + length <= gap_end, "Expected: %d + %d + %d == %d <= %d" % ( gap_start, s, length, gap_start + s + length, gap_end ) - gaps.reverse() - if s > 0: - bisect.insort( gaps, ( s, gap_start, gap_start + s ) ) - if s + length < gap_length: - bisect.insort( gaps, ( gap_length - ( s + length ), gap_start + s + length, gap_end) ) - gaps.reverse() - # And finally set the bits - assert bits[gap_start + s] == 0 - assert bits.next_set( gap_start + s, gap_start+s+length ) == gap_start+s+length - assert( gap_start + s >= 0 and gap_start + s + length <= bits.size ), "Bad interval %d %d %d %d %d" % ( gap_start, s, length, gap_start + s + length, bits.size ) - bits.set_range( gap_start + s, length ) - assert bits.count_range( 0, bits.size ) == sum( lengths ) - return bits - -def overlapping_in_bed( fname, r_chr, r_start, r_stop, chr_col, start_col, end_col ): - rval = [] - for line in open( fname ): - if line.startswith( "#" ) or line.startswith( "track" ): - continue - fields = line.split() - try: - chr, start, stop = fields[chr_col], int( fields[start_col] ), int( fields[end_col] ) - if chr == r_chr and start < r_stop and stop >= r_start: - rval.append( ( chr, max( start, r_start ), min( stop, r_stop ) ) ) - except: - continue - return rval - -def as_bits( region_start, region_length, intervals ): - bits = BitSet( region_length ) - for chr, start, stop in intervals: - bits.set_range( start - region_start, stop - start ) - return bits - -def bit_clone( bits ): - new = BitSet( bits.size ) - new.ior( bits ) - return new - -def count_overlap( bits1, bits2 ): - b = BitSet( bits1.size ) - b |= bits1 - b &= bits2 - return b.count_range( 0, b.size ) - -def interval_lengths( bits ): - end = 0 - while 1: - start = bits.next_set( end ) - if start == bits.size: break - end = bits.next_clear( start ) - yield end - start - -def main(): - #region_fname = sys.argv[1] - region_uid = sys.argv[1] - mask_fname = sys.argv[2] - intervals_fname = sys.argv[3] - out_fname = sys.argv[4] - try: - mask_chr = int( sys.argv[5] ) - 1 - except: - stop_err( "'%s' is an invalid chrom column for 'Intervals to Mask' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[5] ) ) - try: - mask_start = int( sys.argv[6] ) - 1 - except: - stop_err( "'%s' is an invalid start column for 'Intervals to Mask' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[6] ) ) - try: - mask_end = int( sys.argv[7] ) - 1 - except: - stop_err( "'%s' is an invalid end column for 'Intervals to Mask' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[7] ) ) - try: - interval_chr = int( sys.argv[8] ) - 1 - except: - stop_err( "'%s' is an invalid chrom column for 'File to Mimick' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[8] ) ) - try: - interval_start = int( sys.argv[9] ) - 1 - except: - stop_err( "'%s' is an invalid start column for 'File to Mimick' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[9] ) ) - try: - interval_end = int( sys.argv[10] ) - 1 - except: - stop_err( "'%s' is an invalid end column for 'File to Mimick' dataset, click the pencil icon in the history item to edit column settings." % str( sys.argv[10] ) ) - use_mask = sys.argv[11] - available_regions = {} - loc_file = "/depot/data2/galaxy/regions.loc" - - for i, line in enumerate( open( loc_file ) ): - line = line.rstrip( '\r\n' ) - if line and not line.startswith( '#' ): - fields = line.split( '\t' ) - #read each line, if not enough fields, go to next line - try: - build = fields[0] - uid = fields[1] - description = fields[2] - filepath = fields[3] - available_regions[uid] = filepath - except: - continue - - if region_uid not in available_regions: - stop_err( "Invalid region '%s' selected." % region_uid ) - - region_fname = available_regions[region_uid] - try: - out_file = open ( out_fname, "w" ) - except: - stop_err( "Error opening output file '%s'." % out_fname ) - - skipped_lines = 0 - first_invalid_line = 0 - invalid_line = '' - - for line_count, line in enumerate( file( region_fname ) ): - if line and not line.startswith( '#' ): - try: - # Load lengths for all intervals overlapping region - fields = line.split() - r_chr, r_start, r_stop = fields[0], int( fields[1] ), int( fields[2] ) - r_length = r_stop - r_start - # Load the mask - if use_mask == "no_mask": - mask = [] - else: - mask = overlapping_in_bed( mask_fname, r_chr, r_start, r_stop, mask_chr, mask_start, mask_end ) - bits_mask = as_bits( r_start, r_length, mask ) - bits_not_masked = bit_clone( bits_mask ) - bits_not_masked.invert() - # Load the first set - intervals1 = overlapping_in_bed( intervals_fname, r_chr, r_start, r_stop, interval_chr, interval_start, interval_end ) - bits1 = as_bits( r_start, r_length, intervals1 ) - # Intersect it with the mask - bits1.iand( bits_not_masked ) - # Sanity checks - if count_overlap( bits1, bits_mask ) != 0: - stop_err( "Overlap problem, bits: %s, mask: %s" %( str( bits1 ), str( bits_mask ) ) ) - chrom = r_chr - # For each data set - lengths1 = list( interval_lengths( bits1 ) ) - random1 = throw_random_2( lengths1, bits_mask ) - end =0 - while 1: - start = random1.next_set( end ) - if start == random1.size: - break - end = random1.next_clear( start ) - print >>out_file, "%s\t%d\t%d" % ( chrom, start, end ) - except: - skipped_lines += 1 - if not first_invalid_line: - first_invalid_line = line_count - invalid_line = line - continue - out_file.close() - if skipped_lines: - print "Skipped %d invalid lines starting at line # %d: %s" % ( skipped_lines, first_invalid_line, invalid_line ) - -if __name__ == "__main__": main()