diff --git a/lib/galaxy/web/controllers/admin.py b/lib/galaxy/web/controllers/admin.py index e00325885f8..054aa4698de 100644 --- a/lib/galaxy/web/controllers/admin.py +++ b/lib/galaxy/web/controllers/admin.py @@ -1,5 +1,5 @@ -from galaxy.web.base.controller import * +from galaxy.web.base.controller import * import logging, sets, time log = logging.getLogger( __name__ ) @@ -20,51 +20,57 @@ class Admin( BaseController ): msg = self.abandon( **kwd ) users, data = [], [] - for row in trans.model.User.table.select().execute(): - users.append(row) - for row in trans.model.Dataset.table.select().execute(): - data.append(row) - + ut = trans.model.User.table + dt = trans.model.Dataset.table + for row in ut.select().execute(): + users.append( row ) + for row in dt.select().execute(): + data.append( row ) qsize = self.app.job_queue.queue.qsize() return trans.fill_template('admin_main.tmpl', toolbox=self.app.toolbox, users=users, data=data, qsize=qsize,msg=msg) - def abandon( self, **kwd ): + """ + 'Abandons' userless histories older than the specified number of days by setting the 'deleted' + column to 't' in the History table for each History.id whose History.user_id column is null. + A list of each of the affected history records is generated during the process. This list is + then used to find all undeleted datasets that are associated with these histories. Each of + these datasets is then deleted by setting the Dataset.deleted column to 't'. Nothing is removed + from the file system. This function should be executed in preparation for purging datasets using + the purge() function below. + """ params = util.Params(kwd) msg = '' - if params.days: + if params.abandon_days: if params.passwd==self.app.config.admin_pass: days = int(params.abandon_days) - histories_no_users = [] history_count = 0 - for row in self.app.model.History.table.select().execute(): - try: - int(row.user_id) - except: - history = self.app.model.History.get(row.id) - now = time.time() - last = time.mktime( time.strptime( history.update_time.strftime('%a %b %d %H:%M:%S %Y') )) - diff = (now - last) /3600/24 # days - if diff>days: - histories_no_users.append(row.id) - if history.deleted: continue #we don't add to history delete count, but we do want to be able to make sure datasets associated with deleted histories are really deleted + ht = self.app.model.History.table + for row in ht.select( ht.c.user_id==None ).execute(): + now = time.time() + last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) ) + diff = (now-last)/3600/24 # days + if diff > days: + histories_no_users.append( row.id ) + if not row.deleted: + log.warn( "Updating history table, setting id %s to deleted" %str( row.id ) ) + history = self.app.model.History.get( row.id ) history.deleted = True history_count += 1 - dataset_count = 0 - for row in self.app.model.Dataset.table.select().execute(): + dt = self.app.model.Dataset.table + for row in dt.select( dt.c.deleted=='f' ).execute(): if row.history_id in histories_no_users: - #delete dataset - data = self.app.model.Dataset.get(row.id) - if data.deleted: continue + log.warn( "Updating dataset table, setting id %s to deleted" %str( row.id ) ) + data = self.app.model.Dataset.get( row.id ) data.deleted = True dataset_count += 1 try: self.app.model.flush() except: pass - msg = 'deleted %d abandoned histories with %d total datasets' % ( history_count, dataset_count ) + msg = 'Deleted %d histories (including a total of %d datasets )' % ( history_count, dataset_count ) else: msg = 'Invalid password' return msg @@ -77,57 +83,28 @@ class Admin( BaseController ): if params.passwd==self.app.config.admin_pass: days = int(params.purge_days) count = 0 - now = time.time() - for row in self.app.model.Dataset.table.select().execute(): - data = self.app.model.Dataset.get(row.id) - if data.deleted and not data.purged: - last = time.mktime( time.strptime( data.update_time.strftime('%a %b %d %H:%M:%S %Y') )) - diff = (now - last) /3600/24 # days - if diff>days: - data.purge() - count += 1 - try: - self.app.model.flush() - except: - pass - msg = 'Purged %d datasets' % count - else: - msg = 'Invalid password' - return msg - - def delete( self, **kwd ): - params = util.Params(kwd) - msg = '' - if params.days: - if params.passwd==self.app.config.admin_pass: - days = int(params.days) - values = [] - for row in self.app.model.Dataset.table.select().execute(): - values.append(self.app.model.Dataset.get(row.id)) - for row in self.app.model.History.table.select().execute(): - values.append(self.app.model.History.get(row.id)) - #for row in self.app.model.User.table.select().execute(): - # values.append(self.app.model.User.get(row.id)) - count = 0 - for value in values: - now = time.time() - last = time.mktime( time.strptime( value.update_time.strftime('%a %b %d %H:%M:%S %Y') )) - diff = (now - last) /3600/24 # days - if diff>days: - #value.delete() - value.deleted = True + now = time.time() + dt = self.app.model.Dataset.table + for row in dt.select( ( dt.c.purged=='f' ) & ( dt.c.deleted=='t' ) ).execute(): + last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) ) + diff = (now-last)/3600/24 # days + if diff > days: + log.warn( "Purging dataset id %s" %str( row.id ) ) + data = app.model.Dataset.get( row.id ) + data.purge() + data.flush() count += 1 try: self.app.model.flush() except: pass - msg = 'Deleted %d objects' % count + msg = 'purged %d datasets' % count else: msg = 'Invalid password' return msg def tool_reload( self, **kwd ): - params = util.Params(kwd) + params = util.Params( kwd ) if params.passwd==self.app.config.admin_pass: tool_id = params.tool_id self.app.toolbox.reload( tool_id ) diff --git a/lib/galaxy/webapps/reports/base/controller.py b/lib/galaxy/webapps/reports/base/controller.py index 4b77b886ad4..956c97d41bf 100644 --- a/lib/galaxy/webapps/reports/base/controller.py +++ b/lib/galaxy/webapps/reports/base/controller.py @@ -1,27 +1,15 @@ """Contains functionality needed in every webapp interface""" import os, time, logging - # Pieces of Galaxy to make global in every controller -#from galaxy import config, tools, web, model, util from galaxy import web - from Cheetah.Template import Template log = logging.getLogger( __name__ ) class BaseController( object ): """Base class for Galaxy webapp application controllers.""" - beta = False - def __init__( self, app ): """Initialize an interface for application 'app'""" - self.app = app - - #def get_toolbox(self): - # """Returns the application toolbox""" - # return self.app.toolbox - -#Root = BaseController -"""Deprecated: `BaseController` used to be available under the name `Root`""" \ No newline at end of file + self.app = app \ No newline at end of file diff --git a/lib/galaxy/webapps/reports/controllers/system.py b/lib/galaxy/webapps/reports/controllers/system.py index 1914dfe54d5..7fb466132be 100644 --- a/lib/galaxy/webapps/reports/controllers/system.py +++ b/lib/galaxy/webapps/reports/controllers/system.py @@ -1,15 +1,8 @@ -import sys, sets, string, shutil -import re, socket - -from galaxy import web - -from cgi import escape, FieldStorage -import urllib import operator, os from galaxy.webapps.reports.base.controller import * -import logging, sets, time +import logging log = logging.getLogger( __name__ ) class System( BaseController ): @@ -23,29 +16,24 @@ class System( BaseController ): while True: df_line = df_file.readline() df_line = df_line.strip() - #log.debug("df_line: '%s'" %df_line) if df_line: df_line = df_line.lower() if 'filesystem' in df_line or 'proc' in df_line: continue elif is_sym_link: - #log.debug("We have a symlink...") if ':' in df_line and '/' in df_line: mount = df_line - #log.debug("mount: '%s'" %mount) else: try: disk_size, disk_used, disk_avail, disk_cap_pct, file_system = df_line.split() break except: - #log.debug("In symlink try, df_line: '%s'" %df_line) pass else: try: file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount = df_line.split() break except: - #log.debug("In 2nd try, df_line: '%s'" %df_line) pass else: break # EOF diff --git a/scripts/cleanup_datasets.py b/scripts/cleanup_datasets.py index e9ef05c38fa..70bb61ae3dd 100644 --- a/scripts/cleanup_datasets.py +++ b/scripts/cleanup_datasets.py @@ -1,8 +1,5 @@ #!/usr/bin/env python2.4 #Dan Blankenberg -""" -Allows dataset cleanup. -""" import sys, os, time, ConfigParser from optparse import OptionParser @@ -10,79 +7,91 @@ import galaxy.app def main(): parser = OptionParser() - parser.add_option("-d", "--days", dest="days", action="store", type="int", help="number of days (30)", default=30) - parser.add_option("-a", "--abandon", action="store_true", dest="abandon", default=False, help="abandon old, unowned (userless) histories") - parser.add_option("-p", "--purge", action="store_true", dest="purge", default=False, help="purge old deleted datasets") - (options, args) = parser.parse_args() + parser.add_option( "-d", "--days", dest="days", action="store", type="int", help="number of days (30)", default=30 ) + parser.add_option( "-a", "--abandon", action="store_true", dest="abandon", default=False, help="abandon old, unowned (userless) histories" ) + parser.add_option( "-p", "--purge", action="store_true", dest="purge", default=False, help="purge old deleted datasets" ) + ( options, args ) = parser.parse_args() ini_file = args[0] - if not (options.purge ^ options.abandon): + if not ( options.purge ^ options.abandon ): parser.print_help() sys.exit(0) - conf_parser = ConfigParser.ConfigParser({'here':os.getcwd()}) - conf_parser.read(ini_file) + conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} ) + conf_parser.read( ini_file ) configuration = {} - for key, value in conf_parser.items("app:main"): configuration[key] = value + for key, value in conf_parser.items("app:main"): + configuration[key] = value app = galaxy.app.UniverseApplication( global_conf = ini_file, **configuration ) - msg = None + msg = '' if options.purge: - msg = purge(app, options.days) + msg = purge( app, options.days ) elif options.abandon: - msg = abandon(app, options.days) + msg = abandon( app, options.days ) app.shutdown() if msg: - print - print "Number of days buffered: %i" % options.days - print msg + print "\n%s older than %i days.\n" % ( msg, options.days ) sys.exit(0) def abandon( app, days ): - """ Abandons userless histories older than specified number of days """ + """ + 'Abandons' userless histories older than the specified number of days by setting the 'deleted' + column to 't' in the History table for each History.id whose History.user_id column is null. + A list of each of the affected history records is generated during the process. This list is + then used to find all undeleted datasets that are associated with these histories. Each of + these datasets is then deleted by setting the Dataset.deleted column to 't'. Nothing is removed + from the file system. This function should be executed in preparation for purging datasets using + the purge() function below. + """ histories_no_users = [] history_count = 0 - for row in app.model.History.table.select().execute(): - try: - int(row.user_id) - except: - history = app.model.History.get(row.id) - now = time.time() - last = time.mktime( time.strptime( history.update_time.strftime('%a %b %d %H:%M:%S %Y') )) - diff = (now - last) /3600/24 # days - if diff>days: - histories_no_users.append(row.id) - if history.deleted: continue #we don't add to history delete count, but we do want to be able to make sure datasets associated with deleted histories are really deleted + ht = app.model.History.table + # Generate a list of userless histories, deleting the histories along the way + for row in ht.select( ht.c.user_id==None ).execute(): + now = time.time() + last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) ) + diff = (now-last)/3600/24 # days + if diff > days: + histories_no_users.append( row.id ) + if not row.deleted: + print "Updating history table, setting id %s to deleted" %str( row.id ) + history = app.model.History.get( row.id ) history.deleted = True history_count += 1 - dataset_count = 0 - for row in app.model.Dataset.table.select().execute(): + dt = app.model.Dataset.table + # Delete all datasets associated with previously deleted userless histories + for row in dt.select( dt.c.deleted=='f' ).execute(): if row.history_id in histories_no_users: - #delete dataset - data = app.model.Dataset.get(row.id) - if data.deleted: continue + print "Updating dataset table, setting id %s to deleted" %str( row.id ) + data = app.model.Dataset.get( row.id ) data.deleted = True dataset_count += 1 try: app.model.flush() except: pass - msg = 'deleted %d abandoned histories with %d total datasets' % ( history_count, dataset_count ) + msg = 'Deleted %d histories (including a total of %d datasets )' % ( history_count, dataset_count ) return msg def purge( app, days ): - """ Purges deleted datasets older than specified number of days """ + """ + Purges deleted datasets older than specified number of days by executing the Dataset.purge() function. + This will update the Dataset table, setting Dataset.deleted to 't', Dataset.purged to 't', and + Dataset.file_size to 0. The dataset file will then be removed from the file system. + """ count = 0 - now = time.time() - for row in list(app.model.Dataset.table.select().execute()): - data = app.model.Dataset.get(row.id) - if data.deleted and not data.purged: - last = time.mktime( time.strptime( data.update_time.strftime('%a %b %d %H:%M:%S %Y') )) - diff = (now - last) /3600/24 # days - if diff>days: - data.purge() - data.flush() - count += 1 + now = time.time() + dt = app.model.Dataset.table + for row in dt.select( ( dt.c.purged=='f' ) & ( dt.c.deleted=='t' ) ).execute(): + last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) ) + diff = (now-last)/3600/24 # days + if diff > days: + print "Purging dataset id %s" %str( row.id ) + data = app.model.Dataset.get( row.id ) + data.purge() + data.flush() + count += 1 try: app.model.flush() except: @@ -90,6 +99,5 @@ def purge( app, days ): msg = 'Purged %d datasets' % count return msg - if __name__ == "__main__": main() \ No newline at end of file diff --git a/templates/admin_main.tmpl b/templates/admin_main.tmpl index b902fcc5070..f28caa5241d 100644 --- a/templates/admin_main.tmpl +++ b/templates/admin_main.tmpl @@ -1,61 +1,56 @@ - - -Galaxy Administration - - - - - - - - - -
-

Galaxy Administration

- #if $msg: -

$msg

- #end if -

- #set $ulen = len($users) - #set $dlen = len($data) -

    -
  • $ulen users
  • -
  • $dlen data entries
  • -
  • $qsize jobs in the queue
  • -
-

-
-
-

Admin password:

-

- Purge deleted datasets older than days. - -

-#* -

- Delete objects older than days. - -

-*# -

- Delete abandoned histories older than days. - -

-

- Reload tool: - -

-
-
- - + + Galaxy Administration + + + + + + + + + + + +
+

Galaxy Administration

+ #if $msg: +

$msg

+ #end if +

+ #set $ulen = len($users) + #set $dlen = len($data) +

    +
  • $ulen users
  • +
  • $dlen data entries
  • +
  • $qsize jobs in the queue
  • +
+

+
+
+

Admin password:

+

+ Purge deleted datasets older than days. + +

+

+ Delete abandoned histories older than days. + +

+

+ Reload tool: + + +

+
+
+ diff --git a/universe_wsgi.ini.sample b/universe_wsgi.ini.sample index 75e17cf85d7..47ded80faa2 100644 --- a/universe_wsgi.ini.sample +++ b/universe_wsgi.ini.sample @@ -35,6 +35,10 @@ job_queue_cleanup_interval = 30 database_file = database/universe.sqlite # You may use a SQLAlchemy connection string to specify an external database instead ## database_connection = postgres:///galaxy_test +## database_engine_option_echo = true +## database_engine_option_echo_pool = true +## database_engine_option_pool_size = 10 +## database_engine_option_max_overflow = 20 # Where dataset files are saved file_path = database/files