mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-21 13:50:20 +08:00
Cleaned up code for cleaning up histories and datasets, should now be faster and require much less memory. Other miscellanwoud code cleanup. Added sample database config options to config file.
This commit is contained in:
Greg Von Kuster
@@ -1,5 +1,5 @@
|
||||
from galaxy.web.base.controller import *
|
||||
|
||||
from galaxy.web.base.controller import *
|
||||
import logging, sets, time
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
@@ -20,51 +20,57 @@ class Admin( BaseController ):
|
||||
msg = self.abandon( **kwd )
|
||||
|
||||
users, data = [], []
|
||||
for row in trans.model.User.table.select().execute():
|
||||
users.append(row)
|
||||
for row in trans.model.Dataset.table.select().execute():
|
||||
data.append(row)
|
||||
|
||||
ut = trans.model.User.table
|
||||
dt = trans.model.Dataset.table
|
||||
for row in ut.select().execute():
|
||||
users.append( row )
|
||||
for row in dt.select().execute():
|
||||
data.append( row )
|
||||
qsize = self.app.job_queue.queue.qsize()
|
||||
return trans.fill_template('admin_main.tmpl', toolbox=self.app.toolbox, users=users, data=data, qsize=qsize,msg=msg)
|
||||
|
||||
|
||||
def abandon( self, **kwd ):
|
||||
"""
|
||||
'Abandons' userless histories older than the specified number of days by setting the 'deleted'
|
||||
column to 't' in the History table for each History.id whose History.user_id column is null.
|
||||
A list of each of the affected history records is generated during the process. This list is
|
||||
then used to find all undeleted datasets that are associated with these histories. Each of
|
||||
these datasets is then deleted by setting the Dataset.deleted column to 't'. Nothing is removed
|
||||
from the file system. This function should be executed in preparation for purging datasets using
|
||||
the purge() function below.
|
||||
"""
|
||||
params = util.Params(kwd)
|
||||
msg = ''
|
||||
if params.days:
|
||||
if params.abandon_days:
|
||||
if params.passwd==self.app.config.admin_pass:
|
||||
days = int(params.abandon_days)
|
||||
|
||||
histories_no_users = []
|
||||
history_count = 0
|
||||
for row in self.app.model.History.table.select().execute():
|
||||
try:
|
||||
int(row.user_id)
|
||||
except:
|
||||
history = self.app.model.History.get(row.id)
|
||||
now = time.time()
|
||||
last = time.mktime( time.strptime( history.update_time.strftime('%a %b %d %H:%M:%S %Y') ))
|
||||
diff = (now - last) /3600/24 # days
|
||||
if diff>days:
|
||||
histories_no_users.append(row.id)
|
||||
if history.deleted: continue #we don't add to history delete count, but we do want to be able to make sure datasets associated with deleted histories are really deleted
|
||||
ht = self.app.model.History.table
|
||||
for row in ht.select( ht.c.user_id==None ).execute():
|
||||
now = time.time()
|
||||
last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) )
|
||||
diff = (now-last)/3600/24 # days
|
||||
if diff > days:
|
||||
histories_no_users.append( row.id )
|
||||
if not row.deleted:
|
||||
log.warn( "Updating history table, setting id %s to deleted" %str( row.id ) )
|
||||
history = self.app.model.History.get( row.id )
|
||||
history.deleted = True
|
||||
history_count += 1
|
||||
|
||||
dataset_count = 0
|
||||
for row in self.app.model.Dataset.table.select().execute():
|
||||
dt = self.app.model.Dataset.table
|
||||
for row in dt.select( dt.c.deleted=='f' ).execute():
|
||||
if row.history_id in histories_no_users:
|
||||
#delete dataset
|
||||
data = self.app.model.Dataset.get(row.id)
|
||||
if data.deleted: continue
|
||||
log.warn( "Updating dataset table, setting id %s to deleted" %str( row.id ) )
|
||||
data = self.app.model.Dataset.get( row.id )
|
||||
data.deleted = True
|
||||
dataset_count += 1
|
||||
try:
|
||||
self.app.model.flush()
|
||||
except:
|
||||
pass
|
||||
msg = 'deleted %d abandoned histories with %d total datasets' % ( history_count, dataset_count )
|
||||
msg = 'Deleted %d histories (including a total of %d datasets )' % ( history_count, dataset_count )
|
||||
else:
|
||||
msg = 'Invalid password'
|
||||
return msg
|
||||
@@ -77,57 +83,28 @@ class Admin( BaseController ):
|
||||
if params.passwd==self.app.config.admin_pass:
|
||||
days = int(params.purge_days)
|
||||
count = 0
|
||||
now = time.time()
|
||||
for row in self.app.model.Dataset.table.select().execute():
|
||||
data = self.app.model.Dataset.get(row.id)
|
||||
if data.deleted and not data.purged:
|
||||
last = time.mktime( time.strptime( data.update_time.strftime('%a %b %d %H:%M:%S %Y') ))
|
||||
diff = (now - last) /3600/24 # days
|
||||
if diff>days:
|
||||
data.purge()
|
||||
count += 1
|
||||
try:
|
||||
self.app.model.flush()
|
||||
except:
|
||||
pass
|
||||
msg = 'Purged %d datasets' % count
|
||||
else:
|
||||
msg = 'Invalid password'
|
||||
return msg
|
||||
|
||||
def delete( self, **kwd ):
|
||||
params = util.Params(kwd)
|
||||
msg = ''
|
||||
if params.days:
|
||||
if params.passwd==self.app.config.admin_pass:
|
||||
days = int(params.days)
|
||||
values = []
|
||||
for row in self.app.model.Dataset.table.select().execute():
|
||||
values.append(self.app.model.Dataset.get(row.id))
|
||||
for row in self.app.model.History.table.select().execute():
|
||||
values.append(self.app.model.History.get(row.id))
|
||||
#for row in self.app.model.User.table.select().execute():
|
||||
# values.append(self.app.model.User.get(row.id))
|
||||
count = 0
|
||||
for value in values:
|
||||
now = time.time()
|
||||
last = time.mktime( time.strptime( value.update_time.strftime('%a %b %d %H:%M:%S %Y') ))
|
||||
diff = (now - last) /3600/24 # days
|
||||
if diff>days:
|
||||
#value.delete()
|
||||
value.deleted = True
|
||||
now = time.time()
|
||||
dt = self.app.model.Dataset.table
|
||||
for row in dt.select( ( dt.c.purged=='f' ) & ( dt.c.deleted=='t' ) ).execute():
|
||||
last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) )
|
||||
diff = (now-last)/3600/24 # days
|
||||
if diff > days:
|
||||
log.warn( "Purging dataset id %s" %str( row.id ) )
|
||||
data = app.model.Dataset.get( row.id )
|
||||
data.purge()
|
||||
data.flush()
|
||||
count += 1
|
||||
try:
|
||||
self.app.model.flush()
|
||||
except:
|
||||
pass
|
||||
msg = 'Deleted %d objects' % count
|
||||
msg = 'purged %d datasets' % count
|
||||
else:
|
||||
msg = 'Invalid password'
|
||||
return msg
|
||||
|
||||
def tool_reload( self, **kwd ):
|
||||
params = util.Params(kwd)
|
||||
params = util.Params( kwd )
|
||||
if params.passwd==self.app.config.admin_pass:
|
||||
tool_id = params.tool_id
|
||||
self.app.toolbox.reload( tool_id )
|
||||
|
||||
@@ -1,27 +1,15 @@
|
||||
"""Contains functionality needed in every webapp interface"""
|
||||
|
||||
import os, time, logging
|
||||
|
||||
# Pieces of Galaxy to make global in every controller
|
||||
#from galaxy import config, tools, web, model, util
|
||||
from galaxy import web
|
||||
|
||||
from Cheetah.Template import Template
|
||||
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
class BaseController( object ):
|
||||
"""Base class for Galaxy webapp application controllers."""
|
||||
|
||||
beta = False
|
||||
|
||||
def __init__( self, app ):
|
||||
"""Initialize an interface for application 'app'"""
|
||||
self.app = app
|
||||
|
||||
#def get_toolbox(self):
|
||||
# """Returns the application toolbox"""
|
||||
# return self.app.toolbox
|
||||
|
||||
#Root = BaseController
|
||||
"""Deprecated: `BaseController` used to be available under the name `Root`"""
|
||||
self.app = app
|
||||
@@ -1,15 +1,8 @@
|
||||
import sys, sets, string, shutil
|
||||
import re, socket
|
||||
|
||||
from galaxy import web
|
||||
|
||||
from cgi import escape, FieldStorage
|
||||
import urllib
|
||||
|
||||
import operator, os
|
||||
from galaxy.webapps.reports.base.controller import *
|
||||
|
||||
import logging, sets, time
|
||||
import logging
|
||||
log = logging.getLogger( __name__ )
|
||||
|
||||
class System( BaseController ):
|
||||
@@ -23,29 +16,24 @@ class System( BaseController ):
|
||||
while True:
|
||||
df_line = df_file.readline()
|
||||
df_line = df_line.strip()
|
||||
#log.debug("df_line: '%s'" %df_line)
|
||||
if df_line:
|
||||
df_line = df_line.lower()
|
||||
if 'filesystem' in df_line or 'proc' in df_line:
|
||||
continue
|
||||
elif is_sym_link:
|
||||
#log.debug("We have a symlink...")
|
||||
if ':' in df_line and '/' in df_line:
|
||||
mount = df_line
|
||||
#log.debug("mount: '%s'" %mount)
|
||||
else:
|
||||
try:
|
||||
disk_size, disk_used, disk_avail, disk_cap_pct, file_system = df_line.split()
|
||||
break
|
||||
except:
|
||||
#log.debug("In symlink try, df_line: '%s'" %df_line)
|
||||
pass
|
||||
else:
|
||||
try:
|
||||
file_system, disk_size, disk_used, disk_avail, disk_cap_pct, mount = df_line.split()
|
||||
break
|
||||
except:
|
||||
#log.debug("In 2nd try, df_line: '%s'" %df_line)
|
||||
pass
|
||||
else:
|
||||
break # EOF
|
||||
|
||||
+55
-47
@@ -1,8 +1,5 @@
|
||||
#!/usr/bin/env python2.4
|
||||
#Dan Blankenberg
|
||||
"""
|
||||
Allows dataset cleanup.
|
||||
"""
|
||||
|
||||
import sys, os, time, ConfigParser
|
||||
from optparse import OptionParser
|
||||
@@ -10,79 +7,91 @@ import galaxy.app
|
||||
|
||||
def main():
|
||||
parser = OptionParser()
|
||||
parser.add_option("-d", "--days", dest="days", action="store", type="int", help="number of days (30)", default=30)
|
||||
parser.add_option("-a", "--abandon", action="store_true", dest="abandon", default=False, help="abandon old, unowned (userless) histories")
|
||||
parser.add_option("-p", "--purge", action="store_true", dest="purge", default=False, help="purge old deleted datasets")
|
||||
(options, args) = parser.parse_args()
|
||||
parser.add_option( "-d", "--days", dest="days", action="store", type="int", help="number of days (30)", default=30 )
|
||||
parser.add_option( "-a", "--abandon", action="store_true", dest="abandon", default=False, help="abandon old, unowned (userless) histories" )
|
||||
parser.add_option( "-p", "--purge", action="store_true", dest="purge", default=False, help="purge old deleted datasets" )
|
||||
( options, args ) = parser.parse_args()
|
||||
ini_file = args[0]
|
||||
|
||||
if not (options.purge ^ options.abandon):
|
||||
if not ( options.purge ^ options.abandon ):
|
||||
parser.print_help()
|
||||
sys.exit(0)
|
||||
|
||||
conf_parser = ConfigParser.ConfigParser({'here':os.getcwd()})
|
||||
conf_parser.read(ini_file)
|
||||
conf_parser = ConfigParser.ConfigParser( {'here':os.getcwd()} )
|
||||
conf_parser.read( ini_file )
|
||||
configuration = {}
|
||||
for key, value in conf_parser.items("app:main"): configuration[key] = value
|
||||
for key, value in conf_parser.items("app:main"):
|
||||
configuration[key] = value
|
||||
app = galaxy.app.UniverseApplication( global_conf = ini_file, **configuration )
|
||||
msg = None
|
||||
msg = ''
|
||||
if options.purge:
|
||||
msg = purge(app, options.days)
|
||||
msg = purge( app, options.days )
|
||||
elif options.abandon:
|
||||
msg = abandon(app, options.days)
|
||||
msg = abandon( app, options.days )
|
||||
app.shutdown()
|
||||
if msg:
|
||||
print
|
||||
print "Number of days buffered: %i" % options.days
|
||||
print msg
|
||||
print "\n%s older than %i days.\n" % ( msg, options.days )
|
||||
sys.exit(0)
|
||||
|
||||
def abandon( app, days ):
|
||||
""" Abandons userless histories older than specified number of days """
|
||||
"""
|
||||
'Abandons' userless histories older than the specified number of days by setting the 'deleted'
|
||||
column to 't' in the History table for each History.id whose History.user_id column is null.
|
||||
A list of each of the affected history records is generated during the process. This list is
|
||||
then used to find all undeleted datasets that are associated with these histories. Each of
|
||||
these datasets is then deleted by setting the Dataset.deleted column to 't'. Nothing is removed
|
||||
from the file system. This function should be executed in preparation for purging datasets using
|
||||
the purge() function below.
|
||||
"""
|
||||
histories_no_users = []
|
||||
history_count = 0
|
||||
for row in app.model.History.table.select().execute():
|
||||
try:
|
||||
int(row.user_id)
|
||||
except:
|
||||
history = app.model.History.get(row.id)
|
||||
now = time.time()
|
||||
last = time.mktime( time.strptime( history.update_time.strftime('%a %b %d %H:%M:%S %Y') ))
|
||||
diff = (now - last) /3600/24 # days
|
||||
if diff>days:
|
||||
histories_no_users.append(row.id)
|
||||
if history.deleted: continue #we don't add to history delete count, but we do want to be able to make sure datasets associated with deleted histories are really deleted
|
||||
ht = app.model.History.table
|
||||
# Generate a list of userless histories, deleting the histories along the way
|
||||
for row in ht.select( ht.c.user_id==None ).execute():
|
||||
now = time.time()
|
||||
last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) )
|
||||
diff = (now-last)/3600/24 # days
|
||||
if diff > days:
|
||||
histories_no_users.append( row.id )
|
||||
if not row.deleted:
|
||||
print "Updating history table, setting id %s to deleted" %str( row.id )
|
||||
history = app.model.History.get( row.id )
|
||||
history.deleted = True
|
||||
history_count += 1
|
||||
|
||||
dataset_count = 0
|
||||
for row in app.model.Dataset.table.select().execute():
|
||||
dt = app.model.Dataset.table
|
||||
# Delete all datasets associated with previously deleted userless histories
|
||||
for row in dt.select( dt.c.deleted=='f' ).execute():
|
||||
if row.history_id in histories_no_users:
|
||||
#delete dataset
|
||||
data = app.model.Dataset.get(row.id)
|
||||
if data.deleted: continue
|
||||
print "Updating dataset table, setting id %s to deleted" %str( row.id )
|
||||
data = app.model.Dataset.get( row.id )
|
||||
data.deleted = True
|
||||
dataset_count += 1
|
||||
try:
|
||||
app.model.flush()
|
||||
except:
|
||||
pass
|
||||
msg = 'deleted %d abandoned histories with %d total datasets' % ( history_count, dataset_count )
|
||||
msg = 'Deleted %d histories (including a total of %d datasets )' % ( history_count, dataset_count )
|
||||
return msg
|
||||
|
||||
def purge( app, days ):
|
||||
""" Purges deleted datasets older than specified number of days """
|
||||
"""
|
||||
Purges deleted datasets older than specified number of days by executing the Dataset.purge() function.
|
||||
This will update the Dataset table, setting Dataset.deleted to 't', Dataset.purged to 't', and
|
||||
Dataset.file_size to 0. The dataset file will then be removed from the file system.
|
||||
"""
|
||||
count = 0
|
||||
now = time.time()
|
||||
for row in list(app.model.Dataset.table.select().execute()):
|
||||
data = app.model.Dataset.get(row.id)
|
||||
if data.deleted and not data.purged:
|
||||
last = time.mktime( time.strptime( data.update_time.strftime('%a %b %d %H:%M:%S %Y') ))
|
||||
diff = (now - last) /3600/24 # days
|
||||
if diff>days:
|
||||
data.purge()
|
||||
data.flush()
|
||||
count += 1
|
||||
now = time.time()
|
||||
dt = app.model.Dataset.table
|
||||
for row in dt.select( ( dt.c.purged=='f' ) & ( dt.c.deleted=='t' ) ).execute():
|
||||
last = time.mktime( time.strptime( row.update_time.strftime( '%a %b %d %H:%M:%S %Y' ) ) )
|
||||
diff = (now-last)/3600/24 # days
|
||||
if diff > days:
|
||||
print "Purging dataset id %s" %str( row.id )
|
||||
data = app.model.Dataset.get( row.id )
|
||||
data.purge()
|
||||
data.flush()
|
||||
count += 1
|
||||
try:
|
||||
app.model.flush()
|
||||
except:
|
||||
@@ -90,6 +99,5 @@ def purge( app, days ):
|
||||
msg = 'Purged %d datasets' % count
|
||||
return msg
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+53
-58
@@ -1,61 +1,56 @@
|
||||
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Galaxy Administration</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<link href="$h.url_for('/static/universe.css')" rel="stylesheet" type="text/css" />
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
<table align="center" width="60%" class="border" cellpadding="5" cellspacing="5">
|
||||
<tr><td>
|
||||
<h3 align="center">Galaxy Administration</h3>
|
||||
#if $msg:
|
||||
<p class="ok_bgr">$msg</p>
|
||||
#end if
|
||||
<p>
|
||||
#set $ulen = len($users)
|
||||
#set $dlen = len($data)
|
||||
<ul>
|
||||
<li>$ulen users</li>
|
||||
<li>$dlen data entries</li>
|
||||
<li>$qsize jobs in the queue</li>
|
||||
</ul>
|
||||
</p>
|
||||
</td></tr>
|
||||
<tr><td>
|
||||
<form method="post" action="admin">
|
||||
<p>Admin password: <input type="password" name="passwd" size="8"> </p>
|
||||
<p>
|
||||
Purge deleted datasets older than <input type="textfield" value="30" size="3" name="purge_days"> days.
|
||||
<button name="action" value="purge">Purge</button>
|
||||
</p>
|
||||
#*
|
||||
<p>
|
||||
Delete objects older than <input type="textfield" value="120" size="3" name="days"> days.
|
||||
<button name="action" value="delete">Delete</button>
|
||||
</p>
|
||||
*#
|
||||
<p>
|
||||
Delete abandoned histories older than <input type="textfield" value="14" size="3" name="abandon_days"> days.
|
||||
<button name="action" value="abandon">Abandon</button>
|
||||
</p>
|
||||
<p>
|
||||
Reload tool: <select name="tool_id">
|
||||
#for $i, $section in enumerate( $toolbox.sections )
|
||||
<optgroup label="$section.name">
|
||||
#for $t in $section.tools
|
||||
<option value="$t.id">$t.name</option>
|
||||
#end for
|
||||
#end for
|
||||
</select>
|
||||
<button name="action" value="tool_reload">Reload</button>
|
||||
</p>
|
||||
</form>
|
||||
</td></tr>
|
||||
</table>
|
||||
|
||||
</body>
|
||||
<head>
|
||||
<title>Galaxy Administration</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1" />
|
||||
<link href="$h.url_for('/static/universe.css')" rel="stylesheet" type="text/css" />
|
||||
</head>
|
||||
<body>
|
||||
<table align="center" width="60%" class="border" cellpadding="5" cellspacing="5">
|
||||
<tr>
|
||||
<td>
|
||||
<h3 align="center">Galaxy Administration</h3>
|
||||
#if $msg:
|
||||
<p class="ok_bgr">$msg</p>
|
||||
#end if
|
||||
<p>
|
||||
#set $ulen = len($users)
|
||||
#set $dlen = len($data)
|
||||
<ul>
|
||||
<li>$ulen users</li>
|
||||
<li>$dlen data entries</li>
|
||||
<li>$qsize jobs in the queue</li>
|
||||
</ul>
|
||||
</p>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<form method="post" action="admin">
|
||||
<p>Admin password: <input type="password" name="passwd" size="8"></p>
|
||||
<p>
|
||||
Purge deleted datasets older than <input type="textfield" value="30" size="3" name="purge_days"> days.
|
||||
<button name="action" value="purge">Purge</button>
|
||||
</p>
|
||||
<p>
|
||||
Delete abandoned histories older than <input type="textfield" value="30" size="3" name="abandon_days"> days.
|
||||
<button name="action" value="abandon">Abandon</button>
|
||||
</p>
|
||||
<p>
|
||||
Reload tool:
|
||||
<select name="tool_id">
|
||||
#for $i, $section in enumerate( $toolbox.sections )
|
||||
<optgroup label="$section.name">
|
||||
#for $t in $section.tools
|
||||
<option value="$t.id">$t.name</option>
|
||||
#end for
|
||||
#end for
|
||||
</select>
|
||||
<button name="action" value="tool_reload">Reload</button>
|
||||
</p>
|
||||
</form>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -35,6 +35,10 @@ job_queue_cleanup_interval = 30
|
||||
database_file = database/universe.sqlite
|
||||
# You may use a SQLAlchemy connection string to specify an external database instead
|
||||
## database_connection = postgres:///galaxy_test
|
||||
## database_engine_option_echo = true
|
||||
## database_engine_option_echo_pool = true
|
||||
## database_engine_option_pool_size = 10
|
||||
## database_engine_option_max_overflow = 20
|
||||
|
||||
# Where dataset files are saved
|
||||
file_path = database/files
|
||||
|
||||
Reference in New Issue
Block a user