mirror of
https://github.com/galaxyproject/galaxy.git
synced 2026-09-24 16:30:27 +08:00
The Galaxy Monitor, and its cron wrapper.
This commit is contained in:
Nate Coraor
Executable
+208
@@ -0,0 +1,208 @@
|
||||
#!/bin/sh
|
||||
#set -xv
|
||||
#
|
||||
# Runs the scripts/check_galaxy.py script in a way that's easy to handle from cron
|
||||
#
|
||||
|
||||
# defaults (note: default sleep is below since it depends on debug)
|
||||
DEBUG=0
|
||||
STAGGER=0
|
||||
INTERVAL=3
|
||||
MAIL=
|
||||
PAGE=
|
||||
BARDARG=0
|
||||
# get commandline opts
|
||||
while getopts dsi:l:m:p: optname
|
||||
do
|
||||
case $optname in
|
||||
d) DEBUG=1 ;;
|
||||
s) STAGGER=1 ;;
|
||||
i) INTERVAL=$OPTARG ;;
|
||||
l) SLEEP=$OPTARG ;;
|
||||
m) MAIL="$MAIL $OPTARG" ;;
|
||||
p) PAGE="$PAGE $OPTARG" ;;
|
||||
*) BADARG=1 ;;
|
||||
esac
|
||||
done
|
||||
shift `expr $OPTIND - 1`
|
||||
|
||||
if [ -z "$1" -o "$BADARG" ]; then
|
||||
cat <<EOF
|
||||
usage: `basename $0` [-ds] [-i interval] [-m email_address]+ [-p pager_address]+ <galaxy_host>"
|
||||
-d Print debugging information.
|
||||
-s Stagger mailing the pagers/emails, instead of all at once when
|
||||
there's a problem. Useful for running check_galaxy at night.
|
||||
-i <interval> The number of times this wrapper should execute before mailing
|
||||
the next address, when staggering is enabled. Mail is sent
|
||||
every <interval> runs of the program, so the actual time
|
||||
between emails is:
|
||||
time = (<interval>) * (how often wrapper runs from cron)
|
||||
-l <seconds> This wrapper runs check_galaxy a second time if the first check
|
||||
fails, in case the problem is intermittent. <seconds> is how
|
||||
many seconds to sleep between checks.
|
||||
-m <address> Email addresses to send the full check_galaxy output to, if
|
||||
Galaxy is down. Use multiple -m options to specify multiple
|
||||
addresses. When staggering, email will be sent in the order
|
||||
which you specify -m options on the command line.
|
||||
-p <address> Like -m, but sends just the last line of check_galaxy's output.
|
||||
Useful for pagers. When staggering is enabled and both -m and
|
||||
-p options are present, the first -m address and the first -p
|
||||
address are mailed simultaneously, followed by the second -m
|
||||
and second -p, and so on.
|
||||
<galaxy_host> The hostname of the Galaxy server to check. Use a : if running
|
||||
on a non-80 port (e.g. galaxy.example.com:8080).
|
||||
EOF
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -z "$SLEEP" ]; then
|
||||
if [ $DEBUG ]; then
|
||||
SLEEP=2
|
||||
else
|
||||
SLEEP=60
|
||||
fi
|
||||
fi
|
||||
|
||||
# globals
|
||||
CRON_DIR=`dirname $0`
|
||||
SCRIPTS_DIR="$CRON_DIR/../scripts"
|
||||
CHECK_GALAXY="$SCRIPTS_DIR/check_galaxy.py"
|
||||
VAR="$HOME/.check_galaxy"
|
||||
|
||||
# sanity
|
||||
if [ ! -f $CHECK_GALAXY ]; then
|
||||
[ $DEBUG = 1 ] && echo "$CHECK_GALAXY missing"
|
||||
exit 0
|
||||
fi
|
||||
if [ ! -d $VAR ]; then
|
||||
[ $DEBUG = 1 ] && echo "$VAR is missing"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# Do any other systems' default ps not take BSD ps args?
|
||||
case `uname -s` in
|
||||
SunOS) PS="/usr/ucb/ps" ;;
|
||||
*) PS="ps" ;;
|
||||
esac
|
||||
|
||||
NOTIFIED_MAIL="$VAR/$1/mail"
|
||||
NOTIFIED_PAGE="$VAR/$1/page"
|
||||
MUTEX="$VAR/$1/wrap.mutex"
|
||||
COUNT="$VAR/$1/wrap.count"
|
||||
STAGGER_FILE="$VAR/$1/wrap.stagger"
|
||||
for dir in $VAR/$1 $NOTIFIED_MAIL $NOTIFIED_PAGE; do
|
||||
if [ ! -d $dir ]; then
|
||||
mkdir $dir
|
||||
if [ $? -ne 0 ]; then
|
||||
[ $DEBUG = 1 ] && echo "unable to create dir: $dir"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ $STAGGER ]; then
|
||||
if [ -f "$STAGGER_FILE" ]; then
|
||||
STAGGER_COUNT=`cat $STAGGER_FILE`
|
||||
else
|
||||
STAGGER_COUNT=$INTERVAL
|
||||
fi
|
||||
fi
|
||||
|
||||
# only run one at once
|
||||
if [ -f $MUTEX ]; then
|
||||
pid=`cat $MUTEX`
|
||||
$PS p $pid >/dev/null 2>&1
|
||||
if [ $? -eq 0 ]; then
|
||||
if [ -f $COUNT ]; then
|
||||
count=`cat $COUNT`
|
||||
else
|
||||
count=0
|
||||
fi
|
||||
if [ "$count" -eq 3 ]; then
|
||||
echo "A check_galaxy process for $1 has been running for an unusually long time. Something is broken." \
|
||||
| mail -s "$1 problems" $MAIL
|
||||
fi
|
||||
expr $count + 1 > $COUNT
|
||||
exit 0
|
||||
else
|
||||
# stale mutex
|
||||
rm -f $MUTEX
|
||||
fi
|
||||
fi
|
||||
|
||||
rm -f $COUNT
|
||||
echo $$ > $MUTEX
|
||||
|
||||
[ $DEBUG = 1 ] && echo "running first check"
|
||||
first_try=`$CHECK_GALAXY $1 2>&1`
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
# if failure, wait and try again
|
||||
[ $DEBUG = 1 ] && echo "first check failed, sleeping $SLEEP seconds for second run"
|
||||
sleep $SLEEP
|
||||
else
|
||||
# if successful
|
||||
[ $DEBUG = 1 ] && echo "first check succeeded"
|
||||
for file in $NOTIFIED_MAIL/* $NOTIFIED_PAGE/*; do
|
||||
recip=`basename $file`
|
||||
# the literal string including the * will be passed if the dir is empty
|
||||
[ "$recip" = '*' ] && continue
|
||||
echo "$1 is now okay" | mail -s "$1 OK" $recip
|
||||
rm -f $file
|
||||
[ $DEBUG = 1 ] && echo "up: mailed $recip"
|
||||
done
|
||||
rm -f $MUTEX $STAGGER_FILE
|
||||
exit 0
|
||||
fi
|
||||
|
||||
[ $DEBUG = 1 ] && echo "running second check"
|
||||
second_try=`$CHECK_GALAXY $1 2>&1`
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
[ $DEBUG = 1 ] && echo "second check failed"
|
||||
if [ $STAGGER = 1 ]; then
|
||||
if [ "$STAGGER_COUNT" -eq "$INTERVAL" ]; then
|
||||
# send notification this run
|
||||
echo 1 > $STAGGER_FILE
|
||||
else
|
||||
# don't send notification this run
|
||||
[ $DEBUG = 1 ] && echo "$1 is down, but it's not time to send an email. STAGGER_COUNT was $STAGGER_COUNT"
|
||||
expr $STAGGER_COUNT + 1 > $STAGGER_FILE
|
||||
rm -f $MUTEX
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
for recip in $MAIL; do
|
||||
if [ ! -f "$NOTIFIED_MAIL/$recip" ]; then
|
||||
cat <<HERE | mail -s "$1 problems" $recip
|
||||
$second_try
|
||||
HERE
|
||||
touch "$NOTIFIED_MAIL/$recip"
|
||||
[ $DEBUG = 1 ] && echo "dn: mailed $recip"
|
||||
[ $STAGGER = 1 ] && break
|
||||
fi
|
||||
done
|
||||
for recip in $PAGE; do
|
||||
if [ ! -f "$NOTIFIED_PAGE/$recip" ]; then
|
||||
cat <<HERE | tail -1 | mail -s "$1 problems" $recip
|
||||
$second_try
|
||||
HERE
|
||||
touch "$NOTIFIED_PAGE/$recip"
|
||||
[ $DEBUG = 1 ] && echo "dn: mailed $recip"
|
||||
[ $STAGGER = 1 ] && break
|
||||
fi
|
||||
done
|
||||
else
|
||||
[ $DEBUG = 1 ] && echo "second check succeeded"
|
||||
for file in $NOTIFIED_MAIL/* $NOTIFIED_PAGE/*; do
|
||||
recip=`basename $file`
|
||||
[ "$recip" = '*' ] && continue
|
||||
echo "$1 is now okay" | mail -s "$1 OK" $recip
|
||||
rm -f $file
|
||||
[ $DEBUG = 1 ] && echo "up: mailed $recip"
|
||||
done
|
||||
rm -f $STAGGER_FILE
|
||||
fi
|
||||
|
||||
rm -f $MUTEX
|
||||
exit 0
|
||||
Executable
+357
@@ -0,0 +1,357 @@
|
||||
#!/usr/bin/env python
|
||||
"""
|
||||
check_python can be run by hand, although it is meant to run from cron
|
||||
via the check_galaxy.sh script in Galaxy's cron/ directory.
|
||||
"""
|
||||
|
||||
import socket, sys, os, time, tempfile, filecmp, htmllib, formatter
|
||||
from user import home
|
||||
|
||||
# options
|
||||
if os.environ.has_key( "DEBUG" ):
|
||||
debug = os.environ["DEBUG"]
|
||||
else:
|
||||
debug = False
|
||||
scripts_dir = os.path.abspath( os.path.dirname( sys.argv[0] ) )
|
||||
test_data_dir = os.path.join( scripts_dir, "..", "test-data" )
|
||||
# what tools to run - not so pretty
|
||||
tools = {
|
||||
"gops_intersect_1" :
|
||||
[
|
||||
{
|
||||
"inputs" :
|
||||
(
|
||||
os.path.join( test_data_dir, "1.bed" ),
|
||||
os.path.join( test_data_dir, "2.bed" )
|
||||
)
|
||||
},
|
||||
{ "check_file" : os.path.join( test_data_dir, "gops-intersect.dat" ) },
|
||||
{
|
||||
"tool_run_options" :
|
||||
{
|
||||
"input1" : "1.bed",
|
||||
"input2" : "2.bed",
|
||||
"min" : "1",
|
||||
"returntype" : ""
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
# handle arg(s)
|
||||
if len(sys.argv) < 2:
|
||||
print "usage: check_galaxy.py <server>"
|
||||
sys.exit(1)
|
||||
|
||||
if sys.argv[1].find(".") < 0:
|
||||
server = "%s.g2.bx.psu.edu" % sys.argv[1]
|
||||
maint = "/errordocument/502/%s/maint" % sys.argv[1]
|
||||
else:
|
||||
server = sys.argv[1]
|
||||
maint = None
|
||||
|
||||
# state information
|
||||
var_dir = os.path.join( home, ".check_galaxy", server )
|
||||
if not os.access( var_dir, os.F_OK ):
|
||||
os.makedirs( var_dir )
|
||||
|
||||
# get user/pass
|
||||
login_file = os.path.join( var_dir, "login" )
|
||||
try:
|
||||
f = open( login_file, 'r' )
|
||||
except:
|
||||
print "Please create the file:"
|
||||
print " ", login_file
|
||||
print "This should contain a username and password to log in to"
|
||||
print "Galaxy with, on one line, separated by whitespace, e.g.:"
|
||||
print ""
|
||||
print "check_galaxy@example.com password"
|
||||
print ""
|
||||
print "If the user does not exist, check_galaxy will create it"
|
||||
print "for you."
|
||||
sys.exit( 1 )
|
||||
( username, password ) = f.readline().split()
|
||||
|
||||
# find/import twill
|
||||
lib_dir = os.path.join( scripts_dir, "..", "lib" )
|
||||
eggs_dir = os.path.join( scripts_dir, "..", "eggs", "py%s-noplatform" %sys.version[:3] )
|
||||
sys.path.append( lib_dir )
|
||||
sys.path.append( eggs_dir )
|
||||
import pkg_resources
|
||||
pkg_resources.require( "twill" )
|
||||
import twill
|
||||
import twill.commands as tc
|
||||
|
||||
# default timeout for twill browser is never
|
||||
socket.setdefaulttimeout(300)
|
||||
|
||||
# user-agent
|
||||
tc.agent("Mozilla/5.0 (compatible; check_galaxy/0.1)")
|
||||
|
||||
class Browser:
|
||||
|
||||
def __init__(self):
|
||||
self.server = server
|
||||
self.maint = maint
|
||||
self.tool = None
|
||||
self.tool_opts = None
|
||||
self.id = None
|
||||
self.status = None
|
||||
self.check_file = None
|
||||
self.hid = None
|
||||
self.cookie_jar = os.path.join( var_dir, "cookie_jar" )
|
||||
dprint("cookie jar path: %s" % self.cookie_jar)
|
||||
if not os.access(self.cookie_jar, os.R_OK):
|
||||
dprint("no cookie jar at above path, creating")
|
||||
tc.save_cookies(self.cookie_jar)
|
||||
tc.load_cookies(self.cookie_jar)
|
||||
|
||||
def get(self, path):
|
||||
tc.go("http://%s%s" % (self.server, path))
|
||||
tc.code(200)
|
||||
|
||||
def reset(self):
|
||||
self.get("/history_new")
|
||||
self.tool = None
|
||||
self.tool_opts = None
|
||||
self.id = None
|
||||
self.status = None
|
||||
self.check_file = None
|
||||
#self.get("/history")
|
||||
self.get("/root/history_options")
|
||||
p = hidParser()
|
||||
p.feed(tc.browser.get_html())
|
||||
if p.hid is not None:
|
||||
self.hid = p.hid
|
||||
else:
|
||||
raise Exception, "Unable to determine hid after creating new history"
|
||||
|
||||
def check_redir(self, url):
|
||||
try:
|
||||
tc.get_browser()._browser.set_handle_redirect(False)
|
||||
tc.go(url)
|
||||
tc.code(302)
|
||||
tc.get_browser()._browser.set_handle_redirect(True)
|
||||
dprint( "%s is returning redirect (302)" % url )
|
||||
return(True)
|
||||
except twill.errors.TwillAssertionError, e:
|
||||
tc.get_browser()._browser.set_handle_redirect(True)
|
||||
dprint( "%s is not returning redirect (302): %s" % (url, e) )
|
||||
code = tc.browser.get_code()
|
||||
if code == 502:
|
||||
is_maint = self.check_maint()
|
||||
if is_maint:
|
||||
dprint( "Galaxy is down, but a maint file was found, so not sending alert" )
|
||||
sys.exit(0)
|
||||
else:
|
||||
print "Galaxy is down (code 502)"
|
||||
sys.exit(1)
|
||||
return(False)
|
||||
|
||||
# checks for a maint file
|
||||
def check_maint(self):
|
||||
if self.maint is None:
|
||||
#dprint( "Warning: unable to check maint file for %s" % self.server )
|
||||
return(False)
|
||||
try:
|
||||
self.get(self.maint)
|
||||
return(True)
|
||||
except twill.errors.TwillAssertionError, e:
|
||||
return(False)
|
||||
|
||||
def login(self, user, pw):
|
||||
self.get("/user/login")
|
||||
tc.fv("1", "email", user)
|
||||
tc.fv("1", "password", pw)
|
||||
tc.submit("Login")
|
||||
tc.code(200)
|
||||
if len(tc.browser.get_all_forms()) > 0:
|
||||
# uh ohs, fail
|
||||
p = userParser()
|
||||
p.feed(tc.browser.get_html())
|
||||
if p.no_user:
|
||||
dprint("user does not exist, will try creating")
|
||||
self.create_user(user, pw)
|
||||
elif p.bad_pw:
|
||||
raise Exception, "Password is incorrect"
|
||||
else:
|
||||
raise Exception, "Unknown error logging in"
|
||||
tc.save_cookies(self.cookie_jar)
|
||||
|
||||
def create_user(self, user, pw):
|
||||
self.get("/user/create")
|
||||
tc.fv("1", "email", user)
|
||||
tc.fv("1", "password", pw)
|
||||
tc.fv("1", "confirm", pw)
|
||||
tc.submit("Create")
|
||||
tc.code(200)
|
||||
if len(tc.browser.get_all_forms()) > 0:
|
||||
p = userParser()
|
||||
p.feed(tc.browser.get_html())
|
||||
if p.already_exists:
|
||||
raise Exception, 'The user you were trying to create already exists'
|
||||
|
||||
def upload(self, file):
|
||||
self.get("/tool_runner/index?tool_id=upload1")
|
||||
tc.fv("1","file_type", "bed")
|
||||
tc.formfile("1","file_data", file)
|
||||
tc.submit("runtool_btn")
|
||||
tc.code(200)
|
||||
|
||||
def runtool(self):
|
||||
self.get("/tool_runner/index?tool_id=%s" % self.tool)
|
||||
for k, v in self.tool_opts.items():
|
||||
tc.fv("1", k, v)
|
||||
tc.submit("runtool_btn")
|
||||
tc.code(200)
|
||||
|
||||
def wait(self):
|
||||
sleep_amount = 1
|
||||
count = 0
|
||||
maxiter = 16
|
||||
while count < maxiter:
|
||||
count += 1
|
||||
self.get("/root/history")
|
||||
page = tc.browser.get_html()
|
||||
if page.find( '<!-- running: do not change this comment, used by TwillTestCase.wait -->' ) > -1:
|
||||
time.sleep( sleep_amount )
|
||||
sleep_amount += 1
|
||||
else:
|
||||
break
|
||||
if count == maxiter:
|
||||
raise Exception, "Tool never finished"
|
||||
|
||||
def check_status(self):
|
||||
self.get("/root/history")
|
||||
p = historyParser()
|
||||
p.feed(tc.browser.get_html())
|
||||
if p.status != "ok":
|
||||
raise Exception, "JOB %s NOT OK: %s" % (p.id, p.status)
|
||||
self.id = p.id
|
||||
self.status = p.status
|
||||
#return((p.id, p.status))
|
||||
|
||||
def diff(self):
|
||||
self.get("/datasets/%s/display/index" % self.id)
|
||||
data = tc.browser.get_html()
|
||||
tmp = tempfile.mkstemp()
|
||||
dprint("tmp file: %s" % tmp[1])
|
||||
tmpfh = os.fdopen(tmp[0], 'w')
|
||||
tmpfh.write(data)
|
||||
tmpfh.close()
|
||||
if filecmp.cmp(tmp[1], self.check_file):
|
||||
dprint("Tool output is as expected")
|
||||
else:
|
||||
if not debug:
|
||||
os.remove(tmp[1])
|
||||
raise Exception, "Tool output differs from expected"
|
||||
if not debug:
|
||||
os.remove(tmp[1])
|
||||
|
||||
def delete_history(self):
|
||||
self.get("/history_delete?id=%s" % self.hid)
|
||||
|
||||
class userParser(htmllib.HTMLParser):
|
||||
def __init__(self):
|
||||
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
|
||||
self.in_span = False
|
||||
self.no_user = False
|
||||
self.bad_pw = False
|
||||
self.already_exists = False
|
||||
def start_span(self, attrs):
|
||||
self.in_span = True
|
||||
def end_span(self):
|
||||
self.in_span = False
|
||||
def handle_data(self, data):
|
||||
if self.in_span:
|
||||
if data == "No such user":
|
||||
self.no_user = True
|
||||
elif data == "Invalid password":
|
||||
self.bad_pw = True
|
||||
elif data == "User with that email already exists":
|
||||
self.already_exists = True
|
||||
|
||||
class historyParser(htmllib.HTMLParser):
|
||||
def __init__(self):
|
||||
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
|
||||
self.status = None
|
||||
self.id = None
|
||||
def start_div(self, attrs):
|
||||
# find the top history item
|
||||
for i in attrs:
|
||||
if i[0] == "class" and i[1].startswith("historyItemWrapper historyItem historyItem-"):
|
||||
self.status = i[1].rsplit("historyItemWrapper historyItem historyItem-", 1)[1]
|
||||
dprint("status: %s" % self.status)
|
||||
if i[0] == "id" and i[1].startswith("historyItem-"):
|
||||
self.id = i[1].rsplit("historyItem-", 1)[1]
|
||||
dprint("id: %s" % self.id)
|
||||
if self.status is not None:
|
||||
self.reset()
|
||||
|
||||
class hidParser(htmllib.HTMLParser):
|
||||
def __init__(self):
|
||||
htmllib.HTMLParser.__init__(self, formatter.NullFormatter())
|
||||
self.hid = None
|
||||
def start_a(self, attrs):
|
||||
for i in attrs:
|
||||
if i[0] == "href" and i[1].startswith("/history_delete?id="):
|
||||
self.hid = i[1].rsplit("/history_delete?id=", 1)[1]
|
||||
dprint("history id: %s" % self.hid)
|
||||
if self.hid is not None:
|
||||
self.reset()
|
||||
|
||||
def dprint(str):
|
||||
if debug:
|
||||
print str
|
||||
|
||||
# do stuff here
|
||||
if __name__ == "__main__":
|
||||
|
||||
dprint("checking %s" % server)
|
||||
|
||||
b = Browser()
|
||||
|
||||
# login (or not)
|
||||
if b.check_redir("http://%s/user/account" % server):
|
||||
# if the account page redirects, we need to login
|
||||
dprint("not logged in... logging in")
|
||||
b.login(username, password)
|
||||
else:
|
||||
# we are logged in
|
||||
dprint("we are already logged in (via cookies), hooray!")
|
||||
|
||||
for tool, params in tools.iteritems():
|
||||
|
||||
check_file = ""
|
||||
|
||||
# make sure history and state is clean
|
||||
b.reset()
|
||||
b.tool = tool
|
||||
|
||||
# get all the tool run conditions
|
||||
for dict in params:
|
||||
for k, v in dict.items():
|
||||
if k == 'inputs':
|
||||
for file in v:
|
||||
b.upload(file)
|
||||
elif k == 'check_file':
|
||||
b.check_file = v
|
||||
elif k == 'tool_run_options':
|
||||
b.tool_opts = v
|
||||
else:
|
||||
raise Exception, "Unknown key in tools dict: %s" % k
|
||||
|
||||
b.runtool()
|
||||
b.wait()
|
||||
#st = b.check_status()
|
||||
b.check_status()
|
||||
b.diff()
|
||||
b.delete_history()
|
||||
|
||||
# by this point, everything else has succeeded. there should be no maint.
|
||||
is_maint = b.check_maint()
|
||||
if is_maint:
|
||||
print "Galaxy is up and fully functional, but a maint file is in place."
|
||||
sys.exit(1)
|
||||
|
||||
sys.exit(0)
|
||||
Reference in New Issue
Block a user