opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #6262 from nsoranzo/release_18.01_fix_run_scripts

Browse Source 
[18.01] Various fixes for the run scripts
This commit is contained in:
Nate Coraor
2018-06-06 10:52:01 -04:00
committed by GitHub
parent b137d5cf34 aaff474142
commit a0004db720
8 changed files with 81 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/dependencies/__init__.py
+13 -2
View File
@@ -1,13 +1,19 @@
"""
Determine what optional dependencies are needed.
"""
from __future__ import print_function
import sys
from os.path import dirname, join
from xml.etree import ElementTree
import pkg_resources
from galaxy.util import asbool
from galaxy.util.properties import load_app_properties
from galaxy.util.properties import (
find_config_file,
load_app_properties
)
class ConditionalDependencies(object):
@@ -127,7 +133,12 @@ class ConditionalDependencies(object):
self.config['watch_tool_data_dir'] in install_set)
def optional(config_file):
def optional(config_file=None):
if not config_file:
config_file = find_config_file(['galaxy', 'universe_wsgi'])
if not config_file:
print("galaxy.dependencies.optional: no config file found", file=sys.stderr)
return []
rval = []
conditional = ConditionalDependencies(config_file)
for opt in conditional.conditional_reqs:
run.sh
+19 -31
View File
@@ -18,9 +18,9 @@ then
GALAXY_LOCAL_ENV_FILE='./config/local_env.sh'
fi
if [ -f $GALAXY_LOCAL_ENV_FILE ];
if [ -f "$GALAXY_LOCAL_ENV_FILE" ];
then
. $GALAXY_LOCAL_ENV_FILE
. "$GALAXY_LOCAL_ENV_FILE"
fi
GALAXY_PID=${GALAXY_PID:-galaxy.pid}
@@ -46,30 +46,15 @@ if [ -n "$GALAXY_UNIVERSE_CONFIG_DIR" ]; then
python ./scripts/build_universe_config.py "$GALAXY_UNIVERSE_CONFIG_DIR"
fi
if [ -z "$GALAXY_CONFIG_FILE" ]; then
if [ -f universe_wsgi.ini ]; then
GALAXY_CONFIG_FILE=universe_wsgi.ini
elif [ -f config/galaxy.ini ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
elif [ -f config/galaxy.yml ]; then
GALAXY_CONFIG_FILE=config/galaxy.yml
elif [ -f config/galaxy.ini.sample -a -z "$GALAXY_UWSGI" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini.sample
fi
export GALAXY_CONFIG_FILE
fi
set_galaxy_config_file_var
if [ -n "$GALAXY_CONFIG_FILE" ]; then
config_file_arg="-c $GALAXY_CONFIG_FILE"
fi
if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then
if [ "$INITIALIZE_TOOL_DEPENDENCIES" -eq 1 ]; then
# Install Conda environment if needed.
python ./scripts/manage_tool_dependencies.py $config_file_arg init_if_needed
python ./scripts/manage_tool_dependencies.py init_if_needed
fi
[ -n "$GALAXY_UWSGI" ] && APP_WEBSERVER='uwsgi'
find_server ${GALAXY_CONFIG_FILE:-none} galaxy
find_server "${GALAXY_CONFIG_FILE:-none}" galaxy
if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then
servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo)
@@ -78,30 +63,33 @@ if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then
exit 1
fi
for server in $servers; do
echo "Executing: python $server_args --server-name=\"$server\" --pid-file=\"$server.pid\" --log-file=\"$server.log\""
python $server_args --server-name="$server" --pid-file="$server.pid" --log-file="$server.log"
if [ -n "$wait_arg_set" -a -n "$daemon_or_restart_arg_set" ]; then
python ./scripts/paster.py serve "$GALAXY_CONFIG_FILE" --server-name="$server" --pid-file="$server.pid" --log-file="$server.log" $paster_args
while true; do
sleep 1
printf "."
# Grab the current pid from the pid file
if ! current_pid_in_file=$(cat "$server.pid"); then
# Grab the current pid from the pid file and remove any trailing space
if ! current_pid_in_file=$(sed -e 's/[[:space:]]*$//' "$server.pid"); then
echo "A Galaxy process died, interrupting" >&2
exit 1
fi
if [ -n "$current_pid_in_file" ]; then
echo "Found PID $current_pid_in_file in '$server.pid', monitoring '$server.log'"
else
echo "No PID found in '$server.pid' yet"
continue
fi
# Search for all pids in the logs and tail for the last one
latest_pid=$(egrep '^Starting server in PID [0-9]+\.$' "$server.log" -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1)
latest_pid=$(grep '^Starting server in PID [0-9]\+\.$' "$server.log" | sed 's/^Starting server in PID \([0-9]\+\).$/\1/' | tail -n 1)
# If they're equivalent, then the current pid file agrees with our logs
# and we've succesfully started
[ -n "$latest_pid" ] && [ "$latest_pid" -eq "$current_pid_in_file" ] && break
done
echo
else
echo "Handling $server with log file $server.log..."
python ./scripts/paster.py serve "$GALAXY_CONFIG_FILE" --server-name="$server" --pid-file="$server.pid" --log-file="$server.log" $paster_args
fi
done
else
echo "executing: $run_server $server_args"
echo "Executing: $run_server $server_args $pid_log_paster_args"
# args are properly quoted so use eval
eval $run_server $server_args
eval $run_server $server_args $pid_log_paster_args
fi
run_reports.sh
+3 -11
View File
@@ -31,15 +31,7 @@ run_common_start_up
setup_python
if [ -z "$GALAXY_REPORTS_CONFIG" ]; then
if [ -f reports_wsgi.ini ]; then
GALAXY_REPORTS_CONFIG=reports_wsgi.ini
elif [ -f config/reports_wsgi.ini ]; then
GALAXY_REPORTS_CONFIG=config/reports_wsgi.ini
elif [ -f config/reports.ini ]; then
GALAXY_REPORTS_CONFIG=config/reports.ini
elif [ -f config/reports.yml ]; then
GALAXY_REPORTS_CONFIG=config/reports.yml
fi
GALAXY_REPORTS_CONFIG=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['reports', 'reports_wsgi']) or '')")
export GALAXY_REPORTS_CONFIG
fi
@@ -48,5 +40,5 @@ if [ -n "$GALAXY_REPORTS_CONFIG_DIR" ]; then
fi
find_server ${GALAXY_REPORTS_CONFIG:-none} reports
echo "executing: $run_server $server_args"
eval $run_server $server_args
echo "Executing: $run_server $server_args $pid_log_paster_args"
eval $run_server $server_args $pid_log_paster_args
run_tool_shed.sh
+5 -11
View File
@@ -17,25 +17,19 @@ run_common_start_up
setup_python
tool_shed=`./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $parser_args`
tool_shed=$(./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $parser_args)
args=$parser_args
if [ $? -eq 0 ] ; then
bash ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh $parser_args
args=`echo $@ | sed "s#-\?-bootstrap_from_tool_shed $tool_shed##"`
args=$(echo $@ | sed "s#-\?-bootstrap_from_tool_shed $tool_shed##")
fi
if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then
if [ -f tool_shed_wsgi.ini ]; then
TOOL_SHED_CONFIG_FILE=tool_shed_wsgi.ini
elif [ -f config/tool_shed.ini ]; then
TOOL_SHED_CONFIG_FILE=config/tool_shed.ini
elif [ -f config/tool_shed.yml ]; then
TOOL_SHED_CONFIG_FILE=config/tool_shed.yml
fi
TOOL_SHED_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['tool_shed', 'tool_shed_wsgi']) or '')")
export TOOL_SHED_CONFIG_FILE
fi
find_server ${TOOL_SHED_CONFIG_FILE:-none} tool_shed
echo "executing: $run_server $server_args"
eval $run_server $server_args
echo "Executing: $run_server $server_args $pid_log_paster_args"
eval $run_server $server_args $pid_log_paster_args
scripts/common_startup.sh
-11
View File
@@ -96,17 +96,6 @@ else
DEV_WHEELS=1
fi
: ${GALAXY_CONFIG_FILE:=config/galaxy.yml}
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
fi
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=universe_wsgi.ini
fi
if [ ! -f "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=config/galaxy.yml.sample
fi
: ${GALAXY_VIRTUAL_ENV:=.venv}
if [ $SET_VENV -eq 1 -a $CREATE_VENV -eq 1 ]; then
scripts/common_startup_functions.sh
+25 -14
View File
@@ -22,30 +22,33 @@ parse_common_args() {
--stop-daemon|stop)
common_startup_args="$common_startup_args --stop-daemon"
paster_args="$paster_args --stop-daemon"
uwsgi_args="$uwsgi_args --stop $PID_FILE"
pid_log_paster_args="--pid-file \"$PID_FILE\""
uwsgi_args="$uwsgi_args --stop \"$PID_FILE\""
stop_daemon_arg_set=1
shift
;;
--restart|restart)
if [ "$1" = "--restart" ]
then
paster_args="$paster_args restart"
else
paster_args="$paster_args $1"
fi
uwsgi_args="$uwsgi_args --reload $PID_FILE"
paster_args="$paster_args restart"
pid_log_paster_args="--pid-file \"$PID_FILE\" --log-file \"$LOG_FILE\""
uwsgi_args="$uwsgi_args --reload \"$PID_FILE\""
restart_arg_set=1
daemon_or_restart_arg_set=1
shift
;;
--daemon|start)
paster_args="$paster_args --daemon"
pid_log_paster_args="--pid-file \"$PID_FILE\" --log-file \"$LOG_FILE\""
# --daemonize2 waits until after the application has loaded
# to daemonize, thus it stops if any errors are found
uwsgi_args="--master --daemonize2 $LOG_FILE --pidfile2 $PID_FILE $uwsgi_args"
uwsgi_args="--master --daemonize2 \"$LOG_FILE\" --pidfile2 \"$PID_FILE\" $uwsgi_args"
daemon_or_restart_arg_set=1
shift
;;
--status|status)
paster_args="$paster_args $1"
pid_log_paster_args="--pid-file \"$PID_FILE\""
shift
;;
--wait)
wait_arg_set=1
shift
@@ -84,9 +87,16 @@ setup_python() {
python ./scripts/check_python.py || exit 1
}
set_galaxy_config_file_var() {
if [ -z "$GALAXY_CONFIG_FILE" ]; then
GALAXY_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['galaxy', 'universe_wsgi']) or '')")
export GALAXY_CONFIG_FILE
fi
}
find_server() {
server_config="$1"
server_app="$2"
server_config=$1
server_app=$2
arg_getter_args=
default_webserver="paste"
case "$server_config" in
@@ -107,16 +117,17 @@ find_server() {
echo 'ERROR: Could not find uwsgi executable'
exit 1
fi
[ "$server_config" != "none" ] && arg_getter_args="-c $server_config"
[ "$server_config" != "none" ] && arg_getter_args="-c \"$server_config\""
[ -n "$server_app" ] && arg_getter_args="--app $server_app"
run_server="$UWSGI"
server_args=
if [ -z "$stop_daemon_arg_set" -a -z "$restart_arg_set" ]; then
server_args="$(python ./scripts/get_uwsgi_args.py $arg_getter_args)"
server_args="$(eval python ./scripts/get_uwsgi_args.py $arg_getter_args)"
fi
server_args="$server_args $uwsgi_args"
pid_log_paster_args=""
else
run_server="python"
server_args="./scripts/paster.py serve $server_config --pid-file $PID_FILE --log-file $LOG_FILE $paster_args"
server_args="./scripts/paster.py serve \"$server_config\" $paster_args"
fi
}
scripts/common_startup_variables.sh
-38
View File
@@ -1,38 +0,0 @@
#!/usr/bin/env bash
# This file is sourced in the maintenance.sh script and could be sourced in ./run.sh or ./run_reports.sh
SCRIPTLOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
# If there is a file that defines a shell environment specific to this
# instance of Galaxy, source the file.
if [ -z $GALAXY_LOCAL_ENV_FILE ];
then
GALAXY_LOCAL_ENV_FILE="$SCRIPTLOCATION/../config/local_env.sh"
fi
if [ -f $GALAXY_LOCAL_ENV_FILE ];
then
echo "Activating local env file: $GALAXY_LOCAL_ENV_FILE"
. $GALAXY_LOCAL_ENV_FILE
fi
# If there is a .venv/ directory, assume it contains a virtualenv that we
# should run this instance in.
GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-$SCRIPTLOCATION/../.venv}"
if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ];
then
[ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; }
echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV"
. "$GALAXY_VIRTUAL_ENV/bin/activate"
fi
if [ -z "$GALAXY_CONFIG_FILE" ]; then
if [ -f "$SCRIPTLOCATION/../universe_wsgi.ini" ]; then
GALAXY_CONFIG_FILE=universe_wsgi.ini
elif [ -f "$SCRIPTLOCATION/../config/galaxy.ini" ]; then
GALAXY_CONFIG_FILE=config/galaxy.ini
else
GALAXY_CONFIG_FILE=config/galaxy.ini.sample
fi
export GALAXY_CONFIG_FILE
fi
echo "GALAXY_CONFIG_FILE: ${GALAXY_CONFIG_FILE}"
scripts/maintenance.sh
+16 -12
View File
@@ -52,8 +52,13 @@ do
esac
done
SCRIPTLOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source ${SCRIPTLOCATION}/common_startup_variables.sh
cd "$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"/../
. scripts/common_startup_functions.sh
setup_python
set_galaxy_config_file_var
if [ "$DRYRUN" = true ]; then
MODE="--info_only"
@@ -61,14 +66,14 @@ else
MODE="-r"
fi
MAINTENANCE_LOG="$SCRIPTLOCATION/../maintenance.log"
MAINTENANCE_LOG=maintenance.log
COMMANDS=(
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_userless_histories >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_histories >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_folders >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_datasets >> $MAINTENANCE_LOG"
"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG")
"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_userless_histories >> $MAINTENANCE_LOG"
"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_histories >> $MAINTENANCE_LOG"
"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG"
"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_folders >> $MAINTENANCE_LOG"
"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_datasets >> $MAINTENANCE_LOG"
"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG")
printf "\nDry run: $DRYRUN\nDays: $DAYS\n\n"
echo "Will run following commands and output in $MAINTENANCE_LOG"
@@ -77,15 +82,14 @@ for (( i = 0; i < ${#COMMANDS[@]} ; i++ )); do
done
if [ "$DRYRUN" = false ]; then
echo "python $SCRIPTLOCATION/set_user_disk_usage.py >> $MAINTENANCE_LOG"
echo "python scripts/set_user_disk_usage.py >> $MAINTENANCE_LOG"
fi
# Run the commands
cd $SCRIPTLOCATION/../
for (( i = 0; i < ${#COMMANDS[@]} ; i++ )); do
eval "${COMMANDS[$i]}"
done
if [ "$DRYRUN" = false ]; then
python $SCRIPTLOCATION/set_user_disk_usage.py >> $MAINTENANCE_LOG
python scripts/set_user_disk_usage.py >> $MAINTENANCE_LOG
fi