diff --git a/lib/galaxy/dependencies/__init__.py b/lib/galaxy/dependencies/__init__.py index 102d40c0ca2..c5d6b7a13f3 100644 --- a/lib/galaxy/dependencies/__init__.py +++ b/lib/galaxy/dependencies/__init__.py @@ -1,13 +1,19 @@ """ Determine what optional dependencies are needed. """ +from __future__ import print_function + +import sys from os.path import dirname, join from xml.etree import ElementTree import pkg_resources from galaxy.util import asbool -from galaxy.util.properties import load_app_properties +from galaxy.util.properties import ( + find_config_file, + load_app_properties +) class ConditionalDependencies(object): @@ -127,7 +133,12 @@ class ConditionalDependencies(object): self.config['watch_tool_data_dir'] in install_set) -def optional(config_file): +def optional(config_file=None): + if not config_file: + config_file = find_config_file(['galaxy', 'universe_wsgi']) + if not config_file: + print("galaxy.dependencies.optional: no config file found", file=sys.stderr) + return [] rval = [] conditional = ConditionalDependencies(config_file) for opt in conditional.conditional_reqs: diff --git a/run.sh b/run.sh index ea1a8e98052..0210375afac 100755 --- a/run.sh +++ b/run.sh @@ -18,9 +18,9 @@ then GALAXY_LOCAL_ENV_FILE='./config/local_env.sh' fi -if [ -f $GALAXY_LOCAL_ENV_FILE ]; +if [ -f "$GALAXY_LOCAL_ENV_FILE" ]; then - . $GALAXY_LOCAL_ENV_FILE + . "$GALAXY_LOCAL_ENV_FILE" fi GALAXY_PID=${GALAXY_PID:-galaxy.pid} @@ -46,30 +46,15 @@ if [ -n "$GALAXY_UNIVERSE_CONFIG_DIR" ]; then python ./scripts/build_universe_config.py "$GALAXY_UNIVERSE_CONFIG_DIR" fi -if [ -z "$GALAXY_CONFIG_FILE" ]; then - if [ -f universe_wsgi.ini ]; then - GALAXY_CONFIG_FILE=universe_wsgi.ini - elif [ -f config/galaxy.ini ]; then - GALAXY_CONFIG_FILE=config/galaxy.ini - elif [ -f config/galaxy.yml ]; then - GALAXY_CONFIG_FILE=config/galaxy.yml - elif [ -f config/galaxy.ini.sample -a -z "$GALAXY_UWSGI" ]; then - GALAXY_CONFIG_FILE=config/galaxy.ini.sample - fi - export GALAXY_CONFIG_FILE -fi +set_galaxy_config_file_var -if [ -n "$GALAXY_CONFIG_FILE" ]; then - config_file_arg="-c $GALAXY_CONFIG_FILE" -fi - -if [ $INITIALIZE_TOOL_DEPENDENCIES -eq 1 ]; then +if [ "$INITIALIZE_TOOL_DEPENDENCIES" -eq 1 ]; then # Install Conda environment if needed. - python ./scripts/manage_tool_dependencies.py $config_file_arg init_if_needed + python ./scripts/manage_tool_dependencies.py init_if_needed fi [ -n "$GALAXY_UWSGI" ] && APP_WEBSERVER='uwsgi' -find_server ${GALAXY_CONFIG_FILE:-none} galaxy +find_server "${GALAXY_CONFIG_FILE:-none}" galaxy if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then servers=$(sed -n 's/^\[server:\(.*\)\]/\1/ p' "$GALAXY_CONFIG_FILE" | xargs echo) @@ -78,30 +63,33 @@ if [ "$run_server" = "python" -a -n "$GALAXY_RUN_ALL" ]; then exit 1 fi for server in $servers; do + echo "Executing: python $server_args --server-name=\"$server\" --pid-file=\"$server.pid\" --log-file=\"$server.log\"" + python $server_args --server-name="$server" --pid-file="$server.pid" --log-file="$server.log" if [ -n "$wait_arg_set" -a -n "$daemon_or_restart_arg_set" ]; then - python ./scripts/paster.py serve "$GALAXY_CONFIG_FILE" --server-name="$server" --pid-file="$server.pid" --log-file="$server.log" $paster_args while true; do sleep 1 - printf "." - # Grab the current pid from the pid file - if ! current_pid_in_file=$(cat "$server.pid"); then + # Grab the current pid from the pid file and remove any trailing space + if ! current_pid_in_file=$(sed -e 's/[[:space:]]*$//' "$server.pid"); then echo "A Galaxy process died, interrupting" >&2 exit 1 fi + if [ -n "$current_pid_in_file" ]; then + echo "Found PID $current_pid_in_file in '$server.pid', monitoring '$server.log'" + else + echo "No PID found in '$server.pid' yet" + continue + fi # Search for all pids in the logs and tail for the last one - latest_pid=$(egrep '^Starting server in PID [0-9]+\.$' "$server.log" -o | sed 's/Starting server in PID //g;s/\.$//g' | tail -n 1) + latest_pid=$(grep '^Starting server in PID [0-9]\+\.$' "$server.log" | sed 's/^Starting server in PID \([0-9]\+\).$/\1/' | tail -n 1) # If they're equivalent, then the current pid file agrees with our logs # and we've succesfully started [ -n "$latest_pid" ] && [ "$latest_pid" -eq "$current_pid_in_file" ] && break done echo - else - echo "Handling $server with log file $server.log..." - python ./scripts/paster.py serve "$GALAXY_CONFIG_FILE" --server-name="$server" --pid-file="$server.pid" --log-file="$server.log" $paster_args fi done else - echo "executing: $run_server $server_args" + echo "Executing: $run_server $server_args $pid_log_paster_args" # args are properly quoted so use eval - eval $run_server $server_args + eval $run_server $server_args $pid_log_paster_args fi diff --git a/run_reports.sh b/run_reports.sh index 9692904d41b..de869ad0e48 100755 --- a/run_reports.sh +++ b/run_reports.sh @@ -31,15 +31,7 @@ run_common_start_up setup_python if [ -z "$GALAXY_REPORTS_CONFIG" ]; then - if [ -f reports_wsgi.ini ]; then - GALAXY_REPORTS_CONFIG=reports_wsgi.ini - elif [ -f config/reports_wsgi.ini ]; then - GALAXY_REPORTS_CONFIG=config/reports_wsgi.ini - elif [ -f config/reports.ini ]; then - GALAXY_REPORTS_CONFIG=config/reports.ini - elif [ -f config/reports.yml ]; then - GALAXY_REPORTS_CONFIG=config/reports.yml - fi + GALAXY_REPORTS_CONFIG=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['reports', 'reports_wsgi']) or '')") export GALAXY_REPORTS_CONFIG fi @@ -48,5 +40,5 @@ if [ -n "$GALAXY_REPORTS_CONFIG_DIR" ]; then fi find_server ${GALAXY_REPORTS_CONFIG:-none} reports -echo "executing: $run_server $server_args" -eval $run_server $server_args +echo "Executing: $run_server $server_args $pid_log_paster_args" +eval $run_server $server_args $pid_log_paster_args diff --git a/run_tool_shed.sh b/run_tool_shed.sh index e5bd35749e8..06675f0a955 100755 --- a/run_tool_shed.sh +++ b/run_tool_shed.sh @@ -17,25 +17,19 @@ run_common_start_up setup_python -tool_shed=`./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $parser_args` +tool_shed=$(./scripts/tool_shed/bootstrap_tool_shed/parse_run_sh_args.sh $parser_args) args=$parser_args if [ $? -eq 0 ] ; then bash ./scripts/tool_shed/bootstrap_tool_shed/bootstrap_tool_shed.sh $parser_args - args=`echo $@ | sed "s#-\?-bootstrap_from_tool_shed $tool_shed##"` + args=$(echo $@ | sed "s#-\?-bootstrap_from_tool_shed $tool_shed##") fi if [ -z "$TOOL_SHED_CONFIG_FILE" ]; then - if [ -f tool_shed_wsgi.ini ]; then - TOOL_SHED_CONFIG_FILE=tool_shed_wsgi.ini - elif [ -f config/tool_shed.ini ]; then - TOOL_SHED_CONFIG_FILE=config/tool_shed.ini - elif [ -f config/tool_shed.yml ]; then - TOOL_SHED_CONFIG_FILE=config/tool_shed.yml - fi + TOOL_SHED_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['tool_shed', 'tool_shed_wsgi']) or '')") export TOOL_SHED_CONFIG_FILE fi find_server ${TOOL_SHED_CONFIG_FILE:-none} tool_shed -echo "executing: $run_server $server_args" -eval $run_server $server_args +echo "Executing: $run_server $server_args $pid_log_paster_args" +eval $run_server $server_args $pid_log_paster_args diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index eacee81e392..d943f615acc 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -96,17 +96,6 @@ else DEV_WHEELS=1 fi -: ${GALAXY_CONFIG_FILE:=config/galaxy.yml} -if [ ! -f "$GALAXY_CONFIG_FILE" ]; then - GALAXY_CONFIG_FILE=config/galaxy.ini -fi -if [ ! -f "$GALAXY_CONFIG_FILE" ]; then - GALAXY_CONFIG_FILE=universe_wsgi.ini -fi -if [ ! -f "$GALAXY_CONFIG_FILE" ]; then - GALAXY_CONFIG_FILE=config/galaxy.yml.sample -fi - : ${GALAXY_VIRTUAL_ENV:=.venv} if [ $SET_VENV -eq 1 -a $CREATE_VENV -eq 1 ]; then diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 5d0fa87f8b1..fca6db04e75 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -22,30 +22,33 @@ parse_common_args() { --stop-daemon|stop) common_startup_args="$common_startup_args --stop-daemon" paster_args="$paster_args --stop-daemon" - uwsgi_args="$uwsgi_args --stop $PID_FILE" + pid_log_paster_args="--pid-file \"$PID_FILE\"" + uwsgi_args="$uwsgi_args --stop \"$PID_FILE\"" stop_daemon_arg_set=1 shift ;; --restart|restart) - if [ "$1" = "--restart" ] - then - paster_args="$paster_args restart" - else - paster_args="$paster_args $1" - fi - uwsgi_args="$uwsgi_args --reload $PID_FILE" + paster_args="$paster_args restart" + pid_log_paster_args="--pid-file \"$PID_FILE\" --log-file \"$LOG_FILE\"" + uwsgi_args="$uwsgi_args --reload \"$PID_FILE\"" restart_arg_set=1 daemon_or_restart_arg_set=1 shift ;; --daemon|start) paster_args="$paster_args --daemon" + pid_log_paster_args="--pid-file \"$PID_FILE\" --log-file \"$LOG_FILE\"" # --daemonize2 waits until after the application has loaded # to daemonize, thus it stops if any errors are found - uwsgi_args="--master --daemonize2 $LOG_FILE --pidfile2 $PID_FILE $uwsgi_args" + uwsgi_args="--master --daemonize2 \"$LOG_FILE\" --pidfile2 \"$PID_FILE\" $uwsgi_args" daemon_or_restart_arg_set=1 shift ;; + --status|status) + paster_args="$paster_args $1" + pid_log_paster_args="--pid-file \"$PID_FILE\"" + shift + ;; --wait) wait_arg_set=1 shift @@ -84,9 +87,16 @@ setup_python() { python ./scripts/check_python.py || exit 1 } +set_galaxy_config_file_var() { + if [ -z "$GALAXY_CONFIG_FILE" ]; then + GALAXY_CONFIG_FILE=$(PYTHONPATH=lib python -c "from __future__ import print_function; from galaxy.util.properties import find_config_file; print(find_config_file(['galaxy', 'universe_wsgi']) or '')") + export GALAXY_CONFIG_FILE + fi +} + find_server() { - server_config="$1" - server_app="$2" + server_config=$1 + server_app=$2 arg_getter_args= default_webserver="paste" case "$server_config" in @@ -107,16 +117,17 @@ find_server() { echo 'ERROR: Could not find uwsgi executable' exit 1 fi - [ "$server_config" != "none" ] && arg_getter_args="-c $server_config" + [ "$server_config" != "none" ] && arg_getter_args="-c \"$server_config\"" [ -n "$server_app" ] && arg_getter_args="--app $server_app" run_server="$UWSGI" server_args= if [ -z "$stop_daemon_arg_set" -a -z "$restart_arg_set" ]; then - server_args="$(python ./scripts/get_uwsgi_args.py $arg_getter_args)" + server_args="$(eval python ./scripts/get_uwsgi_args.py $arg_getter_args)" fi server_args="$server_args $uwsgi_args" + pid_log_paster_args="" else run_server="python" - server_args="./scripts/paster.py serve $server_config --pid-file $PID_FILE --log-file $LOG_FILE $paster_args" + server_args="./scripts/paster.py serve \"$server_config\" $paster_args" fi } diff --git a/scripts/common_startup_variables.sh b/scripts/common_startup_variables.sh deleted file mode 100755 index dd3e9a95b1b..00000000000 --- a/scripts/common_startup_variables.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env bash - -# This file is sourced in the maintenance.sh script and could be sourced in ./run.sh or ./run_reports.sh -SCRIPTLOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -# If there is a file that defines a shell environment specific to this -# instance of Galaxy, source the file. -if [ -z $GALAXY_LOCAL_ENV_FILE ]; -then - GALAXY_LOCAL_ENV_FILE="$SCRIPTLOCATION/../config/local_env.sh" -fi - -if [ -f $GALAXY_LOCAL_ENV_FILE ]; -then - echo "Activating local env file: $GALAXY_LOCAL_ENV_FILE" - . $GALAXY_LOCAL_ENV_FILE -fi - -# If there is a .venv/ directory, assume it contains a virtualenv that we -# should run this instance in. -GALAXY_VIRTUAL_ENV="${GALAXY_VIRTUAL_ENV:-$SCRIPTLOCATION/../.venv}" -if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ]; -then - [ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; } - echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV" - . "$GALAXY_VIRTUAL_ENV/bin/activate" -fi -if [ -z "$GALAXY_CONFIG_FILE" ]; then - if [ -f "$SCRIPTLOCATION/../universe_wsgi.ini" ]; then - GALAXY_CONFIG_FILE=universe_wsgi.ini - elif [ -f "$SCRIPTLOCATION/../config/galaxy.ini" ]; then - GALAXY_CONFIG_FILE=config/galaxy.ini - else - GALAXY_CONFIG_FILE=config/galaxy.ini.sample - fi - export GALAXY_CONFIG_FILE -fi - -echo "GALAXY_CONFIG_FILE: ${GALAXY_CONFIG_FILE}" diff --git a/scripts/maintenance.sh b/scripts/maintenance.sh index 757da5efd00..8ca4771ed54 100755 --- a/scripts/maintenance.sh +++ b/scripts/maintenance.sh @@ -52,8 +52,13 @@ do esac done -SCRIPTLOCATION="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" -source ${SCRIPTLOCATION}/common_startup_variables.sh +cd "$(cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd)"/../ + +. scripts/common_startup_functions.sh + +setup_python + +set_galaxy_config_file_var if [ "$DRYRUN" = true ]; then MODE="--info_only" @@ -61,14 +66,14 @@ else MODE="-r" fi -MAINTENANCE_LOG="$SCRIPTLOCATION/../maintenance.log" +MAINTENANCE_LOG=maintenance.log COMMANDS=( -"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_userless_histories >> $MAINTENANCE_LOG" -"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_histories >> $MAINTENANCE_LOG" -"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG" -"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_folders >> $MAINTENANCE_LOG" -"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_datasets >> $MAINTENANCE_LOG" -"python $SCRIPTLOCATION/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG") +"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_userless_histories >> $MAINTENANCE_LOG" +"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_histories >> $MAINTENANCE_LOG" +"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG" +"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_folders >> $MAINTENANCE_LOG" +"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --delete_datasets >> $MAINTENANCE_LOG" +"python scripts/cleanup_datasets/cleanup_datasets.py $GALAXY_CONFIG_FILE -d $DAYS $MODE --purge_datasets >> $MAINTENANCE_LOG") printf "\nDry run: $DRYRUN\nDays: $DAYS\n\n" echo "Will run following commands and output in $MAINTENANCE_LOG" @@ -77,15 +82,14 @@ for (( i = 0; i < ${#COMMANDS[@]} ; i++ )); do done if [ "$DRYRUN" = false ]; then - echo "python $SCRIPTLOCATION/set_user_disk_usage.py >> $MAINTENANCE_LOG" + echo "python scripts/set_user_disk_usage.py >> $MAINTENANCE_LOG" fi # Run the commands -cd $SCRIPTLOCATION/../ for (( i = 0; i < ${#COMMANDS[@]} ; i++ )); do eval "${COMMANDS[$i]}" done if [ "$DRYRUN" = false ]; then - python $SCRIPTLOCATION/set_user_disk_usage.py >> $MAINTENANCE_LOG + python scripts/set_user_disk_usage.py >> $MAINTENANCE_LOG fi