From b43236d70b74d854a2317c23552d5fc856658b8c Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 20 Feb 2020 16:44:37 -0500 Subject: [PATCH 1/5] Fix shed image resolution to actually test base-path images as intended (and then fail through to static/images) --- .../galaxy/controllers/shed_tool_static.py | 40 +++++++++++++------ 1 file changed, 27 insertions(+), 13 deletions(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py b/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py index d938f86c958..bcc722f81db 100644 --- a/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py +++ b/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py @@ -9,6 +9,12 @@ from galaxy.webapps.base.controller import BaseUIController log = logging.getLogger(__name__) +def _asset_exists_and_is_safe(repo_path, asset_path): + if not safe_contains(repo_path, asset_path): + raise RequestParameterInvalidException() + return os.path.exists(asset_path) + + class ShedToolStatic(BaseUIController): @web.expose @@ -26,17 +32,25 @@ class ShedToolStatic(BaseUIController): """ guid = '/'.join([shed, 'repos', owner, repo, tool, version]) tool = trans.app.toolbox.get_tool(guid) - repo_path = tool._repository_dir - if 'static/images' not in image_file: - path = join(repo_path, 'static', 'images', image_file) + repo_path = os.path.abspath(tool._repository_dir) + asset_path = os.path.abspath(join(repo_path, image_file)) + + # test specified image_file path exactly first, then fail through to + # other locations. + # Might want to swap this around and check for static/images first if + # that's the new(?) standard. + if not _asset_exists_and_is_safe(repo_path, asset_path): + if 'static/images' not in image_file: + asset_path = join(repo_path, 'static', 'images', image_file) + if not _asset_exists_and_is_safe(repo_path, asset_path): + asset_path = None + + if asset_path: + ext = os.path.splitext(image_file)[-1].lstrip('.') + if ext: + mime = trans.app.datatypes_registry.get_mimetype_by_extension(ext) + if mime: + trans.response.set_content_type(mime) + return open(asset_path, 'rb') else: - path = join(repo_path, image_file) - if not safe_contains(os.path.abspath(repo_path), os.path.abspath(path)): - raise RequestParameterInvalidException() - ext = os.path.splitext(image_file)[-1].lstrip('.') - if ext: - mime = trans.app.datatypes_registry.get_mimetype_by_extension(ext) - if mime: - trans.response.set_content_type(mime) - if os.path.exists(path): - return open(path, 'rb') + return None From 6ed152d9763a076fb87e3e36df0a19d95cfd0f79 Mon Sep 17 00:00:00 2001 From: Dannon Baker Date: Thu, 20 Feb 2020 17:50:46 -0500 Subject: [PATCH 2/5] Throw exception for invalid requested image instead of just returning nothing, so we can catch these in sentry. --- lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py b/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py index bcc722f81db..042e8184f9c 100644 --- a/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py +++ b/lib/galaxy/webapps/galaxy/controllers/shed_tool_static.py @@ -53,4 +53,4 @@ class ShedToolStatic(BaseUIController): trans.response.set_content_type(mime) return open(asset_path, 'rb') else: - return None + raise RequestParameterInvalidException() From f61ae2c42ae201ff177eb842a693f9488385ff14 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sat, 22 Feb 2020 02:22:17 +0000 Subject: [PATCH 3/5] Don't check if python is from conda when `CONDA_EXE` env var is set Since at least version 3.7.4 of the `python` package from the `defaults` conda channel, `python -V` doesn't reveal its origin any more. Also some fixes suggested by `shellcheck`. --- scripts/common_startup_functions.sh | 16 ++++++++-------- tox.ini | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 2a1255bd894..1ec379a2be1 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -99,7 +99,7 @@ setup_python() { # should run this instance in. : ${GALAXY_VIRTUAL_ENV:=.venv} # $GALAXY_CONDA_ENV isn't set here to avoid running the version check if not using Conda - if [ -d "$GALAXY_VIRTUAL_ENV" -a -z "$skip_venv" ]; then + if [ -d "$GALAXY_VIRTUAL_ENV" ] && [ -z "$skip_venv" ]; then [ -n "$PYTHONPATH" ] && { echo 'Unsetting $PYTHONPATH'; unset PYTHONPATH; } echo "Activating virtualenv at $GALAXY_VIRTUAL_ENV" . "$GALAXY_VIRTUAL_ENV/bin/activate" @@ -148,7 +148,7 @@ find_server() { APP_WEBSERVER=${APP_WEBSERVER:-$default_webserver} if [ "$APP_WEBSERVER" = "uwsgi" ]; then # Look for uwsgi - if [ -z "$skip_venv" -a -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then + if [ -z "$skip_venv" ] && [ -x $GALAXY_VIRTUAL_ENV/bin/uwsgi ]; then UWSGI=$GALAXY_VIRTUAL_ENV/bin/uwsgi elif command -v uwsgi >/dev/null 2>&1; then UWSGI=uwsgi @@ -160,7 +160,7 @@ find_server() { [ -n "$server_app" ] && arg_getter_args="$arg_getter_args --app $server_app" run_server="$UWSGI" server_args= - if [ -z "$stop_daemon_arg_set" -a -z "$restart_arg_set" ]; then + if [ -z "$stop_daemon_arg_set" ] && [ -z "$restart_arg_set" ]; then server_args="$(eval python ./scripts/get_uwsgi_args.py $arg_getter_args)" fi server_args="$server_args $uwsgi_args" @@ -193,13 +193,13 @@ find_server() { # to the `conda` script in the base environment. Thus in Conda 4.4, it may not be possible to locate `conda` even if you # are using Conda. set_conda_exe() { - [ -z "$_CONDA_EXE_SET" ] || return 0 + [ -n "$CONDA_EXE" ] || [ -n "$_CONDA_EXE_SET" ] && return 0 if python -V 2>&1 | grep -q -e 'Anaconda' -e 'Continuum Analytics' || \ python -c 'import sys; print(sys.version.replace("\n", " "))' | grep -q -e 'packaged by conda-forge' ; then - : ${CONDA_EXE:=$(command -v conda)} + CONDA_EXE=$(command -v conda) if [ -z "$CONDA_EXE" ]; then echo "WARNING: \`python\` is from conda, but the \`conda\` command cannot be found." - pydir="$(dirname $(command -v python))" + pydir="$(dirname "$(command -v python)")" for CONDA_EXE in $pydir/conda $pydir/../../../bin/conda; do [ -x "$CONDA_EXE" ] && break || unset CONDA_EXE done @@ -209,7 +209,7 @@ set_conda_exe() { echo " $ conda activate base" else echo "Guessed conda location: $CONDA_EXE" - PATH="$(dirname $CONDA_EXE):$PATH" + PATH="$(dirname "$CONDA_EXE"):$PATH" fi else echo "Found conda at: $CONDA_EXE" @@ -220,7 +220,7 @@ set_conda_exe() { set_conda_info() { # cache conda info to avoid the cost of running it multiple times - if [ -z "$__CONDA_INFO" -o "$1" = "reset" ]; then + if [ -z "$__CONDA_INFO" ]; then __CONDA_INFO="$(${CONDA_EXE:-conda} info --json)" fi } diff --git a/tox.ini b/tox.ini index 6db822ade1a..3fe6643ade9 100644 --- a/tox.ini +++ b/tox.ini @@ -10,7 +10,7 @@ commands = lint: bash .ci/flake8_wrapper.sh unit: bash run_tests.sh -u whitelist_externals = bash -passenv = CI +passenv = CI CONDA_EXE setenv = py{35,36,37}-first_startup: GALAXY_VIRTUAL_ENV=.venv3 unit: GALAXY_VIRTUAL_ENV={envdir} From da256571e679df5e107a02c6c000399d51fe6786 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sun, 23 Feb 2020 01:21:05 +0000 Subject: [PATCH 4/5] Source the full path of conda activate script Otherwise, if a virtualenv is active on top of conda base env (e.g. when running py37-first_startup tox test in TravisCI), the wrong `activate` script is sourced. Also fix `CONDA_PREFIX` environment variable definition when using a shell not supported by conda. --- scripts/common_startup_functions.sh | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/scripts/common_startup_functions.sh b/scripts/common_startup_functions.sh index 1ec379a2be1..fef20d27261 100644 --- a/scripts/common_startup_functions.sh +++ b/scripts/common_startup_functions.sh @@ -88,9 +88,9 @@ conda_activate() { echo " starting Galaxy." PATH="$(get_conda_env_path $GALAXY_CONDA_ENV)/bin:$PATH" CONDA_DEFAULT_ENV="$GALAXY_CONDA_ENV" - CONDA_PREFIX="$(get_conda_root_path)" + CONDA_PREFIX="$(get_conda_active_prefix)" else - source activate "$GALAXY_CONDA_ENV" + source "$(get_conda_root_prefix)"/bin/activate "$GALAXY_CONDA_ENV" fi } @@ -225,7 +225,13 @@ set_conda_info() { fi } -get_conda_root_path() { +get_conda_active_prefix() { + set_conda_info + printf "%s" "$__CONDA_INFO" \ + | python -c "import json, sys; print(json.load(sys.stdin)['active_prefix'])" +} + +get_conda_root_prefix() { set_conda_info printf "%s" "$__CONDA_INFO" \ | python -c "import json, sys; print(json.load(sys.stdin)['root_prefix'])" From acb0674b5e87c6c1c507ae9994387d6bd73cfff4 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Mon, 24 Feb 2020 02:11:02 +0000 Subject: [PATCH 5/5] Upgrade Checkout GitHub Action --- .github/workflows/converter_tests.yaml | 2 +- .github/workflows/integration.yaml | 2 +- .github/workflows/mulled.yaml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/converter_tests.yaml b/.github/workflows/converter_tests.yaml index a8cd3f3f7a4..64926d007d3 100644 --- a/.github/workflows/converter_tests.yaml +++ b/.github/workflows/converter_tests.yaml @@ -9,7 +9,7 @@ jobs: matrix: python-version: [3.7] steps: - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-python@v1 diff --git a/.github/workflows/integration.yaml b/.github/workflows/integration.yaml index 4e24129f482..dc865635994 100644 --- a/.github/workflows/integration.yaml +++ b/.github/workflows/integration.yaml @@ -32,7 +32,7 @@ jobs: if: matrix.subset == 'kubernetes' run: | kubectl get pods - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-python@v1 diff --git a/.github/workflows/mulled.yaml b/.github/workflows/mulled.yaml index f31e310452b..4a0ba77ab1f 100644 --- a/.github/workflows/mulled.yaml +++ b/.github/workflows/mulled.yaml @@ -15,7 +15,7 @@ jobs: with: path: .tox/mulled key: tox-mulled-${{ matrix.python-version }} - - uses: actions/checkout@v1 + - uses: actions/checkout@v2 with: fetch-depth: 1 - uses: actions/setup-python@v1