From 2409312f8c4276c72ba8a46d71dd484edd1fa2fd Mon Sep 17 00:00:00 2001 From: Arash Date: Mon, 6 Oct 2025 13:32:19 +0200 Subject: [PATCH] Makes uv optional with pip fallbacks --- lib/galaxy/dependencies/update.sh | 31 +++-- .../dependencies/update_lint_requirements.sh | 33 ++---- packages/package-build-install.sh | 50 ++++---- packages/test.sh | 52 ++++---- scripts/common_startup.sh | 112 +++++++++--------- test/release.sh | 19 +-- 6 files changed, 139 insertions(+), 158 deletions(-) diff --git a/lib/galaxy/dependencies/update.sh b/lib/galaxy/dependencies/update.sh index be0cdbf9675..bb0977fa456 100755 --- a/lib/galaxy/dependencies/update.sh +++ b/lib/galaxy/dependencies/update.sh @@ -28,29 +28,26 @@ do done shift $((OPTIND - 1)) -# Install uv for fast operation -if ! command -v uv >/dev/null; then - echo "Installing uv..." - if command -v curl >/dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv - elif command -v wget >/dev/null; then - wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv - else - python3 -m pip install uv - fi - export PATH="$HOME/.local/bin:$PATH" +# Create a virtual environment in a tmp directory and install uv into it +if command -v uv >/dev/null; then + uv="$(command -v uv)" +else + uv_venv=$(mktemp -d "${TMPDIR:-/tmp}/uv_venv.XXXXXXXXXX") + python3 -m venv "${uv_venv}" + "${uv_venv}/bin/python" -m pip install uv + uv="${uv_venv}/bin/uv" fi # Run uv (this may update pyproject.toml and uv.lock). if [ -n "$pkg" ]; then - uv lock --upgrade-package "$pkg" + ${uv} lock --upgrade-package "$pkg" else - uv lock --upgrade + ${uv} lock --upgrade fi # Update pinned requirements files. UV_EXPORT_OPTIONS='--frozen --no-annotate --no-hashes' -uv export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt" -uv export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt" -uv export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt" -uv export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt" +${uv} export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt" +${uv} export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt" +${uv} export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt" +${uv} export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt" diff --git a/lib/galaxy/dependencies/update_lint_requirements.sh b/lib/galaxy/dependencies/update_lint_requirements.sh index 959820225bc..16f043f62b8 100755 --- a/lib/galaxy/dependencies/update_lint_requirements.sh +++ b/lib/galaxy/dependencies/update_lint_requirements.sh @@ -10,28 +10,19 @@ THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" update_pinned_reqs() { VENV=$(mktemp -d "${TMPDIR:-/tmp}/$1_venv.XXXXXXXXXX") - - # Install uv for fast package installation - if ! command -v uv >/dev/null; then - echo "Installing uv..." - if command -v curl >/dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv - elif command -v wget >/dev/null; then - wget -qO- https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv - else - python3.9 -m pip install uv - fi - export PATH="$HOME/.local/bin:$PATH" + if command -v uv >/dev/null; then + uv venv "${VENV}" --python python3.9 + . "${VENV}/bin/activate" + uv pip install -r "${THIS_DIRECTORY}/$1-requirements.txt" + uv pip freeze | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt" + else + python3.9 -m venv "${VENV}" + . "${VENV}/bin/activate" + pip install --upgrade pip setuptools + pip install -r "${THIS_DIRECTORY}/$1-requirements.txt" + # The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 + pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt" fi - - # Use uv venv for fast virtual environment creation - uv venv "${VENV}" --python python3.9 - . "${VENV}/bin/activate" - - # No need to upgrade pip/setuptools - uv handles everything - uv pip install -r "${THIS_DIRECTORY}/$1-requirements.txt" - # The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 - uv pip freeze | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt" rm -rf "${VENV}" } diff --git a/packages/package-build-install.sh b/packages/package-build-install.sh index 380c341ac5c..abe7ce44269 100755 --- a/packages/package-build-install.sh +++ b/packages/package-build-install.sh @@ -11,21 +11,6 @@ set -euo pipefail : ${PIP_EXTRA_ARGS:=--extra-index-url https://wheels.galaxyproject.org} #: ${SETUP_VENV:=true} -# Ensure uv is installed -ensure_uv() { - if ! command -v uv >/dev/null; then - echo "Installing uv..." - if command -v curl >/dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv - elif command -v wget >/dev/null; then - wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv - else - python3 -m pip install uv - fi - export PATH="$HOME/.local/bin:$PATH" - fi -} - INSTALL=true EDITABLE=false META=false @@ -52,7 +37,7 @@ do h) echo "usage: $0 [-bem] [up_to_package]" echo " -b build only, no install" - echo " -e install packages in \"editable\" mode (uv pip install -e)" + echo " -e install packages in \"editable\" mode (pip install -e)" echo " -m install galaxy metapackage, installing pinned deps in meta/requirements.txt" exit 0 ;; @@ -80,21 +65,28 @@ while read package; do printf "\n========= PACKAGE %s =========\n\n" "$package" pushd $package if $EDITABLE; then - # Install package in editable mode using uv (much faster than pip) - ensure_uv - uv pip install -e . + if command -v uv >/dev/null; then + uv pip install -e . + else + pip install -e . + fi else if [ ! -d "$VENV" ]; then - # Install uv for fast venv creation and package management - ensure_uv - - # Use uv venv for fast virtual environment creation - uv venv "$VENV" --python python3 - "${VENV}/bin/uv" pip install -r <(grep -v test-requirements.txt dev-requirements.txt) + if command -v uv >/dev/null; then + uv venv "$VENV" + uv pip install --python "${VENV}/bin/python" -r <(grep -v test-requirements.txt dev-requirements.txt) + else + python3 -m venv "$VENV" + "${VENV}/bin/pip" install -r <(grep -v test-requirements.txt dev-requirements.txt) + fi fi make dist if $INSTALL && ! $META; then - uv pip install dist/*.whl + if command -v uv >/dev/null; then + uv pip install dist/*.whl + else + pip install dist/*.whl + fi fi fi popd @@ -104,5 +96,9 @@ done < "$PACKAGE_LIST_FILE" if $INSTALL && $META && ! $EDITABLE; then WHEELHOUSE=$(mktemp -d -t gxpkgwheelhouseXXXXXX) cp */dist/*.whl "$WHEELHOUSE" - uv pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl + if command -v uv >/dev/null; then + uv pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl + else + pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl + fi fi diff --git a/packages/test.sh b/packages/test.sh index e23beb8e6cc..55edc133c8a 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -36,27 +36,23 @@ cd "$(dirname "$0")" TEST_PYTHON=${TEST_PYTHON:-"python3"} TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} -# Install uv for fast package installation -if ! command -v uv >/dev/null; then - echo "Installing uv..." - if command -v curl >/dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv - elif command -v wget >/dev/null; then - wget -qO- https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv - else - "$TEST_PYTHON" -m pip install uv - fi - export PATH="$HOME/.local/bin:$PATH" +if command -v uv >/dev/null; then + uv venv "$TEST_ENV_DIR" --python "$TEST_PYTHON" +else + "$TEST_PYTHON" -m venv "$TEST_ENV_DIR" fi -# Use uv venv for much faster virtual environment creation -uv venv "$TEST_ENV_DIR" --python "$TEST_PYTHON" # shellcheck disable=SC1091 . "${TEST_ENV_DIR}/bin/activate" - -# Note: No need to upgrade pip anymore since we're using uv exclusively +if ! command -v uv >/dev/null; then + pip install --upgrade pip setuptools wheel +fi if [ $FOR_PULSAR -eq 0 ]; then - uv pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt + if command -v uv >/dev/null; then + uv pip install -r ../lib/galaxy/dependencies/pinned-typecheck-requirements.txt + else + pip install -r ../lib/galaxy/dependencies/pinned-typecheck-requirements.txt + fi fi # Ensure ordered by dependency DAG @@ -80,13 +76,29 @@ while read -r package_dir || [ -n "$package_dir" ]; do # https://stackoverflow. # Install extras (if needed) if [ "$package_dir" = "util" ]; then - uv pip install '.[image-util,template,jstree,config-template,test]' + if command -v uv >/dev/null; then + uv pip install '.[image-util,template,jstree,config-template,test]' + else + pip install '.[image-util,template,jstree,config-template,test]' + fi elif [ "$package_dir" = "tool_util" ]; then - uv pip install '.[cwl,mulled,edam,extended-assertions,test]' + if command -v uv >/dev/null; then + uv pip install '.[cwl,mulled,edam,extended-assertions,test]' + else + pip install '.[cwl,mulled,edam,extended-assertions,test]' + fi elif grep -q 'test =' setup.cfg 2>/dev/null; then - uv pip install '.[test]' + if command -v uv >/dev/null; then + uv pip install '.[test]' + else + pip install '.[test]' + fi else - uv pip install . + if command -v uv >/dev/null; then + uv pip install . + else + pip install . + fi fi if [ $FOR_PULSAR -eq 0 ]; then diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 58018577b0f..972e68021d4 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -4,22 +4,6 @@ set -e # The caller may do this as well, but since common_startup.sh can be called independently, we need to do it here . ./scripts/common_startup_functions.sh -# Ensure uv is installed (used throughout this script) -ensure_uv() { - if ! command -v uv >/dev/null; then - echo "Installing uv..." - local python_cmd="${1:-python3}" - if command -v curl >/dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}" - elif command -v wget >/dev/null; then - wget -qO- https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}" - else - "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}" - fi - export PATH="$HOME/.local/bin:$PATH" - fi -} - DEV_WHEELS=0 FETCH_WHEELS=1 CREATE_VENV=1 @@ -54,7 +38,7 @@ RMFILES=" " MIN_PYTHON_VERSION=3.9 -MIN_UV_VERSION=0.5.0 +MIN_PIP_VERSION=20.3 # return true if $1 is in $2 else false in_dir() { @@ -128,15 +112,16 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV" echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an" echo "existing environment before starting Galaxy." - # Note: uv will be installed via standalone installer, not conda - $CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}" + $CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}" "pip>=${MIN_PIP_VERSION}" unset __CONDA_INFO fi conda_activate fi - # Install uv first if not available, then use uv venv for fast virtual environment creation - ensure_uv python - uv venv "$GALAXY_VIRTUAL_ENV" --python python3 + if command -v uv >/dev/null; then + uv venv "$GALAXY_VIRTUAL_ENV" + else + python3 -m venv "$GALAXY_VIRTUAL_ENV" + fi else # If $GALAXY_VIRTUAL_ENV does not exist, and there is no conda available, attempt to create it. @@ -147,35 +132,36 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then echo "using Python: $GALAXY_PYTHON" echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_VIRTUAL_ENV to an" echo "existing environment before starting Galaxy." - - # Install uv first if not available, then use uv venv for fast virtual environment creation - ensure_uv "$GALAXY_PYTHON" - - # Use uv venv for much faster virtual environment creation - if ! uv venv "$GALAXY_VIRTUAL_ENV" --python "$GALAXY_PYTHON"; then - echo "Creating the Python virtual environment using uv failed." - echo "Trying with virtualenv now as fallback." - if command -v virtualenv >/dev/null; then - virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV" - else - # Download virtualenv zipapp as last resort - vurl="https://bootstrap.pypa.io/virtualenv/${MIN_PYTHON_VERSION}/virtualenv.pyz" - vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX) - vsrc="$vtmp/$(basename $vurl)" - echo "Fetching $vurl" - if command -v curl >/dev/null; then - curl -L -o "$vsrc" "$vurl" - elif command -v wget >/dev/null; then - wget -O "$vsrc" "$vurl" + if command -v uv >/dev/null; then + uv venv "$GALAXY_VIRTUAL_ENV" --python "$GALAXY_PYTHON" + else + # First try to use the venv standard library module, although it is + # not always installed by default on Linux distributions. + if ! "$GALAXY_PYTHON" -m venv "$GALAXY_VIRTUAL_ENV"; then + echo "Creating the Python virtual environment using the venv standard library module failed." + echo "Trying with virtualenv now." + if command -v virtualenv >/dev/null; then + virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV" else - "$GALAXY_PYTHON" -c "try: - from urllib import urlretrieve - except: - from urllib.request import urlretrieve - urlretrieve('$vurl', '$vsrc')" + # Download virtualenv zipapp + vurl="https://bootstrap.pypa.io/virtualenv/${MIN_PYTHON_VERSION}/virtualenv.pyz" + vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX) + vsrc="$vtmp/$(basename $vurl)" + echo "Fetching $vurl" + if command -v curl >/dev/null; then + curl -L -o "$vsrc" "$vurl" + elif command -v wget >/dev/null; then + wget -O "$vsrc" "$vurl" + else + "$GALAXY_PYTHON" -c "try: + from urllib import urlretrieve + except: + from urllib.request import urlretrieve + urlretrieve('$vurl', '$vsrc')" + fi + "$GALAXY_PYTHON" "$vsrc" "$GALAXY_VIRTUAL_ENV" + rm -rf "$vtmp" fi - "$GALAXY_PYTHON" "$vsrc" "$GALAXY_VIRTUAL_ENV" - rm -rf "$vtmp" fi fi fi @@ -202,22 +188,30 @@ fi [ "$CI" = 'true' ] && export PIP_PROGRESS_BAR=off if [ $FETCH_WHEELS -eq 1 ]; then - # Ensure uv is installed - uv is much faster than pip (10-100x) - ensure_uv python - - # Use uv pip install for fast package installation - # Note: uv pip install is significantly faster than pip install # shellcheck disable=SC2086 - uv pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + if command -v uv >/dev/null; then + uv pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + else + python -m pip install "pip>=$MIN_PIP_VERSION" wheel + pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + fi set_galaxy_config_file_var GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))") if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then - if uv pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then - echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2." - uv pip uninstall psycopg2 psycopg2-binary + if command -v uv >/dev/null; then + if uv pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then + echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2." + uv pip uninstall psycopg2 psycopg2-binary + fi + echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | uv pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + else + if pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then + echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2." + pip uninstall -y psycopg2 psycopg2-binary + fi + echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" fi - echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | uv pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" fi fi diff --git a/test/release.sh b/test/release.sh index bd124c527e5..07891ee2750 100755 --- a/test/release.sh +++ b/test/release.sh @@ -131,21 +131,12 @@ function make_forks() { function create_venv() { if [ ! -d "$VENV" ]; then - # Install uv for fast venv creation and package management - if ! command -v uv >/dev/null; then - echo "Installing uv..." - if command -v curl >/dev/null; then - curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv - elif command -v wget >/dev/null; then - wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv - else - python3 -m pip install uv - fi - export PATH="$HOME/.local/bin:$PATH" + if command -v uv >/dev/null; then + log_exec uv venv "$VENV" + else + log_exec python3 -m venv "$VENV" + log_exec "${VENV}/bin/pip" install wheel packaging fi - - log_exec uv venv "$VENV" - log_exec "${VENV}/bin/uv" pip install wheel packaging fi . "${VENV}/bin/activate" }