Makes uv optional with pip fallbacks

This commit is contained in:
Arash
2025-10-29 12:45:42 +00:00
committed by Nicola Soranzo
parent 478b88a7d4
commit 2409312f8c
6 changed files with 139 additions and 158 deletions
+14 -17
View File
@@ -28,29 +28,26 @@ do
done
shift $((OPTIND - 1))
# Install uv for fast operation
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
else
python3 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
# Create a virtual environment in a tmp directory and install uv into it
if command -v uv >/dev/null; then
uv="$(command -v uv)"
else
uv_venv=$(mktemp -d "${TMPDIR:-/tmp}/uv_venv.XXXXXXXXXX")
python3 -m venv "${uv_venv}"
"${uv_venv}/bin/python" -m pip install uv
uv="${uv_venv}/bin/uv"
fi
# Run uv (this may update pyproject.toml and uv.lock).
if [ -n "$pkg" ]; then
uv lock --upgrade-package "$pkg"
${uv} lock --upgrade-package "$pkg"
else
uv lock --upgrade
${uv} lock --upgrade
fi
# Update pinned requirements files.
UV_EXPORT_OPTIONS='--frozen --no-annotate --no-hashes'
uv export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt"
uv export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt"
uv export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt"
uv export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt"
${uv} export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt"
${uv} export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt"
${uv} export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt"
${uv} export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt"
@@ -10,28 +10,19 @@ THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)"
update_pinned_reqs() {
VENV=$(mktemp -d "${TMPDIR:-/tmp}/$1_venv.XXXXXXXXXX")
# Install uv for fast package installation
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv
else
python3.9 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
if command -v uv >/dev/null; then
uv venv "${VENV}" --python python3.9
. "${VENV}/bin/activate"
uv pip install -r "${THIS_DIRECTORY}/$1-requirements.txt"
uv pip freeze | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt"
else
python3.9 -m venv "${VENV}"
. "${VENV}/bin/activate"
pip install --upgrade pip setuptools
pip install -r "${THIS_DIRECTORY}/$1-requirements.txt"
# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331
pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt"
fi
# Use uv venv for fast virtual environment creation
uv venv "${VENV}" --python python3.9
. "${VENV}/bin/activate"
# No need to upgrade pip/setuptools - uv handles everything
uv pip install -r "${THIS_DIRECTORY}/$1-requirements.txt"
# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331
uv pip freeze | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt"
rm -rf "${VENV}"
}
+23 -27
View File
@@ -11,21 +11,6 @@ set -euo pipefail
: ${PIP_EXTRA_ARGS:=--extra-index-url https://wheels.galaxyproject.org}
#: ${SETUP_VENV:=true}
# Ensure uv is installed
ensure_uv() {
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
else
python3 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
fi
}
INSTALL=true
EDITABLE=false
META=false
@@ -52,7 +37,7 @@ do
h)
echo "usage: $0 [-bem] [up_to_package]"
echo " -b build only, no install"
echo " -e install packages in \"editable\" mode (uv pip install -e)"
echo " -e install packages in \"editable\" mode (pip install -e)"
echo " -m install galaxy metapackage, installing pinned deps in meta/requirements.txt"
exit 0
;;
@@ -80,21 +65,28 @@ while read package; do
printf "\n========= PACKAGE %s =========\n\n" "$package"
pushd $package
if $EDITABLE; then
# Install package in editable mode using uv (much faster than pip)
ensure_uv
uv pip install -e .
if command -v uv >/dev/null; then
uv pip install -e .
else
pip install -e .
fi
else
if [ ! -d "$VENV" ]; then
# Install uv for fast venv creation and package management
ensure_uv
# Use uv venv for fast virtual environment creation
uv venv "$VENV" --python python3
"${VENV}/bin/uv" pip install -r <(grep -v test-requirements.txt dev-requirements.txt)
if command -v uv >/dev/null; then
uv venv "$VENV"
uv pip install --python "${VENV}/bin/python" -r <(grep -v test-requirements.txt dev-requirements.txt)
else
python3 -m venv "$VENV"
"${VENV}/bin/pip" install -r <(grep -v test-requirements.txt dev-requirements.txt)
fi
fi
make dist
if $INSTALL && ! $META; then
uv pip install dist/*.whl
if command -v uv >/dev/null; then
uv pip install dist/*.whl
else
pip install dist/*.whl
fi
fi
fi
popd
@@ -104,5 +96,9 @@ done < "$PACKAGE_LIST_FILE"
if $INSTALL && $META && ! $EDITABLE; then
WHEELHOUSE=$(mktemp -d -t gxpkgwheelhouseXXXXXX)
cp */dist/*.whl "$WHEELHOUSE"
uv pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl
if command -v uv >/dev/null; then
uv pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl
else
pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl
fi
fi
+32 -20
View File
@@ -36,27 +36,23 @@ cd "$(dirname "$0")"
TEST_PYTHON=${TEST_PYTHON:-"python3"}
TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)}
# Install uv for fast package installation
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv
else
"$TEST_PYTHON" -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
if command -v uv >/dev/null; then
uv venv "$TEST_ENV_DIR" --python "$TEST_PYTHON"
else
"$TEST_PYTHON" -m venv "$TEST_ENV_DIR"
fi
# Use uv venv for much faster virtual environment creation
uv venv "$TEST_ENV_DIR" --python "$TEST_PYTHON"
# shellcheck disable=SC1091
. "${TEST_ENV_DIR}/bin/activate"
# Note: No need to upgrade pip anymore since we're using uv exclusively
if ! command -v uv >/dev/null; then
pip install --upgrade pip setuptools wheel
fi
if [ $FOR_PULSAR -eq 0 ]; then
uv pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
if command -v uv >/dev/null; then
uv pip install -r ../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
else
pip install -r ../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
fi
fi
# Ensure ordered by dependency DAG
@@ -80,13 +76,29 @@ while read -r package_dir || [ -n "$package_dir" ]; do # https://stackoverflow.
# Install extras (if needed)
if [ "$package_dir" = "util" ]; then
uv pip install '.[image-util,template,jstree,config-template,test]'
if command -v uv >/dev/null; then
uv pip install '.[image-util,template,jstree,config-template,test]'
else
pip install '.[image-util,template,jstree,config-template,test]'
fi
elif [ "$package_dir" = "tool_util" ]; then
uv pip install '.[cwl,mulled,edam,extended-assertions,test]'
if command -v uv >/dev/null; then
uv pip install '.[cwl,mulled,edam,extended-assertions,test]'
else
pip install '.[cwl,mulled,edam,extended-assertions,test]'
fi
elif grep -q 'test =' setup.cfg 2>/dev/null; then
uv pip install '.[test]'
if command -v uv >/dev/null; then
uv pip install '.[test]'
else
pip install '.[test]'
fi
else
uv pip install .
if command -v uv >/dev/null; then
uv pip install .
else
pip install .
fi
fi
if [ $FOR_PULSAR -eq 0 ]; then
+53 -59
View File
@@ -4,22 +4,6 @@ set -e
# The caller may do this as well, but since common_startup.sh can be called independently, we need to do it here
. ./scripts/common_startup_functions.sh
# Ensure uv is installed (used throughout this script)
ensure_uv() {
if ! command -v uv >/dev/null; then
echo "Installing uv..."
local python_cmd="${1:-python3}"
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}"
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}"
else
"$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}"
fi
export PATH="$HOME/.local/bin:$PATH"
fi
}
DEV_WHEELS=0
FETCH_WHEELS=1
CREATE_VENV=1
@@ -54,7 +38,7 @@ RMFILES="
"
MIN_PYTHON_VERSION=3.9
MIN_UV_VERSION=0.5.0
MIN_PIP_VERSION=20.3
# return true if $1 is in $2 else false
in_dir() {
@@ -128,15 +112,16 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV"
echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an"
echo "existing environment before starting Galaxy."
# Note: uv will be installed via standalone installer, not conda
$CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}"
$CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}" "pip>=${MIN_PIP_VERSION}"
unset __CONDA_INFO
fi
conda_activate
fi
# Install uv first if not available, then use uv venv for fast virtual environment creation
ensure_uv python
uv venv "$GALAXY_VIRTUAL_ENV" --python python3
if command -v uv >/dev/null; then
uv venv "$GALAXY_VIRTUAL_ENV"
else
python3 -m venv "$GALAXY_VIRTUAL_ENV"
fi
else
# If $GALAXY_VIRTUAL_ENV does not exist, and there is no conda available, attempt to create it.
@@ -147,35 +132,36 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
echo "using Python: $GALAXY_PYTHON"
echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_VIRTUAL_ENV to an"
echo "existing environment before starting Galaxy."
# Install uv first if not available, then use uv venv for fast virtual environment creation
ensure_uv "$GALAXY_PYTHON"
# Use uv venv for much faster virtual environment creation
if ! uv venv "$GALAXY_VIRTUAL_ENV" --python "$GALAXY_PYTHON"; then
echo "Creating the Python virtual environment using uv failed."
echo "Trying with virtualenv now as fallback."
if command -v virtualenv >/dev/null; then
virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV"
else
# Download virtualenv zipapp as last resort
vurl="https://bootstrap.pypa.io/virtualenv/${MIN_PYTHON_VERSION}/virtualenv.pyz"
vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX)
vsrc="$vtmp/$(basename $vurl)"
echo "Fetching $vurl"
if command -v curl >/dev/null; then
curl -L -o "$vsrc" "$vurl"
elif command -v wget >/dev/null; then
wget -O "$vsrc" "$vurl"
if command -v uv >/dev/null; then
uv venv "$GALAXY_VIRTUAL_ENV" --python "$GALAXY_PYTHON"
else
# First try to use the venv standard library module, although it is
# not always installed by default on Linux distributions.
if ! "$GALAXY_PYTHON" -m venv "$GALAXY_VIRTUAL_ENV"; then
echo "Creating the Python virtual environment using the venv standard library module failed."
echo "Trying with virtualenv now."
if command -v virtualenv >/dev/null; then
virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV"
else
"$GALAXY_PYTHON" -c "try:
from urllib import urlretrieve
except:
from urllib.request import urlretrieve
urlretrieve('$vurl', '$vsrc')"
# Download virtualenv zipapp
vurl="https://bootstrap.pypa.io/virtualenv/${MIN_PYTHON_VERSION}/virtualenv.pyz"
vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX)
vsrc="$vtmp/$(basename $vurl)"
echo "Fetching $vurl"
if command -v curl >/dev/null; then
curl -L -o "$vsrc" "$vurl"
elif command -v wget >/dev/null; then
wget -O "$vsrc" "$vurl"
else
"$GALAXY_PYTHON" -c "try:
from urllib import urlretrieve
except:
from urllib.request import urlretrieve
urlretrieve('$vurl', '$vsrc')"
fi
"$GALAXY_PYTHON" "$vsrc" "$GALAXY_VIRTUAL_ENV"
rm -rf "$vtmp"
fi
"$GALAXY_PYTHON" "$vsrc" "$GALAXY_VIRTUAL_ENV"
rm -rf "$vtmp"
fi
fi
fi
@@ -202,22 +188,30 @@ fi
[ "$CI" = 'true' ] && export PIP_PROGRESS_BAR=off
if [ $FETCH_WHEELS -eq 1 ]; then
# Ensure uv is installed - uv is much faster than pip (10-100x)
ensure_uv python
# Use uv pip install for fast package installation
# Note: uv pip install is significantly faster than pip install
# shellcheck disable=SC2086
uv pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
if command -v uv >/dev/null; then
uv pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
else
python -m pip install "pip>=$MIN_PIP_VERSION" wheel
pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
fi
set_galaxy_config_file_var
GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))")
if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then
if uv pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then
echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2."
uv pip uninstall psycopg2 psycopg2-binary
if command -v uv >/dev/null; then
if uv pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then
echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2."
uv pip uninstall psycopg2 psycopg2-binary
fi
echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | uv pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
else
if pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then
echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2."
pip uninstall -y psycopg2 psycopg2-binary
fi
echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
fi
echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | uv pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
fi
fi
+5 -14
View File
@@ -131,21 +131,12 @@ function make_forks() {
function create_venv() {
if [ ! -d "$VENV" ]; then
# Install uv for fast venv creation and package management
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
else
python3 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
if command -v uv >/dev/null; then
log_exec uv venv "$VENV"
else
log_exec python3 -m venv "$VENV"
log_exec "${VENV}/bin/pip" install wheel packaging
fi
log_exec uv venv "$VENV"
log_exec "${VENV}/bin/uv" pip install wheel packaging
fi
. "${VENV}/bin/activate"
}