opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-24 16:30:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Enhance package management by integrating 'uv' for faster installation and virtual environment creation across scripts

Browse Source
This commit is contained in:
Arash
2025-10-29 12:45:42 +00:00
committed by Nicola Soranzo
parent 17aa2bc302
commit 478b88a7d4
7 changed files with 143 additions and 44 deletions
Download Patch File Download Diff File Expand all files Collapse all files
lib/galaxy/dependencies/update.sh
+18 -11
View File
@@ -28,22 +28,29 @@ do
done
shift $((OPTIND - 1))
# Create a virtual environment in a tmp directory and install uv into it
uv_venv=$(mktemp -d "${TMPDIR:-/tmp}/uv_venv.XXXXXXXXXX")
python3 -m venv "${uv_venv}"
"${uv_venv}/bin/python" -m pip install uv
uv="${uv_venv}/bin/uv"
# Install uv for fast operation
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
else
python3 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
fi
# Run uv (this may update pyproject.toml and uv.lock).
if [ -n "$pkg" ]; then
${uv} lock --upgrade-package "$pkg"
uv lock --upgrade-package "$pkg"
else
${uv} lock --upgrade
uv lock --upgrade
fi
# Update pinned requirements files.
UV_EXPORT_OPTIONS='--frozen --no-annotate --no-hashes'
${uv} export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt"
${uv} export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt"
${uv} export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt"
${uv} export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt"
uv export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt"
uv export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt"
uv export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt"
uv export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt"
lib/galaxy/dependencies/update_lint_requirements.sh
+20 -4
View File
@@ -10,12 +10,28 @@ THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)"
update_pinned_reqs() {
VENV=$(mktemp -d "${TMPDIR:-/tmp}/$1_venv.XXXXXXXXXX")
python3.9 -m venv "${VENV}"
# Install uv for fast package installation
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv
else
python3.9 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
fi
# Use uv venv for fast virtual environment creation
uv venv "${VENV}" --python python3.9
. "${VENV}/bin/activate"
pip install --upgrade pip setuptools
pip install -r "${THIS_DIRECTORY}/$1-requirements.txt"
# No need to upgrade pip/setuptools - uv handles everything
uv pip install -r "${THIS_DIRECTORY}/$1-requirements.txt"
# The grep below is needed to workaround https://github.com/pypa/pip/issues/8331
pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt"
uv pip freeze | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt"
rm -rf "${VENV}"
}
packages/package-build-install.sh
+27 -6
View File
@@ -11,6 +11,21 @@ set -euo pipefail
: ${PIP_EXTRA_ARGS:=--extra-index-url https://wheels.galaxyproject.org}
#: ${SETUP_VENV:=true}
# Ensure uv is installed
ensure_uv() {
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
else
python3 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
fi
}
INSTALL=true
EDITABLE=false
META=false
@@ -37,7 +52,7 @@ do
h)
echo "usage: $0 [-bem] [up_to_package]"
echo " -b build only, no install"
echo " -e install packages in \"editable\" mode (pip install -e)"
echo " -e install packages in \"editable\" mode (uv pip install -e)"
echo " -m install galaxy metapackage, installing pinned deps in meta/requirements.txt"
exit 0
;;
@@ -65,15 +80,21 @@ while read package; do
printf "\n========= PACKAGE %s =========\n\n" "$package"
pushd $package
if $EDITABLE; then
pip install -e .
# Install package in editable mode using uv (much faster than pip)
ensure_uv
uv pip install -e .
else
if [ ! -d "$VENV" ]; then
python3 -m venv "$VENV"
"${VENV}/bin/pip" install -r <(grep -v test-requirements.txt dev-requirements.txt)
# Install uv for fast venv creation and package management
ensure_uv
# Use uv venv for fast virtual environment creation
uv venv "$VENV" --python python3
"${VENV}/bin/uv" pip install -r <(grep -v test-requirements.txt dev-requirements.txt)
fi
make dist
if $INSTALL && ! $META; then
pip install dist/*.whl
uv pip install dist/*.whl
fi
fi
popd
@@ -83,5 +104,5 @@ done < "$PACKAGE_LIST_FILE"
if $INSTALL && $META && ! $EDITABLE; then
WHEELHOUSE=$(mktemp -d -t gxpkgwheelhouseXXXXXX)
cp */dist/*.whl "$WHEELHOUSE"
pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl
uv pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl
fi
packages/test.sh
+22 -7
View File
@@ -36,12 +36,27 @@ cd "$(dirname "$0")"
TEST_PYTHON=${TEST_PYTHON:-"python3"}
TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)}
"$TEST_PYTHON" -m venv "$TEST_ENV_DIR"
# Install uv for fast package installation
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv
else
"$TEST_PYTHON" -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
fi
# Use uv venv for much faster virtual environment creation
uv venv "$TEST_ENV_DIR" --python "$TEST_PYTHON"
# shellcheck disable=SC1091
. "${TEST_ENV_DIR}/bin/activate"
pip install --upgrade pip setuptools wheel
# Note: No need to upgrade pip anymore since we're using uv exclusively
if [ $FOR_PULSAR -eq 0 ]; then
pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
uv pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt
fi
# Ensure ordered by dependency DAG
@@ -65,13 +80,13 @@ while read -r package_dir || [ -n "$package_dir" ]; do # https://stackoverflow.
# Install extras (if needed)
if [ "$package_dir" = "util" ]; then
pip install '.[image-util,template,jstree,config-template,test]'
uv pip install '.[image-util,template,jstree,config-template,test]'
elif [ "$package_dir" = "tool_util" ]; then
pip install '.[cwl,mulled,edam,extended-assertions,test]'
uv pip install '.[cwl,mulled,edam,extended-assertions,test]'
elif grep -q 'test =' setup.cfg 2>/dev/null; then
pip install '.[test]'
uv pip install '.[test]'
else
pip install .
uv pip install .
fi
if [ $FOR_PULSAR -eq 0 ]; then
pyproject.toml
+1
View File
@@ -261,3 +261,4 @@ default-groups = []
extra-index-url = ["https://wheels.galaxyproject.org/simple"]
index-strategy = "unsafe-best-match"
package = false
required-version = ">=0.5.0" # Minimum uv version for fast package management
scripts/common_startup.sh
+40 -14
View File
@@ -4,6 +4,22 @@ set -e
# The caller may do this as well, but since common_startup.sh can be called independently, we need to do it here
. ./scripts/common_startup_functions.sh
# Ensure uv is installed (used throughout this script)
ensure_uv() {
if ! command -v uv >/dev/null; then
echo "Installing uv..."
local python_cmd="${1:-python3}"
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}"
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}"
else
"$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}"
fi
export PATH="$HOME/.local/bin:$PATH"
fi
}
DEV_WHEELS=0
FETCH_WHEELS=1
CREATE_VENV=1
@@ -38,7 +54,7 @@ RMFILES="
"
MIN_PYTHON_VERSION=3.9
MIN_PIP_VERSION=20.3
MIN_UV_VERSION=0.5.0
# return true if $1 is in $2 else false
in_dir() {
@@ -112,12 +128,15 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV"
echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an"
echo "existing environment before starting Galaxy."
$CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}" "pip>=${MIN_PIP_VERSION}"
# Note: uv will be installed via standalone installer, not conda
$CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}"
unset __CONDA_INFO
fi
conda_activate
fi
python3 -m venv "$GALAXY_VIRTUAL_ENV"
# Install uv first if not available, then use uv venv for fast virtual environment creation
ensure_uv python
uv venv "$GALAXY_VIRTUAL_ENV" --python python3
else
# If $GALAXY_VIRTUAL_ENV does not exist, and there is no conda available, attempt to create it.
@@ -128,15 +147,18 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then
echo "using Python: $GALAXY_PYTHON"
echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_VIRTUAL_ENV to an"
echo "existing environment before starting Galaxy."
# First try to use the venv standard library module, although it is
# not always installed by default on Linux distributions.
if ! "$GALAXY_PYTHON" -m venv "$GALAXY_VIRTUAL_ENV"; then
echo "Creating the Python virtual environment using the venv standard library module failed."
echo "Trying with virtualenv now."
# Install uv first if not available, then use uv venv for fast virtual environment creation
ensure_uv "$GALAXY_PYTHON"
# Use uv venv for much faster virtual environment creation
if ! uv venv "$GALAXY_VIRTUAL_ENV" --python "$GALAXY_PYTHON"; then
echo "Creating the Python virtual environment using uv failed."
echo "Trying with virtualenv now as fallback."
if command -v virtualenv >/dev/null; then
virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV"
else
# Download virtualenv zipapp
# Download virtualenv zipapp as last resort
vurl="https://bootstrap.pypa.io/virtualenv/${MIN_PYTHON_VERSION}/virtualenv.pyz"
vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX)
vsrc="$vtmp/$(basename $vurl)"
@@ -180,18 +202,22 @@ fi
[ "$CI" = 'true' ] && export PIP_PROGRESS_BAR=off
if [ $FETCH_WHEELS -eq 1 ]; then
python -m pip install "pip>=$MIN_PIP_VERSION" wheel
# Ensure uv is installed - uv is much faster than pip (10-100x)
ensure_uv python
# Use uv pip install for fast package installation
# Note: uv pip install is significantly faster than pip install
# shellcheck disable=SC2086
pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
uv pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
set_galaxy_config_file_var
GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))")
if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then
if pip list --format=columns | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then
if uv pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then
echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2."
pip uninstall -y psycopg2 psycopg2-binary
uv pip uninstall psycopg2 psycopg2-binary
fi
echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | uv pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}"
fi
fi
test/release.sh
+15 -2
View File
@@ -131,8 +131,21 @@ function make_forks() {
function create_venv() {
if [ ! -d "$VENV" ]; then
log_exec python3 -m venv "$VENV"
log_exec "${VENV}/bin/pip" install wheel packaging
# Install uv for fast venv creation and package management
if ! command -v uv >/dev/null; then
echo "Installing uv..."
if command -v curl >/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
elif command -v wget >/dev/null; then
wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv
else
python3 -m pip install uv
fi
export PATH="$HOME/.local/bin:$PATH"
fi
log_exec uv venv "$VENV"
log_exec "${VENV}/bin/uv" pip install wheel packaging
fi
. "${VENV}/bin/activate"
}