diff --git a/lib/galaxy/dependencies/update.sh b/lib/galaxy/dependencies/update.sh index 5003735a491..be0cdbf9675 100755 --- a/lib/galaxy/dependencies/update.sh +++ b/lib/galaxy/dependencies/update.sh @@ -28,22 +28,29 @@ do done shift $((OPTIND - 1)) -# Create a virtual environment in a tmp directory and install uv into it -uv_venv=$(mktemp -d "${TMPDIR:-/tmp}/uv_venv.XXXXXXXXXX") -python3 -m venv "${uv_venv}" -"${uv_venv}/bin/python" -m pip install uv -uv="${uv_venv}/bin/uv" +# Install uv for fast operation +if ! command -v uv >/dev/null; then + echo "Installing uv..." + if command -v curl >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv + elif command -v wget >/dev/null; then + wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv + else + python3 -m pip install uv + fi + export PATH="$HOME/.local/bin:$PATH" +fi # Run uv (this may update pyproject.toml and uv.lock). if [ -n "$pkg" ]; then - ${uv} lock --upgrade-package "$pkg" + uv lock --upgrade-package "$pkg" else - ${uv} lock --upgrade + uv lock --upgrade fi # Update pinned requirements files. UV_EXPORT_OPTIONS='--frozen --no-annotate --no-hashes' -${uv} export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt" -${uv} export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt" -${uv} export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt" -${uv} export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt" +uv export ${UV_EXPORT_OPTIONS} --no-dev > "$this_directory/pinned-requirements.txt" +uv export ${UV_EXPORT_OPTIONS} --only-group=test > "$this_directory/pinned-test-requirements.txt" +uv export ${UV_EXPORT_OPTIONS} --only-group=dev > "$this_directory/dev-requirements.txt" +uv export ${UV_EXPORT_OPTIONS} --only-group=typecheck > "$this_directory/pinned-typecheck-requirements.txt" diff --git a/lib/galaxy/dependencies/update_lint_requirements.sh b/lib/galaxy/dependencies/update_lint_requirements.sh index 9fc43257355..959820225bc 100755 --- a/lib/galaxy/dependencies/update_lint_requirements.sh +++ b/lib/galaxy/dependencies/update_lint_requirements.sh @@ -10,12 +10,28 @@ THIS_DIRECTORY="$(cd "$(dirname "$0")" > /dev/null && pwd)" update_pinned_reqs() { VENV=$(mktemp -d "${TMPDIR:-/tmp}/$1_venv.XXXXXXXXXX") - python3.9 -m venv "${VENV}" + + # Install uv for fast package installation + if ! command -v uv >/dev/null; then + echo "Installing uv..." + if command -v curl >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv + elif command -v wget >/dev/null; then + wget -qO- https://astral.sh/uv/install.sh | sh || python3.9 -m pip install uv + else + python3.9 -m pip install uv + fi + export PATH="$HOME/.local/bin:$PATH" + fi + + # Use uv venv for fast virtual environment creation + uv venv "${VENV}" --python python3.9 . "${VENV}/bin/activate" - pip install --upgrade pip setuptools - pip install -r "${THIS_DIRECTORY}/$1-requirements.txt" + + # No need to upgrade pip/setuptools - uv handles everything + uv pip install -r "${THIS_DIRECTORY}/$1-requirements.txt" # The grep below is needed to workaround https://github.com/pypa/pip/issues/8331 - pip freeze -l | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt" + uv pip freeze | grep -v 'pkg_resources==0.0.0' > "${THIS_DIRECTORY}/pinned-$1-requirements.txt" rm -rf "${VENV}" } diff --git a/packages/package-build-install.sh b/packages/package-build-install.sh index 37e6c9d31a2..380c341ac5c 100755 --- a/packages/package-build-install.sh +++ b/packages/package-build-install.sh @@ -11,6 +11,21 @@ set -euo pipefail : ${PIP_EXTRA_ARGS:=--extra-index-url https://wheels.galaxyproject.org} #: ${SETUP_VENV:=true} +# Ensure uv is installed +ensure_uv() { + if ! command -v uv >/dev/null; then + echo "Installing uv..." + if command -v curl >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv + elif command -v wget >/dev/null; then + wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv + else + python3 -m pip install uv + fi + export PATH="$HOME/.local/bin:$PATH" + fi +} + INSTALL=true EDITABLE=false META=false @@ -37,7 +52,7 @@ do h) echo "usage: $0 [-bem] [up_to_package]" echo " -b build only, no install" - echo " -e install packages in \"editable\" mode (pip install -e)" + echo " -e install packages in \"editable\" mode (uv pip install -e)" echo " -m install galaxy metapackage, installing pinned deps in meta/requirements.txt" exit 0 ;; @@ -65,15 +80,21 @@ while read package; do printf "\n========= PACKAGE %s =========\n\n" "$package" pushd $package if $EDITABLE; then - pip install -e . + # Install package in editable mode using uv (much faster than pip) + ensure_uv + uv pip install -e . else if [ ! -d "$VENV" ]; then - python3 -m venv "$VENV" - "${VENV}/bin/pip" install -r <(grep -v test-requirements.txt dev-requirements.txt) + # Install uv for fast venv creation and package management + ensure_uv + + # Use uv venv for fast virtual environment creation + uv venv "$VENV" --python python3 + "${VENV}/bin/uv" pip install -r <(grep -v test-requirements.txt dev-requirements.txt) fi make dist if $INSTALL && ! $META; then - pip install dist/*.whl + uv pip install dist/*.whl fi fi popd @@ -83,5 +104,5 @@ done < "$PACKAGE_LIST_FILE" if $INSTALL && $META && ! $EDITABLE; then WHEELHOUSE=$(mktemp -d -t gxpkgwheelhouseXXXXXX) cp */dist/*.whl "$WHEELHOUSE" - pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl + uv pip install $PIP_EXTRA_ARGS --find-links "$WHEELHOUSE" meta/dist/*.whl fi diff --git a/packages/test.sh b/packages/test.sh index 7d3d5916302..e23beb8e6cc 100755 --- a/packages/test.sh +++ b/packages/test.sh @@ -36,12 +36,27 @@ cd "$(dirname "$0")" TEST_PYTHON=${TEST_PYTHON:-"python3"} TEST_ENV_DIR=${TEST_ENV_DIR:-$(mktemp -d -t gxpkgtestenvXXXXXX)} -"$TEST_PYTHON" -m venv "$TEST_ENV_DIR" +# Install uv for fast package installation +if ! command -v uv >/dev/null; then + echo "Installing uv..." + if command -v curl >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv + elif command -v wget >/dev/null; then + wget -qO- https://astral.sh/uv/install.sh | sh || "$TEST_PYTHON" -m pip install uv + else + "$TEST_PYTHON" -m pip install uv + fi + export PATH="$HOME/.local/bin:$PATH" +fi + +# Use uv venv for much faster virtual environment creation +uv venv "$TEST_ENV_DIR" --python "$TEST_PYTHON" # shellcheck disable=SC1091 . "${TEST_ENV_DIR}/bin/activate" -pip install --upgrade pip setuptools wheel + +# Note: No need to upgrade pip anymore since we're using uv exclusively if [ $FOR_PULSAR -eq 0 ]; then - pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt + uv pip install -r../lib/galaxy/dependencies/pinned-typecheck-requirements.txt fi # Ensure ordered by dependency DAG @@ -65,13 +80,13 @@ while read -r package_dir || [ -n "$package_dir" ]; do # https://stackoverflow. # Install extras (if needed) if [ "$package_dir" = "util" ]; then - pip install '.[image-util,template,jstree,config-template,test]' + uv pip install '.[image-util,template,jstree,config-template,test]' elif [ "$package_dir" = "tool_util" ]; then - pip install '.[cwl,mulled,edam,extended-assertions,test]' + uv pip install '.[cwl,mulled,edam,extended-assertions,test]' elif grep -q 'test =' setup.cfg 2>/dev/null; then - pip install '.[test]' + uv pip install '.[test]' else - pip install . + uv pip install . fi if [ $FOR_PULSAR -eq 0 ]; then diff --git a/pyproject.toml b/pyproject.toml index e504f874d69..94a17893a35 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -261,3 +261,4 @@ default-groups = [] extra-index-url = ["https://wheels.galaxyproject.org/simple"] index-strategy = "unsafe-best-match" package = false +required-version = ">=0.5.0" # Minimum uv version for fast package management diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 63f278dbfde..58018577b0f 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -4,6 +4,22 @@ set -e # The caller may do this as well, but since common_startup.sh can be called independently, we need to do it here . ./scripts/common_startup_functions.sh +# Ensure uv is installed (used throughout this script) +ensure_uv() { + if ! command -v uv >/dev/null; then + echo "Installing uv..." + local python_cmd="${1:-python3}" + if command -v curl >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}" + elif command -v wget >/dev/null; then + wget -qO- https://astral.sh/uv/install.sh | sh || "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}" + else + "$python_cmd" -m pip install "uv>=${MIN_UV_VERSION}" + fi + export PATH="$HOME/.local/bin:$PATH" + fi +} + DEV_WHEELS=0 FETCH_WHEELS=1 CREATE_VENV=1 @@ -38,7 +54,7 @@ RMFILES=" " MIN_PYTHON_VERSION=3.9 -MIN_PIP_VERSION=20.3 +MIN_UV_VERSION=0.5.0 # return true if $1 is in $2 else false in_dir() { @@ -112,12 +128,15 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then echo "Creating Conda environment for Galaxy: $GALAXY_CONDA_ENV" echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_CONDA_ENV to an" echo "existing environment before starting Galaxy." - $CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}" "pip>=${MIN_PIP_VERSION}" + # Note: uv will be installed via standalone installer, not conda + $CONDA_EXE create --yes --override-channels --channel conda-forge --name "$GALAXY_CONDA_ENV" "python=${GALAXY_CONDA_PYTHON_VERSION}" unset __CONDA_INFO fi conda_activate fi - python3 -m venv "$GALAXY_VIRTUAL_ENV" + # Install uv first if not available, then use uv venv for fast virtual environment creation + ensure_uv python + uv venv "$GALAXY_VIRTUAL_ENV" --python python3 else # If $GALAXY_VIRTUAL_ENV does not exist, and there is no conda available, attempt to create it. @@ -128,15 +147,18 @@ if [ $SET_VENV -eq 1 ] && [ $CREATE_VENV -eq 1 ]; then echo "using Python: $GALAXY_PYTHON" echo "To avoid this, use the --no-create-venv flag or set \$GALAXY_VIRTUAL_ENV to an" echo "existing environment before starting Galaxy." - # First try to use the venv standard library module, although it is - # not always installed by default on Linux distributions. - if ! "$GALAXY_PYTHON" -m venv "$GALAXY_VIRTUAL_ENV"; then - echo "Creating the Python virtual environment using the venv standard library module failed." - echo "Trying with virtualenv now." + + # Install uv first if not available, then use uv venv for fast virtual environment creation + ensure_uv "$GALAXY_PYTHON" + + # Use uv venv for much faster virtual environment creation + if ! uv venv "$GALAXY_VIRTUAL_ENV" --python "$GALAXY_PYTHON"; then + echo "Creating the Python virtual environment using uv failed." + echo "Trying with virtualenv now as fallback." if command -v virtualenv >/dev/null; then virtualenv -p "$GALAXY_PYTHON" "$GALAXY_VIRTUAL_ENV" else - # Download virtualenv zipapp + # Download virtualenv zipapp as last resort vurl="https://bootstrap.pypa.io/virtualenv/${MIN_PYTHON_VERSION}/virtualenv.pyz" vtmp=$(mktemp -d -t galaxy-virtualenv-XXXXXX) vsrc="$vtmp/$(basename $vurl)" @@ -180,18 +202,22 @@ fi [ "$CI" = 'true' ] && export PIP_PROGRESS_BAR=off if [ $FETCH_WHEELS -eq 1 ]; then - python -m pip install "pip>=$MIN_PIP_VERSION" wheel + # Ensure uv is installed - uv is much faster than pip (10-100x) + ensure_uv python + + # Use uv pip install for fast package installation + # Note: uv pip install is significantly faster than pip install # shellcheck disable=SC2086 - pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + uv pip install $requirement_args --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" set_galaxy_config_file_var GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "from __future__ import print_function; import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))") if [ -n "$GALAXY_CONDITIONAL_DEPENDENCIES" ]; then - if pip list --format=columns | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then + if uv pip list | grep "psycopg2[\(\ ]*2.7.3" > /dev/null; then echo "An older version of psycopg2 (non-binary, version 2.7.3) has been detected. Galaxy now uses psycopg2-binary, which will be installed after removing psycopg2." - pip uninstall -y psycopg2 psycopg2-binary + uv pip uninstall psycopg2 psycopg2-binary fi - echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" + echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | uv pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" fi fi diff --git a/test/release.sh b/test/release.sh index 595f7c462bb..bd124c527e5 100755 --- a/test/release.sh +++ b/test/release.sh @@ -131,8 +131,21 @@ function make_forks() { function create_venv() { if [ ! -d "$VENV" ]; then - log_exec python3 -m venv "$VENV" - log_exec "${VENV}/bin/pip" install wheel packaging + # Install uv for fast venv creation and package management + if ! command -v uv >/dev/null; then + echo "Installing uv..." + if command -v curl >/dev/null; then + curl -LsSf https://astral.sh/uv/install.sh | sh || python3 -m pip install uv + elif command -v wget >/dev/null; then + wget -qO- https://astral.sh/uv/install.sh | sh || python3 -m pip install uv + else + python3 -m pip install uv + fi + export PATH="$HOME/.local/bin:$PATH" + fi + + log_exec uv venv "$VENV" + log_exec "${VENV}/bin/uv" pip install wheel packaging fi . "${VENV}/bin/activate" }