From 4eefd885eccd0c42c7a74c8415add3a948a91135 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 22 Dec 2015 14:00:25 -0500 Subject: [PATCH 1/2] Improve startup options related to wheel/dependency/virtualenv management. Drop hard requirement on lzo (doubt anyone other than usegalaxy.org is using it, but it will be added as a extras option for bxlab/bx-python for anyone who is). Fixes #1150 --- .../dependencies/pinned-requirements.txt | 68 +++++++++++++++ lib/galaxy/dependencies/requirements.txt | 86 +++++++++---------- run.sh | 2 +- run_tests.sh | 10 ++- scripts/common_startup.sh | 64 +++++++++----- 5 files changed, 163 insertions(+), 67 deletions(-) create mode 100644 lib/galaxy/dependencies/pinned-requirements.txt diff --git a/lib/galaxy/dependencies/pinned-requirements.txt b/lib/galaxy/dependencies/pinned-requirements.txt new file mode 100644 index 00000000000..998490dbc5d --- /dev/null +++ b/lib/galaxy/dependencies/pinned-requirements.txt @@ -0,0 +1,68 @@ +# packages with C extensions +bx-python==0.7.3 +MarkupSafe==0.23 +PyYAML==3.11 +SQLAlchemy==1.0.8 +# Mercurial >= 3.5 changed the bundle format, which breaks hg push of TS repositories +mercurial==3.4.2 +numpy==1.9.2 +pycrypto==2.6.1 + +# Install python_lzo if you want to support indexed access to lzo-compressed +# locally cached maf files via bx-python +#python_lzo==1.8 + +# pure Python packages +Paste==2.0.2 +PasteDeploy==1.5.2 +docutils==0.12 +wchartype==0.1 +repoze.lru==0.6 +Routes==2.2 +WebOb==1.4.1 +WebHelpers==1.3 +Mako==1.0.2 +pytz==2015.4 +Babel==2.0 +Beaker==1.7.0 + +# Cheetah and dependencies +Cheetah==2.4.4 +Markdown==2.6.3 + +# BioBlend and dependencies +bioblend==0.6.1 +boto==2.38.0 +requests==2.8.1 +requests-toolbelt==0.4.0 + +# kombu and dependencies +kombu==3.0.30 +amqp==1.4.8 +anyjson==0.3.3 + +# sqlalchemy-migrate and dependencies +sqlalchemy-migrate==0.10.0 +decorator==4.0.2 +Tempita==0.5.3dev +sqlparse==0.1.16 +pbr==1.8.0 +six==1.9.0 # six is also a Pulsar client dep +Parsley==1.3 +nose==1.3.7 +SVGFig==1.1.6 + +# Fabric and dependencies +Fabric==1.10.2 +paramiko==1.15.2 +ecdsa==0.13 +# pycrypto, but it is a direct Galaxy dependency as well + +# Can't use Whoosh latest due to a bug but need to backport a bugfix from a +# newer release. +# +# https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff +Whoosh==2.4.1+gx1 + +# Flexible BAM index naming +pysam==0.8.3+gx1 diff --git a/lib/galaxy/dependencies/requirements.txt b/lib/galaxy/dependencies/requirements.txt index 020bbaa260f..7deb751c777 100644 --- a/lib/galaxy/dependencies/requirements.txt +++ b/lib/galaxy/dependencies/requirements.txt @@ -1,65 +1,63 @@ # packages with C extensions -bx-python==0.7.3 -MarkupSafe==0.23 -PyYAML==3.11 -SQLAlchemy==1.0.8 +# numpy should be installed before bx to enable extra features in bx +numpy +bx-python +MarkupSafe +PyYAML +SQLAlchemy # Mercurial >= 3.5 changed the bundle format, which breaks hg push of TS repositories -mercurial==3.4.2 -numpy==1.9.2 -pycrypto==2.6.1 -python_lzo==1.8 +mercurial<3.5 +pycrypto -# extra functionality from Dannon -pysam==0.8.3+gx1 +# Install python_lzo if you want to support indexed access to lzo-compressed +# locally cached maf files via bx-python +#python_lzo # pure Python packages -Paste==2.0.2 -PasteDeploy==1.5.2 -docutils==0.12 -wchartype==0.1 -repoze.lru==0.6 -Routes==2.2 -WebOb==1.4.1 -WebHelpers==1.3 -Mako==1.0.2 -pytz==2015.4 -Babel==2.0 -Beaker==1.7.0 +Paste +PasteDeploy +docutils +wchartype +repoze.lru +Routes +WebOb +WebHelpers +Mako +pytz +Babel +Beaker # Cheetah and dependencies -Cheetah==2.4.4 -Markdown==2.6.3 +Cheetah # BioBlend and dependencies -bioblend==0.6.1 -boto==2.38.0 -requests==2.8.1 -requests-toolbelt==0.4.0 +bioblend +boto +requests # kombu and dependencies -kombu==3.0.30 -amqp==1.4.8 -anyjson==0.3.3 +kombu # sqlalchemy-migrate and dependencies -sqlalchemy-migrate==0.10.0 -decorator==4.0.2 -Tempita==0.5.3dev -sqlparse==0.1.16 -pbr==1.8.0 -six==1.9.0 # six is also a Pulsar client dep -Parsley==1.3 -nose==1.3.7 -SVGFig==1.1.6 +sqlalchemy-migrate +decorator +Tempita +sqlparse +six +Parsley +nose +SVGFig # Fabric and dependencies -Fabric==1.10.2 -paramiko==1.15.2 -ecdsa==0.13 -# pycrypto, but it is a direct Galaxy dependency as well +Fabric + +# We still pin these dependencies because of modifications to the upstream packages # Can't use Whoosh latest due to a bug but need to backport a bugfix from a # newer release. # # https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff Whoosh==2.4.1+gx1 + +# Flexible BAM index naming +pysam==0.8.3+gx1 diff --git a/run.sh b/run.sh index 9a655b50968..8e821f238a3 100755 --- a/run.sh +++ b/run.sh @@ -18,7 +18,7 @@ fi while : do case "$1" in - --skip-eggs|--skip-wheels|--skip-samples|--dev-wheels) + --skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip) common_startup_args="$common_startup_args $1" shift ;; diff --git a/run_tests.sh b/run_tests.sh index ac508e4a470..b91e618f010 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -288,6 +288,14 @@ do skip_venv='--skip-venv' shift ;; + --no-create-venv) + no_create_venv='--no-create-venv' + shift + ;; + --no-replace-pip) + no_replace_pip='--no-replace-pip' + shift + ;; --skip-common-startup) # Don't run ./scripts/common_startup.sh (presumably it has already # been done, or you know what you're doing). @@ -314,7 +322,7 @@ if [ -z "$skip_common_startup" ]; then GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=$GALAXY_TEST_DBURI export GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION fi - ./scripts/common_startup.sh $skip_venv --dev-wheels || exit 1 + ./scripts/common_startup.sh $skip_venv $no_create_venv $no_replace_pip --dev-wheels || exit 1 fi if [ -z "$skip_venv" -a -d .venv ]; diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 8f4846dc09b..98365595b31 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -4,12 +4,16 @@ set -e DEV_WHEELS=0 FETCH_WHEELS=1 SET_VENV=1 +CREATE_VENV=1 +REPLACE_PIP=1 COPY_SAMPLE_FILES=1 for arg in "$@"; do [ "$arg" = "--skip-eggs" ] && FETCH_WHEELS=0 [ "$arg" = "--skip-wheels" ] && FETCH_WHEELS=0 [ "$arg" = "--dev-wheels" ] && DEV_WHEELS=1 [ "$arg" = "--skip-venv" ] && SET_VENV=0 + [ "$arg" = "--no-create-venv" ] && CREATE_VENV=0 + [ "$arg" = "--no-replace-pip" ] && REPLACE_PIP=0 [ "$arg" = "--stop-daemon" ] && FETCH_WHEELS=0 [ "$arg" = "--skip-samples" ] && COPY_SAMPLE_FILES=0 done @@ -57,7 +61,7 @@ if [ ! -f $GALAXY_CONFIG_FILE ]; then GALAXY_CONFIG_FILE=config/galaxy.ini.sample fi -if [ $SET_VENV -eq 1 ]; then +if [ $SET_VENV -eq 1 -a $CREATE_VENV -eq 1 ]; then # If .venv does not exist, attempt to create it. if [ ! -d .venv ] then @@ -66,50 +70,68 @@ if [ $SET_VENV -eq 1 ]; then if command -v virtualenv >/dev/null; then virtualenv .venv else - vvers=13.1.0 + vvers=13.1.2 vurl="https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${vvers}.tar.gz" + vsha="aabc8ef18cddbd8a2a9c7f92bc43e2fea54b1147330d65db920ef3ce9812e3dc" vtmp=`mktemp -d -t galaxy-virtualenv-XXXXXX` vsrc="$vtmp/`basename $vurl`" - ( if command -v curl >/dev/null; then - curl --silent -o $vsrc $vurl + # SSL certificates are not checked to prevent problems with messed + # up client cert environments. We verify the download using a known + # good sha256 sum instead. + echo "Fetching $vurl" + if command -v curl >/dev/null; then + curl --insecure -L -o $vsrc $vurl elif command -v wget >/dev/null; then - wget --quiet -O $vsrc $vurl + wget --no-check-certificate -O $vsrc $vurl else python -c "import urllib; urllib.urlretrieve('$vurl', '$vsrc')" - fi ) && - tar zxf $vsrc -C $vtmp && - python $vtmp/virtualenv-$vvers/virtualenv.py .venv && + fi + echo "Verifying $vsrc checksum is $vsha" + if command -v sha256sum >/dev/null; then + echo "$vsha $vsrc" | sha256sum -c --strict + else + python -c "import hashlib; assert hashlib.sha256(open('$vsrc', 'rb').read()).hexdigest() == '$vsha', '$vsrc: invalid checksum'" + fi + tar zxf $vsrc -C $vtmp + python $vtmp/virtualenv-$vvers/virtualenv.py .venv rm -rf $vtmp fi fi +fi +if [ $SET_VENV -eq 1 ]; then # If there is a .venv/ directory, assume it contains a virtualenv that we # should run this instance in. if [ -d .venv ]; then printf "Activating virtualenv at %s/.venv\n" $(pwd) . .venv/bin/activate + # Because it's a virtualenv, we assume $PYTHONPATH is unnecessary for + # anything in the venv to work correctly, and having it set can cause + # problems when there are conflicts with Galaxy's dependencies outside + # the venv (e.g. virtualenv-burrito's pip and six). + # + # If you are skipping the venv setup we shall assume you know what + # you're doing and will deal with any conflicts. + unset PYTHONPATH + fi + + if [ -z "$VIRTUAL_ENV" ]; then + echo "ERROR: A virtualenv cannot be found. Please create a virtualenv in .venv, or activate one." + exit 1 fi fi -if [ -z "$VIRTUAL_ENV" ]; then - echo "ERROR: A virtualenv cannot be found or created. Please create a virtualenv in .venv, or activate one." - exit 1 -fi - - : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"} -if [ $FETCH_WHEELS -eq 1 ]; then - # Because it's a virtualenv, we assume $PYTHONPATH is unnecessary for - # anything in the venv to work correctly, and having it set can cause - # problems when there are conflicts with Galaxy's dependencies outside the - # venv (e.g. virtualenv-burrito's pip and six) - unset PYTHONPATH +if [ $SET_VENV -eq 1 -a $REPLACE_PIP -eq 1 ]; then pip_version=`pip --version | awk '{print $2}'` pre=`python -c "from pkg_resources import parse_version; from sys import stdout; stdout.write('--pre') if parse_version('$pip_version') >= parse_version('1.4') else stdout.write('')"` pip install $pre --no-index --find-links ${GALAXY_WHEELS_INDEX_URL}/pip --upgrade pip # binary-compatibility.cfg may need to be created (e.g. on CentOS) - [ ! -f ${VIRTUAL_ENV}/binary-compatibility.cfg ] && python ./scripts/binary_compatibility.py -o ${VIRTUAL_ENV}/binary-compatibility.cfg + [ -n "$VIRTUAL_ENV" -a ! -f ${VIRTUAL_ENV}/binary-compatibility.cfg ] && python ./scripts/binary_compatibility.py -o ${VIRTUAL_ENV}/binary-compatibility.cfg +fi + +if [ $FETCH_WHEELS -eq 1 ]; then pip install -r requirements.txt --index-url ${GALAXY_WHEELS_INDEX_URL} GALAXY_CONDITIONAL_DEPENDENCIES=`PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))"` [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url ${GALAXY_WHEELS_INDEX_URL} From 9934a45134283d307814058fa09e4b26d3d8452f Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Tue, 22 Dec 2015 14:20:19 -0500 Subject: [PATCH 2/2] Run tox/Travis with --no-create-venv instead of --skip-venv --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index f34ee71ab2c..0e0676b4b19 100644 --- a/tox.ini +++ b/tox.ini @@ -20,7 +20,7 @@ deps = flake8 [testenv:py27-unit] -commands = bash run_tests.sh --skip-venv -u +commands = bash run_tests.sh --no-create-venv -u whitelist_externals = bash deps = nose @@ -28,7 +28,7 @@ deps = mock [testenv:py26-unit] -commands = bash run_tests.sh --skip-venv -u +commands = bash run_tests.sh --no-create-venv -u whitelist_externals = bash deps = unittest2