Merge pull request #1389 from natefoo/wheel-startup-option-improvements

Improve startup options related to wheel/dependency/virtualenv management.

lib/galaxy/dependencies/pinned-requirements.txt

@@ -0,0 +1,68 @@
+# packages with C extensions
+bx-python==0.7.3
+MarkupSafe==0.23
+PyYAML==3.11
+SQLAlchemy==1.0.8
+# Mercurial >= 3.5 changed the bundle format, which breaks hg push of TS repositories
+mercurial==3.4.2
+numpy==1.9.2
+pycrypto==2.6.1
+
+# Install python_lzo if you want to support indexed access to lzo-compressed
+# locally cached maf files via bx-python
+#python_lzo==1.8
+
+# pure Python packages
+Paste==2.0.2
+PasteDeploy==1.5.2
+docutils==0.12
+wchartype==0.1
+repoze.lru==0.6
+Routes==2.2
+WebOb==1.4.1
+WebHelpers==1.3
+Mako==1.0.2
+pytz==2015.4
+Babel==2.0
+Beaker==1.7.0
+
+# Cheetah and dependencies
+Cheetah==2.4.4
+Markdown==2.6.3
+
+# BioBlend and dependencies
+bioblend==0.6.1
+boto==2.38.0
+requests==2.8.1
+requests-toolbelt==0.4.0
+
+# kombu and dependencies
+kombu==3.0.30
+amqp==1.4.8
+anyjson==0.3.3
+
+# sqlalchemy-migrate and dependencies
+sqlalchemy-migrate==0.10.0
+decorator==4.0.2
+Tempita==0.5.3dev
+sqlparse==0.1.16
+pbr==1.8.0
+six==1.9.0 # six is also a Pulsar client dep
+Parsley==1.3
+nose==1.3.7
+SVGFig==1.1.6
+
+# Fabric and dependencies
+Fabric==1.10.2
+paramiko==1.15.2
+ecdsa==0.13
+# pycrypto, but it is a direct Galaxy dependency as well
+
+# Can't use Whoosh latest due to a bug but need to backport a bugfix from a
+# newer release.
+#
+# https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff
+Whoosh==2.4.1+gx1
+
+# Flexible BAM index naming
+pysam==0.8.3+gx1

lib/galaxy/dependencies/requirements.txt

@@ -1,65 +1,63 @@
 # packages with C extensions
-bx-python==0.7.3
-MarkupSafe==0.23
-PyYAML==3.11
-SQLAlchemy==1.0.8
+# numpy should be installed before bx to enable extra features in bx
+numpy
+bx-python
+MarkupSafe
+PyYAML
+SQLAlchemy
 # Mercurial >= 3.5 changed the bundle format, which breaks hg push of TS repositories
-mercurial==3.4.2
-numpy==1.9.2
-pycrypto==2.6.1
-python_lzo==1.8
+mercurial<3.5
+pycrypto
 
-# extra functionality from Dannon
-pysam==0.8.3+gx1
+# Install python_lzo if you want to support indexed access to lzo-compressed
+# locally cached maf files via bx-python
+#python_lzo
 
 # pure Python packages
-Paste==2.0.2
-PasteDeploy==1.5.2
-docutils==0.12
-wchartype==0.1
-repoze.lru==0.6
-Routes==2.2
-WebOb==1.4.1
-WebHelpers==1.3
-Mako==1.0.2
-pytz==2015.4
-Babel==2.0
-Beaker==1.7.0
+Paste
+PasteDeploy
+docutils
+wchartype
+repoze.lru
+Routes
+WebOb
+WebHelpers
+Mako
+pytz
+Babel
+Beaker
 
 # Cheetah and dependencies
-Cheetah==2.4.4
-Markdown==2.6.3
+Cheetah
 
 # BioBlend and dependencies
-bioblend==0.6.1
-boto==2.38.0
-requests==2.8.1
-requests-toolbelt==0.4.0
+bioblend
+boto
+requests
 
 # kombu and dependencies
-kombu==3.0.30
-amqp==1.4.8
-anyjson==0.3.3
+kombu
 
 # sqlalchemy-migrate and dependencies
-sqlalchemy-migrate==0.10.0
-decorator==4.0.2
-Tempita==0.5.3dev
-sqlparse==0.1.16
-pbr==1.8.0
-six==1.9.0 # six is also a Pulsar client dep
-Parsley==1.3
-nose==1.3.7
-SVGFig==1.1.6
+sqlalchemy-migrate
+decorator
+Tempita
+sqlparse
+six
+Parsley
+nose
+SVGFig
 
 # Fabric and dependencies
-Fabric==1.10.2
-paramiko==1.15.2
-ecdsa==0.13
-# pycrypto, but it is a direct Galaxy dependency as well
+Fabric
+
+# We still pin these dependencies because of modifications to the upstream packages
 
 # Can't use Whoosh latest due to a bug but need to backport a bugfix from a
 # newer release.
 #
 # https://bitbucket.org/mchaput/whoosh/pull-requests/50/fix-lru-thread-safety/diff
 Whoosh==2.4.1+gx1
+
+# Flexible BAM index naming
+pysam==0.8.3+gx1

run.sh

@@ -18,7 +18,7 @@ fi
 while :
 do
     case "$1" in
-        --skip-eggs|--skip-wheels|--skip-samples|--dev-wheels)
+        --skip-eggs|--skip-wheels|--skip-samples|--dev-wheels|--no-create-venv|--no-replace-pip)
             common_startup_args="$common_startup_args $1"
             shift
             ;;

run_tests.sh

@@ -288,6 +288,14 @@ do
           skip_venv='--skip-venv'
           shift
           ;;
+      --no-create-venv)
+          no_create_venv='--no-create-venv'
+          shift
+          ;;
+      --no-replace-pip)
+          no_replace_pip='--no-replace-pip'
+          shift
+          ;;
       --skip-common-startup)
           # Don't run ./scripts/common_startup.sh (presumably it has already
           # been done, or you know what you're doing).
@@ -314,7 +322,7 @@ if [ -z "$skip_common_startup" ]; then
             GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION=$GALAXY_TEST_DBURI
             export GALAXY_CONFIG_OVERRIDE_DATABASE_CONNECTION
     fi
-    ./scripts/common_startup.sh $skip_venv --dev-wheels || exit 1
+    ./scripts/common_startup.sh $skip_venv $no_create_venv $no_replace_pip --dev-wheels || exit 1
 fi
 
 if [ -z "$skip_venv" -a -d .venv ];

scripts/common_startup.sh

@@ -4,12 +4,16 @@ set -e
 DEV_WHEELS=0
 FETCH_WHEELS=1
 SET_VENV=1
+CREATE_VENV=1
+REPLACE_PIP=1
 COPY_SAMPLE_FILES=1
 for arg in "$@"; do
     [ "$arg" = "--skip-eggs" ] && FETCH_WHEELS=0
     [ "$arg" = "--skip-wheels" ] && FETCH_WHEELS=0
     [ "$arg" = "--dev-wheels" ] && DEV_WHEELS=1
     [ "$arg" = "--skip-venv" ] && SET_VENV=0
+    [ "$arg" = "--no-create-venv" ] && CREATE_VENV=0
+    [ "$arg" = "--no-replace-pip" ] && REPLACE_PIP=0
     [ "$arg" = "--stop-daemon" ] && FETCH_WHEELS=0
     [ "$arg" = "--skip-samples" ] && COPY_SAMPLE_FILES=0
 done
@@ -57,7 +61,7 @@ if [ ! -f $GALAXY_CONFIG_FILE ]; then
     GALAXY_CONFIG_FILE=config/galaxy.ini.sample
 fi
 
-if [ $SET_VENV -eq 1 ]; then
+if [ $SET_VENV -eq 1 -a $CREATE_VENV -eq 1 ]; then
     # If .venv does not exist, attempt to create it.
     if [ ! -d .venv ]
     then
@@ -66,50 +70,68 @@ if [ $SET_VENV -eq 1 ]; then
         if command -v virtualenv >/dev/null; then
             virtualenv .venv
         else
-            vvers=13.1.0
+            vvers=13.1.2
             vurl="https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${vvers}.tar.gz"
+            vsha="aabc8ef18cddbd8a2a9c7f92bc43e2fea54b1147330d65db920ef3ce9812e3dc"
             vtmp=`mktemp -d -t galaxy-virtualenv-XXXXXX`
             vsrc="$vtmp/`basename $vurl`"
-            ( if command -v curl >/dev/null; then
-                curl --silent -o $vsrc $vurl
+            # SSL certificates are not checked to prevent problems with messed
+            # up client cert environments. We verify the download using a known
+            # good sha256 sum instead.
+            echo "Fetching $vurl"
+            if command -v curl >/dev/null; then
+                curl --insecure -L -o $vsrc $vurl
             elif command -v wget >/dev/null; then
-                wget --quiet -O $vsrc $vurl
+                wget --no-check-certificate -O $vsrc $vurl
             else
                 python -c "import urllib; urllib.urlretrieve('$vurl', '$vsrc')"
-            fi ) &&
-            tar zxf $vsrc -C $vtmp &&
-            python $vtmp/virtualenv-$vvers/virtualenv.py .venv &&
+            fi
+            echo "Verifying $vsrc checksum is $vsha"
+            if command -v sha256sum >/dev/null; then
+                echo "$vsha $vsrc" | sha256sum -c --strict
+            else
+                python -c "import hashlib; assert hashlib.sha256(open('$vsrc', 'rb').read()).hexdigest() == '$vsha', '$vsrc: invalid checksum'"
+            fi
+            tar zxf $vsrc -C $vtmp
+            python $vtmp/virtualenv-$vvers/virtualenv.py .venv
             rm -rf $vtmp
         fi
     fi
+fi
 
+if [ $SET_VENV -eq 1 ]; then
     # If there is a .venv/ directory, assume it contains a virtualenv that we
     # should run this instance in.
     if [ -d .venv ];
     then
         printf "Activating virtualenv at %s/.venv\n" $(pwd)
         . .venv/bin/activate
+        # Because it's a virtualenv, we assume $PYTHONPATH is unnecessary for
+        # anything in the venv to work correctly, and having it set can cause
+        # problems when there are conflicts with Galaxy's dependencies outside
+        # the venv (e.g. virtualenv-burrito's pip and six).
+        #
+        # If you are skipping the venv setup we shall assume you know what
+        # you're doing and will deal with any conflicts.
+        unset PYTHONPATH
+    fi
+
+    if [ -z "$VIRTUAL_ENV" ]; then
+        echo "ERROR: A virtualenv cannot be found. Please create a virtualenv in .venv, or activate one."
+        exit 1
     fi
 fi
 
-if [ -z "$VIRTUAL_ENV" ]; then
-    echo "ERROR: A virtualenv cannot be found or created. Please create a virtualenv in .venv, or activate one."
-    exit 1
-fi
-
-
 : ${GALAXY_WHEELS_INDEX_URL:="https://wheels.galaxyproject.org/simple"}
-if [ $FETCH_WHEELS -eq 1 ]; then
-    # Because it's a virtualenv, we assume $PYTHONPATH is unnecessary for
-    # anything in the venv to work correctly, and having it set can cause
-    # problems when there are conflicts with Galaxy's dependencies outside the
-    # venv (e.g. virtualenv-burrito's pip and six)
-    unset PYTHONPATH
+if [ $SET_VENV -eq 1 -a $REPLACE_PIP -eq 1 ]; then
     pip_version=`pip --version | awk '{print $2}'`
     pre=`python -c "from pkg_resources import parse_version; from sys import stdout; stdout.write('--pre') if parse_version('$pip_version') >= parse_version('1.4') else stdout.write('')"`
     pip install $pre --no-index --find-links ${GALAXY_WHEELS_INDEX_URL}/pip --upgrade pip
     # binary-compatibility.cfg may need to be created (e.g. on CentOS)
-    [ ! -f ${VIRTUAL_ENV}/binary-compatibility.cfg ] && python ./scripts/binary_compatibility.py -o ${VIRTUAL_ENV}/binary-compatibility.cfg
+    [ -n "$VIRTUAL_ENV" -a ! -f ${VIRTUAL_ENV}/binary-compatibility.cfg ] && python ./scripts/binary_compatibility.py -o ${VIRTUAL_ENV}/binary-compatibility.cfg
+fi
+
+if [ $FETCH_WHEELS -eq 1 ]; then
     pip install -r requirements.txt --index-url ${GALAXY_WHEELS_INDEX_URL}
     GALAXY_CONDITIONAL_DEPENDENCIES=`PYTHONPATH=lib python -c "import galaxy.dependencies; print '\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE'))"`
     [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url ${GALAXY_WHEELS_INDEX_URL}

tox.ini

@@ -20,7 +20,7 @@ deps = flake8
 
 
 [testenv:py27-unit]
-commands = bash run_tests.sh --skip-venv -u
+commands = bash run_tests.sh --no-create-venv -u
 whitelist_externals = bash
 deps =
     nose
@@ -28,7 +28,7 @@ deps =
     mock
 
 [testenv:py26-unit]
-commands = bash run_tests.sh --skip-venv -u
+commands = bash run_tests.sh --no-create-venv -u
 whitelist_externals = bash
 deps =
     unittest2