From b463be78a5066c7e6c389c43af2bd863f05e4a80 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Jan 2018 10:22:21 -0500 Subject: [PATCH 1/3] Add nodeenv to dev requirements --- lib/galaxy/dependencies/pipfiles/develop/Pipfile | 1 + lib/galaxy/dependencies/pipfiles/develop/Pipfile.lock | 8 +++++++- .../pipfiles/develop/pinned-hashed-requirements.txt | 1 + .../dependencies/pipfiles/develop/pinned-requirements.txt | 1 + 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/lib/galaxy/dependencies/pipfiles/develop/Pipfile b/lib/galaxy/dependencies/pipfiles/develop/Pipfile index 7f13cc074a7..9d6b30bff88 100644 --- a/lib/galaxy/dependencies/pipfiles/develop/Pipfile +++ b/lib/galaxy/dependencies/pipfiles/develop/Pipfile @@ -23,3 +23,4 @@ Sphinx = "*" sphinx_rtd_theme = "*" lxml = "*" recommonmark = "*" +nodeenv = "*" diff --git a/lib/galaxy/dependencies/pipfiles/develop/Pipfile.lock b/lib/galaxy/dependencies/pipfiles/develop/Pipfile.lock index 800b6d1495a..b89afcf0ea1 100644 --- a/lib/galaxy/dependencies/pipfiles/develop/Pipfile.lock +++ b/lib/galaxy/dependencies/pipfiles/develop/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "7da6298d5a5d49c87941125b449f37cc26c44d80f94d3d35a211463aa1efcf10" + "sha256": "5050d55a5cfaa22fddfa05d7edd3b2102a632d322266917f36a2c05f2522bcbb" }, "host-environment-markers": { "implementation_name": "cpython", @@ -156,6 +156,12 @@ ], "version": "==2.0.0" }, + "nodeenv": { + "hashes": [ + "sha256:98835dab727f94a713eacc7234e3db6777a55cafb60f391485011899e5c818df" + ], + "version": "==1.2.0" + }, "nose": { "hashes": [ "sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a", diff --git a/lib/galaxy/dependencies/pipfiles/develop/pinned-hashed-requirements.txt b/lib/galaxy/dependencies/pipfiles/develop/pinned-hashed-requirements.txt index 7542c0a2e35..24faed6e9b1 100644 --- a/lib/galaxy/dependencies/pipfiles/develop/pinned-hashed-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/develop/pinned-hashed-requirements.txt @@ -33,3 +33,4 @@ chardet==3.0.4 --hash=sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15 nose==1.3.7 --hash=sha256:dadcddc0aefbf99eea214e0f1232b94f2fa9bd98fa8353711dacb112bfcbbb2a --hash=sha256:9ff7c6cc443f8c51994b34a667bbcf45afd6d945be7477b52e97516fd17c53ac --hash=sha256:f1bffef9cbc82628f6e7d7b40d7e255aefaa1adb6a1b1d26c69a8b79e6208a98 requests==2.18.4 --hash=sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b --hash=sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e idna==2.6 --hash=sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4 --hash=sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f +nodeenv==1.2.0 --hash=sha256:98835dab727f94a713eacc7234e3db6777a55cafb60f391485011899e5c818df diff --git a/lib/galaxy/dependencies/pipfiles/develop/pinned-requirements.txt b/lib/galaxy/dependencies/pipfiles/develop/pinned-requirements.txt index 7319651e3e3..a0aeac6bb0f 100644 --- a/lib/galaxy/dependencies/pipfiles/develop/pinned-requirements.txt +++ b/lib/galaxy/dependencies/pipfiles/develop/pinned-requirements.txt @@ -33,3 +33,4 @@ chardet==3.0.4 nose==1.3.7 requests==2.18.4 idna==2.6 +nodeenv==1.2.0 From 977bbf1481b2fd36512f869c01f0af6ba4099a92 Mon Sep 17 00:00:00 2001 From: Nate Coraor Date: Wed, 24 Jan 2018 12:12:56 -0500 Subject: [PATCH 2/3] Install node and yarn in common_startup.sh using nodeenv and the venv's npm. --- scripts/common_startup.sh | 92 +++++++++++++++++++++++++++++---------- 1 file changed, 68 insertions(+), 24 deletions(-) diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 381a03d00fd..61325a209a6 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -49,6 +49,18 @@ RMFILES=" lib/pkg_resources.pyc " +# return true if $1 is in $VIRTUAL_ENV else false +in_venv() { + case $1 in + $VIRTUAL_ENV*) + return 0 + ;; + ''|*) + return 1 + ;; + esac +} + if [ $COPY_SAMPLE_FILES -eq 1 ]; then # Create any missing config/location files for sample in $SAMPLES; do @@ -65,30 +77,23 @@ for rmfile in $RMFILES; do [ -f "$rmfile" ] && rm -f "$rmfile" done -# Check client build state. -if [ $SKIP_CLIENT_BUILD -eq 0 ]; then - gitbranch=$(git rev-parse --abbrev-ref HEAD) - if [ "$gitbranch" = "dev" ]; then - # We're on dev. This branch (only, currently) doesn't have build - # artifacts. We should probabably swap to a list of releases? - # Compare hash. - if [ -f static/client_build_hash.txt ]; then - githash=$(git rev-parse HEAD) - statichash=$(cat static/client_build_hash.txt) - if [ "$githash" = "$statichash" ]; then - SKIP_CLIENT_BUILD=1 - fi - fi - else - # Not on dev. We're not going to bug people about building. - SKIP_CLIENT_BUILD=1 - fi - if [ $SKIP_CLIENT_BUILD -eq 0 ]; then - echo "The Galaxy client build is out of date. Please run 'make client' or your choice of client build target (client-*)." - echo "If you're sure you'd like to skip this check, you can run galaxy with the --skip-client-build flag, though this is not recommended as the client and server code will potentially be out of sync." - echo "See ./client/README.md in the Galaxy repository for more information, including how to get help if you're having trouble." - exit 1 - fi +# Determine branch (if using git) +if command -v git >/dev/null; then + GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) + case $GIT_BRANCH in + release_*|master) + # All non-release branches should build the client as necessary + SKIP_CLIENT_BUILD=1 + ;; + *) + # Ensure nodeenv is installed + DEV_WHEELS=1 + # SKIP_CLIENT_BUILD will default to false, but can be overridden by the command line argument + ;; + esac +else + GIT_BRANCH=0 + SKIP_CLIENT_BUILD=1 fi : ${GALAXY_CONFIG_FILE:=config/galaxy.yml} @@ -183,3 +188,42 @@ if [ $FETCH_WHEELS -eq 1 ]; then GALAXY_CONDITIONAL_DEPENDENCIES=$(PYTHONPATH=lib python -c "import galaxy.dependencies; print('\n'.join(galaxy.dependencies.optional('$GALAXY_CONFIG_FILE')))") [ -z "$GALAXY_CONDITIONAL_DEPENDENCIES" ] || echo "$GALAXY_CONDITIONAL_DEPENDENCIES" | pip install -r /dev/stdin --index-url "${GALAXY_WHEELS_INDEX_URL}" --extra-index-url "${PYPI_INDEX_URL}" fi + +# Check client build state. +if [ $SKIP_CLIENT_BUILD -eq 0 ]; then + if [ -f static/client_build_hash.txt ]; then + # Compare hash. + githash=$(git rev-parse HEAD) + statichash=$(cat static/client_build_hash.txt) + if [ "$githash" = "$statichash" ]; then + SKIP_CLIENT_BUILD=1 + else + echo "The Galaxy client is out of date and will be built now." + fi + else + echo "The Galaxy client has not yet been built and will be built now." + fi +fi + +# Build client if necessary. +if [ $SKIP_CLIENT_BUILD -eq 0 ]; then + # Ensure dependencies are installed + if [ -n "$VIRTUAL_ENV" ]; then + if ! in_venv `command -v node`; then + echo "Installing node into $VIRTUAL_ENV with nodeenv." + nodeenv -p + fi + if ! in_venv `command -v yarn`; then + echo "Installing yarn into $VIRTUAL_ENV with npm." + npm install --global yarn + fi + else + echo "WARNING: Galaxy client build needed but there is no virtualenv enabled. Build may fail." + fi + + # Build client + if ! make client; then + echo "ERROR: Galaxy client build failed. See ./client/README.md for more information, including how to get help." + exit 1 + fi +fi From eba3fb4b35570fc704dbb9a70c733293a2ee4514 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Thu, 8 Feb 2018 11:50:38 -0500 Subject: [PATCH 3/3] Fix common_startup.sh for galaxy tarballs Do not assume git is installed and the `.git` directory is present. Fix https://github.com/galaxyproject/galaxy/issues/5466 --- client/package.json | 2 +- scripts/common_startup.sh | 26 ++++++++++++++++---------- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/client/package.json b/client/package.json index 84841a190b8..c96fa5edb14 100644 --- a/client/package.json +++ b/client/package.json @@ -36,7 +36,7 @@ "build-production": "NODE_ENV=production gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack-production\" \"yarn run gulp clean && yarn run gulp-production\" && yarn run save-build-hash", "build-production-maps": "NODE_ENV=production gulp stage-libs && concurrently \"yarn run style\" \"yarn run webpack-production-maps\" \"yarn run gulp clean && yarn run gulp-production-maps\" && yarn run save-build-hash", "webpack": "webpack -d", - "save-build-hash": "git rev-parse HEAD > ../static/client_build_hash.txt", + "save-build-hash": "(git rev-parse HEAD 2>/dev/null || echo '') >../static/client_build_hash.txt", "webpack-watch": "webpack -d --watch", "webpack-production": "webpack -p", "webpack-production-maps": "GXY_BUILD_SOURCEMAPS=1 webpack -p", diff --git a/scripts/common_startup.sh b/scripts/common_startup.sh index 61325a209a6..3bd01a110b1 100755 --- a/scripts/common_startup.sh +++ b/scripts/common_startup.sh @@ -78,7 +78,7 @@ for rmfile in $RMFILES; do done # Determine branch (if using git) -if command -v git >/dev/null; then +if command -v git >/dev/null && [ -d .git ]; then GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD) case $GIT_BRANCH in release_*|master) @@ -93,7 +93,7 @@ if command -v git >/dev/null; then esac else GIT_BRANCH=0 - SKIP_CLIENT_BUILD=1 + DEV_WHEELS=1 fi : ${GALAXY_CONFIG_FILE:=config/galaxy.yml} @@ -120,7 +120,7 @@ if [ $SET_VENV -eq 1 -a $CREATE_VENV -eq 1 ]; then # Ensure Python is a supported version before creating .venv python ./scripts/check_python.py || exit 1 if command -v virtualenv >/dev/null; then - virtualenv -p python2.7 "$GALAXY_VIRTUAL_ENV" + virtualenv -p "$(command -v python)" "$GALAXY_VIRTUAL_ENV" else vvers=13.1.2 vurl="https://pypi.python.org/packages/source/v/virtualenv/virtualenv-${vvers}.tar.gz" @@ -192,13 +192,19 @@ fi # Check client build state. if [ $SKIP_CLIENT_BUILD -eq 0 ]; then if [ -f static/client_build_hash.txt ]; then - # Compare hash. - githash=$(git rev-parse HEAD) - statichash=$(cat static/client_build_hash.txt) - if [ "$githash" = "$statichash" ]; then + # If git is not used and static/client_build_hash.txt is present, next + # client rebuilds must be done manually by the admin + if [ "$GIT_BRANCH" = "0" ]; then SKIP_CLIENT_BUILD=1 else - echo "The Galaxy client is out of date and will be built now." + # Compare hash. + githash=$(git rev-parse HEAD) + statichash=$(cat static/client_build_hash.txt) + if [ "$githash" = "$statichash" ]; then + SKIP_CLIENT_BUILD=1 + else + echo "The Galaxy client is out of date and will be built now." + fi fi else echo "The Galaxy client has not yet been built and will be built now." @@ -209,11 +215,11 @@ fi if [ $SKIP_CLIENT_BUILD -eq 0 ]; then # Ensure dependencies are installed if [ -n "$VIRTUAL_ENV" ]; then - if ! in_venv `command -v node`; then + if ! in_venv "$(command -v node)"; then echo "Installing node into $VIRTUAL_ENV with nodeenv." nodeenv -p fi - if ! in_venv `command -v yarn`; then + if ! in_venv "$(command -v yarn)"; then echo "Installing yarn into $VIRTUAL_ENV with npm." npm install --global yarn fi