diff --git a/.circleci/config.yml b/.circleci/config.yml index e5da1bca335..d7e557b6896 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -93,6 +93,15 @@ jobs: - run: sudo apt-get install -y libxml2-utils - *install_tox - run: tox -e validate_test_tools + test_galaxy_release: + docker: + - image: circleci/python:3.6 + <<: *set_workdir + steps: + - *restore_repo_cache + - *install_tox + - *install_ffprobe + - run: tox -e test_galaxy_release test_galaxy_packages: docker: - image: circleci/python:3.6 diff --git a/Makefile b/Makefile index 0cc62b722cf..eecec849d45 100644 --- a/Makefile +++ b/Makefile @@ -2,14 +2,8 @@ VENV?=.venv # Source virtualenv to execute command (flake8, sphinx, twine, etc...) IN_VENV=if [ -f "$(VENV)/bin/activate" ]; then . "$(VENV)/bin/activate"; fi; -RELEASE_CURR:=16.01 -RELEASE_CURR_MINOR_NEXT:=$(shell $(IN_VENV) python scripts/bootstrap_history.py --print-next-minor-version) -RELEASE_NEXT:=16.04 -# TODO: This needs to be updated with create_release_rc -#RELEASE_NEXT_BRANCH:=release_$(RELEASE_NEXT) -RELEASE_NEXT_BRANCH:=dev +RELEASE_CURR:=21.05 RELEASE_UPSTREAM:=upstream -MY_UPSTREAM:=origin CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py PROJECT_URL?=https://github.com/galaxyproject/galaxy DOCS_DIR=doc @@ -180,97 +174,13 @@ serve-selenium-notebooks: ## Serve testing notebooks for Jupyter cd lib && export PYTHONPATH=`pwd`; jupyter notebook --notebook-dir=galaxy_test/selenium/jupyter # Release Targets -release-create-rc: release-ensure-upstream ## Create a release-candidate branch - git checkout dev - git pull --ff-only $(RELEASE_UPSTREAM) dev - git push $(MY_UPSTREAM) dev - git checkout -b release_$(RELEASE_CURR) - git push $(MY_UPSTREAM) release_$(RELEASE_CURR) - git push $(RELEASE_UPSTREAM) release_$(RELEASE_CURR) - git checkout -b version-$(RELEASE_CURR) - sed -i.bak -e "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_CURR)\"/" lib/galaxy/version.py - sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = \"rc1\"/" lib/galaxy/version.py - rm -f lib/galaxy/version.py.bak - git add lib/galaxy/version.py - git commit -m "Update version to $(RELEASE_CURR).rc1" - git checkout dev +release-create-rc: ## Create a release-candidate branch or new release-candidate version + $(IN_VENV) ./scripts/release.sh -c - git checkout -b version-$(RELEASE_NEXT).dev - sed -i.bak -e "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_NEXT)\"/" lib/galaxy/version.py - rm -f lib/galaxy/version.py.bak - git add lib/galaxy/version.py - git commit -m "Update version to $(RELEASE_NEXT).dev" +release-create: ## Create a release branch + $(IN_VENV) ./scripts/release.sh - -git merge version-$(RELEASE_CURR) - git checkout --ours lib/galaxy/version.py - git add lib/galaxy/version.py - git commit -m "Merge branch 'version-$(RELEASE_CURR)' into version-$(RELEASE_NEXT).dev" - git push $(MY_UPSTREAM) version-$(RELEASE_CURR):version-$(RELEASE_CURR) - git push $(MY_UPSTREAM) version-$(RELEASE_NEXT).dev:version-$(RELEASE_NEXT).dev - git checkout dev - # TODO: Use hub to automate these PR creations or push directly. - @echo "Open a PR from version-$(RELEASE_CURR) of your fork to release_$(RELEASE_CURR)" - @echo "Open a PR from version-$(RELEASE_NEXT).dev of your fork to dev" - -release-create: release-ensure-upstream ## Create a release branch - git checkout master - git pull --ff-only $(RELEASE_UPSTREAM) master - git push $(MY_UPSTREAM) master - git checkout release_$(RELEASE_CURR) - git pull --ff-only $(RELEASE_UPSTREAM) release_$(RELEASE_CURR) - #git push $(MY_UPSTREAM) release_$(RELEASE_CURR) - git checkout dev - git pull --ff-only $(RELEASE_UPSTREAM) dev - #git push $(MY_UPSTREAM) dev - # Test run of merging. If there are conflicts, it will fail here. - git merge release_$(RELEASE_CURR) - git checkout release_$(RELEASE_CURR) - sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = None/" lib/galaxy/version.py - rm -f lib/galaxy/version.py.bak - git add lib/galaxy/version.py - git commit -m "Update version to $(RELEASE_CURR)" - git tag -m "Tag version $(RELEASE_CURR)" v$(RELEASE_CURR) - - git checkout dev - -git merge release_$(RELEASE_CURR) - git checkout --ours lib/galaxy/version.py - git add lib/galaxy/version.py - git commit -m "Merge branch 'release_$(RELEASE_CURR)' into dev" - git checkout master - git merge release_$(RELEASE_CURR) - git push $(RELEASE_UPSTREAM) release_$(RELEASE_CURR):release_$(RELEASE_CURR) - git push $(RELEASE_UPSTREAM) dev:dev - git push $(RELEASE_UPSTREAM) master:master - git push $(RELEASE_UPSTREAM) --tags - -release-create-point: ## Create a point release - git pull --ff-only $(RELEASE_UPSTREAM) master - git push $(MY_UPSTREAM) master - git checkout release_$(RELEASE_CURR) - git pull --ff-only $(RELEASE_UPSTREAM) release_$(RELEASE_CURR) - #git push $(MY_UPSTREAM) release_$(RELEASE_CURR) - git checkout $(RELEASE_NEXT_BRANCH) - git pull --ff-only $(RELEASE_UPSTREAM) $(RELEASE_NEXT_BRANCH) - #git push $(MY_UPSTREAM) $(RELEASE_NEXT_BRANCH) - git merge release_$(RELEASE_CURR) - git checkout release_$(RELEASE_CURR) - sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = \"$(RELEASE_CURR_MINOR_NEXT)\"/" lib/galaxy/version.py - rm -f lib/galaxy/version.py.bak - git add lib/galaxy/version.py - git commit -m "Update version to $(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)" - git tag -m "Tag version $(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)" v$(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT) - git checkout $(RELEASE_NEXT_BRANCH) - -git merge release_$(RELEASE_CURR) - git checkout --ours lib/galaxy/version.py - git add lib/galaxy/version.py - git commit -m "Merge branch 'release_$(RELEASE_CURR)' into $(RELEASE_NEXT_BRANCH)" - git checkout master - git merge release_$(RELEASE_CURR) - #git push origin release_$(RELEASE_CURR):release_$(RELEASE_CURR) - #git push origin $(RELEASE_NEXT_BRANCH):release_$(RELEASE_NEXT_BRANCH) - #git push origin master:master - #git push origin --tags - git checkout release_$(RELEASE_CURR) +release-create-point: release-create ## Create a point release .PHONY: help diff --git a/packages/package.Makefile b/packages/package.Makefile index 16014364d2e..f924ba81fe7 100644 --- a/packages/package.Makefile +++ b/packages/package.Makefile @@ -9,7 +9,7 @@ SOURCE_DIR?=galaxy BUILD_SCRIPTS_DIR=scripts DEV_RELEASE?=0 VERSION?=$(shell DEV_RELEASE=$(DEV_RELEASE) python $(BUILD_SCRIPTS_DIR)/print_version_for_release.py $(SOURCE_DIR) $(DEV_RELEASE)) -PROJECT_NAME?="galaxy-$(shell basename $(CURDIR))" +PROJECT_NAME?=galaxy-$(shell basename $(CURDIR)) PROJECT_NAME:=$(subst _,-,$(PROJECT_NAME)) BRANCH?=$(shell git rev-parse --abbrev-ref HEAD) TEST_DIR?=tests diff --git a/scripts/bootstrap_history.py b/scripts/bootstrap_history.py index 494911839ea..c891d710099 100644 --- a/scripts/bootstrap_history.py +++ b/scripts/bootstrap_history.py @@ -200,14 +200,15 @@ RELEASE_ISSUE_TEMPLATE = string.Template(""" - [X] **Prep** - - [X] ~~Create this release issue ``make release-issue RELEASE_CURR=${version}``.~~ + - [X] ~~Create this release issue ``make release-issue``.~~ - [X] ~~Set freeze date (${freeze_date}).~~ - [ ] **Branch Release (on or around ${freeze_date})** - [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged, delayed, or closed. - make release-check-blocking-prs RELEASE_CURR=${version} + make release-check-blocking-prs + - [ ] Merge the latest release into dev and push upstream. make release-merge-stable-to-next RELEASE_PREVIOUS=release_${previous_version} @@ -215,12 +216,13 @@ RELEASE_ISSUE_TEMPLATE = string.Template(""" - [ ] Create and push release branch: - make release-create-rc RELEASE_CURR=${version} RELEASE_NEXT=${next_version} + make release-create-rc - [ ] Open PRs from your fork of branch ``version-${version}`` to upstream ``release_${version}`` and of ``version-${next_version}.dev`` to ``dev``. - - [ ] Update ``MILESTONE_NUMBER`` in the [ maintenance bot](https://github.com/galaxyproject/galaxy/blob/dev/.github/workflows/maintenance_bot.yaml) to `${next_version}` so it properly tags new PRs. + - [ ] Update ``MILESTONE_NUMBER`` in the [maintenance bot](https://github.com/galaxyproject/galaxy/blob/dev/.github/workflows/maintenance_bot.yaml) to `${next_version}` so it properly tags new PRs. - [ ] **Issue Review Timeline Notes** + - [ ] Ensure any security fixes will be ready prior to ${freeze_date} + 1 week, to allow time for notification prior to release. - [ ] Ensure ownership of outstanding bugfixes and track progress during freeze. @@ -265,11 +267,11 @@ RELEASE_ISSUE_TEMPLATE = string.Template(""" - [ ] Ensure all PRs merged into the pre-release branch during the freeze have [milestones attached](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_{version}+is%3Amerged+no%3Amilestone) and that they are the not [${next_version} milestones](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_{version}+is%3Amerged+milestone%3A{next_version}) - [ ] Ensure release notes include all PRs added during the freeze by re-running the release note bootstrapping: - make release-bootstrap-history RELEASE_CURR=${version} + make release-bootstrap-history - [ ] Ensure previous release is merged into current. [GitHub branch comparison](https://github.com/galaxyproject/galaxy/compare/release_${version}...release_${previous_version}) - [ ] Create and push release tag: - make release-create RELEASE_CURR=${version} + make release-create - [ ] Add the branch `*/release_{version}` to Jenkins documentation build [configuration matrix](https://jenkins.galaxyproject.org/job/galaxy-sphinx-by-branch/configure). - [ ] Trigger the [branch documentation build](https://jenkins.galaxyproject.org/job/galaxy-sphinx-by-branch/) @@ -294,7 +296,7 @@ RELEASE_ISSUE_TEMPLATE = string.Template(""" - [ ] **Prepare for next release** - [ ] Close milestone ``${version}`` and ensure milestone ``${next_version}`` exists. - - [ ] Create release issue for next version ``make release-issue RELEASE_CURR=${next_version}``. + - [ ] Create release issue for next version ``make release-issue``. - [ ] Schedule committer meeting to discuss re-alignment of priorities. - [ ] Close this issue. """) @@ -314,21 +316,6 @@ GROUPPED_TAGS = OrderedDict([ # https://api.github.com/repos/galaxyproject/galaxy/compare/release_15.05...dev -def print_next_minor_version(): - minor_version_str = None - with open(GALAXY_VERSION_FILE) as f: - for line in f: - result = re.match(r'VERSION_MINOR = "(.*)"', line) - if result: - minor_version_str = result.group(1) - break - try: - minor_version = int(minor_version_str) - except (TypeError, ValueError): - minor_version = 0 - print(minor_version + 1) - - def release_issue(argv): release_name = argv[2] previous_release = _previous_release(release_name) @@ -511,10 +498,6 @@ def main(argv, seen_prs=None): newest_release = None seen_prs = seen_prs or set() - if argv[1] == "--print-next-minor-version": - print_next_minor_version() - return - if argv[1] == "--check-blocking-prs": check_blocking_prs(argv) return diff --git a/scripts/release.sh b/scripts/release.sh new file mode 100755 index 00000000000..587f5e52902 --- /dev/null +++ b/scripts/release.sh @@ -0,0 +1,623 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s extglob + +# TODO: upload packages +# TODO: set dev version in 21.01 (+ 20.09?) branches + +: ${VENV:=.venv} +: ${FORK_REMOTE:=origin} +: ${UPSTREAM_REMOTE:=upstream} +: ${UPSTREAM_REMOTE_URL:=git@github.com:galaxyproject/galaxy.git} +: ${DEV_BRANCH:=dev} +: ${STABLE_BRANCH:=master} + +# Vars for local releases +: ${RELEASE_LOCAL_TAG:=local} +: ${RELEASE_LOCAL_COMMIT:=$(git rev-parse --short HEAD)} +: ${RELEASE_LOCAL_VERSION:=${RELEASE_LOCAL_TAG}$(date -u +%Y%m%dT%H%M%SZ).${RELEASE_LOCAL_COMMIT}} + +# Only use this for dev/testing/CI to ignore forward merge conflicts, skip confirmation and package builds, etc. +: ${TEST_MODE:=false} + +VERIFY_PACKAGES=(wheel packaging) + +BRANCH_CURR=$(git branch --show-current) + +: ${RELEASE_CURR:=$(grep '^VERSION_MAJOR' lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/")} +RELEASE_NEXT= +RELEASE_CURR_MINOR= +RELEASE_CURR_MINOR_NEXT= +RELEASE_CURR_MINOR_NEXT_DEV= +RELEASE_TYPE= +PACKAGE_VERSION= + +declare -a CLEAN_BRANCHES +declare -a CLEAN_TAGS +CLEAN_BRANCHES=() +CLEAN_TAGS=() + +declare -a PUSH_BRANCHES +PUSH_BRANCHES=() + +WORKING_DIR_CLEAN=false +ERROR=false + +NEXT_RELEASE_TEMPLATE="import datetime; print((datetime.datetime.strptime('RELEASE_CURR', '%y.%m').date() + datetime.timedelta(days=(31 * 4))).strftime('%y.%m'))" +PACKAGE_VERSION_TEMPLATE="import packaging.version; print('.'.join(map(str, packaging.version.parse('VERSION').release)))" +PACKAGE_DEV_VERSION_TEMPLATE="import packaging.version; print(packaging.version.parse('VERSION'))" + + +while getopts ":clr:" opt; do + case "$opt" in + c) + RELEASE_TYPE='rc' + ;; + r) + RELEASE_CURR="$OPTARG" + ;; + l) + RELEASE_TYPE='local' + ;; + *) + echo "usage: $(basename "$0") [-c (force rc)] [-l (local release)] [-r release version]" + exit 1 + ;; + esac +done + + +function trap_handler() { + { set +x; } 2>/dev/null + local file + $ERROR && log_func=log_error || log_func=log + $log_func "Cleaning up..." + if $WORKING_DIR_CLEAN; then + log_exec git reset -- . + log_exec git checkout -- . + fi + [ "$(git branch --show-current)" == "$BRANCH_CURR" ] || log_exec git checkout "$BRANCH_CURR" + [ "${#CLEAN_BRANCHES[@]}" -eq 0 ] || git branch -D "${CLEAN_BRANCHES[@]}" + $ERROR && exit 1 || exit 0 +} + + +function trap_handler_err() { + ERROR=true + for tag in "${CLEAN_TAGS[@]}"; do + log_exec git tag -d "$tag" + done + trap_handler +} + + +function trap_handler_ok() { + # if $ERROR is true then the trap handler already ran + $ERROR || trap_handler +} + + +trap "trap_handler_err" SIGTERM SIGINT ERR +trap "trap_handler_ok" EXIT + + +function log() { + [ -t 0 ] && echo -e '\033[1;32m#' "$@" '\033[0m' 1>&2 || echo '#' "$@" 1>&2 +} + + +function log_warning() { + [ -t 0 ] && echo -e '\033[1;33mWARNING:' "$@" '\033[0m' 1>&2 || echo 'WARNING:' "$@" 1>&2 +} + + +function log_error() { + [ -t 0 ] && echo -e '\033[1;31mERROR:' "$@" '\033[0m' 1>&2 || echo 'ERROR:' "$@" 1>&2 +} + + +function log_debug() { + echo "####" "$@" 1>&2 +} + + +function log_exec() { + local rc + set -x + "$@" + { rc=$?; set +x; } 2>/dev/null + return $rc +} + + +function fork_owner() { + local url=$(git remote get-url "$FORK_REMOTE") + case "$url" in + https://github.com/*) + echo "$url" | awk -F/ '{print $4}' + ;; + git@github.com:*) + echo "$url" | awk -F: '{print $2}' | awk -F/ '{print $1}' + ;; + file:///*) + echo '__DEV_TEST_USER__' + ;; + *) + log_error "Cannot parse remote '$FORK_REMOTE' url for owner username: $url" + exit 1 + ;; + esac +} + + +function ensure_upstream() { + if ! git remote -v | grep -E "^${UPSTREAM_REMOTE}\s+" >/dev/null; then + log_warning "Remote ${UPSTREAM_REMOTE} does not exist, will be added with URL: ${UPSTREAM_REMOTE_URL}" + log_exec git remote add "$UPSTREAM_REMOTE" "$UPSTREAM_REMOTE_URL" + fi +} + + +function ensure_prereqs() { + local pip_list + log "Checking for uncommitted local modifications..." + log_exec git diff --stat --exit-code || { log_error "Some files have changes"; exit 1; } + WORKING_DIR_CLEAN=true + log "Checking for required packages..." + if [ -z "${VIRTUAL_ENV:-}" ]; then + [ -d "$VENV" ] || { log_error "Missing venv, please create: ${VENV}"; exit 1; } + . "${VENV}/bin/activate" + fi + pip_list=$(log_exec "${VENV}/bin/pip" list) + for package in ${VERIFY_PACKAGES[@]}; do + echo "$pip_list" | grep -E "^${package}\s+" || { log_error "Package '${package}' missing from venv: ${VENV}" ; exit 1; } + done +} + + +function user_verify_release() { + log "Release Details" + local future + case "${RELEASE_NEXT:-}" in + '') + future="${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT_DEV}" + ;; + *) + future="${RELEASE_NEXT}.dev0" + ;; + esac + cat <= stable branch release '${stable}', merging 'release_${RELEASE_CURR}' to '${STABLE_BRANCH}'" + log_exec git merge -m "Merge branch 'release_${RELEASE_CURR}' into '${STABLE_BRANCH}'" "__release_${RELEASE_CURR}" + PUSH_BRANCHES+=("__stable:${STABLE_BRANCH}") + else + log "Release '${RELEASE_CURR}' < stable branch release '${stable}', skipping merge to to '${STABLE_BRANCH}'" + fi + git checkout "$branch_curr" +} + + +function _perform_forward_merge() { + local curr="$1" + local next="$(release_next "$curr")" + local curr_branch="${UPSTREAM_REMOTE}/release_${curr}" + local next_branch="${UPSTREAM_REMOTE}/release_${next}" + local curr_local_branch="__release_${curr}" + local next_local_branch="__release_${next}" + local recurse=true + log "Performing forward merge of ${curr} to ${next}" + branch_exists "$curr_local_branch" || { log_error "Missing expected branch: ${curr_local_branch}"; exit 1; } + branch_exists "$next_local_branch" && { log_error "Unexpected branch exists: ${next_local_branch}"; exit 1; } + PUSH_BRANCHES+=("${curr_local_branch}:release_${curr}") + if ! branch_exists "$next_branch"; then + next_branch="${UPSTREAM_REMOTE}/${DEV_BRANCH}" + recurse=false + PUSH_BRANCHES+=("${next_local_branch}:${DEV_BRANCH}") + fi + git_checkout_temp "$next_local_branch" "$next_branch" + # This should necessarily result in conflicts merging version.py + log_exec git merge -X ours -m "Merge branch 'release_${curr}' into 'release_${next}'" "$curr_local_branch" + if $recurse; then + _perform_forward_merge "$next" + fi +} + + +function perform_forward_merge() { + local curr="$1" + local branch_curr=$(git branch --show-current) + _perform_forward_merge "$@" + declare -p PUSH_BRANCHES + git checkout "$branch_curr" +} + + +function push_merged() { + local curr="$1" + for branch in "${PUSH_BRANCHES[@]}"; do + log "Pushing '${branch}' to remote '${UPSTREAM_REMOTE}'" + log_exec git push "$UPSTREAM_REMOTE" "$branch" + done + if [ "$RELEASE_TYPE" == 'initial' -o "$RELEASE_TYPE" == 'point' ]; then + log_exec git push --tags "$UPSTREAM_REMOTE" + fi +} + + +function set_package_version_var() { + local package_version_minor + local dev_release='false' + case "$RELEASE_CURR_MINOR_NEXT" in + [0-9]*) + package_version_minor="$RELEASE_CURR_MINOR_NEXT" + ;; + dev) + # TODO: you can remove this case once the current dev branch minor version is updated to dev0 + package_version_minor="0dev0" + dev_release='true' + ;; + *) + package_version_minor="0${RELEASE_CURR_MINOR_NEXT}" + dev_release='true' + ;; + esac + PACKAGE_VERSION=$(packaging_version "${RELEASE_CURR}.${package_version_minor}" "$dev_release") +} + + +function increment_minor() { + local minor="$1" + case "$minor" in + +([0-9])) + [ "$RELEASE_TYPE" != 'rc' ] || { + log_error "Cannot create rc after release (current version: ${RELEASE_CURR}.${minor})"; + exit 1; } + echo "$((minor + 1)).dev0" + ;; + +([0-9]).dev*) + [ "$RELEASE_TYPE" != 'rc' ] || { + log_error "Cannot create rc after release (current version: ${RELEASE_CURR}.${minor})"; + exit 1; } + echo "${minor%.*}" + ;; + rc*) + if [ "$RELEASE_TYPE" == 'rc' ]; then + echo "rc$((${minor#rc*} + 1))" + else + echo '' + fi + ;; + None|'') + [ "$RELEASE_TYPE" != 'rc' ] || { + log_error "Cannot create rc after release (current version: ${RELEASE_CURR}.0)"; + exit 1; } + echo '1.dev0' + ;; + *) + log_error "Don't know how to increment minor version: ${minor}" + exit 1 + ;; + esac +} + + +function get_version_major() { + grep '^VERSION_MAJOR' lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/" +} + + +function get_version_minor() { + grep '^VERSION_MINOR' lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/" | tr -d '[[:space:]]' +} + + +function set_version_vars() { + RELEASE_CURR="$(get_version_major)" + RELEASE_CURR_MINOR="$(get_version_minor)" + [[ "$RELEASE_CURR_MINOR" =~ .*"None"$ ]] && RELEASE_CURR_MINOR='0' + : ${RELEASE_TYPE:=point} + RELEASE_CURR_MINOR_NEXT="$(increment_minor "$RELEASE_CURR_MINOR")" + [ -n "$RELEASE_CURR_MINOR_NEXT" ] || RELEASE_TYPE='initial' + RELEASE_CURR_MINOR_NEXT_DEV="$(increment_minor "${RELEASE_CURR_MINOR_NEXT}")" + set_package_version_var +} + + +function update_galaxy_version() { + local key="$1" + local val="$2" + local match="${3:-.*}" + log "Updating lib/galaxy/version.py for ${key} = ${val}..." + log_exec sed -E -i -e "s/^${key} = ${match}/${key} = \"$val\"/" lib/galaxy/version.py +} + + +function packages_make_all() { + local dir + # subshell to preserve cwd + ( + cd packages/ + for dir in *; do + [ ! -d "$dir" -o ! -f "${dir}/setup.py" ] && continue + # can't use log_exec here because we want to capture output + echo + make -C "$dir" "$@" 1>&2 + make -C "$dir" "$@" >"${dir}/make-${1}.log" 2>&1 + done + ) +} + + +function update_package_versions() { + local package_version="${1:-$PACKAGE_VERSION}" + local project_file + log "Updating package versions to '${package_version}'..." + ( + cd packages/ + for dir in *; do + project_file="${dir}/galaxy/project_galaxy_${dir}.py" + if [ -f "${project_file}" ]; then + sed -i -e "s/^__version__ =.*/__version__ = \"${package_version}\"/" "$project_file" + fi + done + ) + #git diff -- packages/ +} + + +function perform_version_update() { + local tag_version= + update_package_versions + git add -- packages/ + log 'Cleaning package dirs...' + packages_make_all clean + log 'Building packages (logs in packages/*/make-dist.log)...' + if $TEST_MODE; then + log_debug "Test mode skipping package build" + else + packages_make_all dist + fi + log 'Committing version changes...' + log_exec git commit -m "Update version to ${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT}" + case "$RELEASE_TYPE" in + initial) + tag_version="${RELEASE_CURR}" + ;; + point) + tag_version="${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT}" + ;; + esac + if [ -n "$tag_version" ]; then + log_exec git tag -m "Tag version ${tag_version}" "v${tag_version}" + CLEAN_TAGS+=("v${tag_version}") + fi +} + + +function perform_version_update_dev() { + [ "$RELEASE_TYPE" == 'initial' -o "$RELEASE_TYPE" == 'point' ] || return 0 + log "Incrementing release version to '${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT_DEV}' for development of next point release" + update_galaxy_version 'VERSION_MINOR' "$RELEASE_CURR_MINOR_NEXT_DEV" + log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true + git add -- lib/galaxy/version.py + update_package_versions + git add -- packages/ + log_exec git commit -m "Update version to ${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT_DEV}" +} + + +function create_release_rc_initial() { + RELEASE_TYPE='rc-initial' + RELEASE_NEXT="$(release_next "$RELEASE_CURR")" + RELEASE_CURR_MINOR_NEXT='rc1' + set_package_version_var + user_verify_release + + local _dev_branch="${UPSTREAM_REMOTE}/${DEV_BRANCH}" + git_checkout_temp "__release_${RELEASE_CURR}" "$_dev_branch" + + update_galaxy_version 'VERSION_MAJOR' "$RELEASE_CURR" + update_galaxy_version 'VERSION_MINOR' "$RELEASE_CURR_MINOR_NEXT" + log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true + git add -- lib/galaxy/version.py + + perform_version_update + + # Increment version in dev branch + git_checkout_temp "__release_${RELEASE_NEXT}" "$_dev_branch" + + update_galaxy_version 'VERSION_MAJOR' "$RELEASE_NEXT" + update_galaxy_version 'VERSION_MINOR' 'dev0' + log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true + git add -- lib/galaxy/version.py + log_exec sed -i -e "s/^RELEASE_CURR:=.*/RELEASE_CURR:=${RELEASE_NEXT}/" Makefile + log_exec git diff --exit-code && { log_error 'Missing expected Makefile changes'; exit 1; } || true + git add -- Makefile + local package_version=$(packaging_version "${RELEASE_NEXT}.0dev0" "true") + update_package_versions "$package_version" + git add -- packages/ + log_exec git commit -m "Update version to ${RELEASE_NEXT}.dev0" + + # Resolve merge conflicts + log_exec git merge -X ours -m "Merge branch 'release_${RELEASE_CURR}' into 'dev'" "__release_${RELEASE_CURR}" + + # Push branches for PR + local owner=$(fork_owner) + local curr_remote_branch="version-${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT}" + local next_remote_branch="version-${RELEASE_NEXT}.dev" + log_exec git push "$FORK_REMOTE" "__release_${RELEASE_CURR}:${curr_remote_branch}" + log_exec git push "$FORK_REMOTE" "__release_${RELEASE_NEXT}:${next_remote_branch}" + log_exec git push "$UPSTREAM_REMOTE" "refs/remotes/${UPSTREAM_REMOTE}/${DEV_BRANCH}:refs/heads/release_${RELEASE_CURR}" + + #https://github.com/galaxyproject/galaxy/compare/{branch}...{fork_owner}:{branch} + log "Open a PR from ${owner}/galaxy:${curr_remote_branch} to galaxyproject/galaxy:release_${RELEASE_CURR}" + echo "- https://github.com/galaxyproject/galaxy/compare/release_${RELEASE_CURR}...${owner}:${curr_remote_branch}" 1>&2 + log "Open a PR from ${owner}/galaxy:${next_remote_branch} to galaxyproject/galaxy:dev" + echo "- https://github.com/galaxyproject/galaxy/compare/dev...${owner}:${next_remote_branch}" 1>&2 +} + + +function create_release_local() { + RELEASE_CURR_MINOR="$(get_version_minor)" + RELEASE_CURR_MINOR_NEXT="${RELEASE_CURR_MINOR}+${RELEASE_LOCAL_VERSION}" + set_package_version_var + user_verify_release + # Append the local portion if this is a point release or the local portion already exists + update_galaxy_version 'VERSION_MINOR' "\1+${RELEASE_LOCAL_VERSION}" "['\"]([^+]*)[^'\"\]*['\"]$" + # Set the minor version to 0 and append the local version if the release version is not set (e.g. .0) + update_galaxy_version 'VERSION_MINOR' "0+${RELEASE_LOCAL_VERSION}" 'None' + update_galaxy_version 'VERSION_MINOR' "0+${RELEASE_LOCAL_VERSION}" '""' + log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true + update_package_versions + ##git add -- lib/galaxy/version.py packages/*/galaxy/project_*.py + packages_make_all clean + log 'Building packages (logs in packages/*/make-dist.log)...' + packages_make_all dist + ##git commit -m "Update version to $(RELEASE_CURR)" + ##git tag -m "Tag version $(RELEASE_CURR)" v$(RELEASE_CURR) +} + + +function create_release_normal() { + local _release_curr="$RELEASE_CURR" + local _release_branch="${UPSTREAM_REMOTE}/release_${RELEASE_CURR}" + git_checkout_temp "__release_${RELEASE_CURR}" "$_release_branch" + set_version_vars + [ "$_release_curr" == "$RELEASE_CURR" ] || { + log_error "Release is incorrect in branch ${_release_branch}: ${_release_curr} != ${RELEASE_CURR}"; + exit 1; } + + user_verify_release + test_forward_merge "$RELEASE_CURR" + + update_galaxy_version 'VERSION_MINOR' "$RELEASE_CURR_MINOR_NEXT" + log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true + git add -- lib/galaxy/version.py + + perform_version_update + perform_stable_merge + perform_version_update_dev + perform_forward_merge "$RELEASE_CURR" + push_merged "$RELEASE_CURR" +} + + +function create_release() { + log 'Fetching upstream changes...' + git fetch "$UPSTREAM_REMOTE" + if [ "$RELEASE_TYPE" == 'local' ]; then + create_release_local + elif ! branch_exists "${UPSTREAM_REMOTE}/release_${RELEASE_CURR}"; then + create_release_rc_initial + else + create_release_normal + fi +} + + +function main() { + ensure_upstream + ensure_prereqs + create_release +} + + +main diff --git a/test/release.sh b/test/release.sh new file mode 100755 index 00000000000..fcb6b1f8f99 --- /dev/null +++ b/test/release.sh @@ -0,0 +1,308 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Necessary for testing the release script +export TEST_MODE=true + +: ${ORIGIN:=origin} +: ${STABLE_BRANCH:=master} + +REPO_ROOT= +FORK_ROOT=$(mktemp -d -t galaxy_release_test_XXXXXXXX) +DEV_BRANCH="$(git branch --show-current)" + +TEST_RELEASE_PREV= +TEST_RELEASE_PREV_MINOR= +TEST_RELEASE_CURR='99.01' +TEST_RELEASE_NEXT='99.05' +TEST_RELEASE_NEXT_NEXT='99.09' + +: ${VENV:=${FORK_ROOT}/venv} +export VENV + +[ -n "$DEV_BRANCH" ] || { echo 'ERROR: Cannot determine current branch'; exit 1; } + + +function trap_handler() { + [ -z "$FORK_ROOT" ] || rm -rf "$FORK_ROOT" +} +trap "trap_handler" EXIT + + +function log() { + [ -t 0 ] && echo -e '\033[1;35m#' "$@" '\033[0m' 1>&2 || echo '#' "$@" 1>&2 +} + + +function log_exec() { + local rc + set -x + "$@" + { rc=$?; set +x; } 2>/dev/null + return $rc +} + + +function log_function() { + local func_name="$1"; shift + log ">>>> ${func_name}()" + $func_name "$@" + log "<<<< ${func_name}()" +} + + +function get_repo_root() { + ( + cd "$(dirname "$0")/.." + pwd + ) +} + + +function get_stable_version() { + local part="$1" + ( + cd_fork work + local restore_branch="$(git branch --show-current)" + git checkout -q --no-track -b __stable_version_check "upstream/${STABLE_BRANCH}" + grep "^VERSION_${part}" lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/" + git checkout -q "$restore_branch" + git branch -q -D __stable_version_check + ) +} + + +function cd_fork() { + log_exec cd "${FORK_ROOT}/${1}" +} + + +function make_forks() { + [ -n "$REPO_ROOT" ] || REPO_ROOT=$(get_repo_root) + + # Use a "work" clone to prevent modifications to current clone + git clone --no-checkout "${REPO_ROOT}" "${FORK_ROOT}/work" + ( + cd_fork work + log "Checking out ref '${DEV_BRANCH}' as 'dev'" + log_exec git checkout --no-track -b dev "$DEV_BRANCH" + ) + + # Create bare origin and upstream repos + for repo in origin upstream; do + log_exec git clone --bare "${FORK_ROOT}/work" "${FORK_ROOT}/${repo}" + ( + cd_fork "$repo" + log_exec git remote remove origin + ) + done + + # Set remotes on work repo + ( + cd_fork work + git remote remove origin + for repo in origin upstream; do + log_exec git remote add "$repo" "file://${FORK_ROOT}/${repo}" + done + ) + + # Fetch release branches to upstream repo + ( + cd_fork upstream + log_exec git fetch --no-tags "${REPO_ROOT}" refs/remotes/upstream/${STABLE_BRANCH}:${STABLE_BRANCH} + ) + + # Set current (previous) stable release from stable branch + ( + cd_fork work + git fetch --no-tags upstream + ) + TEST_RELEASE_PREV=$(get_stable_version MAJOR) + TEST_RELEASE_PREV_MINOR=$(get_stable_version MINOR) +} + + +function create_venv() { + if [ ! -d "$VENV" ]; then + log_exec python3 -m venv "$VENV" + log_exec "${VENV}/bin/pip" install wheel packaging + fi + . "${VENV}/bin/activate" +} + + +function verify_version() { + local major="$1" + local minor="$2" + local ref="$3" + local _ref="$3" + local restore_branch="$(git branch --show-current)" + [ -n "$(git tag -l $ref)" ] || _ref="upstream/${ref}" + log_exec git checkout --no-track -b "__${ref}" "$_ref" + if grep -q "^VERSION_MAJOR = \"${major}\"$" lib/galaxy/version.py; then + log "**** Major version '${major}' is correct at ref '${ref}'" + else + log "**** Major version '${major}' is incorrect at ref '${ref}':" + log "$(grep '^VERSION_MAJOR' lib/galaxy/version.py)" + exit 1 + fi + if grep -q "^VERSION_MINOR = \"${minor}\"$" lib/galaxy/version.py; then + log "**** Minor version '${minor}' is correct at ref '${ref}'" + else + log "**** Minor version '${minor}' is incorrect at ref '${ref}':" + log "$(grep '^VERSION_MINOR' lib/galaxy/version.py)" + exit 1 + fi + log_exec git checkout "$restore_branch" + log_exec git branch -D "__${ref}" +} + + +function verify_makefile_version() { + local major="$1" + local ref="$2" + local _ref="$2" + local restore_branch="$(git branch --show-current)" + [ -n "$(git tag -l $ref)" ] || _ref="upstream/${ref}" + log_exec git checkout --no-track -b "__${ref}" "$_ref" + if grep -q "^RELEASE_CURR:=${major}$" Makefile; then + log "**** RELEASE_CURR '${major}' is correct in Makefile at ref '${ref}'" + else + log "**** RELEASE_CURR '${major}' is incorrect in Makefile at ref '${ref}':" + log "$(grep '^RELEASE_CURR' Makefile)" + exit 1 + fi + log_exec git checkout "$restore_branch" + log_exec git branch -D "__${ref}" +} + + +function test_rc() { + log "Test creation of ${TEST_RELEASE_CURR}.rc1..." + ( + cd_fork work + log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR" + # simulate merging RC PRs + log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_CURR}.rc1:refs/heads/release_${TEST_RELEASE_CURR} + log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_NEXT}.dev:refs/heads/dev + + # verify major minor ref (branch/tag) + verify_version "$TEST_RELEASE_CURR" 'rc1' "release_${TEST_RELEASE_CURR}" + verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev' + verify_version "$TEST_RELEASE_PREV" "$TEST_RELEASE_PREV_MINOR" "$STABLE_BRANCH" # should not have changed + verify_makefile_version "$TEST_RELEASE_NEXT" 'dev' + ) +} + + +function test_rc_point() { + log "Test creation of ${TEST_RELEASE_CURR}.rc2..." + ( + cd_fork work + log_exec "${REPO_ROOT}/scripts/release.sh" -c -r "$TEST_RELEASE_CURR" + # verify major minor ref (branch/tag) + verify_version "$TEST_RELEASE_CURR" 'rc2' "release_${TEST_RELEASE_CURR}" + verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev' + verify_version "$TEST_RELEASE_PREV" "$TEST_RELEASE_PREV_MINOR" "$STABLE_BRANCH" # should not have changed + verify_makefile_version "$TEST_RELEASE_NEXT" 'dev' + ) +} + + +function test_initial() { + log "Test creation of ${TEST_RELEASE_CURR} initial (.0)..." + ( + cd_fork work + log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR" + # re-fetch from upstream to ensure tags are correct + log_exec git tag -d $(git tag -l) + log_exec git fetch upstream + + # verify major minor ref (branch/tag) + verify_version "$TEST_RELEASE_CURR" '' "v${TEST_RELEASE_CURR}" + verify_version "$TEST_RELEASE_CURR" '1.dev0' "release_${TEST_RELEASE_CURR}" + verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev' + verify_version "$TEST_RELEASE_CURR" '' "$STABLE_BRANCH" + verify_makefile_version "$TEST_RELEASE_NEXT" 'dev' + ) +} + + +function test_point() { + log "Test creation of ${TEST_RELEASE_CURR}.1..." + ( + cd_fork work + log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR" + log_exec git tag -d $(git tag -l) + log_exec git fetch upstream + + # verify major minor ref (branch/tag) + verify_version "$TEST_RELEASE_CURR" '1' "$STABLE_BRANCH" + verify_version "$TEST_RELEASE_CURR" '1' "v${TEST_RELEASE_CURR}.1" + verify_version "$TEST_RELEASE_CURR" '2.dev0' "release_${TEST_RELEASE_CURR}" + verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev' + verify_makefile_version "$TEST_RELEASE_NEXT" 'dev' + ) +} + + +function test_next() { + log "Test creation of ${TEST_RELEASE_NEXT} rc1 -> initial (.0)..." + ( + cd_fork work + # create RC + log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_NEXT" + # simulate merging RC PRs + log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_NEXT}.rc1:refs/heads/release_${TEST_RELEASE_NEXT} + log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_NEXT_NEXT}.dev:refs/heads/dev + # create initial + log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_NEXT" + log_exec git tag -d $(git tag -l) + log_exec git fetch upstream + + # verify major minor ref (branch/tag) + verify_version "$TEST_RELEASE_NEXT" '' "$STABLE_BRANCH" + verify_version "$TEST_RELEASE_NEXT" '' "v${TEST_RELEASE_NEXT}" + verify_version "$TEST_RELEASE_NEXT" '1.dev0' "release_${TEST_RELEASE_NEXT}" + verify_version "$TEST_RELEASE_NEXT_NEXT" 'dev0' 'dev' + verify_makefile_version "$TEST_RELEASE_NEXT" "release_${TEST_RELEASE_NEXT}" + verify_makefile_version "$TEST_RELEASE_NEXT_NEXT" 'dev' + ) +} + + +function test_prev() { + log "Test creation of ${TEST_RELEASE_CURR}.2..." + ( + cd_fork work + log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR" + log_exec git tag -d $(git tag -l) + log_exec git fetch upstream + + # verify major minor ref (branch/tag) + verify_version "$TEST_RELEASE_NEXT" '' "$STABLE_BRANCH" + verify_version "$TEST_RELEASE_CURR" '2' "v${TEST_RELEASE_CURR}.2" + verify_version "$TEST_RELEASE_CURR" '3.dev0' "release_${TEST_RELEASE_CURR}" + verify_version "$TEST_RELEASE_NEXT" '1.dev0' "release_${TEST_RELEASE_NEXT}" + verify_version "$TEST_RELEASE_NEXT_NEXT" 'dev0' 'dev' + verify_makefile_version "$TEST_RELEASE_NEXT" "release_${TEST_RELEASE_NEXT}" + verify_makefile_version "$TEST_RELEASE_NEXT_NEXT" 'dev' + ) +} + + +function main() { + log "TIP: test output is magenta, release script output is green/red" + log_function make_forks + log_function create_venv + log_function test_rc + log_function test_rc_point + log_function test_initial + log_function test_point + log_function test_next + log_function test_prev + log "OK" +} + + +main diff --git a/tox.ini b/tox.ini index 0c67927158b..cf7989ab335 100644 --- a/tox.ini +++ b/tox.ini @@ -47,6 +47,9 @@ commands = bash .ci/check_mako.sh [testenv:test_galaxy_packages] commands = bash packages/test.sh +[testenv:test_galaxy_release] +commands = bash test/release.sh + [testenv:validate_test_tools] commands = bash .ci/validate_test_tools.sh