opensource/galaxy
3
0
Fork 0
mirror of https://github.com/galaxyproject/galaxy.git synced 2026-09-21 13:50:20 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #11641 from natefoo/release-process-overhaul

Browse Source 
Overhaul make-based release process
This commit is contained in:
Marius van den Beek
2021-04-06 16:45:12 +02:00
committed by GitHub
parent 3c1f887f4f cc1f471490
commit 32cb259274
7 changed files with 959 additions and 123 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.circleci/config.yml
+9
View File
@@ -93,6 +93,15 @@ jobs:
- run: sudo apt-get install -y libxml2-utils
- *install_tox
- run: tox -e validate_test_tools
test_galaxy_release:
docker:
- image: circleci/python:3.6
<<: *set_workdir
steps:
- *restore_repo_cache
- *install_tox
- *install_ffprobe
- run: tox -e test_galaxy_release
test_galaxy_packages:
docker:
- image: circleci/python:3.6
Makefile
+6 -96
View File
@@ -2,14 +2,8 @@
VENV?=.venv
# Source virtualenv to execute command (flake8, sphinx, twine, etc...)
IN_VENV=if [ -f "$(VENV)/bin/activate" ]; then . "$(VENV)/bin/activate"; fi;
RELEASE_CURR:=16.01
RELEASE_CURR_MINOR_NEXT:=$(shell $(IN_VENV) python scripts/bootstrap_history.py --print-next-minor-version)
RELEASE_NEXT:=16.04
# TODO: This needs to be updated with create_release_rc
#RELEASE_NEXT_BRANCH:=release_$(RELEASE_NEXT)
RELEASE_NEXT_BRANCH:=dev
RELEASE_CURR:=21.05
RELEASE_UPSTREAM:=upstream
MY_UPSTREAM:=origin
CONFIG_MANAGE=$(IN_VENV) python lib/galaxy/config/config_manage.py
PROJECT_URL?=https://github.com/galaxyproject/galaxy
DOCS_DIR=doc
@@ -180,97 +174,13 @@ serve-selenium-notebooks: ## Serve testing notebooks for Jupyter
cd lib && export PYTHONPATH=`pwd`; jupyter notebook --notebook-dir=galaxy_test/selenium/jupyter
# Release Targets
release-create-rc: release-ensure-upstream ## Create a release-candidate branch
git checkout dev
git pull --ff-only $(RELEASE_UPSTREAM) dev
git push $(MY_UPSTREAM) dev
git checkout -b release_$(RELEASE_CURR)
git push $(MY_UPSTREAM) release_$(RELEASE_CURR)
git push $(RELEASE_UPSTREAM) release_$(RELEASE_CURR)
git checkout -b version-$(RELEASE_CURR)
sed -i.bak -e "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_CURR)\"/" lib/galaxy/version.py
sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = \"rc1\"/" lib/galaxy/version.py
rm -f lib/galaxy/version.py.bak
git add lib/galaxy/version.py
git commit -m "Update version to $(RELEASE_CURR).rc1"
git checkout dev
release-create-rc: ## Create a release-candidate branch or new release-candidate version
$(IN_VENV) ./scripts/release.sh -c
git checkout -b version-$(RELEASE_NEXT).dev
sed -i.bak -e "s/^VERSION_MAJOR = .*/VERSION_MAJOR = \"$(RELEASE_NEXT)\"/" lib/galaxy/version.py
rm -f lib/galaxy/version.py.bak
git add lib/galaxy/version.py
git commit -m "Update version to $(RELEASE_NEXT).dev"
release-create: ## Create a release branch
$(IN_VENV) ./scripts/release.sh
-git merge version-$(RELEASE_CURR)
git checkout --ours lib/galaxy/version.py
git add lib/galaxy/version.py
git commit -m "Merge branch 'version-$(RELEASE_CURR)' into version-$(RELEASE_NEXT).dev"
git push $(MY_UPSTREAM) version-$(RELEASE_CURR):version-$(RELEASE_CURR)
git push $(MY_UPSTREAM) version-$(RELEASE_NEXT).dev:version-$(RELEASE_NEXT).dev
git checkout dev
# TODO: Use hub to automate these PR creations or push directly.
@echo "Open a PR from version-$(RELEASE_CURR) of your fork to release_$(RELEASE_CURR)"
@echo "Open a PR from version-$(RELEASE_NEXT).dev of your fork to dev"
release-create: release-ensure-upstream ## Create a release branch
git checkout master
git pull --ff-only $(RELEASE_UPSTREAM) master
git push $(MY_UPSTREAM) master
git checkout release_$(RELEASE_CURR)
git pull --ff-only $(RELEASE_UPSTREAM) release_$(RELEASE_CURR)
#git push $(MY_UPSTREAM) release_$(RELEASE_CURR)
git checkout dev
git pull --ff-only $(RELEASE_UPSTREAM) dev
#git push $(MY_UPSTREAM) dev
# Test run of merging. If there are conflicts, it will fail here.
git merge release_$(RELEASE_CURR)
git checkout release_$(RELEASE_CURR)
sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = None/" lib/galaxy/version.py
rm -f lib/galaxy/version.py.bak
git add lib/galaxy/version.py
git commit -m "Update version to $(RELEASE_CURR)"
git tag -m "Tag version $(RELEASE_CURR)" v$(RELEASE_CURR)
git checkout dev
-git merge release_$(RELEASE_CURR)
git checkout --ours lib/galaxy/version.py
git add lib/galaxy/version.py
git commit -m "Merge branch 'release_$(RELEASE_CURR)' into dev"
git checkout master
git merge release_$(RELEASE_CURR)
git push $(RELEASE_UPSTREAM) release_$(RELEASE_CURR):release_$(RELEASE_CURR)
git push $(RELEASE_UPSTREAM) dev:dev
git push $(RELEASE_UPSTREAM) master:master
git push $(RELEASE_UPSTREAM) --tags
release-create-point: ## Create a point release
git pull --ff-only $(RELEASE_UPSTREAM) master
git push $(MY_UPSTREAM) master
git checkout release_$(RELEASE_CURR)
git pull --ff-only $(RELEASE_UPSTREAM) release_$(RELEASE_CURR)
#git push $(MY_UPSTREAM) release_$(RELEASE_CURR)
git checkout $(RELEASE_NEXT_BRANCH)
git pull --ff-only $(RELEASE_UPSTREAM) $(RELEASE_NEXT_BRANCH)
#git push $(MY_UPSTREAM) $(RELEASE_NEXT_BRANCH)
git merge release_$(RELEASE_CURR)
git checkout release_$(RELEASE_CURR)
sed -i.bak -e "s/^VERSION_MINOR = .*/VERSION_MINOR = \"$(RELEASE_CURR_MINOR_NEXT)\"/" lib/galaxy/version.py
rm -f lib/galaxy/version.py.bak
git add lib/galaxy/version.py
git commit -m "Update version to $(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)"
git tag -m "Tag version $(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)" v$(RELEASE_CURR).$(RELEASE_CURR_MINOR_NEXT)
git checkout $(RELEASE_NEXT_BRANCH)
-git merge release_$(RELEASE_CURR)
git checkout --ours lib/galaxy/version.py
git add lib/galaxy/version.py
git commit -m "Merge branch 'release_$(RELEASE_CURR)' into $(RELEASE_NEXT_BRANCH)"
git checkout master
git merge release_$(RELEASE_CURR)
#git push origin release_$(RELEASE_CURR):release_$(RELEASE_CURR)
#git push origin $(RELEASE_NEXT_BRANCH):release_$(RELEASE_NEXT_BRANCH)
#git push origin master:master
#git push origin --tags
git checkout release_$(RELEASE_CURR)
release-create-point: release-create ## Create a point release
.PHONY: help
packages/package.Makefile
+1 -1
View File
@@ -9,7 +9,7 @@ SOURCE_DIR?=galaxy
BUILD_SCRIPTS_DIR=scripts
DEV_RELEASE?=0
VERSION?=$(shell DEV_RELEASE=$(DEV_RELEASE) python $(BUILD_SCRIPTS_DIR)/print_version_for_release.py $(SOURCE_DIR) $(DEV_RELEASE))
PROJECT_NAME?="galaxy-$(shell basename $(CURDIR))"
PROJECT_NAME?=galaxy-$(shell basename $(CURDIR))
PROJECT_NAME:=$(subst _,-,$(PROJECT_NAME))
BRANCH?=$(shell git rev-parse --abbrev-ref HEAD)
TEST_DIR?=tests
scripts/bootstrap_history.py
+9 -26
View File
@@ -200,14 +200,15 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
- [X] **Prep**
- [X] ~~Create this release issue ``make release-issue RELEASE_CURR=${version}``.~~
- [X] ~~Create this release issue ``make release-issue``.~~
- [X] ~~Set freeze date (${freeze_date}).~~
- [ ] **Branch Release (on or around ${freeze_date})**
- [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged, delayed, or closed.
make release-check-blocking-prs RELEASE_CURR=${version}
make release-check-blocking-prs
- [ ] Merge the latest release into dev and push upstream.
make release-merge-stable-to-next RELEASE_PREVIOUS=release_${previous_version}
@@ -215,12 +216,13 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
- [ ] Create and push release branch:
make release-create-rc RELEASE_CURR=${version} RELEASE_NEXT=${next_version}
make release-create-rc
- [ ] Open PRs from your fork of branch ``version-${version}`` to upstream ``release_${version}`` and of ``version-${next_version}.dev`` to ``dev``.
- [ ] Update ``MILESTONE_NUMBER`` in the [ maintenance bot](https://github.com/galaxyproject/galaxy/blob/dev/.github/workflows/maintenance_bot.yaml) to `${next_version}` so it properly tags new PRs.
- [ ] Update ``MILESTONE_NUMBER`` in the [maintenance bot](https://github.com/galaxyproject/galaxy/blob/dev/.github/workflows/maintenance_bot.yaml) to `${next_version}` so it properly tags new PRs.
- [ ] **Issue Review Timeline Notes**
- [ ] Ensure any security fixes will be ready prior to ${freeze_date} + 1 week, to allow time for notification prior to release.
- [ ] Ensure ownership of outstanding bugfixes and track progress during freeze.
@@ -265,11 +267,11 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
- [ ] Ensure all PRs merged into the pre-release branch during the freeze have [milestones attached](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_{version}+is%3Amerged+no%3Amilestone) and that they are the not [${next_version} milestones](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Aclosed+base%3Arelease_{version}+is%3Amerged+milestone%3A{next_version})
- [ ] Ensure release notes include all PRs added during the freeze by re-running the release note bootstrapping:
make release-bootstrap-history RELEASE_CURR=${version}
make release-bootstrap-history
- [ ] Ensure previous release is merged into current. [GitHub branch comparison](https://github.com/galaxyproject/galaxy/compare/release_${version}...release_${previous_version})
- [ ] Create and push release tag:
make release-create RELEASE_CURR=${version}
make release-create
- [ ] Add the branch `*/release_{version}` to Jenkins documentation build [configuration matrix](https://jenkins.galaxyproject.org/job/galaxy-sphinx-by-branch/configure).
- [ ] Trigger the [branch documentation build](https://jenkins.galaxyproject.org/job/galaxy-sphinx-by-branch/)
@@ -294,7 +296,7 @@ RELEASE_ISSUE_TEMPLATE = string.Template("""
- [ ] **Prepare for next release**
- [ ] Close milestone ``${version}`` and ensure milestone ``${next_version}`` exists.
- [ ] Create release issue for next version ``make release-issue RELEASE_CURR=${next_version}``.
- [ ] Create release issue for next version ``make release-issue``.
- [ ] Schedule committer meeting to discuss re-alignment of priorities.
- [ ] Close this issue.
""")
@@ -314,21 +316,6 @@ GROUPPED_TAGS = OrderedDict([
# https://api.github.com/repos/galaxyproject/galaxy/compare/release_15.05...dev
def print_next_minor_version():
minor_version_str = None
with open(GALAXY_VERSION_FILE) as f:
for line in f:
result = re.match(r'VERSION_MINOR = "(.*)"', line)
if result:
minor_version_str = result.group(1)
break
try:
minor_version = int(minor_version_str)
except (TypeError, ValueError):
minor_version = 0
print(minor_version + 1)
def release_issue(argv):
release_name = argv[2]
previous_release = _previous_release(release_name)
@@ -511,10 +498,6 @@ def main(argv, seen_prs=None):
newest_release = None
seen_prs = seen_prs or set()
if argv[1] == "--print-next-minor-version":
print_next_minor_version()
return
if argv[1] == "--check-blocking-prs":
check_blocking_prs(argv)
return
scripts/release.sh
Executable
+623
View File
@@ -0,0 +1,623 @@
#!/usr/bin/env bash
set -euo pipefail
shopt -s extglob
# TODO: upload packages
# TODO: set dev version in 21.01 (+ 20.09?) branches
: ${VENV:=.venv}
: ${FORK_REMOTE:=origin}
: ${UPSTREAM_REMOTE:=upstream}
: ${UPSTREAM_REMOTE_URL:=git@github.com:galaxyproject/galaxy.git}
: ${DEV_BRANCH:=dev}
: ${STABLE_BRANCH:=master}
# Vars for local releases
: ${RELEASE_LOCAL_TAG:=local}
: ${RELEASE_LOCAL_COMMIT:=$(git rev-parse --short HEAD)}
: ${RELEASE_LOCAL_VERSION:=${RELEASE_LOCAL_TAG}$(date -u +%Y%m%dT%H%M%SZ).${RELEASE_LOCAL_COMMIT}}
# Only use this for dev/testing/CI to ignore forward merge conflicts, skip confirmation and package builds, etc.
: ${TEST_MODE:=false}
VERIFY_PACKAGES=(wheel packaging)
BRANCH_CURR=$(git branch --show-current)
: ${RELEASE_CURR:=$(grep '^VERSION_MAJOR' lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/")}
RELEASE_NEXT=
RELEASE_CURR_MINOR=
RELEASE_CURR_MINOR_NEXT=
RELEASE_CURR_MINOR_NEXT_DEV=
RELEASE_TYPE=
PACKAGE_VERSION=
declare -a CLEAN_BRANCHES
declare -a CLEAN_TAGS
CLEAN_BRANCHES=()
CLEAN_TAGS=()
declare -a PUSH_BRANCHES
PUSH_BRANCHES=()
WORKING_DIR_CLEAN=false
ERROR=false
NEXT_RELEASE_TEMPLATE="import datetime; print((datetime.datetime.strptime('RELEASE_CURR', '%y.%m').date() + datetime.timedelta(days=(31 * 4))).strftime('%y.%m'))"
PACKAGE_VERSION_TEMPLATE="import packaging.version; print('.'.join(map(str, packaging.version.parse('VERSION').release)))"
PACKAGE_DEV_VERSION_TEMPLATE="import packaging.version; print(packaging.version.parse('VERSION'))"
while getopts ":clr:" opt; do
case "$opt" in
c)
RELEASE_TYPE='rc'
;;
r)
RELEASE_CURR="$OPTARG"
;;
l)
RELEASE_TYPE='local'
;;
*)
echo "usage: $(basename "$0") [-c (force rc)] [-l (local release)] [-r release version]"
exit 1
;;
esac
done
function trap_handler() {
{ set +x; } 2>/dev/null
local file
$ERROR && log_func=log_error || log_func=log
$log_func "Cleaning up..."
if $WORKING_DIR_CLEAN; then
log_exec git reset -- .
log_exec git checkout -- .
fi
[ "$(git branch --show-current)" == "$BRANCH_CURR" ] || log_exec git checkout "$BRANCH_CURR"
[ "${#CLEAN_BRANCHES[@]}" -eq 0 ] || git branch -D "${CLEAN_BRANCHES[@]}"
$ERROR && exit 1 || exit 0
}
function trap_handler_err() {
ERROR=true
for tag in "${CLEAN_TAGS[@]}"; do
log_exec git tag -d "$tag"
done
trap_handler
}
function trap_handler_ok() {
# if $ERROR is true then the trap handler already ran
$ERROR || trap_handler
}
trap "trap_handler_err" SIGTERM SIGINT ERR
trap "trap_handler_ok" EXIT
function log() {
[ -t 0 ] && echo -e '\033[1;32m#' "$@" '\033[0m' 1>&2 || echo '#' "$@" 1>&2
}
function log_warning() {
[ -t 0 ] && echo -e '\033[1;33mWARNING:' "$@" '\033[0m' 1>&2 || echo 'WARNING:' "$@" 1>&2
}
function log_error() {
[ -t 0 ] && echo -e '\033[1;31mERROR:' "$@" '\033[0m' 1>&2 || echo 'ERROR:' "$@" 1>&2
}
function log_debug() {
echo "####" "$@" 1>&2
}
function log_exec() {
local rc
set -x
"$@"
{ rc=$?; set +x; } 2>/dev/null
return $rc
}
function fork_owner() {
local url=$(git remote get-url "$FORK_REMOTE")
case "$url" in
https://github.com/*)
echo "$url" | awk -F/ '{print $4}'
;;
git@github.com:*)
echo "$url" | awk -F: '{print $2}' | awk -F/ '{print $1}'
;;
file:///*)
echo '__DEV_TEST_USER__'
;;
*)
log_error "Cannot parse remote '$FORK_REMOTE' url for owner username: $url"
exit 1
;;
esac
}
function ensure_upstream() {
if ! git remote -v | grep -E "^${UPSTREAM_REMOTE}\s+" >/dev/null; then
log_warning "Remote ${UPSTREAM_REMOTE} does not exist, will be added with URL: ${UPSTREAM_REMOTE_URL}"
log_exec git remote add "$UPSTREAM_REMOTE" "$UPSTREAM_REMOTE_URL"
fi
}
function ensure_prereqs() {
local pip_list
log "Checking for uncommitted local modifications..."
log_exec git diff --stat --exit-code || { log_error "Some files have changes"; exit 1; }
WORKING_DIR_CLEAN=true
log "Checking for required packages..."
if [ -z "${VIRTUAL_ENV:-}" ]; then
[ -d "$VENV" ] || { log_error "Missing venv, please create: ${VENV}"; exit 1; }
. "${VENV}/bin/activate"
fi
pip_list=$(log_exec "${VENV}/bin/pip" list)
for package in ${VERIFY_PACKAGES[@]}; do
echo "$pip_list" | grep -E "^${package}\s+" || { log_error "Package '${package}' missing from venv: ${VENV}" ; exit 1; }
done
}
function user_verify_release() {
log "Release Details"
local future
case "${RELEASE_NEXT:-}" in
'')
future="${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT_DEV}"
;;
*)
future="${RELEASE_NEXT}.dev0"
;;
esac
cat <<EOF
Release Type: ${RELEASE_TYPE}
Major Version: ${RELEASE_CURR}
Minor Version: ${RELEASE_CURR_MINOR_NEXT}
Package Version: ${PACKAGE_VERSION}
Future Version: ${future}
EOF
if $TEST_MODE; then
log "Confirmation not requested, proceeding..."
else
log "Press any key to confirm or ^C to exit"
read -n 1 -s
fi
}
function release_next() {
local curr="$1"
python3 -c "${NEXT_RELEASE_TEMPLATE/RELEASE_CURR/$curr}"
}
function packaging_version() {
local version="$1"
local dev_release="$2"
if $dev_release; then
python3 -c "${PACKAGE_DEV_VERSION_TEMPLATE/VERSION/$version}"
else
python3 -c "${PACKAGE_VERSION_TEMPLATE/VERSION/$version}"
fi
}
function git_checkout_temp() {
local name="$1"
local ref="$2"
log_exec git checkout --no-track -b "$name" "$ref"
CLEAN_BRANCHES+=("$name")
}
function branch_exists() {
local branch="$1"
local ref_base
case "$branch" in
*/*)
ref_base='refs/remotes'
;;
*)
ref_base='refs/heads'
;;
esac
git for-each-ref --format='%(refname:short)' "${ref_base}/${branch}" | grep -q "^${branch}\$"
}
function _test_forward_merge() {
local curr="$1"
local next="$(release_next "$curr")"
local curr_branch="${UPSTREAM_REMOTE}/release_${curr}"
local next_branch="${UPSTREAM_REMOTE}/release_${next}"
local curr_local_branch="__release_merge_test_${curr}"
local next_local_branch="__release_merge_test_${next}"
local recurse=true
local strategy=
log "Testing forward merge of ${curr} to ${next}"
if ! branch_exists "$curr_local_branch"; then
branch_exists "$curr_branch" || { log_error "No existing branch for merge test: ${curr_branch}"; exit 1; }
git_checkout_temp "$curr_local_branch" "$curr_branch"
fi
if ! branch_exists "$next_branch"; then
next_branch="${UPSTREAM_REMOTE}/${DEV_BRANCH}"
recurse=false
fi
git_checkout_temp "$next_local_branch" "$next_branch"
# Test the merge even if ignoring just to test the code path
$TEST_MODE && strategy='-X ours'
log_exec git merge $strategy -m 'test merge; please ignore' "$curr_local_branch" || {
log_error "Merging unmodified ${curr} to ${next} failed, resolve upstream first!"; exit 1; }
if $recurse; then
_test_forward_merge "$next"
fi
}
function test_forward_merge() {
local branch_curr=$(git branch --show-current)
_test_forward_merge "$@"
git checkout "$branch_curr"
}
function perform_stable_merge() {
[ "$RELEASE_TYPE" == 'initial' -o "$RELEASE_TYPE" == 'point' ] || return 0
local branch_curr=$(git branch --show-current)
git_checkout_temp '__stable' "${UPSTREAM_REMOTE}/${STABLE_BRANCH}"
local stable=$(get_version_major)
local curr_int=$(echo "$RELEASE_CURR" | tr -d .)
local stable_int=$(echo "$stable" | tr -d .)
if [ "$curr_int" -ge "$stable_int" ]; then
log "Release '${RELEASE_CURR}' >= stable branch release '${stable}', merging 'release_${RELEASE_CURR}' to '${STABLE_BRANCH}'"
log_exec git merge -m "Merge branch 'release_${RELEASE_CURR}' into '${STABLE_BRANCH}'" "__release_${RELEASE_CURR}"
PUSH_BRANCHES+=("__stable:${STABLE_BRANCH}")
else
log "Release '${RELEASE_CURR}' < stable branch release '${stable}', skipping merge to to '${STABLE_BRANCH}'"
fi
git checkout "$branch_curr"
}
function _perform_forward_merge() {
local curr="$1"
local next="$(release_next "$curr")"
local curr_branch="${UPSTREAM_REMOTE}/release_${curr}"
local next_branch="${UPSTREAM_REMOTE}/release_${next}"
local curr_local_branch="__release_${curr}"
local next_local_branch="__release_${next}"
local recurse=true
log "Performing forward merge of ${curr} to ${next}"
branch_exists "$curr_local_branch" || { log_error "Missing expected branch: ${curr_local_branch}"; exit 1; }
branch_exists "$next_local_branch" && { log_error "Unexpected branch exists: ${next_local_branch}"; exit 1; }
PUSH_BRANCHES+=("${curr_local_branch}:release_${curr}")
if ! branch_exists "$next_branch"; then
next_branch="${UPSTREAM_REMOTE}/${DEV_BRANCH}"
recurse=false
PUSH_BRANCHES+=("${next_local_branch}:${DEV_BRANCH}")
fi
git_checkout_temp "$next_local_branch" "$next_branch"
# This should necessarily result in conflicts merging version.py
log_exec git merge -X ours -m "Merge branch 'release_${curr}' into 'release_${next}'" "$curr_local_branch"
if $recurse; then
_perform_forward_merge "$next"
fi
}
function perform_forward_merge() {
local curr="$1"
local branch_curr=$(git branch --show-current)
_perform_forward_merge "$@"
declare -p PUSH_BRANCHES
git checkout "$branch_curr"
}
function push_merged() {
local curr="$1"
for branch in "${PUSH_BRANCHES[@]}"; do
log "Pushing '${branch}' to remote '${UPSTREAM_REMOTE}'"
log_exec git push "$UPSTREAM_REMOTE" "$branch"
done
if [ "$RELEASE_TYPE" == 'initial' -o "$RELEASE_TYPE" == 'point' ]; then
log_exec git push --tags "$UPSTREAM_REMOTE"
fi
}
function set_package_version_var() {
local package_version_minor
local dev_release='false'
case "$RELEASE_CURR_MINOR_NEXT" in
[0-9]*)
package_version_minor="$RELEASE_CURR_MINOR_NEXT"
;;
dev)
# TODO: you can remove this case once the current dev branch minor version is updated to dev0
package_version_minor="0dev0"
dev_release='true'
;;
*)
package_version_minor="0${RELEASE_CURR_MINOR_NEXT}"
dev_release='true'
;;
esac
PACKAGE_VERSION=$(packaging_version "${RELEASE_CURR}.${package_version_minor}" "$dev_release")
}
function increment_minor() {
local minor="$1"
case "$minor" in
+([0-9]))
[ "$RELEASE_TYPE" != 'rc' ] || {
log_error "Cannot create rc after release (current version: ${RELEASE_CURR}.${minor})";
exit 1; }
echo "$((minor + 1)).dev0"
;;
+([0-9]).dev*)
[ "$RELEASE_TYPE" != 'rc' ] || {
log_error "Cannot create rc after release (current version: ${RELEASE_CURR}.${minor})";
exit 1; }
echo "${minor%.*}"
;;
rc*)
if [ "$RELEASE_TYPE" == 'rc' ]; then
echo "rc$((${minor#rc*} + 1))"
else
echo ''
fi
;;
None|'')
[ "$RELEASE_TYPE" != 'rc' ] || {
log_error "Cannot create rc after release (current version: ${RELEASE_CURR}.0)";
exit 1; }
echo '1.dev0'
;;
*)
log_error "Don't know how to increment minor version: ${minor}"
exit 1
;;
esac
}
function get_version_major() {
grep '^VERSION_MAJOR' lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/"
}
function get_version_minor() {
grep '^VERSION_MINOR' lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/" | tr -d '[[:space:]]'
}
function set_version_vars() {
RELEASE_CURR="$(get_version_major)"
RELEASE_CURR_MINOR="$(get_version_minor)"
[[ "$RELEASE_CURR_MINOR" =~ .*"None"$ ]] && RELEASE_CURR_MINOR='0'
: ${RELEASE_TYPE:=point}
RELEASE_CURR_MINOR_NEXT="$(increment_minor "$RELEASE_CURR_MINOR")"
[ -n "$RELEASE_CURR_MINOR_NEXT" ] || RELEASE_TYPE='initial'
RELEASE_CURR_MINOR_NEXT_DEV="$(increment_minor "${RELEASE_CURR_MINOR_NEXT}")"
set_package_version_var
}
function update_galaxy_version() {
local key="$1"
local val="$2"
local match="${3:-.*}"
log "Updating lib/galaxy/version.py for ${key} = ${val}..."
log_exec sed -E -i -e "s/^${key} = ${match}/${key} = \"$val\"/" lib/galaxy/version.py
}
function packages_make_all() {
local dir
# subshell to preserve cwd
(
cd packages/
for dir in *; do
[ ! -d "$dir" -o ! -f "${dir}/setup.py" ] && continue
# can't use log_exec here because we want to capture output
echo + make -C "$dir" "$@" 1>&2
make -C "$dir" "$@" >"${dir}/make-${1}.log" 2>&1
done
)
}
function update_package_versions() {
local package_version="${1:-$PACKAGE_VERSION}"
local project_file
log "Updating package versions to '${package_version}'..."
(
cd packages/
for dir in *; do
project_file="${dir}/galaxy/project_galaxy_${dir}.py"
if [ -f "${project_file}" ]; then
sed -i -e "s/^__version__ =.*/__version__ = \"${package_version}\"/" "$project_file"
fi
done
)
#git diff -- packages/
}
function perform_version_update() {
local tag_version=
update_package_versions
git add -- packages/
log 'Cleaning package dirs...'
packages_make_all clean
log 'Building packages (logs in packages/*/make-dist.log)...'
if $TEST_MODE; then
log_debug "Test mode skipping package build"
else
packages_make_all dist
fi
log 'Committing version changes...'
log_exec git commit -m "Update version to ${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT}"
case "$RELEASE_TYPE" in
initial)
tag_version="${RELEASE_CURR}"
;;
point)
tag_version="${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT}"
;;
esac
if [ -n "$tag_version" ]; then
log_exec git tag -m "Tag version ${tag_version}" "v${tag_version}"
CLEAN_TAGS+=("v${tag_version}")
fi
}
function perform_version_update_dev() {
[ "$RELEASE_TYPE" == 'initial' -o "$RELEASE_TYPE" == 'point' ] || return 0
log "Incrementing release version to '${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT_DEV}' for development of next point release"
update_galaxy_version 'VERSION_MINOR' "$RELEASE_CURR_MINOR_NEXT_DEV"
log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true
git add -- lib/galaxy/version.py
update_package_versions
git add -- packages/
log_exec git commit -m "Update version to ${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT_DEV}"
}
function create_release_rc_initial() {
RELEASE_TYPE='rc-initial'
RELEASE_NEXT="$(release_next "$RELEASE_CURR")"
RELEASE_CURR_MINOR_NEXT='rc1'
set_package_version_var
user_verify_release
local _dev_branch="${UPSTREAM_REMOTE}/${DEV_BRANCH}"
git_checkout_temp "__release_${RELEASE_CURR}" "$_dev_branch"
update_galaxy_version 'VERSION_MAJOR' "$RELEASE_CURR"
update_galaxy_version 'VERSION_MINOR' "$RELEASE_CURR_MINOR_NEXT"
log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true
git add -- lib/galaxy/version.py
perform_version_update
# Increment version in dev branch
git_checkout_temp "__release_${RELEASE_NEXT}" "$_dev_branch"
update_galaxy_version 'VERSION_MAJOR' "$RELEASE_NEXT"
update_galaxy_version 'VERSION_MINOR' 'dev0'
log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true
git add -- lib/galaxy/version.py
log_exec sed -i -e "s/^RELEASE_CURR:=.*/RELEASE_CURR:=${RELEASE_NEXT}/" Makefile
log_exec git diff --exit-code && { log_error 'Missing expected Makefile changes'; exit 1; } || true
git add -- Makefile
local package_version=$(packaging_version "${RELEASE_NEXT}.0dev0" "true")
update_package_versions "$package_version"
git add -- packages/
log_exec git commit -m "Update version to ${RELEASE_NEXT}.dev0"
# Resolve merge conflicts
log_exec git merge -X ours -m "Merge branch 'release_${RELEASE_CURR}' into 'dev'" "__release_${RELEASE_CURR}"
# Push branches for PR
local owner=$(fork_owner)
local curr_remote_branch="version-${RELEASE_CURR}.${RELEASE_CURR_MINOR_NEXT}"
local next_remote_branch="version-${RELEASE_NEXT}.dev"
log_exec git push "$FORK_REMOTE" "__release_${RELEASE_CURR}:${curr_remote_branch}"
log_exec git push "$FORK_REMOTE" "__release_${RELEASE_NEXT}:${next_remote_branch}"
log_exec git push "$UPSTREAM_REMOTE" "refs/remotes/${UPSTREAM_REMOTE}/${DEV_BRANCH}:refs/heads/release_${RELEASE_CURR}"
#https://github.com/galaxyproject/galaxy/compare/{branch}...{fork_owner}:{branch}
log "Open a PR from ${owner}/galaxy:${curr_remote_branch} to galaxyproject/galaxy:release_${RELEASE_CURR}"
echo "- https://github.com/galaxyproject/galaxy/compare/release_${RELEASE_CURR}...${owner}:${curr_remote_branch}" 1>&2
log "Open a PR from ${owner}/galaxy:${next_remote_branch} to galaxyproject/galaxy:dev"
echo "- https://github.com/galaxyproject/galaxy/compare/dev...${owner}:${next_remote_branch}" 1>&2
}
function create_release_local() {
RELEASE_CURR_MINOR="$(get_version_minor)"
RELEASE_CURR_MINOR_NEXT="${RELEASE_CURR_MINOR}+${RELEASE_LOCAL_VERSION}"
set_package_version_var
user_verify_release
# Append the local portion if this is a point release or the local portion already exists
update_galaxy_version 'VERSION_MINOR' "\1+${RELEASE_LOCAL_VERSION}" "['\"]([^+]*)[^'\"\]*['\"]$"
# Set the minor version to 0 and append the local version if the release version is not set (e.g. .0)
update_galaxy_version 'VERSION_MINOR' "0+${RELEASE_LOCAL_VERSION}" 'None'
update_galaxy_version 'VERSION_MINOR' "0+${RELEASE_LOCAL_VERSION}" '""'
log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true
update_package_versions
##git add -- lib/galaxy/version.py packages/*/galaxy/project_*.py
packages_make_all clean
log 'Building packages (logs in packages/*/make-dist.log)...'
packages_make_all dist
##git commit -m "Update version to $(RELEASE_CURR)"
##git tag -m "Tag version $(RELEASE_CURR)" v$(RELEASE_CURR)
}
function create_release_normal() {
local _release_curr="$RELEASE_CURR"
local _release_branch="${UPSTREAM_REMOTE}/release_${RELEASE_CURR}"
git_checkout_temp "__release_${RELEASE_CURR}" "$_release_branch"
set_version_vars
[ "$_release_curr" == "$RELEASE_CURR" ] || {
log_error "Release is incorrect in branch ${_release_branch}: ${_release_curr} != ${RELEASE_CURR}";
exit 1; }
user_verify_release
test_forward_merge "$RELEASE_CURR"
update_galaxy_version 'VERSION_MINOR' "$RELEASE_CURR_MINOR_NEXT"
log_exec git diff --exit-code && { log_error 'Missing expected version.py changes'; exit 1; } || true
git add -- lib/galaxy/version.py
perform_version_update
perform_stable_merge
perform_version_update_dev
perform_forward_merge "$RELEASE_CURR"
push_merged "$RELEASE_CURR"
}
function create_release() {
log 'Fetching upstream changes...'
git fetch "$UPSTREAM_REMOTE"
if [ "$RELEASE_TYPE" == 'local' ]; then
create_release_local
elif ! branch_exists "${UPSTREAM_REMOTE}/release_${RELEASE_CURR}"; then
create_release_rc_initial
else
create_release_normal
fi
}
function main() {
ensure_upstream
ensure_prereqs
create_release
}
main
test/release.sh
Executable
+308
View File
@@ -0,0 +1,308 @@
#!/usr/bin/env bash
set -euo pipefail
# Necessary for testing the release script
export TEST_MODE=true
: ${ORIGIN:=origin}
: ${STABLE_BRANCH:=master}
REPO_ROOT=
FORK_ROOT=$(mktemp -d -t galaxy_release_test_XXXXXXXX)
DEV_BRANCH="$(git branch --show-current)"
TEST_RELEASE_PREV=
TEST_RELEASE_PREV_MINOR=
TEST_RELEASE_CURR='99.01'
TEST_RELEASE_NEXT='99.05'
TEST_RELEASE_NEXT_NEXT='99.09'
: ${VENV:=${FORK_ROOT}/venv}
export VENV
[ -n "$DEV_BRANCH" ] || { echo 'ERROR: Cannot determine current branch'; exit 1; }
function trap_handler() {
[ -z "$FORK_ROOT" ] || rm -rf "$FORK_ROOT"
}
trap "trap_handler" EXIT
function log() {
[ -t 0 ] && echo -e '\033[1;35m#' "$@" '\033[0m' 1>&2 || echo '#' "$@" 1>&2
}
function log_exec() {
local rc
set -x
"$@"
{ rc=$?; set +x; } 2>/dev/null
return $rc
}
function log_function() {
local func_name="$1"; shift
log ">>>> ${func_name}()"
$func_name "$@"
log "<<<< ${func_name}()"
}
function get_repo_root() {
(
cd "$(dirname "$0")/.."
pwd
)
}
function get_stable_version() {
local part="$1"
(
cd_fork work
local restore_branch="$(git branch --show-current)"
git checkout -q --no-track -b __stable_version_check "upstream/${STABLE_BRANCH}"
grep "^VERSION_${part}" lib/galaxy/version.py | sed -E -e "s/^[^'\"]*['\"]([^'\"]*)['\"]$/\1/"
git checkout -q "$restore_branch"
git branch -q -D __stable_version_check
)
}
function cd_fork() {
log_exec cd "${FORK_ROOT}/${1}"
}
function make_forks() {
[ -n "$REPO_ROOT" ] || REPO_ROOT=$(get_repo_root)
# Use a "work" clone to prevent modifications to current clone
git clone --no-checkout "${REPO_ROOT}" "${FORK_ROOT}/work"
(
cd_fork work
log "Checking out ref '${DEV_BRANCH}' as 'dev'"
log_exec git checkout --no-track -b dev "$DEV_BRANCH"
)
# Create bare origin and upstream repos
for repo in origin upstream; do
log_exec git clone --bare "${FORK_ROOT}/work" "${FORK_ROOT}/${repo}"
(
cd_fork "$repo"
log_exec git remote remove origin
)
done
# Set remotes on work repo
(
cd_fork work
git remote remove origin
for repo in origin upstream; do
log_exec git remote add "$repo" "file://${FORK_ROOT}/${repo}"
done
)
# Fetch release branches to upstream repo
(
cd_fork upstream
log_exec git fetch --no-tags "${REPO_ROOT}" refs/remotes/upstream/${STABLE_BRANCH}:${STABLE_BRANCH}
)
# Set current (previous) stable release from stable branch
(
cd_fork work
git fetch --no-tags upstream
)
TEST_RELEASE_PREV=$(get_stable_version MAJOR)
TEST_RELEASE_PREV_MINOR=$(get_stable_version MINOR)
}
function create_venv() {
if [ ! -d "$VENV" ]; then
log_exec python3 -m venv "$VENV"
log_exec "${VENV}/bin/pip" install wheel packaging
fi
. "${VENV}/bin/activate"
}
function verify_version() {
local major="$1"
local minor="$2"
local ref="$3"
local _ref="$3"
local restore_branch="$(git branch --show-current)"
[ -n "$(git tag -l $ref)" ] || _ref="upstream/${ref}"
log_exec git checkout --no-track -b "__${ref}" "$_ref"
if grep -q "^VERSION_MAJOR = \"${major}\"$" lib/galaxy/version.py; then
log "**** Major version '${major}' is correct at ref '${ref}'"
else
log "**** Major version '${major}' is incorrect at ref '${ref}':"
log "$(grep '^VERSION_MAJOR' lib/galaxy/version.py)"
exit 1
fi
if grep -q "^VERSION_MINOR = \"${minor}\"$" lib/galaxy/version.py; then
log "**** Minor version '${minor}' is correct at ref '${ref}'"
else
log "**** Minor version '${minor}' is incorrect at ref '${ref}':"
log "$(grep '^VERSION_MINOR' lib/galaxy/version.py)"
exit 1
fi
log_exec git checkout "$restore_branch"
log_exec git branch -D "__${ref}"
}
function verify_makefile_version() {
local major="$1"
local ref="$2"
local _ref="$2"
local restore_branch="$(git branch --show-current)"
[ -n "$(git tag -l $ref)" ] || _ref="upstream/${ref}"
log_exec git checkout --no-track -b "__${ref}" "$_ref"
if grep -q "^RELEASE_CURR:=${major}$" Makefile; then
log "**** RELEASE_CURR '${major}' is correct in Makefile at ref '${ref}'"
else
log "**** RELEASE_CURR '${major}' is incorrect in Makefile at ref '${ref}':"
log "$(grep '^RELEASE_CURR' Makefile)"
exit 1
fi
log_exec git checkout "$restore_branch"
log_exec git branch -D "__${ref}"
}
function test_rc() {
log "Test creation of ${TEST_RELEASE_CURR}.rc1..."
(
cd_fork work
log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR"
# simulate merging RC PRs
log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_CURR}.rc1:refs/heads/release_${TEST_RELEASE_CURR}
log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_NEXT}.dev:refs/heads/dev
# verify major minor ref (branch/tag)
verify_version "$TEST_RELEASE_CURR" 'rc1' "release_${TEST_RELEASE_CURR}"
verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev'
verify_version "$TEST_RELEASE_PREV" "$TEST_RELEASE_PREV_MINOR" "$STABLE_BRANCH" # should not have changed
verify_makefile_version "$TEST_RELEASE_NEXT" 'dev'
)
}
function test_rc_point() {
log "Test creation of ${TEST_RELEASE_CURR}.rc2..."
(
cd_fork work
log_exec "${REPO_ROOT}/scripts/release.sh" -c -r "$TEST_RELEASE_CURR"
# verify major minor ref (branch/tag)
verify_version "$TEST_RELEASE_CURR" 'rc2' "release_${TEST_RELEASE_CURR}"
verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev'
verify_version "$TEST_RELEASE_PREV" "$TEST_RELEASE_PREV_MINOR" "$STABLE_BRANCH" # should not have changed
verify_makefile_version "$TEST_RELEASE_NEXT" 'dev'
)
}
function test_initial() {
log "Test creation of ${TEST_RELEASE_CURR} initial (.0)..."
(
cd_fork work
log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR"
# re-fetch from upstream to ensure tags are correct
log_exec git tag -d $(git tag -l)
log_exec git fetch upstream
# verify major minor ref (branch/tag)
verify_version "$TEST_RELEASE_CURR" '' "v${TEST_RELEASE_CURR}"
verify_version "$TEST_RELEASE_CURR" '1.dev0' "release_${TEST_RELEASE_CURR}"
verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev'
verify_version "$TEST_RELEASE_CURR" '' "$STABLE_BRANCH"
verify_makefile_version "$TEST_RELEASE_NEXT" 'dev'
)
}
function test_point() {
log "Test creation of ${TEST_RELEASE_CURR}.1..."
(
cd_fork work
log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR"
log_exec git tag -d $(git tag -l)
log_exec git fetch upstream
# verify major minor ref (branch/tag)
verify_version "$TEST_RELEASE_CURR" '1' "$STABLE_BRANCH"
verify_version "$TEST_RELEASE_CURR" '1' "v${TEST_RELEASE_CURR}.1"
verify_version "$TEST_RELEASE_CURR" '2.dev0' "release_${TEST_RELEASE_CURR}"
verify_version "$TEST_RELEASE_NEXT" 'dev0' 'dev'
verify_makefile_version "$TEST_RELEASE_NEXT" 'dev'
)
}
function test_next() {
log "Test creation of ${TEST_RELEASE_NEXT} rc1 -> initial (.0)..."
(
cd_fork work
# create RC
log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_NEXT"
# simulate merging RC PRs
log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_NEXT}.rc1:refs/heads/release_${TEST_RELEASE_NEXT}
log_exec git push upstream refs/remotes/origin/version-${TEST_RELEASE_NEXT_NEXT}.dev:refs/heads/dev
# create initial
log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_NEXT"
log_exec git tag -d $(git tag -l)
log_exec git fetch upstream
# verify major minor ref (branch/tag)
verify_version "$TEST_RELEASE_NEXT" '' "$STABLE_BRANCH"
verify_version "$TEST_RELEASE_NEXT" '' "v${TEST_RELEASE_NEXT}"
verify_version "$TEST_RELEASE_NEXT" '1.dev0' "release_${TEST_RELEASE_NEXT}"
verify_version "$TEST_RELEASE_NEXT_NEXT" 'dev0' 'dev'
verify_makefile_version "$TEST_RELEASE_NEXT" "release_${TEST_RELEASE_NEXT}"
verify_makefile_version "$TEST_RELEASE_NEXT_NEXT" 'dev'
)
}
function test_prev() {
log "Test creation of ${TEST_RELEASE_CURR}.2..."
(
cd_fork work
log_exec "${REPO_ROOT}/scripts/release.sh" -r "$TEST_RELEASE_CURR"
log_exec git tag -d $(git tag -l)
log_exec git fetch upstream
# verify major minor ref (branch/tag)
verify_version "$TEST_RELEASE_NEXT" '' "$STABLE_BRANCH"
verify_version "$TEST_RELEASE_CURR" '2' "v${TEST_RELEASE_CURR}.2"
verify_version "$TEST_RELEASE_CURR" '3.dev0' "release_${TEST_RELEASE_CURR}"
verify_version "$TEST_RELEASE_NEXT" '1.dev0' "release_${TEST_RELEASE_NEXT}"
verify_version "$TEST_RELEASE_NEXT_NEXT" 'dev0' 'dev'
verify_makefile_version "$TEST_RELEASE_NEXT" "release_${TEST_RELEASE_NEXT}"
verify_makefile_version "$TEST_RELEASE_NEXT_NEXT" 'dev'
)
}
function main() {
log "TIP: test output is magenta, release script output is green/red"
log_function make_forks
log_function create_venv
log_function test_rc
log_function test_rc_point
log_function test_initial
log_function test_point
log_function test_next
log_function test_prev
log "OK"
}
main
tox.ini
+3
View File
@@ -47,6 +47,9 @@ commands = bash .ci/check_mako.sh
[testenv:test_galaxy_packages]
commands = bash packages/test.sh
[testenv:test_galaxy_release]
commands = bash test/release.sh
[testenv:validate_test_tools]
commands = bash .ci/validate_test_tools.sh