diff --git a/Makefile b/Makefile index 309cd737748..d7770ffe451 100644 --- a/Makefile +++ b/Makefile @@ -5,11 +5,55 @@ RELEASE_NEXT:=16.04 #RELEASE_NEXT_BRANCH:=release_$(RELEASE_NEXT) RELEASE_NEXT_BRANCH:=dev RELEASE_UPSTREAM:=upstream +# Location of virtualenv used for development. +VENV?=.venv +# Source virtualenv to execute command (flake8, sphinx, twine, etc...) +IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi; +PROJECT_URL?=https://github.com/galaxyproject/galaxy GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01 all: help @echo "This makefile is primarily used for building Galaxy's JS client. A sensible all target is not yet implemented." +help: + @echo "client - rebuild all client artifacts" + @echo "docs - generate Sphinx HTML documentation, including API docs" + @echo "open-docs - generate Sphinx HTML documentation and open in browser" + @echo "open-project - open project on github" + @echo "lint - check style using tox and flake8 for Python 2 and Python 3" + @echo "release-check-metadata - check github PR metadata for target release" + @echo "release-bootstrap-history - bootstrap history for a new release" + +docs: + $(IN_VENV) $(MAKE) -C docs clean + $(IN_VENV) $(MAKE) -C docs html + +_open-docs: + open docs/_build/html/index.html || xdg-open docs/_build/html/index.html + +open-docs: docs _open-docs + +open-project: + open $(PROJECT_URL) || xdg-open $(PROJECT_URL) + +lint: + $(IN_VENV) tox -e py27-lint && tox -e py34-lint + +release-issue: + $(IN_VENV) python scripts/bootstrap_history.py --create-release-issue $(RELEASE_CURR) + +release-check-metadata: + $(IN_VENV) python scripts/bootstrap_history.py --check-release $(RELEASE_CURR) + +release-check-blocking-issues: + $(IN_VENV) python scripts/bootstrap_history.py --check-blocking-issues $(RELEASE_CURR) + +release-check-blocking-prs: + $(IN_VENV) python scripts/bootstrap_history.py --check-blocking-prs $(RELEASE_CURR) + +release-bootstrap-history: + $(IN_VENV) python scripts/bootstrap_history.py --release $(RELEASE_CURR) + npm-deps: cd client && npm install @@ -35,7 +79,7 @@ clean-grunt-docker-image: ## Remove grunt docker image # Release Targets -create_release_rc: ## Create a release-candidate branch +release-create-rc: ## Create a release-candidate branch git checkout dev git pull --ff-only ${RELEASE_UPSTREAM} dev git push origin dev diff --git a/doc/source/releases/index.rst b/doc/source/releases/index.rst index 42b7250dd40..ef87cedbb04 100644 --- a/doc/source/releases/index.rst +++ b/doc/source/releases/index.rst @@ -4,6 +4,8 @@ Releases .. toctree:: :maxdepth: 1 +.. annoucements + 16.04_announce 16.01_announce 15.10_announce 15.07_announce diff --git a/scripts/bootstrap_history.py b/scripts/bootstrap_history.py index 3f37d41192b..f8345292761 100644 --- a/scripts/bootstrap_history.py +++ b/scripts/bootstrap_history.py @@ -2,6 +2,7 @@ # Little script to make HISTORY.rst more easy to format properly, lots TODO # pull message down and embed, use arg parse, handle multiple, etc... +import ast import calendar import datetime import os @@ -22,17 +23,14 @@ except ImportError: PROJECT_DIRECTORY = os.path.join(os.path.dirname(__file__), os.pardir) +SOURCE_DIR = os.path.join(PROJECT_DIRECTORY, "lib") +GALAXY_SOURCE_DIR = os.path.join(SOURCE_DIR, "galaxy") +GALAXY_VERSION_FILE = os.path.join(GALAXY_SOURCE_DIR, "version.py") PROJECT_OWNER = "galaxyproject" PROJECT_NAME = "galaxy" PROJECT_URL = "https://github.com/%s/%s" % (PROJECT_OWNER, PROJECT_NAME) PROJECT_API = "https://api.github.com/repos/%s/%s/" % (PROJECT_OWNER, PROJECT_NAME) - -RELEASES = [ - ("15.05", "b16ac25cdc0f2b64d6af34ea1e6ff253d8a71ee4"), - ("15.07", "e44c8db9dea56b8d1a2f941ce572b0f14e999d4c"), - ("15.10", "ef55279d58eced4b90632a572a8fc5a227ceefb9"), - ("16.01", "cc23adb0a962f1c9780b838604718319a0a71888"), -] +RELEASES_PATH = os.path.join(PROJECT_DIRECTORY, "doc", "source", "releases") # Uncredit pull requestors... kind of arbitrary at this point. DEVTEAM = [ @@ -151,6 +149,79 @@ Schedule * Planned Release Date: ${release_date} """) + +RELEASE_ISSUE_TEMPLATE = string.Template(""" + +- [X] **Prep** + + - [X] ~~Create this release issue ``make release-issue RELEASE_CURR=${version}``.~~ + - [X] ~~Set freeze date (${freeze_date}).~~ + +- [ ] **Branch Release (on or around ${freeze_date})** + + - [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged, delayed, or closed. + + make release-check-blocking-prs RELEASE_CURR=${version} + - [ ] Merge the latest release into dev. + + git fetch upstream && git checkout dev && git merge --ff-only upstream dev && git merge upstream release_${previous_version} + - [ ] Create and push release branch: + + make release-create-rc RELEASE_CURR=${version} RELEASE_NEXT=${next_version} + +- [ ] **Deploy and Test Release** + + - [ ] Deploy to test (${freeze_date} + 1 day). + - [ ] Deploy to usegalaxy.org (${freeze_date} + 1 week). + +- [ ] **Create Release Notes** + + - [ ] Review merged PRs and ensure they all milestones attached. [Link](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Amerged+no%3Amilestone) + - [ ] Checkout release branch + + git checkout $version -b ${version}_release_notes + - [ ] Check for obvious missing metadata in release PRs + + make release-check-metadata RELEASE_CURR=${version} + - [ ] Bootstrap the release notes + + make release-bootstrap-history RELEASE_CURR=${version} + - [ ] Open newly created files and manually curate major topics and release notes. + - [ ] Commit release notes. + + git add docs/; git commit -m "Release notes for $version"; git push upstream ${version}_release_notes + - [ ] Open a pull request for new release note branch. + - [ ] Merge release note pull request. + +- [ ] **Do Release** + + - [ ] Ensure all [blocking milestone issues](https://github.com/galaxyproject/galaxy/issues?q=is%3Aopen+is%3Aissue+milestone%3A${version}) have been resolved. + + make release-check-blocking-issues RELEASE_CURR=${version} + - [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged or closed. + + make release-check-blocking-prs RELEASE_CURR=${version} + - [ ] Do release (TODO: create a check a list for this...) + +- [ ] **Announce Release** + + - [ ] Stage annoucement content (Wiki, Biostars, Bit.ly link) on annouce date to capture date tags. Note: all final content does not need to be completed to do this. + - [ ] Finalize https://github.com/galaxyproject/galaxy/blob/dev/doc/source/releases/${version}_announce.rst + - [ ] Post release notes to https://docs.galaxyproject.org/en/master/releases/index.html + - [ ] Create wiki *highlights* and post to http://galaxyproject.org News (w/ RSS) and NewsBriefs + - [ ] Tweet wiki news *highlights* (or RTD?) via bit.ly link to https://twitter.com/galaxyproject/ + - [ ] Post *highlights* type News to Galaxy Biostars https://biostar.usegalaxy.org + - [ ] Email *highlights* to galaxy-dev and galaxy-announce @lists.galaxyproject.org + - [ ] Adjust http://getgalaxy.org text and links to match current master branch + +- [ ] **Prepare for next release** + + - [ ] Ensure milestone ``${next_version}`` exists. + - [ ] Create release issue for next version ``make release-issue RELEASE_CURR=${next_version}``. + - [ ] Close this issue. + +""") + # https://api.github.com/repos/galaxyproject/galaxy/pulls?base=dev&state=closed # https://api.github.com/repos/galaxyproject/galaxy/pulls?base=release_15.07&state=closed # https://api.github.com/repos/galaxyproject/galaxy/compare/release_15.05...dev @@ -162,6 +233,31 @@ def commit_time(commit_hash): return datetime.datetime.strptime(req["commit"]["committer"]["date"], "%Y-%m-%dT%H:%M:%SZ") +def release_issue(argv): + release_name = argv[2] + previous_release = _previous_release(release_name) + new_version_params = _next_version_params(release_name) + next_version = new_version_params["version"] + freeze_date, release_date = _release_dates(release_name) + release_issue_template_params = dict( + version=release_name, + next_version=next_version, + previous_version=previous_release, + freeze_date=freeze_date, + ) + release_issue_contents = RELEASE_ISSUE_TEMPLATE.safe_substitute(**release_issue_template_params) + github = _github_client() + github.issues.create( + data=dict( + title="Publication of Galaxy Release v %s" % release_name, + body=release_issue_contents, + ), + user=PROJECT_OWNER, + repo=PROJECT_NAME, + ) + return release_issue + + def do_release(argv): release_name = argv[2] release_file = _release_file(release_name + ".rst") @@ -179,6 +275,72 @@ def do_release(argv): announce_file = _release_file(release_name + "_announce.rst") open(announce_file, "w").write(announce_info.encode("utf-8")) + next_version_params = _next_version_params(release_name) + next_version = next_version_params["version"] + next_release_file = _release_file(next_version + "_announce.rst") + + next_announce = NEXT_TEMPLATE.substitute(**next_version_params) + open(next_release_file, "w").write(next_announce.encode("utf-8")) + releases_index = _release_file("index.rst") + releases_index_contents = open(releases_index, "r").read() + releases_index_contents = releases_index_contents.replace(".. annoucements\n", ".. annoucements\n" + next_version + "_announce\n" ) + with open(releases_index, "w") as f: + f.write(releases_index_contents) + + for pr in _get_prs(release_name): + # 2015-06-29 18:32:13 2015-04-22 19:11:53 2015-08-12 21:15:45 + as_dict = { + "title": pr.title, + "number": pr.number, + "head": pr.head, + } + main([argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)]) + + +def check_release(argv): + github = _github_client() + release_name = argv[2] + for pr in _get_prs(release_name): + _text_target(github, pr) + + +def check_blocking_prs(argv): + release_name = argv[2] + block = 0 + for pr in _get_prs(release_name, state="open"): + print "WARN: Blocking PR| %s" % _pr_to_str(pr) + block = 1 + + sys.exit(block) + + +def check_blocking_issues(argv): + release_name = argv[2] + block = 0 + github = _github_client() + issues = github.issues.list( + state="open", + ) + for page in issues: + for issue in page: + if issue.milestone and issue.milestone.title == release_name and "Publication of Galaxy Release" not in issue.title: + print "WARN: Blocking issue| %s" % _issue_to_str(issue) + block = 1 + + sys.exit(block) + + +def _pr_to_str(pr): + return "PR #%s (%s) %s" % (pr.number, pr.title, pr.html_url) + + +def _issue_to_str(pr): + return "Issue #%s (%s) %s" % (pr.number, pr.title, pr.html_url) + + +def _next_version_params(release_name): + month = int(release_name.split(".")[1]) + year = release_name.split(".")[0] next_month = (((month - 1) + 3) % 12) + 1 next_month_name = calendar.month_name[next_month] if next_month < 3: @@ -186,43 +348,39 @@ def do_release(argv): else: next_year = year next_version = "%s.%02d" % (next_year, next_month) - first_of_next_month = datetime.date(int(next_year) + 2000, next_month, 1) - freeze_date = next_weekday(first_of_next_month, 0) - release_date = next_weekday(first_of_next_month, 0) + datetime.timedelta(21) - - next_release_file = _release_file(next_version + "_announce.rst") - next_announce = NEXT_TEMPLATE.substitute( + freeze_date, release_date = _release_dates(next_version) + return dict( version=next_version, year=next_year, month_name=next_month_name, freeze_date=freeze_date, release_date=release_date, ) - open(next_release_file, "w").write(next_announce.encode("utf-8")) - - for page in _get_prs(): - for pr in page: - merged_at = pr.merged_at - milestone = pr.milestone - if not merged_at or not milestone or milestone['title'] != release_name: - continue - # 2015-06-29 18:32:13 2015-04-22 19:11:53 2015-08-12 21:15:45 - as_dict = { - "title": pr.title, - "number": pr.number, - "head": pr.head, - } - main([argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)]) -def _get_prs(): +def _release_dates(version): + year, month = version.split(".") + first_of_month = datetime.date(int(year) + 2000, int(month), 1) + freeze_date = next_weekday(first_of_month, 0) + release_date = next_weekday(first_of_month, 0) + datetime.timedelta(21) + return freeze_date, release_date + + +def _get_prs(release_name, state="closed"): github = _github_client() pull_requests = github.pull_requests.list( - state='closed', + state=state, user=PROJECT_OWNER, repo=PROJECT_NAME, ) - return pull_requests + for page in pull_requests: + for pr in page: + merged_at = pr.merged_at + milestone = pr.milestone + proper_state = state != "closed" or merged_at + if not proper_state or not milestone or milestone['title'] != release_name: + continue + yield pr def main(argv): @@ -231,10 +389,26 @@ def main(argv): github = _github_client() newest_release = None + if argv[1] == "--check-blocking-prs": + check_blocking_prs(argv) + return + + if argv[1] == "--check-blocking-issues": + check_blocking_issues(argv) + return + + if argv[1] == "--create-release-issue": + release_issue(argv) + return + if argv[1] == "--release": do_release(argv) return + if argv[1] == "--check-release": + check_release(argv) + return + if argv[1] == "--release_file": newest_release = argv[2] argv = [argv[0]] + argv[3:] @@ -245,15 +419,14 @@ def main(argv): else: req = None - releases_path = os.path.join(PROJECT_DIRECTORY, "doc", "source", "releases") if newest_release is None: - newest_release = sorted(os.listdir(releases_path))[-1] - history_path = os.path.join(releases_path, newest_release) + newest_release = sorted(os.listdir(RELEASES_PATH))[-1] + history_path = os.path.join(RELEASES_PATH, newest_release) history = open(history_path, "r").read().decode("utf-8") - def extend(from_str, line): + def extend(from_str, line, source=history): from_str += "\n" - return history.replace(from_str, from_str + line + "\n" ) + return source.replace(from_str, from_str + line + "\n" ) ident = argv[1] @@ -295,58 +468,12 @@ def main(argv): text = ".. _Pull Request {0}: {1}/pull/{0}".format(pull_request, PROJECT_URL) history = extend(".. github_links", text) if owner: - to_doc += "\n(Thanks to `@%s `__.)" % ( + to_doc += "\n(thanks to `@%s `__.)" % ( owner, owner, ) to_doc += "\n`Pull Request {0}`_".format(pull_request) if github: - labels = [] - try: - labels = github.issues.labels.list_by_issue(int(pull_request), user=PROJECT_OWNER, repo=PROJECT_NAME) - except Exception: - pass - is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False - for label in labels: - label_name = label.name.lower() - if label_name == "minor": - is_minor = True - elif label_name == "major": - is_major = True - elif label_name == "merge": - is_merge = True - elif label_name == "kind/bug": - is_bug = True - elif label_name == "kind/feature": - is_feature = True - elif label_name == "kind/enhancement": - is_enhancement = True - elif label_name in ["kind/testing", "kind/refactoring"]: - is_small_enhancement = True - - is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement - - if not( is_bug or is_some_kind_of_enhancement or is_minor or is_merge ): - print "No kind/ or minor or merge label found for %s" % pull_request - text_target = None - - if is_minor or is_merge: - return - - if is_some_kind_of_enhancement and is_major: - text_target = "major_feature" - elif is_feature: - text_target = "feature" - elif is_enhancement: - text_target = "enhancement" - elif is_some_kind_of_enhancement: - text_target = "small_enhancement" - elif is_major: - text_target = "major_bug" - elif is_bug: - text_target = "bug" - else: - print "Logic problem, cannot determine section for %s" % pull_request - text_target = None + _text_target(github, pull_request) elif ident.startswith("issue"): issue = ident[len("issue"):] text = ".. _Issue {0}: {1}/issues/{0}".format(issue, PROJECT_URL) @@ -363,6 +490,97 @@ def main(argv): open(history_path, "w").write(history.encode("utf-8")) +def _text_target(github, pull_request): + labels = [] + try: + labels = github.issues.labels.list_by_issue(int(pull_request.number), user=PROJECT_OWNER, repo=PROJECT_NAME) + except Exception as e: + print e + is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False + for label in labels: + label_name = label.name.lower() + if label_name == "minor": + is_minor = True + elif label_name == "major": + is_major = True + elif label_name == "merge": + is_merge = True + elif label_name == "kind/bug": + is_bug = True + elif label_name == "kind/feature": + is_feature = True + elif label_name == "kind/enhancement": + is_enhancement = True + elif label_name in ["kind/testing", "kind/refactoring"]: + is_small_enhancement = True + + is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement + + if not( is_bug or is_some_kind_of_enhancement or is_minor or is_merge ): + print "No kind/ or minor or merge label found for %s" % _pr_to_str(pull_request) + text_target = None + + if is_minor or is_merge: + return + + if is_some_kind_of_enhancement and is_major: + text_target = "major_feature" + elif is_feature: + text_target = "feature" + elif is_enhancement: + text_target = "enhancement" + elif is_some_kind_of_enhancement: + text_target = "small_enhancement" + elif is_major: + text_target = "major_bug" + elif is_bug: + text_target = "bug" + else: + print "Logic problem, cannot determine section for %s" % pull_request + text_target = None + return text_target + + +def _previous_release(to): + previous_release = None + for release in _releases(): + if release == to: + break + + previous_release = release + + return previous_release + + +def _latest_release(): + return _releases()[-1] + + +def _releases(): + all_files = sorted(os.listdir(RELEASES_PATH)) + release_note_file_pattern = re.compile(r"\d+\.\d+.rst") + release_note_files = filter(lambda f: release_note_file_pattern.match(f), all_files) + return sorted(map(lambda f: f.rstrip('.rst'), release_note_files)) + + +def _get_major_version(): + with open(GALAXY_VERSION_FILE, 'rb') as f: + init_contents = f.read().decode('utf-8') + + def get_var(var_name): + pattern = re.compile(r'%s\s+=\s+(.*)' % var_name) + match = pattern.search(init_contents).group(1) + return str(ast.literal_eval(match)) + return get_var("VERSION_MAJOR") + + +def _get_release_name(argv): + if len(argv) > 2: + return argv[2] + else: + return _get_major_version() + + def _github_client(): if Github: github_json = os.path.expanduser("~/.github.json")