Work toward automating deployment.

Create new makefile targets for release management.

 - ``release-create-issue``: Automates creating a release checklist - https://github.com/jmchilton/galaxy/issues/41)
   Check list contains actual tailored ``make`` commands and links (for PRs and issues).
 - ``release-check-metadata``: Checks milestone PRs for required metadata needed for generating bootstrapped release notes.
 ` ``release-bootstrap-history``: Bootstrap release notes from PRs and metadata.
 - ``release-check-blocking-prs``: Check for PRs blocking release, should happen before branching release and before tagging release.
 - ``release-check-blocking-issues``: Check for Issues blocking release, should happen before tagging release. (Only allowed issue is the release creation issue.)

The issue created with ``release-create-issue`` documents how and when to execute subsequent ``make`` targets (including the target to create the issue for the next release).

This PR contains a bunch of other enhancements to the bootstrap_history.py script to enable these targets and renames ``create_release_rc`` Makefile target so all release related targets start with the prefix ``release-``.

Conflicts:
	Makefile

Makefile

@@ -5,11 +5,55 @@ RELEASE_NEXT:=16.04
 #RELEASE_NEXT_BRANCH:=release_$(RELEASE_NEXT)
 RELEASE_NEXT_BRANCH:=dev
 RELEASE_UPSTREAM:=upstream
+# Location of virtualenv used for development.
+VENV?=.venv
+# Source virtualenv to execute command (flake8, sphinx, twine, etc...)
+IN_VENV=if [ -f $(VENV)/bin/activate ]; then . $(VENV)/bin/activate; fi;
+PROJECT_URL?=https://github.com/galaxyproject/galaxy
 GRUNT_DOCKER_NAME:=galaxy/client-builder:16.01
 
 all: help
 	@echo "This makefile is primarily used for building Galaxy's JS client. A sensible all target is not yet implemented."
 
+help:
+	@echo "client - rebuild all client artifacts"
+	@echo "docs - generate Sphinx HTML documentation, including API docs"
+	@echo "open-docs - generate Sphinx HTML documentation and open in browser"
+	@echo "open-project - open project on github"
+	@echo "lint - check style using tox and flake8 for Python 2 and Python 3"
+	@echo "release-check-metadata - check github PR metadata for target release"
+	@echo "release-bootstrap-history - bootstrap history for a new release"
+
+docs:
+	$(IN_VENV) $(MAKE) -C docs clean
+	$(IN_VENV) $(MAKE) -C docs html
+
+_open-docs:
+	open docs/_build/html/index.html || xdg-open docs/_build/html/index.html
+
+open-docs: docs _open-docs
+
+open-project:
+	open $(PROJECT_URL) || xdg-open $(PROJECT_URL)
+
+lint:
+	$(IN_VENV) tox -e py27-lint && tox -e py34-lint
+
+release-issue:
+	$(IN_VENV) python scripts/bootstrap_history.py --create-release-issue $(RELEASE_CURR)
+
+release-check-metadata:
+	$(IN_VENV) python scripts/bootstrap_history.py --check-release $(RELEASE_CURR)
+
+release-check-blocking-issues:
+	$(IN_VENV) python scripts/bootstrap_history.py --check-blocking-issues $(RELEASE_CURR)
+
+release-check-blocking-prs:
+	$(IN_VENV) python scripts/bootstrap_history.py --check-blocking-prs $(RELEASE_CURR)
+
+release-bootstrap-history:
+	$(IN_VENV) python scripts/bootstrap_history.py --release $(RELEASE_CURR)
+
 npm-deps:
 	cd client && npm install
 
@@ -35,7 +79,7 @@ clean-grunt-docker-image: ## Remove grunt docker image
 
 
 # Release Targets
-create_release_rc: ## Create a release-candidate branch
+release-create-rc: ## Create a release-candidate branch 
 	git checkout dev
 	git pull --ff-only ${RELEASE_UPSTREAM} dev
 	git push origin dev

doc/source/releases/index.rst

@@ -4,6 +4,8 @@ Releases
 .. toctree::
    :maxdepth: 1
 
+.. annoucements
+   16.04_announce
    16.01_announce
    15.10_announce
    15.07_announce

scripts/bootstrap_history.py

@@ -2,6 +2,7 @@
 # Little script to make HISTORY.rst more easy to format properly, lots TODO
 # pull message down and embed, use arg parse, handle multiple, etc...
 
+import ast
 import calendar
 import datetime
 import os
@@ -22,17 +23,14 @@ except ImportError:
 
 
 PROJECT_DIRECTORY = os.path.join(os.path.dirname(__file__), os.pardir)
+SOURCE_DIR = os.path.join(PROJECT_DIRECTORY, "lib")
+GALAXY_SOURCE_DIR = os.path.join(SOURCE_DIR, "galaxy")
+GALAXY_VERSION_FILE = os.path.join(GALAXY_SOURCE_DIR, "version.py")
 PROJECT_OWNER = "galaxyproject"
 PROJECT_NAME = "galaxy"
 PROJECT_URL = "https://github.com/%s/%s" % (PROJECT_OWNER, PROJECT_NAME)
 PROJECT_API = "https://api.github.com/repos/%s/%s/" % (PROJECT_OWNER, PROJECT_NAME)
-
-RELEASES = [
-    ("15.05", "b16ac25cdc0f2b64d6af34ea1e6ff253d8a71ee4"),
-    ("15.07", "e44c8db9dea56b8d1a2f941ce572b0f14e999d4c"),
-    ("15.10", "ef55279d58eced4b90632a572a8fc5a227ceefb9"),
-    ("16.01", "cc23adb0a962f1c9780b838604718319a0a71888"),
-]
+RELEASES_PATH = os.path.join(PROJECT_DIRECTORY, "doc", "source", "releases")
 
 # Uncredit pull requestors... kind of arbitrary at this point.
 DEVTEAM = [
@@ -151,6 +149,79 @@ Schedule
  * Planned Release Date: ${release_date}
 """)
 
+
+RELEASE_ISSUE_TEMPLATE = string.Template("""
+
+- [X] **Prep**
+
+      - [X] ~~Create this release issue ``make release-issue RELEASE_CURR=${version}``.~~
+      - [X] ~~Set freeze date (${freeze_date}).~~
+
+- [ ] **Branch Release (on or around ${freeze_date})**
+
+      - [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged, delayed, or closed.
+
+            make release-check-blocking-prs RELEASE_CURR=${version}
+      - [ ] Merge the latest release into dev.
+
+            git fetch upstream && git checkout dev && git merge --ff-only upstream dev && git merge upstream release_${previous_version}
+      - [ ] Create and push release branch:
+
+            make release-create-rc RELEASE_CURR=${version} RELEASE_NEXT=${next_version}
+
+- [ ] **Deploy and Test Release**
+
+      - [ ] Deploy to test (${freeze_date} + 1 day).
+      - [ ] Deploy to usegalaxy.org (${freeze_date} + 1 week).
+
+- [ ] **Create Release Notes**
+
+      - [ ] Review merged PRs and ensure they all milestones attached. [Link](https://github.com/galaxyproject/galaxy/pulls?q=is%3Apr+is%3Amerged+no%3Amilestone)
+      - [ ] Checkout release branch
+
+            git checkout $version -b ${version}_release_notes
+      - [ ] Check for obvious missing metadata in release PRs
+
+            make release-check-metadata RELEASE_CURR=${version}
+      - [ ] Bootstrap the release notes
+
+            make release-bootstrap-history RELEASE_CURR=${version}
+      - [ ] Open newly created files and manually curate major topics and release notes.
+      - [ ] Commit release notes.
+
+            git add docs/; git commit -m "Release notes for $version"; git push upstream ${version}_release_notes
+      - [ ] Open a pull request for new release note branch.
+      - [ ] Merge release note pull request.
+
+- [ ] **Do Release**
+
+      - [ ] Ensure all [blocking milestone issues](https://github.com/galaxyproject/galaxy/issues?q=is%3Aopen+is%3Aissue+milestone%3A${version}) have been resolved.
+
+            make release-check-blocking-issues RELEASE_CURR=${version}
+      - [ ] Ensure all [blocking milestone PRs](https://github.com/galaxyproject/galaxy/pulls?q=is%3Aopen+is%3Apr+milestone%3A${version}) have been merged or closed.
+
+            make release-check-blocking-prs RELEASE_CURR=${version}
+      - [ ] Do release (TODO: create a check a list for this...)
+
+- [ ] **Announce Release**
+
+      - [ ] Stage annoucement content (Wiki, Biostars, Bit.ly link) on annouce date to capture date tags. Note: all final content does not need to be completed to do this.
+      - [ ] Finalize https://github.com/galaxyproject/galaxy/blob/dev/doc/source/releases/${version}_announce.rst
+      - [ ] Post release notes to https://docs.galaxyproject.org/en/master/releases/index.html
+      - [ ] Create wiki *highlights* and post to http://galaxyproject.org News (w/ RSS) and NewsBriefs
+      - [ ] Tweet wiki news *highlights* (or RTD?) via bit.ly link to https://twitter.com/galaxyproject/
+      - [ ] Post *highlights* type News to Galaxy Biostars https://biostar.usegalaxy.org
+      - [ ] Email *highlights* to galaxy-dev and galaxy-announce @lists.galaxyproject.org
+      - [ ] Adjust http://getgalaxy.org text and links to match current master branch
+
+- [ ] **Prepare for next release**
+
+      - [ ] Ensure milestone ``${next_version}`` exists.
+      - [ ] Create release issue for next version ``make release-issue RELEASE_CURR=${next_version}``.
+      - [ ] Close this issue.
+
+""")
+
 # https://api.github.com/repos/galaxyproject/galaxy/pulls?base=dev&state=closed
 # https://api.github.com/repos/galaxyproject/galaxy/pulls?base=release_15.07&state=closed
 # https://api.github.com/repos/galaxyproject/galaxy/compare/release_15.05...dev
@@ -162,6 +233,31 @@ def commit_time(commit_hash):
     return datetime.datetime.strptime(req["commit"]["committer"]["date"], "%Y-%m-%dT%H:%M:%SZ")
 
 
+def release_issue(argv):
+    release_name = argv[2]
+    previous_release = _previous_release(release_name)
+    new_version_params = _next_version_params(release_name)
+    next_version = new_version_params["version"]
+    freeze_date, release_date = _release_dates(release_name)
+    release_issue_template_params = dict(
+        version=release_name,
+        next_version=next_version,
+        previous_version=previous_release,
+        freeze_date=freeze_date,
+    )
+    release_issue_contents = RELEASE_ISSUE_TEMPLATE.safe_substitute(**release_issue_template_params)
+    github = _github_client()
+    github.issues.create(
+        data=dict(
+            title="Publication of Galaxy Release v %s" % release_name,
+            body=release_issue_contents,
+        ),
+        user=PROJECT_OWNER,
+        repo=PROJECT_NAME,
+    )
+    return release_issue
+
+
 def do_release(argv):
     release_name = argv[2]
     release_file = _release_file(release_name + ".rst")
@@ -179,6 +275,72 @@ def do_release(argv):
     announce_file = _release_file(release_name + "_announce.rst")
     open(announce_file, "w").write(announce_info.encode("utf-8"))
 
+    next_version_params = _next_version_params(release_name)
+    next_version = next_version_params["version"]
+    next_release_file = _release_file(next_version + "_announce.rst")
+
+    next_announce = NEXT_TEMPLATE.substitute(**next_version_params)
+    open(next_release_file, "w").write(next_announce.encode("utf-8"))
+    releases_index = _release_file("index.rst")
+    releases_index_contents = open(releases_index, "r").read()
+    releases_index_contents = releases_index_contents.replace(".. annoucements\n", ".. annoucements\n" + next_version + "_announce\n" )
+    with open(releases_index, "w") as f:
+        f.write(releases_index_contents)
+
+    for pr in _get_prs(release_name):
+        # 2015-06-29 18:32:13 2015-04-22 19:11:53 2015-08-12 21:15:45
+        as_dict = {
+            "title": pr.title,
+            "number": pr.number,
+            "head": pr.head,
+        }
+        main([argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)])
+
+
+def check_release(argv):
+    github = _github_client()
+    release_name = argv[2]
+    for pr in _get_prs(release_name):
+        _text_target(github, pr)
+
+
+def check_blocking_prs(argv):
+    release_name = argv[2]
+    block = 0
+    for pr in _get_prs(release_name, state="open"):
+        print "WARN: Blocking PR| %s" % _pr_to_str(pr)
+        block = 1
+
+    sys.exit(block)
+
+
+def check_blocking_issues(argv):
+    release_name = argv[2]
+    block = 0
+    github = _github_client()
+    issues = github.issues.list(
+        state="open",
+    )
+    for page in issues:
+        for issue in page:
+            if issue.milestone and issue.milestone.title == release_name and "Publication of Galaxy Release" not in issue.title:
+                print "WARN: Blocking issue| %s" % _issue_to_str(issue)
+                block = 1
+
+    sys.exit(block)
+
+
+def _pr_to_str(pr):
+    return "PR #%s (%s) %s" % (pr.number, pr.title, pr.html_url)
+
+
+def _issue_to_str(pr):
+    return "Issue #%s (%s) %s" % (pr.number, pr.title, pr.html_url)
+
+
+def _next_version_params(release_name):
+    month = int(release_name.split(".")[1])
+    year = release_name.split(".")[0]
     next_month = (((month - 1) + 3) % 12) + 1
     next_month_name = calendar.month_name[next_month]
     if next_month < 3:
@@ -186,43 +348,39 @@ def do_release(argv):
     else:
         next_year = year
     next_version = "%s.%02d" % (next_year, next_month)
-    first_of_next_month = datetime.date(int(next_year) + 2000, next_month, 1)
-    freeze_date = next_weekday(first_of_next_month, 0)
-    release_date = next_weekday(first_of_next_month, 0) + datetime.timedelta(21)
-
-    next_release_file = _release_file(next_version + "_announce.rst")
-    next_announce = NEXT_TEMPLATE.substitute(
+    freeze_date, release_date = _release_dates(next_version)
+    return dict(
         version=next_version,
         year=next_year,
         month_name=next_month_name,
         freeze_date=freeze_date,
         release_date=release_date,
     )
-    open(next_release_file, "w").write(next_announce.encode("utf-8"))
-
-    for page in _get_prs():
-        for pr in page:
-            merged_at = pr.merged_at
-            milestone = pr.milestone
-            if not merged_at or not milestone or milestone['title'] != release_name:
-                continue
-            # 2015-06-29 18:32:13 2015-04-22 19:11:53 2015-08-12 21:15:45
-            as_dict = {
-                "title": pr.title,
-                "number": pr.number,
-                "head": pr.head,
-            }
-            main([argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)])
 
 
-def _get_prs():
+def _release_dates(version):
+    year, month = version.split(".")
+    first_of_month = datetime.date(int(year) + 2000, int(month), 1)
+    freeze_date = next_weekday(first_of_month, 0)
+    release_date = next_weekday(first_of_month, 0) + datetime.timedelta(21)
+    return freeze_date, release_date
+
+
+def _get_prs(release_name, state="closed"):
     github = _github_client()
     pull_requests = github.pull_requests.list(
-        state='closed',
+        state=state,
         user=PROJECT_OWNER,
         repo=PROJECT_NAME,
     )
-    return pull_requests
+    for page in pull_requests:
+        for pr in page:
+            merged_at = pr.merged_at
+            milestone = pr.milestone
+            proper_state = state != "closed" or merged_at
+            if not proper_state or not milestone or milestone['title'] != release_name:
+                continue
+            yield pr
 
 
 def main(argv):
@@ -231,10 +389,26 @@ def main(argv):
     github = _github_client()
     newest_release = None
 
+    if argv[1] == "--check-blocking-prs":
+        check_blocking_prs(argv)
+        return
+
+    if argv[1] == "--check-blocking-issues":
+        check_blocking_issues(argv)
+        return
+
+    if argv[1] == "--create-release-issue":
+        release_issue(argv)
+        return
+
     if argv[1] == "--release":
         do_release(argv)
         return
 
+    if argv[1] == "--check-release":
+        check_release(argv)
+        return
+
     if argv[1] == "--release_file":
         newest_release = argv[2]
         argv = [argv[0]] + argv[3:]
@@ -245,15 +419,14 @@ def main(argv):
     else:
         req = None
 
-    releases_path = os.path.join(PROJECT_DIRECTORY, "doc", "source", "releases")
     if newest_release is None:
-        newest_release = sorted(os.listdir(releases_path))[-1]
-    history_path = os.path.join(releases_path, newest_release)
+        newest_release = sorted(os.listdir(RELEASES_PATH))[-1]
+    history_path = os.path.join(RELEASES_PATH, newest_release)
     history = open(history_path, "r").read().decode("utf-8")
 
-    def extend(from_str, line):
+    def extend(from_str, line, source=history):
         from_str += "\n"
-        return history.replace(from_str, from_str + line + "\n" )
+        return source.replace(from_str, from_str + line + "\n" )
 
     ident = argv[1]
 
@@ -295,58 +468,12 @@ def main(argv):
         text = ".. _Pull Request {0}: {1}/pull/{0}".format(pull_request, PROJECT_URL)
         history = extend(".. github_links", text)
         if owner:
-            to_doc += "\n(Thanks to `@%s <https://github.com/%s>`__.)" % (
+            to_doc += "\n(thanks to `@%s <https://github.com/%s>`__.)" % (
                 owner, owner,
             )
         to_doc += "\n`Pull Request {0}`_".format(pull_request)
         if github:
-            labels = []
-            try:
-                labels = github.issues.labels.list_by_issue(int(pull_request), user=PROJECT_OWNER, repo=PROJECT_NAME)
-            except Exception:
-                pass
-            is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False
-            for label in labels:
-                label_name = label.name.lower()
-                if label_name == "minor":
-                    is_minor = True
-                elif label_name == "major":
-                    is_major = True
-                elif label_name == "merge":
-                    is_merge = True
-                elif label_name == "kind/bug":
-                    is_bug = True
-                elif label_name == "kind/feature":
-                    is_feature = True
-                elif label_name == "kind/enhancement":
-                    is_enhancement = True
-                elif label_name in ["kind/testing", "kind/refactoring"]:
-                        is_small_enhancement = True
-
-            is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement
-
-            if not( is_bug or is_some_kind_of_enhancement or is_minor or is_merge ):
-                print "No kind/ or minor or merge label found for %s" % pull_request
-                text_target = None
-
-            if is_minor or is_merge:
-                return
-
-            if is_some_kind_of_enhancement and is_major:
-                text_target = "major_feature"
-            elif is_feature:
-                text_target = "feature"
-            elif is_enhancement:
-                text_target = "enhancement"
-            elif is_some_kind_of_enhancement:
-                text_target = "small_enhancement"
-            elif is_major:
-                text_target = "major_bug"
-            elif is_bug:
-                text_target = "bug"
-            else:
-                print "Logic problem, cannot determine section for %s" % pull_request
-                text_target = None
+            _text_target(github, pull_request)
     elif ident.startswith("issue"):
         issue = ident[len("issue"):]
         text = ".. _Issue {0}: {1}/issues/{0}".format(issue, PROJECT_URL)
@@ -363,6 +490,97 @@ def main(argv):
     open(history_path, "w").write(history.encode("utf-8"))
 
 
+def _text_target(github, pull_request):
+    labels = []
+    try:
+        labels = github.issues.labels.list_by_issue(int(pull_request.number), user=PROJECT_OWNER, repo=PROJECT_NAME)
+    except Exception as e:
+        print e
+    is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False
+    for label in labels:
+        label_name = label.name.lower()
+        if label_name == "minor":
+            is_minor = True
+        elif label_name == "major":
+            is_major = True
+        elif label_name == "merge":
+            is_merge = True
+        elif label_name == "kind/bug":
+            is_bug = True
+        elif label_name == "kind/feature":
+            is_feature = True
+        elif label_name == "kind/enhancement":
+            is_enhancement = True
+        elif label_name in ["kind/testing", "kind/refactoring"]:
+            is_small_enhancement = True
+
+    is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement
+
+    if not( is_bug or is_some_kind_of_enhancement or is_minor or is_merge ):
+        print "No kind/ or minor or merge label found for %s" % _pr_to_str(pull_request)
+        text_target = None
+
+    if is_minor or is_merge:
+        return
+
+    if is_some_kind_of_enhancement and is_major:
+        text_target = "major_feature"
+    elif is_feature:
+        text_target = "feature"
+    elif is_enhancement:
+        text_target = "enhancement"
+    elif is_some_kind_of_enhancement:
+        text_target = "small_enhancement"
+    elif is_major:
+        text_target = "major_bug"
+    elif is_bug:
+        text_target = "bug"
+    else:
+        print "Logic problem, cannot determine section for %s" % pull_request
+        text_target = None
+    return text_target
+
+
+def _previous_release(to):
+    previous_release = None
+    for release in _releases():
+        if release == to:
+            break
+
+        previous_release = release
+
+    return previous_release
+
+
+def _latest_release():
+    return _releases()[-1]
+
+
+def _releases():
+    all_files = sorted(os.listdir(RELEASES_PATH))
+    release_note_file_pattern = re.compile(r"\d+\.\d+.rst")
+    release_note_files = filter(lambda f: release_note_file_pattern.match(f), all_files)
+    return sorted(map(lambda f: f.rstrip('.rst'), release_note_files))
+
+
+def _get_major_version():
+    with open(GALAXY_VERSION_FILE, 'rb') as f:
+        init_contents = f.read().decode('utf-8')
+
+        def get_var(var_name):
+            pattern = re.compile(r'%s\s+=\s+(.*)' % var_name)
+            match = pattern.search(init_contents).group(1)
+            return str(ast.literal_eval(match))
+        return get_var("VERSION_MAJOR")
+
+
+def _get_release_name(argv):
+    if len(argv) > 2:
+        return argv[2]
+    else:
+        return _get_major_version()
+
+
 def _github_client():
     if Github:
         github_json = os.path.expanduser("~/.github.json")