From 038d78a8f620e87644f5bf70d58212b48b29a13f Mon Sep 17 00:00:00 2001 From: John Chilton Date: Fri, 24 May 2019 10:19:40 -0400 Subject: [PATCH] Fixes and enhancements for release note generation. - Auto-bootstrap user announcements for datatypes, tools, and viz. (follow up on #7527) - Many fixes for things broken in #7527 - Treat procedures as an enhancment for categorizing things. - Speed up release note generation a bit by not considering really, really old PRs. --- scripts/bootstrap_history.py | 111 ++++++++++++++++++++++++++--------- 1 file changed, 84 insertions(+), 27 deletions(-) diff --git a/scripts/bootstrap_history.py b/scripts/bootstrap_history.py index 901c50aeb66..154b4e34de5 100644 --- a/scripts/bootstrap_history.py +++ b/scripts/bootstrap_history.py @@ -38,7 +38,7 @@ DEVTEAM = [ "davebx", "martenson", "jmchilton", "tnabtaf", "natefoo", "jgoecks", "guerler", "jennaj", "nekrut", "jxtx", - "VJalili" + "VJalili", "WilliamHolden", "Nerdinacan" ] TEMPLATE = string.Template(""" @@ -143,15 +143,21 @@ Highlights Feature description. -New Visualisations +New Visualizations =========================================================== +.. visualizations + New Datatypes =========================================================== +.. datatypes + Builtin Tool Updates =========================================================== +.. tools + Release Notes =========================================================== @@ -174,6 +180,9 @@ Schedule * Planned Release Date: ${release_date} """) +PRS_TEMPLATE = """ +.. github_links +""" RELEASE_ISSUE_TEMPLATE = string.Template(""" @@ -337,7 +346,7 @@ def do_release(argv): release=release_name ) announce_file = _release_file(release_name + "_announce.rst") - open(announce_file, "w").write(announce_info.encode("utf-8")) + _write_file(announce_file, announce_info) announce_user_info = ANNOUNCE_USER_TEMPLATE.substitute( month_name=month_name, @@ -345,7 +354,10 @@ def do_release(argv): release=release_name ) announce_user_file = _release_file(release_name + "_announce_user.rst") - open(announce_user_file, "w").write(announce_user_info.encode("utf-8")) + _write_file(announce_user_file, announce_user_info) + + prs_file = _release_file(release_name + "_prs.rst") + _write_file(prs_file, PRS_TEMPLATE) next_version_params = _next_version_params(release_name) next_version = next_version_params["version"] @@ -354,10 +366,9 @@ def do_release(argv): next_announce = NEXT_TEMPLATE.substitute(**next_version_params) open(next_release_file, "w").write(next_announce.encode("utf-8")) releases_index = _release_file("index.rst") - releases_index_contents = open(releases_index, "r").read() + releases_index_contents = _read_file(releases_index) releases_index_contents = releases_index_contents.replace(".. announcements\n", ".. announcements\n " + next_version + "_announce\n") - with open(releases_index, "w") as f: - f.write(releases_index_contents) + _write_file(releases_index, releases_index_contents) for pr in _get_prs(release_name): # 2015-06-29 18:32:13 2015-04-22 19:11:53 2015-08-12 21:15:45 @@ -365,15 +376,16 @@ def do_release(argv): "title": pr.title, "number": pr.number, "head": pr.head, + "labels": _pr_to_labels(pr), } - main([argv[0], "--release_file", "%s_prs.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)]) + main([argv[0], "--release_file", "%s.rst" % release_name, "--request", as_dict, "pr" + str(pr.number)]) def check_release(argv): github = _github_client() release_name = argv[2] for pr in _get_prs(release_name): - _text_target(github, pr) + _text_target(github, pr, labels=_pr_to_labels(pr)) def check_blocking_prs(argv): @@ -451,8 +463,16 @@ def _get_prs(release_name, state="closed"): user=PROJECT_OWNER, repo=PROJECT_NAME, ) + reached_old_prs = False + for page in pull_requests: + if reached_old_prs: + break + for pr in page: + if pr.created_at < datetime.datetime(2016, 11, 1, 0, 0): + reached_old_prs = True + pass merged_at = pr.merged_at milestone = pr.milestone proper_state = state != "closed" or merged_at @@ -504,10 +524,15 @@ def main(argv): if newest_release is None: newest_release = sorted(os.listdir(RELEASES_PATH))[-1] history_path = os.path.join(RELEASES_PATH, newest_release) - history = open(history_path, "r").read().decode("utf-8") + user_announce_path = history_path[0:-len(".rst")] + "_announce_user.rst" + prs_path = history_path[0:-len(".rst")] + "_prs.rst" - def extend(from_str, line, source=history): - from_str += "\n" + history = _read_file(history_path) + user_announce = _read_file(user_announce_path) + prs_content = _read_file(prs_path) + + def extend_target(target, line, source=history): + from_str = ".. %s\n" % target return source.replace(from_str, from_str + line + "\n") ident = argv[1] @@ -548,48 +573,72 @@ def main(argv): if owner in DEVTEAM: owner = None text = ".. _Pull Request {0}: {1}/pull/{0}".format(pull_request, PROJECT_URL) - history = extend(".. github_links", text) + prs_content = extend_target("github_links", text, prs_content) if owner: to_doc += "\n(thanks to `@%s `__)." % ( owner, owner, ) to_doc += "\n`Pull Request {0}`_".format(pull_request) if github: - text_target = _text_target(github, pull_request) + labels = None + if req and 'labels' in req: + labels = req['labels'] + text_target = _text_target(github, pull_request, labels=labels) elif ident.startswith("issue"): issue = ident[len("issue"):] text = ".. _Issue {0}: {1}/issues/{0}".format(issue, PROJECT_URL) - history = extend(".. github_links", text) + prs_content = extend_target("github_links", text, prs_content) to_doc += "`Issue {0}`_".format(issue) else: short_rev = ident[:7] text = ".. _{0}: {1}/commit/{0}".format(short_rev, PROJECT_URL) - history = extend(".. github_links", text) + prs_content = extend_target("github_links", text, prs_content) to_doc += "{0}_".format(short_rev) to_doc = wrap(to_doc) - history = extend(".. %s\n" % text_target, to_doc, history) - open(history_path, "w").write(history.encode("utf-8")) + history = extend_target(text_target, to_doc, history) + if req and 'labels' in req: + labels = req['labels'] + if 'area/datatypes' in labels: + user_announce = extend_target("datatypes", to_doc, user_announce) + if 'area/visualizations' in labels: + user_announce = extend_target("visualizations", to_doc, user_announce) + if 'area/tools' in labels: + user_announce = extend_target("tools", to_doc, user_announce) + _write_file(history_path, history) + _write_file(prs_path, prs_content) + _write_file(user_announce_path, user_announce) -def _text_target(github, pull_request): - labels = [] +def _read_file(path): + with open(path, "r") as f: + return f.read().decode("utf-8") + + +def _write_file(path, contents): + with open(path, "w") as f: + f.write(contents.encode("utf-8")) + + +def _text_target(github, pull_request, labels=None): pr_number = None if isinstance(pull_request, string_types): pr_number = pull_request else: pr_number = pull_request.number - try: - labels = github.issues.labels.list_by_issue(int(pr_number), user=PROJECT_OWNER, repo=PROJECT_NAME) - except Exception as e: - print(e) + if labels is None: + labels = [] + try: + labels = github.issues.labels.list_by_issue(int(pr_number), user=PROJECT_OWNER, repo=PROJECT_NAME) + labels = [l.name.lower() for l in labels] + except Exception as e: + print(e) is_bug = is_enhancement = is_feature = is_minor = is_major = is_merge = is_small_enhancement = False if len(labels) == 0: print('No labels found for %s' % pr_number) return None - for label in labels: - label_name = label.name.lower() + for label_name in labels: if label_name == "minor": is_minor = True elif label_name == "major": @@ -604,11 +653,14 @@ def _text_target(github, pull_request): is_enhancement = True elif label_name in ["kind/testing", "kind/refactoring"]: is_small_enhancement = True + elif label_name == "procedures": + # Treat procedures as an implicit enhancement. + is_enhancement = True is_some_kind_of_enhancement = is_enhancement or is_feature or is_small_enhancement if not(is_bug or is_some_kind_of_enhancement or is_minor or is_merge): - print("No kind/ or minor or merge label found for %s" % _pr_to_str(pull_request)) + print("No 'kind/*' or 'minor' or 'merge' or 'procedures' label found for %s" % _pr_to_str(pull_request)) text_target = None if is_minor or is_merge: @@ -632,6 +684,11 @@ def _text_target(github, pull_request): return text_target +def _pr_to_labels(pr): + labels = [l["name"].lower() for l in pr.labels] + return labels + + def _previous_release(to): previous_release = None for release in _releases():