Merge pull request #10258 from hexylena/release-diff

Add script to diff two release config files.

doc/source/releases/20.09-config.rst

@@ -0,0 +1,30 @@
+Configuration Changes
+=====================
+
+Changed
+-------
+
+The following configuration options have been changed
+
+config/tool_shed.yml.sample
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-  uwsgi.static-safe has changed from ``client/galaxy/images`` to ``client/src/assets``
+
+config/galaxy.yml.sample
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+-  uwsgi.static-safe has changed from ``client/galaxy/images`` to ``client/src/assets``
+
+config/reports.yml.sample
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-  uwsgi.static-safe has changed from ``client/galaxy/images`` to ``client/src/assets``
+
+
+New Configuration Files
+-----------------------
+
+The following files are new, or recently converted to yaml
+
+-  ``config/trs_servers_conf.yml.sample``

scripts/release-diff.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python
+import subprocess
+import argparse
+from pathlib import Path
+import glob
+import yaml
+from yaml import SafeLoader
+
+
+def flatten(d, path):
+    """
+    Flatten a dictionary into ('some/path/to/key', value)
+
+    >>> flatten({'a': {'b': 2}, 'q': 3}, [])
+    [('a.b', 2), ('q', 3)]
+    """
+
+    if isinstance(d, dict):
+        for k, v in d.items():
+            yield from flatten(v, path + [k])
+    else:
+        yield (".".join(path), d)
+
+
+def flat_dict(d):
+    return dict(flatten(d, []))
+
+
+# Load without the includes since we can't follow those across git revisions.
+class MockOrderedLoader(SafeLoader):
+    def include(self, node):
+        return {}
+
+
+MockOrderedLoader.add_constructor("!include", MockOrderedLoader.include)
+
+
+def diff_files(old, new):
+    # Flatten them
+    old_kv = flat_dict(old)
+    new_kv = flat_dict(new)
+
+    # Compare them
+    old_k = set(old_kv.keys())
+    new_k = set(new_kv.keys())
+
+    added = new_k - old_k
+    removed = old_k - new_k
+    shared = old_k & new_k
+    changed = [(k, old_kv[k], new_kv[k]) for k in shared if old_kv[k] != new_kv[k]]
+
+    return added, removed, changed
+
+
+def _report_dict(title, subheading, data, mapper):
+    print(title)
+    print("-" * len(title))
+    print()
+    print(subheading)
+    print()
+    for fn in data:
+        print(fn)
+        print("~" * len(fn))
+        print()
+        for k in data[fn]:
+            print(mapper(k))
+        print()
+    print()
+
+
+def report_diff(added, changed, removed, new_files):
+    # Print out report
+    if added or changed or removed:
+        print("Configuration Changes")
+        print("=====================")
+        print()
+
+    if added:
+        _report_dict(
+            "Added",
+            "The following configuration options are new",
+            added,
+            lambda x: f"-  {x}"
+        )
+
+    if changed:
+        _report_dict(
+            "Changed",
+            "The following configuration options have been changed",
+            changed,
+            lambda x: f"-  {x[0]} has changed from ``{x[1]}`` to ``{x[2]}``"
+        )
+
+    if removed:
+        _report_dict(
+            "Removed",
+            "The following configuration options have been completely removed",
+            removed,
+            lambda x: f"-  {x}"
+        )
+
+    if new_files:
+        print("New Configuration Files")
+        print("-----------------------")
+        print()
+        print(f"The following files are new, or recently converted to yaml")
+        print()
+        for k in new_files:
+            print(f"-  ``{k}``")
+
+
+def load_at_time(path, revision=None):
+    if revision is not None:
+        return subprocess.check_output(
+            ["git", "show", f"{revision}:{path}"], stderr=subprocess.STDOUT
+        )
+    else:
+        with open(path, "r") as handle:
+            return handle.read()
+
+
+def main(old_revision, new_revision=None):
+    files_to_diff = glob.glob("config/*.yml.sample")
+    added = {}
+    removed = {}
+    changed = {}
+    new_files = []
+
+    for file in files_to_diff:
+        real_path = Path(file).resolve().relative_to(Path.cwd())
+        try:
+            old_contents = yaml.load(
+                load_at_time(real_path, old_revision), Loader=MockOrderedLoader
+            )
+            new_contents = yaml.load(
+                load_at_time(real_path, new_revision), Loader=MockOrderedLoader
+            )
+
+            (a, r, c) = diff_files(old_contents, new_contents)
+            if a:
+                added[file] = a
+
+            if r:
+                removed[file] = r
+
+            if c:
+                changed[file] = c
+
+        except subprocess.CalledProcessError:
+            new_files.append(file)
+
+    report_diff(added, changed, removed, new_files)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Diff yaml configuration files between two points in time."
+    )
+    parser.add_argument("old_revision", help="Old revision")
+    parser.add_argument(
+        "--new_revision",
+        help="New revision (defaults to whatever is currently in tree)",
+    )
+    args = parser.parse_args()
+    main(args.old_revision, args.new_revision)