From f7f44371f63841cbc0eccbe4ed1ea265b2b0dcd9 Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Tue, 22 Sep 2020 11:11:30 +0200
Subject: [PATCH 1/7] Add script to diff two release config files.

---
 scripts/release-diff.py | 92 +++++++++++++++++++++++++++++++++++++++++
 1 file changed, 92 insertions(+)
 create mode 100644 scripts/release-diff.py

diff --git a/scripts/release-diff.py b/scripts/release-diff.py
new file mode 100644
index 00000000000..4419ff3fdcd
--- /dev/null
+++ b/scripts/release-diff.py
@@ -0,0 +1,92 @@
+#!/usr/bin/env python
+import sys
+import yaml
+from yaml import SafeLoader
+
+
+def flatten(d, path):
+    """
+    Flatten a dictionary into ('some/path/to/key', value)
+
+    >>> flatten({'a': {'b': 2}, 'q': 3}, [])
+    [('a.b', 2), ('q', 3)]
+    """
+
+    if isinstance(d, dict):
+        for k, v in d.items():
+            yield from flatten(v, path + [k])
+    elif isinstance(d, list):
+        for x in d:
+            yield from flatten(x, path + [x])
+    else:
+        yield (".".join(path), d)
+
+
+def flat_dict(d):
+    return dict(flatten(d, []))
+
+
+# Load without the includes since we can't follow those across git revisions.
+class MockOrderedLoader(SafeLoader):
+    def include(self, node):
+        return {}
+
+
+MockOrderedLoader.add_constructor("!include", MockOrderedLoader.include)
+
+# Load our two files
+with open(sys.argv[1], "r") as handle:
+    old = yaml.load(handle, Loader=MockOrderedLoader)
+
+with open(sys.argv[2], "r") as handle:
+    new = yaml.load(handle, Loader=MockOrderedLoader)
+
+
+# Flatten them
+old_kv = flat_dict(old)
+new_kv = flat_dict(new)
+
+# Compare them
+old_k = set(old_kv.keys())
+new_k = set(new_kv.keys())
+
+added = new_k - old_k
+removed = old_k - new_k
+shared = old_k & new_k
+changed = [(k, old_kv[k], new_kv[k]) for k in shared if old_kv[k] != new_kv[k]]
+
+# Print out report
+if added or changed or removed:
+    print("Configuration Changes")
+    print("=====================")
+    print()
+
+if added:
+    print("Added")
+    print("-----")
+    print()
+    print("The following configuration options are new")
+    print()
+    for k in added:
+        print(f"-  {k}")
+    print()
+
+if changed:
+    print("Changed")
+    print("-------")
+    print()
+    print("The following configuration options have been changed")
+    print()
+    for (k, o, n) in changed:
+        print(f"-  {k} has changed from ``{o}`` to ``{n}``")
+    print()
+
+if removed:
+    print("Removed")
+    print("-------")
+    print()
+    print("The following configuration options have been completely removed")
+    print()
+    for k in removed:
+        print(f"-  {k}")
+    print()

From 6a1edfdb35b296e90867417cf25464b0532c392f Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Tue, 22 Sep 2020 11:39:35 +0200
Subject: [PATCH 2/7] make it loop over yml

---
 scripts/release-diff.py | 93 ++++++++++++++++++++++++++++++-----------
 1 file changed, 69 insertions(+), 24 deletions(-)

diff --git a/scripts/release-diff.py b/scripts/release-diff.py
index 4419ff3fdcd..bd75e493501 100644
--- a/scripts/release-diff.py
+++ b/scripts/release-diff.py
@@ -1,8 +1,12 @@
 #!/usr/bin/env python
+import subprocess
 import sys
+from pathlib import Path
+import glob
 import yaml
 from yaml import SafeLoader
 
+old_version = sys.argv[1]
 
 def flatten(d, path):
     """
@@ -15,9 +19,10 @@ def flatten(d, path):
     if isinstance(d, dict):
         for k, v in d.items():
             yield from flatten(v, path + [k])
-    elif isinstance(d, list):
-        for x in d:
-            yield from flatten(x, path + [x])
+    # elif isinstance(d, list):
+        # yield ('.'.join(path), d)
+        # for i, x in enumerate(d):
+            # yield from flatten(x, path + [i])
     else:
         yield (".".join(path), d)
 
@@ -34,26 +39,51 @@ class MockOrderedLoader(SafeLoader):
 
 MockOrderedLoader.add_constructor("!include", MockOrderedLoader.include)
 
-# Load our two files
-with open(sys.argv[1], "r") as handle:
-    old = yaml.load(handle, Loader=MockOrderedLoader)
+# git show v20.01:lib/galaxy/config/sample/galaxy.yml.sample
 
-with open(sys.argv[2], "r") as handle:
-    new = yaml.load(handle, Loader=MockOrderedLoader)
+def diff_files(old, new):
+    # Flatten them
+    old_kv = flat_dict(old)
+    new_kv = flat_dict(new)
+
+    # Compare them
+    old_k = set(old_kv.keys())
+    new_k = set(new_kv.keys())
+
+    added = new_k - old_k
+    removed = old_k - new_k
+    shared = old_k & new_k
+    changed = [(k, old_kv[k], new_kv[k]) for k in shared if old_kv[k] != new_kv[k]]
+
+    return added, removed, changed
 
 
-# Flatten them
-old_kv = flat_dict(old)
-new_kv = flat_dict(new)
 
-# Compare them
-old_k = set(old_kv.keys())
-new_k = set(new_kv.keys())
+files_to_diff = glob.glob('config/*.yml.sample')
+added = {}
+removed = {}
+changed = {}
 
-added = new_k - old_k
-removed = old_k - new_k
-shared = old_k & new_k
-changed = [(k, old_kv[k], new_kv[k]) for k in shared if old_kv[k] != new_kv[k]]
+for file in files_to_diff:
+    real_path = Path(file).resolve().relative_to(Path.cwd())
+    try:
+        contents = subprocess.check_output(['git', 'show', f'{old_version}:{real_path}'])
+        old = yaml.load(contents, Loader=MockOrderedLoader)
+        with open(real_path, 'r') as handle:
+            new = yaml.load(handle, Loader=MockOrderedLoader)
+
+        (a, r, c) = diff_files(old, new)
+        if a:
+            added[file] = a
+
+        if r:
+            removed[file] = r
+
+        if c:
+            changed[file] = c
+
+    except subprocess.CalledProcessError:
+        print(f"{file} did not exist in that revision.")
 
 # Print out report
 if added or changed or removed:
@@ -67,8 +97,13 @@ if added:
     print()
     print("The following configuration options are new")
     print()
-    for k in added:
-        print(f"-  {k}")
+    for fn in added:
+        print(fn)
+        print('~' * len(fn))
+        print()
+        for k in added[fn]:
+            print(f"-  {k}")
+        print()
     print()
 
 if changed:
@@ -77,8 +112,13 @@ if changed:
     print()
     print("The following configuration options have been changed")
     print()
-    for (k, o, n) in changed:
-        print(f"-  {k} has changed from ``{o}`` to ``{n}``")
+    for fn in changed:
+        print(fn)
+        print('~' * len(fn))
+        print()
+        for (k, o, n) in changed[fn]:
+            print(f"-  {k} has changed from ``{o}`` to ``{n}``")
+        print()
     print()
 
 if removed:
@@ -87,6 +127,11 @@ if removed:
     print()
     print("The following configuration options have been completely removed")
     print()
-    for k in removed:
-        print(f"-  {k}")
+    for fn in removed:
+        print(fn)
+        print('~' * len(fn))
+        print()
+        for k in removed[fn]:
+            print(f"-  {k}")
+        print()
     print()

From 64892baf5cc35b1efec8553e1e0f54d175a484e3 Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Tue, 22 Sep 2020 11:46:29 +0200
Subject: [PATCH 3/7] add new files

---
 scripts/release-diff.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/scripts/release-diff.py b/scripts/release-diff.py
index bd75e493501..353156d771c 100644
--- a/scripts/release-diff.py
+++ b/scripts/release-diff.py
@@ -63,11 +63,12 @@ files_to_diff = glob.glob('config/*.yml.sample')
 added = {}
 removed = {}
 changed = {}
+new_files = []
 
 for file in files_to_diff:
     real_path = Path(file).resolve().relative_to(Path.cwd())
     try:
-        contents = subprocess.check_output(['git', 'show', f'{old_version}:{real_path}'])
+        contents = subprocess.check_output(['git', 'show', f'{old_version}:{real_path}'], stderr=subprocess.STDOUT)
         old = yaml.load(contents, Loader=MockOrderedLoader)
         with open(real_path, 'r') as handle:
             new = yaml.load(handle, Loader=MockOrderedLoader)
@@ -83,7 +84,7 @@ for file in files_to_diff:
             changed[file] = c
 
     except subprocess.CalledProcessError:
-        print(f"{file} did not exist in that revision.")
+        new_files.append(file)
 
 # Print out report
 if added or changed or removed:
@@ -135,3 +136,12 @@ if removed:
             print(f"-  {k}")
         print()
     print()
+
+if new_files:
+    print("New Configuration Files")
+    print("-----------------------")
+    print()
+    print(f"The following files are new, or recently converted to yaml since the {old_version}")
+    print()
+    for k in new_files:
+        print(f"-  ``{k}``")

From 721ad64a61412daae3737e5b4e44619dc3d2ff2b Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Tue, 22 Sep 2020 11:59:19 +0200
Subject: [PATCH 4/7] improvements

---
 scripts/release-diff.py | 152 ++++++++++++++++++++--------------------
 1 file changed, 75 insertions(+), 77 deletions(-)

diff --git a/scripts/release-diff.py b/scripts/release-diff.py
index 353156d771c..ba064177b80 100644
--- a/scripts/release-diff.py
+++ b/scripts/release-diff.py
@@ -1,12 +1,12 @@
 #!/usr/bin/env python
 import subprocess
+import argparse
 import sys
 from pathlib import Path
 import glob
 import yaml
 from yaml import SafeLoader
 
-old_version = sys.argv[1]
 
 def flatten(d, path):
     """
@@ -58,90 +58,88 @@ def diff_files(old, new):
     return added, removed, changed
 
 
-
-files_to_diff = glob.glob('config/*.yml.sample')
-added = {}
-removed = {}
-changed = {}
-new_files = []
-
-for file in files_to_diff:
-    real_path = Path(file).resolve().relative_to(Path.cwd())
-    try:
-        contents = subprocess.check_output(['git', 'show', f'{old_version}:{real_path}'], stderr=subprocess.STDOUT)
-        old = yaml.load(contents, Loader=MockOrderedLoader)
-        with open(real_path, 'r') as handle:
-            new = yaml.load(handle, Loader=MockOrderedLoader)
-
-        (a, r, c) = diff_files(old, new)
-        if a:
-            added[file] = a
-
-        if r:
-            removed[file] = r
-
-        if c:
-            changed[file] = c
-
-    except subprocess.CalledProcessError:
-        new_files.append(file)
-
-# Print out report
-if added or changed or removed:
-    print("Configuration Changes")
-    print("=====================")
+def _report_dict(title, subheading, data, mapper):
+    print(title)
+    print("-" * len(title))
     print()
-
-if added:
-    print("Added")
-    print("-----")
+    print(subheading)
     print()
-    print("The following configuration options are new")
-    print()
-    for fn in added:
+    for fn in data:
         print(fn)
         print('~' * len(fn))
         print()
-        for k in added[fn]:
-            print(f"-  {k}")
+        for k in data[fn]:
+            print(mapper(k))
         print()
     print()
 
-if changed:
-    print("Changed")
-    print("-------")
-    print()
-    print("The following configuration options have been changed")
-    print()
-    for fn in changed:
-        print(fn)
-        print('~' * len(fn))
-        print()
-        for (k, o, n) in changed[fn]:
-            print(f"-  {k} has changed from ``{o}`` to ``{n}``")
-        print()
-    print()
 
-if removed:
-    print("Removed")
-    print("-------")
-    print()
-    print("The following configuration options have been completely removed")
-    print()
-    for fn in removed:
-        print(fn)
-        print('~' * len(fn))
+def report_diff(added, changed, removed, new_files):
+    # Print out report
+    if added or changed or removed:
+        print("Configuration Changes")
+        print("=====================")
         print()
-        for k in removed[fn]:
-            print(f"-  {k}")
-        print()
-    print()
 
-if new_files:
-    print("New Configuration Files")
-    print("-----------------------")
-    print()
-    print(f"The following files are new, or recently converted to yaml since the {old_version}")
-    print()
-    for k in new_files:
-        print(f"-  ``{k}``")
+    if added:
+        _report_dict("Added", "The following configuration options are new", added, lambda x: f'-  {x}')
+
+    if changed:
+        _report_dict("Changed", "The following configuration options have been changed", changed, lambda x: f'-  {x[0]} has changed from ``{x[1]}`` to ``{x[2]}``')
+
+    if removed:
+        _report_dict("Removed", "The following configuration options have been completely removed", removed, lambda x: f'-  {x}')
+
+    if new_files:
+        print("New Configuration Files")
+        print("-----------------------")
+        print()
+        print(f"The following files are new, or recently converted to yaml")
+        print()
+        for k in new_files:
+            print(f"-  ``{k}``")
+
+
+def load_at_time(path, revision=None):
+    if revision is not None:
+        return subprocess.check_output(['git', 'show', f'{revision}:{path}'], stderr=subprocess.STDOUT)
+    else:
+        with open(path, 'r') as handle:
+            return handle.read()
+
+
+def main(old_revision, new_revision=None):
+    files_to_diff = glob.glob('config/*.yml.sample')
+    added = {}
+    removed = {}
+    changed = {}
+    new_files = []
+
+    for file in files_to_diff:
+        real_path = Path(file).resolve().relative_to(Path.cwd())
+        try:
+            old_contents = yaml.load(load_at_time(real_path, old_revision), Loader=MockOrderedLoader)
+            new_contents = yaml.load(load_at_time(real_path, new_revision), Loader=MockOrderedLoader)
+
+            (a, r, c) = diff_files(old_contents, new_contents)
+            if a:
+                added[file] = a
+
+            if r:
+                removed[file] = r
+
+            if c:
+                changed[file] = c
+
+        except subprocess.CalledProcessError:
+            new_files.append(file)
+
+    report_diff(added, removed, changed, new_files)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Diff yaml configuration files between two points in time.')
+    parser.add_argument("old_revision", help="Old revision")
+    parser.add_argument("--new_revision", help="New revision (defaults to whatever is currently in tree)")
+    args = parser.parse_args()
+    main(args.old_revision, args.new_revision)

From 01a2aa4bd9d75c4b3e5146d95ab190185fd8f056 Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Tue, 22 Sep 2020 12:01:07 +0200
Subject: [PATCH 5/7] small bug

---
 scripts/release-diff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scripts/release-diff.py b/scripts/release-diff.py
index ba064177b80..b1a635bb234 100644
--- a/scripts/release-diff.py
+++ b/scripts/release-diff.py
@@ -134,7 +134,7 @@ def main(old_revision, new_revision=None):
         except subprocess.CalledProcessError:
             new_files.append(file)
 
-    report_diff(added, removed, changed, new_files)
+    report_diff(added, changed, removed, new_files)
 
 
 if __name__ == '__main__':

From cac94a0f24549f47f05486f33d977352ae7f6eaf Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Tue, 22 Sep 2020 12:06:06 +0200
Subject: [PATCH 6/7] output

---
 doc/source/releases/20.09-config.rst | 30 +++++++++++++++
 scripts/release-diff.py              | 56 +++++++++++++++++++---------
 2 files changed, 68 insertions(+), 18 deletions(-)
 create mode 100644 doc/source/releases/20.09-config.rst

diff --git a/doc/source/releases/20.09-config.rst b/doc/source/releases/20.09-config.rst
new file mode 100644
index 00000000000..be7cb7d3852
--- /dev/null
+++ b/doc/source/releases/20.09-config.rst
@@ -0,0 +1,30 @@
+Configuration Changes
+=====================
+
+Changed
+-------
+
+The following configuration options have been changed
+
+config/tool_shed.yml.sample
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-  uwsgi.static-safe has changed from ``client/galaxy/images`` to ``client/src/assets``
+
+config/galaxy.yml.sample
+~~~~~~~~~~~~~~~~~~~~~~~~
+
+-  uwsgi.static-safe has changed from ``client/galaxy/images`` to ``client/src/assets``
+
+config/reports.yml.sample
+~~~~~~~~~~~~~~~~~~~~~~~~~
+
+-  uwsgi.static-safe has changed from ``client/galaxy/images`` to ``client/src/assets``
+
+
+New Configuration Files
+-----------------------
+
+The following files are new, or recently converted to yaml
+
+-  ``config/trs_servers_conf.yml.sample``
diff --git a/scripts/release-diff.py b/scripts/release-diff.py
index b1a635bb234..50a9918bc0f 100644
--- a/scripts/release-diff.py
+++ b/scripts/release-diff.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 import subprocess
 import argparse
-import sys
 from pathlib import Path
 import glob
 import yaml
@@ -19,10 +18,6 @@ def flatten(d, path):
     if isinstance(d, dict):
         for k, v in d.items():
             yield from flatten(v, path + [k])
-    # elif isinstance(d, list):
-        # yield ('.'.join(path), d)
-        # for i, x in enumerate(d):
-            # yield from flatten(x, path + [i])
     else:
         yield (".".join(path), d)
 
@@ -39,7 +34,6 @@ class MockOrderedLoader(SafeLoader):
 
 MockOrderedLoader.add_constructor("!include", MockOrderedLoader.include)
 
-# git show v20.01:lib/galaxy/config/sample/galaxy.yml.sample
 
 def diff_files(old, new):
     # Flatten them
@@ -66,7 +60,7 @@ def _report_dict(title, subheading, data, mapper):
     print()
     for fn in data:
         print(fn)
-        print('~' * len(fn))
+        print("~" * len(fn))
         print()
         for k in data[fn]:
             print(mapper(k))
@@ -82,13 +76,28 @@ def report_diff(added, changed, removed, new_files):
         print()
 
     if added:
-        _report_dict("Added", "The following configuration options are new", added, lambda x: f'-  {x}')
+        _report_dict(
+            "Added",
+            "The following configuration options are new",
+            added,
+            lambda x: f"-  {x}",
+        )
 
     if changed:
-        _report_dict("Changed", "The following configuration options have been changed", changed, lambda x: f'-  {x[0]} has changed from ``{x[1]}`` to ``{x[2]}``')
+        _report_dict(
+            "Changed",
+            "The following configuration options have been changed",
+            changed,
+            lambda x: f"-  {x[0]} has changed from ``{x[1]}`` to ``{x[2]}``",
+        )
 
     if removed:
-        _report_dict("Removed", "The following configuration options have been completely removed", removed, lambda x: f'-  {x}')
+        _report_dict(
+            "Removed",
+            "The following configuration options have been completely removed",
+            removed,
+            lambda x: f"-  {x}",
+        )
 
     if new_files:
         print("New Configuration Files")
@@ -102,14 +111,16 @@ def report_diff(added, changed, removed, new_files):
 
 def load_at_time(path, revision=None):
     if revision is not None:
-        return subprocess.check_output(['git', 'show', f'{revision}:{path}'], stderr=subprocess.STDOUT)
+        return subprocess.check_output(
+            ["git", "show", f"{revision}:{path}"], stderr=subprocess.STDOUT
+        )
     else:
-        with open(path, 'r') as handle:
+        with open(path, "r") as handle:
             return handle.read()
 
 
 def main(old_revision, new_revision=None):
-    files_to_diff = glob.glob('config/*.yml.sample')
+    files_to_diff = glob.glob("config/*.yml.sample")
     added = {}
     removed = {}
     changed = {}
@@ -118,8 +129,12 @@ def main(old_revision, new_revision=None):
     for file in files_to_diff:
         real_path = Path(file).resolve().relative_to(Path.cwd())
         try:
-            old_contents = yaml.load(load_at_time(real_path, old_revision), Loader=MockOrderedLoader)
-            new_contents = yaml.load(load_at_time(real_path, new_revision), Loader=MockOrderedLoader)
+            old_contents = yaml.load(
+                load_at_time(real_path, old_revision), Loader=MockOrderedLoader
+            )
+            new_contents = yaml.load(
+                load_at_time(real_path, new_revision), Loader=MockOrderedLoader
+            )
 
             (a, r, c) = diff_files(old_contents, new_contents)
             if a:
@@ -137,9 +152,14 @@ def main(old_revision, new_revision=None):
     report_diff(added, changed, removed, new_files)
 
 
-if __name__ == '__main__':
-    parser = argparse.ArgumentParser(description='Diff yaml configuration files between two points in time.')
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Diff yaml configuration files between two points in time."
+    )
     parser.add_argument("old_revision", help="Old revision")
-    parser.add_argument("--new_revision", help="New revision (defaults to whatever is currently in tree)")
+    parser.add_argument(
+        "--new_revision",
+        help="New revision (defaults to whatever is currently in tree)",
+    )
     args = parser.parse_args()
     main(args.old_revision, args.new_revision)

From 483b21d9066e1a96573489c6e90017ffa90bb457 Mon Sep 17 00:00:00 2001
From: Helena Rasche <hxr@hx42.org>
Date: Tue, 22 Sep 2020 12:10:04 +0200
Subject: [PATCH 7/7] remove trailing commas

---
 scripts/release-diff.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/scripts/release-diff.py b/scripts/release-diff.py
index 50a9918bc0f..b90536312dc 100644
--- a/scripts/release-diff.py
+++ b/scripts/release-diff.py
@@ -80,7 +80,7 @@ def report_diff(added, changed, removed, new_files):
             "Added",
             "The following configuration options are new",
             added,
-            lambda x: f"-  {x}",
+            lambda x: f"-  {x}"
         )
 
     if changed:
@@ -88,7 +88,7 @@ def report_diff(added, changed, removed, new_files):
             "Changed",
             "The following configuration options have been changed",
             changed,
-            lambda x: f"-  {x[0]} has changed from ``{x[1]}`` to ``{x[2]}``",
+            lambda x: f"-  {x[0]} has changed from ``{x[1]}`` to ``{x[2]}``"
         )
 
     if removed:
@@ -96,7 +96,7 @@ def report_diff(added, changed, removed, new_files):
             "Removed",
             "The following configuration options have been completely removed",
             removed,
-            lambda x: f"-  {x}",
+            lambda x: f"-  {x}"
         )
 
     if new_files: