Factor out yaml processing out of config_manage

- Create new utility module for processing yaml
- Move yaml processing into new utility module
- Factor out Schema classes out of config_manage into schema module
- Refactor yaml reading to accommodate a unit test (minor change)
- Add unit test for new schema module

Overall: this is first step towards refactoring config.

lib/galaxy/config/config_manage.py

@@ -25,8 +25,15 @@ except ImportError:
 if __name__ == '__main__':
     sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir)))
 
+from galaxy.config.schema import AppSchema, Schema
 from galaxy.util import safe_makedirs
 from galaxy.util.properties import nice_config_parser
+from galaxy.util.yaml_util import (
+    ordered_dump,
+    ordered_load,
+    OPTION_DEFAULTS,
+)
+
 
 DESCRIPTION = "Convert configuration files."
 
@@ -311,63 +318,6 @@ App.schema = property(_schema)
 OptionValue = namedtuple("OptionValue", ["name", "value", "option"])
 
 
-UNKNOWN_OPTION = {
-    "type": "str",
-    "required": False,
-    "unknown_option": True,
-    "desc": "Unknown option, may want to remove or report to Galaxy team."
-}
-
-OPTION_DEFAULTS = {
-    "type": "str",
-    "unknown_option": False,
-    "default": None,
-    "desc": None,
-}
-
-
-class Schema(object):
-
-    def __init__(self, mapping):
-        self.app_schema = mapping
-
-    def get_app_option(self, name):
-        try:
-            raw_option = self.app_schema[name]
-        except KeyError:
-            raw_option = UNKNOWN_OPTION
-        option = OPTION_DEFAULTS.copy()
-        option.update(raw_option)
-        return option
-
-
-class AppSchema(Schema):
-
-    def __init__(self, app_desc):
-        schema_path = app_desc.schema_path
-        app_name = app_desc.app_name
-        with open(schema_path, "r") as f:
-            config_all = _ordered_load(f)
-        self.raw_schema = config_all
-        app_schema = config_all["mapping"][app_name]
-        super(AppSchema, self).__init__(app_schema["mapping"])
-        self.description = config_all.get("desc", None)
-        self.reloadable_options = self._load_reloadable_options(app_schema["mapping"])
-
-    def get_reloadable_option_defaults(self):
-        option_dict = {}
-        for key in self.reloadable_options:
-            option_dict[key] = self.get_app_option(key)["default"]
-        return option_dict
-
-    def _load_reloadable_options(self, mapping):
-        reloadable_options = []
-        for key, option in mapping.items():
-            if option.get("reloadable", False):
-                reloadable_options.append(key)
-        return reloadable_options
-
-
 GALAXY_APP = App(
     ["universe_wsgi.ini", "config/galaxy.ini"],
     "8080",
@@ -490,7 +440,7 @@ def _build_uwsgi_schema(args, app_desc):
         "mapping": options
     }
     path = os.path.join(args.galaxy_root, UWSGI_SCHEMA_PATH)
-    contents = _ordered_dump(schema)
+    contents = ordered_dump(schema)
     _write_to_file(args, contents, path)
 
 
@@ -553,12 +503,12 @@ def _validate(args, app_desc):
     if raw_config.get(app_desc.app_name, None) is None:
         raw_config[app_desc.app_name] = {}
         config_p = tempfile.NamedTemporaryFile(delete=False, suffix=".yml")
-        _ordered_dump(raw_config, config_p)
+        ordered_dump(raw_config, config_p)
         config_p.flush()
         path = config_p.name
 
     fp = tempfile.NamedTemporaryFile(delete=False, suffix=".yml")
-    _ordered_dump(app_desc.schema.raw_schema, fp)
+    ordered_dump(app_desc.schema.raw_schema, fp)
     fp.flush()
     name = fp.name
     if Core is None:
@@ -734,7 +684,7 @@ def _order_load_path(path):
     """Load (with ``_ordered_load``) on specified path (a YAML file)."""
     with open(path, "r") as f:
         # Allow empty mapping (not allowed by pykawlify)
-        raw_config = _ordered_load(f)
+        raw_config = ordered_load(f)
         return raw_config
 
 
@@ -829,43 +779,6 @@ def _warn(message):
     print("WARNING: %s" % message)
 
 
-def _ordered_load(stream):
-
-    class OrderedLoader(yaml.Loader):
-
-        def __init__(self, stream):
-            self._root = os.path.split(stream.name)[0]
-            super(OrderedLoader, self).__init__(stream)
-
-        def include(self, node):
-            filename = os.path.join(self._root, self.construct_scalar(node))
-            with open(filename, 'r') as f:
-                return yaml.load(f, OrderedLoader)
-
-    def construct_mapping(loader, node):
-        loader.flatten_mapping(node)
-        return OrderedDict(loader.construct_pairs(node))
-
-    OrderedLoader.add_constructor(
-        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-        construct_mapping)
-    OrderedLoader.add_constructor('!include', OrderedLoader.include)
-
-    return yaml.load(stream, OrderedLoader)
-
-
-def _ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds):
-    class OrderedDumper(Dumper):
-        pass
-
-    def _dict_representer(dumper, data):
-        return dumper.represent_mapping(
-            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
-            list(data.items()))
-    OrderedDumper.add_representer(OrderedDict, _dict_representer)
-    return yaml.dump(data, stream, OrderedDumper, **kwds)
-
-
 ACTIONS = {
     "convert": _run_conversion,
     "build_sample_yaml": _build_sample_yaml,

lib/galaxy/config/schema.py

@@ -0,0 +1,54 @@
+from galaxy.util.yaml_util import (
+    ordered_load,
+    OPTION_DEFAULTS,
+)
+
+
+UNKNOWN_OPTION = {
+    "type": "str",
+    "required": False,
+    "unknown_option": True,
+    "desc": "Unknown option, may want to remove or report to Galaxy team."
+}
+
+
+class Schema(object):
+
+    def __init__(self, mapping):
+        self.app_schema = mapping
+
+    def get_app_option(self, name):
+        try:
+            raw_option = self.app_schema[name]
+        except KeyError:
+            raw_option = UNKNOWN_OPTION
+        option = OPTION_DEFAULTS.copy()
+        option.update(raw_option)
+        return option
+
+
+class AppSchema(Schema):
+
+    def __init__(self, app_desc):
+        self.raw_schema = self._read_schema(app_desc.schema_path)
+        self.description = self.raw_schema.get("desc", None)
+        app_schema = self.raw_schema['mapping'][app_desc.app_name]['mapping']
+        super(AppSchema, self).__init__(app_schema)
+        self.reloadable_options = self._load_reloadable_options(app_schema)  # TODO redo
+
+    def _read_schema(self, path):
+        with open(path, "r") as f:
+            return ordered_load(f)
+
+    def get_reloadable_option_defaults(self):  # TODO redo
+        option_dict = {}
+        for key in self.reloadable_options:
+            option_dict[key] = self.get_app_option(key)["default"]
+        return option_dict
+
+    def _load_reloadable_options(self, mapping):  # TODO redo
+        reloadable_options = []
+        for key, option in mapping.items():
+            if option.get("reloadable", False):
+                reloadable_options.append(key)
+        return reloadable_options

lib/galaxy/util/yaml_util.py

@@ -0,0 +1,49 @@
+import os
+import yaml
+from collections import OrderedDict
+
+
+OPTION_DEFAULTS = {
+    "type": "str",
+    "unknown_option": False,
+    "default": None,
+    "desc": None,
+}
+
+
+def ordered_load(stream):
+    def construct_mapping(loader, node):
+        loader.flatten_mapping(node)
+        return OrderedDict(loader.construct_pairs(node))
+
+    OrderedLoader.add_constructor(
+        yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+        construct_mapping)
+    OrderedLoader.add_constructor('!include', OrderedLoader.include)
+
+    return yaml.load(stream, OrderedLoader)
+
+
+class OrderedLoader(yaml.Loader):
+    # This class was pulled out of ordered_load() for the sake of
+    # mocking __init__ in a unit test.
+    def __init__(self, stream):
+        self._root = os.path.split(stream.name)[0]
+        super(OrderedLoader, self).__init__(stream)
+
+    def include(self, node):
+        filename = os.path.join(self._root, self.construct_scalar(node))
+        with open(filename, 'r') as f:
+            return yaml.load(f, OrderedLoader)
+
+
+def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds):
+    class OrderedDumper(Dumper):
+        pass
+
+    def _dict_representer(dumper, data):
+        return dumper.represent_mapping(
+            yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
+            list(data.items()))
+    OrderedDumper.add_representer(OrderedDict, _dict_representer)
+    return yaml.dump(data, stream, OrderedDumper, **kwds)

test/unit/config/test_schema.py

@@ -0,0 +1,43 @@
+from galaxy.config.schema import AppSchema
+from galaxy.util.yaml_util import (
+    ordered_load,
+    OrderedLoader,
+)
+
+MOCK_YAML = '''
+    type: map
+    desc: mocked schema
+    foo: bar
+    mapping:
+      mockgalaxy:
+        type: map
+        mapping:
+          option:
+            attr1: a
+            attr2: b
+    '''
+
+
+class MockGalaxyApp():
+    app_name = 'mockgalaxy'
+    schema_path = None
+
+
+def test_schema_is_loaded(monkeypatch):
+
+    def mock_read_schema(self, path):
+        return ordered_load(MOCK_YAML)
+
+    def mock_init(self, stream):
+        super(OrderedLoader, self).__init__(stream)
+
+    monkeypatch.setattr(AppSchema, '_read_schema', mock_read_schema)
+    monkeypatch.setattr(OrderedLoader, '__init__', mock_init)
+
+    loaded_schema = AppSchema(MockGalaxyApp())
+    data = ordered_load(MOCK_YAML)
+
+    assert loaded_schema.description == data['desc']
+    assert loaded_schema.raw_schema['foo'] == 'bar'
+    assert loaded_schema.app_schema['option']['attr1'] == 'a'
+    assert loaded_schema.get_app_option('option')['attr2'] == 'b'