diff --git a/lib/galaxy/config/config_manage.py b/lib/galaxy/config/config_manage.py index 4e84debca86..26b3ed3de31 100644 --- a/lib/galaxy/config/config_manage.py +++ b/lib/galaxy/config/config_manage.py @@ -25,8 +25,15 @@ except ImportError: if __name__ == '__main__': sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), os.pardir, os.pardir))) +from galaxy.config.schema import AppSchema, Schema from galaxy.util import safe_makedirs from galaxy.util.properties import nice_config_parser +from galaxy.util.yaml_util import ( + ordered_dump, + ordered_load, + OPTION_DEFAULTS, +) + DESCRIPTION = "Convert configuration files." @@ -311,63 +318,6 @@ App.schema = property(_schema) OptionValue = namedtuple("OptionValue", ["name", "value", "option"]) -UNKNOWN_OPTION = { - "type": "str", - "required": False, - "unknown_option": True, - "desc": "Unknown option, may want to remove or report to Galaxy team." -} - -OPTION_DEFAULTS = { - "type": "str", - "unknown_option": False, - "default": None, - "desc": None, -} - - -class Schema(object): - - def __init__(self, mapping): - self.app_schema = mapping - - def get_app_option(self, name): - try: - raw_option = self.app_schema[name] - except KeyError: - raw_option = UNKNOWN_OPTION - option = OPTION_DEFAULTS.copy() - option.update(raw_option) - return option - - -class AppSchema(Schema): - - def __init__(self, app_desc): - schema_path = app_desc.schema_path - app_name = app_desc.app_name - with open(schema_path, "r") as f: - config_all = _ordered_load(f) - self.raw_schema = config_all - app_schema = config_all["mapping"][app_name] - super(AppSchema, self).__init__(app_schema["mapping"]) - self.description = config_all.get("desc", None) - self.reloadable_options = self._load_reloadable_options(app_schema["mapping"]) - - def get_reloadable_option_defaults(self): - option_dict = {} - for key in self.reloadable_options: - option_dict[key] = self.get_app_option(key)["default"] - return option_dict - - def _load_reloadable_options(self, mapping): - reloadable_options = [] - for key, option in mapping.items(): - if option.get("reloadable", False): - reloadable_options.append(key) - return reloadable_options - - GALAXY_APP = App( ["universe_wsgi.ini", "config/galaxy.ini"], "8080", @@ -490,7 +440,7 @@ def _build_uwsgi_schema(args, app_desc): "mapping": options } path = os.path.join(args.galaxy_root, UWSGI_SCHEMA_PATH) - contents = _ordered_dump(schema) + contents = ordered_dump(schema) _write_to_file(args, contents, path) @@ -553,12 +503,12 @@ def _validate(args, app_desc): if raw_config.get(app_desc.app_name, None) is None: raw_config[app_desc.app_name] = {} config_p = tempfile.NamedTemporaryFile(delete=False, suffix=".yml") - _ordered_dump(raw_config, config_p) + ordered_dump(raw_config, config_p) config_p.flush() path = config_p.name fp = tempfile.NamedTemporaryFile(delete=False, suffix=".yml") - _ordered_dump(app_desc.schema.raw_schema, fp) + ordered_dump(app_desc.schema.raw_schema, fp) fp.flush() name = fp.name if Core is None: @@ -734,7 +684,7 @@ def _order_load_path(path): """Load (with ``_ordered_load``) on specified path (a YAML file).""" with open(path, "r") as f: # Allow empty mapping (not allowed by pykawlify) - raw_config = _ordered_load(f) + raw_config = ordered_load(f) return raw_config @@ -829,43 +779,6 @@ def _warn(message): print("WARNING: %s" % message) -def _ordered_load(stream): - - class OrderedLoader(yaml.Loader): - - def __init__(self, stream): - self._root = os.path.split(stream.name)[0] - super(OrderedLoader, self).__init__(stream) - - def include(self, node): - filename = os.path.join(self._root, self.construct_scalar(node)) - with open(filename, 'r') as f: - return yaml.load(f, OrderedLoader) - - def construct_mapping(loader, node): - loader.flatten_mapping(node) - return OrderedDict(loader.construct_pairs(node)) - - OrderedLoader.add_constructor( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - construct_mapping) - OrderedLoader.add_constructor('!include', OrderedLoader.include) - - return yaml.load(stream, OrderedLoader) - - -def _ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): - class OrderedDumper(Dumper): - pass - - def _dict_representer(dumper, data): - return dumper.represent_mapping( - yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, - list(data.items())) - OrderedDumper.add_representer(OrderedDict, _dict_representer) - return yaml.dump(data, stream, OrderedDumper, **kwds) - - ACTIONS = { "convert": _run_conversion, "build_sample_yaml": _build_sample_yaml, diff --git a/lib/galaxy/config/schema.py b/lib/galaxy/config/schema.py new file mode 100644 index 00000000000..21f45ef0031 --- /dev/null +++ b/lib/galaxy/config/schema.py @@ -0,0 +1,54 @@ +from galaxy.util.yaml_util import ( + ordered_load, + OPTION_DEFAULTS, +) + + +UNKNOWN_OPTION = { + "type": "str", + "required": False, + "unknown_option": True, + "desc": "Unknown option, may want to remove or report to Galaxy team." +} + + +class Schema(object): + + def __init__(self, mapping): + self.app_schema = mapping + + def get_app_option(self, name): + try: + raw_option = self.app_schema[name] + except KeyError: + raw_option = UNKNOWN_OPTION + option = OPTION_DEFAULTS.copy() + option.update(raw_option) + return option + + +class AppSchema(Schema): + + def __init__(self, app_desc): + self.raw_schema = self._read_schema(app_desc.schema_path) + self.description = self.raw_schema.get("desc", None) + app_schema = self.raw_schema['mapping'][app_desc.app_name]['mapping'] + super(AppSchema, self).__init__(app_schema) + self.reloadable_options = self._load_reloadable_options(app_schema) # TODO redo + + def _read_schema(self, path): + with open(path, "r") as f: + return ordered_load(f) + + def get_reloadable_option_defaults(self): # TODO redo + option_dict = {} + for key in self.reloadable_options: + option_dict[key] = self.get_app_option(key)["default"] + return option_dict + + def _load_reloadable_options(self, mapping): # TODO redo + reloadable_options = [] + for key, option in mapping.items(): + if option.get("reloadable", False): + reloadable_options.append(key) + return reloadable_options diff --git a/lib/galaxy/util/yaml_util.py b/lib/galaxy/util/yaml_util.py new file mode 100644 index 00000000000..ed5f1b8795c --- /dev/null +++ b/lib/galaxy/util/yaml_util.py @@ -0,0 +1,49 @@ +import os +import yaml +from collections import OrderedDict + + +OPTION_DEFAULTS = { + "type": "str", + "unknown_option": False, + "default": None, + "desc": None, +} + + +def ordered_load(stream): + def construct_mapping(loader, node): + loader.flatten_mapping(node) + return OrderedDict(loader.construct_pairs(node)) + + OrderedLoader.add_constructor( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + construct_mapping) + OrderedLoader.add_constructor('!include', OrderedLoader.include) + + return yaml.load(stream, OrderedLoader) + + +class OrderedLoader(yaml.Loader): + # This class was pulled out of ordered_load() for the sake of + # mocking __init__ in a unit test. + def __init__(self, stream): + self._root = os.path.split(stream.name)[0] + super(OrderedLoader, self).__init__(stream) + + def include(self, node): + filename = os.path.join(self._root, self.construct_scalar(node)) + with open(filename, 'r') as f: + return yaml.load(f, OrderedLoader) + + +def ordered_dump(data, stream=None, Dumper=yaml.Dumper, **kwds): + class OrderedDumper(Dumper): + pass + + def _dict_representer(dumper, data): + return dumper.represent_mapping( + yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, + list(data.items())) + OrderedDumper.add_representer(OrderedDict, _dict_representer) + return yaml.dump(data, stream, OrderedDumper, **kwds) diff --git a/test/unit/config/test_schema.py b/test/unit/config/test_schema.py new file mode 100644 index 00000000000..bf47defa942 --- /dev/null +++ b/test/unit/config/test_schema.py @@ -0,0 +1,43 @@ +from galaxy.config.schema import AppSchema +from galaxy.util.yaml_util import ( + ordered_load, + OrderedLoader, +) + +MOCK_YAML = ''' + type: map + desc: mocked schema + foo: bar + mapping: + mockgalaxy: + type: map + mapping: + option: + attr1: a + attr2: b + ''' + + +class MockGalaxyApp(): + app_name = 'mockgalaxy' + schema_path = None + + +def test_schema_is_loaded(monkeypatch): + + def mock_read_schema(self, path): + return ordered_load(MOCK_YAML) + + def mock_init(self, stream): + super(OrderedLoader, self).__init__(stream) + + monkeypatch.setattr(AppSchema, '_read_schema', mock_read_schema) + monkeypatch.setattr(OrderedLoader, '__init__', mock_init) + + loaded_schema = AppSchema(MockGalaxyApp()) + data = ordered_load(MOCK_YAML) + + assert loaded_schema.description == data['desc'] + assert loaded_schema.raw_schema['foo'] == 'bar' + assert loaded_schema.app_schema['option']['attr1'] == 'a' + assert loaded_schema.get_app_option('option')['attr2'] == 'b'