diff --git a/lib/galaxy/exceptions/__init__.py b/lib/galaxy/exceptions/__init__.py index 9f47371b36d..cac23f38ea2 100644 --- a/lib/galaxy/exceptions/__init__.py +++ b/lib/galaxy/exceptions/__init__.py @@ -203,9 +203,13 @@ class NotImplemented(MessageException): err_code = error_codes.NOT_IMPLEMENTED -# non-web exceptions +class InvalidFileFormatError(MessageException): + status_code = 500 + err_code = error_codes.INVALID_FILE_FORMAT +# non-web exceptions + class ContainerCLIError(Exception): def __init__(self, msg=None, stdout=None, stderr=None, returncode=None, command=None, subprocess_command=None, **kwargs): diff --git a/lib/galaxy/exceptions/error_codes.json b/lib/galaxy/exceptions/error_codes.json index 5b8c2bbce2a..382d95155cc 100644 --- a/lib/galaxy/exceptions/error_codes.json +++ b/lib/galaxy/exceptions/error_codes.json @@ -149,6 +149,11 @@ "code": 500004, "message": "Tool execution failed due to an internal server error." }, + { + "name": "INVALID_FILE_FORMAT", + "code": 500005, + "message": "File format not supported for this operation." + }, { "name": "NOT_IMPLEMENTED", "code": 501001, diff --git a/lib/galaxy/util/properties.py b/lib/galaxy/util/properties.py index 7cc131145ac..0ccd352f087 100644 --- a/lib/galaxy/util/properties.py +++ b/lib/galaxy/util/properties.py @@ -12,6 +12,7 @@ import yaml from six import iteritems, string_types from six.moves.configparser import ConfigParser +from galaxy.exceptions import InvalidFileFormatError from galaxy.util.path import extensions, has_ext, joinext @@ -63,38 +64,24 @@ def find_config_file(names, exts=None, dirs=None, include_samples=False): def load_app_properties( - kwds={}, + kwds=None, ini_file=None, ini_section=None, config_file=None, config_section=None, config_prefix="GALAXY_CONFIG_" ): - properties = kwds.copy() if kwds else {} - properties['__file__'] = None if config_file is None: config_file = ini_file config_section = ini_section + # read from file or init w/no file if config_file: - properties['__file__'] = os.path.abspath(config_file) - if not has_ext(config_file, 'yaml', aliases=True, ignore='sample'): - if config_section is None: - config_section = "app:main" - parser = nice_config_parser(config_file) - if parser.has_section(config_section): - properties.update(dict(parser.items(config_section))) - else: - properties.update(parser.defaults()) - else: - if config_section is None: - config_section = "galaxy" - - with open(config_file, "r") as f: - raw_properties = yaml.safe_load(f) - properties = __default_properties(config_file) - properties.update(raw_properties.get(config_section) or {}) + properties = read_properties_from_file(config_file, config_section) + else: + properties = {'__file__': None} + # update from env override_prefix = "%sOVERRIDE_" % config_prefix for key in os.environ: if key.startswith(override_prefix): @@ -105,9 +92,40 @@ def load_app_properties( if config_key not in properties: properties[config_key] = os.environ[key] + # update from kwds + if kwds: + properties.update(kwds) + return properties +def read_properties_from_file(config_file, config_section=None): + properties = {} + if has_ext(config_file, 'yaml', aliases=True, ignore='sample'): + if config_section is None: + config_section = "galaxy" + properties.update(__default_properties(config_file)) + raw_properties = _read_from_yaml_file(config_file) + if raw_properties: + properties.update(raw_properties.get(config_section) or {}) + elif has_ext(config_file, 'ini', aliases=True, ignore='sample'): + if config_section is None: + config_section = "app:main" + parser = nice_config_parser(config_file) # default properties loaded w/parser + if parser.has_section(config_section): + properties.update(dict(parser.items(config_section))) + else: + properties.update(parser.defaults()) + else: + raise InvalidFileFormatError() + return properties + + +def _read_from_yaml_file(path): + with open(path, "r") as f: + return yaml.safe_load(f) + + def nice_config_parser(path): parser = NicerConfigParser(path, defaults=__default_properties(path)) parser.optionxform = str # Don't lower-case keys diff --git a/test/unit/util/test_properties.py b/test/unit/util/test_properties.py new file mode 100644 index 00000000000..6a0cfc61bd6 --- /dev/null +++ b/test/unit/util/test_properties.py @@ -0,0 +1,115 @@ +import pytest + +from galaxy.exceptions import InvalidFileFormatError +from galaxy.util import properties +from galaxy.util.properties import read_properties_from_file + +KEY1, KEY2, KEY3, KEY4, KEY5, KEY6 = 'k1', 'k2', 'k3', 'k4', 'k5', 'k6' +VAL1, VAL2, VAL3, VAL4, VAL5, VAL6 = 1, 2, 3, 4, 5, 6 +OTHER_SECTION = 'other' + + +@pytest.fixture +def mock_properties(monkeypatch): + # Keys in defaults dict and other dicts should not intersect. The tests assume + # that len(dict1.update(dict2)) == len(dict1) + len(dict2). So, for example, if + # defaults = {foo: 1, bar:2} and other_section = {foo:3}, the final dict will have len=2. + def mock_default_properties(path): + return {KEY1: VAL1, KEY2: VAL2} + + def mock_read_from_yaml(path): + return { + 'galaxy': {KEY3: VAL3, KEY4: VAL4}, + OTHER_SECTION: {KEY5: VAL5, KEY6: VAL6} + } + + class MockConfigParser(): + def __init__(self): + self._items = { + 'app:main': [(KEY3, VAL3), (KEY4, VAL4)], + OTHER_SECTION: [(KEY5, VAL5), (KEY6, VAL6)] + } + + def has_section(self, section): + return section in self._items.keys() + + def items(self, section): + return self._items[section] + self.defaults() + + def defaults(self): + return [(KEY1, VAL1), (KEY2, VAL2)] + + def mock_nice_config_parser(path): + return MockConfigParser() + + monkeypatch.setattr(properties, '_read_from_yaml_file', mock_read_from_yaml) + monkeypatch.setattr(properties, 'nice_config_parser', mock_nice_config_parser) + monkeypatch.setattr(properties, '__default_properties', mock_default_properties) + + +def test_read_galaxy_properties_from_yaml(mock_properties): + file = 'foo.yaml' + result = read_properties_from_file(file) + + assert len(result) == 4 + assert result[KEY1] == VAL1 + assert result[KEY2] == VAL2 + assert result[KEY3] == VAL3 + assert result[KEY4] == VAL4 + + +def test_read_other_properties_from_yaml(mock_properties): + file = 'foo.yaml' + result = read_properties_from_file(file, config_section=OTHER_SECTION) + + assert len(result) == 4 + assert result[KEY1] == VAL1 + assert result[KEY2] == VAL2 + assert result[KEY5] == VAL5 + assert result[KEY6] == VAL6 + + +def test_get_default_properties_from_yaml(mock_properties): + file = 'foo.yaml' + result = read_properties_from_file(file, config_section="section's not here, man") + + assert len(result) == 2 + assert result[KEY1] == VAL1 + assert result[KEY2] == VAL2 + + +def test_read_galaxy_properties_from_ini(mock_properties): + file = 'foo.ini' + result = read_properties_from_file(file) + + assert len(result) == 4 + assert result[KEY1] == VAL1 + assert result[KEY2] == VAL2 + assert result[KEY3] == VAL3 + assert result[KEY4] == VAL4 + + +def test_read_other_properties_from_ini(mock_properties): + file = 'foo.ini' + result = read_properties_from_file(file, config_section=OTHER_SECTION) + + assert len(result) == 4 + assert result[KEY1] == VAL1 + assert result[KEY2] == VAL2 + assert result[KEY5] == VAL5 + assert result[KEY6] == VAL6 + + +def test_get_default_properties_from_ini(mock_properties): + file = 'foo.ini' + result = read_properties_from_file(file, config_section="section's not here, man") + + assert len(result) == 2 + assert result[KEY1] == VAL1 + assert result[KEY2] == VAL2 + + +def test_dont_read_properties_from_other_format(mock_properties): + file = 'invalid' + with pytest.raises(InvalidFileFormatError): + read_properties_from_file(file) diff --git a/test/unit/test_utils.py b/test/unit/util/test_utils.py similarity index 100% rename from test/unit/test_utils.py rename to test/unit/util/test_utils.py