Mercurial > hgrepos > Python > libs > ConfigMix
changeset 144:7e6ec99d5ff5
Allow comments as keys and filter them by default
| author | Franz Glasner <hg@dom66.de> |
|---|---|
| date | Fri, 13 Apr 2018 09:51:02 +0200 |
| parents | 252645c69c7b |
| children | d44ee758e31b |
| files | configmix/__init__.py configmix/json.py tests/data/conf1.ini tests/data/conf1.json tests/data/conf1.py tests/data/conf1.yml tests/data/conf20.yml tests/data/conf22.ini tests/data/conf23.json tests/test.py |
| diffstat | 10 files changed, 126 insertions(+), 21 deletions(-) [+] |
line wrap: on
line diff
--- a/configmix/__init__.py Mon Apr 09 09:35:04 2018 +0200 +++ b/configmix/__init__.py Fri Apr 13 09:51:02 2018 +0200 @@ -20,6 +20,7 @@ import copy +from .compat import u from .config import Configuration @@ -28,6 +29,15 @@ "Configuration"] +COMMENTS = [u("__comment"), + u("__doc"), +] +"""Prefixes for comment configuration keys that are to be handled as +comments + +""" + + def load(*files): """Load the given configuration files, merge them in the given order and return the resulting configuration dictionary. @@ -168,7 +178,7 @@ return result -def merge(user, default): +def merge(user, default, filter_comments=True): """Logically merge the configuration in `user` into `default`. :param ~configmix.config.Configuration user: @@ -176,73 +186,120 @@ into `default` :param ~configmix.config.Configuration default: the base configuration where `user` is logically merged into + :param bool filter_comments: flag whether to filter comment keys that + start with any of the items in :data:`COMMENTS` :returns: `user` with the necessary amendments from `default`. If `user` is ``None`` then `default` is returned. - .. note:: The configuration in `default` is not changed but the - configuration given in `user` is changed **inplace**. + .. note:: The configuration in `user` is augmented/changed + **inplace**. + + The configuration in `default` will be changed **inplace** + when filtering out comments (which is the default). From http://stackoverflow.com/questions/823196/yaml-merge-in-python """ if user is None: + if filter_comments: + _filter_comments(default) return default + if filter_comments: + _filter_comments(user) if isinstance(user, dict) and isinstance(default, dict): for k, v in default.items(): + if filter_comments and _is_comment(k): + continue if k not in user: user[k] = v else: - user[k] = _merge(user[k], v) + user[k] = _merge(user[k], v, filter_comments) return user -def _merge(user, default): +def _merge(user, default, filter_comments): """Recursion helper for :meth:`merge` """ if isinstance(user, dict) and isinstance(default, dict): for k, v in default.items(): + if filter_comments and _is_comment(k): + continue if k not in user: user[k] = v else: - user[k] = _merge(user[k], v) + user[k] = _merge(user[k], v, filter_comments) return user -def safe_merge(user, default): +def safe_merge(user, default, filter_comments=True): """A more safe version of :func:`merge` that makes deep copies of the returned container objects. - No given argument is ever changed inplace. Every object from `default` - is decoupled from the result -- so changing the `default` configuration - lates does not yield into a merged configuration later. + Contrary to :func:`merge` no given argument is ever changed + inplace. Every object from `default` is decoupled from the result + -- so changing the `default` configuration later does not yield + into a merged configuration later. """ if user is None: + if filter_comments: + _filter_comments(default) return copy.deepcopy(default) user = copy.deepcopy(user) + if filter_comments: + _filter_comments(user) if isinstance(user, dict) and isinstance(default, dict): for k, v in default.items(): + if filter_comments and _is_comment(k): + continue if k not in user: user[k] = copy.deepcopy(v) else: - user[k] = _safe_merge(user[k], v) + user[k] = _safe_merge(user[k], v, filter_comments) return user -def _safe_merge(user, default): +def _safe_merge(user, default, filter_comments): """Recursion helper for :meth:`safe_merge` """ if isinstance(user, dict) and isinstance(default, dict): for k, v in default.items(): + if filter_comments and _is_comment(k): + continue if k not in user: user[k] = copy.deepcopy(v) else: - user[k] = _safe_merge(user[k], v) + user[k] = _safe_merge(user[k], v, filter_comments) return user +def _filter_comments(d): + """Recursively filter comments keys in the dict `d`. + + Comment keys are keys that start with any of the items in + :data:`COMMENTS`. + + """ + if not isinstance(d, dict): + return + # use a copy of the keys because we change `d` while iterating + for k in list(d.keys()): + if _is_comment(k): + del d[k] + else: + if isinstance(d[k], dict): + _filter_comments(d[k]) + + +def _is_comment(k): + for i in COMMENTS: + if k.startswith(i): + return True + return False + + # # Init loader defaults #
--- a/configmix/json.py Mon Apr 09 09:35:04 2018 +0200 +++ b/configmix/json.py Fri Apr 13 09:51:02 2018 +0200 @@ -38,8 +38,6 @@ def load(filename, encoding="utf-8"): """Load a single JSON file with name `filename` and encoding `encoding`. - .. todo:: Allow comments in JSON files - """ with io.open(filename, mode="rt", encoding=encoding) as jsfp: #
--- a/tests/data/conf1.ini Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/data/conf1.ini Fri Apr 13 09:51:02 2018 +0200 @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- [config] +__comment1 = Comment 1 key1 = the value key2 = :int:2 key3 = :float:5.7 key4 = :bool:yes key5 = :bool:off key6 = :int:0o377 +__comment2 = Comment no 2 key7 = Umlaute: ÄÖÜäöüß
--- a/tests/data/conf1.json Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/data/conf1.json Fri Apr 13 09:51:02 2018 +0200 @@ -1,8 +1,10 @@ -{"key1": "the value", +{"__comment1": "Comment 1", + "key1": "the value", "key2": 2, "key3": 5.7, "key4": true, "key5": false, "key6": 255, + "__comment2": "Comment no 2", "key7": "Umlaute: ÄÖÜäöüß" }
--- a/tests/data/conf1.py Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/data/conf1.py Fri Apr 13 09:51:02 2018 +0200 @@ -1,9 +1,11 @@ # -*- coding: utf-8 -*- +__comment1 = u"Comment 1" key1 = u"the value" key2 = 2 key3 = 5.7 key4 = True key5 = False key6 = 0o377 +__comment2 = u"Comment no 2" key7 = u"Umlaute: ÄÖÜäöüß"
--- a/tests/data/conf1.yml Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/data/conf1.yml Fri Apr 13 09:51:02 2018 +0200 @@ -2,10 +2,12 @@ %YAML 1.1 --- +__comment1: Comment 1 key1: the value key2: 2 key3: 5.7 key4: true key5: false key6: 0377 +__comment2: Comment no 2 key7: 'Umlaute: ÄÖÜäöüß'
--- a/tests/data/conf20.yml Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/data/conf20.yml Fri Apr 13 09:51:02 2018 +0200 @@ -6,6 +6,7 @@ --- process: + __doc1: A first comment umask: 0o0027 intl: @@ -17,8 +18,10 @@ db: + __comment1: A second comment user: name: <My Username> + __doc2: A third comment pwd: <My Password> catalog: <MyDbName>
--- a/tests/data/conf22.ini Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/data/conf22.ini Fri Apr 13 09:51:02 2018 +0200 @@ -3,6 +3,7 @@ [config.db.user] name = the_database_user_2 +__doc2 = Overwriting a comment pwd=the-database-password-2 [config.db.locinfo.ro]
--- a/tests/data/conf23.json Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/data/conf23.json Fri Apr 13 09:51:02 2018 +0200 @@ -2,6 +2,7 @@ "db": { "user": { "name": "the_database_user_2", + "__doc2": "comment", "pwd": "the-database-password-2" }, "locinfo": {
--- a/tests/test.py Mon Apr 09 09:35:04 2018 +0200 +++ b/tests/test.py Fri Apr 13 09:51:02 2018 +0200 @@ -43,6 +43,22 @@ self.assertEqual(u("Umlaute: ÄÖÜäöüß"), cfg.get("key7")) + def __check_comment(self, cfg): + # Check comments: low level comments are *not* filtered + self.assertEqual(u("Comment 1"), cfg.get("__comment1")) + self.assertEqual(u("Comment no 2"), cfg.get("__comment2")) + + def __check_no_comment(self, cfg): + + def _c(name): + def _f(): + cfg[u(name)] + return _f + + # Variables with leading underscores are *not* imported by default + self.assertRaises(KeyError, _c("__comment1")) + self.assertRaises(KeyError, _c("__comment2")) + def __check_tree(self, cfg): self.assertEqual(u("in the root namespace"), cfg.get("key1")) @@ -57,10 +73,12 @@ def test01_ini_types(self): cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini")) self.__check_types(cfg) + self.__check_comment(cfg) def test02_py_types(self): cfg = configmix.py.load(os.path.join(TESTDATADIR, "conf1.py")) self.__check_types(cfg) + self.__check_no_comment(cfg) def test03_yaml_types(self): with io.open(os.path.join(TESTDATADIR, "conf1.yml"), "rt", @@ -69,11 +87,13 @@ if configmix.yaml.OrderedDict: self.assertTrue(isinstance(cfg, configmix.yaml.OrderedDict)) self.__check_types(cfg) + self.__check_comment(cfg) def test04_json_types(self): cfg = configmix.json.load(os.path.join(TESTDATADIR, "conf1.json")) self.assertTrue(isinstance(cfg, configmix.json.DictImpl)) self.__check_types(cfg) + self.__check_comment(cfg) def test05_py_export_all(self): # When __all__ is given only it's keys are exported @@ -218,18 +238,35 @@ self.assertRaises(KeyError, _look) + def test05_comments(self): + cfg = self._load(os.path.join(TESTDATADIR, "conf20.yml"), + os.path.join(TESTDATADIR, "conf21.yml"), + os.path.join(TESTDATADIR, "conf22.ini"), + os.path.join(TESTDATADIR, "conf23.json")) + + def _c(name): + def _f(): + cfg.getvar_s(name) + return _f + + # Variables with leading underscores are *not* imported by default + self.assertEqual(0o0027, int(cfg.getvar_s("process.umask"), 0)) + self.assertRaises(KeyError, _c("process.__doc1")) + self.assertRaises(KeyError, _c("db.__comment1")) + self.assertRaises(KeyError, _c("db.user.__doc2")) + class T02LoadAndMerge(_T02MixinLoadAndMerge, unittest.TestCase): def setUp(self): self._load = configmix.load - def test05_identity(self): + def test06_identity(self): cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini")) cfg2 = configmix.merge(cfg, None) self.assertEqual(id(cfg), id(cfg2)) - def test06_identity(self): + def test07_identity(self): cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini")) cfg2 = configmix.merge(cfg, {}) self.assertEqual(id(cfg), id(cfg2)) @@ -240,12 +277,12 @@ def setUp(self): self._load = configmix.safe_load - def test05_deepcopy(self): + def test06_deepcopy(self): cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini")) cfg2 = configmix.safe_merge(cfg, None) self.assertNotEqual(id(cfg), id(cfg2)) - def test06_deepcopy(self): + def test07_deepcopy(self): cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini")) cfg2 = configmix.safe_merge(cfg, {}) self.assertNotEqual(id(cfg), id(cfg2)) @@ -290,7 +327,7 @@ def _g(): return cfg.getvar_s("key7") - + self.assertRaises(KeyError, _g)
