changeset 144:7e6ec99d5ff5

Allow comments as keys and filter them by default
author Franz Glasner <hg@dom66.de>
date Fri, 13 Apr 2018 09:51:02 +0200
parents 252645c69c7b
children d44ee758e31b
files configmix/__init__.py configmix/json.py tests/data/conf1.ini tests/data/conf1.json tests/data/conf1.py tests/data/conf1.yml tests/data/conf20.yml tests/data/conf22.ini tests/data/conf23.json tests/test.py
diffstat 10 files changed, 126 insertions(+), 21 deletions(-) [+]
line wrap: on
line diff
--- a/configmix/__init__.py	Mon Apr 09 09:35:04 2018 +0200
+++ b/configmix/__init__.py	Fri Apr 13 09:51:02 2018 +0200
@@ -20,6 +20,7 @@
 
 import copy
 
+from .compat import u
 from .config import Configuration
 
 
@@ -28,6 +29,15 @@
            "Configuration"]
 
 
+COMMENTS = [u("__comment"),
+            u("__doc"),
+]
+"""Prefixes for comment configuration keys that are to be handled as
+comments
+
+"""
+
+
 def load(*files):
     """Load the given configuration files, merge them in the given order
     and return the resulting configuration dictionary.
@@ -168,7 +178,7 @@
         return result
 
 
-def merge(user, default):
+def merge(user, default, filter_comments=True):
     """Logically merge the configuration in `user` into `default`.
 
     :param ~configmix.config.Configuration user:
@@ -176,73 +186,120 @@
                 into `default`
     :param ~configmix.config.Configuration default:
                 the base configuration where `user` is logically merged into
+    :param bool filter_comments: flag whether to filter comment keys that
+                   start with any of the items in :data:`COMMENTS`
     :returns: `user` with the necessary amendments from `default`.
               If `user` is ``None`` then `default` is returned.
 
-    .. note:: The configuration in `default` is not changed but the
-              configuration given in `user` is changed **inplace**.
+    .. note:: The configuration in `user` is augmented/changed
+              **inplace**.
+
+              The configuration in `default` will be changed **inplace**
+              when filtering out comments (which is the default).
 
     From http://stackoverflow.com/questions/823196/yaml-merge-in-python
 
     """
     if user is None:
+        if filter_comments:
+            _filter_comments(default)
         return default
+    if filter_comments:
+        _filter_comments(user)
     if isinstance(user, dict) and isinstance(default, dict):
         for k, v in default.items():
+            if filter_comments and _is_comment(k):
+                continue
             if k not in user:
                 user[k] = v
             else:
-                user[k] = _merge(user[k], v)
+                user[k] = _merge(user[k], v, filter_comments)
     return user
 
 
-def _merge(user, default):
+def _merge(user, default, filter_comments):
     """Recursion helper for :meth:`merge`
 
     """
     if isinstance(user, dict) and isinstance(default, dict):
         for k, v in default.items():
+            if filter_comments and _is_comment(k):
+                continue
             if k not in user:
                 user[k] = v
             else:
-                user[k] = _merge(user[k], v)
+                user[k] = _merge(user[k], v, filter_comments)
     return user
 
 
-def safe_merge(user, default):
+def safe_merge(user, default, filter_comments=True):
     """A more safe version of :func:`merge` that makes deep copies of
     the returned container objects.
 
-    No given argument is ever changed inplace. Every object from `default`
-    is decoupled from the result -- so changing the `default` configuration
-    lates does not yield into a merged configuration later.
+    Contrary to :func:`merge` no given argument is ever changed
+    inplace. Every object from `default` is decoupled from the result
+    -- so changing the `default` configuration later does not yield
+    into a merged configuration later.
 
     """
     if user is None:
+        if filter_comments:
+            _filter_comments(default)
         return copy.deepcopy(default)
     user = copy.deepcopy(user)
+    if filter_comments:
+        _filter_comments(user)
     if isinstance(user, dict) and isinstance(default, dict):
         for k, v in default.items():
+            if filter_comments and _is_comment(k):
+                continue
             if k not in user:
                 user[k] = copy.deepcopy(v)
             else:
-                user[k] = _safe_merge(user[k], v)
+                user[k] = _safe_merge(user[k], v, filter_comments)
     return user
 
 
-def _safe_merge(user, default):
+def _safe_merge(user, default, filter_comments):
     """Recursion helper for :meth:`safe_merge`
 
     """
     if isinstance(user, dict) and isinstance(default, dict):
         for k, v in default.items():
+            if filter_comments and _is_comment(k):
+                continue
             if k not in user:
                 user[k] = copy.deepcopy(v)
             else:
-                user[k] = _safe_merge(user[k], v)
+                user[k] = _safe_merge(user[k], v, filter_comments)
     return user
 
 
+def _filter_comments(d):
+    """Recursively filter comments keys in the dict `d`.
+
+    Comment keys are keys that start with any of the items in
+    :data:`COMMENTS`.
+
+    """
+    if not isinstance(d, dict):
+        return
+    # use a copy of the keys because we change `d` while iterating
+    for k in list(d.keys()):
+        if _is_comment(k):
+            del d[k]
+        else:
+            if isinstance(d[k], dict):
+                _filter_comments(d[k])
+
+
+def _is_comment(k):
+    for i in COMMENTS:
+        if k.startswith(i):
+            return True
+    return False
+
+
 #
 # Init loader defaults
 #
--- a/configmix/json.py	Mon Apr 09 09:35:04 2018 +0200
+++ b/configmix/json.py	Fri Apr 13 09:51:02 2018 +0200
@@ -38,8 +38,6 @@
 def load(filename, encoding="utf-8"):
     """Load a single JSON file with name `filename` and encoding `encoding`.
 
-    .. todo:: Allow comments in JSON files
-
     """
     with io.open(filename, mode="rt", encoding=encoding) as jsfp:
         #
--- a/tests/data/conf1.ini	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/data/conf1.ini	Fri Apr 13 09:51:02 2018 +0200
@@ -1,10 +1,12 @@
 # -*- coding: utf-8 -*-
 
 [config]
+__comment1 = Comment 1
 key1 = the value
 key2 = :int:2
 key3 = :float:5.7
 key4 = :bool:yes
 key5 = :bool:off
 key6 = :int:0o377
+__comment2 = Comment no 2
 key7 = Umlaute: ÄÖÜäöüß
--- a/tests/data/conf1.json	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/data/conf1.json	Fri Apr 13 09:51:02 2018 +0200
@@ -1,8 +1,10 @@
-{"key1": "the value",
+{"__comment1": "Comment 1",
+ "key1": "the value",
  "key2": 2,
  "key3": 5.7,
  "key4": true,
  "key5": false,
  "key6": 255,
+ "__comment2": "Comment no 2",
  "key7": "Umlaute: ÄÖÜäöüß"
 }
--- a/tests/data/conf1.py	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/data/conf1.py	Fri Apr 13 09:51:02 2018 +0200
@@ -1,9 +1,11 @@
 # -*- coding: utf-8 -*-
 
+__comment1 = u"Comment 1"
 key1 = u"the value"
 key2 = 2
 key3 = 5.7
 key4 = True
 key5 = False
 key6 = 0o377
+__comment2 = u"Comment no 2"
 key7 = u"Umlaute: ÄÖÜäöüß"
--- a/tests/data/conf1.yml	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/data/conf1.yml	Fri Apr 13 09:51:02 2018 +0200
@@ -2,10 +2,12 @@
 %YAML 1.1
 ---
 
+__comment1: Comment 1
 key1: the value
 key2: 2
 key3: 5.7
 key4: true
 key5: false
 key6: 0377
+__comment2: Comment no 2
 key7: 'Umlaute: ÄÖÜäöüß'
--- a/tests/data/conf20.yml	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/data/conf20.yml	Fri Apr 13 09:51:02 2018 +0200
@@ -6,6 +6,7 @@
 ---
 
 process:
+  __doc1: A first comment
   umask: 0o0027
 
 intl:
@@ -17,8 +18,10 @@
 
 
 db:
+  __comment1: A second comment
   user:
     name: <My Username>
+    __doc2: A third comment
     pwd: <My Password>
 
   catalog: <MyDbName>
--- a/tests/data/conf22.ini	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/data/conf22.ini	Fri Apr 13 09:51:02 2018 +0200
@@ -3,6 +3,7 @@
 
 [config.db.user]
 name = the_database_user_2
+__doc2 = Overwriting a comment
 pwd=the-database-password-2
 
 [config.db.locinfo.ro]
--- a/tests/data/conf23.json	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/data/conf23.json	Fri Apr 13 09:51:02 2018 +0200
@@ -2,6 +2,7 @@
  "db": {
      "user": {
 	 "name": "the_database_user_2",
+	 "__doc2": "comment",
 	 "pwd": "the-database-password-2"
      },
      "locinfo": {
--- a/tests/test.py	Mon Apr 09 09:35:04 2018 +0200
+++ b/tests/test.py	Fri Apr 13 09:51:02 2018 +0200
@@ -43,6 +43,22 @@
         self.assertEqual(u("Umlaute: ÄÖÜäöüß"),
                          cfg.get("key7"))
 
+    def __check_comment(self, cfg):
+        # Check comments: low level comments are *not* filtered
+        self.assertEqual(u("Comment 1"), cfg.get("__comment1"))
+        self.assertEqual(u("Comment no 2"), cfg.get("__comment2"))
+
+    def __check_no_comment(self, cfg):
+
+        def _c(name):
+            def _f():
+                cfg[u(name)]
+            return _f
+
+        # Variables with leading underscores are *not*  imported by default
+        self.assertRaises(KeyError, _c("__comment1"))
+        self.assertRaises(KeyError, _c("__comment2"))
+
     def __check_tree(self, cfg):
         self.assertEqual(u("in the root namespace"),
                          cfg.get("key1"))
@@ -57,10 +73,12 @@
     def test01_ini_types(self):
         cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini"))
         self.__check_types(cfg)
+        self.__check_comment(cfg)
 
     def test02_py_types(self):
         cfg = configmix.py.load(os.path.join(TESTDATADIR, "conf1.py"))
         self.__check_types(cfg)
+        self.__check_no_comment(cfg)
 
     def test03_yaml_types(self):
         with io.open(os.path.join(TESTDATADIR, "conf1.yml"), "rt",
@@ -69,11 +87,13 @@
             if configmix.yaml.OrderedDict:
                 self.assertTrue(isinstance(cfg, configmix.yaml.OrderedDict))
             self.__check_types(cfg)
+            self.__check_comment(cfg)
 
     def test04_json_types(self):
         cfg = configmix.json.load(os.path.join(TESTDATADIR, "conf1.json"))
         self.assertTrue(isinstance(cfg, configmix.json.DictImpl))
         self.__check_types(cfg)
+        self.__check_comment(cfg)
 
     def test05_py_export_all(self):
         # When __all__ is given only it's keys are exported
@@ -218,18 +238,35 @@
 
         self.assertRaises(KeyError, _look)
 
+    def test05_comments(self):
+        cfg = self._load(os.path.join(TESTDATADIR, "conf20.yml"),
+                         os.path.join(TESTDATADIR, "conf21.yml"),
+                         os.path.join(TESTDATADIR, "conf22.ini"),
+                         os.path.join(TESTDATADIR, "conf23.json"))
+
+        def _c(name):
+            def _f():
+                cfg.getvar_s(name)
+            return _f
+
+        # Variables with leading underscores are *not*  imported by default
+        self.assertEqual(0o0027, int(cfg.getvar_s("process.umask"), 0))
+        self.assertRaises(KeyError, _c("process.__doc1"))
+        self.assertRaises(KeyError, _c("db.__comment1"))
+        self.assertRaises(KeyError, _c("db.user.__doc2"))
+
 
 class T02LoadAndMerge(_T02MixinLoadAndMerge, unittest.TestCase):
 
     def setUp(self):
         self._load = configmix.load
 
-    def test05_identity(self):
+    def test06_identity(self):
         cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini"))
         cfg2 = configmix.merge(cfg, None)
         self.assertEqual(id(cfg), id(cfg2))
 
-    def test06_identity(self):
+    def test07_identity(self):
         cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini"))
         cfg2 = configmix.merge(cfg, {})
         self.assertEqual(id(cfg), id(cfg2))
@@ -240,12 +277,12 @@
     def setUp(self):
         self._load = configmix.safe_load
 
-    def test05_deepcopy(self):
+    def test06_deepcopy(self):
         cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini"))
         cfg2 = configmix.safe_merge(cfg, None)
         self.assertNotEqual(id(cfg), id(cfg2))
 
-    def test06_deepcopy(self):
+    def test07_deepcopy(self):
         cfg = configmix.ini.load(os.path.join(TESTDATADIR, "conf1.ini"))
         cfg2 = configmix.safe_merge(cfg, {})
         self.assertNotEqual(id(cfg), id(cfg2))
@@ -290,7 +327,7 @@
 
         def _g():
             return cfg.getvar_s("key7")
-            
+
         self.assertRaises(KeyError, _g)