# HG changeset patch # User Franz Glasner # Date 1619422962 -7200 # Node ID f529ca46dd5004b85b5af09dfb15595988a792c1 # Parent d8361dd70d2dc1ce4e9ac74ca327885f7d798343 Implemented the "ref" namespace to get configuration tree references. BUGS: - Tests should be done more thoroughly and extensively - Interaction of tree references and variable substitution should be tested more properly - Documentation is missing yet diff -r d8361dd70d2d -r f529ca46dd50 CHANGES.txt --- a/CHANGES.txt Sun Apr 25 18:05:26 2021 +0200 +++ b/CHANGES.txt Mon Apr 26 09:42:42 2021 +0200 @@ -22,6 +22,12 @@ Moved some important public constants from :py:mod:`configmix` into the :py:mod:`configmix.constants` module. + .. change:: + :tags: feature + + Configuration tree references are implemented in the ``ref`` + namespace + .. changelog:: :version: 0.13 :released: 2021-04-21 diff -r d8361dd70d2d -r f529ca46dd50 configmix/config.py --- a/configmix/config.py Sun Apr 25 18:05:26 2021 +0200 +++ b/configmix/config.py Mon Apr 26 09:42:42 2021 +0200 @@ -22,9 +22,14 @@ from ordereddict import OrderedDict as ConfigurationBase except ImportError: ConfigurationBase = dict +try: + from urllib.parse import urlsplit +except ImportError: + from urlparse import urlsplit from .variables import lookup_varns, lookup_filter from .compat import u +from .constants import REF_NAMESPACE _MARKER = object() @@ -55,6 +60,16 @@ """ + # Speed + _TEXTTYPE = type(u("")) + _STARTTOK = u(b"{{") + _ENDTOK = u(b"}}") + _NS_SEPARATOR = u(b':') + _FILTER_SEPARATOR = u(b'|') + _STARTTOK_REF = _STARTTOK + REF_NAMESPACE + _NS_SEPARATOR + _ENDTOK_REF = _ENDTOK + _DOT = u(b'.') + def getvar(self, varname, default=_MARKER): """Get a variable of the form ``[ns:][[key1.]key2.]name`` - including variables from other namespaces. @@ -67,7 +82,10 @@ if not varns: lookupfn = self._lookupvar else: - lookupfn = lookup_varns(varns) + if varns == REF_NAMESPACE: + lookupfn = self._lookupref + else: + lookupfn = lookup_varns(varns) varvalue = lookupfn(varname) except KeyError: if default is _MARKER: @@ -139,14 +157,14 @@ return s def _split_ns(self, s): - nameparts = s.split(':', 1) + nameparts = s.split(self._NS_SEPARATOR, 1) if len(nameparts) == 1: return (None, s, ) else: return (nameparts[0], nameparts[1], ) def _split_filters(self, s): - nameparts = s.split('|') + nameparts = s.split(self._FILTER_SEPARATOR) if len(nameparts) == 1: return (s, [], ) else: @@ -160,9 +178,9 @@ """ parts = key.split('.') try: - v = self[parts[0]] + v = self.expand_if_reference(self[parts[0]]) for p in parts[1:]: - v = v[p] + v = self.expand_if_reference(v[p]) except TypeError: raise KeyError( "Configuration variable %r not found" @@ -174,8 +192,41 @@ return default return v - # Speed - _TEXTTYPE = type(u("")) + def _lookupref(self, key, default=_MARKER): + """ + `key` must be a reference URI without any (namespace) prefixes + and suffixes + + """ + return self.expand_ref_uri(key, default=default) + + def expand_if_reference(self, v, default=_MARKER): + """Check whether `v` is a reference and -- if true -- then expand it. + + `v` must match the pattern ``{{{ref:}}}`` + + All non-matching texttypes and all non-texttypes are returned + unchanged. + + """ + if not isinstance(v, self._TEXTTYPE): + return v + if not v.startswith(self._STARTTOK_REF) \ + or not v.endswith(self._ENDTOK_REF): + return v + return self.expand_ref_uri( + v[len(self._STARTTOK_REF):-len(self._ENDTOK_REF)], + default=default) + + def expand_ref_uri(self, uri, default=_MARKER): + pu = urlsplit(uri) + if pu.scheme or pu.netloc or pu.path or pu.query: + raise ValueError("only fragment-only URIs are supported") + if not pu.fragment: + return self + if pu.fragment.startswith(self._DOT): + raise ValueError("relative refs not supported") + return self.getvar(pu.fragment, default=default) def substitute_variables_in_obj(self, obj): """Recursively expand variables in the object tree `obj`.""" @@ -199,10 +250,6 @@ else: return obj - # Speed - _STARTTOK = u(b"{{") - _ENDTOK = u(b"}}") - def expand_variable(self, s): """Expand variables in the single string `s`""" start = s.find(self._STARTTOK, 0) diff -r d8361dd70d2d -r f529ca46dd50 configmix/constants.py --- a/configmix/constants.py Sun Apr 25 18:05:26 2021 +0200 +++ b/configmix/constants.py Mon Apr 26 09:42:42 2021 +0200 @@ -30,3 +30,9 @@ key-value is to be deleted when configurations are merged """ + +REF_NAMESPACE = u("ref") +"""Special internal namespace used for implementation of tree +`references` + +""" diff -r d8361dd70d2d -r f529ca46dd50 configmix/variables.py --- a/configmix/variables.py Sun Apr 25 18:05:26 2021 +0200 +++ b/configmix/variables.py Mon Apr 26 09:42:42 2021 +0200 @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- # :- -# :Copyright: (c) 2015-2020, Franz Glasner. All rights reserved. +# :Copyright: (c) 2015-2021, Franz Glasner. All rights reserved. # :License: BSD-3-Clause. See LICENSE.txt for details. # :- """Variable interpolation: implementation of namespaces and filters @@ -18,6 +18,7 @@ from functools import wraps from .compat import PY2, native_os_str_to_text, u +from .constants import REF_NAMESPACE _MARKER = object() @@ -74,7 +75,13 @@ """Register a new variable namespace `name` and it's implementing function `fn` + ..note:: This function checks that `name` is not the special + namespace :data:`~configmix.constants.REF_NAMESPACE`. + """ + if name == REF_NAMESPACE: + raise ValueError( + "the special namespace `%s' is not allowed here" % REF_NAMESPACE) _varns_registry[name] = fn @@ -86,7 +93,13 @@ :returns: the implementing function :exception KeyError: if the namespace `name` doesn't exist + ..note:: This function checks that `name` is not the special + namespace :data:`~configmix.constants.REF_NAMESPACE`. + """ + if name == REF_NAMESPACE: + raise ValueError( + "the special namespace `%s' is not allowed here" % REF_NAMESPACE) return _varns_registry[_normalized(name)] diff -r d8361dd70d2d -r f529ca46dd50 tests/test.py --- a/tests/test.py Sun Apr 25 18:05:26 2021 +0200 +++ b/tests/test.py Mon Apr 26 09:42:42 2021 +0200 @@ -503,5 +503,67 @@ self.assertEqual("22", cfg.getvar_s("key103")) +class T06References(unittest.TestCase): + + def setUp(self): + self._reset() + self._cfg = configmix.load( + os.path.join(TESTDATADIR, "conf20.yml"), + os.path.join(TESTDATADIR, "conf21.yml"), + os.path.join(TESTDATADIR, "conf22.ini"), + os.path.join(TESTDATADIR, "conf23.json"), + os.path.join(TESTDATADIR, "conf24.toml"), + os.path.join(TESTDATADIR, "reference-style.yml")) + + def tearDown(self): + self._reset() + + def _reset(self): + configmix.clear_assoc() + for pat, fmode in configmix.DEFAULT_ASSOC: + configmix.set_assoc(pat, fmode) + + def test01_reference_without_expansions(self): + self.assertTrue(isinstance(self._cfg.getvar("wsgi.profiler"), dict)) + self.assertTrue(isinstance( + self._cfg.getvar("wsgi.profiler.params"), dict)) + self.assertEqual("werkzeug", + self._cfg.getvar("wsgi.profiler.params.type")) + self.assertTrue(self._cfg.getvar("wsgi.profiler.params.params.evalex")) + self.assertEqual(self._cfg.getvar("wsgi.debugger"), + self._cfg.getvar("wsgi.profiler.params")) + + def test02_reference__with_expansions(self): + self.assertTrue(isinstance(self._cfg.getvar_s("wsgi.profiler"), dict)) + self.assertTrue(isinstance( + self._cfg.getvar_s("wsgi.profiler.params"), dict)) + self.assertTrue( + self._cfg.getvar_s("wsgi.profiler.params.params.evalex")) + self.assertEqual("werkzeug", + self._cfg.getvar_s("wsgi.profiler.params.type")) + + def test03_no_direct_attribute_access_to_expanded_references(self): + self.assertEqual( + "{{ref:#wsgi.debugger}}", + self._cfg.wsgi.profiler.params) + try: + self._cfg.wsgi.profiler.params.type + except AttributeError: + pass + else: + self.fail("no attribute error seen") + + def test04_indirect_recursive_references(self): + self.assertEqual( + "werkzeug", + self._cfg.getvar_s("testref.here.params.type")) + self.assertTrue( + self._cfg.getvar_s("testref.here.params.params.evalex")) + + def test05_recursive_expansion(self): + c = self._cfg.getvar_s("testref") + self.assertTrue(c["here"]["params"]["params"]["evalex"]) + + if __name__ == "__main__": unittest.main()