# HG changeset patch # User Franz Glasner # Date 1552523716 -3600 # Node ID b5ce9a8461bf6977772eb3302c59b198db29a1dd # Parent 6ca90e80f4f42a54f33152a5b4d0e9598adc6288 Use the filesystem encoding explicitely where appropriate. diff -r 6ca90e80f4f4 -r b5ce9a8461bf CHANGES.txt --- a/CHANGES.txt Thu Mar 14 00:21:30 2019 +0100 +++ b/CHANGES.txt Thu Mar 14 01:35:16 2019 +0100 @@ -1,7 +1,7 @@ .. -*- coding: utf-8; mode: rst; indent-tabs-mode: nil; -*- .. -.. Valid tags: breaking, feature, bugfix, test, doc +.. Valid tags: breaking, feature, bugfix, misc, test, doc .. .. _changelog: @@ -38,6 +38,11 @@ with ``__doc`` or ``__comment``. .. change:: + :tags: misc + + Use the filesystem encoding where appripriate. + + .. change:: :tags: doc Begin the documentation with `Sphinx `_ diff -r 6ca90e80f4f4 -r b5ce9a8461bf configmix/__init__.py --- a/configmix/__init__.py Thu Mar 14 00:21:30 2019 +0100 +++ b/configmix/__init__.py Thu Mar 14 01:35:16 2019 +0100 @@ -21,7 +21,7 @@ import copy -from .compat import u +from .compat import u, u2fs from .config import Configuration @@ -73,7 +73,7 @@ def _load_yaml(filename): from . import yaml - with open(filename, "rb") as yf: + with open(u2fs(filename), "rb") as yf: return yaml.safe_load(yf) diff -r 6ca90e80f4f4 -r b5ce9a8461bf configmix/compat.py --- a/configmix/compat.py Thu Mar 14 00:21:30 2019 +0100 +++ b/configmix/compat.py Thu Mar 14 01:35:16 2019 +0100 @@ -10,13 +10,15 @@ from __future__ import division, absolute_import, print_function import sys +import os import locale __all__ = ["PY2", "text_to_native_os_str", "native_os_str_to_text", - "u"] + "u", + "u2fs"] PY2 = sys.version_info[0] <= 2 @@ -26,6 +28,9 @@ _OS_ENCODING = locale.getpreferredencoding() + _FS_ENCODING = sys.getfilesystemencoding() or _OS_ENCODING + + def text_to_native_os_str(s, encoding=None): if isinstance(s, unicode): return s.encode(encoding or _OS_ENCODING) @@ -43,6 +48,21 @@ else: return s.decode(encoding) + + def u2fs(s, force=False): + """Convert a text (Unicode) string to the filesystem encoding. + + .. note:: If `s` is already a byte string be permissive and + return `s` unchanged. + + """ + if isinstance(s, str): + return s + if not force and os.name in ("nt", "ce"): + # WinNT and CE have native Unicode support: nothing to convert + return s + return s.encode(_FS_ENCODING) + else: def text_to_native_os_str(s, encoding=None): @@ -52,8 +72,20 @@ def native_os_str_to_text(s, encoding=None): return s + def u(s, encoding="utf-8"): if isinstance(s, str): return s else: return s.decode(encoding) + + + def u2fs(s, force=False): + """Convert a text (Unicode) string to the filesystem encoding. + + .. note:: The filesystem encoding on Python 3 is a Unicode text + string. The function is a noop when called on Python 3. + + """ + assert isinstance(s, str) + return s diff -r 6ca90e80f4f4 -r b5ce9a8461bf configmix/ini.py --- a/configmix/ini.py Thu Mar 14 00:21:30 2019 +0100 +++ b/configmix/ini.py Thu Mar 14 01:35:16 2019 +0100 @@ -32,7 +32,7 @@ except ImportError: DictImpl = dict -from .compat import PY2, u +from .compat import PY2, u, u2fs __all__ = ["INIConfigParser", "NoSectionError", "NoOptionError", @@ -50,16 +50,13 @@ _ConfigParserBase.__init__(self) if executable is None: executable = sys.argv[0] - if PY2: - if isinstance(filename, str): - filename = filename.decode(locale.getpreferredencoding()) - if isinstance(executable, str): - executable = executable.decode(locale.getpreferredencoding()) + filename = u(filename, locale.getpreferredencoding()) + executable = u(executable, locale.getpreferredencoding()) self.executable = os.path.normpath(os.path.abspath(executable)) if encoding is None: encoding = locale.getpreferredencoding() self.encoding = encoding - with io.open(filename, mode="rt", encoding=self.encoding) as cf: + with io.open(u2fs(filename), mode="rt", encoding=self.encoding) as cf: self.read_file(cf, filename) def optionxform(self, option): @@ -91,9 +88,7 @@ if hasattr(self, "filename"): raise RuntimeError("already initialized") filename = os.path.normpath(os.path.abspath(filename)) - if PY2: - if isinstance(filename, str): - filename = filename.decode(locale.getpreferredencoding()) + filename = u(filename, locale.getpreferredencoding()) self.set(None, u("self"), filename) self.set(None, u("here"), os.path.dirname(filename)) self.set(None, u("root"), os.path.dirname(self.executable)) diff -r 6ca90e80f4f4 -r b5ce9a8461bf configmix/json.py --- a/configmix/json.py Thu Mar 14 00:21:30 2019 +0100 +++ b/configmix/json.py Thu Mar 14 01:35:16 2019 +0100 @@ -19,6 +19,8 @@ except ImportError: DictImpl = dict +from .compat import u2fs + __all__ = ["load"] @@ -39,7 +41,7 @@ """Load a single JSON file with name `filename` and encoding `encoding`. """ - with io.open(filename, mode="rt", encoding=encoding) as jsfp: + with io.open(u2fs(filename), mode="rt", encoding=encoding) as jsfp: # # The scanner (not to be changed yet) does only recognize decimal # integers yet. diff -r 6ca90e80f4f4 -r b5ce9a8461bf configmix/py.py --- a/configmix/py.py Thu Mar 14 00:21:30 2019 +0100 +++ b/configmix/py.py Thu Mar 14 01:35:16 2019 +0100 @@ -18,7 +18,7 @@ except ImportError: DictImpl = dict -from .compat import PY2 +from .compat import PY2, u2fs __all__ = ["load"] @@ -43,9 +43,7 @@ gcontext = DictImpl() lcontext = DictImpl() if PY2: - filename2 = filename.encode(locale.getpreferredencoding()) - if PY2: - execfile(filename2, gcontext, lcontext) + execfile(u2fs(filename, True), gcontext, lcontext) else: # "rb" mode allows Python to derive the encoding automatically with open(filename, "rb") as vf: diff -r 6ca90e80f4f4 -r b5ce9a8461bf doc/conf.py --- a/doc/conf.py Thu Mar 14 00:21:30 2019 +0100 +++ b/doc/conf.py Thu Mar 14 01:35:16 2019 +0100 @@ -211,7 +211,8 @@ # -- Options for changelog --------------------------------------------------- -changelog_inner_tag_sort = ['breaking', 'feature', 'bugfix', 'test', 'doc'] +changelog_inner_tag_sort = ['breaking', 'feature', 'bugfix', 'misc', + 'test', 'doc'] def setup(app):