changeset 570:e15e86c47a27

ADD: Configure.py ind ninjy_syntax.py from py3-extension-tests
author Franz Glasner <fzglas.hg@dom66.de>
date Sat, 08 Jan 2022 17:57:36 +0100
parents 1bdf88437555
children aa240af27589
files Configure.py ninja_syntax.py
diffstat 2 files changed, 673 insertions(+), 0 deletions(-) [+]
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/Configure.py	Sat Jan 08 17:57:36 2022 +0100
@@ -0,0 +1,431 @@
+#
+#
+# Important Triplets:
+#
+# clang-cl (clang-cl /clang:-dumpmachine)
+#
+#   x86_64-pc-windows-msvc
+#   i386-pc-windows-msvc
+#
+# clang on FreeBSD (clang -dumpmachine):
+#
+#   x86_64-unknown-freebsd12.2
+#   i386-unknown-freebsd12.2
+#
+# NOTE: gcc also known "-dumpmachine"
+#
+#
+
+from __future__ import print_function, absolute_import
+
+import argparse
+import collections
+import datetime
+import copy
+import getopt
+import os
+import sys
+
+import ninja_syntax
+
+tool = build = host = None
+
+host = None
+
+
+#
+# Global build variables (ordered because they must be written ordered
+# -- and with simple attribute access
+#
+class BuildVars(collections.OrderedDict):
+
+    def __getattr__(self, n):
+        try:
+            return self[n]
+        except KeyError:
+            raise AttributeError(n)
+
+    def __setattr__(self, n, v):
+        # private v
+        if n.startswith("_OrderedDict__"):
+            return super(BuildVars, self).__setattr__(n, v)
+        self[n] = v
+
+
+def make_obj_name(name, newext):
+    bn = os.path.basename(name)
+    if not bn:
+        return bn
+    root, ext = os.path.splitext(bn)
+    return root + newext
+
+options = argparse.Namespace(
+    user_includes = [],
+    sys_includes = [],
+    sys_libpath = [],
+    user_libpath = [],
+    link_with_python = None,
+    python_limited_api = None,
+)
+
+gbv = BuildVars()
+gbv.intdir = "_builddir-test"
+gbv.srcdir = "src"
+gbv.builddir = "$intdir"
+gbv.pxx3dir = "pxx3"
+
+opts, args = getopt.getopt(
+    sys.argv[1:],
+    "B:H:t:I:L:",
+    ["build=",
+     "host=",
+     "tool=",
+     "include=",
+     "libpath=",
+     "sys-include=",
+     "sys-libpath=",
+     "CXX=",
+     "LINK=",
+     "intdir=",              # intermediate files
+     "builddir=",            # Ninja builddir
+     "link-with-python=",    # link with libpython
+     "python-limited-api=",  # Use Py_LIMITED_API
+     ])
+for opt, val in opts:
+    if opt in ("-t", "--tool"):
+        if tool is None:
+            tool = argparse.Namespace(local=False, msvc=False, clang=False)
+        if val == "msvc":
+            tool.msvc = True
+            tool.compile_only = "/c"
+            tool.define_format = "/D {}"
+            tool.include_format = "/I {}"
+            tool.lib_format = "{}"
+            tool.libpath_format = "/libpath:{}"
+            tool.dependencies = "msvc"
+        elif val == "clang-cl":
+            tool.msvc = tool.clang = True
+            tool.compile_only = "/c"
+            tool.define_format = "/D {}"
+            tool.include_format = "/I {}"
+            tool.lib_format = "{}"
+            tool.libpath_format = "/libpath:{}"
+            tool.dependencies = "msvc"
+        elif val == "clang":
+            tool.clang = True
+            tool.compile_only = "-c"
+            tool.define_format = "-D{}"
+            tool.include_format = "-I{}"
+            tool.lib_format = "-l{}"
+            tool.libpath_format = "-L{}"
+            tool.dependencies = "gcc"
+        elif val in ("local", "posix"):
+            tool.local = True
+            tool.compile_only = "-c"
+            tool.define_format = "-D{}"
+            tool.include_format = "-I{}"
+            tool.lib_format = "-l{}"
+            tool.libpath_format = "-L{}"
+            tool.dependencies = "gcc"
+        else:
+            raise getopt.GetoptError("unknown tool value: {}".format(val), opt)
+    elif opt in ("-B", "--build"):
+        build = argparse.Namespace(type=None,
+                                   posix=False, windows=False,
+                                   pathmod=None)
+        if val == "windows":
+            # build on Windows with clang-cl
+            build.windows = True
+            build.type = "windows"
+        elif val == "posix":
+            build.posix = True
+            build.type = "posix"
+        else:
+            raise getopt.GetoptError("unknwon build value: {}".format(val),
+                                     opt)
+    elif opt in ("-H", "--host"):
+        if host is None:
+            host = argparse.Namespace(windows=False)
+        if val == "windows":
+            host.type = "windows"
+            host.windows = True
+            host.posix = False
+            host.objext = ".obj"
+            host.pydext = ".pyd"
+        elif val == "posix":
+            host.type = "posix"
+            host.windows = False
+            host.posix = True
+            host.objext = ".o"
+            host.pydext = ".so"
+        else:
+            raise getopt.GetoptError("unknown host value: {}".format(val),
+                                     opt)
+    elif opt in ("-I", "--include"):
+        options.user_includes.append(val)
+    elif opt in ("-L", "--libpath"):
+        options.user_libpath.append(val)
+    elif opt == "--sys-include":
+        options.sys_includes.append(val)
+    elif opt == "--sys-libpath":
+        options.sys_libpath.append(val)
+    elif opt == "--CXX":
+        gbv.cxx = val
+    elif opt == "--LINK":
+        gbv.link = val
+    elif opt == "--intdir":
+        gbv.intdir = val
+    elif opt == "--builddir":
+        gbv.builddir = val
+    elif opt == "--link-with-python":
+        options.link_with_python = val
+    elif opt == "--python-limited-api":
+        if val.lower().startswith("0x"):
+            options.python_limited_api = val
+        else:
+            options.python_limited_api = "0x03040000"
+    else:
+        raise getopt.GetoptError("Unhandled option `{}'".format(opt), opt)
+
+if tool is None:
+    print("ERROR: no tool given", file=sys.stderr)
+    sys.exit(1)
+
+if build.windows and host.posix:
+    print("ERROR: cross-compiling on Windows not supported", file=sys.stderr)
+    sys.exit(1)
+
+if build.windows:
+    import ntpath as pathmod
+else:
+    import posixpath as pathmod
+build.pathmod = pathmod
+
+if tool.msvc:
+    if tool.clang:
+        if not getattr(gbv, "cxx", None):
+            gbv.cxx = "clang-cl"
+        if not getattr(gbv, "link", None):
+            gbv.link = "lld-link"
+    else:
+        gbv.cxx = "cl"
+        gbv.link = "link"
+elif tool.clang:
+    gbv.cxx = "clang++"
+    gbv.link = "clang++"   # link C++ through the compiler
+elif tool.local:
+    gbv.cxx = "c++"
+    gbv.link = "c++"       # link through the compiler
+else:
+    raise RuntimeError("tool condition is not handled")
+
+ext1_sources = [
+    "$srcdir/ext1/testext1.cpp",
+    "$pxx3dir/shared/thread.cpp",
+]
+
+ext2_sources = [
+    "$srcdir/ext2/testext2.cpp",
+    "$srcdir/ext2/hashes.cpp",    
+    "$pxx3dir/shared/thread.cpp",
+    "$pxx3dir/shared/module.cpp",
+    "$pxx3dir/shared/xcept.cpp",
+    "$pxx3dir/shared/cfunctions.cpp",
+    "$pxx3dir/shared/misc.cpp",
+    "$pxx3dir/shared/exttype.cpp",    
+    "$pxx3dir/shared/allocator.cpp",    
+]
+
+ccflags = []
+cxxflags = []
+ccwarnings = []
+ldflags = []
+
+defines = [
+    "PY_SSIZE_T_CLEAN",
+    "HAVE_THREADS",
+]
+if options.python_limited_api:
+    defines.append("Py_LIMITED_API={}".format(options.python_limited_api))
+
+# XXX TBD: handle debug/release build _DEBUG/NDEBUG
+
+includes = []
+includes.extend(options.sys_includes)
+includes.extend(options.user_includes)
+
+includes.append("$pxx3dir/include")
+
+libpath = []
+libpath.extend(options.sys_libpath)
+libpath.extend(options.user_libpath)
+
+libs = []
+if host.windows:
+    if tool.msvc:
+        # automatically included via #pragma
+        # libs.append("python3.lib")
+        pass
+else:
+    if options.link_with_python:
+        libs.append(options.link_with_python)
+
+if host.windows:
+    defines.append("WIN32")
+    # XXX TBD Handle arch -> WIN64
+    defines.append("WIN64")
+    defines.append("_WINDOWS")
+    # for a user dll
+    defines.append("_USRDLL")
+    defines.append("_WINDLL")
+
+    defines.append("WIN32_LEAN_AND_MEAN")
+    defines.append("_WIN32_WINNT=0x0501")    # WinXP
+
+    if tool.msvc:
+        # XXX TBD warnings
+
+        defines.append("_CRT_SECURE_NO_WARNINGS")
+
+        ccflags.append("/Zi")
+        ccflags.append("/MD")   # link to dll runtime
+        ccflags.append("/EHsc")
+        ccflags.append("/Gy")   # enable function level linking
+
+        cxxflags.append("/TP")
+        #cxxflags.append("/std:c++latest")
+
+        # XXX TBD machine
+        ccflags.append("-m64")
+
+        ldflags.append("/dll")
+        ldflags.append("/debug")    # PDB output
+        # 32-bit: -> 5.01   64-bit: 5.02
+        ldflags.append("/subsystem:windows,5.02")
+        ldflags.append("/incremental:no")
+        #
+        ldflags.append("/manifest:NO")
+
+
+    if tool.clang:
+        ccflags.append("-fms-compatibility-version=16.00")
+
+        ccwarnings.append("-Wno-nonportable-include-path")
+        ccwarnings.append("-Wno-microsoft-template")
+        ccwarnings.append("-Wno-pragma-pack")
+elif host.posix:
+    defines.append("PIC")
+
+    ccwarnings.extend(["-Wall", "-Wextra", "-pedantic"])
+
+    ccflags.append("-g")
+    ccflags.append("-fPIC")
+    ccflags.append("-fvisibility=hidden")
+    ccflags.append("-pthread")
+
+    if tool.clang:  # || tool.gcc
+        ccflags.append("-ffunction-sections")
+        ccflags.append("-fdata-sections")
+
+    if tool.clang:
+        ccflags.append("-faddrsig")    # use with --icf=all/safe when linking
+
+    ldflags.append("-shared")
+    ldflags.append("-Wl,-z,relro,-z,now")
+    ldflags.append("-Wl,--build-id=sha1")
+    # XXX TBD only when building in debug code
+    if options.link_with_python:
+        ldflags.append("-Wl,-z,defs")
+
+    if tool.clang: # || tool.gcc
+        ldflags.append("-Wl,--gc-sections")
+
+    if tool.clang:
+        ldflags.append("-Wl,--icf=safe")
+
+gbv.cppdefines = [tool.define_format.format(d) for d in defines]
+gbv.includes = [tool.include_format.format(pathmod.normpath(i))
+                for i in includes]
+gbv.ccflags = ccflags
+gbv.cxxflags = cxxflags
+gbv.ccwarnings = ccwarnings
+gbv.ldflags = ldflags
+gbv.ldlibpath = [tool.libpath_format.format(pathmod.normpath(l))
+                 for l in libpath]
+gbv.ldlibs = [tool.lib_format.format(l) for l in libs]
+
+n = ninja_syntax.Writer(sys.stdout)
+
+n.comment('This file is used to build test Python extensions.')
+n.comment(
+    'It is generated by {} at {}Z.'.format(
+        os.path.basename(__file__),
+        datetime.datetime.utcnow().isoformat()))
+n.newline()
+n.comment('Created using command: {!r}'.format(sys.argv))
+n.newline()
+for k, v in gbv.items():
+    n.variable(k, v)
+n.newline()
+if tool.msvc:
+    # Note: this includes clang-cl
+    n.rule("compile-pyextension-unit",
+           "$cxx /nologo /showIncludes /c $cppdefines $ccwarnings $includes $ccflags $cxxflags /Fd$intdir/$intsubdir/ /Fo$out $in",
+           deps=tool.dependencies)
+else:
+    n.rule("compile-pyextension-unit",
+           "$cxx -MD -MF $intdir/_deps -MT $out $cppdefines $ccwarnings $includes $ccflags $cxxflags -c -o $out $in",
+           deps=tool.dependencies,
+           depfile="$intdir/_deps")
+n.newline()
+if tool.msvc:
+    # XXX TBD: in "release" builds use /pdbaltpath:$out.pdb
+    n.rule("link-pyextension", "$link /nologo $ldflags $ldlibpath /implib:$intdir/$out.lib /pdb:$intdir/$out.pdb /out:$out $in $ldlibs")
+else:
+    n.rule("link-pyextension", "$link $cppdefines $ccwarnings $ccflags $cxxflags $ldflags -o $out $in $ldlibpath $ldlibs")
+n.newline()
+
+n.comment("testext1")
+for f in ext1_sources:
+    n.build(pathmod.normpath("$intdir/$intsubdir/"+make_obj_name(f, host.objext)),
+            "compile-pyextension-unit",
+            inputs=pathmod.normpath(f),
+            variables={"intsubdir": "ext1"})
+n.newline()
+linkinputs = [pathmod.normpath("$intdir/ext1/"+make_obj_name(f, host.objext))
+              for f in ext1_sources]
+if tool.msvc:
+    implicit_outputs = [
+        pathmod.normpath("$intdir/testext1"+host.pydext+".lib"),
+        pathmod.normpath("$intdir/testext1"+host.pydext+".pdb")]
+    if not tool.clang:
+        implicit_outputs.append(pathmod.normpath("$intdir/testext1"+host.pydext+".exp"))
+else:
+    implicit_outputs = None
+n.build("testext1"+host.pydext,
+        "link-pyextension",
+        inputs=linkinputs,
+        implicit_outputs=implicit_outputs)
+n.newline()
+
+n.comment("testext2")
+for f in ext2_sources:
+    n.build(pathmod.normpath("$intdir/$intsubdir/"+make_obj_name(f, host.objext)),
+            "compile-pyextension-unit",
+            inputs=pathmod.normpath(f),
+            variables={"intsubdir": "ext2"})
+n.newline()
+linkinputs = [pathmod.normpath("$intdir/ext2/"+make_obj_name(f, host.objext))
+              for f in ext2_sources]
+if tool.msvc:
+    implicit_outputs = [
+        pathmod.normpath("$intdir/testext2"+host.pydext+".lib"),
+        pathmod.normpath("$intdir/testext2"+host.pydext+".pdb")]
+    if not tool.clang:
+        implicit_outputs.append(pathmod.normpath("$intdir/testext2"+host.pydext+".exp"))
+else:
+    implicit_outputs = None
+n.build("testext2"+host.pydext,
+        "link-pyextension",
+        inputs=linkinputs,
+        implicit_outputs=implicit_outputs)
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/ninja_syntax.py	Sat Jan 08 17:57:36 2022 +0100
@@ -0,0 +1,242 @@
+# -*- coding: utf-8 -*-
+#
+# Copyright 2011 Google Inc. All Rights Reserved.
+# Copyright 2020 Franz Glasner. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+r"""Python module for generating .ninja files.
+
+Note that this is emphatically not a required piece of Ninja; it's
+just a helpful utility for build-file-generation systems that already
+use Python.
+
+:Copyright: \(c) 2001 Google Inc. All Rights Reserved.
+:Copyright: \(c) 2020 Franz Glasner. All Rights Reserved.
+:License:   Apache License 2.0. See :ref:`LICENSE.txt <license>` for details.
+
+"""
+
+from __future__ import absolute_import
+
+import re
+import textwrap
+
+
+def escape_path(word):
+    """Escape `word` according to Ninja's path interpretation rules.
+
+    Escapes not only ``$`` but also spaces and colons.
+    """
+    return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:')
+
+
+class Writer(object):
+
+    """An object-oriented writer of a Ninja build file.
+
+    Can act as its own context manager.
+    """
+
+    def __init__(self, output, width=78):
+        """
+        :param output: an opened file-alike where to write into
+        :param int width: the width where line-wrapping should happen
+
+        `output` just needs a `write()` and -- if :meth:`~.close` is called
+        -- a `close()` method.
+        """
+        self.output = output
+        self.width = width
+
+    def newline(self):
+        self.output.write('\n')
+
+    def comment(self, text):
+        for line in textwrap.wrap(text, self.width - 2, break_long_words=False,
+                                  break_on_hyphens=False):
+            self.output.write('# ' + line + '\n')
+
+    def variable(self, key, value, indent=0):
+        if value is None:
+            return
+        if isinstance(value, list):
+            value = ' '.join(filter(None, value))  # Filter out empty strings.
+        self._line('%s = %s' % (key, value), indent)
+
+    def pool(self, name, depth):
+        self._line('pool %s' % name)
+        self.variable('depth', depth, indent=1)
+
+    def rule(self, name, command, description=None, depfile=None,
+             generator=False, pool=None, restat=False, rspfile=None,
+             rspfile_content=None, deps=None):
+        self._line('rule %s' % name)
+        self.variable('command', command, indent=1)
+        if description:
+            self.variable('description', description, indent=1)
+        if depfile:
+            self.variable('depfile', depfile, indent=1)
+        if generator:
+            self.variable('generator', '1', indent=1)
+        if pool:
+            self.variable('pool', pool, indent=1)
+        if restat:
+            self.variable('restat', '1', indent=1)
+        if rspfile:
+            self.variable('rspfile', rspfile, indent=1)
+        if rspfile_content:
+            self.variable('rspfile_content', rspfile_content, indent=1)
+        if deps:
+            self.variable('deps', deps, indent=1)
+
+    def build(self, outputs, rule, inputs=None, implicit=None, order_only=None,
+              variables=None, implicit_outputs=None, pool=None):
+        outputs = as_list(outputs)
+        out_outputs = [escape_path(x) for x in outputs]
+        all_inputs = [escape_path(x) for x in as_list(inputs)]
+
+        if implicit:
+            implicit = [escape_path(x) for x in as_list(implicit)]
+            all_inputs.append('|')
+            all_inputs.extend(implicit)
+        if order_only:
+            order_only = [escape_path(x) for x in as_list(order_only)]
+            all_inputs.append('||')
+            all_inputs.extend(order_only)
+        if implicit_outputs:
+            implicit_outputs = [escape_path(x)
+                                for x in as_list(implicit_outputs)]
+            out_outputs.append('|')
+            out_outputs.extend(implicit_outputs)
+
+        self._line('build %s: %s' % (' '.join(out_outputs),
+                                     ' '.join([rule] + all_inputs)))
+        if pool is not None:
+            self._line('  pool = %s' % pool)
+
+        if variables:
+            if isinstance(variables, dict):
+                iterator = iter(variables.items())
+            else:
+                iterator = iter(variables)
+
+            for key, val in iterator:
+                self.variable(key, val, indent=1)
+
+        return outputs
+
+    def include(self, path):
+        self._line('include %s' % path)
+
+    def subninja(self, path):
+        self._line('subninja %s' % path)
+
+    def default(self, paths):
+        self._line('default %s' % ' '.join(as_list(paths)))
+
+    def _count_dollars_before_index(self, s, i):
+        """Returns the number of '$' characters right in front of s[i]."""
+        dollar_count = 0
+        dollar_index = i - 1
+        while dollar_index > 0 and s[dollar_index] == '$':
+            dollar_count += 1
+            dollar_index -= 1
+        return dollar_count
+
+    def _line(self, text, indent=0):
+        """Write 'text' word-wrapped at self.width characters."""
+        leading_space = '  ' * indent
+        while len(leading_space) + len(text) > self.width:
+            # The text is too wide; wrap if possible.
+
+            # Find the rightmost space that would obey our width constraint and
+            # that's not an escaped space.
+            available_space = self.width - len(leading_space) - len(' $')
+            space = available_space
+            while True:
+                space = text.rfind(' ', 0, space)
+                if (space < 0
+                        or self._count_dollars_before_index(text, space) % 2 == 0):  # noqa: E501
+                    break
+
+            if space < 0:
+                #
+                # No such space;
+                # just use the first unescaped space we can find.
+                #
+                space = available_space - 1
+                while True:
+                    space = text.find(' ', space + 1)
+                    if (space < 0
+                            or self._count_dollars_before_index(text, space) % 2 == 0):  # noqa: E501
+                        break
+            if space < 0:
+                # Give up on breaking.
+                break
+
+            self.output.write(leading_space + text[0:space] + ' $\n')
+            text = text[space+1:]
+
+            # Subsequent lines are continuations, so indent them.
+            leading_space = '  ' * (indent+2)
+
+        self.output.write(leading_space + text + '\n')
+
+    def close(self):
+        """Close the associated output"""
+        self.output.close()
+
+    def __enter__(self):
+        """Return `self`"""
+        return self
+
+    def __exit__(self, *args, **kwds):
+        """Close the associated output"""
+        self.close()
+
+
+def as_list(input):
+    """Ensure that `input` is always a :class:`list` -- wrap if needed
+
+    :rtype: list
+
+    """
+
+    if input is None:
+        return []
+    if isinstance(input, list):
+        return input
+    return [input]
+
+
+def escape(string):
+    """Escape a string such that it can be embedded into a Ninja file without
+    further interpretation."""
+    assert '\n' not in string, 'Ninja syntax does not allow newlines'
+    # We only have one special metacharacter: '$'.
+    return string.replace('$', '$$')
+
+
+def expand(string, vars, local_vars={}):
+    """Expand a string containing $vars as Ninja would.
+
+    Note: doesn't handle the full Ninja variable syntax, but it's enough
+    to make configure.py's use of it work.
+    """
+    def exp(m):
+        var = m.group(1)
+        if var == '$':
+            return '$'
+        return local_vars.get(var, vars.get(var, ''))
+    return re.sub(r'\$(\$|\w*)', exp, string)