# HG changeset patch # User Franz Glasner # Date 1641661056 -3600 # Node ID e15e86c47a27d1e9758bf0b4905b76408431b5be # Parent 1bdf884375558815e76719572cc59b440645b265 ADD: Configure.py ind ninjy_syntax.py from py3-extension-tests diff -r 1bdf88437555 -r e15e86c47a27 Configure.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/Configure.py Sat Jan 08 17:57:36 2022 +0100 @@ -0,0 +1,431 @@ +# +# +# Important Triplets: +# +# clang-cl (clang-cl /clang:-dumpmachine) +# +# x86_64-pc-windows-msvc +# i386-pc-windows-msvc +# +# clang on FreeBSD (clang -dumpmachine): +# +# x86_64-unknown-freebsd12.2 +# i386-unknown-freebsd12.2 +# +# NOTE: gcc also known "-dumpmachine" +# +# + +from __future__ import print_function, absolute_import + +import argparse +import collections +import datetime +import copy +import getopt +import os +import sys + +import ninja_syntax + +tool = build = host = None + +host = None + + +# +# Global build variables (ordered because they must be written ordered +# -- and with simple attribute access +# +class BuildVars(collections.OrderedDict): + + def __getattr__(self, n): + try: + return self[n] + except KeyError: + raise AttributeError(n) + + def __setattr__(self, n, v): + # private v + if n.startswith("_OrderedDict__"): + return super(BuildVars, self).__setattr__(n, v) + self[n] = v + + +def make_obj_name(name, newext): + bn = os.path.basename(name) + if not bn: + return bn + root, ext = os.path.splitext(bn) + return root + newext + +options = argparse.Namespace( + user_includes = [], + sys_includes = [], + sys_libpath = [], + user_libpath = [], + link_with_python = None, + python_limited_api = None, +) + +gbv = BuildVars() +gbv.intdir = "_builddir-test" +gbv.srcdir = "src" +gbv.builddir = "$intdir" +gbv.pxx3dir = "pxx3" + +opts, args = getopt.getopt( + sys.argv[1:], + "B:H:t:I:L:", + ["build=", + "host=", + "tool=", + "include=", + "libpath=", + "sys-include=", + "sys-libpath=", + "CXX=", + "LINK=", + "intdir=", # intermediate files + "builddir=", # Ninja builddir + "link-with-python=", # link with libpython + "python-limited-api=", # Use Py_LIMITED_API + ]) +for opt, val in opts: + if opt in ("-t", "--tool"): + if tool is None: + tool = argparse.Namespace(local=False, msvc=False, clang=False) + if val == "msvc": + tool.msvc = True + tool.compile_only = "/c" + tool.define_format = "/D {}" + tool.include_format = "/I {}" + tool.lib_format = "{}" + tool.libpath_format = "/libpath:{}" + tool.dependencies = "msvc" + elif val == "clang-cl": + tool.msvc = tool.clang = True + tool.compile_only = "/c" + tool.define_format = "/D {}" + tool.include_format = "/I {}" + tool.lib_format = "{}" + tool.libpath_format = "/libpath:{}" + tool.dependencies = "msvc" + elif val == "clang": + tool.clang = True + tool.compile_only = "-c" + tool.define_format = "-D{}" + tool.include_format = "-I{}" + tool.lib_format = "-l{}" + tool.libpath_format = "-L{}" + tool.dependencies = "gcc" + elif val in ("local", "posix"): + tool.local = True + tool.compile_only = "-c" + tool.define_format = "-D{}" + tool.include_format = "-I{}" + tool.lib_format = "-l{}" + tool.libpath_format = "-L{}" + tool.dependencies = "gcc" + else: + raise getopt.GetoptError("unknown tool value: {}".format(val), opt) + elif opt in ("-B", "--build"): + build = argparse.Namespace(type=None, + posix=False, windows=False, + pathmod=None) + if val == "windows": + # build on Windows with clang-cl + build.windows = True + build.type = "windows" + elif val == "posix": + build.posix = True + build.type = "posix" + else: + raise getopt.GetoptError("unknwon build value: {}".format(val), + opt) + elif opt in ("-H", "--host"): + if host is None: + host = argparse.Namespace(windows=False) + if val == "windows": + host.type = "windows" + host.windows = True + host.posix = False + host.objext = ".obj" + host.pydext = ".pyd" + elif val == "posix": + host.type = "posix" + host.windows = False + host.posix = True + host.objext = ".o" + host.pydext = ".so" + else: + raise getopt.GetoptError("unknown host value: {}".format(val), + opt) + elif opt in ("-I", "--include"): + options.user_includes.append(val) + elif opt in ("-L", "--libpath"): + options.user_libpath.append(val) + elif opt == "--sys-include": + options.sys_includes.append(val) + elif opt == "--sys-libpath": + options.sys_libpath.append(val) + elif opt == "--CXX": + gbv.cxx = val + elif opt == "--LINK": + gbv.link = val + elif opt == "--intdir": + gbv.intdir = val + elif opt == "--builddir": + gbv.builddir = val + elif opt == "--link-with-python": + options.link_with_python = val + elif opt == "--python-limited-api": + if val.lower().startswith("0x"): + options.python_limited_api = val + else: + options.python_limited_api = "0x03040000" + else: + raise getopt.GetoptError("Unhandled option `{}'".format(opt), opt) + +if tool is None: + print("ERROR: no tool given", file=sys.stderr) + sys.exit(1) + +if build.windows and host.posix: + print("ERROR: cross-compiling on Windows not supported", file=sys.stderr) + sys.exit(1) + +if build.windows: + import ntpath as pathmod +else: + import posixpath as pathmod +build.pathmod = pathmod + +if tool.msvc: + if tool.clang: + if not getattr(gbv, "cxx", None): + gbv.cxx = "clang-cl" + if not getattr(gbv, "link", None): + gbv.link = "lld-link" + else: + gbv.cxx = "cl" + gbv.link = "link" +elif tool.clang: + gbv.cxx = "clang++" + gbv.link = "clang++" # link C++ through the compiler +elif tool.local: + gbv.cxx = "c++" + gbv.link = "c++" # link through the compiler +else: + raise RuntimeError("tool condition is not handled") + +ext1_sources = [ + "$srcdir/ext1/testext1.cpp", + "$pxx3dir/shared/thread.cpp", +] + +ext2_sources = [ + "$srcdir/ext2/testext2.cpp", + "$srcdir/ext2/hashes.cpp", + "$pxx3dir/shared/thread.cpp", + "$pxx3dir/shared/module.cpp", + "$pxx3dir/shared/xcept.cpp", + "$pxx3dir/shared/cfunctions.cpp", + "$pxx3dir/shared/misc.cpp", + "$pxx3dir/shared/exttype.cpp", + "$pxx3dir/shared/allocator.cpp", +] + +ccflags = [] +cxxflags = [] +ccwarnings = [] +ldflags = [] + +defines = [ + "PY_SSIZE_T_CLEAN", + "HAVE_THREADS", +] +if options.python_limited_api: + defines.append("Py_LIMITED_API={}".format(options.python_limited_api)) + +# XXX TBD: handle debug/release build _DEBUG/NDEBUG + +includes = [] +includes.extend(options.sys_includes) +includes.extend(options.user_includes) + +includes.append("$pxx3dir/include") + +libpath = [] +libpath.extend(options.sys_libpath) +libpath.extend(options.user_libpath) + +libs = [] +if host.windows: + if tool.msvc: + # automatically included via #pragma + # libs.append("python3.lib") + pass +else: + if options.link_with_python: + libs.append(options.link_with_python) + +if host.windows: + defines.append("WIN32") + # XXX TBD Handle arch -> WIN64 + defines.append("WIN64") + defines.append("_WINDOWS") + # for a user dll + defines.append("_USRDLL") + defines.append("_WINDLL") + + defines.append("WIN32_LEAN_AND_MEAN") + defines.append("_WIN32_WINNT=0x0501") # WinXP + + if tool.msvc: + # XXX TBD warnings + + defines.append("_CRT_SECURE_NO_WARNINGS") + + ccflags.append("/Zi") + ccflags.append("/MD") # link to dll runtime + ccflags.append("/EHsc") + ccflags.append("/Gy") # enable function level linking + + cxxflags.append("/TP") + #cxxflags.append("/std:c++latest") + + # XXX TBD machine + ccflags.append("-m64") + + ldflags.append("/dll") + ldflags.append("/debug") # PDB output + # 32-bit: -> 5.01 64-bit: 5.02 + ldflags.append("/subsystem:windows,5.02") + ldflags.append("/incremental:no") + # + ldflags.append("/manifest:NO") + + + if tool.clang: + ccflags.append("-fms-compatibility-version=16.00") + + ccwarnings.append("-Wno-nonportable-include-path") + ccwarnings.append("-Wno-microsoft-template") + ccwarnings.append("-Wno-pragma-pack") +elif host.posix: + defines.append("PIC") + + ccwarnings.extend(["-Wall", "-Wextra", "-pedantic"]) + + ccflags.append("-g") + ccflags.append("-fPIC") + ccflags.append("-fvisibility=hidden") + ccflags.append("-pthread") + + if tool.clang: # || tool.gcc + ccflags.append("-ffunction-sections") + ccflags.append("-fdata-sections") + + if tool.clang: + ccflags.append("-faddrsig") # use with --icf=all/safe when linking + + ldflags.append("-shared") + ldflags.append("-Wl,-z,relro,-z,now") + ldflags.append("-Wl,--build-id=sha1") + # XXX TBD only when building in debug code + if options.link_with_python: + ldflags.append("-Wl,-z,defs") + + if tool.clang: # || tool.gcc + ldflags.append("-Wl,--gc-sections") + + if tool.clang: + ldflags.append("-Wl,--icf=safe") + +gbv.cppdefines = [tool.define_format.format(d) for d in defines] +gbv.includes = [tool.include_format.format(pathmod.normpath(i)) + for i in includes] +gbv.ccflags = ccflags +gbv.cxxflags = cxxflags +gbv.ccwarnings = ccwarnings +gbv.ldflags = ldflags +gbv.ldlibpath = [tool.libpath_format.format(pathmod.normpath(l)) + for l in libpath] +gbv.ldlibs = [tool.lib_format.format(l) for l in libs] + +n = ninja_syntax.Writer(sys.stdout) + +n.comment('This file is used to build test Python extensions.') +n.comment( + 'It is generated by {} at {}Z.'.format( + os.path.basename(__file__), + datetime.datetime.utcnow().isoformat())) +n.newline() +n.comment('Created using command: {!r}'.format(sys.argv)) +n.newline() +for k, v in gbv.items(): + n.variable(k, v) +n.newline() +if tool.msvc: + # Note: this includes clang-cl + n.rule("compile-pyextension-unit", + "$cxx /nologo /showIncludes /c $cppdefines $ccwarnings $includes $ccflags $cxxflags /Fd$intdir/$intsubdir/ /Fo$out $in", + deps=tool.dependencies) +else: + n.rule("compile-pyextension-unit", + "$cxx -MD -MF $intdir/_deps -MT $out $cppdefines $ccwarnings $includes $ccflags $cxxflags -c -o $out $in", + deps=tool.dependencies, + depfile="$intdir/_deps") +n.newline() +if tool.msvc: + # XXX TBD: in "release" builds use /pdbaltpath:$out.pdb + n.rule("link-pyextension", "$link /nologo $ldflags $ldlibpath /implib:$intdir/$out.lib /pdb:$intdir/$out.pdb /out:$out $in $ldlibs") +else: + n.rule("link-pyextension", "$link $cppdefines $ccwarnings $ccflags $cxxflags $ldflags -o $out $in $ldlibpath $ldlibs") +n.newline() + +n.comment("testext1") +for f in ext1_sources: + n.build(pathmod.normpath("$intdir/$intsubdir/"+make_obj_name(f, host.objext)), + "compile-pyextension-unit", + inputs=pathmod.normpath(f), + variables={"intsubdir": "ext1"}) +n.newline() +linkinputs = [pathmod.normpath("$intdir/ext1/"+make_obj_name(f, host.objext)) + for f in ext1_sources] +if tool.msvc: + implicit_outputs = [ + pathmod.normpath("$intdir/testext1"+host.pydext+".lib"), + pathmod.normpath("$intdir/testext1"+host.pydext+".pdb")] + if not tool.clang: + implicit_outputs.append(pathmod.normpath("$intdir/testext1"+host.pydext+".exp")) +else: + implicit_outputs = None +n.build("testext1"+host.pydext, + "link-pyextension", + inputs=linkinputs, + implicit_outputs=implicit_outputs) +n.newline() + +n.comment("testext2") +for f in ext2_sources: + n.build(pathmod.normpath("$intdir/$intsubdir/"+make_obj_name(f, host.objext)), + "compile-pyextension-unit", + inputs=pathmod.normpath(f), + variables={"intsubdir": "ext2"}) +n.newline() +linkinputs = [pathmod.normpath("$intdir/ext2/"+make_obj_name(f, host.objext)) + for f in ext2_sources] +if tool.msvc: + implicit_outputs = [ + pathmod.normpath("$intdir/testext2"+host.pydext+".lib"), + pathmod.normpath("$intdir/testext2"+host.pydext+".pdb")] + if not tool.clang: + implicit_outputs.append(pathmod.normpath("$intdir/testext2"+host.pydext+".exp")) +else: + implicit_outputs = None +n.build("testext2"+host.pydext, + "link-pyextension", + inputs=linkinputs, + implicit_outputs=implicit_outputs) diff -r 1bdf88437555 -r e15e86c47a27 ninja_syntax.py --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/ninja_syntax.py Sat Jan 08 17:57:36 2022 +0100 @@ -0,0 +1,242 @@ +# -*- coding: utf-8 -*- +# +# Copyright 2011 Google Inc. All Rights Reserved. +# Copyright 2020 Franz Glasner. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +r"""Python module for generating .ninja files. + +Note that this is emphatically not a required piece of Ninja; it's +just a helpful utility for build-file-generation systems that already +use Python. + +:Copyright: \(c) 2001 Google Inc. All Rights Reserved. +:Copyright: \(c) 2020 Franz Glasner. All Rights Reserved. +:License: Apache License 2.0. See :ref:`LICENSE.txt ` for details. + +""" + +from __future__ import absolute_import + +import re +import textwrap + + +def escape_path(word): + """Escape `word` according to Ninja's path interpretation rules. + + Escapes not only ``$`` but also spaces and colons. + """ + return word.replace('$ ', '$$ ').replace(' ', '$ ').replace(':', '$:') + + +class Writer(object): + + """An object-oriented writer of a Ninja build file. + + Can act as its own context manager. + """ + + def __init__(self, output, width=78): + """ + :param output: an opened file-alike where to write into + :param int width: the width where line-wrapping should happen + + `output` just needs a `write()` and -- if :meth:`~.close` is called + -- a `close()` method. + """ + self.output = output + self.width = width + + def newline(self): + self.output.write('\n') + + def comment(self, text): + for line in textwrap.wrap(text, self.width - 2, break_long_words=False, + break_on_hyphens=False): + self.output.write('# ' + line + '\n') + + def variable(self, key, value, indent=0): + if value is None: + return + if isinstance(value, list): + value = ' '.join(filter(None, value)) # Filter out empty strings. + self._line('%s = %s' % (key, value), indent) + + def pool(self, name, depth): + self._line('pool %s' % name) + self.variable('depth', depth, indent=1) + + def rule(self, name, command, description=None, depfile=None, + generator=False, pool=None, restat=False, rspfile=None, + rspfile_content=None, deps=None): + self._line('rule %s' % name) + self.variable('command', command, indent=1) + if description: + self.variable('description', description, indent=1) + if depfile: + self.variable('depfile', depfile, indent=1) + if generator: + self.variable('generator', '1', indent=1) + if pool: + self.variable('pool', pool, indent=1) + if restat: + self.variable('restat', '1', indent=1) + if rspfile: + self.variable('rspfile', rspfile, indent=1) + if rspfile_content: + self.variable('rspfile_content', rspfile_content, indent=1) + if deps: + self.variable('deps', deps, indent=1) + + def build(self, outputs, rule, inputs=None, implicit=None, order_only=None, + variables=None, implicit_outputs=None, pool=None): + outputs = as_list(outputs) + out_outputs = [escape_path(x) for x in outputs] + all_inputs = [escape_path(x) for x in as_list(inputs)] + + if implicit: + implicit = [escape_path(x) for x in as_list(implicit)] + all_inputs.append('|') + all_inputs.extend(implicit) + if order_only: + order_only = [escape_path(x) for x in as_list(order_only)] + all_inputs.append('||') + all_inputs.extend(order_only) + if implicit_outputs: + implicit_outputs = [escape_path(x) + for x in as_list(implicit_outputs)] + out_outputs.append('|') + out_outputs.extend(implicit_outputs) + + self._line('build %s: %s' % (' '.join(out_outputs), + ' '.join([rule] + all_inputs))) + if pool is not None: + self._line(' pool = %s' % pool) + + if variables: + if isinstance(variables, dict): + iterator = iter(variables.items()) + else: + iterator = iter(variables) + + for key, val in iterator: + self.variable(key, val, indent=1) + + return outputs + + def include(self, path): + self._line('include %s' % path) + + def subninja(self, path): + self._line('subninja %s' % path) + + def default(self, paths): + self._line('default %s' % ' '.join(as_list(paths))) + + def _count_dollars_before_index(self, s, i): + """Returns the number of '$' characters right in front of s[i].""" + dollar_count = 0 + dollar_index = i - 1 + while dollar_index > 0 and s[dollar_index] == '$': + dollar_count += 1 + dollar_index -= 1 + return dollar_count + + def _line(self, text, indent=0): + """Write 'text' word-wrapped at self.width characters.""" + leading_space = ' ' * indent + while len(leading_space) + len(text) > self.width: + # The text is too wide; wrap if possible. + + # Find the rightmost space that would obey our width constraint and + # that's not an escaped space. + available_space = self.width - len(leading_space) - len(' $') + space = available_space + while True: + space = text.rfind(' ', 0, space) + if (space < 0 + or self._count_dollars_before_index(text, space) % 2 == 0): # noqa: E501 + break + + if space < 0: + # + # No such space; + # just use the first unescaped space we can find. + # + space = available_space - 1 + while True: + space = text.find(' ', space + 1) + if (space < 0 + or self._count_dollars_before_index(text, space) % 2 == 0): # noqa: E501 + break + if space < 0: + # Give up on breaking. + break + + self.output.write(leading_space + text[0:space] + ' $\n') + text = text[space+1:] + + # Subsequent lines are continuations, so indent them. + leading_space = ' ' * (indent+2) + + self.output.write(leading_space + text + '\n') + + def close(self): + """Close the associated output""" + self.output.close() + + def __enter__(self): + """Return `self`""" + return self + + def __exit__(self, *args, **kwds): + """Close the associated output""" + self.close() + + +def as_list(input): + """Ensure that `input` is always a :class:`list` -- wrap if needed + + :rtype: list + + """ + + if input is None: + return [] + if isinstance(input, list): + return input + return [input] + + +def escape(string): + """Escape a string such that it can be embedded into a Ninja file without + further interpretation.""" + assert '\n' not in string, 'Ninja syntax does not allow newlines' + # We only have one special metacharacter: '$'. + return string.replace('$', '$$') + + +def expand(string, vars, local_vars={}): + """Expand a string containing $vars as Ninja would. + + Note: doesn't handle the full Ninja variable syntax, but it's enough + to make configure.py's use of it work. + """ + def exp(m): + var = m.group(1) + if var == '$': + return '$' + return local_vars.get(var, vars.get(var, '')) + return re.sub(r'\$(\$|\w*)', exp, string)