Mercurial > hgrepos > Python2 > PyMuPDF
diff pipcl.py @ 39:a6bc019ac0b2 upstream
ADD: PyMuPDF v1.26.5: the original sdist.
| author | Franz Glasner <fzglas.hg@dom66.de> |
|---|---|
| date | Sat, 11 Oct 2025 11:19:58 +0200 |
| parents | 1d09e1dec1d9 |
| children | 71bcc18e306f |
line wrap: on
line diff
--- a/pipcl.py Mon Sep 15 11:43:07 2025 +0200 +++ b/pipcl.py Sat Oct 11 11:19:58 2025 +0200 @@ -2,23 +2,37 @@ Python packaging operations, including PEP-517 support, for use by a `setup.py` script. -The intention is to take care of as many packaging details as possible so that -setup.py contains only project-specific information, while also giving as much -flexibility as possible. - -For example we provide a function `build_extension()` that can be used to build -a SWIG extension, but we also give access to the located compiler/linker so -that a `setup.py` script can take over the details itself. - -Run doctests with: `python -m doctest pipcl.py` - -For Graal we require that PIPCL_GRAAL_PYTHON is set to non-graal Python (we -build for non-graal except with Graal Python's include paths and library -directory). +Overview: + + The intention is to take care of as many packaging details as possible so + that setup.py contains only project-specific information, while also giving + as much flexibility as possible. + + For example we provide a function `build_extension()` that can be used + to build a SWIG extension, but we also give access to the located + compiler/linker so that a `setup.py` script can take over the details + itself. + +Doctests: + Doctest strings are provided in some comments. + + Test in the usual way with: + python -m doctest pipcl.py + + Test specific functions/classes with: + python pipcl.py --doctest run_if ... + + If no functions or classes are specified, this tests everything. + +Graal: + For Graal we require that PIPCL_GRAAL_PYTHON is set to non-graal Python (we + build for non-graal except with Graal Python's include paths and library + directory). ''' import base64 import codecs +import difflib import glob import hashlib import inspect @@ -55,6 +69,9 @@ by legacy distutils/setuptools and described in: https://pip.pypa.io/en/stable/reference/build-system/setup-py/ + The file pyproject.toml must exist; this is checked if/when fn_build() is + called. + Here is a `doctest` example of using pipcl to create a SWIG extension module. Requires `swig`. @@ -321,63 +338,86 @@ wheel_compresslevel = None, ): ''' - The initial args before `root` define the package - metadata and closely follow the definitions in: + The initial args before `entry_points` define the + package metadata and closely follow the definitions in: https://packaging.python.org/specifications/core-metadata/ Args: name: + Used for metadata `Name`. A string, the name of the Python package. version: + Used for metadata `Version`. A string, the version of the Python package. Also see PEP-440 `Version Identification and Dependency Specification`. platform: + Used for metadata `Platform`. A string or list of strings. supported_platform: + Used for metadata `Supported-Platform`. A string or list of strings. summary: + Used for metadata `Summary`. A string, short description of the package. description: + Used for metadata `Description`. A string. If contains newlines, a detailed description of the package. Otherwise the path of a file containing the detailed description of the package. description_content_type: + Used for metadata `Description-Content-Type`. A string describing markup of `description` arg. For example `text/markdown; variant=GFM`. keywords: + Used for metadata `Keywords`. A string containing comma-separated keywords. home_page: + Used for metadata `Home-page`. URL of home page. download_url: + Used for metadata `Download-URL`. Where this version can be downloaded from. author: + Used for metadata `Author`. Author. author_email: + Used for metadata `Author-email`. Author email. maintainer: + Used for metadata `Maintainer`. Maintainer. maintainer_email: + Used for metadata `Maintainer-email`. Maintainer email. license: + Used for metadata `License`. A string containing the license text. Written into metadata file `COPYING`. Is also written into metadata itself if not multi-line. classifier: + Used for metadata `Classifier`. A string or list of strings. Also see: * https://pypi.org/pypi?%3Aaction=list_classifiers * https://pypi.org/classifiers/ requires_dist: - A string or list of strings. None items are ignored. Also see PEP-508. + Used for metadata `Requires-Dist`. + A string or list of strings, Python packages required + at runtime. None items are ignored. requires_python: + Used for metadata `Requires-Python`. A string or list of strings. requires_external: + Used for metadata `Requires-External`. A string or list of strings. project_url: - A string or list of strings, each of the form: `{name}, {url}`. + Used for metadata `Project-URL`. + A string or list of strings, each of the form: `{name}, + {url}`. provides_extra: + Used for metadata `Provides-Extra`. A string or list of strings. entry_points: @@ -415,8 +455,11 @@ added. `to_` identifies what the file should be called within a wheel - or when installing. If `to_` ends with `/`, the leaf of `from_` - is appended to it (and `from_` must not be a `bytes`). + or when installing. If `to_` is empty or `/` we set it to the + leaf of `from_` (`from_` must not be a `bytes`) - i.e. we place + the file in the root directory of the wheel; otherwise if + `to_` ends with `/` the leaf of `from_` is appended to it (and + `from_` must not be a `bytes`). Initial `$dist-info/` in `_to` is replaced by `{name}-{version}.dist-info/`; this is useful for license files @@ -439,6 +482,11 @@ default being `sysconfig.get_path('platlib')` e.g. `myvenv/lib/python3.9/site-packages/`. + When calling this function, we assert that the file + pyproject.toml exists in the current directory. (We do this + here rather than in pipcl.Package's constructor, as otherwise + importing setup.py from non-package-related code could fail.) + fn_clean: A function taking a single arg `all_` that cleans generated files. `all_` is true iff `--all` is in argv. @@ -457,8 +505,7 @@ It can be convenient to use `pipcl.git_items()`. The specification for sdists requires that the list contains - `pyproject.toml`; we enforce this with a diagnostic rather than - raising an exception, to allow legacy command-line usage. + `pyproject.toml`; we enforce this with a Python assert. tag_python: First element of wheel tag defined in PEP-425. If None we use @@ -528,6 +575,12 @@ assert_str_or_multi( requires_external) assert_str_or_multi( project_url) assert_str_or_multi( provides_extra) + + assert re.match('^([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])\\Z', name, re.IGNORECASE), ( + f'Invalid package name' + f' (https://packaging.python.org/en/latest/specifications/name-normalization/)' + f': {name!r}' + ) # https://packaging.python.org/en/latest/specifications/core-metadata/. assert re.match('([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$', name, re.IGNORECASE), \ @@ -602,7 +655,10 @@ f' metadata_directory={metadata_directory!r}' ) - if sys.implementation.name == 'graalpy': + if os.environ.get('CIBUILDWHEEL') == '1': + # Don't special-case graal builds when running under cibuildwheel. + pass + elif sys.implementation.name == 'graalpy': # We build for Graal by building a native Python wheel with Graal # Python's include paths and library directory. We then rename the # wheel to contain graal's tag etc. @@ -754,7 +810,7 @@ else: items = self.fn_sdist() - prefix = f'{_normalise(self.name)}-{self.version}' + prefix = f'{_normalise2(self.name)}-{self.version}' os.makedirs(sdist_directory, exist_ok=True) tarpath = f'{sdist_directory}/{prefix}.tar.gz' log2(f'Creating sdist: {tarpath}') @@ -796,12 +852,11 @@ assert 0, f'Path is inside sdist_directory={sdist_directory}: {from_!r}' assert os.path.exists(from_), f'Path does not exist: {from_!r}' assert os.path.isfile(from_), f'Path is not a file: {from_!r}' - if to_rel == 'pyproject.toml': - found_pyproject_toml = True add(from_, to_rel) - - if not found_pyproject_toml: - log0(f'Warning: no pyproject.toml specified.') + if to_rel == 'pyproject.toml': + found_pyproject_toml = True + + assert found_pyproject_toml, f'Cannot create sdist because file not specified: pyproject.toml' # Always add a PKG-INFO file. add_string(self._metainfo(), 'PKG-INFO') @@ -826,9 +881,11 @@ Get two-digit python version, e.g. 'cp3.8' for python-3.8.6. ''' if self.tag_python_: - return self.tag_python_ + ret = self.tag_python_ else: - return 'cp' + ''.join(platform.python_version().split('.')[:2]) + ret = 'cp' + ''.join(platform.python_version().split('.')[:2]) + assert '-' not in ret + return ret def tag_abi(self): ''' @@ -884,10 +941,13 @@ ret = ret2 log0( f'tag_platform(): returning {ret=}.') + assert '-' not in ret return ret def wheel_name(self): - return f'{_normalise(self.name)}-{self.version}-{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}.whl' + ret = f'{_normalise2(self.name)}-{self.version}-{self.tag_python()}-{self.tag_abi()}-{self.tag_platform()}.whl' + assert ret.count('-') == 4, f'Expected 4 dash characters in {ret=}.' + return ret def wheel_name_match(self, wheel): ''' @@ -916,7 +976,7 @@ log2(f'py_limited_api; {tag_python=} compatible with {self.tag_python()=}.') py_limited_api_compatible = True - log2(f'{_normalise(self.name) == name=}') + log2(f'{_normalise2(self.name) == name=}') log2(f'{self.version == version=}') log2(f'{self.tag_python() == tag_python=} {self.tag_python()=} {tag_python=}') log2(f'{py_limited_api_compatible=}') @@ -925,7 +985,7 @@ log2(f'{self.tag_platform()=}') log2(f'{tag_platform.split(".")=}') ret = (1 - and _normalise(self.name) == name + and _normalise2(self.name) == name and self.version == version and (self.tag_python() == tag_python or py_limited_api_compatible) and self.tag_abi() == tag_abi @@ -947,6 +1007,9 @@ def _call_fn_build( self, config_settings=None): assert self.fn_build + assert os.path.isfile('pyproject.toml'), ( + 'Cannot create package because file does not exist: pyproject.toml' + ) log2(f'calling self.fn_build={self.fn_build}') if inspect.signature(self.fn_build).parameters: ret = self.fn_build(config_settings) @@ -954,6 +1017,28 @@ ret = self.fn_build() assert isinstance( ret, (list, tuple)), \ f'Expected list/tuple from {self.fn_build} but got: {ret!r}' + + # Check that any extensions that we have built, have same + # py_limited_api value. If package is marked with py_limited_api=True + # then non-py_limited_api extensions seem to fail at runtime on + # Windows. + # + # (We could possibly allow package py_limited_api=False and extensions + # py_limited_api=True, but haven't tested this, and it seems simpler to + # be strict.) + for item in ret: + from_, (to_abs, to_rel) = self._fromto(item) + from_abs = os.path.abspath(from_) + is_py_limited_api = _extensions_to_py_limited_api.get(from_abs) + if is_py_limited_api is not None: + assert bool(self.py_limited_api) == bool(is_py_limited_api), ( + f'Extension was built with' + f' py_limited_api={is_py_limited_api} but pipcl.Package' + f' name={self.name!r} has' + f' py_limited_api={self.py_limited_api}:' + f' {from_abs!r}' + ) + return ret @@ -1052,7 +1137,7 @@ it writes to a slightly different directory. ''' if root is None: - root = f'{self.name}-{self.version}.dist-info' + root = f'{normalise2(self.name)}-{self.version}.dist-info' self._write_info(f'{root}/METADATA') if self.license: with open( f'{root}/COPYING', 'w') as f: @@ -1340,7 +1425,7 @@ ) def _dist_info_dir( self): - return f'{_normalise(self.name)}-{self.version}.dist-info' + return f'{_normalise2(self.name)}-{self.version}.dist-info' def _metainfo(self): ''' @@ -1446,8 +1531,12 @@ `p` is a tuple `(from_, to_)` where `from_` is str/bytes and `to_` is str. If `from_` is a bytes it is contents of file to add, otherwise the path of an existing file; non-absolute paths are assumed to be relative - to `self.root`. If `to_` is empty or ends with `/`, we append the leaf - of `from_` (which must be a str). + to `self.root`. + + If `to_` is empty or `/` we set it to the leaf of `from_` (which must + be a str) - i.e. we place the file in the root directory of the wheel; + otherwise if `to_` ends with `/` we append the leaf of `from_` (which + must be a str). If `to_` starts with `$dist-info/`, we replace this with `self._dist_info_dir()`. @@ -1467,14 +1556,16 @@ from_, to_ = p assert isinstance(from_, (str, bytes)) assert isinstance(to_, str) - if to_.endswith('/') or to_=='': + if to_ == '/' or to_ == '': + to_ = os.path.basename(from_) + elif to_.endswith('/'): to_ += os.path.basename(from_) prefix = '$dist-info/' if to_.startswith( prefix): to_ = f'{self._dist_info_dir()}/{to_[ len(prefix):]}' prefix = '$data/' if to_.startswith( prefix): - to_ = f'{self.name}-{self.version}.data/{to_[ len(prefix):]}' + to_ = f'{_normalise2(self.name)}-{self.version}.data/{to_[ len(prefix):]}' if isinstance(from_, str): from_, _ = self._path_relative_to_root( from_, assert_within_root=False) to_ = self._path_relative_to_root(to_) @@ -1482,11 +1573,13 @@ log2(f'returning {from_=} {to_=}') return from_, to_ +_extensions_to_py_limited_api = dict() def build_extension( name, path_i, outdir, + *, builddir=None, includes=None, defines=None, @@ -1498,6 +1591,7 @@ linker_extra='', swig=None, cpp=True, + source_extra=None, prerequisites_swig=None, prerequisites_compile=None, prerequisites_link=None, @@ -1539,7 +1633,7 @@ A string, or a sequence of library names. Each item is prefixed with `-l` on non-Windows. optimise: - Whether to use compiler optimisations. + Whether to use compiler optimisations and define NDEBUG. debug: Whether to build with debug symbols. compiler_extra: @@ -1550,6 +1644,8 @@ Swig command; if false we use 'swig'. cpp: If true we tell SWIG to generate C++ code instead of C. + source_extra: + Extra source files to build into the shared library, prerequisites_swig: prerequisites_compile: prerequisites_link: @@ -1584,10 +1680,15 @@ `compile_extra` (also `/I` on windows) and use them with swig so that it can see the same header files as C/C++. This is useful when using enviromment variables such as `CC` and `CXX` to set - `compile_extra. + `compile_extra`. py_limited_api: If true we build for current Python's limited API / stable ABI. + Note that we will assert false if this extension is added to a + pipcl.Package that has a different <py_limited_api>, because + on Windows importing a non-py_limited_api extension inside a + py_limited=True package fails. + Returns the leafname of the generated library file within `outdir`, e.g. `_{name}.so` on Unix or `_{name}.cp311-win_amd64.pyd` on Windows. ''' @@ -1599,6 +1700,12 @@ builddir = outdir if not swig: swig = 'swig' + + if source_extra is None: + source_extra = list() + if isinstance(source_extra, str): + source_extra = [source_extra] + includes_text = _flags( includes, '-I') defines_text = _flags( defines, '-D') libpaths_text = _flags( libpaths, '/LIBPATH:', '"') if windows() else _flags( libpaths, '-L') @@ -1608,11 +1715,11 @@ os.makedirs( outdir, exist_ok=True) # Run SWIG. - + # if infer_swig_includes: # Extract include flags from `compiler_extra`. swig_includes_extra = '' - compiler_extra_items = compiler_extra.split() + compiler_extra_items = shlex.split(compiler_extra) i = 0 while i < len(compiler_extra_items): item = compiler_extra_items[i] @@ -1647,75 +1754,130 @@ prerequisites_swig2, ) - so_suffix = _so_suffix(use_so_versioning = not py_limited_api) + if pyodide(): + so_suffix = '.so' + log0(f'pyodide: PEP-3149 suffix untested, so omitting. {_so_suffix()=}.') + else: + so_suffix = _so_suffix(use_so_versioning = not py_limited_api) path_so_leaf = f'_{name}{so_suffix}' path_so = f'{outdir}/{path_so_leaf}' py_limited_api2 = current_py_limited_api() if py_limited_api else None + compiler_command, pythonflags = base_compiler(cpp=cpp) + linker_command, _ = base_linker(cpp=cpp) + # setuptools on Linux seems to use slightly different compile flags: + # + # -fwrapv -O3 -Wall -O2 -g0 -DPY_CALL_TRAMPOLINE + # + + general_flags = '' if windows(): - path_obj = f'{path_so}.obj' - permissive = '/permissive-' EHsc = '/EHsc' T = '/Tp' if cpp else '/Tc' optimise2 = '/DNDEBUG /O2' if optimise else '/D_DEBUG' - debug2 = '' - if debug: - debug2 = '/Zi' # Generate .pdb. - # debug2 = '/Z7' # Embed debug info in .obj files. - + debug2 = '/Zi' if debug else '' py_limited_api3 = f'/DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else '' - # As of 2023-08-23, it looks like VS tools create slightly - # .dll's each time, even with identical inputs. - # - # Some info about this is at: - # https://nikhilism.com/post/2020/windows-deterministic-builds/. - # E.g. an undocumented linker flag `/Brepro`. + else: + if debug: + general_flags += '/Zi' if windows() else ' -g' + if optimise: + general_flags += ' /DNDEBUG /O2' if windows() else ' -O2 -DNDEBUG' + + py_limited_api3 = f'-DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else '' + + if windows(): + pass + elif darwin(): + # MacOS's linker does not like `-z origin`. + rpath_flag = "-Wl,-rpath,@loader_path/" + # Avoid `Undefined symbols for ... "_PyArg_UnpackTuple" ...'. + general_flags += ' -undefined dynamic_lookup' + elif pyodide(): + # Setting `-Wl,-rpath,'$ORIGIN',-z,origin` gives: + # emcc: warning: ignoring unsupported linker flag: `-rpath` [-Wlinkflags] + # wasm-ld: error: unknown -z value: origin # - - command, pythonflags = base_compiler(cpp=cpp) - command = f''' - {command} - # General: - /c # Compiles without linking. - {EHsc} # Enable "Standard C++ exception handling". - - #/MD # Creates a multithreaded DLL using MSVCRT.lib. - {'/MDd' if debug else '/MD'} - - # Input/output files: - {T}{path_cpp} # /Tp specifies C++ source file. - /Fo{path_obj} # Output file. codespell:ignore - - # Include paths: - {includes_text} - {pythonflags.includes} # Include path for Python headers. - - # Code generation: - {optimise2} - {debug2} - {permissive} # Set standard-conformance mode. - - # Diagnostics: - #/FC # Display full path of source code files passed to cl.exe in diagnostic text. - /W3 # Sets which warning level to output. /W3 is IDE default. - /diagnostics:caret # Controls the format of diagnostic messages. - /nologo # - - {defines_text} - {compiler_extra} - - {py_limited_api3} - ''' - run_if( command, path_obj, path_cpp, prerequisites_compile) - - command, pythonflags = base_linker(cpp=cpp) + rpath_flag = "-Wl,-rpath,'$ORIGIN'" + else: + rpath_flag = "-Wl,-rpath,'$ORIGIN',-z,origin" + + # Fun fact - on Linux, if the -L and -l options are before '{path_cpp}' + # they seem to be ignored... + # + path_os = list() + + for path_source in [path_cpp] + source_extra: + path_o = f'{path_source}.obj' if windows() else f'{path_source}.o' + path_os.append(f' {path_o}') + + prerequisites_path = f'{path_o}.d' + + if windows(): + compiler_command2 = f''' + {compiler_command} + # General: + /c # Compiles without linking. + {EHsc} # Enable "Standard C++ exception handling". + + #/MD # Creates a multithreaded DLL using MSVCRT.lib. + {'/MDd' if debug else '/MD'} + + # Input/output files: + {T}{path_source} # /Tp specifies C++ source file. + /Fo{path_o} # Output file. codespell:ignore + + # Include paths: + {includes_text} + {pythonflags.includes} # Include path for Python headers. + + # Code generation: + {optimise2} + {debug2} + {permissive} # Set standard-conformance mode. + + # Diagnostics: + #/FC # Display full path of source code files passed to cl.exe in diagnostic text. + /W3 # Sets which warning level to output. /W3 is IDE default. + /diagnostics:caret # Controls the format of diagnostic messages. + /nologo # + + {defines_text} + {compiler_extra} + + {py_limited_api3} + ''' + + else: + compiler_command2 = f''' + {compiler_command} + -fPIC + {general_flags.strip()} + {pythonflags.includes} + {includes_text} + {defines_text} + -MD -MF {prerequisites_path} + -c {path_source} + -o {path_o} + {compiler_extra} + {py_limited_api3} + ''' + run_if( + compiler_command2, + path_o, + path_source, + [path_source] + _get_prerequisites(prerequisites_path), + ) + + # Link + prerequisites_path = f'{path_so}.d' + if windows(): debug2 = '/DEBUG' if debug else '' base, _ = os.path.splitext(path_so_leaf) - command = f''' - {command} + command2 = f''' + {linker_command} /DLL # Builds a DLL. /EXPORT:PyInit__{name} # Exports a function. /IMPLIB:{base}.lib # Overrides the default import library name. @@ -1725,139 +1887,67 @@ {debug2} /nologo {libs_text} - {path_obj} + {' '.join(path_os)} {linker_extra} ''' - run_if( command, path_so, path_obj, prerequisites_link) - + elif pyodide(): + command2 = f''' + {linker_command} + -MD -MF {prerequisites_path} + -o {path_so} + {' '.join(path_os)} + {libpaths_text} + {libs_text} + {linker_extra} + {pythonflags.ldflags} + {rpath_flag} + ''' else: - - # Not Windows. - # - command, pythonflags = base_compiler(cpp=cpp) - - # setuptools on Linux seems to use slightly different compile flags: - # - # -fwrapv -O3 -Wall -O2 -g0 -DPY_CALL_TRAMPOLINE - # - - general_flags = '' - if debug: - general_flags += ' -g' - if optimise: - general_flags += ' -O2 -DNDEBUG' - - py_limited_api3 = f'-DPy_LIMITED_API={py_limited_api2}' if py_limited_api2 else '' - - if darwin(): - # MacOS's linker does not like `-z origin`. - rpath_flag = "-Wl,-rpath,@loader_path/" - - # Avoid `Undefined symbols for ... "_PyArg_UnpackTuple" ...'. - general_flags += ' -undefined dynamic_lookup' - elif pyodide(): - # Setting `-Wl,-rpath,'$ORIGIN',-z,origin` gives: - # emcc: warning: ignoring unsupported linker flag: `-rpath` [-Wlinkflags] - # wasm-ld: error: unknown -z value: origin - # - log0(f'pyodide: PEP-3149 suffix untested, so omitting. {_so_suffix()=}.') - path_so_leaf = f'_{name}.so' - path_so = f'{outdir}/{path_so_leaf}' - - rpath_flag = '' - else: - rpath_flag = "-Wl,-rpath,'$ORIGIN',-z,origin" - path_so = f'{outdir}/{path_so_leaf}' - # Fun fact - on Linux, if the -L and -l options are before '{path_cpp}' - # they seem to be ignored... - # - prerequisites = list() - - if pyodide(): - # Looks like pyodide's `cc` can't compile and link in one invocation. - prerequisites_compile_path = f'{path_cpp}.o.d' - prerequisites += _get_prerequisites( prerequisites_compile_path) - command = f''' - {command} - -fPIC - {general_flags.strip()} - {pythonflags.includes} - {includes_text} - {defines_text} - -MD -MF {prerequisites_compile_path} - -c {path_cpp} - -o {path_cpp}.o - {compiler_extra} - {py_limited_api3} - ''' - prerequisites_link_path = f'{path_cpp}.o.d' - prerequisites += _get_prerequisites( prerequisites_link_path) - ld, _ = base_linker(cpp=cpp) - command += f''' - && {ld} - {path_cpp}.o - -o {path_so} - -MD -MF {prerequisites_link_path} - {rpath_flag} - {libpaths_text} - {libs_text} - {linker_extra} - {pythonflags.ldflags} - ''' - else: - # We use compiler to compile and link in one command. - prerequisites_path = f'{path_so}.d' - prerequisites = _get_prerequisites(prerequisites_path) - - command = f''' - {command} - -fPIC - -shared - {general_flags.strip()} - {pythonflags.includes} - {includes_text} - {defines_text} - {path_cpp} - -MD -MF {prerequisites_path} - -o {path_so} - {compiler_extra} - {libpaths_text} - {linker_extra} - {pythonflags.ldflags} - {libs_text} - {rpath_flag} - {py_limited_api3} - ''' - command_was_run = run_if( - command, - path_so, - path_cpp, - prerequisites_compile, - prerequisites_link, - prerequisites, - ) - - if command_was_run and darwin(): - # We need to patch up references to shared libraries in `libs`. - sublibraries = list() - for lib in () if libs is None else libs: - for libpath in libpaths: - found = list() - for suffix in '.so', '.dylib': - path = f'{libpath}/lib{os.path.basename(lib)}{suffix}' - if os.path.exists( path): - found.append( path) - if found: - assert len(found) == 1, f'More than one file matches lib={lib!r}: {found}' - sublibraries.append( found[0]) - break - else: - log2(f'Warning: can not find path of lib={lib!r} in libpaths={libpaths}') - macos_patch( path_so, *sublibraries) + command2 = f''' + {linker_command} + -shared + {general_flags.strip()} + -MD -MF {prerequisites_path} + -o {path_so} + {' '.join(path_os)} + {libpaths_text} + {libs_text} + {linker_extra} + {pythonflags.ldflags} + {rpath_flag} + {py_limited_api3} + ''' + link_was_run = run_if( + command2, + path_so, + path_cpp, + *path_os, + *_get_prerequisites(f'{path_so}.d'), + ) + + if link_was_run and darwin(): + # We need to patch up references to shared libraries in `libs`. + sublibraries = list() + for lib in () if libs is None else libs: + for libpath in libpaths: + found = list() + for suffix in '.so', '.dylib': + path = f'{libpath}/lib{os.path.basename(lib)}{suffix}' + if os.path.exists( path): + found.append( path) + if found: + assert len(found) == 1, f'More than one file matches lib={lib!r}: {found}' + sublibraries.append( found[0]) + break + else: + log2(f'Warning: can not find path of lib={lib!r} in libpaths={libpaths}') + macos_patch( path_so, *sublibraries) #run(f'ls -l {path_so}', check=0) #run(f'file {path_so}', check=0) + _extensions_to_py_limited_api[os.path.abspath(path_so)] = py_limited_api + return path_so_leaf @@ -1983,7 +2073,7 @@ ) if not e: branch = out.strip() - log(f'git_info(): directory={directory!r} returning branch={branch!r} sha={sha!r} comment={comment!r}') + log1(f'git_info(): directory={directory!r} returning branch={branch!r} sha={sha!r} comment={comment!r}') return sha, comment, diff, branch @@ -2027,88 +2117,96 @@ def git_get( - remote, local, *, + remote=None, branch=None, + tag=None, + text=None, depth=1, env_extra=None, - tag=None, update=True, submodules=True, - default_remote=None, ): ''' - Ensures that <local> is a git checkout (at either <tag>, or <branch> HEAD) - of a remote repository. - - Exactly one of <branch> and <tag> must be specified, or <remote> must start - with 'git:' and match the syntax described below. + Creates/updates local checkout <local> of remote repository and returns + absolute path of <local>. + + If <text> is set but does not start with 'git:', it is assumed to be an up + to date local checkout, and we return absolute path of <text> without doing + any git operations. Args: + local: + Local directory. Created and/or updated using `git clone` and `git + fetch` etc. remote: Remote git repostitory, for example - 'https://github.com/ArtifexSoftware/mupdf.git'. + 'https://github.com/ArtifexSoftware/mupdf.git'. Can be overridden + by <text>. + branch: + Branch to use; can be overridden by <text>. + tag: + Tag to use; can be overridden by <text>. + text: + If None or empty: + Ignored. - If starts with 'git:', the remaining text should be a command-line - style string containing some or all of these args: - --branch <branch> - --tag <tag> - <remote> - These overrides <branch>, <tag> and <default_remote>. + If starts with 'git:': + The remaining text should be a command-line + style string containing some or all of these args: + --branch <branch> + --tag <tag> + <remote> + These overrides <branch>, <tag> and <remote>. + Otherwise: + <text> is assumed to be a local directory, and we simply return + it as an absolute path without doing any git operations. For example these all clone/update/branch master of https://foo.bar/qwerty.git to local checkout 'foo-local': - git_get('https://foo.bar/qwerty.git', 'foo-local', branch='master') - git_get('git:--branch master https://foo.bar/qwerty.git', 'foo-local') - git_get('git:--branch master', 'foo-local', default_remote='https://foo.bar/qwerty.git') - git_get('git:', 'foo-local', branch='master', default_remote='https://foo.bar/qwerty.git') - - local: - Local directory. If <local>/.git exists, we attempt to run `git - update` in it. - branch: - Branch to use. Is used as default if remote starts with 'git:'. + git_get('foo-local', remote='https://foo.bar/qwerty.git', branch='master') + git_get('foo-local', text='git:--branch master https://foo.bar/qwerty.git') + git_get('foo-local', text='git:--branch master', remote='https://foo.bar/qwerty.git') + git_get('foo-local', text='git:', branch='master', remote='https://foo.bar/qwerty.git') depth: Depth of local checkout when cloning and fetching, or None. env_extra: Dict of extra name=value environment variables to use whenever we run git. - tag: - Tag to use. Is used as default if remote starts with 'git:'. update: If false we do not update existing repository. Might be useful if testing without network access. submodules: If true, we clone with `--recursive --shallow-submodules` and run `git submodule update --init --recursive` before returning. - default_remote: - The remote URL if <remote> starts with 'git:' but does not specify - the remote URL. ''' log0(f'{remote=} {local=} {branch=} {tag=}') - if remote.startswith('git:'): - remote0 = remote - args = iter(shlex.split(remote0[len('git:'):])) - remote = default_remote - while 1: - try: - arg = next(args) - except StopIteration: - break - if arg == '--branch': - branch = next(args) - tag = None - elif arg == '--tag': - tag == next(args) - branch = None - else: - remote = arg - assert remote, f'{default_remote=} and no remote specified in remote={remote0!r}.' - assert branch or tag, f'{branch=} {tag=} and no branch/tag specified in remote={remote0!r}.' + + if text: + if text.startswith('git:'): + args = iter(shlex.split(text[len('git:'):])) + while 1: + try: + arg = next(args) + except StopIteration: + break + if arg == '--branch': + branch = next(args) + tag = None + elif arg == '--tag': + tag = next(args) + branch = None + else: + remote = arg + assert remote, f'<remote> unset and no remote specified in {text=}.' + assert branch or tag, f'<branch> and <tag> unset and no branch/tag specified in {text=}.' + else: + log0(f'Using local directory {text!r}.') + return os.path.abspath(text) - assert (branch and not tag) or (not branch and tag), f'Must specify exactly one of <branch> and <tag>.' + assert (branch and not tag) or (not branch and tag), f'Must specify exactly one of <branch> and <tag>; {branch=} {tag=}.' depth_arg = f' --depth {depth}' if depth else '' @@ -2116,7 +2214,7 @@ # This seems to pull in the entire repository. log0(f'do_update(): attempting to update {local=}.') # Remove any local changes. - run(f'cd {local} && git checkout .', env_extra=env_extra) + run(f'cd {local} && git reset --hard', env_extra=env_extra) if tag: # `-u` avoids `fatal: Refusing to fetch into current branch`. # Using '+' and `revs/tags/` prefix seems to avoid errors like: @@ -2164,6 +2262,7 @@ # Show sha of checkout. run( f'cd {local} && git show --pretty=oneline|head -n 1', check=False) + return os.path.abspath(local) def run( @@ -2452,10 +2551,11 @@ log2(f'### Have removed `-lcrypt` from ldflags: {self.ldflags!r} -> {ldflags2!r}') self.ldflags = ldflags2 - log1(f'{self.includes=}') - log1(f' {includes_=}') - log1(f'{self.ldflags=}') - log1(f' {ldflags_=}') + if 0: + log1(f'{self.includes=}') + log1(f' {includes_=}') + log1(f'{self.ldflags=}') + log1(f' {ldflags_=}') def macos_add_cross_flags(command): @@ -2555,7 +2655,7 @@ return f'x{32 if sys.maxsize == 2**31 - 1 else 64}' -def run_if( command, out, *prerequisites): +def run_if( command, out, *prerequisites, caller=1): ''' Runs a command only if the output file is not up to date. @@ -2585,21 +2685,26 @@ ... os.remove( out) >>> if os.path.exists( f'{out}.cmd'): ... os.remove( f'{out}.cmd') - >>> run_if( f'touch {out}', out) + >>> run_if( f'touch {out}', out, caller=0) pipcl.py:run_if(): Running command because: File does not exist: 'run_if_test_out' pipcl.py:run_if(): Running: touch run_if_test_out True If we repeat, the output file will be up to date so the command is not run: - >>> run_if( f'touch {out}', out) + >>> run_if( f'touch {out}', out, caller=0) pipcl.py:run_if(): Not running command because up to date: 'run_if_test_out' If we change the command, the command is run: - >>> run_if( f'touch {out}', out) - pipcl.py:run_if(): Running command because: Command has changed - pipcl.py:run_if(): Running: touch run_if_test_out + >>> run_if( f'touch {out};', out, caller=0) + pipcl.py:run_if(): Running command because: Command has changed: + pipcl.py:run_if(): @@ -1,2 +1,2 @@ + pipcl.py:run_if(): touch + pipcl.py:run_if(): -run_if_test_out + pipcl.py:run_if(): +run_if_test_out; + pipcl.py:run_if(): + pipcl.py:run_if(): Running: touch run_if_test_out; True If we add a prerequisite that is newer than the output, the command is run: @@ -2608,15 +2713,20 @@ >>> prerequisite = 'run_if_test_prerequisite' >>> run( f'touch {prerequisite}', caller=0) pipcl.py:run(): Running: touch run_if_test_prerequisite - >>> run_if( f'touch {out}', out, prerequisite) - pipcl.py:run_if(): Running command because: Prerequisite is new: 'run_if_test_prerequisite' + >>> run_if( f'touch {out}', out, prerequisite, caller=0) + pipcl.py:run_if(): Running command because: Command has changed: + pipcl.py:run_if(): @@ -1,2 +1,2 @@ + pipcl.py:run_if(): touch + pipcl.py:run_if(): -run_if_test_out; + pipcl.py:run_if(): +run_if_test_out + pipcl.py:run_if(): pipcl.py:run_if(): Running: touch run_if_test_out True If we repeat, the output will be newer than the prerequisite, so the command is not run: - >>> run_if( f'touch {out}', out, prerequisite) + >>> run_if( f'touch {out}', out, prerequisite, caller=0) pipcl.py:run_if(): Not running command because up to date: 'run_if_test_out' ''' doit = False @@ -2633,13 +2743,34 @@ cmd = f.read() else: cmd = None - if command != cmd: + cmd_args = shlex.split(cmd or '') + command_args = shlex.split(command or '') + if command_args != cmd_args: if cmd is None: doit = 'No previous command stored' else: doit = f'Command has changed' if 0: - doit += f': {cmd!r} => {command!r}' + doit += f':\n {cmd!r}\n {command!r}' + if 0: + doit += f'\nbefore:\n' + doit += textwrap.indent(cmd, ' ') + doit += f'\nafter:\n' + doit += textwrap.indent(command, ' ') + if 1: + # Show diff based on commands split into pseudo lines by + # shlex.split(). + doit += ':\n' + lines = difflib.unified_diff( + cmd.split(), + command.split(), + lineterm='', + ) + # Skip initial lines. + assert next(lines) == '--- ' + assert next(lines) == '+++ ' + for line in lines: + doit += f' {line}\n' if not doit: # See whether any prerequisites are newer than target. @@ -2652,9 +2783,9 @@ for p in prerequisites: prerequisites_all += _make_prerequisites( p) if 0: - log2( 'prerequisites_all:') + log2( 'prerequisites_all:', caller=caller+1) for i in prerequisites_all: - log2( f' {i!r}') + log2( f' {i!r}', caller=caller+1) pre_mtime = 0 pre_path = None for prerequisite in prerequisites_all: @@ -2670,7 +2801,7 @@ break if not doit: if pre_mtime > out_mtime: - doit = f'Prerequisite is new: {pre_path!r}' + doit = f'Prerequisite is new: {os.path.abspath(pre_path)!r}' if doit: # Remove `cmd_path` before we run the command, so any failure @@ -2680,16 +2811,16 @@ os.remove( cmd_path) except Exception: pass - log1( f'Running command because: {doit}') - - run( command) + log1( f'Running command because: {doit}', caller=caller+1) + + run( command, caller=caller+1) # Write the command we ran, into `cmd_path`. with open( cmd_path, 'w') as f: f.write( command) return True else: - log1( f'Not running command because up to date: {out!r}') + log1( f'Not running command because up to date: {out!r}', caller=caller+1) if 0: log2( f'out_mtime={time.ctime(out_mtime)} pre_mtime={time.ctime(pre_mtime)}.' @@ -2761,6 +2892,11 @@ return re.sub(r"[-_.]+", "-", name).lower() +def _normalise2(name): + # https://packaging.python.org/en/latest/specifications/binary-distribution-format/ + return _normalise(name).replace('-', '_') + + def _assert_version_pep_440(version): assert re.match( r'^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$', @@ -2790,6 +2926,9 @@ global g_log_line_numbers g_log_line_numbers = bool(yes) +def log(text='', caller=1): + _log(text, 0, caller+1) + def log0(text='', caller=1): _log(text, 0, caller+1) @@ -2813,19 +2952,30 @@ print(f'{filename}:{fr.function}(): {line}', file=sys.stdout, flush=1) -def relpath(path, start=None): +def relpath(path, start=None, allow_up=True): ''' A safe alternative to os.path.relpath(), avoiding an exception on Windows if the drive needs to change - in this case we use os.path.abspath(). + + Args: + path: + Path to be processed. + start: + Start directory or current directory if None. + allow_up: + If false we return absolute path is <path> is not within <start>. ''' if windows(): try: - return os.path.relpath(path, start) + ret = os.path.relpath(path, start) except ValueError: # os.path.relpath() fails if trying to change drives. - return os.path.abspath(path) + ret = os.path.abspath(path) else: - return os.path.relpath(path, start) + ret = os.path.relpath(path, start) + if not allow_up and ret.startswith('../') or ret.startswith('..\\'): + ret = os.path.abspath(path) + return ret def _so_suffix(use_so_versioning=True): @@ -2981,21 +3131,22 @@ for path, id_ in items.items(): id0 = self.items0.get(path) if id0 != id_: - #mtime0, hash0 = id0 - #mtime1, hash1 = id_ - #log0(f'New/modified file {path=}.') - #log0(f' {mtime0=} {"==" if mtime0==mtime1 else "!="} {mtime1=}.') - #log0(f' {hash0=} {"==" if hash0==hash1 else "!="} {hash1=}.') ret.append(path) return ret + def get_n(self, n): + ''' + Returns new files matching <glob_pattern>, asserting that there are + exactly <n>. + ''' + ret = self.get() + assert len(ret) == n, f'{len(ret)=}: {ret}' + return ret def get_one(self): ''' Returns new match of <glob_pattern>, asserting that there is exactly one. ''' - ret = self.get() - assert len(ret) == 1, f'{len(ret)=}' - return ret[0] + return self.get_n(1)[0] def _file_id(self, path): mtime = os.stat(path).st_mtime with open(path, 'rb') as f: @@ -3025,7 +3176,7 @@ Args: swig: - If starts with 'git:', passed as <remote> arg to git_remote(). + If starts with 'git:', passed as <text> arg to git_get(). quick: If true, we do not update/build local checkout if the binary is already present. @@ -3033,9 +3184,8 @@ path to use for checkout. ''' if swig and swig.startswith('git:'): - assert platform.system() != 'Windows' - swig_local = os.path.abspath(swig_local) - # Note that {swig_local}/install/bin/swig doesn't work on MacoS because + assert platform.system() != 'Windows', f'Cannot build swig on Windows.' + # Note that {swig_local}/install/bin/swig doesn't work on MacOS because # {swig_local}/INSTALL is a file and the fs is case-insensitive. swig_binary = f'{swig_local}/install-dir/bin/swig' if quick and os.path.isfile(swig_binary): @@ -3043,10 +3193,10 @@ else: # Clone swig. swig_env_extra = None - git_get( - swig, + swig_local = git_get( swig_local, - default_remote='https://github.com/swig/swig.git', + text=swig, + remote='https://github.com/swig/swig.git', branch='master', ) if darwin(): @@ -3061,10 +3211,10 @@ # > If you need to have bison first in your PATH, run: # > echo 'export PATH="/opt/homebrew/opt/bison/bin:$PATH"' >> ~/.zshrc # - run(f'brew install bison') - PATH = os.environ['PATH'] - PATH = f'/opt/homebrew/opt/bison/bin:{PATH}' - swig_env_extra = dict(PATH=PATH) + swig_env_extra = dict() + macos_add_brew_path('bison', swig_env_extra) + run(f'which bison') + run(f'which bison', env_extra=swig_env_extra) # Build swig. run(f'cd {swig_local} && ./autogen.sh', env_extra=swig_env_extra) run(f'cd {swig_local} && ./configure --prefix={swig_local}/install-dir', env_extra=swig_env_extra) @@ -3076,6 +3226,38 @@ return swig +def macos_add_brew_path(package, env=None, gnubin=True): + ''' + Adds path(s) for Brew <package>'s binaries to env['PATH']. + + Args: + package: + Name of package. We get <package_root> of installed package by + running `brew --prefix <package>`. + env: + The environment dict to modify. If None we use os.environ. If PATH + is not in <env>, we first copy os.environ['PATH'] into <env>. + gnubin: + If true, we also add path to gnu binaries if it exists, + <package_root>/libexe/gnubin. + ''' + if not darwin(): + return + if env is None: + env = os.environ + if 'PATH' not in env: + env['PATH'] = os.environ['PATH'] + package_root = run(f'brew --prefix {package}', capture=1).strip() + def add(path): + if os.path.isdir(path): + log1(f'Adding to $PATH: {path}') + PATH = env['PATH'] + env['PATH'] = f'{path}:{PATH}' + add(f'{package_root}/bin') + if gnubin: + add(f'{package_root}/libexec/gnubin') + + def _show_dict(d): ret = '' for n in sorted(d.keys()): @@ -3119,12 +3301,76 @@ return includes_, ldflags_ +def venv_in(path=None): + ''' + If path is None, returns true if we are in a venv. Otherwise returns true + only if we are in venv <path>. + ''' + if path: + return os.path.abspath(sys.prefix) == os.path.abspath(path) + else: + return sys.prefix != sys.base_prefix + + +def venv_run(args, path, recreate=True, clean=False): + ''' + Runs Python command inside venv and returns termination code. + + Args: + args: + List of args or string command. + path: + Path of venv directory. + recreate: + If false we do not run `<sys.executable> -m venv <path>` if <path> + already exists. This avoids a delay in the common case where <path> + is already set up, but fails if <path> exists but does not contain + a valid venv. + clean: + If true we first delete <path>. + ''' + if clean: + log(f'Removing any existing venv {path}.') + assert path.startswith('venv-') + shutil.rmtree(path, ignore_errors=1) + if recreate or not os.path.isdir(path): + run(f'{sys.executable} -m venv {path}') + + if isinstance(args, str): + args_string = args + elif platform.system() == 'Windows': + # shlex not reliable on Windows so we use Use crude quoting with "...". + args_string = '' + for i, arg in enumerate(args): + assert '"' not in arg + if i: + args_string += ' ' + args_string += f'"{arg}"' + else: + args_string = shlex.join(args) + + if platform.system() == 'Windows': + command = f'{path}\\Scripts\\activate && python {args_string}' + else: + command = f'. {path}/bin/activate && python {args_string}' + e = run(command, check=0) + return e + + if __name__ == '__main__': # Internal-only limited command line support, used if # graal_legacy_python_config is true. # includes, ldflags = sysconfig_python_flags() - if sys.argv[1:] == ['--graal-legacy-python-config', '--includes']: + if sys.argv[1] == '--doctest': + import doctest + if sys.argv[2:]: + for f in sys.argv[2:]: + ff = globals()[f] + doctest.run_docstring_examples(ff, globals()) + else: + doctest.testmod(None) + elif sys.argv[1:] == ['--graal-legacy-python-config', '--includes']: print(includes) elif sys.argv[1:] == ['--graal-legacy-python-config', '--ldflags']: print(ldflags)