Mercurial > hgrepos > Python2 > PyMuPDF
diff scripts/test.py @ 3:2c135c81b16c
MERGE: upstream PyMuPDF 1.26.4 with MuPDF 1.26.7
| author | Franz Glasner <fzglas.hg@dom66.de> |
|---|---|
| date | Mon, 15 Sep 2025 11:44:09 +0200 |
| parents | 1d09e1dec1d9 |
| children | a6bc019ac0b2 |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/scripts/test.py Mon Sep 15 11:44:09 2025 +0200 @@ -0,0 +1,1300 @@ +#! /usr/bin/env python3 + +'''Developer build/test script for PyMuPDF. + +Examples: + + ./PyMuPDF/scripts/test.py --m mupdf build test + Build and test with pre-existing local mupdf/ checkout. + + ./PyMuPDF/scripts/test.py build test + Build and test with default internal download of mupdf. + + ./PyMuPDF/scripts/test.py -m 'git:https://git.ghostscript.com/mupdf.git' build test + Build and test with internal checkout of MuPDF master. + + ./PyMuPDF/scripts/test.py -m 'git:--branch 1.26.x https://github.com/ArtifexSoftware/mupdf.git' build test + Build and test using internal checkout of mupdf 1.26.x branch from + Github. + +Usage: + +* Command line arguments are called parameters if they start with `-`, + otherwise they are called commands. +* Parameters are evaluated first in the order that they were specified. +* Then commands are run in the order in which they were specified. +* Usually command `test` would be specified after a `build`, `install` or + `wheel` command. +* Parameters and commands can be interleaved but it may be clearer to separate + them on the command line. + +Other: + +* If we are not already running inside a Python venv, we automatically create a + venv and re-run ourselves inside it. +* Build/wheel/install commands always install into the venv. +* Tests use whatever PyMuPDF/MuPDF is currently installed in the venv. +* We run tests with pytest. + +* One can generate call traces by setting environment variables in debug + builds. For details see: + https://mupdf.readthedocs.io/en/latest/language-bindings.html#environmental-variables + +Command line args: + + -a <env_name> + Read next space-separated argument(s) from environmental variable + <env_name>. + * Does nothing if <env_name> is unset. + * Useful when running via Github action. + + -b <build> + Set build type for `build` commands. `<build>` should be one of + 'release', 'debug', 'memento'. [This makes `build` set environment + variable `PYMUPDF_SETUP_MUPDF_BUILD_TYPE`, which is used by PyMuPDF's + `setup.py`.] + + --build-flavour <build_flavour> + Combination of 'p', 'b', 'd'. See ../setup.py's description of + PYMUPDF_SETUP_FLAVOUR. Default is 'pbd', i.e. self-contained PyMuPDF + wheels including MuPDF build-time files. + + --build-isolation 0|1 + If true (the default on non-OpenBSD systems), we let pip create and use + its own new venv to build PyMuPDF. Otherwise we force pip to use the + current venv. + + --cibw-archs-linux <archs> + Set CIBW_ARCHS_LINUX, e.g. to `auto64 aarch64`. Default is `auto64` so + this allows control over whether to build linux-aarch64 wheels. + + --cibw-name <cibw_name> + Name to use when installing cibuildwheel, e.g.: + --cibw-name cibuildwheel==3.0.0b1 + Default is `cibuildwheel`, i.e. the current release. + + --cibw-pyodide 0|1 + Experimental, make `cibuild` command build a pyodide wheel. + 2025-05-27: this fails when building mupdf C API - `ld -r -b binary + ...` fails with: + emcc: error: binary: No such file or directory ("binary" was expected to be an input file, based on the commandline arguments provided) + + --cibw-pyodide-version <cibw_pyodide_version> + Override default Pyodide version to use with `cibuildwheel` command. If + empty string we use cibuildwheel's default. + + --cibw-release-1 + Set up so that `cibw` builds all wheels except linux-aarch64, and sdist + if on Linux. + + --cibw-release-2 + Set up so that `cibw` builds only linux-aarch64 wheel. + + -d + Equivalent to `-b debug`. + + --dummy + Sets PYMUPDF_SETUP_DUMMY=1 which makes setup.py build a dummy wheel + with no content. For internal testing only. + + -e <name>=<value> + Add to environment used in build and test commands. Can be specified + multiple times. + + -f 0|1 + If 1 we also test alias `fitz` as well as `pymupdf`. Default is '0'. + + --gdb 0|1 + Run tests under gdb. Requires user interaction. + + --graal + Use graal - run inside a Graal VM instead of a Python venv. + + As of 2025-08-04 we: + * Clone the latest pyenv and build it. + * Use pyenv to install graalpy. + * Use graalpy to create venv. + + [After the first time, suggest `-v 1` to avoid delay from + updating/building pyenv and recreating the graal venv.] + + --help + -h + Show help. + + -I <implementations> + Set PyMuPDF implementations to test. + <implementations> must contain only these individual characters: + 'r' - rebased. + 'R' - rebased without optimisations. + Default is 'r'. Also see `PyMuPDF:tests/run_compound.py`. + + -i <install_version> + Set version installed by the 'install' command. + + -k <expression> + Specify which test(s) to run; passed straight through to pytest's `-k`. + For example `-k test_3354`. + + -m <location> | --mupdf <location> + Location of local mupdf/ directory or 'git:...' to be used + when building PyMuPDF. + + This sets environment variable PYMUPDF_SETUP_MUPDF_BUILD, which is used + by PyMuPDF/setup.py. If not specified PyMuPDF will download its default + mupdf .tgz. + + Additionally if <location> starts with ':' we use the remaining text as + the branch name and add https://github.com/ArtifexSoftware/mupdf.git. + + For example: + + -m "git:--branch master https://github.com/ArtifexSoftware/mupdf.git" + -m :master + + -m "git:--branch 1.26.x https://github.com/ArtifexSoftware/mupdf.git" + -m :1.26.x + + --mupdf-clean 0|1 + If 1 we do a clean MuPDF build. + + -M 0|1 + --build-mupdf 0|1 + Whether to rebuild mupdf when we build PyMuPDF. Default is 1. + + -o <os_names> + Control whether we do nothing on the current platform. + * <os_names> is a comma-separated list of names. + * If <os_names> is empty (the default), we always run normally. + * Otherwise we only run if an item in <os_names> matches (case + insensitive) platform.system(). + * For example `-o linux,darwin` will do nothing unless on Linux or + MacOS. + + -p <pytest-options> + Set pytest options; default is ''. + + -P 0|1 + If 1, automatically install required system packages such as + Valgrind. Default is 0. + + --pybind 0|1 + Experimental, for investigating + https://github.com/pymupdf/PyMuPDF/issues/3869. Runs run basic code + inside C++ pybind. Requires `sudo apt install pybind11-dev` or similar. + + --pyodide-build-version <version> + Version of Python package pyodide-build to use with `pyodide` command. + + If None (the default) `pyodide` uses the latest available version. + 2025-02-13: pyodide_build_version='0.29.3' works. + + -s 0 | 1 + If 1 (the default), build with Python Limited API/Stable ABI. + [This simply sSets $PYMUPDF_SETUP_PY_LIMITED_API, which is used by + PyMuPDF/setup.py.] + + --show-args: + Show sys.argv and exit. For debugging. + + --sync-paths + Do not run anything, instead write required files/directories/checkouts + to stdout, one per line. This is to help with automated running on + remote machines. + + --system-site-packages 0|1 + If 1, use `--system-site-packages` when creating venv. Defaults is 0. + + --swig <swig> + Use <swig> instead of the `swig` command. + + Unix only: + Clone/update/build swig from a git repository using 'git:' prefix. + + We default to https://github.com/swig/swig.git branch master, so these + are all equivalent: + + --swig 'git:--branch master https://github.com/swig/swig.git' + --swig 'git:--branch master' + --swig git: + + 2025-08-18: This fixes building with py_limited_api on python-3.13. + + --swig-quick 0|1 + If 1 and `--swig` starts with 'git:', we do not update/build swig if + already present. + + See description of PYMUPDF_SETUP_SWIG_QUICK in setup.py. + + -t <names> + Pytest test names, comma-separated. Should be relative to PyMuPDF + directory. For example: + -t tests/test_general.py + -t tests/test_general.py::test_subset_fonts + To specify multiple tests, use comma-separated list and/or multiple `-t + <names>` args. + + --timeout <seconds> + Sets timeout when running tests. + + -T <prefix> + Use specified prefix when running pytest, must be one of: + gdb + helgrind + vagrind + + -v <venv> + venv is: + 0 - do not use a venv. + 1 - Use venv. If it already exists, we assume the existing directory + was created by us earlier and is a valid venv containing all + necessary packages; this saves a little time. + 2 - Use venv. + 3 - Use venv but delete it first if it already exists. + The default is 2. + +Commands: + + build + Builds and installs PyMuPDF into venv, using `pip install .../PyMuPDF`. + + buildtest + Same as 'build test'. + + cibw + Build and test PyMuPDF wheel(s) using cibuildwheel. Wheels are placed + in directory `wheelhouse`. + * We do not attempt to install wheels. + * So it is generally not useful to do `cibw test`. + + If CIBW_BUILD is unset, we set it as follows: + * On Github we build and test all supported Python versions. + * Otherwise we build and test the current Python version only. + + If CIBW_ARCHS is unset we set $CIBW_ARCHS_WINDOWS, $CIBW_ARCHS_MACOS + and $CIBW_ARCHS_LINUX to auto64 if they are unset. + + install <pymupdf> + Install with `pip install --force-reinstall <pymupdf>`. + + pyodide + Build Pyodide wheel. We clone `emsdk.git`, set it up, and run + `pyodide build`. This runs our setup.py with CC etc set up + to create Pyodide binaries in a wheel called, for example, + `PyMuPDF-1.23.2-cp311-none-emscripten_3_1_32_wasm32.whl`. + + It seems that sys.version must match the Python version inside emsdk; + as of 2025-02-14 this is 3.12. Otherwise we get build errors such as: + [wasm-validator error in function 723] unexpected false: all used features should be allowed, on ... + + test + Runs PyMuPDF's pytest tests. Default is to test rebased and unoptimised + rebased; use `-i` to change this. + + wheel + Build and install wheel. + + +Environment: + PYMUDF_SCRIPTS_TEST_options + Is prepended to command line args. +''' + +import glob +import os +import platform +import re +import shlex +import shutil +import subprocess +import sys +import textwrap + + +pymupdf_dir_abs = os.path.abspath( f'{__file__}/../..') + +try: + sys.path.insert(0, pymupdf_dir_abs) + import pipcl +finally: + del sys.path[0] + +try: + sys.path.insert(0, f'{pymupdf_dir_abs}/scripts') + import gh_release +finally: + del sys.path[0] + + +pymupdf_dir = pipcl.relpath(pymupdf_dir_abs) + +log = pipcl.log0 +run = pipcl.run + + +def main(argv): + + if github_workflow_unimportant(): + return + + build_isolation = None + cibw_name = None + cibw_pyodide = None + cibw_pyodide_version = None + commands = list() + env_extra = dict() + graal = False + implementations = 'r' + install_version = None + mupdf_sync = None + os_names = list() + system_packages = False + pybind = False + pyodide_build_version = None + pytest_options = '' + pytest_prefix = None + cibw_sdist = None + show_args = False + show_help = False + sync_paths = False + system_site_packages = False + swig = None + swig_quick = None + test_fitz = False + test_names = list() + test_timeout = None + valgrind = False + warnings = list() + venv = 2 + + options = os.environ.get('PYMUDF_SCRIPTS_TEST_options', '') + options = shlex.split(options) + + # Parse args and update the above state. We do this before moving into a + # venv, partly so we can return errors immediately. + # + args = iter(options + argv[1:]) + i = 0 + while 1: + try: + arg = next(args) + except StopIteration: + arg = None + break + + if 0: + pass + + elif arg == '-a': + _name = next(args) + _value = os.environ.get(_name, '') + _args = shlex.split(_value) + list(args) + args = iter(_args) + + elif arg == '-b': + env_extra['PYMUPDF_SETUP_MUPDF_BUILD_TYPE'] = next(args) + + elif arg == '--build-flavour': + env_extra['PYMUPDF_SETUP_FLAVOUR'] = next(args) + + elif arg == '--build-isolation': + build_isolation = int(next(args)) + + elif arg == '--cibw-pyodide-version': + cibw_pyodide_version = next(args) + + elif arg == '--cibw-release-1': + cibw_sdist = True + env_extra['CIBW_ARCHS_LINUX'] = 'auto64' + env_extra['CIBW_ARCHS_MACOS'] = 'auto64' + env_extra['CIBW_ARCHS_WINDOWS'] = 'auto' # win32 and win64. + env_extra['CIBW_SKIP'] = 'pp* *i686 cp36* cp37* *musllinux*aarch64*' + + elif arg == '--cibw-release-2': + env_extra['CIBW_ARCHS_LINUX'] = 'aarch64' + # Testing only first and last python versions because otherwise + # Github times out after 6h. + env_extra['CIBW_BUILD'] = 'cp39* cp313*' + os_names = ['linux'] + + elif arg == '--cibw-archs-linux': + env_extra['CIBW_ARCHS_LINUX'] = next(args) + + elif arg == '--cibw-name': + cibw_name = next(args) + + elif arg == '--cibw-pyodide': + cibw_pyodide = next(args) + + elif arg == '-d': + env_extra['PYMUPDF_SETUP_MUPDF_BUILD_TYPE'] = 'debug' + + elif arg == '--dummy': + env_extra['PYMUPDF_SETUP_DUMMY'] = '1' + env_extra['CIBW_TEST_COMMAND'] = '' + + elif arg == '-e': + _nv = next(args) + assert '=' in _nv, f'-e <name>=<value> does not contain "=": {_nv!r}' + _name, _value = _nv.split('=', 1) + env_extra[_name] = _value + + elif arg == '-f': + test_fitz = int(next(args)) + + elif arg == '--graal': + graal = True + + elif arg in ('-h', '--help'): + show_help = True + + elif arg == '-i': + install_version = next(args) + + elif arg == '-I': + implementations = next(args) + + elif arg == '-k': + pytest_options += f' -k {shlex.quote(next(args))}' + + elif arg in ('-m', '--mupdf'): + _mupdf = next(args) + if _mupdf == '-': + _mupdf = None + elif _mupdf.startswith(':'): + _branch = _mupdf[1:] + _mupdf = 'git:--branch {_branch} https://github.com/ArtifexSoftware/mupdf.git' + os.environ['PYMUPDF_SETUP_MUPDF_BUILD'] = _mupdf + elif _mupdf.startswith('git:') or '://' in _mupdf: + os.environ['PYMUPDF_SETUP_MUPDF_BUILD'] = _mupdf + else: + assert os.path.isdir(_mupdf), f'Not a directory: {_mupdf=}' + os.environ['PYMUPDF_SETUP_MUPDF_BUILD'] = os.path.abspath(_mupdf) + mupdf_sync = _mupdf + + elif arg == '--mupdf-clean': + env_extra['PYMUPDF_SETUP_MUPDF_CLEAN']=next(args) + + elif arg in ('-M', '--build-mupdf'): + env_extra['PYMUPDF_SETUP_MUPDF_REBUILD'] = next(args) + + elif arg == '-o': + os_names += next(args).split(',') + + elif arg == '-p': + pytest_options += f' {next(args)}' + + elif arg == '-P': + system_packages = int(next(args)) + + elif arg == '--pybind': + pybind = int(next(args)) + + elif arg == '--pyodide-build-version': + pyodide_build_version = next(args) + + elif arg == '-s': + _value = next(args) + assert _value in ('0', '1'), f'`-s` must be followed by `0` or `1`, not {_value=}.' + env_extra['PYMUPDF_SETUP_PY_LIMITED_API'] = _value + + elif arg == '--show-args': + show_args = 1 + elif arg == '--sync-paths': + sync_paths = True + + elif arg == '--system-site-packages': + system_site_packages = int(next(args)) + + elif arg == '--swig': + swig = next(args) + + elif arg == '--swig-quick': + swig_quick = int(next(args)) + + elif arg == '-t': + test_names += next(args).split(',') + + elif arg == '--timeout': + test_timeout = float(next(args)) + + elif arg == '-T': + pytest_prefix = next(args) + assert pytest_prefix in ('gdb', 'helgrind', 'valgrind'), \ + f'Unrecognised {pytest_prefix=}, should be one of: gdb valgrind helgrind.' + + elif arg == '-v': + venv = int(next(args)) + assert venv in (0, 1, 2, 3), f'Invalid {venv=} should be 0, 1, 2 or 3.' + + elif arg in ('build', 'cibw', 'install', 'pyodide', 'test', 'wheel'): + commands.append(arg) + + elif arg == 'buildtest': + commands += ['build', 'test'] + + else: + assert 0, f'Unrecognised option/command: {arg=}.' + + # Handle special args --sync-paths, -h, -v, -o first. + # + if sync_paths: + # Just print required files, directories and checkouts. + print(pymupdf_dir) + if mupdf_sync: + print(mupdf_sync) + return + + if show_help: + print(__doc__) + return + + if show_args: + print(f'sys.argv ({len(sys.argv)}):') + for arg in sys.argv: + print(f' {arg!r}') + return + + if os_names: + if platform.system().lower() not in os_names: + log(f'Not running because {platform.system().lower()=} not in {os_names=}') + return + + if commands: + if venv: + # Rerun ourselves inside a venv if not already in a venv. + if not venv_in(): + if graal: + # 2025-07-24: We need the latest pyenv. + graalpy = 'graalpy-24.2.1' + venv_name = f'venv-pymupdf-{graalpy}' + pyenv_dir = f'{pymupdf_dir_abs}/pyenv-git' + os.environ['PYENV_ROOT'] = pyenv_dir + os.environ['PATH'] = f'{pyenv_dir}/bin:{os.environ["PATH"]}' + os.environ['PIPCL_GRAAL_PYTHON'] = sys.executable + + if venv >= 3: + shutil.rmtree(venv_name, ignore_errors=1) + if venv == 1 and os.path.exists(pyenv_dir) and os.path.exists(venv_name): + log(f'{venv=} and {venv_name=} already exists so not building pyenv or creating venv.') + else: + pipcl.git_get('https://github.com/pyenv/pyenv.git', pyenv_dir, branch='master') + run(f'cd {pyenv_dir} && src/configure && make -C src') + run(f'which pyenv') + run(f'pyenv install -v -s {graalpy}') + run(f'{pyenv_dir}/versions/{graalpy}/bin/graalpy -m venv {venv_name}') + e = run(f'. {venv_name}/bin/activate && python {shlex.join(sys.argv)}', + check=False, + ) + else: + venv_name = f'venv-pymupdf-{platform.python_version()}-{int.bit_length(sys.maxsize+1)}' + e = venv_run( + sys.argv, + venv_name, + recreate=(venv>=2), + clean=(venv>=3), + ) + sys.exit(e) + else: + log(f'Warning, no commands specified so nothing to do.') + + # Clone/update/build swig if specified. + swig_binary = pipcl.swig_get(swig, swig_quick) + if swig_binary: + os.environ['PYMUPDF_SETUP_SWIG'] = swig_binary + + # Handle commands. + # + have_installed = False + for command in commands: + log(f'### {command=}.') + if 0: + pass + + elif command in ('build', 'wheel'): + build( + env_extra, + build_isolation=build_isolation, + venv=venv, + wheel=(command=='wheel'), + ) + have_installed = True + + elif command == 'cibw': + # Build wheel(s) with cibuildwheel. + if cibw_pyodide and env_extra.get('CIBW_BUILD') is None: + assert 0, f'Need a Python version for Pyodide.' + CIBW_BUILD = 'cp312*' + env_extra['CIBW_BUILD'] = CIBW_BUILD + log(f'Defaulting to {CIBW_BUILD=} for Pyodide.') + #if cibw_pyodide_version == None: + # cibw_pyodide_version = '0.28.0' + cibuildwheel( + env_extra, + cibw_name or 'cibuildwheel', + cibw_pyodide, + cibw_pyodide_version, + cibw_sdist, + ) + + elif command == 'install': + p = 'pymupdf' + if install_version: + if not install_version.startswith(('==', '>=', '>')): + p = f'{p}==' + p = f'{p}{install_version}' + run(f'pip install --force-reinstall {p}') + have_installed = True + + elif command == 'test': + if not have_installed: + log(f'## Warning: have not built/installed PyMuPDF; testing whatever is already installed.') + test( + env_extra=env_extra, + implementations=implementations, + test_names=test_names, + pytest_options=pytest_options, + test_timeout=test_timeout, + pytest_prefix=pytest_prefix, + test_fitz=test_fitz, + pybind=pybind, + system_packages=system_packages, + venv=venv, + ) + + elif command == 'pyodide': + build_pyodide_wheel(pyodide_build_version=pyodide_build_version) + + else: + assert 0, f'{command=}' + + +def get_env_bool(name, default=0): + v = os.environ.get(name) + if v in ('1', 'true'): + return 1 + elif v in ('0', 'false'): + return 0 + elif v is None: + return default + else: + assert 0, f'Bad environ {name=} {v=}' + +def show_help(): + print(__doc__) + print(venv_info()) + + +def github_workflow_unimportant(): + ''' + Returns true if we are running a Github scheduled workflow but in a + repository not called 'PyMuPDF'. This can be used to avoid consuming + unnecessary Github minutes running workflows on non-main repositories such + as ArtifexSoftware/PyMuPDF-julian. + ''' + GITHUB_EVENT_NAME = os.environ.get('GITHUB_EVENT_NAME') + GITHUB_REPOSITORY = os.environ.get('GITHUB_REPOSITORY') + if GITHUB_EVENT_NAME == 'schedule' and GITHUB_REPOSITORY != 'pymupdf/PyMuPDF': + log(f'## This is an unimportant Github workflow: a scheduled event, not in the main repository `pymupdf/PyMuPDF`.') + log(f'## {GITHUB_EVENT_NAME=}.') + log(f'## {GITHUB_REPOSITORY=}.') + return True + +def venv_info(pytest_args=None): + ''' + Returns string containing information about the venv we use and how to + run tests manually. If specified, `pytest_args` contains the pytest args, + otherwise we use an example. + ''' + pymupdf_dir_rel = gh_release.relpath(pymupdf_dir) + ret = f'Name of venv: {gh_release.venv_name}\n' + if pytest_args is None: + pytest_args = f'{pymupdf_dir_rel}/tests/test_general.py::test_subset_fonts' + if platform.system() == 'Windows': + ret += textwrap.dedent(f''' + Rerun tests manually with rebased implementation: + Enter venv: + {gh_release.venv_name}\\Scripts\\activate + Run specific test in venv: + {gh_release.venv_name}\\Scripts\\python -m pytest {pytest_args} + ''') + else: + ret += textwrap.dedent(f''' + Rerun tests manually with rebased implementation: + Enter venv and run specific test, also under gdb: + . {gh_release.venv_name}/bin/activate + python -m pytest {pytest_args} + gdb --args python -m pytest {pytest_args} + Run without explicitly entering venv, also under gdb: + ./{gh_release.venv_name}/bin/python -m pytest {pytest_args} + gdb --args ./{gh_release.venv_name}/bin/python -m pytest {pytest_args} + ''') + return ret + + +def build( + env_extra, + *, + build_isolation, + venv, + wheel, + ): + print(f'{build_isolation=}') + + if build_isolation is None: + # On OpenBSD libclang is not available on pypi.org, so we need to force + # use of system package py3-llvm with --no-build-isolation, manually + # installing other required packages. + build_isolation = False if platform.system() == 'OpenBSD' else True + + if build_isolation: + # This is the default on non-OpenBSD. + build_isolation_text = '' + else: + # Not using build isolation - i.e. pip will not be using its own clean + # venv, so we need to explicitly install required packages. Manually + # install required packages from pyproject.toml. + sys.path.insert(0, os.path.abspath(f'{__file__}/../..')) + import setup + names = setup.get_requires_for_build_wheel() + del sys.path[0] + if names: + names = ' '.join(names) + if venv == 2: + run( f'python -m pip install --upgrade {names}') + else: + log(f'{venv=}: Not installing packages with pip: {names}') + build_isolation_text = ' --no-build-isolation' + + if wheel: + new_files = pipcl.NewFiles(f'wheelhouse/*.whl') + run(f'pip wheel{build_isolation_text} -w wheelhouse -v {pymupdf_dir_abs}', env_extra=env_extra) + wheel = new_files.get_one() + run(f'pip install --force-reinstall {wheel}') + else: + run(f'pip install{build_isolation_text} -v --force-reinstall {pymupdf_dir_abs}', env_extra=env_extra) + + +def cibuildwheel(env_extra, cibw_name, cibw_pyodide, cibw_pyodide_version, cibw_sdist): + + if cibw_sdist and platform.system() == 'Linux': + log(f'Building sdist.') + run(f'cd {pymupdf_dir_abs} && {sys.executable} setup.py -d wheelhouse sdist', env_extra=env_extra) + sdists = glob.glob(f'{pymupdf_dir_abs}/wheelhouse/pymupdf-*.tar.gz') + log(f'{sdists=}') + assert sdists + + run(f'pip install --upgrade --force-reinstall {cibw_name}') + + # Some general flags. + if 'CIBW_BUILD_VERBOSITY' not in env_extra: + env_extra['CIBW_BUILD_VERBOSITY'] = '1' + if 'CIBW_SKIP' not in env_extra: + env_extra['CIBW_SKIP'] = 'pp* *i686 cp36* cp37* *musllinux* *-win32 *-aarch64' + + # Set what wheels to build, if not already specified. + if 'CIBW_ARCHS' not in env_extra: + if 'CIBW_ARCHS_WINDOWS' not in env_extra: + env_extra['CIBW_ARCHS_WINDOWS'] = 'auto64' + + if 'CIBW_ARCHS_MACOS' not in env_extra: + env_extra['CIBW_ARCHS_MACOS'] = 'auto64' + + if 'CIBW_ARCHS_LINUX' not in env_extra: + env_extra['CIBW_ARCHS_LINUX'] = 'auto64' + + # Tell cibuildwheel not to use `auditwheel` on Linux and MacOS, + # because it cannot cope with us deliberately having required + # libraries in different wheel - specifically in the PyMuPDF wheel. + # + # We cannot use a subset of auditwheel's functionality + # with `auditwheel addtag` because it says `No tags + # to be added` and terminates with non-zero. See: + # https://github.com/pypa/auditwheel/issues/439. + # + env_extra['CIBW_REPAIR_WHEEL_COMMAND_LINUX'] = '' + env_extra['CIBW_REPAIR_WHEEL_COMMAND_MACOS'] = '' + + # Tell cibuildwheel how to test PyMuPDF. + if 'CIBW_TEST_COMMAND' not in env_extra: + env_extra['CIBW_TEST_COMMAND'] = f'python {{project}}/scripts/test.py test' + + # Specify python versions. + CIBW_BUILD = env_extra.get('CIBW_BUILD') + log(f'{CIBW_BUILD=}') + if CIBW_BUILD is None: + if os.environ.get('GITHUB_ACTIONS') == 'true': + # Build/test all supported Python versions. + CIBW_BUILD = 'cp39* cp310* cp311* cp312* cp313*' + else: + # Build/test current Python only. + v = platform.python_version_tuple()[:2] + log(f'{v=}') + CIBW_BUILD = f'cp{"".join(v)}*' + + cibw_pyodide_args = '' + if cibw_pyodide: + cibw_pyodide_args = ' --platform pyodide' + env_extra['HAVE_LIBCRYPTO'] = 'no' + env_extra['PYMUPDF_SETUP_MUPDF_TESSERACT'] = '0' + if cibw_pyodide_version: + # 2025-07-21: there is no --pyodide-version option so we set + # CIBW_PYODIDE_VERSION. + env_extra['CIBW_PYODIDE_VERSION'] = cibw_pyodide_version + env_extra['CIBW_ENABLE'] = 'pyodide-prerelease' + + # Pass all the environment variables we have set, to Linux + # docker. Note that this will miss any settings in the original + # environment. + env_extra['CIBW_ENVIRONMENT_PASS_LINUX'] = ' '.join(sorted(env_extra.keys())) + + # Build for lowest (assumed first) Python version. + # + CIBW_BUILD_0 = CIBW_BUILD.split()[0] + log(f'Building for first Python version {CIBW_BUILD_0}.') + env_extra['CIBW_BUILD'] = CIBW_BUILD_0 + run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_args}', env_extra=env_extra) + + # Tell cibuildwheel to build and test all specified Python versions; it + # will notice that the wheel we built above supports all versions of + # Python, so will not actually do any builds here. + # + env_extra['CIBW_BUILD'] = CIBW_BUILD + run(f'cd {pymupdf_dir} && cibuildwheel{cibw_pyodide_args}', env_extra=env_extra) + run(f'ls -ld {pymupdf_dir}/wheelhouse/*') + + +def build_pyodide_wheel(pyodide_build_version=None): + ''' + Build Pyodide wheel. + + This runs `pyodide build` inside the PyMuPDF directory, which in turn runs + setup.py in a Pyodide build environment. + ''' + log(f'## Building Pyodide wheel.') + + # Our setup.py does not know anything about Pyodide; we set a few + # required environmental variables here. + # + env_extra = dict() + + # Disable libcrypto because not available in Pyodide. + env_extra['HAVE_LIBCRYPTO'] = 'no' + + # Tell MuPDF to build for Pyodide. + env_extra['OS'] = 'pyodide' + + # Build a single wheel without a separate PyMuPDFb wheel. + env_extra['PYMUPDF_SETUP_FLAVOUR'] = 'pb' + + # 2023-08-30: We set PYMUPDF_SETUP_MUPDF_BUILD_TESSERACT=0 because + # otherwise mupdf thirdparty/tesseract/src/ccstruct/dppoint.cpp fails to + # build because `#include "errcode.h"` finds a header inside emsdk. This is + # pyodide bug https://github.com/pyodide/pyodide/issues/3839. It's fixed in + # https://github.com/pyodide/pyodide/pull/3866 but the fix has not reached + # pypi.org's pyodide-build package. E.g. currently in tag 0.23.4, but + # current devuan pyodide-build is pyodide_build-0.23.4. + # + env_extra['PYMUPDF_SETUP_MUPDF_TESSERACT'] = '0' + setup = pyodide_setup(pymupdf_dir, pyodide_build_version=pyodide_build_version) + command = f'{setup} && echo "### Running pyodide build" && pyodide build --exports whole_archive' + + command = command.replace(' && ', '\n && ') + + run(command, env_extra=env_extra) + + # Copy wheel into `wheelhouse/` so it is picked up as a workflow + # artifact. + # + run(f'ls -l {pymupdf_dir}/dist/') + run(f'mkdir -p {pymupdf_dir}/wheelhouse && cp -p {pymupdf_dir}/dist/* {pymupdf_dir}/wheelhouse/') + run(f'ls -l {pymupdf_dir}/wheelhouse/') + + +def pyodide_setup( + directory, + clean=False, + pyodide_build_version=None, + ): + ''' + Returns a command that will set things up for a pyodide build. + + Args: + directory: + Our command cd's into this directory. + clean: + If true we create an entirely new environment. Otherwise + we reuse any existing emsdk repository and venv. + pyodide_build_version: + Version of Python package pyodide-build; if None we use latest + available version. + 2025-02-13: pyodide_build_version='0.29.3' works. + + The returned command does the following: + + * Checkout latest emsdk from https://github.com/emscripten-core/emsdk.git: + * Clone emsdk repository to `emsdk` if not already present. + * Run `git pull -r` inside emsdk checkout. + * Create venv `venv_pyodide_<python_version>` if not already present. + * Activate venv `venv_pyodide_<python_version>`. + * Install/upgrade package `pyodide-build`. + * Run emsdk install scripts and enter emsdk environment. + + Example usage in a build function: + + command = pyodide_setup() + command += ' && pyodide build --exports pyinit' + subprocess.run(command, shell=1, check=1) + ''' + + pv = platform.python_version_tuple()[:2] + assert pv == ('3', '12'), f'Pyodide builds need to be run with Python-3.12 but current Python is {platform.python_version()}.' + command = f'cd {directory}' + + # Clone/update emsdk. We always use the latest emsdk with `git pull`. + # + # 2025-02-13: this works: 2514ec738de72cebbba7f4fdba0cf2fabcb779a5 + # + dir_emsdk = 'emsdk' + if clean: + shutil.rmtree(dir_emsdk, ignore_errors=1) + # 2024-06-25: old `.pyodide-xbuildenv` directory was breaking build, so + # important to remove it here. + shutil.rmtree('.pyodide-xbuildenv', ignore_errors=1) + if not os.path.exists(f'{directory}/{dir_emsdk}'): + command += f' && echo "### Cloning emsdk.git"' + command += f' && git clone https://github.com/emscripten-core/emsdk.git {dir_emsdk}' + command += f' && echo "### Updating checkout {dir_emsdk}"' + command += f' && (cd {dir_emsdk} && git pull -r)' + command += f' && echo "### Checkout {dir_emsdk} is:"' + command += f' && (cd {dir_emsdk} && git show -s --oneline)' + + # Create and enter Python venv. + # + python = sys.executable + venv_pyodide = f'venv_pyodide_{sys.version_info[0]}.{sys.version_info[1]}' + + if not os.path.exists( f'{directory}/{venv_pyodide}'): + command += f' && echo "### Creating venv {venv_pyodide}"' + command += f' && {python} -m venv {venv_pyodide}' + command += f' && . {venv_pyodide}/bin/activate' + command += f' && echo "### Installing Python packages."' + command += f' && python -m pip install --upgrade pip wheel pyodide-build' + if pyodide_build_version: + command += f'=={pyodide_build_version}' + + # Run emsdk install scripts and enter emsdk environment. + # + command += f' && cd {dir_emsdk}' + command += ' && PYODIDE_EMSCRIPTEN_VERSION=$(pyodide config get emscripten_version)' + command += ' && echo "### PYODIDE_EMSCRIPTEN_VERSION is: $PYODIDE_EMSCRIPTEN_VERSION"' + command += ' && echo "### Running ./emsdk install"' + command += ' && ./emsdk install ${PYODIDE_EMSCRIPTEN_VERSION}' + command += ' && echo "### Running ./emsdk activate"' + command += ' && ./emsdk activate ${PYODIDE_EMSCRIPTEN_VERSION}' + command += ' && echo "### Running ./emsdk_env.sh"' + command += ' && . ./emsdk_env.sh' # Need leading `./` otherwise weird 'Not found' error. + + command += ' && cd ..' + return command + + +def test( + *, + env_extra, + implementations, + venv=False, + test_names=None, + pytest_options=None, + test_timeout=None, + pytest_prefix=None, + test_fitz=True, + pytest_k=None, + pybind=False, + system_packages=False, + ): + if pybind: + cpp_path = 'pymupdf_test_pybind.cpp' + cpp_exe = 'pymupdf_test_pybind.exe' + cpp = textwrap.dedent(''' + #include <pybind11/embed.h> + + int main() + { + pybind11::scoped_interpreter guard{}; + pybind11::exec(R"( + print('Hello world', flush=1) + import pymupdf + pymupdf.JM_mupdf_show_warnings = 1 + print(f'{pymupdf.version=}', flush=1) + doc = pymupdf.Document() + pymupdf.mupdf.fz_warn('Dummy warning.') + pymupdf.mupdf.fz_warn('Dummy warning.') + pymupdf.mupdf.fz_warn('Dummy warning.') + print(f'{doc=}', flush=1) + )"); + } + ''') + def fs_read(path): + try: + with open(path) as f: + return f.read() + except Exception: + return + def fs_remove(path): + try: + os.remove(path) + except Exception: + pass + cpp_existing = fs_read(cpp_path) + if cpp == cpp_existing: + log(f'Not creating {cpp_exe} because unchanged: {cpp_path}') + else: + with open(cpp_path, 'w') as f: + f.write(cpp) + def getmtime(path): + try: + return os.path.getmtime(path) + except Exception: + return 0 + python_config = f'{os.path.realpath(sys.executable)}-config' + # `--embed` adds `-lpython3.11` to the link command, which appears to + # be necessary when building an executable. + flags = run(f'{python_config} --cflags --ldflags --embed', capture=1) + build_command = f'c++ {cpp_path} -o {cpp_exe} -g -W -Wall {flags}' + build_path = f'{cpp_exe}.cmd' + build_command_prev = fs_read(build_path) + if build_command != build_command_prev or getmtime(cpp_path) >= getmtime(cpp_exe): + fs_remove(build_path) + run(build_command) + with open(build_path, 'w') as f: + f.write(build_command) + run(f'./{cpp_exe}') + return + + pymupdf_dir_rel = gh_release.relpath(pymupdf_dir) + if not pytest_options and pytest_prefix == 'valgrind': + pytest_options = '-sv' + if pytest_k: + pytest_options += f' -k {shlex.quote(pytest_k)}' + pytest_arg = '' + if test_names: + for test_name in test_names: + pytest_arg += f' {pymupdf_dir_rel}/{test_name}' + else: + pytest_arg += f' {pymupdf_dir_rel}/tests' + python = gh_release.relpath(sys.executable) + log('Running tests with tests/run_compound.py and pytest.') + + PYODIDE_ROOT = os.environ.get('PYODIDE_ROOT') + if PYODIDE_ROOT is not None: + log(f'Not installing test packages because {PYODIDE_ROOT=}.') + command = f'{pytest_options} {pytest_arg} -s' + args = shlex.split(command) + print(f'{PYODIDE_ROOT=} so calling pytest.main(args).') + print(f'{command=}') + print(f'args are ({len(args)}):') + for arg in args: + print(f' {arg!r}') + import pytest + pytest.main(args) + return + + if venv >= 2: + run(f'pip install --upgrade {gh_release.test_packages}') + else: + log(f'{venv=}: Not installing test packages: {gh_release.test_packages}') + run_compound_args = '' + + if implementations: + run_compound_args += f' -i {implementations}' + + if test_timeout: + run_compound_args += f' -t {test_timeout}' + + if pytest_prefix in ('valgrind', 'helgrind'): + if system_packages: + log('Installing valgrind.') + run(f'sudo apt update') + run(f'sudo apt install --upgrade valgrind') + run(f'valgrind --version') + + command = f'{python} {pymupdf_dir_rel}/tests/run_compound.py{run_compound_args}' + + if pytest_prefix is None: + pass + elif pytest_prefix == 'gdb': + command += ' gdb --args' + elif pytest_prefix == 'valgrind': + env_extra['PYMUPDF_RUNNING_ON_VALGRIND'] = '1' + env_extra['PYTHONMALLOC'] = 'malloc' + command += ( + f' valgrind' + f' --suppressions={pymupdf_dir_abs}/valgrind.supp' + f' --trace-children=no' + f' --num-callers=20' + f' --error-exitcode=100' + f' --errors-for-leak-kinds=none' + f' --fullpath-after=' + ) + elif pytest_prefix == 'helgrind': + env_extra['PYMUPDF_RUNNING_ON_VALGRIND'] = '1' + env_extra['PYTHONMALLOC'] = 'malloc' + command = ( + f' valgrind' + f' --tool=helgrind' + f' --trace-children=no' + f' --num-callers=20' + f' --error-exitcode=100' + f' --fullpath-after=' + ) + else: + assert 0, f'Unrecognised {pytest_prefix=}' + + if platform.system() == 'Windows': + # `python -m pytest` doesn't seem to work. + command += ' pytest' + else: + # On OpenBSD `pip install pytest` doesn't seem to install the pytest + # command, so we use `python -m pytest ...`. + command += f' {python} -m pytest' + + command += f' {pytest_options} {pytest_arg}' + + # Always start by removing any test_*_fitz.py files. + for p in glob.glob(f'{pymupdf_dir_rel}/tests/test_*_fitz.py'): + print(f'Removing {p=}') + os.remove(p) + if test_fitz: + # Create copies of each test file, modified to use `pymupdf` + # instead of `fitz`. + for p in glob.glob(f'{pymupdf_dir_rel}/tests/test_*.py'): + if os.path.basename(p).startswith('test_fitz_'): + # Don't recursively generate test_fitz_fitz_foo.py, + # test_fitz_fitz_fitz_foo.py, ... etc. + continue + branch, leaf = os.path.split(p) + p2 = f'{branch}/{leaf[:5]}fitz_{leaf[5:]}' + print(f'Converting {p=} to {p2=}.') + with open(p, encoding='utf8') as f: + text = f.read() + text2 = re.sub("([^\'])\\bpymupdf\\b", '\\1fitz', text) + if p.replace(os.sep, '/') == f'{pymupdf_dir_rel}/tests/test_docs_samples.py'.replace(os.sep, '/'): + assert text2 == text + else: + assert text2 != text, f'Unexpectedly unchanged when creating {p!r} => {p2!r}' + with open(p2, 'w', encoding='utf8') as f: + f.write(text2) + try: + log(f'Running tests with tests/run_compound.py and pytest.') + run(command, env_extra=env_extra, timeout=test_timeout) + + except subprocess.TimeoutExpired as e: + log(f'Timeout when running tests.') + raise + finally: + log(f'\n' + f'[As of 2024-10-10 we get warnings from pytest/Python such as:\n' + f' DeprecationWarning: builtin type SwigPyPacked has no __module__ attribute\n' + f'This seems to be due to Swig\'s handling of Py_LIMITED_API.\n' + f'For details see https://github.com/swig/swig/issues/2881.\n' + f']' + ) + log('\n' + venv_info(pytest_args=f'{pytest_options} {pytest_arg}')) + + +def get_pyproject_required(ppt=None): + ''' + Returns space-separated names of required packages in pyproject.toml. We + do not do a proper parse and rely on the packages being in a single line. + ''' + if ppt is None: + ppt = os.path.abspath(f'{__file__}/../../pyproject.toml') + with open(ppt) as f: + for line in f: + m = re.match('^requires = \\[(.*)\\]$', line) + if m: + names = m.group(1).replace(',', ' ').replace('"', '') + return names + else: + assert 0, f'Failed to find "requires" line in {ppt}' + +def wrap_get_requires_for_build_wheel(dir_): + ''' + Returns space-separated list of required + packages. Looks at `dir_`/pyproject.toml and calls + `dir_`/setup.py:get_requires_for_build_wheel(). + ''' + dir_abs = os.path.abspath(dir_) + ret = list() + ppt = os.path.join(dir_abs, 'pyproject.toml') + if os.path.exists(ppt): + ret += get_pyproject_required(ppt) + if os.path.exists(os.path.join(dir_abs, 'setup.py')): + sys.path.insert(0, dir_abs) + try: + from setup import get_requires_for_build_wheel as foo + for i in foo(): + ret.append(i) + finally: + del sys.path[0] + return ' '.join(ret) + + +def venv_in(path=None): + ''' + If path is None, returns true if we are in a venv. Otherwise returns true + only if we are in venv <path>. + ''' + if path: + return os.path.abspath(sys.prefix) == os.path.abspath(path) + else: + return sys.prefix != sys.base_prefix + + +def venv_run(args, path, recreate=True, clean=False): + ''' + Runs command inside venv and returns termination code. + + Args: + args: + List of args. + path: + Name of venv. + recreate: + If false we do not run `<sys.executable> -m venv <path>` if <path> + already exists. This avoids a delay in the common case where <path> + is already set up, but fails if <path> exists but does not contain + a valid venv. + clean: + If true we first delete <path>. + ''' + if clean: + log(f'Removing any existing venv {path}.') + assert path.startswith('venv-') + shutil.rmtree(path, ignore_errors=1) + if recreate or not os.path.isdir(path): + run(f'{sys.executable} -m venv {path}') + if platform.system() == 'Windows': + command = f'{path}\\Scripts\\activate && python' + # shlex not reliable on Windows. + # Use crude quoting with "...". Seems to work. + for arg in args: + assert '"' not in arg + command += f' "{arg}"' + else: + command = f'. {path}/bin/activate && python {shlex.join(args)}' + e = run(command, check=0) + return e + + +if __name__ == '__main__': + try: + sys.exit(main(sys.argv)) + except (subprocess.CalledProcessError, subprocess.TimeoutExpired) as e: + # Terminate relatively quietly, failed commands will usually have + # generated diagnostics. + log(f'{e}') + sys.exit(1) + # Other exceptions should not happen, and will generate a full Python + # backtrace etc here.
