Mercurial > hgrepos > Python2 > PyMuPDF

comparison mupdf-source/scripts/pipcl.py @ 2:b50eed0cc0ef upstream

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
ADD: MuPDF v1.26.7: the MuPDF source as downloaded by a default build of PyMuPDF 1.26.4. The directory name has changed: no version number in the expanded directory now.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 15 Sep 2025 11:43:07 +0200
parents
children 59f1bd90b2a0
comparison
equal deleted inserted replaced
1:1d09e1dec1d9 2:b50eed0cc0ef
1 '''
2 Python packaging operations, including PEP-517 support, for use by a `setup.py`
3 script.
4
5 The intention is to take care of as many packaging details as possible so that
6 setup.py contains only project-specific information, while also giving as much
7 flexibility as possible.
8
9 For example we provide a function `build_extension()` that can be used to build
10 a SWIG extension, but we also give access to the located compiler/linker so
11 that a `setup.py` script can take over the details itself.
12
13 Run doctests with: `python -m doctest pipcl.py`
14 '''
15
16 import base64
17 import glob
18 import hashlib
19 import inspect
20 import io
21 import os
22 import platform
23 import re
24 import shutil
25 import site
26 import subprocess
27 import sys
28 import sysconfig
29 import tarfile
30 import textwrap
31 import time
32 import zipfile
33
34 import wdev
35
36
37 class Package:
38 '''
39 Our constructor takes a definition of a Python package similar to that
40 passed to `distutils.core.setup()` or `setuptools.setup()` (name, version,
41 summary etc) plus callbacks for building, getting a list of sdist
42 filenames, and cleaning.
43
44 We provide methods that can be used to implement a Python package's
45 `setup.py` supporting PEP-517.
46
47 We also support basic command line handling for use
48 with a legacy (pre-PEP-517) pip, as implemented
49 by legacy distutils/setuptools and described in:
50 https://pip.pypa.io/en/stable/reference/build-system/setup-py/
51
52 Here is a `doctest` example of using pipcl to create a SWIG extension
53 module. Requires `swig`.
54
55 Create an empty test directory:
56
57 >>> import os
58 >>> import shutil
59 >>> shutil.rmtree('pipcl_test', ignore_errors=1)
60 >>> os.mkdir('pipcl_test')
61
62 Create a `setup.py` which uses `pipcl` to define an extension module.
63
64 >>> import textwrap
65 >>> with open('pipcl_test/setup.py', 'w') as f:
66 ... _ = f.write(textwrap.dedent("""
67 ... import sys
68 ... import pipcl
69 ...
70 ... def build():
71 ... so_leaf = pipcl.build_extension(
72 ... name = 'foo',
73 ... path_i = 'foo.i',
74 ... outdir = 'build',
75 ... )
76 ... return [
77 ... ('build/foo.py', 'foo/__init__.py'),
78 ... ('cli.py', 'foo/__main__.py'),
79 ... (f'build/{so_leaf}', f'foo/'),
80 ... ('README', '$dist-info/'),
81 ... (b'Hello world', 'foo/hw.txt'),
82 ... ]
83 ...
84 ... def sdist():
85 ... return [
86 ... 'foo.i',
87 ... 'bar.i',
88 ... 'setup.py',
89 ... 'pipcl.py',
90 ... 'wdev.py',
91 ... 'README',
92 ... (b'Hello word2', 'hw2.txt'),
93 ... ]
94 ...
95 ... p = pipcl.Package(
96 ... name = 'foo',
97 ... version = '1.2.3',
98 ... fn_build = build,
99 ... fn_sdist = sdist,
100 ... entry_points = (
101 ... { 'console_scripts': [
102 ... 'foo_cli = foo.__main__:main',
103 ... ],
104 ... }),
105 ... )
106 ...
107 ... build_wheel = p.build_wheel
108 ... build_sdist = p.build_sdist
109 ...
110 ... # Handle old-style setup.py command-line usage:
111 ... if __name__ == '__main__':
112 ... p.handle_argv(sys.argv)
113 ... """))
114
115 Create the files required by the above `setup.py` - the SWIG `.i` input
116 file, the README file, and copies of `pipcl.py` and `wdev.py`.
117
118 >>> with open('pipcl_test/foo.i', 'w') as f:
119 ... _ = f.write(textwrap.dedent("""
120 ... %include bar.i
121 ... %{
122 ... #include <stdio.h>
123 ... #include <string.h>
124 ... int bar(const char* text)
125 ... {
126 ... printf("bar(): text: %s\\\\n", text);
127 ... int len = (int) strlen(text);
128 ... printf("bar(): len=%i\\\\n", len);
129 ... fflush(stdout);
130 ... return len;
131 ... }
132 ... %}
133 ... int bar(const char* text);
134 ... """))
135
136 >>> with open('pipcl_test/bar.i', 'w') as f:
137 ... _ = f.write( '\\n')
138
139 >>> with open('pipcl_test/README', 'w') as f:
140 ... _ = f.write(textwrap.dedent("""
141 ... This is Foo.
142 ... """))
143
144 >>> with open('pipcl_test/cli.py', 'w') as f:
145 ... _ = f.write(textwrap.dedent("""
146 ... def main():
147 ... print('pipcl_test:main().')
148 ... if __name__ == '__main__':
149 ... main()
150 ... """))
151
152 >>> root = os.path.dirname(__file__)
153 >>> _ = shutil.copy2(f'{root}/pipcl.py', 'pipcl_test/pipcl.py')
154 >>> _ = shutil.copy2(f'{root}/wdev.py', 'pipcl_test/wdev.py')
155
156 Use `setup.py`'s command-line interface to build and install the extension
157 module into root `pipcl_test/install`.
158
159 >>> _ = subprocess.run(
160 ... f'cd pipcl_test && {sys.executable} setup.py --root install install',
161 ... shell=1, check=1)
162
163 The actual install directory depends on `sysconfig.get_path('platlib')`:
164
165 >>> if windows():
166 ... install_dir = 'pipcl_test/install'
167 ... else:
168 ... install_dir = f'pipcl_test/install/{sysconfig.get_path("platlib").lstrip(os.sep)}'
169 >>> assert os.path.isfile( f'{install_dir}/foo/__init__.py')
170
171 Create a test script which asserts that Python function call `foo.bar(s)`
172 returns the length of `s`, and run it with `PYTHONPATH` set to the install
173 directory:
174
175 >>> with open('pipcl_test/test.py', 'w') as f:
176 ... _ = f.write(textwrap.dedent("""
177 ... import sys
178 ... import foo
179 ... text = 'hello'
180 ... print(f'test.py: calling foo.bar() with text={text!r}')
181 ... sys.stdout.flush()
182 ... l = foo.bar(text)
183 ... print(f'test.py: foo.bar() returned: {l}')
184 ... assert l == len(text)
185 ... """))
186 >>> r = subprocess.run(
187 ... f'{sys.executable} pipcl_test/test.py',
188 ... shell=1, check=1, text=1,
189 ... stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
190 ... env=os.environ | dict(PYTHONPATH=install_dir),
191 ... )
192 >>> print(r.stdout)
193 test.py: calling foo.bar() with text='hello'
194 bar(): text: hello
195 bar(): len=5
196 test.py: foo.bar() returned: 5
197 <BLANKLINE>
198
199 Check that building sdist and wheel succeeds. For now we don't attempt to
200 check that the sdist and wheel actually work.
201
202 >>> _ = subprocess.run(
203 ... f'cd pipcl_test && {sys.executable} setup.py sdist',
204 ... shell=1, check=1)
205
206 >>> _ = subprocess.run(
207 ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
208 ... shell=1, check=1)
209
210 Check that rebuild does nothing.
211
212 >>> t0 = os.path.getmtime('pipcl_test/build/foo.py')
213 >>> _ = subprocess.run(
214 ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
215 ... shell=1, check=1)
216 >>> t = os.path.getmtime('pipcl_test/build/foo.py')
217 >>> assert t == t0
218
219 Check that touching bar.i forces rebuild.
220
221 >>> os.utime('pipcl_test/bar.i')
222 >>> _ = subprocess.run(
223 ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
224 ... shell=1, check=1)
225 >>> t = os.path.getmtime('pipcl_test/build/foo.py')
226 >>> assert t > t0
227
228 Check that touching foo.i.cpp does not run swig, but does recompile/link.
229
230 >>> t0 = time.time()
231 >>> os.utime('pipcl_test/build/foo.i.cpp')
232 >>> _ = subprocess.run(
233 ... f'cd pipcl_test && {sys.executable} setup.py bdist_wheel',
234 ... shell=1, check=1)
235 >>> assert os.path.getmtime('pipcl_test/build/foo.py') <= t0
236 >>> so = glob.glob('pipcl_test/build/*.so')
237 >>> assert len(so) == 1
238 >>> so = so[0]
239 >>> assert os.path.getmtime(so) > t0
240
241 Check `entry_points` causes creation of command `foo_cli` when we install
242 from our wheel using pip. [As of 2024-02-24 using pipcl's CLI interface
243 directly with `setup.py install` does not support entry points.]
244
245 >>> print('Creating venv.', file=sys.stderr)
246 >>> _ = subprocess.run(
247 ... f'cd pipcl_test && {sys.executable} -m venv pylocal',
248 ... shell=1, check=1)
249
250 >>> print('Installing from wheel into venv using pip.', file=sys.stderr)
251 >>> _ = subprocess.run(
252 ... f'. pipcl_test/pylocal/bin/activate && pip install pipcl_test/dist/*.whl',
253 ... shell=1, check=1)
254
255 >>> print('Running foo_cli.', file=sys.stderr)
256 >>> _ = subprocess.run(
257 ... f'. pipcl_test/pylocal/bin/activate && foo_cli',
258 ... shell=1, check=1)
259
260 Wheels and sdists
261
262 Wheels:
263 We generate wheels according to:
264 https://packaging.python.org/specifications/binary-distribution-format/
265
266 * `{name}-{version}.dist-info/RECORD` uses sha256 hashes.
267 * We do not generate other `RECORD*` files such as
268 `RECORD.jws` or `RECORD.p7s`.
269 * `{name}-{version}.dist-info/WHEEL` has:
270
271 * `Wheel-Version: 1.0`
272 * `Root-Is-Purelib: false`
273 * No support for signed wheels.
274
275 Sdists:
276 We generate sdist's according to:
277 https://packaging.python.org/specifications/source-distribution-format/
278 '''
279 def __init__(self,
280 name,
281 version,
282 *,
283 platform = None,
284 supported_platform = None,
285 summary = None,
286 description = None,
287 description_content_type = None,
288 keywords = None,
289 home_page = None,
290 download_url = None,
291 author = None,
292 author_email = None,
293 maintainer = None,
294 maintainer_email = None,
295 license = None,
296 classifier = None,
297 requires_dist = None,
298 requires_python = None,
299 requires_external = None,
300 project_url = None,
301 provides_extra = None,
302
303 entry_points = None,
304
305 root = None,
306 fn_build = None,
307 fn_clean = None,
308 fn_sdist = None,
309 tag_python = None,
310 tag_abi = None,
311 tag_platform = None,
312
313 wheel_compression = zipfile.ZIP_DEFLATED,
314 wheel_compresslevel = None,
315 ):
316 '''
317 The initial args before `root` define the package
318 metadata and closely follow the definitions in:
319 https://packaging.python.org/specifications/core-metadata/
320
321 Args:
322
323 name:
324 A string, the name of the Python package.
325 version:
326 A string, the version of the Python package. Also see PEP-440
327 `Version Identification and Dependency Specification`.
328 platform:
329 A string or list of strings.
330 supported_platform:
331 A string or list of strings.
332 summary:
333 A string, short description of the package.
334 description:
335 A string, a detailed description of the package.
336 description_content_type:
337 A string describing markup of `description` arg. For example
338 `text/markdown; variant=GFM`.
339 keywords:
340 A string containing comma-separated keywords.
341 home_page:
342 URL of home page.
343 download_url:
344 Where this version can be downloaded from.
345 author:
346 Author.
347 author_email:
348 Author email.
349 maintainer:
350 Maintainer.
351 maintainer_email:
352 Maintainer email.
353 license:
354 A string containing the license text. Written into metadata
355 file `COPYING`. Is also written into metadata itself if not
356 multi-line.
357 classifier:
358 A string or list of strings. Also see:
359
360 * https://pypi.org/pypi?%3Aaction=list_classifiers
361 * https://pypi.org/classifiers/
362
363 requires_dist:
364 A string or list of strings. Also see PEP-508.
365 requires_python:
366 A string or list of strings.
367 requires_external:
368 A string or list of strings.
369 project_url:
370 A string or list of strings, each of the form: `{name}, {url}`.
371 provides_extra:
372 A string or list of strings.
373
374 entry_points:
375 String or dict specifying *.dist-info/entry_points.txt, for
376 example:
377
378 ```
379 [console_scripts]
380 foo_cli = foo.__main__:main
381 ```
382
383 or:
384
385 { 'console_scripts': [
386 'foo_cli = foo.__main__:main',
387 ],
388 }
389
390 See: https://packaging.python.org/en/latest/specifications/entry-points/
391
392 root:
393 Root of package, defaults to current directory.
394
395 fn_build:
396 A function taking no args, or a single `config_settings` dict
397 arg (as described in PEP-517), that builds the package.
398
399 Should return a list of items; each item should be a tuple
400 `(from_, to_)`, or a single string `path` which is treated as
401 the tuple `(path, path)`.
402
403 `from_` can be a string or a `bytes`. If a string it should
404 be the path to a file; a relative path is treated as relative
405 to `root`. If a `bytes` it is the contents of the file to be
406 added.
407
408 `to_` identifies what the file should be called within a wheel
409 or when installing. If `to_` ends with `/`, the leaf of `from_`
410 is appended to it (and `from_` must not be a `bytes`).
411
412 Initial `$dist-info/` in `_to` is replaced by
413 `{name}-{version}.dist-info/`; this is useful for license files
414 etc.
415
416 Initial `$data/` in `_to` is replaced by
417 `{name}-{version}.data/`. We do not enforce particular
418 subdirectories, instead it is up to `fn_build()` to specify
419 specific subdirectories such as `purelib`, `headers`,
420 `scripts`, `data` etc.
421
422 If we are building a wheel (e.g. `python setup.py bdist_wheel`,
423 or PEP-517 pip calls `self.build_wheel()`), we add file `from_`
424 to the wheel archive with name `to_`.
425
426 If we are installing (e.g. `install` command in
427 the argv passed to `self.handle_argv()`), then
428 we copy `from_` to `{sitepackages}/{to_}`, where
429 `sitepackages` is the installation directory, the
430 default being `sysconfig.get_path('platlib')` e.g.
431 `myvenv/lib/python3.9/site-packages/`.
432
433 fn_clean:
434 A function taking a single arg `all_` that cleans generated
435 files. `all_` is true iff `--all` is in argv.
436
437 For safety and convenience, can also returns a list of
438 files/directory paths to be deleted. Relative paths are
439 interpreted as relative to `root`. All paths are asserted to be
440 within `root`.
441
442 fn_sdist:
443 A function taking no args, or a single `config_settings` dict
444 arg (as described in PEP517), that returns a list of items to
445 be copied into the sdist. The list should be in the same format
446 as returned by `fn_build`.
447
448 It can be convenient to use `pipcl.git_items()`.
449
450 The specification for sdists requires that the list contains
451 `pyproject.toml`; we enforce this with a diagnostic rather than
452 raising an exception, to allow legacy command-line usage.
453
454 tag_python:
455 First element of wheel tag defined in PEP-425. If None we use
456 `cp{version}`.
457
458 For example if code works with any Python version, one can use
459 'py3'.
460
461 tag_abi:
462 Second element of wheel tag defined in PEP-425. If None we use
463 `none`.
464
465 tag_platform:
466 Third element of wheel tag defined in PEP-425. Default is
467 `os.environ('AUDITWHEEL_PLAT')` if set, otherwise derived
468 from `setuptools.distutils.util.get_platform()` (was
469 `distutils.util.get_platform()` as specified in the PEP), e.g.
470 `openbsd_7_0_amd64`.
471
472 For pure python packages use: `tag_platform=any`
473
474 wheel_compression:
475 Used as `zipfile.ZipFile()`'s `compression` parameter when
476 creating wheels.
477
478 wheel_compresslevel:
479 Used as `zipfile.ZipFile()`'s `compresslevel` parameter when
480 creating wheels.
481
482 '''
483 assert name
484 assert version
485
486 def assert_str( v):
487 if v is not None:
488 assert isinstance( v, str), f'Not a string: {v!r}'
489 def assert_str_or_multi( v):
490 if v is not None:
491 assert isinstance( v, (str, tuple, list)), f'Not a string, tuple or list: {v!r}'
492
493 assert_str( name)
494 assert_str( version)
495 assert_str_or_multi( platform)
496 assert_str_or_multi( supported_platform)
497 assert_str( summary)
498 assert_str( description)
499 assert_str( description_content_type)
500 assert_str( keywords)
501 assert_str( home_page)
502 assert_str( download_url)
503 assert_str( author)
504 assert_str( author_email)
505 assert_str( maintainer)
506 assert_str( maintainer_email)
507 assert_str( license)
508 assert_str_or_multi( classifier)
509 assert_str_or_multi( requires_dist)
510 assert_str( requires_python)
511 assert_str_or_multi( requires_external)
512 assert_str_or_multi( project_url)
513 assert_str_or_multi( provides_extra)
514
515 # https://packaging.python.org/en/latest/specifications/core-metadata/.
516 assert re.match('([A-Z0-9]|[A-Z0-9][A-Z0-9._-]*[A-Z0-9])$', name, re.IGNORECASE), \
517 f'Bad name: {name!r}'
518
519 _assert_version_pep_440(version)
520
521 # https://packaging.python.org/en/latest/specifications/binary-distribution-format/
522 if tag_python:
523 assert '-' not in tag_python
524 if tag_abi:
525 assert '-' not in tag_abi
526 if tag_platform:
527 assert '-' not in tag_platform
528
529 self.name = name
530 self.version = version
531 self.platform = platform
532 self.supported_platform = supported_platform
533 self.summary = summary
534 self.description = description
535 self.description_content_type = description_content_type
536 self.keywords = keywords
537 self.home_page = home_page
538 self.download_url = download_url
539 self.author = author
540 self.author_email = author_email
541 self.maintainer = maintainer
542 self.maintainer_email = maintainer_email
543 self.license = license
544 self.classifier = classifier
545 self.requires_dist = requires_dist
546 self.requires_python = requires_python
547 self.requires_external = requires_external
548 self.project_url = project_url
549 self.provides_extra = provides_extra
550 self.entry_points = entry_points
551
552 self.root = os.path.abspath(root if root else os.getcwd())
553 self.fn_build = fn_build
554 self.fn_clean = fn_clean
555 self.fn_sdist = fn_sdist
556 self.tag_python = tag_python
557 self.tag_abi = tag_abi
558 self.tag_platform = tag_platform
559
560 self.wheel_compression = wheel_compression
561 self.wheel_compresslevel = wheel_compresslevel
562
563
564 def build_wheel(self,
565 wheel_directory,
566 config_settings=None,
567 metadata_directory=None,
568 ):
569 '''
570 A PEP-517 `build_wheel()` function.
571
572 Also called by `handle_argv()` to handle the `bdist_wheel` command.
573
574 Returns leafname of generated wheel within `wheel_directory`.
575 '''
576 log2(
577 f' wheel_directory={wheel_directory!r}'
578 f' config_settings={config_settings!r}'
579 f' metadata_directory={metadata_directory!r}'
580 )
581
582 # Get two-digit python version, e.g. 'cp3.8' for python-3.8.6.
583 #
584 if self.tag_python:
585 tag_python = self.tag_python
586 else:
587 tag_python = 'cp' + ''.join(platform.python_version().split('.')[:2])
588
589 # ABI tag.
590 if self.tag_abi:
591 tag_abi = self.tag_abi
592 else:
593 tag_abi = 'none'
594
595 # Find platform tag used in wheel filename.
596 #
597 tag_platform = None
598 if not tag_platform:
599 tag_platform = self.tag_platform
600 if not tag_platform:
601 # Prefer this to PEP-425. Appears to be undocumented,
602 # but set in manylinux docker images and appears
603 # to be used by cibuildwheel and auditwheel, e.g.
604 # https://github.com/rapidsai/shared-action-workflows/issues/80
605 tag_platform = os.environ.get( 'AUDITWHEEL_PLAT')
606 if not tag_platform:
607 # PEP-425. On Linux gives `linux_x86_64` which is rejected by
608 # pypi.org.
609 #
610 import setuptools
611 tag_platform = setuptools.distutils.util.get_platform().replace('-', '_').replace('.', '_')
612
613 # We need to patch things on MacOS.
614 #
615 # E.g. `foo-1.2.3-cp311-none-macosx_13_x86_64.whl`
616 # causes `pip` to fail with: `not a supported wheel on this
617 # platform`. We seem to need to add `_0` to the OS version.
618 #
619 m = re.match( '^(macosx_[0-9]+)(_[^0-9].+)$', tag_platform)
620 if m:
621 tag_platform2 = f'{m.group(1)}_0{m.group(2)}'
622 log2( f'Changing from {tag_platform!r} to {tag_platform2!r}')
623 tag_platform = tag_platform2
624
625 # Final tag is, for example, 'cp39-none-win32', 'cp39-none-win_amd64'
626 # or 'cp38-none-openbsd_6_8_amd64'.
627 #
628 tag = f'{tag_python}-{tag_abi}-{tag_platform}'
629
630 path = f'{wheel_directory}/{self.name}-{self.version}-{tag}.whl'
631
632 # Do a build and get list of files to copy into the wheel.
633 #
634 items = list()
635 if self.fn_build:
636 items = self._call_fn_build(config_settings)
637
638 log2(f'Creating wheel: {path}')
639 os.makedirs(wheel_directory, exist_ok=True)
640 record = _Record()
641 with zipfile.ZipFile(path, 'w', self.wheel_compression, self.wheel_compresslevel) as z:
642
643 def add(from_, to_):
644 if isinstance(from_, str):
645 z.write(from_, to_)
646 record.add_file(from_, to_)
647 elif isinstance(from_, bytes):
648 z.writestr(to_, from_)
649 record.add_content(from_, to_)
650 else:
651 assert 0
652
653 def add_str(content, to_):
654 add(content.encode('utf8'), to_)
655
656 dist_info_dir = self._dist_info_dir()
657
658 # Add the files returned by fn_build().
659 #
660 for item in items:
661 from_, (to_abs, to_rel) = self._fromto(item)
662 add(from_, to_rel)
663
664 # Add <name>-<version>.dist-info/WHEEL.
665 #
666 add_str(
667 f'Wheel-Version: 1.0\n'
668 f'Generator: pipcl\n'
669 f'Root-Is-Purelib: false\n'
670 f'Tag: {tag}\n'
671 ,
672 f'{dist_info_dir}/WHEEL',
673 )
674 # Add <name>-<version>.dist-info/METADATA.
675 #
676 add_str(self._metainfo(), f'{dist_info_dir}/METADATA')
677
678 # Add <name>-<version>.dist-info/COPYING.
679 if self.license:
680 add_str(self.license, f'{dist_info_dir}/COPYING')
681
682 # Add <name>-<version>.dist-info/entry_points.txt.
683 entry_points_text = self._entry_points_text()
684 if entry_points_text:
685 add_str(entry_points_text, f'{dist_info_dir}/entry_points.txt')
686
687 # Update <name>-<version>.dist-info/RECORD. This must be last.
688 #
689 z.writestr(f'{dist_info_dir}/RECORD', record.get(f'{dist_info_dir}/RECORD'))
690
691 st = os.stat(path)
692 log1( f'Have created wheel size={st.st_size}: {path}')
693 if g_verbose >= 2:
694 with zipfile.ZipFile(path, compression=self.wheel_compression) as z:
695 log2(f'Contents are:')
696 for zi in sorted(z.infolist(), key=lambda z: z.filename):
697 log2(f' {zi.file_size: 10d} {zi.filename}')
698
699 return os.path.basename(path)
700
701
702 def build_sdist(self,
703 sdist_directory,
704 formats,
705 config_settings=None,
706 ):
707 '''
708 A PEP-517 `build_sdist()` function.
709
710 Also called by `handle_argv()` to handle the `sdist` command.
711
712 Returns leafname of generated archive within `sdist_directory`.
713 '''
714 log2(
715 f' sdist_directory={sdist_directory!r}'
716 f' formats={formats!r}'
717 f' config_settings={config_settings!r}'
718 )
719 if formats and formats != 'gztar':
720 raise Exception( f'Unsupported: formats={formats}')
721 items = list()
722 if self.fn_sdist:
723 if inspect.signature(self.fn_sdist).parameters:
724 items = self.fn_sdist(config_settings)
725 else:
726 items = self.fn_sdist()
727
728 prefix = f'{self.name}-{self.version}'
729 os.makedirs(sdist_directory, exist_ok=True)
730 tarpath = f'{sdist_directory}/{prefix}.tar.gz'
731 log2(f'Creating sdist: {tarpath}')
732
733 with tarfile.open(tarpath, 'w:gz') as tar:
734
735 names_in_tar = list()
736 def check_name(name):
737 if name in names_in_tar:
738 raise Exception(f'Name specified twice: {name}')
739 names_in_tar.append(name)
740
741 def add(from_, name):
742 check_name(name)
743 if isinstance(from_, str):
744 log2( f'Adding file: {os.path.relpath(from_)} => {name}')
745 tar.add( from_, f'{prefix}/{name}', recursive=False)
746 elif isinstance(from_, bytes):
747 log2( f'Adding: {name}')
748 ti = tarfile.TarInfo(f'{prefix}/{name}')
749 ti.size = len(from_)
750 ti.mtime = time.time()
751 tar.addfile(ti, io.BytesIO(from_))
752 else:
753 assert 0
754
755 def add_string(text, name):
756 textb = text.encode('utf8')
757 return add(textb, name)
758
759 found_pyproject_toml = False
760 for item in items:
761 from_, (to_abs, to_rel) = self._fromto(item)
762 if isinstance(from_, bytes):
763 add(from_, to_rel)
764 else:
765 if from_.startswith(f'{os.path.abspath(sdist_directory)}/'):
766 # Source files should not be inside <sdist_directory>.
767 assert 0, f'Path is inside sdist_directory={sdist_directory}: {from_!r}'
768 assert os.path.exists(from_), f'Path does not exist: {from_!r}'
769 assert os.path.isfile(from_), f'Path is not a file: {from_!r}'
770 if to_rel == 'pyproject.toml':
771 found_pyproject_toml = True
772 add(from_, to_rel)
773
774 if not found_pyproject_toml:
775 log0(f'Warning: no pyproject.toml specified.')
776
777 # Always add a PKG-INFO file.
778 add_string(self._metainfo(), 'PKG-INFO')
779
780 if self.license:
781 if 'COPYING' in names_in_tar:
782 log2(f'Not writing .license because file already in sdist: COPYING')
783 else:
784 add_string(self.license, 'COPYING')
785
786 log1( f'Have created sdist: {tarpath}')
787 return os.path.basename(tarpath)
788
789 def _entry_points_text(self):
790 if self.entry_points:
791 if isinstance(self.entry_points, str):
792 return self.entry_points
793 ret = ''
794 for key, values in self.entry_points.items():
795 ret += f'[{key}]\n'
796 for value in values:
797 ret += f'{value}\n'
798 return ret
799
800 def _call_fn_build( self, config_settings=None):
801 assert self.fn_build
802 log2(f'calling self.fn_build={self.fn_build}')
803 if inspect.signature(self.fn_build).parameters:
804 ret = self.fn_build(config_settings)
805 else:
806 ret = self.fn_build()
807 assert isinstance( ret, (list, tuple)), \
808 f'Expected list/tuple from {self.fn_build} but got: {ret!r}'
809 return ret
810
811
812 def _argv_clean(self, all_):
813 '''
814 Called by `handle_argv()`.
815 '''
816 if not self.fn_clean:
817 return
818 paths = self.fn_clean(all_)
819 if paths:
820 if isinstance(paths, str):
821 paths = paths,
822 for path in paths:
823 if not os.path.isabs(path):
824 path = ps.path.join(self.root, path)
825 path = os.path.abspath(path)
826 assert path.startswith(self.root+os.sep), \
827 f'path={path!r} does not start with root={self.root+os.sep!r}'
828 log2(f'Removing: {path}')
829 shutil.rmtree(path, ignore_errors=True)
830
831
832 def install(self, record_path=None, root=None):
833 '''
834 Called by `handle_argv()` to handle `install` command..
835 '''
836 log2( f'{record_path=} {root=}')
837
838 # Do a build and get list of files to install.
839 #
840 items = list()
841 if self.fn_build:
842 items = self._call_fn_build( dict())
843
844 root2 = install_dir(root)
845 log2( f'{root2=}')
846
847 log1( f'Installing into: {root2!r}')
848 dist_info_dir = self._dist_info_dir()
849
850 if not record_path:
851 record_path = f'{root2}/{dist_info_dir}/RECORD'
852 record = _Record()
853
854 def add_file(from_, to_abs, to_rel):
855 os.makedirs( os.path.dirname( to_abs), exist_ok=True)
856 if isinstance(from_, bytes):
857 log2(f'Copying content into {to_abs}.')
858 with open(to_abs, 'wb') as f:
859 f.write(from_)
860 record.add_content(from_, to_rel)
861 else:
862 log0(f'{from_=}')
863 log2(f'Copying from {os.path.relpath(from_, self.root)} to {to_abs}')
864 shutil.copy2( from_, to_abs)
865 record.add_file(from_, to_rel)
866
867 def add_str(content, to_abs, to_rel):
868 log2( f'Writing to: {to_abs}')
869 os.makedirs( os.path.dirname( to_abs), exist_ok=True)
870 with open( to_abs, 'w') as f:
871 f.write( content)
872 record.add_content(content, to_rel)
873
874 for item in items:
875 from_, (to_abs, to_rel) = self._fromto(item)
876 log0(f'{from_=} {to_abs=} {to_rel=}')
877 to_abs2 = f'{root2}/{to_rel}'
878 add_file( from_, to_abs2, to_rel)
879
880 add_str( self._metainfo(), f'{root2}/{dist_info_dir}/METADATA', f'{dist_info_dir}/METADATA')
881
882 if self.license:
883 add_str( self.license, f'{root2}/{dist_info_dir}/COPYING', f'{dist_info_dir}/COPYING')
884
885 entry_points_text = self._entry_points_text()
886 if entry_points_text:
887 add_str(
888 entry_points_text,
889 f'{root2}/{dist_info_dir}/entry_points.txt',
890 f'{dist_info_dir}/entry_points.txt',
891 )
892
893 log2( f'Writing to: {record_path}')
894 with open(record_path, 'w') as f:
895 f.write(record.get())
896
897 log2(f'Finished.')
898
899
900 def _argv_dist_info(self, root):
901 '''
902 Called by `handle_argv()`. There doesn't seem to be any documentation
903 for `setup.py dist_info`, but it appears to be like `egg_info` except
904 it writes to a slightly different directory.
905 '''
906 if root is None:
907 root = f'{self.name}-{self.version}.dist-info'
908 self._write_info(f'{root}/METADATA')
909 if self.license:
910 with open( f'{root}/COPYING', 'w') as f:
911 f.write( self.license)
912
913
914 def _argv_egg_info(self, egg_base):
915 '''
916 Called by `handle_argv()`.
917 '''
918 if egg_base is None:
919 egg_base = '.'
920 self._write_info(f'{egg_base}/.egg-info')
921
922
923 def _write_info(self, dirpath=None):
924 '''
925 Writes egg/dist info to files in directory `dirpath` or `self.root` if
926 `None`.
927 '''
928 if dirpath is None:
929 dirpath = self.root
930 log2(f'Creating files in directory {dirpath}')
931 os.makedirs(dirpath, exist_ok=True)
932 with open(os.path.join(dirpath, 'PKG-INFO'), 'w') as f:
933 f.write(self._metainfo())
934
935 # These don't seem to be required?
936 #
937 #with open(os.path.join(dirpath, 'SOURCES.txt', 'w') as f:
938 # pass
939 #with open(os.path.join(dirpath, 'dependency_links.txt', 'w') as f:
940 # pass
941 #with open(os.path.join(dirpath, 'top_level.txt', 'w') as f:
942 # f.write(f'{self.name}\n')
943 #with open(os.path.join(dirpath, 'METADATA', 'w') as f:
944 # f.write(self._metainfo())
945
946
947 def handle_argv(self, argv):
948 '''
949 Attempt to handles old-style (pre PEP-517) command line passed by
950 old releases of pip to a `setup.py` script, and manual running of
951 `setup.py`.
952
953 This is partial support at best.
954 '''
955 global g_verbose
956 #log2(f'argv: {argv}')
957
958 class ArgsRaise:
959 pass
960
961 class Args:
962 '''
963 Iterates over argv items.
964 '''
965 def __init__( self, argv):
966 self.items = iter( argv)
967 def next( self, eof=ArgsRaise):
968 '''
969 Returns next arg. If no more args, we return <eof> or raise an
970 exception if <eof> is ArgsRaise.
971 '''
972 try:
973 return next( self.items)
974 except StopIteration:
975 if eof is ArgsRaise:
976 raise Exception('Not enough args')
977 return eof
978
979 command = None
980 opt_all = None
981 opt_dist_dir = 'dist'
982 opt_egg_base = None
983 opt_formats = None
984 opt_install_headers = None
985 opt_record = None
986 opt_root = None
987
988 args = Args(argv[1:])
989
990 while 1:
991 arg = args.next(None)
992 if arg is None:
993 break
994
995 elif arg in ('-h', '--help', '--help-commands'):
996 log0(textwrap.dedent('''
997 Usage:
998 [<options>...] <command> [<options>...]
999 Commands:
1000 bdist_wheel
1001 Creates a wheel called
1002 <dist-dir>/<name>-<version>-<details>.whl, where
1003 <dist-dir> is "dist" or as specified by --dist-dir,
1004 and <details> encodes ABI and platform etc.
1005 clean
1006 Cleans build files.
1007 dist_info
1008 Creates files in <name>-<version>.dist-info/ or
1009 directory specified by --egg-base.
1010 egg_info
1011 Creates files in .egg-info/ or directory
1012 directory specified by --egg-base.
1013 install
1014 Builds and installs. Writes installation
1015 information to <record> if --record was
1016 specified.
1017 sdist
1018 Make a source distribution:
1019 <dist-dir>/<name>-<version>.tar.gz
1020 Options:
1021 --all
1022 Used by "clean".
1023 --compile
1024 Ignored.
1025 --dist-dir | -d <dist-dir>
1026 Default is "dist".
1027 --egg-base <egg-base>
1028 Used by "egg_info".
1029 --formats <formats>
1030 Used by "sdist".
1031 --install-headers <directory>
1032 Ignored.
1033 --python-tag <python-tag>
1034 Ignored.
1035 --record <record>
1036 Used by "install".
1037 --root <path>
1038 Used by "install".
1039 --single-version-externally-managed
1040 Ignored.
1041 --verbose -v
1042 Extra diagnostics.
1043 Other:
1044 windows-vs [-y <year>] [-v <version>] [-g <grade] [--verbose]
1045 Windows only; looks for matching Visual Studio.
1046 windows-python [-v <version>] [--verbose]
1047 Windows only; looks for matching Python.
1048 '''))
1049 return
1050
1051 elif arg in ('bdist_wheel', 'clean', 'dist_info', 'egg_info', 'install', 'sdist'):
1052 assert command is None, 'Two commands specified: {command} and {arg}.'
1053 command = arg
1054
1055 elif arg == '--all': opt_all = True
1056 elif arg == '--compile': pass
1057 elif arg == '--dist-dir' or arg == '-d': opt_dist_dir = args.next()
1058 elif arg == '--egg-base': opt_egg_base = args.next()
1059 elif arg == '--formats': opt_formats = args.next()
1060 elif arg == '--install-headers': opt_install_headers = args.next()
1061 elif arg == '--python-tag': pass
1062 elif arg == '--record': opt_record = args.next()
1063 elif arg == '--root': opt_root = args.next()
1064 elif arg == '--single-version-externally-managed': pass
1065 elif arg == '--verbose' or arg == '-v': g_verbose += 1
1066
1067 elif arg == 'windows-vs':
1068 command = arg
1069 break
1070 elif arg == 'windows-python':
1071 command = arg
1072 break
1073 else:
1074 raise Exception(f'Unrecognised arg: {arg}')
1075
1076 assert command, 'No command specified'
1077
1078 log1(f'Handling command={command}')
1079 if 0: pass
1080 elif command == 'bdist_wheel': self.build_wheel(opt_dist_dir)
1081 elif command == 'clean': self._argv_clean(opt_all)
1082 elif command == 'dist_info': self._argv_dist_info(opt_egg_base)
1083 elif command == 'egg_info': self._argv_egg_info(opt_egg_base)
1084 elif command == 'install': self.install(opt_record, opt_root)
1085 elif command == 'sdist': self.build_sdist(opt_dist_dir, opt_formats)
1086
1087 elif command == 'windows-python':
1088 version = None
1089 while 1:
1090 arg = args.next(None)
1091 if arg is None:
1092 break
1093 elif arg == '-v':
1094 version = args.next()
1095 elif arg == '--verbose':
1096 g_verbose += 1
1097 else:
1098 assert 0, f'Unrecognised {arg=}'
1099 python = wdev.WindowsPython(version=version)
1100 print(f'Python is:\n{python.description_ml(" ")}')
1101
1102 elif command == 'windows-vs':
1103 grade = None
1104 version = None
1105 year = None
1106 while 1:
1107 arg = args.next(None)
1108 if arg is None:
1109 break
1110 elif arg == '-g':
1111 grade = args.next()
1112 elif arg == '-v':
1113 version = args.next()
1114 elif arg == '-y':
1115 year = args.next()
1116 elif arg == '--verbose':
1117 g_verbose += 1
1118 else:
1119 assert 0, f'Unrecognised {arg=}'
1120 vs = wdev.WindowsVS(year=year, grade=grade, version=version)
1121 print(f'Visual Studio is:\n{vs.description_ml(" ")}')
1122
1123 else:
1124 assert 0, f'Unrecognised command: {command}'
1125
1126 log2(f'Finished handling command: {command}')
1127
1128
1129 def __str__(self):
1130 return ('{'
1131 f'name={self.name!r}'
1132 f' version={self.version!r}'
1133 f' platform={self.platform!r}'
1134 f' supported_platform={self.supported_platform!r}'
1135 f' summary={self.summary!r}'
1136 f' description={self.description!r}'
1137 f' description_content_type={self.description_content_type!r}'
1138 f' keywords={self.keywords!r}'
1139 f' home_page={self.home_page!r}'
1140 f' download_url={self.download_url!r}'
1141 f' author={self.author!r}'
1142 f' author_email={self.author_email!r}'
1143 f' maintainer={self.maintainer!r}'
1144 f' maintainer_email={self.maintainer_email!r}'
1145 f' license={self.license!r}'
1146 f' classifier={self.classifier!r}'
1147 f' requires_dist={self.requires_dist!r}'
1148 f' requires_python={self.requires_python!r}'
1149 f' requires_external={self.requires_external!r}'
1150 f' project_url={self.project_url!r}'
1151 f' provides_extra={self.provides_extra!r}'
1152
1153 f' root={self.root!r}'
1154 f' fn_build={self.fn_build!r}'
1155 f' fn_sdist={self.fn_sdist!r}'
1156 f' fn_clean={self.fn_clean!r}'
1157 f' tag_python={self.tag_python!r}'
1158 f' tag_abi={self.tag_abi!r}'
1159 f' tag_platform={self.tag_platform!r}'
1160 '}'
1161 )
1162
1163 def _dist_info_dir( self):
1164 return f'{self.name}-{self.version}.dist-info'
1165
1166 def _metainfo(self):
1167 '''
1168 Returns text for `.egg-info/PKG-INFO` file, or `PKG-INFO` in an sdist
1169 `.tar.gz` file, or `...dist-info/METADATA` in a wheel.
1170 '''
1171 # 2021-04-30: Have been unable to get multiline content working on
1172 # test.pypi.org so we currently put the description as the body after
1173 # all the other headers.
1174 #
1175 ret = ['']
1176 def add(key, value):
1177 if value is None:
1178 return
1179 if isinstance( value, (tuple, list)):
1180 for v in value:
1181 add( key, v)
1182 return
1183 if key == 'License' and '\n' in value:
1184 # This is ok because we write `self.license` into
1185 # *.dist-info/COPYING.
1186 #
1187 log1( f'Omitting license because contains newline(s).')
1188 return
1189 assert '\n' not in value, f'key={key} value contains newline: {value!r}'
1190 if key == 'Project-URL':
1191 assert value.count(',') == 1, f'For {key=}, should have one comma in {value!r}.'
1192 ret[0] += f'{key}: {value}\n'
1193 #add('Description', self.description)
1194 add('Metadata-Version', '2.1')
1195
1196 # These names are from:
1197 # https://packaging.python.org/specifications/core-metadata/
1198 #
1199 for name in (
1200 'Name',
1201 'Version',
1202 'Platform',
1203 'Supported-Platform',
1204 'Summary',
1205 'Description-Content-Type',
1206 'Keywords',
1207 'Home-page',
1208 'Download-URL',
1209 'Author',
1210 'Author-email',
1211 'Maintainer',
1212 'Maintainer-email',
1213 'License',
1214 'Classifier',
1215 'Requires-Dist',
1216 'Requires-Python',
1217 'Requires-External',
1218 'Project-URL',
1219 'Provides-Extra',
1220 ):
1221 identifier = name.lower().replace( '-', '_')
1222 add( name, getattr( self, identifier))
1223
1224 ret = ret[0]
1225
1226 # Append description as the body
1227 if self.description:
1228 ret += '\n' # Empty line separates headers from body.
1229 ret += self.description.strip()
1230 ret += '\n'
1231 return ret
1232
1233 def _path_relative_to_root(self, path, assert_within_root=True):
1234 '''
1235 Returns `(path_abs, path_rel)`, where `path_abs` is absolute path and
1236 `path_rel` is relative to `self.root`.
1237
1238 Interprets `path` as relative to `self.root` if not absolute.
1239
1240 We use `os.path.realpath()` to resolve any links.
1241
1242 if `assert_within_root` is true, assert-fails if `path` is not within
1243 `self.root`.
1244 '''
1245 if os.path.isabs(path):
1246 p = path
1247 else:
1248 p = os.path.join(self.root, path)
1249 p = os.path.realpath(os.path.abspath(p))
1250 if assert_within_root:
1251 assert p.startswith(self.root+os.sep) or p == self.root, \
1252 f'Path not within root={self.root+os.sep!r}: {path=} {p=}'
1253 p_rel = os.path.relpath(p, self.root)
1254 return p, p_rel
1255
1256 def _fromto(self, p):
1257 '''
1258 Returns `(from_, (to_abs, to_rel))`.
1259
1260 If `p` is a string we convert to `(p, p)`. Otherwise we assert that
1261 `p` is a tuple `(from_, to_)` where `from_` is str/bytes and `to_` is
1262 str. If `from_` is a bytes it is contents of file to add, otherwise the
1263 path of an existing file; non-absolute paths are assumed to be relative
1264 to `self.root`. If `to_` is empty or ends with `/`, we append the leaf
1265 of `from_` (which must be a str).
1266
1267 If `to_` starts with `$dist-info/`, we replace this with
1268 `self._dist_info_dir()`.
1269
1270 If `to_` starts with `$data/`, we replace this with
1271 `{self.name}-{self.version}.data/`.
1272
1273 We assert that `to_abs` is `within self.root`.
1274
1275 `to_rel` is derived from the `to_abs` and is relative to self.root`.
1276 '''
1277 ret = None
1278 if isinstance(p, str):
1279 p = p, p
1280 assert isinstance(p, tuple) and len(p) == 2
1281
1282 from_, to_ = p
1283 assert isinstance(from_, (str, bytes))
1284 assert isinstance(to_, str)
1285 if to_.endswith('/') or to_=='':
1286 to_ += os.path.basename(from_)
1287 prefix = '$dist-info/'
1288 if to_.startswith( prefix):
1289 to_ = f'{self._dist_info_dir()}/{to_[ len(prefix):]}'
1290 prefix = '$data/'
1291 if to_.startswith( prefix):
1292 to_ = f'{self.name}-{self.version}.data/{to_[ len(prefix):]}'
1293 if isinstance(from_, str):
1294 from_, _ = self._path_relative_to_root( from_, assert_within_root=False)
1295 to_ = self._path_relative_to_root(to_)
1296 assert isinstance(from_, (str, bytes))
1297 log2(f'returning {from_=} {to_=}')
1298 return from_, to_
1299
1300
1301 def build_extension(
1302 name,
1303 path_i,
1304 outdir,
1305 builddir=None,
1306 includes=None,
1307 defines=None,
1308 libpaths=None,
1309 libs=None,
1310 optimise=True,
1311 debug=False,
1312 compiler_extra='',
1313 linker_extra='',
1314 swig='swig',
1315 cpp=True,
1316 prerequisites_swig=None,
1317 prerequisites_compile=None,
1318 prerequisites_link=None,
1319 infer_swig_includes=True,
1320 ):
1321 '''
1322 Builds a Python extension module using SWIG. Works on Windows, Linux, MacOS
1323 and OpenBSD.
1324
1325 On Unix, sets rpath when linking shared libraries.
1326
1327 Args:
1328 name:
1329 Name of generated extension module.
1330 path_i:
1331 Path of input SWIG `.i` file. Internally we use swig to generate a
1332 corresponding `.c` or `.cpp` file.
1333 outdir:
1334 Output directory for generated files:
1335
1336 * `{outdir}/{name}.py`
1337 * `{outdir}/_{name}.so` # Unix
1338 * `{outdir}/_{name}.*.pyd` # Windows
1339 We return the leafname of the `.so` or `.pyd` file.
1340 builddir:
1341 Where to put intermediate files, for example the .cpp file
1342 generated by swig and `.d` dependency files. Default is `outdir`.
1343 includes:
1344 A string, or a sequence of extra include directories to be prefixed
1345 with `-I`.
1346 defines:
1347 A string, or a sequence of extra preprocessor defines to be
1348 prefixed with `-D`.
1349 libpaths
1350 A string, or a sequence of library paths to be prefixed with
1351 `/LIBPATH:` on Windows or `-L` on Unix.
1352 libs
1353 A string, or a sequence of library names to be prefixed with `-l`.
1354 optimise:
1355 Whether to use compiler optimisations.
1356 debug:
1357 Whether to build with debug symbols.
1358 compiler_extra:
1359 Extra compiler flags.
1360 linker_extra:
1361 Extra linker flags.
1362 swig:
1363 Base swig command.
1364 cpp:
1365 If true we tell SWIG to generate C++ code instead of C.
1366 prerequisites_swig:
1367 prerequisites_compile:
1368 prerequisites_link:
1369
1370 [These are mainly for use on Windows. On other systems we
1371 automatically generate dynamic dependencies using swig/compile/link
1372 commands' `-MD` and `-MF` args.]
1373
1374 Sequences of extra input files/directories that should force
1375 running of swig, compile or link commands if they are newer than
1376 any existing generated SWIG `.i` file, compiled object file or
1377 shared library file.
1378
1379 If present, the first occurrence of `True` or `False` forces re-run
1380 or no re-run. Any occurrence of None is ignored. If an item is a
1381 directory path we look for newest file within the directory tree.
1382
1383 If not a sequence, we convert into a single-item list.
1384
1385 prerequisites_swig
1386
1387 We use swig's -MD and -MF args to generate dynamic dependencies
1388 automatically, so this is not usually required.
1389
1390 prerequisites_compile
1391 prerequisites_link
1392
1393 On non-Windows we use cc's -MF and -MF args to generate dynamic
1394 dependencies so this is not usually required.
1395 infer_swig_includes:
1396 If true, we extract `-I<path>` and `-I <path>` args from
1397 `compile_extra` (also `/I` on windows) and use them with swig so
1398 that it can see the same header files as C/C++. This is useful
1399 when using enviromment variables such as `CC` and `CXX` to set
1400 `compile_extra.
1401
1402 Returns the leafname of the generated library file within `outdir`, e.g.
1403 `_{name}.so` on Unix or `_{name}.cp311-win_amd64.pyd` on Windows.
1404 '''
1405 if builddir is None:
1406 builddir = outdir
1407 includes_text = _flags( includes, '-I')
1408 defines_text = _flags( defines, '-D')
1409 libpaths_text = _flags( libpaths, '/LIBPATH:', '"') if windows() else _flags( libpaths, '-L')
1410 libs_text = _flags( libs, '-l')
1411 path_cpp = f'{builddir}/{os.path.basename(path_i)}'
1412 path_cpp += '.cpp' if cpp else '.c'
1413 os.makedirs( outdir, exist_ok=True)
1414
1415 # Run SWIG.
1416
1417 if infer_swig_includes:
1418 # Extract include flags from `compiler_extra`.
1419 swig_includes_extra = ''
1420 compiler_extra_items = compiler_extra.split()
1421 i = 0
1422 while i < len(compiler_extra_items):
1423 item = compiler_extra_items[i]
1424 # Swig doesn't seem to like a space after `I`.
1425 if item == '-I' or (windows() and item == '/I'):
1426 swig_includes_extra += f' -I{compiler_extra_items[i+1]}'
1427 i += 1
1428 elif item.startswith('-I') or (windows() and item.startswith('/I')):
1429 swig_includes_extra += f' -I{compiler_extra_items[i][2:]}'
1430 i += 1
1431 swig_includes_extra = swig_includes_extra.strip()
1432 deps_path = f'{path_cpp}.d'
1433 prerequisites_swig2 = _get_prerequisites( deps_path)
1434 run_if(
1435 f'''
1436 {swig}
1437 -Wall
1438 {"-c++" if cpp else ""}
1439 -python
1440 -module {name}
1441 -outdir {outdir}
1442 -o {path_cpp}
1443 -MD -MF {deps_path}
1444 {includes_text}
1445 {swig_includes_extra}
1446 {path_i}
1447 '''
1448 ,
1449 path_cpp,
1450 path_i,
1451 prerequisites_swig,
1452 prerequisites_swig2,
1453 )
1454
1455 path_so_leaf = f'_{name}{_so_suffix()}'
1456 path_so = f'{outdir}/{path_so_leaf}'
1457
1458 if windows():
1459 path_obj = f'{path_so}.obj'
1460
1461 permissive = '/permissive-'
1462 EHsc = '/EHsc'
1463 T = '/Tp' if cpp else '/Tc'
1464 optimise2 = '/DNDEBUG /O2' if optimise else '/D_DEBUG'
1465 debug2 = ''
1466 if debug:
1467 debug2 = '/Zi' # Generate .pdb.
1468 # debug2 = '/Z7' # Embed debug info in .obj files.
1469
1470 # As of 2023-08-23, it looks like VS tools create slightly
1471 # .dll's each time, even with identical inputs.
1472 #
1473 # Some info about this is at:
1474 # https://nikhilism.com/post/2020/windows-deterministic-builds/.
1475 # E.g. an undocumented linker flag `/Brepro`.
1476 #
1477
1478 command, pythonflags = base_compiler(cpp=cpp)
1479 command = f'''
1480 {command}
1481 # General:
1482 /c # Compiles without linking.
1483 {EHsc} # Enable "Standard C++ exception handling".
1484
1485 #/MD # Creates a multithreaded DLL using MSVCRT.lib.
1486 {'/MDd' if debug else '/MD'}
1487
1488 # Input/output files:
1489 {T}{path_cpp} # /Tp specifies C++ source file.
1490 /Fo{path_obj} # Output file. codespell:ignore
1491
1492 # Include paths:
1493 {includes_text}
1494 {pythonflags.includes} # Include path for Python headers.
1495
1496 # Code generation:
1497 {optimise2}
1498 {debug2}
1499 {permissive} # Set standard-conformance mode.
1500
1501 # Diagnostics:
1502 #/FC # Display full path of source code files passed to cl.exe in diagnostic text.
1503 /W3 # Sets which warning level to output. /W3 is IDE default.
1504 /diagnostics:caret # Controls the format of diagnostic messages.
1505 /nologo #
1506
1507 {defines_text}
1508 {compiler_extra}
1509 '''
1510 run_if( command, path_obj, path_cpp, prerequisites_compile)
1511
1512 command, pythonflags = base_linker(cpp=cpp)
1513 debug2 = '/DEBUG' if debug else ''
1514 base, _ = os.path.splitext(path_so_leaf)
1515 command = f'''
1516 {command}
1517 /DLL # Builds a DLL.
1518 /EXPORT:PyInit__{name} # Exports a function.
1519 /IMPLIB:{base}.lib # Overrides the default import library name.
1520 {libpaths_text}
1521 {pythonflags.ldflags}
1522 /OUT:{path_so} # Specifies the output file name.
1523 {debug2}
1524 /nologo
1525 {libs_text}
1526 {path_obj}
1527 {linker_extra}
1528 '''
1529 run_if( command, path_so, path_obj, prerequisites_link)
1530
1531 else:
1532
1533 # Not Windows.
1534 #
1535 command, pythonflags = base_compiler(cpp=cpp)
1536
1537 # setuptools on Linux seems to use slightly different compile flags:
1538 #
1539 # -fwrapv -O3 -Wall -O2 -g0 -DPY_CALL_TRAMPOLINE
1540 #
1541
1542 general_flags = ''
1543 if debug:
1544 general_flags += ' -g'
1545 if optimise:
1546 general_flags += ' -O2 -DNDEBUG'
1547
1548 if darwin():
1549 # MacOS's linker does not like `-z origin`.
1550 rpath_flag = "-Wl,-rpath,@loader_path/"
1551
1552 # Avoid `Undefined symbols for ... "_PyArg_UnpackTuple" ...'.
1553 general_flags += ' -undefined dynamic_lookup'
1554 elif pyodide():
1555 # Setting `-Wl,-rpath,'$ORIGIN',-z,origin` gives:
1556 # emcc: warning: ignoring unsupported linker flag: `-rpath` [-Wlinkflags]
1557 # wasm-ld: error: unknown -z value: origin
1558 #
1559 log0(f'pyodide: PEP-3149 suffix untested, so omitting. {_so_suffix()=}.')
1560 path_so_leaf = f'_{name}.so'
1561 path_so = f'{outdir}/{path_so_leaf}'
1562
1563 rpath_flag = ''
1564 else:
1565 rpath_flag = "-Wl,-rpath,'$ORIGIN',-z,origin"
1566 path_so = f'{outdir}/{path_so_leaf}'
1567 # Fun fact - on Linux, if the -L and -l options are before '{path_cpp}'
1568 # they seem to be ignored...
1569 #
1570 prerequisites = list()
1571
1572 if pyodide():
1573 # Looks like pyodide's `cc` can't compile and link in one invocation.
1574 prerequisites_compile_path = f'{path_cpp}.o.d'
1575 prerequisites += _get_prerequisites( prerequisites_compile_path)
1576 command = f'''
1577 {command}
1578 -fPIC
1579 {general_flags.strip()}
1580 {pythonflags.includes}
1581 {includes_text}
1582 {defines_text}
1583 -MD -MF {prerequisites_compile_path}
1584 -c {path_cpp}
1585 -o {path_cpp}.o
1586 {compiler_extra}
1587 '''
1588 prerequisites_link_path = f'{path_cpp}.o.d'
1589 prerequisites += _get_prerequisites( prerequisites_link_path)
1590 ld, _ = base_linker(cpp=cpp)
1591 command += f'''
1592 && {ld}
1593 {path_cpp}.o
1594 -o {path_so}
1595 -MD -MF {prerequisites_link_path}
1596 {rpath_flag}
1597 {libpaths_text}
1598 {libs_text}
1599 {linker_extra}
1600 {pythonflags.ldflags}
1601 '''
1602 else:
1603 # We use compiler to compile and link in one command.
1604 prerequisites_path = f'{path_so}.d'
1605 prerequisites = _get_prerequisites(prerequisites_path)
1606
1607 command = f'''
1608 {command}
1609 -fPIC
1610 -shared
1611 {general_flags.strip()}
1612 {pythonflags.includes}
1613 {includes_text}
1614 {defines_text}
1615 {path_cpp}
1616 -MD -MF {prerequisites_path}
1617 -o {path_so}
1618 {compiler_extra}
1619 {libpaths_text}
1620 {linker_extra}
1621 {pythonflags.ldflags}
1622 {libs_text}
1623 {rpath_flag}
1624 '''
1625 command_was_run = run_if(
1626 command,
1627 path_so,
1628 path_cpp,
1629 prerequisites_compile,
1630 prerequisites_link,
1631 prerequisites,
1632 )
1633
1634 if command_was_run and darwin():
1635 # We need to patch up references to shared libraries in `libs`.
1636 sublibraries = list()
1637 for lib in () if libs is None else libs:
1638 for libpath in libpaths:
1639 found = list()
1640 for suffix in '.so', '.dylib':
1641 path = f'{libpath}/lib{os.path.basename(lib)}{suffix}'
1642 if os.path.exists( path):
1643 found.append( path)
1644 if found:
1645 assert len(found) == 1, f'More than one file matches lib={lib!r}: {found}'
1646 sublibraries.append( found[0])
1647 break
1648 else:
1649 log2(f'Warning: can not find path of lib={lib!r} in libpaths={libpaths}')
1650 macos_patch( path_so, *sublibraries)
1651
1652 #run(f'ls -l {path_so}', check=0)
1653 #run(f'file {path_so}', check=0)
1654
1655 return path_so_leaf
1656
1657
1658 # Functions that might be useful.
1659 #
1660
1661
1662 def base_compiler(vs=None, pythonflags=None, cpp=False, use_env=True):
1663 '''
1664 Returns basic compiler command and PythonFlags.
1665
1666 Args:
1667 vs:
1668 Windows only. A `wdev.WindowsVS` instance or None to use default
1669 `wdev.WindowsVS` instance.
1670 pythonflags:
1671 A `pipcl.PythonFlags` instance or None to use default
1672 `pipcl.PythonFlags` instance.
1673 cpp:
1674 If true we return C++ compiler command instead of C. On Windows
1675 this has no effect - we always return `cl.exe`.
1676 use_env:
1677 If true we return '$CC' or '$CXX' if the corresponding
1678 environmental variable is set (without evaluating with `getenv()`
1679 or `os.environ`).
1680
1681 Returns `(cc, pythonflags)`:
1682 cc:
1683 C or C++ command. On Windows this is of the form
1684 `{vs.vcvars}&&{vs.cl}`; otherwise it is typically `cc` or `c++`.
1685 pythonflags:
1686 The `pythonflags` arg or a new `pipcl.PythonFlags` instance.
1687 '''
1688 if not pythonflags:
1689 pythonflags = PythonFlags()
1690 cc = None
1691 if use_env:
1692 if cpp:
1693 if os.environ.get( 'CXX'):
1694 cc = '$CXX'
1695 else:
1696 if os.environ.get( 'CC'):
1697 cc = '$CC'
1698 if cc:
1699 pass
1700 elif windows():
1701 if not vs:
1702 vs = wdev.WindowsVS()
1703 cc = f'"{vs.vcvars}"&&"{vs.cl}"'
1704 elif wasm():
1705 cc = 'em++' if cpp else 'emcc'
1706 else:
1707 cc = 'c++' if cpp else 'cc'
1708 cc = macos_add_cross_flags( cc)
1709 return cc, pythonflags
1710
1711
1712 def base_linker(vs=None, pythonflags=None, cpp=False, use_env=True):
1713 '''
1714 Returns basic linker command.
1715
1716 Args:
1717 vs:
1718 Windows only. A `wdev.WindowsVS` instance or None to use default
1719 `wdev.WindowsVS` instance.
1720 pythonflags:
1721 A `pipcl.PythonFlags` instance or None to use default
1722 `pipcl.PythonFlags` instance.
1723 cpp:
1724 If true we return C++ linker command instead of C. On Windows this
1725 has no effect - we always return `link.exe`.
1726 use_env:
1727 If true we use `os.environ['LD']` if set.
1728
1729 Returns `(linker, pythonflags)`:
1730 linker:
1731 Linker command. On Windows this is of the form
1732 `{vs.vcvars}&&{vs.link}`; otherwise it is typically `cc` or `c++`.
1733 pythonflags:
1734 The `pythonflags` arg or a new `pipcl.PythonFlags` instance.
1735 '''
1736 if not pythonflags:
1737 pythonflags = PythonFlags()
1738 linker = None
1739 if use_env:
1740 if os.environ.get( 'LD'):
1741 linker = '$LD'
1742 if linker:
1743 pass
1744 elif windows():
1745 if not vs:
1746 vs = wdev.WindowsVS()
1747 linker = f'"{vs.vcvars}"&&"{vs.link}"'
1748 elif wasm():
1749 linker = 'em++' if cpp else 'emcc'
1750 else:
1751 linker = 'c++' if cpp else 'cc'
1752 linker = macos_add_cross_flags( linker)
1753 return linker, pythonflags
1754
1755
1756 def git_items( directory, submodules=False):
1757 '''
1758 Returns list of paths for all files known to git within a `directory`.
1759
1760 Args:
1761 directory:
1762 Must be somewhere within a git checkout.
1763 submodules:
1764 If true we also include git submodules.
1765
1766 Returns:
1767 A list of paths for all files known to git within `directory`. Each
1768 path is relative to `directory`. `directory` must be somewhere within a
1769 git checkout.
1770
1771 We run a `git ls-files` command internally.
1772
1773 This function can be useful for the `fn_sdist()` callback.
1774 '''
1775 command = 'cd ' + directory + ' && git ls-files'
1776 if submodules:
1777 command += ' --recurse-submodules'
1778 log1(f'Running {command=}')
1779 text = subprocess.check_output( command, shell=True)
1780 ret = []
1781 for path in text.decode('utf8').strip().split( '\n'):
1782 path2 = os.path.join(directory, path)
1783 # Sometimes git ls-files seems to list empty/non-existent directories
1784 # within submodules.
1785 #
1786 if not os.path.exists(path2):
1787 log2(f'Ignoring git ls-files item that does not exist: {path2}')
1788 elif os.path.isdir(path2):
1789 log2(f'Ignoring git ls-files item that is actually a directory: {path2}')
1790 else:
1791 ret.append(path)
1792 return ret
1793
1794
1795 def run( command, capture=False, check=1, verbose=1):
1796 '''
1797 Runs a command using `subprocess.run()`.
1798
1799 Args:
1800 command:
1801 A string, the command to run.
1802
1803 Multiple lines in `command` are are treated as a single command.
1804
1805 * If a line starts with `#` it is discarded.
1806 * If a line contains ` #`, the trailing text is discarded.
1807
1808 When running the command, on Windows newlines are replaced by
1809 spaces; otherwise each line is terminated by a backslash character.
1810 capture:
1811 If true, we include the command's output in our return value.
1812 check:
1813 If true we raise an exception on error; otherwise we include the
1814 command's returncode in our return value.
1815 verbose:
1816 If true we show the command.
1817 Returns:
1818 check capture Return
1819 --------------------------
1820 false false returncode
1821 false true (returncode, output)
1822 true false None or raise exception
1823 true true output or raise exception
1824 '''
1825 lines = _command_lines( command)
1826 nl = '\n'
1827 if verbose:
1828 log1( f'Running: {nl.join(lines)}')
1829 sep = ' ' if windows() else ' \\\n'
1830 command2 = sep.join( lines)
1831 cp = subprocess.run(
1832 command2,
1833 shell=True,
1834 stdout=subprocess.PIPE if capture else None,
1835 stderr=subprocess.STDOUT if capture else None,
1836 check=check,
1837 encoding='utf8',
1838 )
1839 if check:
1840 return cp.stdout if capture else None
1841 else:
1842 return (cp.returncode, cp.stdout) if capture else cp.returncode
1843
1844
1845 def darwin():
1846 return sys.platform.startswith( 'darwin')
1847
1848 def windows():
1849 return platform.system() == 'Windows'
1850
1851 def wasm():
1852 return os.environ.get( 'OS') in ('wasm', 'wasm-mt')
1853
1854 def pyodide():
1855 return os.environ.get( 'PYODIDE') == '1'
1856
1857 def linux():
1858 return platform.system() == 'Linux'
1859
1860 def openbsd():
1861 return platform.system() == 'OpenBSD'
1862
1863 class PythonFlags:
1864 '''
1865 Compile/link flags for the current python, for example the include path
1866 needed to get `Python.h`.
1867
1868 The 'PIPCL_PYTHON_CONFIG' environment variable allows to override
1869 the location of the python-config executable.
1870
1871 Members:
1872 .includes:
1873 String containing compiler flags for include paths.
1874 .ldflags:
1875 String containing linker flags for library paths.
1876 '''
1877 def __init__(self):
1878
1879 if windows():
1880 wp = wdev.WindowsPython()
1881 self.includes = f'/I"{wp.include}"'
1882 self.ldflags = f'/LIBPATH:"{wp.libs}"'
1883
1884 elif pyodide():
1885 _include_dir = os.environ[ 'PYO3_CROSS_INCLUDE_DIR']
1886 _lib_dir = os.environ[ 'PYO3_CROSS_LIB_DIR']
1887 self.includes = f'-I {_include_dir}'
1888 self.ldflags = f'-L {_lib_dir}'
1889
1890 else:
1891 python_config = os.environ.get("PIPCL_PYTHON_CONFIG")
1892 if not python_config:
1893 # We use python-config which appears to work better than pkg-config
1894 # because it copes with multiple installed python's, e.g.
1895 # manylinux_2014's /opt/python/cp*-cp*/bin/python*.
1896 #
1897 # But... on non-macos it seems that we should not attempt to specify
1898 # libpython on the link command. The manylinux docker containers
1899 # don't actually contain libpython.so, and it seems that this
1900 # deliberate. And the link command runs ok.
1901 #
1902 python_exe = os.path.realpath( sys.executable)
1903 if darwin():
1904 # Basic install of dev tools with `xcode-select --install` doesn't
1905 # seem to provide a `python3-config` or similar, but there is a
1906 # `python-config.py` accessible via sysconfig.
1907 #
1908 # We try different possibilities and use the last one that
1909 # works.
1910 #
1911 python_config = None
1912 for pc in (
1913 f'python3-config',
1914 f'{sys.executable} {sysconfig.get_config_var("srcdir")}/python-config.py',
1915 f'{python_exe}-config',
1916 ):
1917 e = subprocess.run(
1918 f'{pc} --includes',
1919 shell=1,
1920 stdout=subprocess.DEVNULL,
1921 stderr=subprocess.DEVNULL,
1922 check=0,
1923 ).returncode
1924 log2(f'{e=} from {pc!r}.')
1925 if e == 0:
1926 python_config = pc
1927 assert python_config, f'Cannot find python-config'
1928 else:
1929 python_config = f'{python_exe}-config'
1930 log2(f'Using {python_config=}.')
1931 try:
1932 self.includes = run( f'{python_config} --includes', capture=1, verbose=0).strip()
1933 except Exception as e:
1934 raise Exception('We require python development tools to be installed.') from e
1935 self.ldflags = run( f'{python_config} --ldflags', capture=1, verbose=0).strip()
1936 if linux():
1937 # It seems that with python-3.10 on Linux, we can get an
1938 # incorrect -lcrypt flag that on some systems (e.g. WSL)
1939 # causes:
1940 #
1941 # ImportError: libcrypt.so.2: cannot open shared object file: No such file or directory
1942 #
1943 ldflags2 = self.ldflags.replace(' -lcrypt ', ' ')
1944 if ldflags2 != self.ldflags:
1945 log2(f'### Have removed `-lcrypt` from ldflags: {self.ldflags!r} -> {ldflags2!r}')
1946 self.ldflags = ldflags2
1947
1948 log2(f'{self.includes=}')
1949 log2(f'{self.ldflags=}')
1950
1951
1952 def macos_add_cross_flags(command):
1953 '''
1954 If running on MacOS and environment variables ARCHFLAGS is set
1955 (indicating we are cross-building, e.g. for arm64), returns
1956 `command` with extra flags appended. Otherwise returns unchanged
1957 `command`.
1958 '''
1959 if darwin():
1960 archflags = os.environ.get( 'ARCHFLAGS')
1961 if archflags:
1962 command = f'{command} {archflags}'
1963 log2(f'Appending ARCHFLAGS to command: {command}')
1964 return command
1965 return command
1966
1967
1968 def macos_patch( library, *sublibraries):
1969 '''
1970 If running on MacOS, patches `library` so that all references to items in
1971 `sublibraries` are changed to `@rpath/{leafname}`. Does nothing on other
1972 platforms.
1973
1974 library:
1975 Path of shared library.
1976 sublibraries:
1977 List of paths of shared libraries; these have typically been
1978 specified with `-l` when `library` was created.
1979 '''
1980 log2( f'macos_patch(): library={library} sublibraries={sublibraries}')
1981 if not darwin():
1982 return
1983 if not sublibraries:
1984 return
1985 subprocess.run( f'otool -L {library}', shell=1, check=1)
1986 command = 'install_name_tool'
1987 names = []
1988 for sublibrary in sublibraries:
1989 name = subprocess.run(
1990 f'otool -D {sublibrary}',
1991 shell=1,
1992 check=1,
1993 capture_output=1,
1994 encoding='utf8',
1995 ).stdout.strip()
1996 name = name.split('\n')
1997 assert len(name) == 2 and name[0] == f'{sublibrary}:', f'{name=}'
1998 name = name[1]
1999 # strip trailing so_name.
2000 leaf = os.path.basename(name)
2001 m = re.match('^(.+[.]((so)|(dylib)))[0-9.]*$', leaf)
2002 assert m
2003 log2(f'Changing {leaf=} to {m.group(1)}')
2004 leaf = m.group(1)
2005 command += f' -change {name} @rpath/{leaf}'
2006 command += f' {library}'
2007 log2( f'Running: {command}')
2008 subprocess.run( command, shell=1, check=1)
2009 subprocess.run( f'otool -L {library}', shell=1, check=1)
2010
2011
2012 # Internal helpers.
2013 #
2014
2015 def _command_lines( command):
2016 '''
2017 Process multiline command by running through `textwrap.dedent()`, removes
2018 comments (lines starting with `#` or ` #` until end of line), removes
2019 entirely blank lines.
2020
2021 Returns list of lines.
2022 '''
2023 command = textwrap.dedent( command)
2024 lines = []
2025 for line in command.split( '\n'):
2026 if line.startswith( '#'):
2027 h = 0
2028 else:
2029 h = line.find( ' #')
2030 if h >= 0:
2031 line = line[:h]
2032 if line.strip():
2033 lines.append(line.rstrip())
2034 return lines
2035
2036
2037 def _cpu_name():
2038 '''
2039 Returns `x32` or `x64` depending on Python build.
2040 '''
2041 #log(f'sys.maxsize={hex(sys.maxsize)}')
2042 return f'x{32 if sys.maxsize == 2**31 - 1 else 64}'
2043
2044
2045 def run_if( command, out, *prerequisites):
2046 '''
2047 Runs a command only if the output file is not up to date.
2048
2049 Args:
2050 command:
2051 The command to run. We write this into a file <out>.cmd so that we
2052 know to run a command if the command itself has changed.
2053 out:
2054 Path of the output file.
2055
2056 prerequisites:
2057 List of prerequisite paths or true/false/None items. If an item
2058 is None it is ignored, otherwise if an item is not a string we
2059 immediately return it cast to a bool.
2060
2061 Returns:
2062 True if we ran the command, otherwise None.
2063
2064
2065 If the output file does not exist, the command is run:
2066
2067 >>> verbose(1)
2068 1
2069 >>> out = 'run_if_test_out'
2070 >>> if os.path.exists( out):
2071 ... os.remove( out)
2072 >>> if os.path.exists( f'{out}.cmd'):
2073 ... os.remove( f'{out}.cmd')
2074 >>> run_if( f'touch {out}', out)
2075 pipcl.py: run_if(): Running command because: File does not exist: 'run_if_test_out'
2076 pipcl.py: run(): Running: touch run_if_test_out
2077 True
2078
2079 If we repeat, the output file will be up to date so the command is not run:
2080
2081 >>> run_if( f'touch {out}', out)
2082 pipcl.py: run_if(): Not running command because up to date: 'run_if_test_out'
2083
2084 If we change the command, the command is run:
2085
2086 >>> run_if( f'touch {out}', out)
2087 pipcl.py: run_if(): Running command because: Command has changed
2088 pipcl.py: run(): Running: touch run_if_test_out
2089 True
2090
2091 If we add a prerequisite that is newer than the output, the command is run:
2092
2093 >>> time.sleep(1)
2094 >>> prerequisite = 'run_if_test_prerequisite'
2095 >>> run( f'touch {prerequisite}')
2096 pipcl.py: run(): Running: touch run_if_test_prerequisite
2097 >>> run_if( f'touch {out}', out, prerequisite)
2098 pipcl.py: run_if(): Running command because: Prerequisite is new: 'run_if_test_prerequisite'
2099 pipcl.py: run(): Running: touch run_if_test_out
2100 True
2101
2102 If we repeat, the output will be newer than the prerequisite, so the
2103 command is not run:
2104
2105 >>> run_if( f'touch {out}', out, prerequisite)
2106 pipcl.py: run_if(): Not running command because up to date: 'run_if_test_out'
2107 '''
2108 doit = False
2109 cmd_path = f'{out}.cmd'
2110
2111 if not doit:
2112 out_mtime = _fs_mtime( out)
2113 if out_mtime == 0:
2114 doit = f'File does not exist: {out!r}'
2115
2116 if not doit:
2117 if os.path.isfile( cmd_path):
2118 with open( cmd_path) as f:
2119 cmd = f.read()
2120 else:
2121 cmd = None
2122 if command != cmd:
2123 if cmd is None:
2124 doit = 'No previous command stored'
2125 else:
2126 doit = f'Command has changed'
2127 if 0:
2128 doit += f': {cmd!r} => {command!r}'
2129
2130 if not doit:
2131 # See whether any prerequisites are newer than target.
2132 def _make_prerequisites(p):
2133 if isinstance( p, (list, tuple)):
2134 return list(p)
2135 else:
2136 return [p]
2137 prerequisites_all = list()
2138 for p in prerequisites:
2139 prerequisites_all += _make_prerequisites( p)
2140 if 0:
2141 log2( 'prerequisites_all:')
2142 for i in prerequisites_all:
2143 log2( f' {i!r}')
2144 pre_mtime = 0
2145 pre_path = None
2146 for prerequisite in prerequisites_all:
2147 if isinstance( prerequisite, str):
2148 mtime = _fs_mtime_newest( prerequisite)
2149 if mtime >= pre_mtime:
2150 pre_mtime = mtime
2151 pre_path = prerequisite
2152 elif prerequisite is None:
2153 pass
2154 elif prerequisite:
2155 doit = str(prerequisite)
2156 break
2157 if not doit:
2158 if pre_mtime > out_mtime:
2159 doit = f'Prerequisite is new: {pre_path!r}'
2160
2161 if doit:
2162 # Remove `cmd_path` before we run the command, so any failure
2163 # will force rerun next time.
2164 #
2165 try:
2166 os.remove( cmd_path)
2167 except Exception:
2168 pass
2169 log1( f'Running command because: {doit}')
2170
2171 run( command)
2172
2173 # Write the command we ran, into `cmd_path`.
2174 with open( cmd_path, 'w') as f:
2175 f.write( command)
2176 return True
2177 else:
2178 log1( f'Not running command because up to date: {out!r}')
2179
2180 if 0:
2181 log2( f'out_mtime={time.ctime(out_mtime)} pre_mtime={time.ctime(pre_mtime)}.'
2182 f' pre_path={pre_path!r}: returning {ret!r}.'
2183 )
2184
2185
2186 def _get_prerequisites(path):
2187 '''
2188 Returns list of prerequisites from Makefile-style dependency file, e.g.
2189 created by `cc -MD -MF <path>`.
2190 '''
2191 ret = list()
2192 if os.path.isfile(path):
2193 with open(path) as f:
2194 for line in f:
2195 for item in line.split():
2196 if item.endswith( (':', '\\')):
2197 continue
2198 ret.append( item)
2199 return ret
2200
2201
2202 def _fs_mtime_newest( path):
2203 '''
2204 path:
2205 If a file, returns mtime of the file. If a directory, returns mtime of
2206 newest file anywhere within directory tree. Otherwise returns 0.
2207 '''
2208 ret = 0
2209 if os.path.isdir( path):
2210 for dirpath, dirnames, filenames in os.walk( path):
2211 for filename in filenames:
2212 path = os.path.join( dirpath, filename)
2213 ret = max( ret, _fs_mtime( path))
2214 else:
2215 ret = _fs_mtime( path)
2216 return ret
2217
2218
2219 def _flags( items, prefix='', quote=''):
2220 '''
2221 Turns sequence into string, prefixing/quoting each item.
2222 '''
2223 if not items:
2224 return ''
2225 if isinstance( items, str):
2226 return items
2227 ret = ''
2228 for item in items:
2229 if ret:
2230 ret += ' '
2231 ret += f'{prefix}{quote}{item}{quote}'
2232 return ret.strip()
2233
2234
2235 def _fs_mtime( filename, default=0):
2236 '''
2237 Returns mtime of file, or `default` if error - e.g. doesn't exist.
2238 '''
2239 try:
2240 return os.path.getmtime( filename)
2241 except OSError:
2242 return default
2243
2244
2245 def _assert_version_pep_440(version):
2246 assert re.match(
2247 r'^([1-9][0-9]*!)?(0|[1-9][0-9]*)(\.(0|[1-9][0-9]*))*((a|b|rc)(0|[1-9][0-9]*))?(\.post(0|[1-9][0-9]*))?(\.dev(0|[1-9][0-9]*))?$',
2248 version,
2249 ), \
2250 f'Bad version: {version!r}.'
2251
2252
2253 g_verbose = int(os.environ.get('PIPCL_VERBOSE', '1'))
2254
2255 def verbose(level=None):
2256 '''
2257 Sets verbose level if `level` is not None.
2258 Returns verbose level.
2259 '''
2260 global g_verbose
2261 if level is not None:
2262 g_verbose = level
2263 return g_verbose
2264
2265 def log0(text=''):
2266 _log(text, 0)
2267
2268 def log1(text=''):
2269 _log(text, 1)
2270
2271 def log2(text=''):
2272 _log(text, 2)
2273
2274 def _log(text, level):
2275 '''
2276 Logs lines with prefix.
2277 '''
2278 if g_verbose >= level:
2279 caller = inspect.stack()[2].function
2280 for line in text.split('\n'):
2281 print(f'pipcl.py: {caller}(): {line}')
2282 sys.stdout.flush()
2283
2284
2285 def _so_suffix():
2286 '''
2287 Filename suffix for shared libraries is defined in pep-3149. The
2288 pep claims to only address posix systems, but the recommended
2289 sysconfig.get_config_var('EXT_SUFFIX') also seems to give the
2290 right string on Windows.
2291 '''
2292 # Example values:
2293 # linux: .cpython-311-x86_64-linux-gnu.so
2294 # macos: .cpython-311-darwin.so
2295 # openbsd: .cpython-310.so
2296 # windows .cp311-win_amd64.pyd
2297 #
2298 # Only Linux and Windows seem to identify the cpu. For example shared
2299 # libraries in numpy-1.25.2-cp311-cp311-macosx_11_0_arm64.whl are called
2300 # things like `numpy/core/_simd.cpython-311-darwin.so`.
2301 #
2302 return sysconfig.get_config_var('EXT_SUFFIX')
2303
2304
2305 def get_soname(path):
2306 '''
2307 If we are on Linux and `path` is softlink and points to a shared library
2308 for which `objdump -p` contains 'SONAME', return the pointee. Otherwise
2309 return `path`. Useful if Linux shared libraries have been created with
2310 `-Wl,-soname,...`, where we need to embed the versioned library.
2311 '''
2312 if linux() and os.path.islink(path):
2313 path2 = os.path.realpath(path)
2314 if subprocess.run(f'objdump -p {path2}|grep SONAME', shell=1, check=0).returncode == 0:
2315 return path2
2316 elif openbsd():
2317 # Return newest .so with version suffix.
2318 sos = glob.glob(f'{path}.*')
2319 log1(f'{sos=}')
2320 sos2 = list()
2321 for so in sos:
2322 suffix = so[len(path):]
2323 if not suffix or re.match('^[.][0-9.]*[0-9]$', suffix):
2324 sos2.append(so)
2325 sos2.sort(key=lambda p: os.path.getmtime(p))
2326 log1(f'{sos2=}')
2327 return sos2[-1]
2328 return path
2329
2330
2331 def install_dir(root=None):
2332 '''
2333 Returns install directory used by `install()`.
2334
2335 This will be `sysconfig.get_path('platlib')`, modified by `root` if not
2336 None.
2337 '''
2338 # todo: for pure-python we should use sysconfig.get_path('purelib') ?
2339 root2 = sysconfig.get_path('platlib')
2340 if root:
2341 if windows():
2342 # If we are in a venv, `sysconfig.get_path('platlib')`
2343 # can be absolute, e.g.
2344 # `C:\\...\\venv-pypackage-3.11.1-64\\Lib\\site-packages`, so it's
2345 # not clear how to append it to `root`. So we just use `root`.
2346 return root
2347 else:
2348 # E.g. if `root` is `install' and `sysconfig.get_path('platlib')`
2349 # is `/usr/local/lib/python3.9/site-packages`, we set `root2` to
2350 # `install/usr/local/lib/python3.9/site-packages`.
2351 #
2352 return os.path.join( root, root2.lstrip( os.sep))
2353 else:
2354 return root2
2355
2356
2357 class _Record:
2358 '''
2359 Internal - builds up text suitable for writing to a RECORD item, e.g.
2360 within a wheel.
2361 '''
2362 def __init__(self):
2363 self.text = ''
2364
2365 def add_content(self, content, to_, verbose=True):
2366 if isinstance(content, str):
2367 content = content.encode('utf8')
2368
2369 # Specification for the line we write is supposed to be in
2370 # https://packaging.python.org/en/latest/specifications/binary-distribution-format
2371 # but it's not very clear.
2372 #
2373 h = hashlib.sha256(content)
2374 digest = h.digest()
2375 digest = base64.urlsafe_b64encode(digest)
2376 digest = digest.rstrip(b'=')
2377 digest = digest.decode('utf8')
2378
2379 self.text += f'{to_},sha256={digest},{len(content)}\n'
2380 if verbose:
2381 log2(f'Adding {to_}')
2382
2383 def add_file(self, from_, to_):
2384 log1(f'Adding file: {os.path.relpath(from_)} => {to_}')
2385 with open(from_, 'rb') as f:
2386 content = f.read()
2387 self.add_content(content, to_, verbose=False)
2388
2389 def get(self, record_path=None):
2390 '''
2391 Returns contents of the RECORD file. If `record_path` is
2392 specified we append a final line `<record_path>,,`; this can be
2393 used to include the RECORD file itself in the contents, with
2394 empty hash and size fields.
2395 '''
2396 ret = self.text
2397 if record_path:
2398 ret += f'{record_path},,\n'
2399 return ret