Mercurial > hgrepos > Python > apps > py-cutils

comparison cutils/shasum.py @ 117:e51f34ad6d71

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Move constant definitions into new module cutils.util.constants
author Franz Glasner <fzglas.hg@dom66.de>
date Sun, 29 Dec 2024 17:39:00 +0100
parents 6378ba9ae949
children 12339ac2148d
comparison
equal deleted inserted replaced
116:1856374bbd40 117:e51f34ad6d71
22 try: 22 try:
23 import mmap 23 import mmap
24 except ImportError: 24 except ImportError:
25 mmap = None 25 mmap = None
26 import os 26 import os
27 try:
28 import pathlib
29 except ImportError:
30 pathlib = None
31 import re 27 import re
32 import stat 28 import stat
33 import sys 29 import sys
34 30
31 from .util import constants
35 from . import (__version__, __revision__) 32 from . import (__version__, __revision__)
36
37
38 PY2 = sys.version_info[0] < 3
39
40 if PY2:
41 PATH_TYPES = (unicode, str) # noqa: F821 (undefined name 'unicode')
42 else:
43 if pathlib:
44 PATH_TYPES = (str, bytes, pathlib.Path)
45 else:
46 PATH_TYPES = (str, bytes)
47
48 READ_CHUNK_SIZE = 2 * 1024 * 1024 # like BUFSIZE_MAX on FreeBSD
49 MAX_AUTO_MAP_SIZE = 8 * 1024 * 1024
50 MAP_WINDOW_SIZE = MAX_AUTO_MAP_SIZE # do not totally trash memory on big files
51 33
52 34
53 def main(argv=None): 35 def main(argv=None):
54 aparser = argparse.ArgumentParser( 36 aparser = argparse.ArgumentParser(
55 description="Python implementation of shasum", 37 description="Python implementation of shasum",
194 opts.algorithm[1], 176 opts.algorithm[1],
195 True, 177 True,
196 opts.base64) 178 opts.base64)
197 else: 179 else:
198 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'): 180 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
199 if PY2: 181 if constants.PY2:
200 if sys.platform == "win32": 182 if sys.platform == "win32":
201 import msvcrt # noqa: E401 183 import msvcrt # noqa: E401
202 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) 184 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
203 source = sys.stdin 185 source = sys.stdin
204 else: 186 else:
263 245
264 def verify_digests_with_checklist(opts): 246 def verify_digests_with_checklist(opts):
265 dest = opts.dest or sys.stdout 247 dest = opts.dest or sys.stdout
266 exit_code = 0 248 exit_code = 0
267 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'): 249 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
268 if PY2: 250 if constants.PY2:
269 if sys.platform == "win32": 251 if sys.platform == "win32":
270 import os, msvcrt # noqa: E401 252 import os, msvcrt # noqa: E401
271 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) 253 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
272 source = sys.stdin 254 source = sys.stdin
273 else: 255 else:
594 576
595 If a file descriptor is given is must support :func:`os.read`. 577 If a file descriptor is given is must support :func:`os.read`.
596 578
597 """ 579 """
598 h = hashobj() 580 h = hashobj()
599 if isinstance(path, PATH_TYPES): 581 if isinstance(path, constants.PATH_TYPES):
600 flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) \ 582 flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) \
601 | getattr(os, "O_SEQUENTIAL", 0) | getattr(os, "O_NOCTTY", 0) 583 | getattr(os, "O_SEQUENTIAL", 0) | getattr(os, "O_NOCTTY", 0)
602 fd = os.open(path, flags) 584 fd = os.open(path, flags)
603 own_fd = True 585 own_fd = True
604 else: 586 else:
615 use_mmap = False 597 use_mmap = False
616 else: 598 else:
617 if stat.S_ISREG(st[stat.ST_MODE]): 599 if stat.S_ISREG(st[stat.ST_MODE]):
618 filesize = st[stat.ST_SIZE] 600 filesize = st[stat.ST_SIZE]
619 if (use_mmap is None) \ 601 if (use_mmap is None) \
620 and (filesize > MAX_AUTO_MAP_SIZE): 602 and (filesize > constants.MAX_AUTO_MAP_SIZE):
621 # 603 #
622 # This is borrowed from FreeBSD's cp(1) implementation: 604 # This is borrowed from FreeBSD's cp(1) implementation:
623 # Mmap and process if less than 8M (the limit is 605 # Mmap and process if less than 8M (the limit is
624 # so we don't totally trash memory on big files. 606 # so we don't totally trash memory on big files.
625 # This is really a minor hack, but it wins some 607 # This is really a minor hack, but it wins some
635 if mmap is None or not use_mmap: 617 if mmap is None or not use_mmap:
636 # No mmap available or wanted -> use traditional low-level file IO 618 # No mmap available or wanted -> use traditional low-level file IO
637 fadvise = getattr(os, "posix_fadvise", None) 619 fadvise = getattr(os, "posix_fadvise", None)
638 if fadvise: 620 if fadvise:
639 fadvise(fd, 0, 0, os.POSIX_FADV_SEQUENTIAL) 621 fadvise(fd, 0, 0, os.POSIX_FADV_SEQUENTIAL)
640 if not PY2: 622 if not constants.PY2:
641 fileobj = io.FileIO(fd, mode="r", closefd=False) 623 fileobj = io.FileIO(fd, mode="r", closefd=False)
642 buf = bytearray(READ_CHUNK_SIZE) 624 buf = bytearray(constants.READ_CHUNK_SIZE)
643 with memoryview(buf) as full_view: 625 with memoryview(buf) as full_view:
644 while True: 626 while True:
645 try: 627 try:
646 n = fileobj.readinto(buf) 628 n = fileobj.readinto(buf)
647 except OSError as e: 629 except OSError as e:
650 errno.EINTR): 632 errno.EINTR):
651 raise 633 raise
652 else: 634 else:
653 if n == 0: 635 if n == 0:
654 break 636 break
655 if n == READ_CHUNK_SIZE: 637 if n == constants.READ_CHUNK_SIZE:
656 h.update(buf) 638 h.update(buf)
657 else: 639 else:
658 with full_view[:n] as partial_view: 640 with full_view[:n] as partial_view:
659 h.update(partial_view) 641 h.update(partial_view)
660 else: 642 else:
661 while True: 643 while True:
662 try: 644 try:
663 buf = os.read(fd, READ_CHUNK_SIZE) 645 buf = os.read(fd, constants.READ_CHUNK_SIZE)
664 except OSError as e: 646 except OSError as e:
665 if e.errno not in (errno.EAGAIN, 647 if e.errno not in (errno.EAGAIN,
666 errno.EWOULDBLOCK, 648 errno.EWOULDBLOCK,
667 errno.EINTR): 649 errno.EINTR):
668 raise 650 raise
676 # 658 #
677 # NOTE: On Windows mmapped files with length 0 are not supported. 659 # NOTE: On Windows mmapped files with length 0 are not supported.
678 # So ensure to not call mmap.mmap() if the file size is 0. 660 # So ensure to not call mmap.mmap() if the file size is 0.
679 # 661 #
680 madvise = getattr(mmap.mmap, "madvise", None) 662 madvise = getattr(mmap.mmap, "madvise", None)
681 if filesize <= MAP_WINDOW_SIZE: 663 if filesize <= constants.MAP_WINDOW_SIZE:
682 mapsize = filesize 664 mapsize = filesize
683 else: 665 else:
684 mapsize = MAP_WINDOW_SIZE 666 mapsize = constants.MAP_WINDOW_SIZE
685 mapoffset = 0 667 mapoffset = 0
686 rest = filesize 668 rest = filesize
687 while rest > 0: 669 while rest > 0:
688 m = mmap.mmap(fd, 670 m = mmap.mmap(fd,
689 mapsize, 671 mapsize,
715 697
716 """ 698 """
717 h = hashobj() 699 h = hashobj()
718 while True: 700 while True:
719 try: 701 try:
720 buf = instream.read(READ_CHUNK_SIZE) 702 buf = instream.read(constants.READ_CHUNK_SIZE)
721 except OSError as e: 703 except OSError as e:
722 if e.errno not in (errno.EAGAIN, errno.EWOULDBLOCK, errno.EINTR): 704 if e.errno not in (errno.EAGAIN, errno.EWOULDBLOCK, errno.EINTR):
723 raise 705 raise
724 else: 706 else:
725 if buf is not None: 707 if buf is not None: