Mercurial > hgrepos > Python > apps > py-cutils

annotate cutils/shasum.py @ 118:12339ac2148d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Move some functions into cutils.util (i.e. algorithms and their aliases)
author Franz Glasner <fzglas.hg@dom66.de>
date Sun, 29 Dec 2024 18:22:22 +0100
parents e51f34ad6d71
children dd4fe912d7e9
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
73
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
1 # -*- coding: utf-8 -*-
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
2 # :-
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
3 # :Copyright: (c) 2020-2022 Franz Glasner
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
4 # :License: BSD-3-Clause
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
5 # :-
113
Franz Glasner <fzglas.hg@dom66.de>
parents: 110
diff changeset
6 r"""Pure Python implementation of `shasum`.
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
7
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
8 """
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
9
72
ae2df602beb4 Make shasum.py and dos2unix sub-modules to the new "cutils" package
Franz Glasner <fzglas.hg@dom66.de>
parents: 71
diff changeset
10 from __future__ import print_function, absolute_import
13
db64e282b049 Implement a version option
Franz Glasner <fzglas.hg@dom66.de>
parents: 12
diff changeset
11
73
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
12
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
13 __all__ = []
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
14
13
db64e282b049 Implement a version option
Franz Glasner <fzglas.hg@dom66.de>
parents: 12
diff changeset
15
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
16 import argparse
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
17 import base64
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
18 import binascii
66
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
19 import errno
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
20 import io
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
21 try:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
22 import mmap
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
23 except ImportError:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
24 mmap = None
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
25 import os
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
26 import re
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
27 import stat
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
28 import sys
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
29
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
30 from . import (__version__, __revision__)
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
31 from . import util
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
32 from .util import constants
95
fc2dd6afd594 Style: canonical global variable and import ordering
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 94
diff changeset
33
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
34
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
35 def main(argv=None):
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
36 aparser = argparse.ArgumentParser(
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
37 description="Python implementation of shasum",
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
38 fromfile_prefix_chars='@')
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
39 aparser.add_argument(
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
40 "--algorithm", "-a", action="store", type=util.argv2algo,
87
b46673c42894 Implement support for BLAKE2b-256 (aka BLAKE2-256).
Franz Glasner <fzglas.hg@dom66.de>
parents: 86
diff changeset
41 help="1 (default), 224, 256, 384, 512, 3-224, 3-256, 3-384, 3-512, blake2b, blake2s, blake2, blake2-256, md5")
7
47b4c98e4d40 Allow algorithm selection for all algorithms in :mod:`hashlib`.
Franz Glasner <fzglas.hg@dom66.de>
parents: 6
diff changeset
42 aparser.add_argument(
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
43 "--base64", action="store_true",
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
44 help="Output checksums in base64 notation, not hexadecimal (OpenBSD).")
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
45 aparser.add_argument(
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
46 "--binary", "-b", action="store_false", dest="text_mode", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
47 help="Read in binary mode (default)")
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
48 aparser.add_argument(
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
49 "--bsd", "-B", action="store_true", dest="bsd", default=False,
17
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
50 help="Write BSD style output. This is also the default output format of :command:`openssl dgst`.")
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
51 aparser.add_argument(
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
52 "--check", "-c", action="store_true",
17
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
53 help="""Read digests from FILEs and check them.
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
54 If this option is specified, the FILE options become checklists. Each
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
55 checklist should contain hash results in a supported format, which will
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
56 be verified against the specified paths. Output consists of the digest
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
57 used, the file name, and an OK, FAILED, or MISSING for the result of
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
58 the comparison. This will validate any of the supported checksums.
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
59 If no file is given, stdin is used.""")
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
60 aparser.add_argument(
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
61 "--checklist", "-C", metavar="CHECKLIST",
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
62 help="""Compare the checksum of each FILE against the checksums in
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
63 the CHECKLIST. Any specified FILE that is not listed in the CHECKLIST will
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
64 generate an error.""")
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
65 aparser.add_argument(
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
66 "--checklist-allow-distinfo", action="store_true",
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
67 dest="allow_distinfo",
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
68 help='Allow FreeBSD "distinfo" formatted checklists: ignore SIZE and TIMESTAMP lines.')
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
69
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
70 aparser.add_argument(
105
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
71 "--follow-symlinks", action="store_true", dest="follow_symlinks",
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
72 help="""Also follow symlinks that resolve to directories. Only effective if `--recurse` is activated.""")
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
73
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
74 aparser.add_argument(
90
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
75 "--mmap", action="store_true", dest="mmap", default=None,
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
76 help="""Use mmap if available. Default is to determine automatically
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
77 from the filesize.""")
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
78 aparser.add_argument(
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
79 "--no-mmap", action="store_false", dest="mmap", default=None,
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
80 help="Dont use mmap.")
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
81
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
82 aparser.add_argument(
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
83 "--recurse", action="store_true",
105
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
84 help="Recurse into sub-directories while interpreting every FILE as a directory.")
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
85
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
86 aparser.add_argument(
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
87 "--reverse", "-r", action="store_false", dest="bsd", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
88 help="Explicitely select normal coreutils style output (to be option compatible with BSD style commands and :command:`openssl dgst -r`)")
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
89 aparser.add_argument(
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
90 "--tag", action="store_true", dest="bsd", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
91 help="Alias for the `--bsd' option (to be compatible with :command:`b2sum`)")
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
92 aparser.add_argument(
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
93 "--text", "-t", action="store_true", dest="text_mode", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
94 help="Read in text mode (not supported)")
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
95 aparser.add_argument(
27
c29bd96dcd4f Put the HG revision into the version output also
Franz Glasner <fzglas.hg@dom66.de>
parents: 26
diff changeset
96 "--version", "-v", action="version", version="%s (rv:%s)" % (__version__, __revision__))
13
db64e282b049 Implement a version option
Franz Glasner <fzglas.hg@dom66.de>
parents: 12
diff changeset
97 aparser.add_argument(
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
98 "files", nargs="*", metavar="FILE")
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
99
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
100 opts = aparser.parse_args(args=argv)
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
101
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
102 if opts.text_mode:
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
103 print("ERROR: text mode not supported", file=sys.stderr)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
104 sys.exit(78) # :manpage:`sysexits(3)` EX_CONFIG
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
105
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
106 if opts.check and opts.checklist:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
107 print("ERROR: only one of --check or --checklist allowed",
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
108 file=sys.stderr)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
109 sys.exit(64) # :manpage:`sysexits(3)` EX_USAGE
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
110
7
47b4c98e4d40 Allow algorithm selection for all algorithms in :mod:`hashlib`.
Franz Glasner <fzglas.hg@dom66.de>
parents: 6
diff changeset
111 if not opts.algorithm:
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
112 opts.algorithm = util.argv2algo("1")
7
47b4c98e4d40 Allow algorithm selection for all algorithms in :mod:`hashlib`.
Franz Glasner <fzglas.hg@dom66.de>
parents: 6
diff changeset
113
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
114 opts.dest = None
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
115
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
116 return shasum(opts)
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
117
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
118
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
119 def gen_opts(files=[], algorithm="SHA1", bsd=False, text_mode=False,
100
f95918115c6b FIX: Implement some new commandline flags in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 96
diff changeset
120 checklist=False, check=False, dest=None, base64=False,
106
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
121 allow_distinfo=False, mmap=None, recurse=False,
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
122 follow_symlinks=False):
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
123 if text_mode:
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
124 raise ValueError("text mode not supported")
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
125 if checklist and check:
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
126 raise ValueError("only one of `checklist' or `check' is allowed")
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
127 opts = argparse.Namespace(files=files,
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
128 algorithm=(util.algotag2algotype(algorithm),
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
129 algorithm),
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
130 bsd=bsd,
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
131 checklist=checklist,
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
132 check=check,
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
133 text_mode=False,
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
134 dest=dest,
100
f95918115c6b FIX: Implement some new commandline flags in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 96
diff changeset
135 base64=base64,
f95918115c6b FIX: Implement some new commandline flags in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 96
diff changeset
136 allow_distinfo=allow_distinfo,
106
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
137 mmap=mmap,
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
138 recurse=recurse,
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
139 follow_symlinks=follow_symlinks)
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
140 return opts
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
141
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
142
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
143 def shasum(opts):
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
144 if opts.check:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
145 return verify_digests_from_files(opts)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
146 elif opts.checklist:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
147 return verify_digests_with_checklist(opts)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
148 else:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
149 return generate_digests(opts)
10
77446cd3ea6f Move the digest generation loop from "main()" into an own function "generate_digests()".
Franz Glasner <fzglas.hg@dom66.de>
parents: 9
diff changeset
150
77446cd3ea6f Move the digest generation loop from "main()" into an own function "generate_digests()".
Franz Glasner <fzglas.hg@dom66.de>
parents: 9
diff changeset
151
77446cd3ea6f Move the digest generation loop from "main()" into an own function "generate_digests()".
Franz Glasner <fzglas.hg@dom66.de>
parents: 9
diff changeset
152 def generate_digests(opts):
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
153 if opts.bsd:
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
154 out = out_bsd
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
155 else:
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
156 out = out_std
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
157 if opts.recurse:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
158 if not opts.files:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
159 opts.files.append(".")
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
160 for dn in opts.files:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
161 if not os.path.isdir(dn):
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
162 if os.path.exists(dn):
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
163 raise OSError(errno.ENOTDIR, "not a directory", dn)
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
164 else:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
165 raise OSError(errno.ENOENT, "directory does not exist", dn)
105
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
166 for dirpath, dirnames, dirfiles in os.walk(
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
167 dn, followlinks=opts.follow_symlinks):
110
3060aa4cb252 When generating digests recursively normalize the output somewhat by sorting directory names and filenames
Franz Glasner <fzglas.hg@dom66.de>
parents: 106
diff changeset
168 dirnames.sort()
3060aa4cb252 When generating digests recursively normalize the output somewhat by sorting directory names and filenames
Franz Glasner <fzglas.hg@dom66.de>
parents: 106
diff changeset
169 dirfiles.sort()
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
170 for fn in dirfiles:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
171 path = os.path.join(dirpath, fn)
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
172 out(opts.dest or sys.stdout,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
173 compute_digest_file(opts.algorithm[0], path,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
174 use_mmap=opts.mmap),
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
175 path,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
176 opts.algorithm[1],
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
177 True,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
178 opts.base64)
4
67d10529ce88 FIX: "-" filename handling now consistent with Perl shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 3
diff changeset
179 else:
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
180 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
181 if constants.PY2:
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
182 if sys.platform == "win32":
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
183 import msvcrt # noqa: E401
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
184 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
185 source = sys.stdin
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
186 else:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
187 source = sys.stdin.buffer
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
188 out(sys.stdout,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
189 compute_digest_stream(opts.algorithm[0], source),
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
190 None,
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
191 opts.algorithm[1],
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
192 True,
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
193 opts.base64)
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
194 else:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
195 for fn in opts.files:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
196 out(opts.dest or sys.stdout,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
197 compute_digest_file(opts.algorithm[0], fn,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
198 use_mmap=opts.mmap),
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
199 fn,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
200 opts.algorithm[1],
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
201 True,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
202 opts.base64)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
203 return 0
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
204
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
205
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
206 def compare_digests_equal(given_digest, expected_digest, algo):
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
207 """Compare a newly computed binary digest `given_digest` with a digest
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
208 string (hex or base64) in `expected_digest`.
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
209
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
210 :param bytes given_digest:
71
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
211 :param expected_digest: digest (as bytes) or hexlified or base64 encoded
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
212 digest (as str)
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
213 :type expected_digest: str or bytes or bytearray
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
214 :param algo: The algorithm (factory)
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
215 :return: `True` if the digests are equal, `False` if not
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
216 :rtype: bool
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
217
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
218 """
71
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
219 if isinstance(expected_digest, (bytes, bytearray)) \
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
220 and len(expected_digest) == algo().digest_size:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
221 exd = expected_digest
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
222 else:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
223 if len(expected_digest) == algo().digest_size * 2:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
224 # hex
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
225 if re.search(r"\A[a-fA-F0-9]+\Z", expected_digest):
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
226 try:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
227 exd = binascii.unhexlify(expected_digest)
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
228 except TypeError:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
229 return False
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
230 else:
52
5935055edea6 More proper formal checks (with regexp) for valid hex and base64 encoding of digests
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 51
diff changeset
231 return False
5935055edea6 More proper formal checks (with regexp) for valid hex and base64 encoding of digests
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 51
diff changeset
232 else:
71
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
233 # base64
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
234 if re.search(
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
235 r"\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?\Z",
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
236 expected_digest):
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
237 try:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
238 exd = base64.b64decode(expected_digest)
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
239 except TypeError:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
240 return False
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
241 else:
52
5935055edea6 More proper formal checks (with regexp) for valid hex and base64 encoding of digests
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 51
diff changeset
242 return False
75
a31de3c65877 Remove the use of "hmac.compare_digest()": there are no secrets to protect here
Franz Glasner <fzglas.hg@dom66.de>
parents: 73
diff changeset
243 return given_digest == exd
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
244
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
245
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
246 def verify_digests_with_checklist(opts):
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
247 dest = opts.dest or sys.stdout
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
248 exit_code = 0
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
249 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
250 if constants.PY2:
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
251 if sys.platform == "win32":
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
252 import os, msvcrt # noqa: E401
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
253 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
254 source = sys.stdin
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
255 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
256 source = sys.stdin.buffer
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
257 pl = get_parsed_digest_line_from_checklist(opts.checklist, opts, None)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
258 if pl is None:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
259 exit_code = 1
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
260 print("-: MISSING", file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
261 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
262 tag, algo, cl_filename, cl_digest = pl
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
263 computed_digest = compute_digest_stream(algo, source)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
264 if compare_digests_equal(computed_digest, cl_digest, algo):
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
265 res = "OK"
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
266 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
267 res = "FAILED"
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
268 exit_code = 1
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
269 print("{}: {}: {}".format(tag, "-", res), file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
270 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
271 for fn in opts.files:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
272 pl = get_parsed_digest_line_from_checklist(opts.checklist, opts, fn)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
273 if pl is None:
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
274 print("{}: MISSING".format(fn), file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
275 exit_code = 1
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
276 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
277 tag, algo, cl_filename, cl_digest = pl
90
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
278 computed_digest = compute_digest_file(algo, fn,
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
279 use_mmap=opts.mmap)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
280 if compare_digests_equal(computed_digest, cl_digest, algo):
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
281 res = "OK"
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
282 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
283 exit_code = 1
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
284 res = "FAILED"
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
285 print("{}: {}: {}".format(tag, fn, res), file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
286 return exit_code
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
287
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
288
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
289 def verify_digests_from_files(opts):
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
290 dest = opts.dest or sys.stdout
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
291 exit_code = 0
20
8f0241ed4a00 Do not append "-" to an empty FILE list any more but check explicitely for an empty list
Franz Glasner <fzglas.hg@dom66.de>
parents: 19
diff changeset
292 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
293 for checkline in sys.stdin:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
294 if not checkline:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
295 continue
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
296 r, fn, tag = handle_checkline(opts, checkline)
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
297 if tag in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
298 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
299 continue
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
300 print("{}: {}: {}".format(tag, fn, r.upper()), file=dest)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
301 if r != "ok" and exit_code == 0:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
302 exit_code = 1
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
303 else:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
304 for fn in opts.files:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
305 with io.open(fn, "rt", encoding="utf-8") as checkfile:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
306 for checkline in checkfile:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
307 if not checkline:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
308 continue
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
309 r, fn, tag = handle_checkline(opts, checkline)
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
310 if tag in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
311 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
312 continue
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
313 print("{}: {}: {}".format(tag, fn, r.upper()), file=dest)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
314 if r != "ok" and exit_code == 0:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
315 exit_code = 1
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
316 return exit_code
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
317
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
318
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
319 def handle_checkline(opts, line):
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
320 """
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
321 :return: a tuple with static "ok", "missing", or "failed", the filename and
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
322 the digest used
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
323 :rtype: tuple(str, str, str)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
324
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
325 """
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
326 parts = parse_digest_line(opts, line)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
327 if not parts:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
328 raise ValueError(
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
329 "improperly formatted digest line: {}".format(line))
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
330 tag, algo, fn, digest = parts
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
331 if tag in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
332 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
333 return (None, None, tag)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
334 try:
90
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
335 d = compute_digest_file(algo, fn, use_mmap=opts.mmap)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
336 if compare_digests_equal(d, digest, algo):
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
337 return ("ok", fn, tag)
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
338 else:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
339 return ("failed", fn, tag)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
340 except EnvironmentError:
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
341 return ("missing", fn, tag)
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
342
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
343
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
344 def get_parsed_digest_line_from_checklist(checklist, opts, filename):
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
345 if filename is None:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
346 filenames = ("-", "stdin", "", )
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
347 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
348 filenames = (
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
349 util.normalize_filename(filename, strip_leading_dot_slash=True),)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
350 with io.open(checklist, "rt", encoding="utf-8") as clf:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
351 for checkline in clf:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
352 if not checkline:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
353 continue
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
354 parts = parse_digest_line(opts, checkline)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
355 if not parts:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
356 raise ValueError(
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
357 "improperly formatted digest line: {}".format(checkline))
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
358 if parts[0] in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
359 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
360 continue
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
361 fn = util.normalize_filename(parts[2], strip_leading_dot_slash=True)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
362 if fn in filenames:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
363 return parts
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
364 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
365 return None
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
366
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
367
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
368 def parse_digest_line(opts, line):
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
369 """Parse a `line` of a digest file and return its parts.
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
370
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
371 This is rather strict. But if `opts.allow_distinfo` is `True` then
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
372 some additional keywords ``SIZE`` and ``TIMESTAMP``are recignized
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
373 and returned. The caller is responsible to handle them.
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
374
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
375 :return: a tuple of the normalized algorithm tag, the algorithm
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
376 constructor, the filename and the hex digest;
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
377 if `line` cannot be parsed successfully `None` is returned
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
378 :rtype: tuple(str, obj, str, str) or None
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
379
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
380 Handles coreutils and BSD-style file formats.
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
381
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
382 """
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
383 # determine checkfile format (BSD or coreutils)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
384 # BSD?
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
385 mo = re.search(r"\A(\S+)\s*\((.*)\)\s*=\s*(.+)\n?\Z", line)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
386 if mo:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
387 # (tag, algorithm, filename, digest)
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
388 if opts.allow_distinfo:
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
389 if mo.group(1) == "SIZE":
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
390 return ("SIZE", None, None, mo.group(3))
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
391 return (mo.group(1),
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
392 util.algotag2algotype(mo.group(1)),
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
393 mo.group(2),
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
394 mo.group(3))
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
395 else:
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
396 if opts.allow_distinfo:
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
397 mo = re.search(r"\ATIMESTAMP\s*=\s*([0-9]+)\s*\n\Z", line)
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
398 if mo:
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
399 return ("TIMESTAMP", None, None, mo.group(1))
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
400
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
401 # coreutils?
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
402 mo = re.search(r"([^\ ]+) [\*\ ]?(.+)\n?\Z", line)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
403 if mo:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
404 # (tag, algorithm, filename, digest)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
405 return (opts.algorithm[1],
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
406 opts.algorithm[0],
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
407 mo.group(2),
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
408 mo.group(1))
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
409 else:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
410 return None
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
411
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
412
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
413 def out_bsd(dest, digest, filename, digestname, binary, use_base64):
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
414 """BSD format output, also :command:`openssl dgst` and
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
415 :command:`b2sum --tag" format output
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
416
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
417 """
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
418 if use_base64:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
419 digest = base64.b64encode(digest).decode("ascii")
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
420 else:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
421 digest = binascii.hexlify(digest).decode("ascii")
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
422 if filename is None:
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
423 print(digest, file=dest)
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
424 else:
19
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
425 print("{} ({}) = {}".format(digestname,
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
426 util.normalize_filename(filename),
19
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
427 digest),
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
428 file=dest)
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
429
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
430
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
431 def out_std(dest, digest, filename, digestname, binary, use_base64):
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
432 """Coreutils format (:command:`shasum` et al.)
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
433
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
434 """
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
435 if use_base64:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
436 digest = base64.b64encode(digest).decode("ascii")
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
437 else:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
438 digest = binascii.hexlify(digest).decode("ascii")
19
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
439 print("{} {}{}".format(
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
440 digest,
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
441 '*' if binary else ' ',
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
442 '-' if filename is None else util.normalize_filename(filename)),
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
443 file=dest)
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
444
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
445
89
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
446 def compute_digest_file(hashobj, path, use_mmap=None):
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
447 """
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
448 :param hashobj: a :mod:`hashlib` compatible hash algorithm type or factory
68
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
449 :param path: filename within the filesystem or a file descriptor opened in
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
450 binary mode (also a socket or pipe)
89
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
451 :param use_mmap: Use the :mod:`mmap` module if available.
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
452 If `None` determine automatically.
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
453 :type use_mmap: bool or None
56
6e91c530545f FIX: Docstring: Computes hashes are now returned as bytes from the low-level functions
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 52
diff changeset
454 :return: the digest in binary form
6e91c530545f FIX: Docstring: Computes hashes are now returned as bytes from the low-level functions
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 52
diff changeset
455 :rtype: bytes
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
456
68
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
457 If a file descriptor is given is must support :func:`os.read`.
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
458
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
459 """
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
460 h = hashobj()
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
461 if isinstance(path, constants.PATH_TYPES):
68
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
462 flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) \
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
463 | getattr(os, "O_SEQUENTIAL", 0) | getattr(os, "O_NOCTTY", 0)
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
464 fd = os.open(path, flags)
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
465 own_fd = True
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
466 else:
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
467 fd = path
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
468 own_fd = False
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
469 try:
68
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
470 try:
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
471 st = os.fstat(fd)
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
472 except TypeError:
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
473 #
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
474 # "fd" is most probably a Python socket object.
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
475 # (a pipe typically supports fstat)
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
476 #
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
477 use_mmap = False
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
478 else:
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
479 if stat.S_ISREG(st[stat.ST_MODE]):
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
480 filesize = st[stat.ST_SIZE]
89
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
481 if (use_mmap is None) \
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
482 and (filesize > constants.MAX_AUTO_MAP_SIZE):
89
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
483 #
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
484 # This is borrowed from FreeBSD's cp(1) implementation:
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
485 # Mmap and process if less than 8M (the limit is
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
486 # so we don't totally trash memory on big files.
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
487 # This is really a minor hack, but it wins some
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
488 # CPU back. Some filesystems, such as smbnetfs,
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
489 # don't support mmap, so this is a best-effort
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
490 # attempt.
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
491 #
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
492 use_mmap = False
68
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
493 else:
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
494 use_mmap = False
89
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
495 if use_mmap is None:
72684020f2f3 By default use mmap only for files up to 8MiB in size.
Franz Glasner <fzglas.hg@dom66.de>
parents: 88
diff changeset
496 use_mmap = True
67
19893b4f42a5 Flag to disable the use of mmap
Franz Glasner <fzglas.hg@dom66.de>
parents: 66
diff changeset
497 if mmap is None or not use_mmap:
84
163de6dd6e05 Enhance comment and -- while being there -- fix a typo
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 83
diff changeset
498 # No mmap available or wanted -> use traditional low-level file IO
94
8352da172a3e Use "fadvise()" with POSIX_FADV_SEQUENTIAL -- if available
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 90
diff changeset
499 fadvise = getattr(os, "posix_fadvise", None)
8352da172a3e Use "fadvise()" with POSIX_FADV_SEQUENTIAL -- if available
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 90
diff changeset
500 if fadvise:
8352da172a3e Use "fadvise()" with POSIX_FADV_SEQUENTIAL -- if available
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 90
diff changeset
501 fadvise(fd, 0, 0, os.POSIX_FADV_SEQUENTIAL)
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
502 if not constants.PY2:
96
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
503 fileobj = io.FileIO(fd, mode="r", closefd=False)
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
504 buf = bytearray(constants.READ_CHUNK_SIZE)
96
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
505 with memoryview(buf) as full_view:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
506 while True:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
507 try:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
508 n = fileobj.readinto(buf)
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
509 except OSError as e:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
510 if e.errno not in (errno.EAGAIN,
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
511 errno.EWOULDBLOCK,
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
512 errno.EINTR):
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
513 raise
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
514 else:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
515 if n == 0:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
516 break
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
517 if n == constants.READ_CHUNK_SIZE:
96
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
518 h.update(buf)
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
519 else:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
520 with full_view[:n] as partial_view:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
521 h.update(partial_view)
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
522 else:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
523 while True:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
524 try:
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
525 buf = os.read(fd, constants.READ_CHUNK_SIZE)
96
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
526 except OSError as e:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
527 if e.errno not in (errno.EAGAIN,
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
528 errno.EWOULDBLOCK,
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
529 errno.EINTR):
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
530 raise
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
531 else:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
532 if len(buf) == 0:
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
533 break
59253a1e6ef2 Employ .readinto() on Python 3
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 95
diff changeset
534 h.update(buf)
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
535 else:
58
ae5b31c10b41 Enhance comments
Franz Glasner <fzglas.hg@dom66.de>
parents: 57
diff changeset
536 #
ae5b31c10b41 Enhance comments
Franz Glasner <fzglas.hg@dom66.de>
parents: 57
diff changeset
537 # Use mmap
ae5b31c10b41 Enhance comments
Franz Glasner <fzglas.hg@dom66.de>
parents: 57
diff changeset
538 #
ae5b31c10b41 Enhance comments
Franz Glasner <fzglas.hg@dom66.de>
parents: 57
diff changeset
539 # NOTE: On Windows mmapped files with length 0 are not supported.
ae5b31c10b41 Enhance comments
Franz Glasner <fzglas.hg@dom66.de>
parents: 57
diff changeset
540 # So ensure to not call mmap.mmap() if the file size is 0.
ae5b31c10b41 Enhance comments
Franz Glasner <fzglas.hg@dom66.de>
parents: 57
diff changeset
541 #
61
c9f9401abc0c Use getattr when trying to get mmap.madvise()
Franz Glasner <fzglas.hg@dom66.de>
parents: 60
diff changeset
542 madvise = getattr(mmap.mmap, "madvise", None)
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
543 if filesize <= constants.MAP_WINDOW_SIZE:
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
544 mapsize = filesize
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
545 else:
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
546 mapsize = constants.MAP_WINDOW_SIZE
57
0fa2067bedb8 Common initialization is now common when computing the hashes from a mmap
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 56
diff changeset
547 mapoffset = 0
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
548 rest = filesize
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
549 while rest > 0:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
550 m = mmap.mmap(fd,
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
551 mapsize,
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
552 access=mmap.ACCESS_READ,
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
553 offset=mapoffset)
60
21d2589c96b9 Use madvise if available to sequentially read a file
Franz Glasner <fzglas.hg@dom66.de>
parents: 59
diff changeset
554 if madvise:
21d2589c96b9 Use madvise if available to sequentially read a file
Franz Glasner <fzglas.hg@dom66.de>
parents: 59
diff changeset
555 madvise(m, mmap.MADV_SEQUENTIAL)
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
556 try:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
557 h.update(m)
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
558 finally:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
559 m.close()
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
560 rest -= mapsize
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
561 mapoffset += mapsize
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
562 if rest < mapsize:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
563 mapsize = rest
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
564 finally:
68
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
565 if own_fd:
4c2da9c74d7c "compute_digest_file()" now also accepts an already opened file descriptor.
Franz Glasner <fzglas.hg@dom66.de>
parents: 67
diff changeset
566 os.close(fd)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
567 return h.digest()
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
568
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
569
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
570 def compute_digest_stream(hashobj, instream):
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
571 """
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
572
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
573 :param hashobj: a :mod:`hashlib` compatible hash algorithm type or factory
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
574 :param instream: a bytes input stream to read the data to be hashed from
56
6e91c530545f FIX: Docstring: Computes hashes are now returned as bytes from the low-level functions
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 52
diff changeset
575 :return: the digest in binary form
6e91c530545f FIX: Docstring: Computes hashes are now returned as bytes from the low-level functions
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 52
diff changeset
576 :rtype: bytes
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
577
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
578 """
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
579 h = hashobj()
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
580 while True:
66
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
581 try:
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
582 buf = instream.read(constants.READ_CHUNK_SIZE)
66
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
583 except OSError as e:
79
8708c34e2723 Handle EINTR also in combination with EAGAIN and EWOULDBLOCK
Franz Glasner <fzglas.hg@dom66.de>
parents: 75
diff changeset
584 if e.errno not in (errno.EAGAIN, errno.EWOULDBLOCK, errno.EINTR):
66
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
585 raise
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
586 else:
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
587 if buf is not None:
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
588 if len(buf) == 0:
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
589 break
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
590 h.update(buf)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
591 return h.digest()
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
592
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
593
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
594 if __name__ == "__main__":
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
595 sys.exit(main())