Mercurial > hgrepos > Python > apps > py-cutils

annotate cutils/shasum.py @ 157:27d1aaf5fe39

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Implement "--mode" flag for "treesum.py" to consider file portable mode bits
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 06 Jan 2025 21:03:10 +0100
parents 1e5127028254
children a813094ae4f5
Ignore whitespace changes - Everywhere: Within whitespace: At end of lines:
rev   line source
73
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
1 # -*- coding: utf-8 -*-
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
2 # :-
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
3 # :Copyright: (c) 2020-2025 Franz Glasner
73
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
4 # :License: BSD-3-Clause
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
5 # :-
113
Franz Glasner <fzglas.hg@dom66.de>
parents: 110
diff changeset
6 r"""Pure Python implementation of `shasum`.
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
7
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
8 """
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
9
72
ae2df602beb4 Make shasum.py and dos2unix sub-modules to the new "cutils" package
Franz Glasner <fzglas.hg@dom66.de>
parents: 71
diff changeset
10 from __future__ import print_function, absolute_import
13
db64e282b049 Implement a version option
Franz Glasner <fzglas.hg@dom66.de>
parents: 12
diff changeset
11
73
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
12
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
13 __all__ = []
c3268f4e752f Adjust all license notes to (a) more literally comply with the BSD3 templates and to the style guide
Franz Glasner <fzglas.hg@dom66.de>
parents: 72
diff changeset
14
13
db64e282b049 Implement a version option
Franz Glasner <fzglas.hg@dom66.de>
parents: 12
diff changeset
15
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
16 import argparse
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
17 import base64
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
18 import binascii
66
c52e5f86b0ab Handle EAGAIN and EWOULDBLOCK when reading files
Franz Glasner <fzglas.hg@dom66.de>
parents: 61
diff changeset
19 import errno
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
20 import io
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
21 import os
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
22 import re
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
23 import sys
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
24
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
25 from . import (__version__, __revision__)
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
26 from . import util
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
27 from .util import constants
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
28 from .util import digest
95
fc2dd6afd594 Style: canonical global variable and import ordering
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 94
diff changeset
29
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
30
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
31 def main(argv=None):
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
32 aparser = argparse.ArgumentParser(
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
33 description="Python implementation of shasum",
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
34 fromfile_prefix_chars='@')
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
35 aparser.add_argument(
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
36 "--algorithm", "-a", action="store", type=util.argv2algo,
120
a548783381b6 More accurate description of --algorithm.
Franz Glasner <fzglas.hg@dom66.de>
parents: 119
diff changeset
37 help="1 (default, aka sha1), 224, 256, 384, 512, 3 (alias for sha3-512), 3-224, 3-256, 3-384, 3-512, blake2b, blake2b-256, blake2s, blake2 (alias for blake2b), blake2-256 (alias for blake2b-256), md5")
7
47b4c98e4d40 Allow algorithm selection for all algorithms in :mod:`hashlib`.
Franz Glasner <fzglas.hg@dom66.de>
parents: 6
diff changeset
38 aparser.add_argument(
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
39 "--base64", action="store_true",
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
40 help="Output checksums in base64 notation, not hexadecimal (OpenBSD).")
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
41 aparser.add_argument(
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
42 "--binary", "-b", action="store_false", dest="text_mode", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
43 help="Read in binary mode (default)")
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
44 aparser.add_argument(
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
45 "--bsd", "-B", action="store_true", dest="bsd", default=False,
17
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
46 help="Write BSD style output. This is also the default output format of :command:`openssl dgst`.")
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
47 aparser.add_argument(
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
48 "--check", "-c", action="store_true",
17
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
49 help="""Read digests from FILEs and check them.
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
50 If this option is specified, the FILE options become checklists. Each
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
51 checklist should contain hash results in a supported format, which will
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
52 be verified against the specified paths. Output consists of the digest
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
53 used, the file name, and an OK, FAILED, or MISSING for the result of
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
54 the comparison. This will validate any of the supported checksums.
184ab1da1307 Extend help messages
Franz Glasner <fzglas.hg@dom66.de>
parents: 14
diff changeset
55 If no file is given, stdin is used.""")
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
56 aparser.add_argument(
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
57 "--checklist", "-C", metavar="CHECKLIST",
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
58 help="""Compare the checksum of each FILE against the checksums in
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
59 the CHECKLIST. Any specified FILE that is not listed in the CHECKLIST will
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
60 generate an error.""")
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
61 aparser.add_argument(
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
62 "--checklist-allow-distinfo", action="store_true",
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
63 dest="allow_distinfo",
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
64 help='Allow FreeBSD "distinfo" formatted checklists: ignore SIZE and TIMESTAMP lines.')
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
65
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
66 aparser.add_argument(
105
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
67 "--follow-symlinks", action="store_true", dest="follow_symlinks",
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
68 help="""Also follow symlinks that resolve to directories. Only effective if `--recurse` is activated.""")
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
69
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
70 aparser.add_argument(
90
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
71 "--mmap", action="store_true", dest="mmap", default=None,
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
72 help="""Use mmap if available. Default is to determine automatically
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
73 from the filesize.""")
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
74 aparser.add_argument(
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
75 "--no-mmap", action="store_false", dest="mmap", default=None,
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
76 help="Dont use mmap.")
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
77
42419f57eda9 Allow to control the use of mmap from the command-line
Franz Glasner <fzglas.hg@dom66.de>
parents: 89
diff changeset
78 aparser.add_argument(
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
79 "--recurse", action="store_true",
105
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
80 help="Recurse into sub-directories while interpreting every FILE as a directory.")
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
81
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
82 aparser.add_argument(
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
83 "--reverse", "-r", action="store_false", dest="bsd", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
84 help="Explicitely select normal coreutils style output (to be option compatible with BSD style commands and :command:`openssl dgst -r`)")
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
85 aparser.add_argument(
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
86 "--tag", action="store_true", dest="bsd", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
87 help="Alias for the `--bsd' option (to be compatible with :command:`b2sum`)")
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
88 aparser.add_argument(
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
89 "--text", "-t", action="store_true", dest="text_mode", default=False,
69
a23371a8780f Writing style: Begin all help messages with an uppercase letter
Franz Glasner <fzglas.hg@dom66.de>
parents: 68
diff changeset
90 help="Read in text mode (not supported)")
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
91 aparser.add_argument(
27
c29bd96dcd4f Put the HG revision into the version output also
Franz Glasner <fzglas.hg@dom66.de>
parents: 26
diff changeset
92 "--version", "-v", action="version", version="%s (rv:%s)" % (__version__, __revision__))
13
db64e282b049 Implement a version option
Franz Glasner <fzglas.hg@dom66.de>
parents: 12
diff changeset
93 aparser.add_argument(
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
94 "files", nargs="*", metavar="FILE")
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
95
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
96 opts = aparser.parse_args(args=argv)
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
97
3
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
98 if opts.text_mode:
5a6ed622846c Add comman line switches for reading in binary and text mode
Franz Glasner <fzglas.hg@dom66.de>
parents: 2
diff changeset
99 print("ERROR: text mode not supported", file=sys.stderr)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
100 sys.exit(78) # :manpage:`sysexits(3)` EX_CONFIG
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
101
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
102 if opts.check and opts.checklist:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
103 print("ERROR: only one of --check or --checklist allowed",
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
104 file=sys.stderr)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
105 sys.exit(64) # :manpage:`sysexits(3)` EX_USAGE
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
106
7
47b4c98e4d40 Allow algorithm selection for all algorithms in :mod:`hashlib`.
Franz Glasner <fzglas.hg@dom66.de>
parents: 6
diff changeset
107 if not opts.algorithm:
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
108 opts.algorithm = util.argv2algo("1")
7
47b4c98e4d40 Allow algorithm selection for all algorithms in :mod:`hashlib`.
Franz Glasner <fzglas.hg@dom66.de>
parents: 6
diff changeset
109
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
110 opts.dest = None
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
111
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
112 return shasum(opts)
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
113
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
114
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
115 def gen_opts(files=[], algorithm="SHA1", bsd=False, text_mode=False,
100
f95918115c6b FIX: Implement some new commandline flags in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 96
diff changeset
116 checklist=False, check=False, dest=None, base64=False,
106
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
117 allow_distinfo=False, mmap=None, recurse=False,
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
118 follow_symlinks=False):
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
119 if text_mode:
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
120 raise ValueError("text mode not supported")
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
121 if checklist and check:
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
122 raise ValueError("only one of `checklist' or `check' is allowed")
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
123 opts = argparse.Namespace(files=files,
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
124 algorithm=(util.algotag2algotype(algorithm),
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
125 algorithm),
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
126 bsd=bsd,
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
127 checklist=checklist,
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
128 check=check,
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
129 text_mode=False,
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
130 dest=dest,
100
f95918115c6b FIX: Implement some new commandline flags in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 96
diff changeset
131 base64=base64,
f95918115c6b FIX: Implement some new commandline flags in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 96
diff changeset
132 allow_distinfo=allow_distinfo,
106
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
133 mmap=mmap,
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
134 recurse=recurse,
5fe6f63f0be7 Implement "--recurse" and "--follow-symlinks" in "gen_opts()" also
Franz Glasner <fzglas.hg@dom66.de>
parents: 105
diff changeset
135 follow_symlinks=follow_symlinks)
45
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
136 return opts
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
137
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
138
b25ef7293bf2 Enhance shasum.py to allow it to be used as Python module from within other programs more easily
Franz Glasner <fzglas.hg@dom66.de>
parents: 43
diff changeset
139 def shasum(opts):
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
140 if opts.check:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
141 return verify_digests_from_files(opts)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
142 elif opts.checklist:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
143 return verify_digests_with_checklist(opts)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
144 else:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
145 return generate_digests(opts)
10
77446cd3ea6f Move the digest generation loop from "main()" into an own function "generate_digests()".
Franz Glasner <fzglas.hg@dom66.de>
parents: 9
diff changeset
146
77446cd3ea6f Move the digest generation loop from "main()" into an own function "generate_digests()".
Franz Glasner <fzglas.hg@dom66.de>
parents: 9
diff changeset
147
77446cd3ea6f Move the digest generation loop from "main()" into an own function "generate_digests()".
Franz Glasner <fzglas.hg@dom66.de>
parents: 9
diff changeset
148 def generate_digests(opts):
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
149 if opts.bsd:
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
150 out = out_bsd
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
151 else:
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
152 out = out_std
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
153 if opts.recurse:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
154 if not opts.files:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
155 opts.files.append(".")
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
156 for dn in opts.files:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
157 if not os.path.isdir(dn):
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
158 if os.path.exists(dn):
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
159 raise OSError(errno.ENOTDIR, "not a directory", dn)
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
160 else:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
161 raise OSError(errno.ENOENT, "directory does not exist", dn)
105
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
162 for dirpath, dirnames, dirfiles in os.walk(
b0631f320efd Implement "--follow-symlinks" to allow to control whether to follow directory symlinks
Franz Glasner <fzglas.hg@dom66.de>
parents: 104
diff changeset
163 dn, followlinks=opts.follow_symlinks):
110
3060aa4cb252 When generating digests recursively normalize the output somewhat by sorting directory names and filenames
Franz Glasner <fzglas.hg@dom66.de>
parents: 106
diff changeset
164 dirnames.sort()
3060aa4cb252 When generating digests recursively normalize the output somewhat by sorting directory names and filenames
Franz Glasner <fzglas.hg@dom66.de>
parents: 106
diff changeset
165 dirfiles.sort()
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
166 for fn in dirfiles:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
167 path = os.path.join(dirpath, fn)
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
168 out(opts.dest or sys.stdout,
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
169 digest.compute_digest_file(
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
170 opts.algorithm[0], path, use_mmap=opts.mmap),
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
171 path,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
172 opts.algorithm[1],
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
173 True,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
174 opts.base64)
4
67d10529ce88 FIX: "-" filename handling now consistent with Perl shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 3
diff changeset
175 else:
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
176 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
177 if constants.PY2:
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
178 if sys.platform == "win32":
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
179 import msvcrt # noqa: E401
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
180 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
181 source = sys.stdin
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
182 else:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
183 source = sys.stdin.buffer
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
184 out(sys.stdout,
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
185 digest.compute_digest_stream(opts.algorithm[0], source),
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
186 None,
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
187 opts.algorithm[1],
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
188 True,
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
189 opts.base64)
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
190 else:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
191 for fn in opts.files:
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
192 out(opts.dest or sys.stdout,
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
193 digest.compute_digest_file(
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
194 opts.algorithm[0], fn, use_mmap=opts.mmap),
104
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
195 fn,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
196 opts.algorithm[1],
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
197 True,
08fd0609fdd4 Implement "--recurse" for shasum: recurse into directories
Franz Glasner <fzglas.hg@dom66.de>
parents: 100
diff changeset
198 opts.base64)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
199 return 0
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
200
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
201
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
202 def compare_digests_equal(given_digest, expected_digest, algo):
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
203 """Compare a newly computed binary digest `given_digest` with a digest
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
204 string (hex or base64) in `expected_digest`.
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
205
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
206 :param bytes given_digest:
71
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
207 :param expected_digest: digest (as bytes) or hexlified or base64 encoded
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
208 digest (as str)
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
209 :type expected_digest: str or bytes or bytearray
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
210 :param algo: The algorithm (factory)
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
211 :return: `True` if the digests are equal, `False` if not
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
212 :rtype: bool
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
213
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
214 """
71
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
215 if isinstance(expected_digest, (bytes, bytearray)) \
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
216 and len(expected_digest) == algo().digest_size:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
217 exd = expected_digest
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
218 else:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
219 if len(expected_digest) == algo().digest_size * 2:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
220 # hex
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
221 if re.search(r"\A[a-fA-F0-9]+\Z", expected_digest):
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
222 try:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
223 exd = binascii.unhexlify(expected_digest)
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
224 except TypeError:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
225 return False
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
226 else:
52
5935055edea6 More proper formal checks (with regexp) for valid hex and base64 encoding of digests
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 51
diff changeset
227 return False
5935055edea6 More proper formal checks (with regexp) for valid hex and base64 encoding of digests
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 51
diff changeset
228 else:
71
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
229 # base64
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
230 if re.search(
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
231 r"\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?\Z",
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
232 expected_digest):
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
233 try:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
234 exd = base64.b64decode(expected_digest)
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
235 except TypeError:
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
236 return False
29fb33aa639a "compare_digests_equal()" now accepts binary (aka un-encoded) expected digests also
Franz Glasner <fzglas.hg@dom66.de>
parents: 70
diff changeset
237 else:
52
5935055edea6 More proper formal checks (with regexp) for valid hex and base64 encoding of digests
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 51
diff changeset
238 return False
75
a31de3c65877 Remove the use of "hmac.compare_digest()": there are no secrets to protect here
Franz Glasner <fzglas.hg@dom66.de>
parents: 73
diff changeset
239 return given_digest == exd
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
240
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
241
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
242 def verify_digests_with_checklist(opts):
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
243 dest = opts.dest or sys.stdout
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
244 exit_code = 0
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
245 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
117
e51f34ad6d71 Move constant definitions into new module cutils.util.constants
Franz Glasner <fzglas.hg@dom66.de>
parents: 113
diff changeset
246 if constants.PY2:
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
247 if sys.platform == "win32":
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
248 import os, msvcrt # noqa: E401
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
249 msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
250 source = sys.stdin
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
251 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
252 source = sys.stdin.buffer
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
253 pl = get_parsed_digest_line_from_checklist(opts.checklist, opts, None)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
254 if pl is None:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
255 exit_code = 1
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
256 print("-: MISSING", file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
257 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
258 tag, algo, cl_filename, cl_digest = pl
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
259 computed_digest = digest.compute_digest_stream(algo, source)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
260 if compare_digests_equal(computed_digest, cl_digest, algo):
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
261 res = "OK"
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
262 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
263 res = "FAILED"
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
264 exit_code = 1
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
265 print("{}: {}: {}".format(tag, "-", res), file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
266 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
267 for fn in opts.files:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
268 pl = get_parsed_digest_line_from_checklist(opts.checklist, opts, fn)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
269 if pl is None:
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
270 print("{}: MISSING".format(fn), file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
271 exit_code = 1
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
272 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
273 tag, algo, cl_filename, cl_digest = pl
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
274 computed_digest = digest.compute_digest_file(
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
275 algo, fn, use_mmap=opts.mmap)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
276 if compare_digests_equal(computed_digest, cl_digest, algo):
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
277 res = "OK"
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
278 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
279 exit_code = 1
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
280 res = "FAILED"
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
281 print("{}: {}: {}".format(tag, fn, res), file=dest)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
282 return exit_code
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
283
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
284
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
285 def verify_digests_from_files(opts):
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
286 dest = opts.dest or sys.stdout
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
287 exit_code = 0
20
8f0241ed4a00 Do not append "-" to an empty FILE list any more but check explicitely for an empty list
Franz Glasner <fzglas.hg@dom66.de>
parents: 19
diff changeset
288 if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'):
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
289 for checkline in sys.stdin:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
290 if not checkline:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
291 continue
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
292 r, fn, tag = handle_checkline(opts, checkline)
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
293 if tag in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
294 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
295 continue
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
296 print("{}: {}: {}".format(tag, fn, r.upper()), file=dest)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
297 if r != "ok" and exit_code == 0:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
298 exit_code = 1
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
299 else:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
300 for fn in opts.files:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
301 with io.open(fn, "rt", encoding="utf-8") as checkfile:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
302 for checkline in checkfile:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
303 if not checkline:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
304 continue
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
305 r, fn, tag = handle_checkline(opts, checkline)
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
306 if tag in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
307 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
308 continue
47
5bec7a5d894a Allow internal output redirection: print() always to explicitely given file objects
Franz Glasner <fzglas.hg@dom66.de>
parents: 45
diff changeset
309 print("{}: {}: {}".format(tag, fn, r.upper()), file=dest)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
310 if r != "ok" and exit_code == 0:
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
311 exit_code = 1
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
312 return exit_code
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
313
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
314
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
315 def handle_checkline(opts, line):
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
316 """
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
317 :return: a tuple with static "ok", "missing", or "failed", the filename and
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
318 the digest used
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
319 :rtype: tuple(str, str, str)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
320
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
321 """
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
322 parts = parse_digest_line(opts, line)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
323 if not parts:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
324 raise ValueError(
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
325 "improperly formatted digest line: {}".format(line))
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
326 tag, algo, fn, digest = parts
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
327 if tag in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
328 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
329 return (None, None, tag)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
330 try:
122
1e5127028254 Move the real computation of digests from files and streams into dedicated submodule cutils.util.digest
Franz Glasner <fzglas.hg@dom66.de>
parents: 120
diff changeset
331 d = digest.compute_digest_file(algo, fn, use_mmap=opts.mmap)
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
332 if compare_digests_equal(d, digest, algo):
23
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
333 return ("ok", fn, tag)
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
334 else:
232063b73e45 Optimized reading of files by using mmap.
Franz Glasner <fzglas.hg@dom66.de>
parents: 22
diff changeset
335 return ("failed", fn, tag)
12
5e2c9123f93f Implemented digest verification: -c or --check option
Franz Glasner <fzglas.hg@dom66.de>
parents: 11
diff changeset
336 except EnvironmentError:
18
285848db0b52 When verifying/checking digests: also print the digest tag used
Franz Glasner <fzglas.hg@dom66.de>
parents: 17
diff changeset
337 return ("missing", fn, tag)
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
338
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
339
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
340 def get_parsed_digest_line_from_checklist(checklist, opts, filename):
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
341 if filename is None:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
342 filenames = ("-", "stdin", "", )
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
343 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
344 filenames = (
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
345 util.normalize_filename(filename, strip_leading_dot_slash=True),)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
346 with io.open(checklist, "rt", encoding="utf-8") as clf:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
347 for checkline in clf:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
348 if not checkline:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
349 continue
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
350 parts = parse_digest_line(opts, checkline)
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
351 if not parts:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
352 raise ValueError(
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
353 "improperly formatted digest line: {}".format(checkline))
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
354 if parts[0] in ("SIZE", "TIMESTAMP"):
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
355 assert opts.allow_distinfo
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
356 continue
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
357 fn = util.normalize_filename(parts[2], strip_leading_dot_slash=True)
22
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
358 if fn in filenames:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
359 return parts
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
360 else:
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
361 return None
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
362
6bdfc5ad4656 Implemented OpenBSD's -C (aka --checklist) option for shasum
Franz Glasner <fzglas.hg@dom66.de>
parents: 21
diff changeset
363
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
364 def parse_digest_line(opts, line):
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
365 """Parse a `line` of a digest file and return its parts.
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
366
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
367 This is rather strict. But if `opts.allow_distinfo` is `True` then
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
368 some additional keywords ``SIZE`` and ``TIMESTAMP``are recignized
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
369 and returned. The caller is responsible to handle them.
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
370
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
371 :return: a tuple of the normalized algorithm tag, the algorithm
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
372 constructor, the filename and the hex digest;
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
373 if `line` cannot be parsed successfully `None` is returned
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
374 :rtype: tuple(str, obj, str, str) or None
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
375
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
376 Handles coreutils and BSD-style file formats.
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
377
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
378 """
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
379 # determine checkfile format (BSD or coreutils)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
380 # BSD?
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
381 mo = re.search(r"\A(\S+)\s*\((.*)\)\s*=\s*(.+)\n?\Z", line)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
382 if mo:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
383 # (tag, algorithm, filename, digest)
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
384 if opts.allow_distinfo:
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
385 if mo.group(1) == "SIZE":
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
386 return ("SIZE", None, None, mo.group(3))
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
387 return (mo.group(1),
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
388 util.algotag2algotype(mo.group(1)),
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
389 mo.group(2),
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
390 mo.group(3))
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
391 else:
83
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
392 if opts.allow_distinfo:
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
393 mo = re.search(r"\ATIMESTAMP\s*=\s*([0-9]+)\s*\n\Z", line)
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
394 if mo:
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
395 return ("TIMESTAMP", None, None, mo.group(1))
05e2bf4796fd Add an option "--checklist-allow-distinfo" to allow FreeBSD "distinfo" formatted files as checkfiles.
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 79
diff changeset
396
21
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
397 # coreutils?
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
398 mo = re.search(r"([^\ ]+) [\*\ ]?(.+)\n?\Z", line)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
399 if mo:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
400 # (tag, algorithm, filename, digest)
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
401 return (opts.algorithm[1],
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
402 opts.algorithm[0],
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
403 mo.group(2),
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
404 mo.group(1))
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
405 else:
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
406 return None
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
407
f2d634270e1c Refactor: parse a line of a digest file within a dedicated funcion
Franz Glasner <fzglas.hg@dom66.de>
parents: 20
diff changeset
408
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
409 def out_bsd(dest, digest, filename, digestname, binary, use_base64):
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
410 """BSD format output, also :command:`openssl dgst` and
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
411 :command:`b2sum --tag" format output
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
412
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
413 """
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
414 if use_base64:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
415 digest = base64.b64encode(digest).decode("ascii")
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
416 else:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
417 digest = binascii.hexlify(digest).decode("ascii")
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
418 if filename is None:
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
419 print(digest, file=dest)
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
420 else:
19
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
421 print("{} ({}) = {}".format(digestname,
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
422 util.normalize_filename(filename),
19
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
423 digest),
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
424 file=dest)
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
425
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
426
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
427 def out_std(dest, digest, filename, digestname, binary, use_base64):
9
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
428 """Coreutils format (:command:`shasum` et al.)
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
429
81f28bf89c26 Some more output selection options and documentation
Franz Glasner <fzglas.hg@dom66.de>
parents: 8
diff changeset
430 """
51
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
431 if use_base64:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
432 digest = base64.b64encode(digest).decode("ascii")
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
433 else:
58d5a0b6e5b3 Implement the OpenBSD variant (with --base64) to encode digests in base64, not hexadecimal
Franz Glasner <f.glasner@feldmann-mg.com>
parents: 47
diff changeset
434 digest = binascii.hexlify(digest).decode("ascii")
19
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
435 print("{} {}{}".format(
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
436 digest,
2f9e702e3f7a Convert backslashes in filenames to forward slashes when creating digests
Franz Glasner <fzglas.hg@dom66.de>
parents: 18
diff changeset
437 '*' if binary else ' ',
118
12339ac2148d Move some functions into cutils.util (i.e. algorithms and their aliases)
Franz Glasner <fzglas.hg@dom66.de>
parents: 117
diff changeset
438 '-' if filename is None else util.normalize_filename(filename)),
5
bbcb225640de Handle standard and BSD-style output formats
Franz Glasner <fzglas.hg@dom66.de>
parents: 4
diff changeset
439 file=dest)
2
5510a39a2d04 Basic hashing with proper binary stdin/stdout support for Py2, Py3 and Windows
Franz Glasner <fzglas.hg@dom66.de>
parents: 1
diff changeset
440
1
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
441
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
442 if __name__ == "__main__":
bbf4e0f5b651 Begin the shasum.py script
Franz Glasner <fzglas.hg@dom66.de>
parents:
diff changeset
443 sys.exit(main())