Mercurial > hgrepos > Python > apps > py-cutils

comparison cutils/treesum.py @ 210:1be3af138183

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Refactor option handling for configuring symlink handling: now all variations are supported
author Franz Glasner <fzglas.hg@dom66.de>
date Thu, 23 Jan 2025 09:16:51 +0100
parents 85e7edea8ac7
children 5bb0b25f8e99
comparison
equal deleted inserted replaced
209:efbf99bd0910 210:1be3af138183
14 14
15 15
16 import argparse 16 import argparse
17 import base64 17 import base64
18 import binascii 18 import binascii
19 import collections
19 import datetime 20 import datetime
20 import logging 21 import logging
21 import os 22 import os
22 import re 23 import re
23 import stat 24 import stat
61 gp.add_argument( 62 gp.add_argument(
62 "--comment", action="append", default=[], 63 "--comment", action="append", default=[],
63 help="Put given comment COMMENT into the output as \"COMMENT\". " 64 help="Put given comment COMMENT into the output as \"COMMENT\". "
64 "Can be given more than once.") 65 "Can be given more than once.")
65 gp.add_argument( 66 gp.add_argument(
66 "--follow-directory-symlinks", "-l", action="store_true", 67 "--follow-directory-symlinks", "-l", action=SymlinkAction,
67 dest="follow_directory_symlinks", 68 const="follow-directory-symlinks",
68 help="Follow symbolic links to directories when walking a " 69 default=FollowSymlinkConfig(False, False, True),
69 "directory tree. Note that this is different from using " 70 dest="follow_symlinks",
70 "\"--logical\" or \"--physical\" for arguments given " 71 help="""Follow symbolic links to directories when walking a
71 "directly on the command line") 72 directory tree. Augments --physical.""")
73 gp.add_argument(
74 "--follow-file-symlinks", action=SymlinkAction,
75 const="follow-file-symlinks",
76 default=FollowSymlinkConfig(False, False, True),
77 dest="follow_symlinks",
78 help="""Follow symbolic links to files when walking a
79 directory tree. Augments --physical.""")
72 gp.add_argument( 80 gp.add_argument(
73 "--full-mode", action="store_true", dest="metadata_full_mode", 81 "--full-mode", action="store_true", dest="metadata_full_mode",
74 help="Consider all mode bits as returned from stat(2) when " 82 help="Consider all mode bits as returned from stat(2) when "
75 "computing directory digests. " 83 "computing directory digests. "
76 "Note that mode bits on symbolic links itself are not " 84 "Note that mode bits on symbolic links itself are not "
81 help="""Put a `GENERATOR' line into the output. 89 help="""Put a `GENERATOR' line into the output.
82 `full' prints full Python and OS/platform version information, 90 `full' prints full Python and OS/platform version information,
83 `normal' prints just whether Python 2 or Python 3 is used, and `none' 91 `normal' prints just whether Python 2 or Python 3 is used, and `none'
84 suppresses the output completely. The default is `normal'.""") 92 suppresses the output completely. The default is `normal'.""")
85 gp.add_argument( 93 gp.add_argument(
86 "--logical", "-L", dest="logical", action="store_true", 94 "--logical", "-L", action=SymlinkAction, dest="follow_symlinks",
87 default=None, 95 const=FollowSymlinkConfig(True, True, True),
88 help="Follow symbolic links given on command line arguments." 96 help="""Follow symbolic links everywhere: on command line
89 " Note that this is a different setting as to follow symbolic" 97 arguments and -- while walking -- directory and file symbolic links.
90 " links to directories when traversing a directory tree.") 98 Overwrites any other symlink related options
99 (--physical, no-follow-directory-symlinks, no-follow-file-symlinks, et al.).
100 """)
91 gp.add_argument( 101 gp.add_argument(
92 "--minimal", nargs="?", const="", default=None, metavar="TAG", 102 "--minimal", nargs="?", const="", default=None, metavar="TAG",
93 help="Produce minimal output only. If a TAG is given and not " 103 help="Produce minimal output only. If a TAG is given and not "
94 "empty use it as the leading \"ROOT (<TAG>)\" output.") 104 "empty use it as the leading \"ROOT (<TAG>)\" output.")
95 gp.add_argument( 105 gp.add_argument(
106 "--mtime", action="store_true", dest="metadata_mtime", 116 "--mtime", action="store_true", dest="metadata_mtime",
107 help="Consider the mtime of files (non-directories) when " 117 help="Consider the mtime of files (non-directories) when "
108 "generating digests for directories. Digests for files are " 118 "generating digests for directories. Digests for files are "
109 "not affected.") 119 "not affected.")
110 gp.add_argument( 120 gp.add_argument(
121 "--no-follow-directory-symlinks", action=SymlinkAction,
122 const="no-follow-directory-symlinks",
123 dest="follow_symlinks",
124 help="""Do not follow symbolic links to directories when walking a
125 directory tree. Augments --logical.""")
126 gp.add_argument(
127 "--no-follow-file-symlinks", action=SymlinkAction,
128 const="no-follow-file-symlinks",
129 dest="follow_symlinks",
130 help="""Dont follow symbolic links to files when walking a
131 directory tree. Augments --logical.""")
132 gp.add_argument(
111 "--no-mmap", action="store_false", dest="mmap", default=None, 133 "--no-mmap", action="store_false", dest="mmap", default=None,
112 help="Dont use mmap.") 134 help="Dont use mmap.")
113 gp.add_argument( 135 gp.add_argument(
114 "--output", "-o", action="store", metavar="OUTPUT", 136 "--output", "-o", action="store", metavar="OUTPUT",
115 help="Put the checksum into given file. " 137 help="Put the checksum into given file. "
116 "If not given or if it is given as `-' then stdout is used.") 138 "If not given or if it is given as `-' then stdout is used.")
117 gp.add_argument( 139 gp.add_argument(
118 "--physical", "-P", dest="logical", action="store_false", 140 "--physical", "-P", action=SymlinkAction, dest="follow_symlinks",
119 default=None, 141 const=FollowSymlinkConfig(False, False, False),
120 help="Do not follow symbolic links given on comment line " 142 help="""Do not follow any symbolic links whether they are given
121 "arguments. This is the default.") 143 on the command line or when walking the directory tree.
144 Overwrites any other symlink related options
145 (--logical, follow-directory-symlinks, follow-file-symlinks, et al.).
146 This is the default.""")
122 gp.add_argument( 147 gp.add_argument(
123 "--print-size", action="store_true", 148 "--print-size", action="store_true",
124 help="""Print the size of a file or the accumulated sizes of 149 help="""Print the size of a file or the accumulated sizes of
125 directory content into the output also. 150 directory content into the output also.
126 The size is not considered when computing digests. For symbolic links 151 The size is not considered when computing digests. For symbolic links
247 logging.captureWarnings(True) 272 logging.captureWarnings(True)
248 273
249 return treesum(opts) 274 return treesum(opts)
250 275
251 276
277 FollowSymlinkConfig = collections.namedtuple("FollowSymlinkConfig",
278 ["command_line",
279 "directory",
280 "file"])
281
282
283 class SymlinkAction(argparse.Action):
284
285 """`type' is fixed here.
286 `dest' is a tuple with three items:
287
288 1. follow symlinks on the command line
289 2. follow directory symlinks while walking
290 3. follow file symlinks while walking (not yet implemented)
291
292 """
293
294 def __init__(self, *args, **kwargs):
295 if "nargs" in kwargs:
296 raise ValueError("`nargs' not allowed")
297 if "type" in kwargs:
298 raise ValueError("`type' not allowed")
299 c = kwargs.get("const", None)
300 if c is None:
301 raise ValueError("a const value is needed")
302 if (not isinstance(c, FollowSymlinkConfig)
303 and c not in ("follow-directory-symlinks",
304 "no-follow-directory-symlinks",
305 "follow-file-symlinks",
306 "no-follow-file-symlinks")):
307 raise ValueError(
308 "invalid value for the `const' configuration value")
309 default = kwargs.get("default", None)
310 if (default is not None
311 and not isinstance(default, FollowSymlinkConfig)):
312 raise TypeError("invalid type for `default'")
313 kwargs["nargs"] = 0
314 super(SymlinkAction, self).__init__(*args, **kwargs)
315
316 def __call__(self, parser, namespace, values, option_string=None):
317 curval = getattr(namespace, self.dest, None)
318 if curval is None:
319 curval = FollowSymlinkConfig(False, False, True)
320 if isinstance(self.const, FollowSymlinkConfig):
321 curval = self.const
322 else:
323 if self.const == "follow-directory-symlinks":
324 curval = FollowSymlinkConfig(
325 curval.command_line, True, curval.file)
326 elif self.const == "no-follow-directory-symlinks":
327 curval = FollowSymlinkConfig(
328 curval.command_line, False, curval.file)
329 elif self.const == "follow-file-symlinks":
330 curval = FollowSymlinkConfig(
331 curval.command_line, curval.directory, True)
332 elif self.const == "no-follow-file-symlinks":
333 curval = FollowSymlinkConfig(
334 curval.command_line, curval.directory, False)
335 else:
336 assert False, "Implementation error: not yet implemented"
337
338 # Not following symlinks to files is not yet supported: reset to True
339 if not curval.file:
340 curval = FollowSymlinkConfig(
341 curval.command_line, curval.directory, True)
342 logging.warning("Coercing options to `follow-file-symlinks'")
343 setattr(namespace, self.dest, curval)
344
345
252 def gen_generate_opts(directories=[], 346 def gen_generate_opts(directories=[],
253 algorithm=util.default_algotag(), 347 algorithm=util.default_algotag(),
254 append_output=False, 348 append_output=False,
255 base64=False, 349 base64=False,
256 comment=[], 350 comment=[],
257 follow_directory_symlinks=False, 351 follow_symlinks=FollowSymlinkConfig(False, False, False),
258 full_mode=False, 352 full_mode=False,
259 generator="normal", 353 generator="normal",
260 logical=None, 354 logical=None,
261 minimal=None, 355 minimal=None,
262 mode=False, 356 mode=False,
264 mtime=False, 358 mtime=False,
265 output=None, 359 output=None,
266 print_size=False, 360 print_size=False,
267 size_only=False, 361 size_only=False,
268 utf8=False): 362 utf8=False):
363 # Not following symlinks to files is not yet supported: reset to True
364 if not isinstance(follow_symlinks, FollowSymlinkConfig):
365 raise TypeError("`follow_symlinks' must be a FollowSymlinkConfig")
366 if not follow_symlinks.file:
367 follow_symlinks = follow_symlinks._make([follow_symlinks.command_line,
368 follow_symlinks.directory,
369 True])
370 logging.warning("Coercing to follow-symlinks-file")
269 opts = argparse.Namespace( 371 opts = argparse.Namespace(
270 directories=directories, 372 directories=directories,
271 algorithm=util.argv2algo(algorithm), 373 algorithm=util.argv2algo(algorithm),
272 append_output=append_output, 374 append_output=append_output,
273 base64=base64, 375 base64=base64,
274 comment=comment, 376 comment=comment,
275 follow_directory_symlinks=follow_directory_symlinks, 377 follow_symlinks=follow_symlinks,
276 generator=generator, 378 generator=generator,
277 logical=logical, 379 logical=logical,
278 minimal=minimal, 380 minimal=minimal,
279 mmap=mmap, 381 mmap=mmap,
280 metadata_full_mode=full_mode, 382 metadata_full_mode=full_mode,
324 out_cm = open(opts.output, "wb") 426 out_cm = open(opts.output, "wb")
325 out_cm = CRC32Output(out_cm) 427 out_cm = CRC32Output(out_cm)
326 428
327 with out_cm as outfp: 429 with out_cm as outfp:
328 for d in opts.directories: 430 for d in opts.directories:
329
330 V1DirectoryTreesumGenerator( 431 V1DirectoryTreesumGenerator(
331 opts.algorithm, opts.mmap, opts.base64, 432 opts.algorithm, opts.mmap, opts.base64,
332 opts.logical, opts.follow_directory_symlinks, 433 opts.follow_symlinks,
333 opts.generator, 434 opts.generator,
334 opts.metadata_mode, 435 opts.metadata_mode,
335 opts.metadata_full_mode, 436 opts.metadata_full_mode,
336 opts.metadata_mtime, 437 opts.metadata_mtime,
337 opts.size_only, 438 opts.size_only,
342 443
343 444
344 class V1DirectoryTreesumGenerator(object): 445 class V1DirectoryTreesumGenerator(object):
345 446
346 def __init__(self, algorithm, use_mmap, use_base64, 447 def __init__(self, algorithm, use_mmap, use_base64,
347 handle_root_logical, follow_directory_symlinks, 448 follow_symlinks,
348 with_generator, 449 with_generator,
349 with_metadata_mode, with_metadata_full_mode, 450 with_metadata_mode, with_metadata_full_mode,
350 with_metadata_mtime, size_only, print_size, utf8_mode, 451 with_metadata_mtime, size_only, print_size, utf8_mode,
351 minimal=None,): 452 minimal=None,):
352 super(V1DirectoryTreesumGenerator, self).__init__() 453 super(V1DirectoryTreesumGenerator, self).__init__()
353 self._algorithm = algorithm 454 self._algorithm = algorithm
354 self._use_mmap = use_mmap 455 self._use_mmap = use_mmap
355 self._use_base64 = use_base64 456 self._use_base64 = use_base64
356 self._handle_root_logical = handle_root_logical 457 self._follow_symlinks = follow_symlinks
357 self._follow_directory_symlinks = follow_directory_symlinks
358 self._with_generator = with_generator 458 self._with_generator = with_generator
359 self._with_metadata_mode = with_metadata_mode 459 self._with_metadata_mode = with_metadata_mode
360 self._with_metadata_full_mode = with_metadata_full_mode 460 self._with_metadata_full_mode = with_metadata_full_mode
361 self._with_metadata_mtime = with_metadata_mtime 461 self._with_metadata_mtime = with_metadata_mtime
362 self._size_only = size_only 462 self._size_only = size_only
402 flags.append("with-metadata-fullmode") 502 flags.append("with-metadata-fullmode")
403 elif self._with_metadata_mode: 503 elif self._with_metadata_mode:
404 flags.append("with-metadata-mode") 504 flags.append("with-metadata-mode")
405 if self._with_metadata_mtime: 505 if self._with_metadata_mtime:
406 flags.append("with-metadata-mtime") 506 flags.append("with-metadata-mtime")
407 if self._handle_root_logical: 507 if self._follow_symlinks.command_line:
408 flags.append("logical") 508 flags.append("follow-symlinks-commandline")
409 if self._follow_directory_symlinks: 509 if self._follow_symlinks.directory:
410 flags.append("follow-directory-symlinks") 510 flags.append("follow-symlinks-directory")
511 if self._follow_symlinks.file:
512 flags.append("follow-symlinks-file")
411 if self._size_only: 513 if self._size_only:
412 flags.append("size-only") 514 flags.append("size-only")
413 if self._utf8_mode: 515 if self._utf8_mode:
414 flags.append("utf8-mode") 516 flags.append("utf8-mode")
415 else: 517 else:
442 else: 544 else:
443 self._outfp.write(format_bsd_line( 545 self._outfp.write(format_bsd_line(
444 "ROOT", None, walk.WalkDirEntry.alt_u8(root), False)) 546 "ROOT", None, walk.WalkDirEntry.alt_u8(root), False))
445 self._outfp.flush() 547 self._outfp.flush()
446 548
447 if not self._handle_root_logical and os.path.islink(root): 549 if not self._follow_symlinks.command_line and os.path.islink(root):
448 linktgt = walk.WalkDirEntry.from_readlink(os.readlink(root)) 550 linktgt = walk.WalkDirEntry.from_readlink(os.readlink(root))
449 linkdgst = self._algorithm[0]() 551 linkdgst = self._algorithm[0]()
450 linkdgst.update( 552 linkdgst.update(
451 util.interpolate_bytes( 553 util.interpolate_bytes(
452 b"%d:%s,", len(linktgt.fspath), linktgt.fspath)) 554 b"%d:%s,", len(linktgt.fspath), linktgt.fspath))
478 self._generate(os.path.normpath(root), tuple()) 580 self._generate(os.path.normpath(root), tuple())
479 self._outfp.write(format_bsd_line( 581 self._outfp.write(format_bsd_line(
480 "CRC32", self._outfp.hexcrcdigest(), None, False)) 582 "CRC32", self._outfp.hexcrcdigest(), None, False))
481 583
482 def _generate(self, root, top): 584 def _generate(self, root, top):
585 # This is currently always True
586 assert self._follow_symlinks.file
587
483 logging.debug("Handling %s/%r", root, top) 588 logging.debug("Handling %s/%r", root, top)
484 path = os.path.join(root, *top) if top else root 589 path = os.path.join(root, *top) if top else root
485 with walk.ScanDir(path) as dirscan: 590 with walk.ScanDir(path) as dirscan:
486 fsobjects = list(dirscan) 591 fsobjects = list(dirscan)
487 if self._utf8_mode: 592 if self._utf8_mode:
491 dir_dgst = self._algorithm[0]() 596 dir_dgst = self._algorithm[0]()
492 dir_size = 0 597 dir_size = 0
493 dir_tainted = False 598 dir_tainted = False
494 for fso in fsobjects: 599 for fso in fsobjects:
495 if fso.is_dir: 600 if fso.is_dir:
496 if fso.is_symlink and not self._follow_directory_symlinks: 601 if fso.is_symlink and not self._follow_symlinks.directory:
497 linktgt = walk.WalkDirEntry.from_readlink( 602 linktgt = walk.WalkDirEntry.from_readlink(
498 os.readlink(fso.path)) 603 os.readlink(fso.path))
499 # linktgt = util.fsencode(os.readlink(fso.path))) 604 # linktgt = util.fsencode(os.readlink(fso.path)))
500 linkdgst = self._algorithm[0]() 605 linkdgst = self._algorithm[0]()
501 if self._utf8_mode: 606 if self._utf8_mode: