Mercurial > hgrepos > Python > apps > py-cutils
changeset 228:a828f320ac58
MERGE: with branch "imports": genpwd.py
| author | Franz Glasner <fzglas.hg@dom66.de> |
|---|---|
| date | Fri, 07 Feb 2025 12:38:14 +0100 |
| parents | 5d992e2a2fbc (diff) 4bb2d0975cfe (current diff) |
| children | 788f14425503 |
| files | |
| diffstat | 40 files changed, 6244 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.hgignore Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,8 @@ +syntax: regexp + +(^|/)_venv.* +\.py[co]$ +^dist/ +^build/ +^py_cutils\.egg\-info/ +^__.*
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.hgkwarchive Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,3 @@ +[patterns] +path:cutils/__init__.py = RCS, reST +path:README.txt = RCS, reST
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.hgsigs Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,13 @@ +ed7e5010990c4e496e4a42f08f08cbd490289250 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAl/KZuEACgkQaMOdASQTpp7xqA//TP4BvVToQ+cClJrBbgom5M2Yt7wA5bVgJBwl47pemnjUKZEbI+7R/PnSe9coYIYuffCKhi3qACYwkh5QuAA1jHQafb2xCJ5s6Ik/t20XhV3j4QDMUAEfEsz44A4mwkYxyf0HajG8KSlAs58D/+/2MDoumyySrDD/A2E1aZZ6J5HNZtMXf2+b66V4BLvPO/EBMbT2F9+hUFLNXwXJMiqipEtagTUlAQxLYLZGHQccLNfMe+A8fu9j8t3F7oBE80uitGk3Wx1BReHonMnTwFBL52MjIVM4akr1xCQu20X//JFpD7HOIoMF7Uxg83IvNhRs0pk44kfOYlH/+9U65PhAn4HiV4OYfcOxtWPmhlcEI85+gqcZ/qc7lX0j8L6VL1b67eY4uWfs5C1d1deYSNIZTF81Bc0ZzMPevKFG8BNTAXSRWOrJu0ksHuJ90fMcZ/b1hkvvqVAgNm7AzJgOjlyyXGpS0MhI1mV/LU4RyHjV5MMgm7tuWJDd41/LhL4ZcdnaxlSaCqQHV7yD+KfPr4ZXd+PySxwoPfh/qn2S+qRzK3aPL+sWfvea3gB+BKMMWoYomG9rTZ4o1FSKenEclteLTUKEPLMzVLdPGRq0uJZhCYM6kobJHwBSJBgt7TFK4jMCt/DfNS/mxiQIpVC5basfNp9kBMv5CV9IcGVegygNaKk= +f7817ff5a62d7de8c81249d1ccd1eead82286ffa 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAl/N60IACgkQaMOdASQTpp5YkhAAmOYGCJ2GfOwHNVKLrAySx1nhpiM7rx0CA8+YXoRfSXf/i26jBVNKoiRkaapwzYzXik0EGAXj7mGcEmLbCSglChupNjPXhhGKCkZvcwT9Js78UfmEGv56DTeXtkcviYjRxvRJx+/NSNjrnIYTKNy6z3xXXm5/grEjAPjDVzNqVwc06qW0kKhNf6KeriY3zTVHiaUac6gtIE+RT/BAz89dZe5AXC95K0/aQMH/MGqeJLj5jH+zuxTRP0hpGYritikxyF19hHk4JAab3t3Q4sw+vfClrXXQc5w2HFchdYzBuqFsHRLfCpm6yddzuD4pPTHB/NKDX6QYgvZOGLtrHVmB48b/kNFiY660o5vXYjjDc+kQWp//T/MtiOBlAWUj3KZ50ghuI4WabDrqQ6aSNMVfVNbCIAPY1sfKrpLLi2k1plERUfw7Aq6qPRUWEcNu+T8MBK65Ggof/9gs84ORAzt0sKcKmaXiewtcpn8M8hDUG3KpzI120c0KenZJiwZpvzm9k6lvUpdpkOh7Ov8kt6Z4bm5lpmRGnnLFexrs4JMWZGoTaLAh9Xnf56KcQyknekhCq0rzmuOyrNdCTSj7pSYpJF1fmUVB9pRY4NsJW2R+Wp1QRsA02WLjH/K5PHaNsclhenRs/aIajtGGHCNEKdvTS5r9bRdWsd8Cb8ac5ymVKyM= +2d9f283f301e0509694f59d281aa40ba42b8f9bf 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAl/PPLEACgkQaMOdASQTpp7ynhAAlJSjvwrvmC8h80gzyE8SCrz3OqittIe0ceu1Ge5+IllcrBNr851s6sz7XUq7yqWW6ojQKiO9DoB2IiEtTcHmK+rsj/xCRB+IyuIJRacrP1Be1rA0oLejYNg/95SnAqepX4MZ1XMjM2sheIT2IucjZ6olIte/oyBWx6ISsFsn0wlEdmlx72WGOi5damnSTkTL1QzC6mL/YsLNqwTV3bqQuNwug7iK3nLJagLXbedb0pOU1XEqutQb7AVt0tfjL5s5GQHVPTnHr+GgORcC3Op0aM2vtLu7+w+pc4JFgFyno6GjWE/mqQSKUlyQXUHhAo9yVZ+BjLKYVGVkDQhNBpF0yq/UpG/tBgylGvRczAfeMKLbYHWJYqU5yNAuG9P3RwcOf5QSX13a/HMeVCgXuVSaVa0nX6hrpOkxakzjlbi6g6CenuB/1Pkbf6X8aWA2YOHfa0+khlZpschOAhzoSd269Lfml5uBLaRdsL04rlLb3C1SO4EEImRv1ySfxGCM8mMm/pD4UPCXmnsv3eAeer6D3+sUHn7qc+spzofr+Uw6aLuFdS7GX/JPL26u/SGIyck6wGnJ9ZozhQVOxvNe/e0opopnRm1Pt2OaW0SKPZeN9Kbtyw5uPCbomBnzEEd0sz9rwleElxHjCWgEF/TnPIsFBFK8tIRWq6dGx6TusJmqVGI= +fbaaa5790ca9ae147a21cf8e1c1efc17854f063d 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmHnymQACgkQaMOdASQTpp6VmA/9ELe0V+qWl4nciABegAgmVhQhNus1pF2wBth7oPiU4yf2/twn/mwccqTcPduqGzK5asi1ZJ0jVBUj731sGEth9LhGamkgXR2zk7jZxHJU7PPkG++yzPNHJ01AwM5hcFSZObB1K29zumvKNlBla0eC79NnbBGcBm7xrbI7UPpKf6/vqqS8SKN5YVLAQT3G79/lnEBYF47o46u5YBWw9ygtFqbJMsxr9vGxWhjg4OWti+mZO/46sbt4JZwM2a+Z6lLtNy66l3zHSR76jJ5B45mA0LXC82GgL1JM0TLU3DKaJWGwRovwfYUJa+jzXolTLH6w9gJCwsQi5z+6V9fWhYVpVhxLINuIzraO5/Q7ukVZWoGfiaGqmewiKEiqMuoKneHWodVIDRs1nervBwQ2SygnlmNHJ+ZdVrWBjUYkSWyf+pV2R3foa0v6Cwn6COj90jnFQYnFsmyRhJKKRSTUijalXNFIJd4YIN9DLXdwoewhc6xZVo/0wgnjpEAQvdtX/DgRAKgLsrnEf+v0QbQma002WfUaRn73fG1whGlCqMD+kJuyKzzL8HCdTrcefmH/Kd/7c+AQB0UQxOqmF1ujPbSHL1qs2lAP4klltglYPx6kJoanEAPspPvcf4NpCN8BgKCXZSeqXFy4prw3L/D7BBhnHr6XwUMRpR+eSc6t1eNeYDw= +d3b054066b33df4dd4544093834b24938e811f29 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmHyR4kACgkQaMOdASQTpp7QTw//fWtPtEaOP4oS5ttfvET0jtW05vGFq4+dqmKaunJiG726aaWVfhdLJHk2U6oIgWaQBtQ7lYM2CeJ5s2pmeeElnvR66xXm7+M87GMoqHikULndaG4lWyJiLU0z66Z4iS4D53WPfSVGNzYjStwJOo05UmAB7bBTZXitHKiXLEFDC5mXQ8gLNmxNkqx3I5zD7+6x+KpqNH3lZKvf4fHGKG2ZznO0D3y8bJUWnRHz2ojomUXkrrQtHhLRTtmFPszaX/QP5+YPdlEBVyYcRBUsugjuc+rg+mWPeyhB2BRNDcS8cP4KYcUlkdDxi0lCkVAPt5x6n7UagPjRkaBX+whwQ3ju4D+9toJEc6KaCzTiPWQwxYRT2AUq+C+aKrCBItI00r7FmzoWHEkU2LH5oal8u+1OGQZ5rXrLHvKOuscAyt7O4kIRazckmUBoEetO43Q+Nq/fPr4HFuhWqTpjS96EgAhOnyD/72X8VqfHkgkFmq/xgBlZrm15W5UhIvC+MDLl3ivq9PPpgDh+YoglQl47hO71TJGmJl2QtXMyz6shYm1czawccwGM3BRbaBYSMKxAsuBgpVLbIPArNVVk9cgZlwY75jFvLG/L0VbN6HSlS/R8TnoS+mA41MjPfwLOOylwZSWaZarjKas4bibdxp2A5Ua94df940mz37z5nU5SX1/CKmM= +3eceeb1d2b467323b3641419486d90f0b8d56107 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmIDpI8ACgkQaMOdASQTpp5/Hg//cX4cdCA0DjFfZ9MSfOOjDeQKGaceAezz6u9xBSVs8f52y9MhKAXZ0e+1t7nalA7dK3XTZEHKbKYByoDzPpRRI7O3SuNm3FY1/LZUcGGCBqKElRxPpAO4GPEg/pTL7yguhwpSh9hj38p1HuPebJ63f0f9XPiOeP6Uvazeae0X8XmU3ih37yNms+ye74R/z1sy5sw7TlvDYFwhdz585x+ufzDqLH5CYTtDffBv7F8X4fr5kT5PzQCamjrS9o3Phpp7gT8tbkt9HoRXSFru2cDz56GhLQ16xbD6Ox+Ov+CKTXKJhz0yqN2TXXdmdK8EOeI9VT2kbfJxh7gHFjvaxlzbjR4866QwznrluAnpK+MCxPth6z32EYLEzutfVEG9PQQMsS7Mw+UWQavla3UxOSV3gHzJaa24mZhsX2SkZpK+RXkUJYi+eXIKfmmMIoNffBj8pBsSjk54gXx33ptTzsn7VwYRy8hoasSqROGrbA3/9SR/Yy9fuRr+0gcmE7z+Bzkvc4vemqOa9tqsyX77Ty0PX5Wi4zYFu39vuEMnregvx/xy9sTY4vSwZPmQ9cvXWmty4hTQCMb6E8J+2qjpKfZD1LocMf5c+69qa3pjj41F4zpd9WxsQc8A9fcs7KR7J7IEgvaTskjmYXKGVr0HBahrJfGeUr/uqhfOayWP9A5ZSZY= +0bced92fca6615fefd131488923907dd3d554385 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmIchZ0ACgkQaMOdASQTpp6RLg//QPVzyXz6miWNS6DLGZ/oAXN0m1kSyzZIEC/W0fntB4TY1eWgGUseg1v3kcQGxke4Pq0yKLfjnBjvlrMDlK6aJ7UKXJ1fxFH2PdWrxwTSc9PoeiBI2hDQwCw/KrPU35HnrPoMC/QJrvIPTklFtDBrXpBjzC+1r0JKWp8qClYTW69VQ6txRb2mUcZKgvw1XCOr2j6TG23fW38OGbV4BIv9XUbu2lMBiN39ZDdMPmxNR/6lzJCMW0tR6uuSyTGiUMlCmdNt890xBSZI/HrmDVA+oj1FZaADFfETS+3awE1o6tT6fcUH2KdEQQMa75b5onVL4uODsoB9vvG8KvwrSVlS8CuXGfZDmCB5XacTB4DfFcV1AVDBiBqDrXRq2zSWTlnWIzoRXL6YzIMJHLuPlMRrx23762N4M9/OTs8/jB97d0+zw4P1r0J7B9/PBQrIz2YnVBgO3a83XOVGThsNJAR1NORv6U2GAHs7xVux3JOBJIeB2KRBf1dJvMIUip8T3esTXpK2343Ts7AXveSLkSwMBq9syCFOoBs+iP8FVOeJSMgCUGV0HLEiCwYmX47EqOxgL6hXXpjp0HsBBVZZ28fvD0x33tAmZcOMtGt2plfSfHAnfDZW4PYY0+19NgyKM+NDxl33C6QLjHSr4gs2mv4D1UENKvmfi4h8a30Fr38IMao= +9511dbc5ae578065ebf4bf6ae622fa89997c7386 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmId2LkACgkQaMOdASQTpp7x3hAAoKqU/fRZKrdzJ+zyM+ADmsgpCMvaU8Y2jRhnQPSNutE22U51nRYyzXCNUjdG4nFBRY9RQwUZu9DBKqd2dy875x5cHvu6HNV3+VNhXGjUpDZ4pMKfCb0fF0zllvgaZYUlBXzgXZGq6XvyDk6Sp/CTct7PewTukAKhQQrbbePwU/sNdu+/Hek3xtDGQlgjVK1LVPqxIpgrThIwhkh3uMRuPbxfpRml2B5ltIjcr7pOiomdowWEsYqVPli+kAb9fwSY3eIcNPq8uQRi5caz0mfM1Bm+D7Rmi37XGb47Wprqd43DqrwdoR5g8x1K2np1MZja2S826mfqrrw+22YdZlPSwiJ7/dCFmmJ30vpTwFx5WnR5Njo8EUtc2kUCJ1aK6qBHtwKZboQSvV+G8LTBk3wpkm8Snl2TNPX5KbSbiLr9s3+7GBTLh4rhWodUEYRea7gG2Bbm/RryttKBzBX6/MjcTP1bWt5EgvPW6LVBujV8PcNP7BDSNrsdlD8WmnkPrSWc9BxGvGLR68hMvcmKPMVWE8MPeHhtouRU343uP3WC31FoSpeyywLiABMVoZNla11WsiPNzAsKUtT9HRp9UBA48L5xoVwGw0QyaYA6wUB01lp3HEh56uND4vby9lVqMwaAvWNTrwKAoIDFtkXQearNw9C8a7DrL4cDMMc/4j5xme8= +a8c8a825890c02a73f686ed50ff9310634cf9e2a 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmJhBToACgkQaMOdASQTpp7AEBAAqOgjV0yhZMsSILXkXVNu5JVdi4ZkEeFQcPwApRsvfUBL28mRsGNDSwtBoOZoFNt+SAtD2Ivq493ZUFxmxO4xR83mD4zP6Lj8pp0gtbh4JgiNqfu3zHv/x6A1xm4wWpNkYA6wpZp6IqXfOF1o5JlfWwrgjsEwpykY0iAd56nRrlRyUXB3rqyGAAwxy63h/sA+znM/FTQRgak/59SEbMX8nL8uEOU/2ABFx2fnIktU/8OWIybleJelTE9uW0YLvrdgvgrLu6LsC2Va3K3MtUitRYM1S9iiPazEpat7CkyBL+0BvOPH/JsuaOf7K3DeDjZXeQNvcnPUmMJGt7QXowjgTxScQ+s0ZZuZUt4oPDPZloO8qPEK2z9RvlQST12w6ddB3PMbic6rOkrpIhWLbvU0WK8XaEdyA/XRNGvFBAZ+wMrLhJkxc9jwSP+JBLNDeXdp9L4Gm2No7jHxnTItfQRcc8wwd8jpBuPZL+01IQVssjD+YjbC8lHRq8dlUTLKMqgTKjwfLu/hjb8HbnzRrAH9BJ3e1w6txfZXwRuUHclq9awqbQl7JaSzUe8/M51jMh9yZG/yMF2QAeMN12Zb9m9Fl9/LUP3VeFnTq3geXptLOdpHTWNBfXaqt8azhG9QQpGGwYLGExc8aVp96F4JDjeCVDP924g8r3RXyWCT7c1prps= +f1e5590a3efb36a1b640d266986273b9076acd7b 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmJiWRMACgkQaMOdASQTpp6FhQ//YKbMjlSrndPjpyvaVsvplDfBsa7+qFCVk9qI6gWCiJOEWyAgDex2zNIGzLLXSWuvsC4dEMxpWt4JDirQcrC6AjOmSPhomprU7ikeh7/hI4ZNm5TSgqXc7pF3gibWr31bFbdQA8+ZJnJOJOfzoV4rfuq2Sr9ox+hlt1W7sjHHUJVFHPp9c4YrVFN5/lftXQg0MvpaBzaJ6cJiW/v4szvbnjsLMzSptvXHfifOIzj+bHuPjoHHjQLCWBNzxqRtdyXGJOV/7gdmEx9LzhgaB+6sBgIX5CkwsbOnf6877U7zPpzdYYbTkWVFbzQ4Gicgu4qurO4btIdwVmN9YzwhChkLFh0Gk29rLAfuBL3W7LVuFDecVptA8p/jRHSXgw80xRxE2bfvz47wRDZ1HsSz4NDEt2o9FsG89hOUUf4E8wK5YlX1e6FdU12faVdtiL54HI+L8y7OiDYT2SmtigZtqjvnplafftrpHdv8oVq2W0kczQpfqv5YKpjVZ2G7DUfrSAhLiw2IQev+no1cRFZSy5V4ul1Ii+PJXY1VPBndeaOBqe1QqRkg4R4EwykyzIuM+9S2U2cRWQYN7MmLu8w6QoROy6zLqohXFznqBjqfZlZAyC4Rd5Hlgj1xSiA5sapPgxfyHLccQf9Qk5B9grXLRFYLsaVPkHtbrV/qL2vkLOxdlag= +186170f5ac56a96a40ebbcd6dde222cb568867de 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmKUcMEACgkQaMOdASQTpp7NOg/+JQub7g8AemLNGg/fe0Hch25xDcKxYZKej/4A0CksTH8vp+EDdC39Z+ObbjLDoQ2ZYFej7t6iNyFuggKF8LBTcHDbJZBwMd8ralUNrVXgNZk+rVOeLP3mhDAaEJhB7kaEOVPPGAlbI7uy5x0D1ml9vygau5i9h0ee2SLZWOIaqF1tJnOAaZyGzaEMdKY7kgzD1ucrg5ffQm4/DIDGFisXVQ5dfuWDHJH0Bqo+wKrhQXhgsH5LJTw76Xnjmmx7D2FzW7IXe0HgwnLv3yfLJJYg6ZA393OgzoUVmwL/Bt8NwXhJ7D7XeAvmUGVxYZc45WPbhYsnOaZNPpaebZeUhn6mh2A195EjkdDP9bdmfgljsBE4wi66QjNk7F5qMObxIiWO/ujeGl/xIXLbhOM1RSuhtRpSk1pIHQCkCypqYEMMqYa8At+jnlQZFqVfmZsEOlMmmgs7CvfxQj2mmVXEV91t9XGRfEXM+B73aPWc4eNUV7dGtoa5d74cDHOrZrR7EHOcjR4JJK9U+G0L3K9H9un8tJIOAFJthEWdAqz0Zr+k1OkB1J/jWSHQqlLJGoj261nlFa1rorXp8h0AbjkpaCMb7nlhPLKGc8OVdwUT9mYTL4tqIJNva17+YZsGDEA5mhyBT0O6+We9XQ9iAfmujeazrIGI6ae9EpNpphVgUGQVd1M= +798c79420f6595a745ad691ecf68bc858a51197b 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmeTvUUACgkQaMOdASQTpp5/DQ/+KaG/+KQmZCGJj+kSyiUM+6lcSc3jgToBHpIrVAqJuHGB2SfcX5FLLOYSKm4S4lcligwMfRULrbtYo06P0lFO4mWA+2usu9463OFS9hDGZwGRFfWU4lfDlvGtcQRt6ZMAg30fEgIzQNwacobrAg0VMzTCIqFUfEbjJ+WUPucL/Nh6i7Hi6sB0AAU01S5Uem7sXQLusD0NDRPKq1vJ9UWyploHm0BFjoAL3zPRdvqYHptbewRaDZE0hLxWUcu5Tn6c16foJFfNau2UIejCQrohlB+Hg4qH+5BBl5XSmAPAq29SGYJSBGIXiSP8OrCjiHaNp+Qs88FDlKfeRxhQPGgzvhiXhRwId4oE+1mKCPCYPqKMcYe+E92jSSg+QrA8eh2DYXlLAz4Y8Cekh49FvuYMbH57HDfT2FohSg4noh3vpGZvjR6TBsBAZwa63S6mYir0U9c25qzN0fNVYJtyJmAFYsnVhRwgBszYI+QoAWHtLtJuQlhFL0sfeZ5D7VNPuL1qhhWeJsLR6osp9lEXCsO5JezKt07ZJYwXYF5bYMevkOFvEtbD7wi+6SOEpO+Qc+1wEF3KHK29bU2dCT8+J7WhDm+G+ErOo0ARR1ljiV75EHPahEzC1UCHUViyfbrX1xhCP47IGx4LDjXBUr1BYOilC3IfYisvHe467naDmAbr984= +393da3f9d8a9a1399bb756193616ce0c65d60df0 0 iQIzBAABCAAdFiEEnJ+K/GJ0KgNZGXPsaMOdASQTpp4FAmeVGN0ACgkQaMOdASQTpp6ldw/+N/JX7oxA7NgK0hlUjUGaTsoIIBdxu0dcsnkl5Jtw+AB4K8jthe6xmz3SgC2PV7oP1ptUK4HRDEZmJ9toBJok8cx6S586sx6lEOXvNnPQFZ2Q3t41/nNFWeyS9arXA8oMe5xozPwKk8oruMmj1zjTCf5OoiWrbkIMbFmyhzymMpLZEgEWEukN63n06pA6MSp0uLKRhCbMuzq8vMfsl77eFdFED/Ec8He7GT0uJTYLSfvHR2fPF/CtyTVEEL/7GdP8lha5Zpx6r3SYlUH0OkU9vKnYdGTLYFqhhhI2/xtEiWW+xrc6dm2dQYfNv0unecyzlDq9QveyNLD/6EyWiyVi59vm+mdHlCtCiPfhGx/C6xYZCBMzj5dZKoQzhZ7n1gxPI2g66gDPewdUtsKWb6EQ6gX+yUXdfA8DSwe/eghqloN0Zi7Ek0N8stylQsGx5ELi4B274fZ3Hk2bVyUCi8bDiz7WIlCvgM7SGaU1OMTG5jJUxiKU53ezj2lsiQ9wEnJ9BE82X0ylcqhah/+Rs8AAv+kCwizJS+9EWhchXqIYHG3FmjCxHTJ0XsS+dLtOMNPe8WVtaO5p+8ncvMlnEG7B4G/v7p4HrrdhYs1so9wF40ohBMxuiwVzoWqTbw6FgOtVMqZSp4+MJ7Qxi0iBOHtWiajK6W1DFpCltAljpDjg94U=
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/.hgtags Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,17 @@ +409dac636805b9384762f02db7dd053e7ea2d629 v0.1 +b80519d583457bbde9c54ab5efe9e1c85f1dea8b v0.2 +801f6585a263fb488cc026d5a579890012d7860d v0.3 +801f6585a263fb488cc026d5a579890012d7860d v0.3 +0000000000000000000000000000000000000000 v0.3 +0000000000000000000000000000000000000000 v0.3 +acedf239289df7a8183098a4b95638c41fa5a9d3 v0.3 +7a3bb86e2d155fc3167a003f60694a1a7c9b9447 v0.3.1 +2e0cf1e7c4833ba462fc31be5082878536a6854f v0.3.2 +44172581bfb8c465f84be783a7cfc5d4b2746c7d v0.3.3 +15c1058c832e8a2c1e9e53c32b578b57379008ac v0.4 +79f49b0602c0dc9b3972f36d34451cda52656d05 v0.4.1 +0ebdd6b01c0854f779d02f407165985afa9edcf0 v0.4.2 +10785998be38ea66181d892baaea548dcba9f24a v0.4.3 +0edbb9a261f7e615822e8635f16784f31d0b9a0c v0.4.4 +798c79420f6595a745ad691ecf68bc858a51197b v0.5 +422a594c588ee6f634cf3864629a1bbf00226ae7 v0.5.1
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/LICENSE.txt Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,59 @@ +Copyright (c) 2020-2025 Franz Glasner + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the + distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived + from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +Licenses for incorporated software +================================== + +crcmod/crcmod2 -- vendored in cutils.crcmod -- MIT License +---------------------------------------------------------- + +Copyright (c) 2010 Raymond L. Buvel +Copyright (c) 2010 Craig McQueen +Copyright (c) 2025 Franz Glasner + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +----------------------------------------------------------------------------
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/MANIFEST.in Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,5 @@ +include .hg* *.txt *.py +graft docs +prune docs/_build +graft tests +global-exclude *.pyc *.pyo
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/README.txt Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,21 @@ +=========== + py-cutils +=========== + +:Author: Franz Glasner +:Version: 0.5.1 +:Date: 2025-01-25 +:Copyright: (c) 2020-2025 Franz Glasner +:License: BSD 3-Clause "New" or "Revised" License +:ID: @(#) $HGid$ + +Pure Python implementations of coreutils and some additional utilities. + +Currently implemented: + +- `dos2unix` as :command:`py-dos2unix` +- `shasum` as :command:`py-shasum` + +Additional utilities: + +- :command:`py-treesum` to compute hash trees for directories.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/__init__.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,18 @@ +# -*- coding: utf-8 -*- +r""" +:Author: Franz Glasner +:Copyright: (c) 2020-2025 Franz Glasner +:License: BSD 3-Clause "New" or "Revised" License. + See :ref:`LICENSE.txt <license>` for details. + If you cannot find LICENSE.txt see + <https://opensource.org/licenses/BSD-3-Clause> +:ID: @(#) $HGid$ + +""" + +__version__ = "0.5.1" + +__revision__ = "|VCSRevision|" +__date__ = "|VCSJustDate|" + +__all__ = ["__version__"]
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/__init__.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,41 @@ +# -*- coding: utf-8; -*- +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- + +from __future__ import absolute_import + +import sys + +if sys.version_info[0] < 3: + from .python2.crcmod import * + from .python2.crcmod import __doc__ +else: + from .python3.crcmod import * + from .python3.crcmod import __doc__ + + +__version__ = "1.7" + + +del sys
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/predefined.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,37 @@ +# -*- coding: utf-8; -*- +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- + +from __future__ import absolute_import + +import sys + +if sys.version_info[0] < 3: + from .python2.predefined import * + from .python2.predefined import __doc__ +else: + from .python3.predefined import * + from .python3.predefined import __doc__ + +del sys
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python2/__init__.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#-----------------------------------------------------------------------------
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python2/_crcfunpy.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,87 @@ +#----------------------------------------------------------------------------- +# Low level CRC functions for use by crcmod. This version is implemented in +# Python for a couple of reasons. 1) Provide a reference implememtation. +# 2) Provide a version that can be used on systems where a C compiler is not +# available for building extension modules. +# +# Copyright (c) 2004 Raymond L. Buvel +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- + +def _crc8(data, crc, table): + crc = crc & 0xFF + for x in data: + crc = table[ord(x) ^ crc] + return crc + +def _crc8r(data, crc, table): + crc = crc & 0xFF + for x in data: + crc = table[ord(x) ^ crc] + return crc + +def _crc16(data, crc, table): + crc = crc & 0xFFFF + for x in data: + crc = table[ord(x) ^ ((crc>>8) & 0xFF)] ^ ((crc << 8) & 0xFF00) + return crc + +def _crc16r(data, crc, table): + crc = crc & 0xFFFF + for x in data: + crc = table[ord(x) ^ (crc & 0xFF)] ^ (crc >> 8) + return crc + +def _crc24(data, crc, table): + crc = crc & 0xFFFFFF + for x in data: + crc = table[ord(x) ^ (int(crc>>16) & 0xFF)] ^ ((crc << 8) & 0xFFFF00) + return crc + +def _crc24r(data, crc, table): + crc = crc & 0xFFFFFF + for x in data: + crc = table[ord(x) ^ int(crc & 0xFF)] ^ (crc >> 8) + return crc + +def _crc32(data, crc, table): + crc = crc & 0xFFFFFFFFL + for x in data: + crc = table[ord(x) ^ (int(crc>>24) & 0xFF)] ^ ((crc << 8) & 0xFFFFFF00L) + return crc + +def _crc32r(data, crc, table): + crc = crc & 0xFFFFFFFFL + for x in data: + crc = table[ord(x) ^ int(crc & 0xFFL)] ^ (crc >> 8) + return crc + +def _crc64(data, crc, table): + crc = crc & 0xFFFFFFFFFFFFFFFFL + for x in data: + crc = table[ord(x) ^ (int(crc>>56) & 0xFF)] ^ ((crc << 8) & 0xFFFFFFFFFFFFFF00L) + return crc + +def _crc64r(data, crc, table): + crc = crc & 0xFFFFFFFFFFFFFFFFL + for x in data: + crc = table[ord(x) ^ int(crc & 0xFFL)] ^ (crc >> 8) + return crc +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python2/crcmod.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,476 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- +'''crcmod is a Python module for gererating objects that compute the Cyclic +Redundancy Check. Any 8, 16, 24, 32, or 64 bit polynomial can be used. + +The following are the public components of this module. + +Crc -- a class that creates instances providing the same interface as the +md5 and sha modules in the Python standard library. These instances also +provide a method for generating a C/C++ function to compute the CRC. + +mkCrcFun -- create a Python function to compute the CRC using the specified +polynomial and initial value. This provides a much simpler interface if +all you need is a function for CRC calculation. +''' + +from __future__ import absolute_import + + +__all__ = '''mkCrcFun Crc'''.split() + + +# Select the appropriate set of low-level CRC functions for this installation. +# If the extension module was not built, drop back to the Python implementation +# even though it is significantly slower. +try: + from . import _crcfunext as _crcfun + _usingExtension = True +except ImportError: + from . import _crcfunpy as _crcfun + _usingExtension = False + +import sys, struct + +#----------------------------------------------------------------------------- +class Crc: + '''Compute a Cyclic Redundancy Check (CRC) using the specified polynomial. + + Instances of this class have the same interface as the md5 and sha modules + in the Python standard library. See the documentation for these modules + for examples of how to use a Crc instance. + + The string representation of a Crc instance identifies the polynomial, + initial value, XOR out value, and the current CRC value. The print + statement can be used to output this information. + + If you need to generate a C/C++ function for use in another application, + use the generateCode method. If you need to generate code for another + language, subclass Crc and override the generateCode method. + + The following are the parameters supplied to the constructor. + + poly -- The generator polynomial to use in calculating the CRC. The value + is specified as a Python integer or long integer. The bits in this integer + are the coefficients of the polynomial. The only polynomials allowed are + those that generate 8, 16, 24, 32, or 64 bit CRCs. + + initCrc -- Initial value used to start the CRC calculation. This initial + value should be the initial shift register value XORed with the final XOR + value. That is equivalent to the CRC result the algorithm should return for + a zero-length string. Defaults to all bits set because that starting value + will take leading zero bytes into account. Starting with zero will ignore + all leading zero bytes. + + rev -- A flag that selects a bit reversed algorithm when True. Defaults to + True because the bit reversed algorithms are more efficient. + + xorOut -- Final value to XOR with the calculated CRC value. Used by some + CRC algorithms. Defaults to zero. + ''' + def __init__(self, poly, initCrc=~0L, rev=True, xorOut=0, initialize=True): + if not initialize: + # Don't want to perform the initialization when using new or copy + # to create a new instance. + return + + (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut) + self.digest_size = sizeBits//8 + self.initCrc = initCrc + self.xorOut = xorOut + + self.poly = poly + self.reverse = rev + + (crcfun, table) = _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut) + self._crc = crcfun + self.table = table + + self.crcValue = self.initCrc + + def __str__(self): + lst = [] + lst.append('poly = 0x%X' % self.poly) + lst.append('reverse = %s' % self.reverse) + fmt = '0x%%0%dX' % (self.digest_size*2) + lst.append('initCrc = %s' % (fmt % self.initCrc)) + lst.append('xorOut = %s' % (fmt % self.xorOut)) + lst.append('crcValue = %s' % (fmt % self.crcValue)) + return '\n'.join(lst) + + def new(self, arg=None): + '''Create a new instance of the Crc class initialized to the same + values as the original instance. The current CRC is set to the initial + value. If a string is provided in the optional arg parameter, it is + passed to the update method. + ''' + n = Crc(poly=None, initialize=False) + n._crc = self._crc + n.digest_size = self.digest_size + n.initCrc = self.initCrc + n.xorOut = self.xorOut + n.table = self.table + n.crcValue = self.initCrc + n.reverse = self.reverse + n.poly = self.poly + if arg is not None: + n.update(arg) + return n + + def copy(self): + '''Create a new instance of the Crc class initialized to the same + values as the original instance. The current CRC is set to the current + value. This allows multiple CRC calculations using a common initial + string. + ''' + c = self.new() + c.crcValue = self.crcValue + return c + + def update(self, data): + '''Update the current CRC value using the string specified as the data + parameter. + ''' + self.crcValue = self._crc(data, self.crcValue) + + def digest(self): + '''Return the current CRC value as a string of bytes. The length of + this string is specified in the digest_size attribute. + ''' + n = self.digest_size + crc = self.crcValue + lst = [] + while n > 0: + lst.append(chr(crc & 0xFF)) + crc = crc >> 8 + n -= 1 + lst.reverse() + return ''.join(lst) + + def hexdigest(self): + '''Return the current CRC value as a string of hex digits. The length + of this string is twice the digest_size attribute. + ''' + n = self.digest_size + crc = self.crcValue + lst = [] + while n > 0: + lst.append('%02X' % (crc & 0xFF)) + crc = crc >> 8 + n -= 1 + lst.reverse() + return ''.join(lst) + + def generateCode(self, functionName, out, dataType=None, crcType=None): + '''Generate a C/C++ function. + + functionName -- String specifying the name of the function. + + out -- An open file-like object with a write method. This specifies + where the generated code is written. + + dataType -- An optional parameter specifying the data type of the input + data to the function. Defaults to UINT8. + + crcType -- An optional parameter specifying the data type of the CRC + value. Defaults to one of UINT8, UINT16, UINT32, or UINT64 depending + on the size of the CRC value. + ''' + if dataType is None: + dataType = 'UINT8' + + if crcType is None: + size = 8*self.digest_size + if size == 24: + size = 32 + crcType = 'UINT%d' % size + + if self.digest_size == 1: + # Both 8-bit CRC algorithms are the same + crcAlgor = 'table[*data ^ (%s)crc]' + elif self.reverse: + # The bit reverse algorithms are all the same except for the data + # type of the crc variable which is specified elsewhere. + crcAlgor = 'table[*data ^ (%s)crc] ^ (crc >> 8)' + else: + # The forward CRC algorithms larger than 8 bits have an extra shift + # operation to get the high byte. + shift = 8*(self.digest_size - 1) + crcAlgor = 'table[*data ^ (%%s)(crc >> %d)] ^ (crc << 8)' % shift + + fmt = '0x%%0%dX' % (2*self.digest_size) + if self.digest_size <= 4: + fmt = fmt + 'U,' + else: + # Need the long long type identifier to keep gcc from complaining. + fmt = fmt + 'ULL,' + + # Select the number of entries per row in the output code. + n = {1:8, 2:8, 3:4, 4:4, 8:2}[self.digest_size] + + lst = [] + for i, val in enumerate(self.table): + if (i % n) == 0: + lst.append('\n ') + lst.append(fmt % val) + + poly = 'polynomial: 0x%X' % self.poly + if self.reverse: + poly = poly + ', bit reverse algorithm' + + if self.xorOut: + # Need to remove the comma from the format. + preCondition = '\n crc = crc ^ %s;' % (fmt[:-1] % self.xorOut) + postCondition = preCondition + else: + preCondition = '' + postCondition = '' + + if self.digest_size == 3: + # The 24-bit CRC needs to be conditioned so that only 24-bits are + # used from the 32-bit variable. + if self.reverse: + preCondition += '\n crc = crc & 0xFFFFFFU;' + else: + postCondition += '\n crc = crc & 0xFFFFFFU;' + + + parms = { + 'dataType' : dataType, + 'crcType' : crcType, + 'name' : functionName, + 'crcAlgor' : crcAlgor % dataType, + 'crcTable' : ''.join(lst), + 'poly' : poly, + 'preCondition' : preCondition, + 'postCondition' : postCondition, + } + out.write(_codeTemplate % parms) + +#----------------------------------------------------------------------------- +def mkCrcFun(poly, initCrc=~0L, rev=True, xorOut=0): + '''Return a function that computes the CRC using the specified polynomial. + + poly -- integer representation of the generator polynomial + initCrc -- default initial CRC value + rev -- when true, indicates that the data is processed bit reversed. + xorOut -- the final XOR value + + The returned function has the following user interface + def crcfun(data, crc=initCrc): + ''' + + # First we must verify the params + (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut) + # Make the function (and table), return the function + return _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut)[0] + +#----------------------------------------------------------------------------- +# Naming convention: +# All function names ending with r are bit reverse variants of the ones +# without the r. + +#----------------------------------------------------------------------------- +# Check the polynomial to make sure that it is acceptable and return the number +# of bits in the CRC. + +def _verifyPoly(poly): + msg = 'The degree of the polynomial must be 8, 16, 24, 32 or 64' + poly = long(poly) # Use a common representation for all operations + for n in (8,16,24,32,64): + low = 1L<<n + high = low*2 + if low <= poly < high: + return n + raise ValueError(msg) + +#----------------------------------------------------------------------------- +# Bit reverse the input value. + +def _bitrev(x, n): + x = long(x) + y = 0L + for i in xrange(n): + y = (y << 1) | (x & 1L) + x = x >> 1 + if ((1L<<n)-1) <= sys.maxint: + return int(y) + return y + +#----------------------------------------------------------------------------- +# The following functions compute the CRC for a single byte. These are used +# to build up the tables needed in the CRC algorithm. Assumes the high order +# bit of the polynomial has been stripped off. + +def _bytecrc(crc, poly, n): + crc = long(crc) + poly = long(poly) + mask = 1L<<(n-1) + for i in xrange(8): + if crc & mask: + crc = (crc << 1) ^ poly + else: + crc = crc << 1 + mask = (1L<<n) - 1 + crc = crc & mask + if mask <= sys.maxint: + return int(crc) + return crc + +def _bytecrc_r(crc, poly, n): + crc = long(crc) + poly = long(poly) + for i in xrange(8): + if crc & 1L: + crc = (crc >> 1) ^ poly + else: + crc = crc >> 1 + mask = (1L<<n) - 1 + crc = crc & mask + if mask <= sys.maxint: + return int(crc) + return crc + +#----------------------------------------------------------------------------- +# The following functions compute the table needed to compute the CRC. The +# table is returned as a list. Note that the array module does not support +# 64-bit integers on a 32-bit architecture as of Python 2.3. +# +# These routines assume that the polynomial and the number of bits in the CRC +# have been checked for validity by the caller. + +def _mkTable(poly, n): + mask = (1L<<n) - 1 + poly = long(poly) & mask + table = [_bytecrc(long(i)<<(n-8),poly,n) for i in xrange(256)] + return table + +def _mkTable_r(poly, n): + mask = (1L<<n) - 1 + poly = _bitrev(long(poly) & mask, n) + table = [_bytecrc_r(long(i),poly,n) for i in xrange(256)] + return table + +#----------------------------------------------------------------------------- +# Map the CRC size onto the functions that handle these sizes. + +_sizeMap = { + 8 : [_crcfun._crc8, _crcfun._crc8r], + 16 : [_crcfun._crc16, _crcfun._crc16r], + 24 : [_crcfun._crc24, _crcfun._crc24r], + 32 : [_crcfun._crc32, _crcfun._crc32r], + 64 : [_crcfun._crc64, _crcfun._crc64r], +} + +#----------------------------------------------------------------------------- +# Build a mapping of size to struct module type code. This table is +# constructed dynamically so that it has the best chance of picking the best +# code to use for the platform we are running on. This should properly adapt +# to 32 and 64 bit machines. + +_sizeToTypeCode = {} + +for typeCode in 'B H I L Q'.split(): + size = {1:8, 2:16, 4:32, 8:64}.get(struct.calcsize(typeCode),None) + if size is not None and size not in _sizeToTypeCode: + _sizeToTypeCode[size] = '256%s' % typeCode + +_sizeToTypeCode[24] = _sizeToTypeCode[32] + +del typeCode, size + +#----------------------------------------------------------------------------- +# The following function validates the parameters of the CRC, namely, +# poly, and initial/final XOR values. +# It returns the size of the CRC (in bits), and "sanitized" initial/final XOR values. + +def _verifyParams(poly, initCrc, xorOut): + sizeBits = _verifyPoly(poly) + + mask = (1L<<sizeBits) - 1 + + # Adjust the initial CRC to the correct data type (unsigned value). + initCrc = long(initCrc) & mask + if mask <= sys.maxint: + initCrc = int(initCrc) + + # Similar for XOR-out value. + xorOut = long(xorOut) & mask + if mask <= sys.maxint: + xorOut = int(xorOut) + + return (sizeBits, initCrc, xorOut) + +#----------------------------------------------------------------------------- +# The following function returns a Python function to compute the CRC. +# +# It must be passed parameters that are already verified & sanitized by +# _verifyParams(). +# +# The returned function calls a low level function that is written in C if the +# extension module could be loaded. Otherwise, a Python implementation is +# used. +# +# In addition to this function, a list containing the CRC table is returned. + +def _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut): + if rev: + tableList = _mkTable_r(poly, sizeBits) + _fun = _sizeMap[sizeBits][1] + else: + tableList = _mkTable(poly, sizeBits) + _fun = _sizeMap[sizeBits][0] + + _table = tableList + if _usingExtension: + _table = struct.pack(_sizeToTypeCode[sizeBits], *tableList) + + if xorOut == 0: + def crcfun(data, crc=initCrc, table=_table, fun=_fun): + return fun(data, crc, table) + else: + def crcfun(data, crc=initCrc, table=_table, fun=_fun): + return xorOut ^ fun(data, xorOut ^ crc, table) + + return crcfun, tableList + +#----------------------------------------------------------------------------- +_codeTemplate = '''// Automatically generated CRC function +// %(poly)s +%(crcType)s +%(name)s(%(dataType)s *data, int len, %(crcType)s crc) +{ + static const %(crcType)s table[256] = {%(crcTable)s + }; + %(preCondition)s + while (len > 0) + { + crc = %(crcAlgor)s; + data++; + len--; + }%(postCondition)s + return crc; +} +''' +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python2/predefined.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,180 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- +''' +crcmod.predefined defines some well-known CRC algorithms. + +To use it, e.g.: + import crcmod.predefined + + crc32func = crcmod.predefined.mkPredefinedCrcFun("crc-32") + crc32class = crcmod.predefined.PredefinedCrc("crc-32") + +crcmod.predefined.Crc is an alias for crcmod.predefined.PredefinedCrc +But if doing 'from crc.predefined import *', only PredefinedCrc is imported. +''' + +from __future__ import absolute_import + +# local imports +from . import crcmod + +__all__ = [ + 'PredefinedCrc', + 'mkPredefinedCrcFun', +] + +REVERSE = True +NON_REVERSE = False + +# +# The following table defines the parameters of well-known CRC algorithms. +# The "Check" value is the CRC for the ASCII byte sequence "123456789". It +# can be used for unit tests. +# +# See also: +# - https://en.wikipedia.org/wiki/Cyclic_redundancy_check +# - https://reveng.sourceforge.io/crc-catalogue/all.htm +# - http://users.ece.cmu.edu/~koopman/crc/index.html +# - https://pycrc.org/models.html +# - https://github.com/marzooqy/anycrc +# +_crc_definitions_table = [ +# Name Identifier-name, Poly Reverse Init-value XOR-out Check + [ 'crc-8', 'Crc8', 0x107, NON_REVERSE, 0x00, 0x00, 0xF4, ], + [ 'crc-8-darc', 'Crc8Darc', 0x139, REVERSE, 0x00, 0x00, 0x15, ], + [ 'crc-8-i-code', 'Crc8ICode', 0x11D, NON_REVERSE, 0xFD, 0x00, 0x7E, ], + [ 'crc-8-itu', 'Crc8Itu', 0x107, NON_REVERSE, 0x55, 0x55, 0xA1, ], + [ 'crc-8-maxim', 'Crc8Maxim', 0x131, REVERSE, 0x00, 0x00, 0xA1, ], + [ 'crc-8-rohc', 'Crc8Rohc', 0x107, REVERSE, 0xFF, 0x00, 0xD0, ], + [ 'crc-8-wcdma', 'Crc8Wcdma', 0x19B, REVERSE, 0x00, 0x00, 0x25, ], + + [ 'crc-16', 'Crc16', 0x18005, REVERSE, 0x0000, 0x0000, 0xBB3D, ], + [ 'crc-16-buypass', 'Crc16Buypass', 0x18005, NON_REVERSE, 0x0000, 0x0000, 0xFEE8, ], + [ 'crc-16-dds-110', 'Crc16Dds110', 0x18005, NON_REVERSE, 0x800D, 0x0000, 0x9ECF, ], + [ 'crc-16-dect', 'Crc16Dect', 0x10589, NON_REVERSE, 0x0001, 0x0001, 0x007E, ], + [ 'crc-16-dnp', 'Crc16Dnp', 0x13D65, REVERSE, 0xFFFF, 0xFFFF, 0xEA82, ], + [ 'crc-16-en-13757', 'Crc16En13757', 0x13D65, NON_REVERSE, 0xFFFF, 0xFFFF, 0xC2B7, ], + [ 'crc-16-genibus', 'Crc16Genibus', 0x11021, NON_REVERSE, 0x0000, 0xFFFF, 0xD64E, ], + [ 'crc-16-maxim', 'Crc16Maxim', 0x18005, REVERSE, 0xFFFF, 0xFFFF, 0x44C2, ], + [ 'crc-16-mcrf4xx', 'Crc16Mcrf4xx', 0x11021, REVERSE, 0xFFFF, 0x0000, 0x6F91, ], + [ 'crc-16-riello', 'Crc16Riello', 0x11021, REVERSE, 0x554D, 0x0000, 0x63D0, ], + [ 'crc-16-t10-dif', 'Crc16T10Dif', 0x18BB7, NON_REVERSE, 0x0000, 0x0000, 0xD0DB, ], + [ 'crc-16-teledisk', 'Crc16Teledisk', 0x1A097, NON_REVERSE, 0x0000, 0x0000, 0x0FB3, ], + [ 'crc-16-usb', 'Crc16Usb', 0x18005, REVERSE, 0x0000, 0xFFFF, 0xB4C8, ], + [ 'x-25', 'CrcX25', 0x11021, REVERSE, 0x0000, 0xFFFF, 0x906E, ], + [ 'xmodem', 'CrcXmodem', 0x11021, NON_REVERSE, 0x0000, 0x0000, 0x31C3, ], + [ 'modbus', 'CrcModbus', 0x18005, REVERSE, 0xFFFF, 0x0000, 0x4B37, ], + + # Note definitions of CCITT are disputable. See: + # http://homepages.tesco.net/~rainstorm/crc-catalogue.htm + # http://web.archive.org/web/20071229021252/http://www.joegeluso.com/software/articles/ccitt.htm + [ 'kermit', 'CrcKermit', 0x11021, REVERSE, 0x0000, 0x0000, 0x2189, ], + [ 'crc-ccitt-false', 'CrcCcittFalse', 0x11021, NON_REVERSE, 0xFFFF, 0x0000, 0x29B1, ], + [ 'crc-aug-ccitt', 'CrcAugCcitt', 0x11021, NON_REVERSE, 0x1D0F, 0x0000, 0xE5CC, ], + + [ 'crc-24', 'Crc24', 0x1864CFB, NON_REVERSE, 0xB704CE, 0x000000, 0x21CF02, ], + [ 'crc-24-flexray-a', 'Crc24FlexrayA', 0x15D6DCB, NON_REVERSE, 0xFEDCBA, 0x000000, 0x7979BD, ], + [ 'crc-24-flexray-b', 'Crc24FlexrayB', 0x15D6DCB, NON_REVERSE, 0xABCDEF, 0x000000, 0x1F23B8, ], + + [ 'crc-32', 'Crc32', 0x104C11DB7, REVERSE, 0x00000000, 0xFFFFFFFF, 0xCBF43926, ], + [ 'crc-32-bzip2', 'Crc32Bzip2', 0x104C11DB7, NON_REVERSE, 0x00000000, 0xFFFFFFFF, 0xFC891918, ], + [ 'crc-32c', 'Crc32C', 0x11EDC6F41, REVERSE, 0x00000000, 0xFFFFFFFF, 0xE3069283, ], + [ 'crc-32d', 'Crc32D', 0x1A833982B, REVERSE, 0x00000000, 0xFFFFFFFF, 0x87315576, ], + [ 'crc-32-mpeg', 'Crc32Mpeg', 0x104C11DB7, NON_REVERSE, 0xFFFFFFFF, 0x00000000, 0x0376E6E7, ], + [ 'posix', 'CrcPosix', 0x104C11DB7, NON_REVERSE, 0xFFFFFFFF, 0xFFFFFFFF, 0x765E7680, ], + [ 'crc-32q', 'Crc32Q', 0x1814141AB, NON_REVERSE, 0x00000000, 0x00000000, 0x3010BF7F, ], + [ 'jamcrc', 'CrcJamCrc', 0x104C11DB7, REVERSE, 0xFFFFFFFF, 0x00000000, 0x340BC6D9, ], + [ 'xfer', 'CrcXfer', 0x1000000AF, NON_REVERSE, 0x00000000, 0x00000000, 0xBD0BE338, ], + +# 64-bit +# Name Identifier-name, Poly Reverse Init-value XOR-out Check + [ 'crc-64', 'Crc64', 0x1000000000000001B, REVERSE, 0x0000000000000000, 0x0000000000000000, 0x46A5A9388A5BEFFE, ], # ISO POLY + # See https://lwn.net/Articles/976030/ (MCRC64, used as CRC-64 in the Linux kernel) + [ 'crc-64-2', 'Crc64_2', 0x1000000000000001B, NON_REVERSE, 0x0000000000000000, 0x0000000000000000, 0xE4FFBEA588933790, ], # fag, ISO POLY + [ 'crc-64-go', 'Crc64Go', 0x1000000000000001B, REVERSE, 0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0xB90956C775A41001, ], # fag, ISO POLY + [ 'crc-64-ecma', 'Crc64Ecma', 0x142F0E1EBA9EA3693, NON_REVERSE, 0x0000000000000000, 0x0000000000000000, 0x6C40DF5F0B497347, ], # fag + [ 'crc-64-we', 'Crc64We', 0x142F0E1EBA9EA3693, NON_REVERSE, 0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0x62EC59E3F1A4F00A, ], + [ 'crc-64-jones', 'Crc64Jones', 0x1AD93D23594C935A9, REVERSE, 0xFFFFFFFFFFFFFFFF, 0x0000000000000000, 0xCAA717168609F281, ], + [ 'crc-64-xz', 'Crc64Xz', 0x142F0E1EBA9EA3693, REVERSE, 0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0x995DC9BBDF1939FA, ], # fag, ECMA POLY + [ 'crc-64-redis', 'Crc64Redis', 0x1AD93D23594C935A9, REVERSE, 0x0000000000000000, 0x0000000000000000, 0xE9C6D914C4b8D9CA, ], # fag +] + + +def _simplify_name(name): + """ + Reduce CRC definition name to a simplified form: + * lowercase + * dashes removed + * spaces removed + * any initial "CRC" string removed + """ + name = name.lower() + name = name.replace('-', '') + name = name.replace(' ', '') + if name.startswith('crc'): + name = name[len('crc'):] + return name + + +_crc_definitions_by_name = {} +_crc_definitions_by_identifier = {} +_crc_definitions = [] + +_crc_table_headings = [ 'name', 'identifier', 'poly', 'reverse', 'init', 'xor_out', 'check' ] + +for table_entry in _crc_definitions_table: + crc_definition = dict(zip(_crc_table_headings, table_entry)) + _crc_definitions.append(crc_definition) + name = _simplify_name(table_entry[0]) + if name in _crc_definitions_by_name: + raise Exception("Duplicate entry for '%s' in CRC table" % name) + _crc_definitions_by_name[name] = crc_definition + _crc_definitions_by_identifier[table_entry[1]] = crc_definition + + +def _get_definition_by_name(crc_name): + definition = _crc_definitions_by_name.get(_simplify_name(crc_name), None) + if not definition: + definition = _crc_definitions_by_identifier.get(crc_name, None) + if not definition: + raise KeyError("Unkown CRC name '%s'" % crc_name) + return definition + + +class PredefinedCrc(crcmod.Crc): + def __init__(self, crc_name): + definition = _get_definition_by_name(crc_name) + crcmod.Crc.__init__(self, poly=definition['poly'], initCrc=definition['init'], rev=definition['reverse'], xorOut=definition['xor_out']) + + +# crcmod.predefined.Crc is an alias for crcmod.predefined.PredefinedCrc +Crc = PredefinedCrc + + +def mkPredefinedCrcFun(crc_name): + definition = _get_definition_by_name(crc_name) + return crcmod.mkCrcFun(poly=definition['poly'], initCrc=definition['init'], rev=definition['reverse'], xorOut=definition['xor_out']) + + +# crcmod.predefined.mkCrcFun is an alias for crcmod.predefined.mkPredefinedCrcFun +mkCrcFun = mkPredefinedCrcFun
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python2/test.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,496 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- +'''Unit tests for crcmod functionality''' + + +from __future__ import absolute_import + +import unittest +import binascii + +from .crcmod import mkCrcFun, Crc +from .crcmod import _usingExtension +from .predefined import PredefinedCrc +from .predefined import mkPredefinedCrcFun +from .predefined import _crc_definitions as _predefined_crc_definitions + + +#----------------------------------------------------------------------------- +# This polynomial was chosen because it is the product of two irreducible +# polynomials. +# g8 = (x^7+x+1)*(x+1) +g8 = 0x185 + +#----------------------------------------------------------------------------- +# The following reproduces all of the entries in the Numerical Recipes table. +# This is the standard CCITT polynomial. +g16 = 0x11021 + +#----------------------------------------------------------------------------- +g24 = 0x15D6DCB + +#----------------------------------------------------------------------------- +# This is the standard AUTODIN-II polynomial which appears to be used in a +# wide variety of standards and applications. +g32 = 0x104C11DB7 + + +#----------------------------------------------------------------------------- +# I was able to locate a couple of 64-bit polynomials on the web. To make it +# easier to input the representation, define a function that builds a +# polynomial from a list of the bits that need to be turned on. + +def polyFromBits(bits): + p = 0L + for n in bits: + p = p | (1L << n) + return p + +# The following is from the paper "An Improved 64-bit Cyclic Redundancy Check +# for Protein Sequences" by David T. Jones + +g64a = polyFromBits([64, 63, 61, 59, 58, 56, 55, 52, 49, 48, 47, 46, 44, 41, + 37, 36, 34, 32, 31, 28, 26, 23, 22, 19, 16, 13, 12, 10, 9, 6, 4, + 3, 0]) + +# The following is from Standard ECMA-182 "Data Interchange on 12,7 mm 48-Track +# Magnetic Tape Cartridges -DLT1 Format-", December 1992. + +g64b = polyFromBits([64, 62, 57, 55, 54, 53, 52, 47, 46, 45, 40, 39, 38, 37, + 35, 33, 32, 31, 29, 27, 24, 23, 22, 21, 19, 17, 13, 12, 10, 9, 7, + 4, 1, 0]) + +#----------------------------------------------------------------------------- +# This class is used to check the CRC calculations against a direct +# implementation using polynomial division. + +class poly: + '''Class implementing polynomials over the field of integers mod 2''' + def __init__(self,p): + p = long(p) + if p < 0: raise ValueError('invalid polynomial') + self.p = p + + def __long__(self): + return self.p + + def __eq__(self,other): + return self.p == other.p + + def __ne__(self,other): + return self.p != other.p + + # To allow sorting of polynomials, use their long integer form for + # comparison + def __cmp__(self,other): + return cmp(self.p, other.p) + + def __nonzero__(self): + return self.p != 0L + + def __neg__(self): + return self # These polynomials are their own inverse under addition + + def __invert__(self): + n = max(self.deg() + 1, 1) + x = (1L << n) - 1 + return poly(self.p ^ x) + + def __add__(self,other): + return poly(self.p ^ other.p) + + def __sub__(self,other): + return poly(self.p ^ other.p) + + def __mul__(self,other): + a = self.p + b = other.p + if a == 0 or b == 0: return poly(0) + x = 0L + while b: + if b&1: + x = x ^ a + a = a<<1 + b = b>>1 + return poly(x) + + def __divmod__(self,other): + u = self.p + m = self.deg() + v = other.p + n = other.deg() + if v == 0: raise ZeroDivisionError('polynomial division by zero') + if n == 0: return (self,poly(0)) + if m < n: return (poly(0),self) + k = m-n + a = 1L << m + v = v << k + q = 0L + while k > 0: + if a & u: + u = u ^ v + q = q | 1L + q = q << 1 + a = a >> 1 + v = v >> 1 + k -= 1 + if a & u: + u = u ^ v + q = q | 1L + return (poly(q),poly(u)) + + def __div__(self,other): + return self.__divmod__(other)[0] + + def __mod__(self,other): + return self.__divmod__(other)[1] + + def __repr__(self): + return 'poly(0x%XL)' % self.p + + def __str__(self): + p = self.p + if p == 0: return '0' + lst = { 0:[], 1:['1'], 2:['x'], 3:['1','x'] }[p&3] + p = p>>2 + n = 2 + while p: + if p&1: lst.append('x^%d' % n) + p = p>>1 + n += 1 + lst.reverse() + return '+'.join(lst) + + def deg(self): + '''return the degree of the polynomial''' + a = self.p + if a == 0: return -1 + n = 0 + while a >= 0x10000L: + n += 16 + a = a >> 16 + a = int(a) + while a > 1: + n += 1 + a = a >> 1 + return n + +#----------------------------------------------------------------------------- +# The following functions compute the CRC using direct polynomial division. +# These functions are checked against the result of the table driven +# algorithms. + +g8p = poly(g8) +x8p = poly(1L<<8) +def crc8p(d): + d = map(ord, d) + p = 0L + for i in d: + p = p*256L + i + p = poly(p) + return long(p*x8p%g8p) + +g16p = poly(g16) +x16p = poly(1L<<16) +def crc16p(d): + d = map(ord, d) + p = 0L + for i in d: + p = p*256L + i + p = poly(p) + return long(p*x16p%g16p) + +g24p = poly(g24) +x24p = poly(1L<<24) +def crc24p(d): + d = map(ord, d) + p = 0L + for i in d: + p = p*256L + i + p = poly(p) + return long(p*x24p%g24p) + +g32p = poly(g32) +x32p = poly(1L<<32) +def crc32p(d): + d = map(ord, d) + p = 0L + for i in d: + p = p*256L + i + p = poly(p) + return long(p*x32p%g32p) + +g64ap = poly(g64a) +x64p = poly(1L<<64) +def crc64ap(d): + d = map(ord, d) + p = 0L + for i in d: + p = p*256L + i + p = poly(p) + return long(p*x64p%g64ap) + +g64bp = poly(g64b) +def crc64bp(d): + d = map(ord, d) + p = 0L + for i in d: + p = p*256L + i + p = poly(p) + return long(p*x64p%g64bp) + + +class KnownAnswerTests(unittest.TestCase): + test_messages = [ + 'T', + 'CatMouse987654321', + ] + + known_answers = [ + [ (g8,0,0), (0xFE, 0x9D) ], + [ (g8,-1,1), (0x4F, 0x9B) ], + [ (g8,0,1), (0xFE, 0x62) ], + [ (g16,0,0), (0x1A71, 0xE556) ], + [ (g16,-1,1), (0x1B26, 0xF56E) ], + [ (g16,0,1), (0x14A1, 0xC28D) ], + [ (g24,0,0), (0xBCC49D, 0xC4B507) ], + [ (g24,-1,1), (0x59BD0E, 0x0AAA37) ], + [ (g24,0,1), (0xD52B0F, 0x1523AB) ], + [ (g32,0,0), (0x6B93DDDB, 0x12DCA0F4) ], + [ (g32,0xFFFFFFFFL,1), (0x41FB859FL, 0xF7B400A7L) ], + [ (g32,0,1), (0x6C0695EDL, 0xC1A40EE5L) ], + [ (g32,0,1,0xFFFFFFFF), (0xBE047A60L, 0x084BFF58L) ], + ] + + def test_known_answers(self): + for crcfun_params, v in self.known_answers: + crcfun = mkCrcFun(*crcfun_params) + self.assertEqual(crcfun('',0), 0, "Wrong answer for CRC parameters %s, input ''" % (crcfun_params,)) + for i, msg in enumerate(self.test_messages): + self.assertEqual(crcfun(msg), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg)) + self.assertEqual(crcfun(msg[4:], crcfun(msg[:4])), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg)) + self.assertEqual(crcfun(msg[-1:], crcfun(msg[:-1])), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg)) + + +class CompareReferenceCrcTest(unittest.TestCase): + test_messages = [ + '', + 'T', + '123456789', + 'CatMouse987654321', + ] + + test_poly_crcs = [ + [ (g8,0,0), crc8p ], + [ (g16,0,0), crc16p ], + [ (g24,0,0), crc24p ], + [ (g32,0,0), crc32p ], + [ (g64a,0,0), crc64ap ], + [ (g64b,0,0), crc64bp ], + ] + + @staticmethod + def reference_crc32(d, crc=0): + """This function modifies the return value of binascii.crc32 + to be an unsigned 32-bit value. I.e. in the range 0 to 2**32-1.""" + # Work around the future warning on constants. + if crc > 0x7FFFFFFFL: + x = int(crc & 0x7FFFFFFFL) + crc = x | -2147483648 + x = binascii.crc32(d,crc) + return long(x) & 0xFFFFFFFFL + + def test_compare_crc32(self): + """The binascii module has a 32-bit CRC function that is used in a wide range + of applications including the checksum used in the ZIP file format. + This test compares the CRC-32 implementation of this crcmod module to + that of binascii.crc32.""" + # The following function should produce the same result as + # self.reference_crc32 which is derived from binascii.crc32. + crc32 = mkCrcFun(g32,0,1,0xFFFFFFFF) + + for msg in self.test_messages: + self.assertEqual(crc32(msg), self.reference_crc32(msg)) + + def test_compare_poly(self): + """Compare various CRCs of this crcmod module to a pure + polynomial-based implementation.""" + for crcfun_params, crc_poly_fun in self.test_poly_crcs: + # The following function should produce the same result as + # the associated polynomial CRC function. + crcfun = mkCrcFun(*crcfun_params) + + for msg in self.test_messages: + self.assertEqual(crcfun(msg), crc_poly_fun(msg)) + + +class CrcClassTest(unittest.TestCase): + """Verify the Crc class""" + + msg = 'CatMouse987654321' + + def test_simple_crc32_class(self): + """Verify the CRC class when not using xorOut""" + crc = Crc(g32) + + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0xFFFFFFFF +xorOut = 0x00000000 +crcValue = 0xFFFFFFFF''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(crc.digest(), '\xff\xff\xff\xff') + self.assertEqual(crc.hexdigest(), 'FFFFFFFF') + + crc.update(self.msg) + self.assertEqual(crc.crcValue, 0xF7B400A7L) + self.assertEqual(crc.digest(), '\xf7\xb4\x00\xa7') + self.assertEqual(crc.hexdigest(), 'F7B400A7') + + # Verify the .copy() method + x = crc.copy() + self.assertTrue(x is not crc) + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0xFFFFFFFF +xorOut = 0x00000000 +crcValue = 0xF7B400A7''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(str(x), str_rep) + + def test_full_crc32_class(self): + """Verify the CRC class when using xorOut""" + + crc = Crc(g32, initCrc=0, xorOut= ~0L) + + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0x00000000 +xorOut = 0xFFFFFFFF +crcValue = 0x00000000''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(crc.digest(), '\x00\x00\x00\x00') + self.assertEqual(crc.hexdigest(), '00000000') + + crc.update(self.msg) + self.assertEqual(crc.crcValue, 0x84BFF58L) + self.assertEqual(crc.digest(), '\x08\x4b\xff\x58') + self.assertEqual(crc.hexdigest(), '084BFF58') + + # Verify the .copy() method + x = crc.copy() + self.assertTrue(x is not crc) + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0x00000000 +xorOut = 0xFFFFFFFF +crcValue = 0x084BFF58''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(str(x), str_rep) + + # Verify the .new() method + y = crc.new() + self.assertTrue(y is not crc) + self.assertTrue(y is not x) + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0x00000000 +xorOut = 0xFFFFFFFF +crcValue = 0x00000000''' + self.assertEqual(str(y), str_rep) + + +class PredefinedCrcTest(unittest.TestCase): + """Verify the predefined CRCs""" + + test_messages_for_known_answers = [ + '', # Test cases below depend on this first entry being the empty string. + 'T', + 'CatMouse987654321', + ] + + known_answers = [ + [ 'crc-aug-ccitt', (0x1D0F, 0xD6ED, 0x5637) ], + [ 'x-25', (0x0000, 0xE4D9, 0x0A91) ], + [ 'crc-32', (0x00000000, 0xBE047A60, 0x084BFF58) ], + ] + + def test_known_answers(self): + for crcfun_name, v in self.known_answers: + crcfun = mkPredefinedCrcFun(crcfun_name) + self.assertEqual(crcfun('',0), 0, "Wrong answer for CRC '%s', input ''" % crcfun_name) + for i, msg in enumerate(self.test_messages_for_known_answers): + self.assertEqual(crcfun(msg), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg)) + self.assertEqual(crcfun(msg[4:], crcfun(msg[:4])), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg)) + self.assertEqual(crcfun(msg[-1:], crcfun(msg[:-1])), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg)) + + def test_class_with_known_answers(self): + for crcfun_name, v in self.known_answers: + for i, msg in enumerate(self.test_messages_for_known_answers): + crc1 = PredefinedCrc(crcfun_name) + crc1.update(msg) + self.assertEqual(crc1.crcValue, v[i], "Wrong answer for crc1 %s, input '%s'" % (crcfun_name,msg)) + + crc2 = crc1.new() + # Check that crc1 maintains its same value, after .new() call. + self.assertEqual(crc1.crcValue, v[i], "Wrong state for crc1 %s, input '%s'" % (crcfun_name,msg)) + # Check that the new class instance created by .new() contains the initialisation value. + # This depends on the first string in self.test_messages_for_known_answers being + # the empty string. + self.assertEqual(crc2.crcValue, v[0], "Wrong state for crc2 %s, input '%s'" % (crcfun_name,msg)) + + crc2.update(msg) + # Check that crc1 maintains its same value, after crc2 has called .update() + self.assertEqual(crc1.crcValue, v[i], "Wrong state for crc1 %s, input '%s'" % (crcfun_name,msg)) + # Check that crc2 contains the right value after calling .update() + self.assertEqual(crc2.crcValue, v[i], "Wrong state for crc2 %s, input '%s'" % (crcfun_name,msg)) + + def test_function_predefined_table(self): + for table_entry in _predefined_crc_definitions: + # Check predefined function + crc_func = mkPredefinedCrcFun(table_entry['name']) + calc_value = crc_func("123456789") + self.assertEqual(calc_value, table_entry['check'], "Wrong answer for CRC '%s'" % table_entry['name']) + + def test_class_predefined_table(self): + for table_entry in _predefined_crc_definitions: + # Check predefined class + crc1 = PredefinedCrc(table_entry['name']) + crc1.update("123456789") + self.assertEqual(crc1.crcValue, table_entry['check'], "Wrong answer for CRC '%s'" % table_entry['name']) + + +def runtests(): + print "Using extension:", _usingExtension + print + unittest.main() + + +if __name__ == '__main__': + runtests()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python3/__init__.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,24 @@ +# -*- coding: utf-8 -*- +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#-----------------------------------------------------------------------------
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python3/_crcfunpy.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,107 @@ +#----------------------------------------------------------------------------- +# Low level CRC functions for use by crcmod. This version is implemented in +# Python for a couple of reasons. 1) Provide a reference implememtation. +# 2) Provide a version that can be used on systems where a C compiler is not +# available for building extension modules. +# +# Copyright (c) 2009 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- + +def _get_buffer_view(in_obj): + if isinstance(in_obj, str): + raise TypeError('Unicode-objects must be encoded before calculating a CRC') + mv = memoryview(in_obj) + if mv.ndim > 1: + raise BufferError('Buffer must be single dimension') + return mv + + +def _crc8(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFF + for x in mv.tobytes(): + crc = table[x ^ crc] + return crc + +def _crc8r(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFF + for x in mv.tobytes(): + crc = table[x ^ crc] + return crc + +def _crc16(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFF + for x in mv.tobytes(): + crc = table[x ^ ((crc>>8) & 0xFF)] ^ ((crc << 8) & 0xFF00) + return crc + +def _crc16r(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFF + for x in mv.tobytes(): + crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8) + return crc + +def _crc24(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFFFF + for x in mv.tobytes(): + crc = table[x ^ (crc>>16 & 0xFF)] ^ ((crc << 8) & 0xFFFF00) + return crc + +def _crc24r(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFFFF + for x in mv.tobytes(): + crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8) + return crc + +def _crc32(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFFFFFF + for x in mv.tobytes(): + crc = table[x ^ ((crc>>24) & 0xFF)] ^ ((crc << 8) & 0xFFFFFF00) + return crc + +def _crc32r(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFFFFFF + for x in mv.tobytes(): + crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8) + return crc + +def _crc64(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFFFFFFFFFFFFFF + for x in mv.tobytes(): + crc = table[x ^ ((crc>>56) & 0xFF)] ^ ((crc << 8) & 0xFFFFFFFFFFFFFF00) + return crc + +def _crc64r(data, crc, table): + mv = _get_buffer_view(data) + crc = crc & 0xFFFFFFFFFFFFFFFF + for x in mv.tobytes(): + crc = table[x ^ (crc & 0xFF)] ^ (crc >> 8) + return crc +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python3/crcmod.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,457 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- +'''crcmod is a Python module for gererating objects that compute the Cyclic +Redundancy Check. Any 8, 16, 24, 32, or 64 bit polynomial can be used. + +The following are the public components of this module. + +Crc -- a class that creates instances providing the same interface as the +algorithms in the hashlib module in the Python standard library. These +instances also provide a method for generating a C/C++ function to compute +the CRC. + +mkCrcFun -- create a Python function to compute the CRC using the specified +polynomial and initial value. This provides a much simpler interface if +all you need is a function for CRC calculation. +''' + +__all__ = '''mkCrcFun Crc'''.split() + +# Select the appropriate set of low-level CRC functions for this installation. +# If the extension module was not built, drop back to the Python implementation +# even though it is significantly slower. +try: + from . import _crcfunext as _crcfun + _usingExtension = True +except ImportError: + from . import _crcfunpy as _crcfun + _usingExtension = False + +import sys, struct + +#----------------------------------------------------------------------------- +class Crc: + '''Compute a Cyclic Redundancy Check (CRC) using the specified polynomial. + + Instances of this class have the same interface as the algorithms in the + hashlib module in the Python standard library. See the documentation of + this module for examples of how to use a Crc instance. + + The string representation of a Crc instance identifies the polynomial, + initial value, XOR out value, and the current CRC value. The print + statement can be used to output this information. + + If you need to generate a C/C++ function for use in another application, + use the generateCode method. If you need to generate code for another + language, subclass Crc and override the generateCode method. + + The following are the parameters supplied to the constructor. + + poly -- The generator polynomial to use in calculating the CRC. The value + is specified as a Python integer. The bits in this integer are the + coefficients of the polynomial. The only polynomials allowed are those + that generate 8, 16, 24, 32, or 64 bit CRCs. + + initCrc -- Initial value used to start the CRC calculation. This initial + value should be the initial shift register value XORed with the final XOR + value. That is equivalent to the CRC result the algorithm should return for + a zero-length string. Defaults to all bits set because that starting value + will take leading zero bytes into account. Starting with zero will ignore + all leading zero bytes. + + rev -- A flag that selects a bit reversed algorithm when True. Defaults to + True because the bit reversed algorithms are more efficient. + + xorOut -- Final value to XOR with the calculated CRC value. Used by some + CRC algorithms. Defaults to zero. + ''' + def __init__(self, poly, initCrc=~0, rev=True, xorOut=0, initialize=True): + if not initialize: + # Don't want to perform the initialization when using new or copy + # to create a new instance. + return + + (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut) + self.digest_size = sizeBits//8 + self.initCrc = initCrc + self.xorOut = xorOut + + self.poly = poly + self.reverse = rev + + (crcfun, table) = _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut) + self._crc = crcfun + self.table = table + + self.crcValue = self.initCrc + + def __str__(self): + lst = [] + lst.append('poly = 0x%X' % self.poly) + lst.append('reverse = %s' % self.reverse) + fmt = '0x%%0%dX' % (self.digest_size*2) + lst.append('initCrc = %s' % (fmt % self.initCrc)) + lst.append('xorOut = %s' % (fmt % self.xorOut)) + lst.append('crcValue = %s' % (fmt % self.crcValue)) + return '\n'.join(lst) + + def new(self, arg=None): + '''Create a new instance of the Crc class initialized to the same + values as the original instance. The current CRC is set to the initial + value. If a string is provided in the optional arg parameter, it is + passed to the update method. + ''' + n = Crc(poly=None, initialize=False) + n._crc = self._crc + n.digest_size = self.digest_size + n.initCrc = self.initCrc + n.xorOut = self.xorOut + n.table = self.table + n.crcValue = self.initCrc + n.reverse = self.reverse + n.poly = self.poly + if arg is not None: + n.update(arg) + return n + + def copy(self): + '''Create a new instance of the Crc class initialized to the same + values as the original instance. The current CRC is set to the current + value. This allows multiple CRC calculations using a common initial + string. + ''' + c = self.new() + c.crcValue = self.crcValue + return c + + def update(self, data): + '''Update the current CRC value using the string specified as the data + parameter. + ''' + self.crcValue = self._crc(data, self.crcValue) + + def digest(self): + '''Return the current CRC value as a string of bytes. The length of + this string is specified in the digest_size attribute. + ''' + n = self.digest_size + crc = self.crcValue + lst = [] + while n > 0: + lst.append(crc & 0xFF) + crc = crc >> 8 + n -= 1 + lst.reverse() + return bytes(lst) + + def hexdigest(self): + '''Return the current CRC value as a string of hex digits. The length + of this string is twice the digest_size attribute. + ''' + n = self.digest_size + crc = self.crcValue + lst = [] + while n > 0: + lst.append('%02X' % (crc & 0xFF)) + crc = crc >> 8 + n -= 1 + lst.reverse() + return ''.join(lst) + + def generateCode(self, functionName, out, dataType=None, crcType=None): + '''Generate a C/C++ function. + + functionName -- String specifying the name of the function. + + out -- An open file-like object with a write method. This specifies + where the generated code is written. + + dataType -- An optional parameter specifying the data type of the input + data to the function. Defaults to UINT8. + + crcType -- An optional parameter specifying the data type of the CRC + value. Defaults to one of UINT8, UINT16, UINT32, or UINT64 depending + on the size of the CRC value. + ''' + if dataType is None: + dataType = 'UINT8' + + if crcType is None: + size = 8*self.digest_size + if size == 24: + size = 32 + crcType = 'UINT%d' % size + + if self.digest_size == 1: + # Both 8-bit CRC algorithms are the same + crcAlgor = 'table[*data ^ (%s)crc]' + elif self.reverse: + # The bit reverse algorithms are all the same except for the data + # type of the crc variable which is specified elsewhere. + crcAlgor = 'table[*data ^ (%s)crc] ^ (crc >> 8)' + else: + # The forward CRC algorithms larger than 8 bits have an extra shift + # operation to get the high byte. + shift = 8*(self.digest_size - 1) + crcAlgor = 'table[*data ^ (%%s)(crc >> %d)] ^ (crc << 8)' % shift + + fmt = '0x%%0%dX' % (2*self.digest_size) + if self.digest_size <= 4: + fmt = fmt + 'U,' + else: + # Need the long long type identifier to keep gcc from complaining. + fmt = fmt + 'ULL,' + + # Select the number of entries per row in the output code. + n = {1:8, 2:8, 3:4, 4:4, 8:2}[self.digest_size] + + lst = [] + for i, val in enumerate(self.table): + if (i % n) == 0: + lst.append('\n ') + lst.append(fmt % val) + + poly = 'polynomial: 0x%X' % self.poly + if self.reverse: + poly = poly + ', bit reverse algorithm' + + if self.xorOut: + # Need to remove the comma from the format. + preCondition = '\n crc = crc ^ %s;' % (fmt[:-1] % self.xorOut) + postCondition = preCondition + else: + preCondition = '' + postCondition = '' + + if self.digest_size == 3: + # The 24-bit CRC needs to be conditioned so that only 24-bits are + # used from the 32-bit variable. + if self.reverse: + preCondition += '\n crc = crc & 0xFFFFFFU;' + else: + postCondition += '\n crc = crc & 0xFFFFFFU;' + + + parms = { + 'dataType' : dataType, + 'crcType' : crcType, + 'name' : functionName, + 'crcAlgor' : crcAlgor % dataType, + 'crcTable' : ''.join(lst), + 'poly' : poly, + 'preCondition' : preCondition, + 'postCondition' : postCondition, + } + out.write(_codeTemplate % parms) + +#----------------------------------------------------------------------------- +def mkCrcFun(poly, initCrc=~0, rev=True, xorOut=0): + '''Return a function that computes the CRC using the specified polynomial. + + poly -- integer representation of the generator polynomial + initCrc -- default initial CRC value + rev -- when true, indicates that the data is processed bit reversed. + xorOut -- the final XOR value + + The returned function has the following user interface + def crcfun(data, crc=initCrc): + ''' + + # First we must verify the params + (sizeBits, initCrc, xorOut) = _verifyParams(poly, initCrc, xorOut) + # Make the function (and table), return the function + return _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut)[0] + +#----------------------------------------------------------------------------- +# Naming convention: +# All function names ending with r are bit reverse variants of the ones +# without the r. + +#----------------------------------------------------------------------------- +# Check the polynomial to make sure that it is acceptable and return the number +# of bits in the CRC. + +def _verifyPoly(poly): + msg = 'The degree of the polynomial must be 8, 16, 24, 32 or 64' + for n in (8,16,24,32,64): + low = 1<<n + high = low*2 + if low <= poly < high: + return n + raise ValueError(msg) + +#----------------------------------------------------------------------------- +# Bit reverse the input value. + +def _bitrev(x, n): + y = 0 + for i in range(n): + y = (y << 1) | (x & 1) + x = x >> 1 + return y + +#----------------------------------------------------------------------------- +# The following functions compute the CRC for a single byte. These are used +# to build up the tables needed in the CRC algorithm. Assumes the high order +# bit of the polynomial has been stripped off. + +def _bytecrc(crc, poly, n): + mask = 1<<(n-1) + for i in range(8): + if crc & mask: + crc = (crc << 1) ^ poly + else: + crc = crc << 1 + mask = (1<<n) - 1 + crc = crc & mask + return crc + +def _bytecrc_r(crc, poly, n): + for i in range(8): + if crc & 1: + crc = (crc >> 1) ^ poly + else: + crc = crc >> 1 + mask = (1<<n) - 1 + crc = crc & mask + return crc + +#----------------------------------------------------------------------------- +# The following functions compute the table needed to compute the CRC. The +# table is returned as a list. Note that the array module does not support +# 64-bit integers on a 32-bit architecture as of Python 2.3. +# +# These routines assume that the polynomial and the number of bits in the CRC +# have been checked for validity by the caller. + +def _mkTable(poly, n): + mask = (1<<n) - 1 + poly = poly & mask + table = [_bytecrc(i<<(n-8),poly,n) for i in range(256)] + return table + +def _mkTable_r(poly, n): + mask = (1<<n) - 1 + poly = _bitrev(poly & mask, n) + table = [_bytecrc_r(i,poly,n) for i in range(256)] + return table + +#----------------------------------------------------------------------------- +# Map the CRC size onto the functions that handle these sizes. + +_sizeMap = { + 8 : [_crcfun._crc8, _crcfun._crc8r], + 16 : [_crcfun._crc16, _crcfun._crc16r], + 24 : [_crcfun._crc24, _crcfun._crc24r], + 32 : [_crcfun._crc32, _crcfun._crc32r], + 64 : [_crcfun._crc64, _crcfun._crc64r], +} + +#----------------------------------------------------------------------------- +# Build a mapping of size to struct module type code. This table is +# constructed dynamically so that it has the best chance of picking the best +# code to use for the platform we are running on. This should properly adapt +# to 32 and 64 bit machines. + +_sizeToTypeCode = {} + +for typeCode in 'B H I L Q'.split(): + size = {1:8, 2:16, 4:32, 8:64}.get(struct.calcsize(typeCode),None) + if size is not None and size not in _sizeToTypeCode: + _sizeToTypeCode[size] = '256%s' % typeCode + +_sizeToTypeCode[24] = _sizeToTypeCode[32] + +del typeCode, size + +#----------------------------------------------------------------------------- +# The following function validates the parameters of the CRC, namely, +# poly, and initial/final XOR values. +# It returns the size of the CRC (in bits), and "sanitized" initial/final XOR values. + +def _verifyParams(poly, initCrc, xorOut): + sizeBits = _verifyPoly(poly) + + mask = (1<<sizeBits) - 1 + + # Adjust the initial CRC to the correct data type (unsigned value). + initCrc = initCrc & mask + + # Similar for XOR-out value. + xorOut = xorOut & mask + + return (sizeBits, initCrc, xorOut) + +#----------------------------------------------------------------------------- +# The following function returns a Python function to compute the CRC. +# +# It must be passed parameters that are already verified & sanitized by +# _verifyParams(). +# +# The returned function calls a low level function that is written in C if the +# extension module could be loaded. Otherwise, a Python implementation is +# used. +# +# In addition to this function, a list containing the CRC table is returned. + +def _mkCrcFun(poly, sizeBits, initCrc, rev, xorOut): + if rev: + tableList = _mkTable_r(poly, sizeBits) + _fun = _sizeMap[sizeBits][1] + else: + tableList = _mkTable(poly, sizeBits) + _fun = _sizeMap[sizeBits][0] + + _table = tableList + if _usingExtension: + _table = struct.pack(_sizeToTypeCode[sizeBits], *tableList) + + if xorOut == 0: + def crcfun(data, crc=initCrc, table=_table, fun=_fun): + return fun(data, crc, table) + else: + def crcfun(data, crc=initCrc, table=_table, fun=_fun): + return xorOut ^ fun(data, xorOut ^ crc, table) + + return crcfun, tableList + +#----------------------------------------------------------------------------- +_codeTemplate = '''// Automatically generated CRC function +// %(poly)s +%(crcType)s +%(name)s(%(dataType)s *data, int len, %(crcType)s crc) +{ + static const %(crcType)s table[256] = {%(crcTable)s + }; + %(preCondition)s + while (len > 0) + { + crc = %(crcAlgor)s; + data++; + len--; + }%(postCondition)s + return crc; +} +''' +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python3/predefined.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,178 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- +''' +crcmod.predefined defines some well-known CRC algorithms. + +To use it, e.g.: + import crcmod.predefined + + crc32func = crcmod.predefined.mkPredefinedCrcFun("crc-32") + crc32class = crcmod.predefined.PredefinedCrc("crc-32") + +crcmod.predefined.Crc is an alias for crcmod.predefined.PredefinedCrc +But if doing 'from crc.predefined import *', only PredefinedCrc is imported. +''' + +# local imports +from . import crcmod + +__all__ = [ + 'PredefinedCrc', + 'mkPredefinedCrcFun', +] + +REVERSE = True +NON_REVERSE = False + +# +# The following table defines the parameters of well-known CRC algorithms. +# The "Check" value is the CRC for the ASCII byte sequence b"123456789". It +# can be used for unit tests. +# +# See also: +# - https://en.wikipedia.org/wiki/Cyclic_redundancy_check +# - https://reveng.sourceforge.io/crc-catalogue/all.htm +# - http://users.ece.cmu.edu/~koopman/crc/index.html +# - https://pycrc.org/models.html +# - https://github.com/marzooqy/anycrc +# +_crc_definitions_table = [ +# Name Identifier-name, Poly Reverse Init-value XOR-out Check + [ 'crc-8', 'Crc8', 0x107, NON_REVERSE, 0x00, 0x00, 0xF4, ], + [ 'crc-8-darc', 'Crc8Darc', 0x139, REVERSE, 0x00, 0x00, 0x15, ], + [ 'crc-8-i-code', 'Crc8ICode', 0x11D, NON_REVERSE, 0xFD, 0x00, 0x7E, ], + [ 'crc-8-itu', 'Crc8Itu', 0x107, NON_REVERSE, 0x55, 0x55, 0xA1, ], + [ 'crc-8-maxim', 'Crc8Maxim', 0x131, REVERSE, 0x00, 0x00, 0xA1, ], + [ 'crc-8-rohc', 'Crc8Rohc', 0x107, REVERSE, 0xFF, 0x00, 0xD0, ], + [ 'crc-8-wcdma', 'Crc8Wcdma', 0x19B, REVERSE, 0x00, 0x00, 0x25, ], + + [ 'crc-16', 'Crc16', 0x18005, REVERSE, 0x0000, 0x0000, 0xBB3D, ], + [ 'crc-16-buypass', 'Crc16Buypass', 0x18005, NON_REVERSE, 0x0000, 0x0000, 0xFEE8, ], + [ 'crc-16-dds-110', 'Crc16Dds110', 0x18005, NON_REVERSE, 0x800D, 0x0000, 0x9ECF, ], + [ 'crc-16-dect', 'Crc16Dect', 0x10589, NON_REVERSE, 0x0001, 0x0001, 0x007E, ], + [ 'crc-16-dnp', 'Crc16Dnp', 0x13D65, REVERSE, 0xFFFF, 0xFFFF, 0xEA82, ], + [ 'crc-16-en-13757', 'Crc16En13757', 0x13D65, NON_REVERSE, 0xFFFF, 0xFFFF, 0xC2B7, ], + [ 'crc-16-genibus', 'Crc16Genibus', 0x11021, NON_REVERSE, 0x0000, 0xFFFF, 0xD64E, ], + [ 'crc-16-maxim', 'Crc16Maxim', 0x18005, REVERSE, 0xFFFF, 0xFFFF, 0x44C2, ], + [ 'crc-16-mcrf4xx', 'Crc16Mcrf4xx', 0x11021, REVERSE, 0xFFFF, 0x0000, 0x6F91, ], + [ 'crc-16-riello', 'Crc16Riello', 0x11021, REVERSE, 0x554D, 0x0000, 0x63D0, ], + [ 'crc-16-t10-dif', 'Crc16T10Dif', 0x18BB7, NON_REVERSE, 0x0000, 0x0000, 0xD0DB, ], + [ 'crc-16-teledisk', 'Crc16Teledisk', 0x1A097, NON_REVERSE, 0x0000, 0x0000, 0x0FB3, ], + [ 'crc-16-usb', 'Crc16Usb', 0x18005, REVERSE, 0x0000, 0xFFFF, 0xB4C8, ], + [ 'x-25', 'CrcX25', 0x11021, REVERSE, 0x0000, 0xFFFF, 0x906E, ], + [ 'xmodem', 'CrcXmodem', 0x11021, NON_REVERSE, 0x0000, 0x0000, 0x31C3, ], + [ 'modbus', 'CrcModbus', 0x18005, REVERSE, 0xFFFF, 0x0000, 0x4B37, ], + + # Note definitions of CCITT are disputable. See: + # http://homepages.tesco.net/~rainstorm/crc-catalogue.htm + # http://web.archive.org/web/20071229021252/http://www.joegeluso.com/software/articles/ccitt.htm + [ 'kermit', 'CrcKermit', 0x11021, REVERSE, 0x0000, 0x0000, 0x2189, ], + [ 'crc-ccitt-false', 'CrcCcittFalse', 0x11021, NON_REVERSE, 0xFFFF, 0x0000, 0x29B1, ], + [ 'crc-aug-ccitt', 'CrcAugCcitt', 0x11021, NON_REVERSE, 0x1D0F, 0x0000, 0xE5CC, ], + + [ 'crc-24', 'Crc24', 0x1864CFB, NON_REVERSE, 0xB704CE, 0x000000, 0x21CF02, ], + [ 'crc-24-flexray-a', 'Crc24FlexrayA', 0x15D6DCB, NON_REVERSE, 0xFEDCBA, 0x000000, 0x7979BD, ], + [ 'crc-24-flexray-b', 'Crc24FlexrayB', 0x15D6DCB, NON_REVERSE, 0xABCDEF, 0x000000, 0x1F23B8, ], + + [ 'crc-32', 'Crc32', 0x104C11DB7, REVERSE, 0x00000000, 0xFFFFFFFF, 0xCBF43926, ], + [ 'crc-32-bzip2', 'Crc32Bzip2', 0x104C11DB7, NON_REVERSE, 0x00000000, 0xFFFFFFFF, 0xFC891918, ], + [ 'crc-32c', 'Crc32C', 0x11EDC6F41, REVERSE, 0x00000000, 0xFFFFFFFF, 0xE3069283, ], + [ 'crc-32d', 'Crc32D', 0x1A833982B, REVERSE, 0x00000000, 0xFFFFFFFF, 0x87315576, ], + [ 'crc-32-mpeg', 'Crc32Mpeg', 0x104C11DB7, NON_REVERSE, 0xFFFFFFFF, 0x00000000, 0x0376E6E7, ], + [ 'posix', 'CrcPosix', 0x104C11DB7, NON_REVERSE, 0xFFFFFFFF, 0xFFFFFFFF, 0x765E7680, ], + [ 'crc-32q', 'Crc32Q', 0x1814141AB, NON_REVERSE, 0x00000000, 0x00000000, 0x3010BF7F, ], + [ 'jamcrc', 'CrcJamCrc', 0x104C11DB7, REVERSE, 0xFFFFFFFF, 0x00000000, 0x340BC6D9, ], + [ 'xfer', 'CrcXfer', 0x1000000AF, NON_REVERSE, 0x00000000, 0x00000000, 0xBD0BE338, ], + +# 64-bit +# Name Identifier-name, Poly Reverse Init-value XOR-out Check + [ 'crc-64', 'Crc64', 0x1000000000000001B, REVERSE, 0x0000000000000000, 0x0000000000000000, 0x46A5A9388A5BEFFE, ], # ISO POLY + # See https://lwn.net/Articles/976030/ (MCRC64, used as CRC-64 in the Linux kernel) + [ 'crc-64-2', 'Crc64_2', 0x1000000000000001B, NON_REVERSE, 0x0000000000000000, 0x0000000000000000, 0xE4FFBEA588933790, ], # fag, ISO POLY + [ 'crc-64-go', 'Crc64Go', 0x1000000000000001B, REVERSE, 0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0xB90956C775A41001, ], # fag, ISO POLY + [ 'crc-64-ecma', 'Crc64Ecma', 0x142F0E1EBA9EA3693, NON_REVERSE, 0x0000000000000000, 0x0000000000000000, 0x6C40DF5F0B497347, ], # fag + [ 'crc-64-we', 'Crc64We', 0x142F0E1EBA9EA3693, NON_REVERSE, 0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0x62EC59E3F1A4F00A, ], + [ 'crc-64-jones', 'Crc64Jones', 0x1AD93D23594C935A9, REVERSE, 0xFFFFFFFFFFFFFFFF, 0x0000000000000000, 0xCAA717168609F281, ], + [ 'crc-64-xz', 'Crc64Xz', 0x142F0E1EBA9EA3693, REVERSE, 0x0000000000000000, 0xFFFFFFFFFFFFFFFF, 0x995DC9BBDF1939FA, ], # fag, ECMA POLY + [ 'crc-64-redis', 'Crc64Redis', 0x1AD93D23594C935A9, REVERSE, 0x0000000000000000, 0x0000000000000000, 0xE9C6D914C4b8D9CA, ], # fag +] + + +def _simplify_name(name): + """ + Reduce CRC definition name to a simplified form: + * lowercase + * dashes removed + * spaces removed + * any initial "CRC" string removed + """ + name = name.lower() + name = name.replace('-', '') + name = name.replace(' ', '') + if name.startswith('crc'): + name = name[len('crc'):] + return name + + +_crc_definitions_by_name = {} +_crc_definitions_by_identifier = {} +_crc_definitions = [] + +_crc_table_headings = [ 'name', 'identifier', 'poly', 'reverse', 'init', 'xor_out', 'check' ] + +for table_entry in _crc_definitions_table: + crc_definition = dict(zip(_crc_table_headings, table_entry)) + _crc_definitions.append(crc_definition) + name = _simplify_name(table_entry[0]) + if name in _crc_definitions_by_name: + raise Exception("Duplicate entry for '{0}' in CRC table".format(name)) + _crc_definitions_by_name[name] = crc_definition + _crc_definitions_by_identifier[table_entry[1]] = crc_definition + + +def _get_definition_by_name(crc_name): + definition = _crc_definitions_by_name.get(_simplify_name(crc_name), None) + if not definition: + definition = _crc_definitions_by_identifier.get(crc_name, None) + if not definition: + raise KeyError("Unkown CRC name '{0}'".format(crc_name)) + return definition + + +class PredefinedCrc(crcmod.Crc): + def __init__(self, crc_name): + definition = _get_definition_by_name(crc_name) + super().__init__(poly=definition['poly'], initCrc=definition['init'], rev=definition['reverse'], xorOut=definition['xor_out']) + + +# crcmod.predefined.Crc is an alias for crcmod.predefined.PredefinedCrc +Crc = PredefinedCrc + + +def mkPredefinedCrcFun(crc_name): + definition = _get_definition_by_name(crc_name) + return crcmod.mkCrcFun(poly=definition['poly'], initCrc=definition['init'], rev=definition['reverse'], xorOut=definition['xor_out']) + + +# crcmod.predefined.mkCrcFun is an alias for crcmod.predefined.mkPredefinedCrcFun +mkCrcFun = mkPredefinedCrcFun
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/python3/test.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,541 @@ +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- +'''Unit tests for crcmod functionality''' + + +import unittest + +from array import array +import binascii + +from .crcmod import mkCrcFun, Crc +from .crcmod import _usingExtension +from .predefined import PredefinedCrc +from .predefined import mkPredefinedCrcFun +from .predefined import _crc_definitions as _predefined_crc_definitions + + +#----------------------------------------------------------------------------- +# This polynomial was chosen because it is the product of two irreducible +# polynomials. +# g8 = (x^7+x+1)*(x+1) +g8 = 0x185 + +#----------------------------------------------------------------------------- +# The following reproduces all of the entries in the Numerical Recipes table. +# This is the standard CCITT polynomial. +g16 = 0x11021 + +#----------------------------------------------------------------------------- +g24 = 0x15D6DCB + +#----------------------------------------------------------------------------- +# This is the standard AUTODIN-II polynomial which appears to be used in a +# wide variety of standards and applications. +g32 = 0x104C11DB7 + + +#----------------------------------------------------------------------------- +# I was able to locate a couple of 64-bit polynomials on the web. To make it +# easier to input the representation, define a function that builds a +# polynomial from a list of the bits that need to be turned on. + +def polyFromBits(bits): + p = 0 + for n in bits: + p = p | (1 << n) + return p + +# The following is from the paper "An Improved 64-bit Cyclic Redundancy Check +# for Protein Sequences" by David T. Jones + +g64a = polyFromBits([64, 63, 61, 59, 58, 56, 55, 52, 49, 48, 47, 46, 44, 41, + 37, 36, 34, 32, 31, 28, 26, 23, 22, 19, 16, 13, 12, 10, 9, 6, 4, + 3, 0]) + +# The following is from Standard ECMA-182 "Data Interchange on 12,7 mm 48-Track +# Magnetic Tape Cartridges -DLT1 Format-", December 1992. + +g64b = polyFromBits([64, 62, 57, 55, 54, 53, 52, 47, 46, 45, 40, 39, 38, 37, + 35, 33, 32, 31, 29, 27, 24, 23, 22, 21, 19, 17, 13, 12, 10, 9, 7, + 4, 1, 0]) + +#----------------------------------------------------------------------------- +# This class is used to check the CRC calculations against a direct +# implementation using polynomial division. + +class poly: + '''Class implementing polynomials over the field of integers mod 2''' + def __init__(self,p): + p = int(p) + if p < 0: raise ValueError('invalid polynomial') + self.p = p + + def __int__(self): + return self.p + + def __eq__(self,other): + return self.p == other.p + + def __ne__(self,other): + return self.p != other.p + + # To allow sorting of polynomials, use their long integer form for + # comparison + def __cmp__(self,other): + return cmp(self.p, other.p) + + def __bool__(self): + return self.p != 0 + + def __neg__(self): + return self # These polynomials are their own inverse under addition + + def __invert__(self): + n = max(self.deg() + 1, 1) + x = (1 << n) - 1 + return poly(self.p ^ x) + + def __add__(self,other): + return poly(self.p ^ other.p) + + def __sub__(self,other): + return poly(self.p ^ other.p) + + def __mul__(self,other): + a = self.p + b = other.p + if a == 0 or b == 0: return poly(0) + x = 0 + while b: + if b&1: + x = x ^ a + a = a<<1 + b = b>>1 + return poly(x) + + def __divmod__(self,other): + u = self.p + m = self.deg() + v = other.p + n = other.deg() + if v == 0: raise ZeroDivisionError('polynomial division by zero') + if n == 0: return (self,poly(0)) + if m < n: return (poly(0),self) + k = m-n + a = 1 << m + v = v << k + q = 0 + while k > 0: + if a & u: + u = u ^ v + q = q | 1 + q = q << 1 + a = a >> 1 + v = v >> 1 + k -= 1 + if a & u: + u = u ^ v + q = q | 1 + return (poly(q),poly(u)) + + def __div__(self,other): + return self.__divmod__(other)[0] + + def __mod__(self,other): + return self.__divmod__(other)[1] + + def __repr__(self): + return 'poly(0x%XL)' % self.p + + def __str__(self): + p = self.p + if p == 0: return '0' + lst = { 0:[], 1:['1'], 2:['x'], 3:['1','x'] }[p&3] + p = p>>2 + n = 2 + while p: + if p&1: lst.append('x^%d' % n) + p = p>>1 + n += 1 + lst.reverse() + return '+'.join(lst) + + def deg(self): + '''return the degree of the polynomial''' + a = self.p + if a == 0: return -1 + n = 0 + while a >= 0x10000: + n += 16 + a = a >> 16 + a = int(a) + while a > 1: + n += 1 + a = a >> 1 + return n + +#----------------------------------------------------------------------------- +# The following functions compute the CRC using direct polynomial division. +# These functions are checked against the result of the table driven +# algorithms. + +g8p = poly(g8) +x8p = poly(1<<8) +def crc8p(d): + p = 0 + for i in d: + p = p*256 + i + p = poly(p) + return int(p*x8p%g8p) + +g16p = poly(g16) +x16p = poly(1<<16) +def crc16p(d): + p = 0 + for i in d: + p = p*256 + i + p = poly(p) + return int(p*x16p%g16p) + +g24p = poly(g24) +x24p = poly(1<<24) +def crc24p(d): + p = 0 + for i in d: + p = p*256 + i + p = poly(p) + return int(p*x24p%g24p) + +g32p = poly(g32) +x32p = poly(1<<32) +def crc32p(d): + p = 0 + for i in d: + p = p*256 + i + p = poly(p) + return int(p*x32p%g32p) + +g64ap = poly(g64a) +x64p = poly(1<<64) +def crc64ap(d): + p = 0 + for i in d: + p = p*256 + i + p = poly(p) + return int(p*x64p%g64ap) + +g64bp = poly(g64b) +def crc64bp(d): + p = 0 + for i in d: + p = p*256 + i + p = poly(p) + return int(p*x64p%g64bp) + + +class KnownAnswerTests(unittest.TestCase): + test_messages = [ + b'T', + b'CatMouse987654321', + ] + + known_answers = [ + [ (g8,0,0), (0xFE, 0x9D) ], + [ (g8,-1,1), (0x4F, 0x9B) ], + [ (g8,0,1), (0xFE, 0x62) ], + [ (g16,0,0), (0x1A71, 0xE556) ], + [ (g16,-1,1), (0x1B26, 0xF56E) ], + [ (g16,0,1), (0x14A1, 0xC28D) ], + [ (g24,0,0), (0xBCC49D, 0xC4B507) ], + [ (g24,-1,1), (0x59BD0E, 0x0AAA37) ], + [ (g24,0,1), (0xD52B0F, 0x1523AB) ], + [ (g32,0,0), (0x6B93DDDB, 0x12DCA0F4) ], + [ (g32,0xFFFFFFFF,1), (0x41FB859F, 0xF7B400A7) ], + [ (g32,0,1), (0x6C0695ED, 0xC1A40EE5) ], + [ (g32,0,1,0xFFFFFFFF), (0xBE047A60, 0x084BFF58) ], + ] + + def test_known_answers(self): + for crcfun_params, v in self.known_answers: + crcfun = mkCrcFun(*crcfun_params) + self.assertEqual(crcfun(b'',0), 0, "Wrong answer for CRC parameters %s, input ''" % (crcfun_params,)) + for i, msg in enumerate(self.test_messages): + self.assertEqual(crcfun(msg), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg)) + self.assertEqual(crcfun(msg[4:], crcfun(msg[:4])), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg)) + self.assertEqual(crcfun(msg[-1:], crcfun(msg[:-1])), v[i], "Wrong answer for CRC parameters %s, input '%s'" % (crcfun_params,msg)) + + +class CompareReferenceCrcTest(unittest.TestCase): + test_messages = [ + b'', + b'T', + b'123456789', + b'CatMouse987654321', + ] + + test_poly_crcs = [ + [ (g8,0,0), crc8p ], + [ (g16,0,0), crc16p ], + [ (g24,0,0), crc24p ], + [ (g32,0,0), crc32p ], + [ (g64a,0,0), crc64ap ], + [ (g64b,0,0), crc64bp ], + ] + + @staticmethod + def reference_crc32(d, crc=0): + """This function modifies the return value of binascii.crc32 + to be an unsigned 32-bit value. I.e. in the range 0 to 2**32-1.""" + # Work around the future warning on constants. + if crc > 0x7FFFFFFF: + x = int(crc & 0x7FFFFFFF) + crc = x | -2147483648 + x = binascii.crc32(d,crc) + return int(x) & 0xFFFFFFFF + + def test_compare_crc32(self): + """The binascii module has a 32-bit CRC function that is used in a wide range + of applications including the checksum used in the ZIP file format. + This test compares the CRC-32 implementation of this crcmod module to + that of binascii.crc32.""" + # The following function should produce the same result as + # self.reference_crc32 which is derived from binascii.crc32. + crc32 = mkCrcFun(g32,0,1,0xFFFFFFFF) + + for msg in self.test_messages: + self.assertEqual(crc32(msg), self.reference_crc32(msg)) + + def test_compare_poly(self): + """Compare various CRCs of this crcmod module to a pure + polynomial-based implementation.""" + for crcfun_params, crc_poly_fun in self.test_poly_crcs: + # The following function should produce the same result as + # the associated polynomial CRC function. + crcfun = mkCrcFun(*crcfun_params) + + for msg in self.test_messages: + self.assertEqual(crcfun(msg), crc_poly_fun(msg)) + + +class CrcClassTest(unittest.TestCase): + """Verify the Crc class""" + + msg = b'CatMouse987654321' + + def test_simple_crc32_class(self): + """Verify the CRC class when not using xorOut""" + crc = Crc(g32) + + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0xFFFFFFFF +xorOut = 0x00000000 +crcValue = 0xFFFFFFFF''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(crc.digest(), b'\xff\xff\xff\xff') + self.assertEqual(crc.hexdigest(), 'FFFFFFFF') + + crc.update(self.msg) + self.assertEqual(crc.crcValue, 0xF7B400A7) + self.assertEqual(crc.digest(), b'\xf7\xb4\x00\xa7') + self.assertEqual(crc.hexdigest(), 'F7B400A7') + + # Verify the .copy() method + x = crc.copy() + self.assertTrue(x is not crc) + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0xFFFFFFFF +xorOut = 0x00000000 +crcValue = 0xF7B400A7''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(str(x), str_rep) + + def test_full_crc32_class(self): + """Verify the CRC class when using xorOut""" + + crc = Crc(g32, initCrc=0, xorOut= ~0) + + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0x00000000 +xorOut = 0xFFFFFFFF +crcValue = 0x00000000''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(crc.digest(), b'\x00\x00\x00\x00') + self.assertEqual(crc.hexdigest(), '00000000') + + crc.update(self.msg) + self.assertEqual(crc.crcValue, 0x84BFF58) + self.assertEqual(crc.digest(), b'\x08\x4b\xff\x58') + self.assertEqual(crc.hexdigest(), '084BFF58') + + # Verify the .copy() method + x = crc.copy() + self.assertTrue(x is not crc) + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0x00000000 +xorOut = 0xFFFFFFFF +crcValue = 0x084BFF58''' + self.assertEqual(str(crc), str_rep) + self.assertEqual(str(x), str_rep) + + # Verify the .new() method + y = crc.new() + self.assertTrue(y is not crc) + self.assertTrue(y is not x) + str_rep = \ +'''poly = 0x104C11DB7 +reverse = True +initCrc = 0x00000000 +xorOut = 0xFFFFFFFF +crcValue = 0x00000000''' + self.assertEqual(str(y), str_rep) + + +class PredefinedCrcTest(unittest.TestCase): + """Verify the predefined CRCs""" + + test_messages_for_known_answers = [ + b'', # Test cases below depend on this first entry being the empty string. + b'T', + b'CatMouse987654321', + ] + + known_answers = [ + [ 'crc-aug-ccitt', (0x1D0F, 0xD6ED, 0x5637) ], + [ 'x-25', (0x0000, 0xE4D9, 0x0A91) ], + [ 'crc-32', (0x00000000, 0xBE047A60, 0x084BFF58) ], + ] + + def test_known_answers(self): + for crcfun_name, v in self.known_answers: + crcfun = mkPredefinedCrcFun(crcfun_name) + self.assertEqual(crcfun(b'',0), 0, "Wrong answer for CRC '%s', input ''" % crcfun_name) + for i, msg in enumerate(self.test_messages_for_known_answers): + self.assertEqual(crcfun(msg), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg)) + self.assertEqual(crcfun(msg[4:], crcfun(msg[:4])), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg)) + self.assertEqual(crcfun(msg[-1:], crcfun(msg[:-1])), v[i], "Wrong answer for CRC %s, input '%s'" % (crcfun_name,msg)) + + def test_class_with_known_answers(self): + for crcfun_name, v in self.known_answers: + for i, msg in enumerate(self.test_messages_for_known_answers): + crc1 = PredefinedCrc(crcfun_name) + crc1.update(msg) + self.assertEqual(crc1.crcValue, v[i], "Wrong answer for crc1 %s, input '%s'" % (crcfun_name,msg)) + + crc2 = crc1.new() + # Check that crc1 maintains its same value, after .new() call. + self.assertEqual(crc1.crcValue, v[i], "Wrong state for crc1 %s, input '%s'" % (crcfun_name,msg)) + # Check that the new class instance created by .new() contains the initialisation value. + # This depends on the first string in self.test_messages_for_known_answers being + # the empty string. + self.assertEqual(crc2.crcValue, v[0], "Wrong state for crc2 %s, input '%s'" % (crcfun_name,msg)) + + crc2.update(msg) + # Check that crc1 maintains its same value, after crc2 has called .update() + self.assertEqual(crc1.crcValue, v[i], "Wrong state for crc1 %s, input '%s'" % (crcfun_name,msg)) + # Check that crc2 contains the right value after calling .update() + self.assertEqual(crc2.crcValue, v[i], "Wrong state for crc2 %s, input '%s'" % (crcfun_name,msg)) + + def test_function_predefined_table(self): + for table_entry in _predefined_crc_definitions: + # Check predefined function + crc_func = mkPredefinedCrcFun(table_entry['name']) + calc_value = crc_func(b"123456789") + self.assertEqual(calc_value, table_entry['check'], "Wrong answer for CRC '%s'" % table_entry['name']) + + def test_class_predefined_table(self): + for table_entry in _predefined_crc_definitions: + # Check predefined class + crc1 = PredefinedCrc(table_entry['name']) + crc1.update(b"123456789") + self.assertEqual(crc1.crcValue, table_entry['check'], "Wrong answer for CRC '%s'" % table_entry['name']) + + +class InputTypesTest(unittest.TestCase): + """Check the various input types that CRC functions can accept.""" + + msg = b'CatMouse987654321' + + check_crc_names = [ + 'crc-aug-ccitt', + 'x-25', + 'crc-32', + ] + + array_check_types = [ + 'B', + 'H', + 'I', + 'L', + ] + + def test_bytearray_input(self): + """Test that bytearray inputs are accepted, as an example + of a type that implements the buffer protocol.""" + for crc_name in self.check_crc_names: + crcfun = mkPredefinedCrcFun(crc_name) + for i in range(len(self.msg) + 1): + test_msg = self.msg[:i] + bytes_answer = crcfun(test_msg) + bytearray_answer = crcfun(bytearray(test_msg)) + self.assertEqual(bytes_answer, bytearray_answer) + + def test_array_input(self): + """Test that array inputs are accepted, as an example + of a type that implements the buffer protocol.""" + for crc_name in self.check_crc_names: + crcfun = mkPredefinedCrcFun(crc_name) + for i in range(len(self.msg) + 1): + test_msg = self.msg[:i] + bytes_answer = crcfun(test_msg) + for array_type in self.array_check_types: + if i % array(array_type).itemsize == 0: + test_array = array(array_type, test_msg) + array_answer = crcfun(test_array) + self.assertEqual(bytes_answer, array_answer) + + def test_unicode_input(self): + """Test that Unicode input raises TypeError""" + for crc_name in self.check_crc_names: + crcfun = mkPredefinedCrcFun(crc_name) + with self.assertRaises(TypeError): + crcfun("123456789") + + +def runtests(): + print("Using extension:", _usingExtension) + print() + unittest.main() + + +if __name__ == '__main__': + runtests()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/crcmod/test.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,40 @@ +# -*- coding: utf-8; -*- +#----------------------------------------------------------------------------- +# Copyright (c) 2010 Raymond L. Buvel +# Copyright (c) 2010 Craig McQueen +# Copyright (c) 2025 Franz Glasner +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +#----------------------------------------------------------------------------- + +from __future__ import absolute_import + +import sys + +if sys.version_info[0] < 3: + from .python2.test import * +else: + from .python3.test import * + + +del sys + + +if __name__ == '__main__': + runtests()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/dos2unix.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,104 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Pure Python implementation of `dos2unix`. + +""" + +from __future__ import print_function, absolute_import + +from . import (__version__, __revision__) + + +__all__ = [] + + +import argparse +import io +import sys + + +def main(argv=None): + aparser = argparse.ArgumentParser( + description="Python implementation of dos2unix", + fromfile_prefix_chars='@') + aparser.add_argument( + "--version", "-V", action="version", + version="%s (rv:%s)" % (__version__, __revision__)) + aparser.add_argument( + "--keepdate", "-k", action="store_true", + help="Keep the date stamp of output file same as input file.") + aparser.add_argument( + "--oldfile", "-o", action="store_false", dest="newfile", default=False, + help="Old file mode. Convert the file and write output to it." + " The program defaults to run in this mode." + " Wildcard names may be used. ") + aparser.add_argument( + "--newfile", "-n", action="store_true", dest="newfile", default=False, + help="New file mode. Convert the infile and write output to outfile." + " File names must be given in pairs and wildcard names should" + " NOT be used or you WILL lose your files.") + aparser.add_argument( + "--quiet", "-q", action="store_true", + help="Quiet mode. Suppress all warning and messages.") + + aparser.add_argument( + "files", nargs="+", metavar="FILE") + + opts = aparser.parse_args(args=argv) + + if opts.keepdate: + raise NotImplementedError("--keepdate, -k") + + return dos2unix(opts) + + +def gen_opts(files=[], newfile=False, keepdate=False, quiet=True): + if keepdate: + raise NotImplementedError("--keepdate, -k") + + if newfile and (len(files) % 2): + raise ValueError("need pairs of files") + + opts = argparse.Namespace(files=files, + newfile=newfile, + keepdate=keepdate, + quiet=quiet) + return opts + + +def dos2unix(opts): + if opts.newfile: + return _convert_copy(opts) + else: + return _convert_inplace(opts) + + +def _convert_inplace(opts): + lines = [] + for filename in opts.files: + with io.open(filename, "rt", encoding="iso-8859-1") as source: + for line in source: + lines.append(line.encode("iso-8859-1")) + with open(filename, "wb") as dest: + for line in lines: + dest.write(line) + + +def _convert_copy(opts): + if len(opts.files) % 2: + print("ERROR: need pairs of files", file=sys.stderr) + return 64 # :manpage:`sysexits(3)` EX_USAGE + idx = 0 + while idx < len(opts.files): + with io.open(opts.files[idx], "rt", encoding="iso-8859-1") as source: + with open(opts.files[idx+1], "wb") as dest: + for line in source: + dest.write(line.encode("iso-8859-1")) + idx += 2 + + +if __name__ == "__main__": + sys.exit(main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/shasum.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,456 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Pure Python implementation of `shasum`. + +""" + +from __future__ import print_function, absolute_import + + +__all__ = [] + + +import argparse +import base64 +import binascii +import errno +import io +import os +import re +import sys + +from . import (__version__, __revision__) +from . import util +from .util import digest as digestmod + + +def main(argv=None): + aparser = argparse.ArgumentParser( + description="Python implementation of shasum", + fromfile_prefix_chars='@') + aparser.add_argument( + "--algorithm", "-a", action="store", type=util.argv2algo, + help="""1 (default, aka sha1), 224, 256, 384, 512, +3 (alias for sha3-512), 3-224, 3-256, 3-384, 3-512, +blake2b, blake2b-256, blake2s, blake2 (alias for blake2b), +blake2-256 (alias for blake2b-256), md5""") + aparser.add_argument( + "--base64", action="store_true", + help="Output checksums in base64 notation, not hexadecimal (OpenBSD).") + aparser.add_argument( + "--binary", "-b", action="store_false", dest="text_mode", + default=False, + help="Read in binary mode (default)") + aparser.add_argument( + "--bsd", "-B", action="store_true", dest="bsd", default=False, + help="""Write BSD style output. This is also the default output format +of :command:`openssl dgst`.""") + aparser.add_argument( + "--check", "-c", action="store_true", + help="""Read digests from FILEs and check them. +If this option is specified, the FILE options become checklists. Each +checklist should contain hash results in a supported format, which will +be verified against the specified paths. Output consists of the digest +used, the file name, and an OK, FAILED, or MISSING for the result of +the comparison. This will validate any of the supported checksums. +If no file is given, stdin is used.""") + aparser.add_argument( + "--checklist", "-C", metavar="CHECKLIST", + help="""Compare the checksum of each FILE against the checksums in +the CHECKLIST. Any specified FILE that is not listed in the CHECKLIST will +generate an error.""") + aparser.add_argument( + "--checklist-allow-distinfo", action="store_true", + dest="allow_distinfo", + help='''Allow FreeBSD "distinfo" formatted checklists: +ignore SIZE and TIMESTAMP lines.''') + + aparser.add_argument( + "--follow-symlinks", action="store_true", dest="follow_symlinks", + help="""Also follow symlinks that resolve to directories. +Only effective if `--recurse` is activated.""") + + aparser.add_argument( + "--mmap", action="store_true", dest="mmap", default=None, + help="""Use mmap if available. Default is to determine automatically + from the filesize.""") + aparser.add_argument( + "--no-mmap", action="store_false", dest="mmap", default=None, + help="Dont use mmap.") + + aparser.add_argument( + "--recurse", action="store_true", + help="""Recurse into sub-directories while interpreting every +FILE as a directory.""") + + aparser.add_argument( + "--reverse", "-r", action="store_false", dest="bsd", default=False, + help="""Explicitely select normal coreutils style output +(to be option compatible with BSD style commands and +:command:`openssl dgst -r`)""") + aparser.add_argument( + "--tag", action="store_true", dest="bsd", default=False, + help="""Alias for the `--bsd' option (to be compatible with +:command:`b2sum`)""") + aparser.add_argument( + "--text", "-t", action="store_true", dest="text_mode", default=False, + help="Read in text mode (not supported)") + aparser.add_argument( + "--version", "-v", action="version", + version="%s (rv:%s)" % (__version__, __revision__)) + aparser.add_argument( + "files", nargs="*", metavar="FILE") + + opts = aparser.parse_args(args=argv) + + if opts.text_mode: + print("ERROR: text mode not supported", file=sys.stderr) + sys.exit(78) # :manpage:`sysexits(3)` EX_CONFIG + + if opts.check and opts.checklist: + print("ERROR: only one of --check or --checklist allowed", + file=sys.stderr) + sys.exit(64) # :manpage:`sysexits(3)` EX_USAGE + + if not opts.algorithm: + opts.algorithm = util.argv2algo("1") + + opts.dest = None + + return shasum(opts) + + +def gen_opts(files=[], algorithm="SHA1", bsd=False, text_mode=False, + checklist=False, check=False, dest=None, base64=False, + allow_distinfo=False, mmap=None, recurse=False, + follow_symlinks=False): + if text_mode: + raise ValueError("text mode not supported") + if checklist and check: + raise ValueError("only one of `checklist' or `check' is allowed") + opts = argparse.Namespace(files=files, + algorithm=(util.algotag2algotype(algorithm), + algorithm), + bsd=bsd, + checklist=checklist, + check=check, + text_mode=False, + dest=dest, + base64=base64, + allow_distinfo=allow_distinfo, + mmap=mmap, + recurse=recurse, + follow_symlinks=follow_symlinks) + return opts + + +def shasum(opts): + if opts.check: + return verify_digests_from_files(opts) + elif opts.checklist: + return verify_digests_with_checklist(opts) + else: + return generate_digests(opts) + + +def generate_digests(opts): + if opts.bsd: + out = out_bsd + else: + out = out_std + if opts.recurse: + if not opts.files: + opts.files.append(".") + for dn in opts.files: + if not os.path.isdir(dn): + if os.path.exists(dn): + raise OSError(errno.ENOTDIR, "not a directory", dn) + else: + raise OSError(errno.ENOENT, "directory does not exist", dn) + for dirpath, dirnames, dirfiles in os.walk( + dn, followlinks=opts.follow_symlinks): + dirnames.sort() + dirfiles.sort() + for fn in dirfiles: + path = os.path.join(dirpath, fn) + out(opts.dest or sys.stdout, + digestmod.compute_digest_file( + opts.algorithm[0], path, use_mmap=opts.mmap), + path, + opts.algorithm[1], + True, + opts.base64) + else: + if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'): + if util.PY2: + if sys.platform == "win32": + import msvcrt # noqa: E401 + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + source = sys.stdin + else: + source = sys.stdin.buffer + out(sys.stdout, + digestmod.compute_digest_stream(opts.algorithm[0], source), + None, + opts.algorithm[1], + True, + opts.base64) + else: + for fn in opts.files: + out(opts.dest or sys.stdout, + digestmod.compute_digest_file( + opts.algorithm[0], fn, use_mmap=opts.mmap), + fn, + opts.algorithm[1], + True, + opts.base64) + return 0 + + +def compare_digests_equal(given_digest, expected_digest, algo): + """Compare a newly computed binary digest `given_digest` with a digest + string (hex or base64) in `expected_digest`. + + :param bytes given_digest: + :param expected_digest: digest (as bytes) or hexlified or base64 encoded + digest (as str) + :type expected_digest: str or bytes or bytearray + :param algo: The algorithm (factory) + :return: `True` if the digests are equal, `False` if not + :rtype: bool + + """ + if isinstance(expected_digest, (bytes, bytearray)) \ + and len(expected_digest) == algo().digest_size: + exd = expected_digest + else: + if len(expected_digest) == algo().digest_size * 2: + # hex + if re.search(r"\A[a-fA-F0-9]+\Z", expected_digest): + try: + exd = binascii.unhexlify(expected_digest) + except TypeError: + return False + else: + return False + else: + # base64 + if re.search( + r"\A(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{2}==)?\Z", # noqa: E501 line too long + expected_digest): + try: + exd = base64.b64decode(expected_digest) + except TypeError: + return False + else: + return False + return given_digest == exd + + +def verify_digests_with_checklist(opts): + dest = opts.dest or sys.stdout + exit_code = 0 + if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'): + if util.PY2: + if sys.platform == "win32": + import os, msvcrt # noqa: E401 + msvcrt.setmode(sys.stdin.fileno(), os.O_BINARY) + source = sys.stdin + else: + source = sys.stdin.buffer + pl = get_parsed_digest_line_from_checklist(opts.checklist, opts, None) + if pl is None: + exit_code = 1 + print("-: MISSING", file=dest) + else: + tag, algo, cl_filename, cl_digest = pl + computed_digest = digestmod.compute_digest_stream(algo, source) + if compare_digests_equal(computed_digest, cl_digest, algo): + res = "OK" + else: + res = "FAILED" + exit_code = 1 + print("{}: {}: {}".format(tag, "-", res), file=dest) + else: + for fn in opts.files: + pl = get_parsed_digest_line_from_checklist( + opts.checklist, opts, fn) + if pl is None: + print("{}: MISSING".format(fn), file=dest) + exit_code = 1 + else: + tag, algo, cl_filename, cl_digest = pl + computed_digest = digestmod.compute_digest_file( + algo, fn, use_mmap=opts.mmap) + if compare_digests_equal(computed_digest, cl_digest, algo): + res = "OK" + else: + exit_code = 1 + res = "FAILED" + print("{}: {}: {}".format(tag, fn, res), file=dest) + return exit_code + + +def verify_digests_from_files(opts): + dest = opts.dest or sys.stdout + exit_code = 0 + if not opts.files or (len(opts.files) == 1 and opts.files[0] == '-'): + for checkline in sys.stdin: + if not checkline: + continue + r, fn, tag = handle_checkline(opts, checkline) + if tag in ("SIZE", "TIMESTAMP"): + assert opts.allow_distinfo + continue + print("{}: {}: {}".format(tag, fn, r.upper()), file=dest) + if r != "ok" and exit_code == 0: + exit_code = 1 + else: + for fn in opts.files: + with io.open(fn, "rt", encoding="utf-8") as checkfile: + for checkline in checkfile: + if not checkline: + continue + r, fn, tag = handle_checkline(opts, checkline) + if tag in ("SIZE", "TIMESTAMP"): + assert opts.allow_distinfo + continue + print("{}: {}: {}".format(tag, fn, r.upper()), file=dest) + if r != "ok" and exit_code == 0: + exit_code = 1 + return exit_code + + +def handle_checkline(opts, line): + """ + :return: a tuple with static "ok", "missing", or "failed", the filename and + the digest used + :rtype: tuple(str, str, str) + + """ + parts = parse_digest_line(opts, line) + if not parts: + raise ValueError( + "improperly formatted digest line: {}".format(line)) + tag, algo, fn, digest = parts + if tag in ("SIZE", "TIMESTAMP"): + assert opts.allow_distinfo + return (None, None, tag) + try: + d = digestmod.compute_digest_file(algo, fn, use_mmap=opts.mmap) + if compare_digests_equal(d, digest, algo): + return ("ok", fn, tag) + else: + return ("failed", fn, tag) + except EnvironmentError: + return ("missing", fn, tag) + + +def get_parsed_digest_line_from_checklist(checklist, opts, filename): + if filename is None: + filenames = ("-", "stdin", "", ) + else: + filenames = ( + util.normalize_filename(filename, strip_leading_dot_slash=True),) + with io.open(checklist, "rt", encoding="utf-8") as clf: + for checkline in clf: + if not checkline: + continue + parts = parse_digest_line(opts, checkline) + if not parts: + raise ValueError( + "improperly formatted digest line: {}".format(checkline)) + if parts[0] in ("SIZE", "TIMESTAMP"): + assert opts.allow_distinfo + continue + fn = util.normalize_filename( + parts[2], strip_leading_dot_slash=True) + if fn in filenames: + return parts + else: + return None + + +def parse_digest_line(opts, line): + """Parse a `line` of a digest file and return its parts. + + This is rather strict. But if `opts.allow_distinfo` is `True` then + some additional keywords ``SIZE`` and ``TIMESTAMP``are recignized + and returned. The caller is responsible to handle them. + + :return: a tuple of the normalized algorithm tag, the algorithm + constructor, the filename and the hex digest; + if `line` cannot be parsed successfully `None` is returned + :rtype: tuple(str, obj, str, str) or None + + Handles coreutils and BSD-style file formats. + + """ + # determine checkfile format (BSD or coreutils) + # BSD? + mo = re.search(r"\A(\S+)\s*\((.*)\)\s*=\s*(.+)\n?\Z", line) + if mo: + # (tag, algorithm, filename, digest) + if opts.allow_distinfo: + if mo.group(1) == "SIZE": + return ("SIZE", None, None, mo.group(3)) + return (mo.group(1), + util.algotag2algotype(mo.group(1)), + mo.group(2), + mo.group(3)) + else: + if opts.allow_distinfo: + mo = re.search(r"\ATIMESTAMP\s*=\s*([0-9]+)\s*\n\Z", line) + if mo: + return ("TIMESTAMP", None, None, mo.group(1)) + + # coreutils? + mo = re.search(r"([^\ ]+) [\*\ ]?(.+)\n?\Z", line) + if mo: + # (tag, algorithm, filename, digest) + return (opts.algorithm[1], + opts.algorithm[0], + mo.group(2), + mo.group(1)) + else: + return None + + +def out_bsd(dest, digest, filename, digestname, binary, use_base64): + """BSD format output, also :command:`openssl dgst` and + :command:`b2sum --tag" format output + + """ + if use_base64: + digest = base64.b64encode(digest).decode("ascii") + else: + digest = binascii.hexlify(digest).decode("ascii") + if filename is None: + print(digest, file=dest) + else: + print("{} ({}) = {}".format(digestname, + util.normalize_filename(filename), + digest), + file=dest) + + +def out_std(dest, digest, filename, digestname, binary, use_base64): + """Coreutils format (:command:`shasum` et al.) + + """ + if use_base64: + digest = base64.b64encode(digest).decode("ascii") + else: + digest = binascii.hexlify(digest).decode("ascii") + print("{} {}{}".format( + digest, + '*' if binary else ' ', + '-' if filename is None else util.normalize_filename(filename)), + file=dest) + + +if __name__ == "__main__": + sys.exit(main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/treesum.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,1334 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Generate and verify checksums for directory trees. + +""" + +from __future__ import print_function, absolute_import + + +__all__ = [] + + +import argparse +import base64 +import binascii +import collections +import datetime +import logging +import os +import re +import stat +import sys +import time +import zlib + +from . import (__version__, __revision__) +from . import util +from .util import cm +from .util import digest +from .util import walk + + +def main(argv=None): + + def _populate_generate_arguments(gp): + """Use to populate command aliases. + + This is because :class:`argparse.ArgumentParser` does not + support them for all supported Python versions. + + """ + gp.add_argument( + "--algorithm", "-a", action="store", type=util.argv2algo, + help="1 (aka sha1), 224, 256 (aka sha256), 384, 512 (aka sha512), " + "3 (alias for sha3-512), 3-224, 3-256, 3-384, 3-512, " + "blake2b, blake2b-256, blake2s, " + "blake2 (alias for blake2b), " + "blake2-256 (alias for blake2b-256), " + "md5. " + "The default depends on availability in hashlib: " + "blake2b-256, sha256 or sha1.") + gp.add_argument( + "--append-output", action="store_true", dest="append_output", + help="Append to the output file instead of overwriting it.") + gp.add_argument( + "--base64", action="store_true", + help="Output checksums in base64 notation, not hexadecimal " + "(OpenBSD).") + gp.add_argument( + "--comment", action="append", default=[], + help="Put given comment COMMENT into the output as \"COMMENT\". " + "Can be given more than once.") + gp.add_argument( + "--follow-directory-symlinks", "-l", action=SymlinkAction, + const="follow-directory-symlinks", + default=FollowSymlinkConfig(False, False, True), + dest="follow_symlinks", + help="""Follow symbolic links to directories when walking a +directory tree. Augments --physical and -p.""") + gp.add_argument( + "--follow-file-symlinks", action=SymlinkAction, + const="follow-file-symlinks", + default=FollowSymlinkConfig(False, False, True), + dest="follow_symlinks", + help="""Follow symbolic links to files when walking a +directory tree. Augments --physical.""") + gp.add_argument( + "--full-mode", action="store_true", dest="metadata_full_mode", + help="Consider all mode bits as returned from stat(2) when " + "computing directory digests. " + "Note that mode bits on symbolic links itself are not " + "considered.") + gp.add_argument( + "--generator", choices=("normal", "full", "none"), + default="normal", + help="""Put a `GENERATOR' line into the output. +`full' prints full Python and OS/platform version information, +`normal' prints just whether Python 2 or Python 3 is used, and `none' +suppresses the output completely. The default is `normal'.""") + gp.add_argument( + "--logical", "-L", action=SymlinkAction, dest="follow_symlinks", + const=FollowSymlinkConfig(True, True, True), + help="""Follow symbolic links everywhere: on command line +arguments and -- while walking -- directory and file symbolic links. +Overwrites any other symlink related options +(--physical,-p, no-follow-directory-symlinks, no-follow-file-symlinks, +et al.). +""") + gp.add_argument( + "--minimal", nargs="?", const="", default=None, metavar="TAG", + help="Produce minimal output only. If a TAG is given and not " + "empty use it as the leading \"ROOT (<TAG>)\" output.") + gp.add_argument( + "--mmap", action="store_true", dest="mmap", default=None, + help="Use mmap if available. Default is to determine " + "automatically from the filesize.") + gp.add_argument( + "--mode", action="store_true", dest="metadata_mode", + help="Consider the permission bits of stat(2) using S_IMODE (i.e. " + "all bits without the filetype bits) when " + "computing directory digests. Note that mode bits on " + "symbolic links itself are not considered.") + gp.add_argument( + "--mtime", action="store_true", dest="metadata_mtime", + help="Consider the mtime of files (non-directories) when " + "generating digests for directories. Digests for files are " + "not affected.") + gp.add_argument( + "--no-follow-directory-symlinks", action=SymlinkAction, + const="no-follow-directory-symlinks", + dest="follow_symlinks", + help="""Do not follow symbolic links to directories when walking a +directory tree. Augments --logical.""") + gp.add_argument( + "--no-follow-file-symlinks", action=SymlinkAction, + const="no-follow-file-symlinks", + dest="follow_symlinks", + help="""Dont follow symbolic links to files when walking a +directory tree. Augments --logical and -p.""") + gp.add_argument( + "--no-mmap", action="store_false", dest="mmap", default=None, + help="Dont use mmap.") + gp.add_argument( + "--output", "-o", action="store", metavar="OUTPUT", + help="Put the checksum into given file. " + "If not given or if it is given as `-' then stdout is used.") + gp.add_argument( + "--physical", "-P", action=SymlinkAction, dest="follow_symlinks", + const=FollowSymlinkConfig(False, False, False), + help="""Do not follow any symbolic links whether they are given +on the command line or when walking the directory tree. +Overwrites any other symlink related options +(--logical, -p, follow-directory-symlinks, follow-file-symlinks, et al.). +This is the default.""") + gp.add_argument( + "-p", action=SymlinkAction, dest="follow_symlinks", + const=FollowSymlinkConfig(False, False, True), + help="""Do not follow any symbolic links to directories, +whether they are given on the command line or when walking the directory tree, +but follow symbolic links to files. +Overwrites any other symlink related options +(--logical, --physical, follow-directory-symlinks, no-follow-file-symlinks, +et al.). +This is the default.""") + gp.add_argument( + "--print-size", action="store_true", + help="""Print the size of a file or the accumulated sizes of +directory content into the output also. +The size is not considered when computing digests. For symbolic links +the size is not printed also.""") + gp.add_argument( + "--size-only", action="store_true", + help="""Print only the size of files and for each directory its +accumulated directory size. Digests are not computed.""") + gp.add_argument( + "--utf8", "--utf-8", action="store_true", + help="""Encode all file paths using UTF-8 instead of +the filesystem encoding. Add some error tag into the path if it cannot +representated in Unicode cleanly.""") + gp.add_argument( + "directories", nargs="*", metavar="DIRECTORY") + + def _populate_info_arguments(ip): + ip.add_argument( + "--last", action="store_true", dest="print_only_last_block", + help="Print only the last block of every given input file") + ip.add_argument( + "digest_files", nargs="+", metavar="TREESUM-DIGEST-FILE") + + parser = argparse.ArgumentParser( + description="Generate and verify checksums for directory trees.", + fromfile_prefix_chars='@', + add_help=False) + + # + # Global options for all sub-commands. + # In a group because this allows a customized title. + # + gparser = parser.add_argument_group(title="Global Options") + gparser.add_argument( + "--debug", action="store_true", + help="Activate debug logging to stderr") + gparser.add_argument( + "-v", "--version", action="version", + version="%s (rv:%s)" % (__version__, __revision__), + help="Show program's version number and exit") + gparser.add_argument( + "-h", "--help", action="help", + help="Show this help message and exit") + + # + # Subcommands + # + subparsers = parser.add_subparsers( + dest="subcommand", + title="Commands", + description="This tool uses subcommands. " + "To see detailed help for a specific subcommand use " + "the -h/--help option after the subcommand name. " + "A list of valid commands and their short descriptions " + "is listed below:", + metavar="COMMAND") + + genparser = subparsers.add_parser( + "generate", + help="Generate checksums for directory trees", + description="Generate checksums for directory trees.") + _populate_generate_arguments(genparser) + # And an alias for "generate" + genparser2 = subparsers.add_parser( + "gen", + help="Alias for \"generate\"", + description="Generate checksums for directory trees. " + "This is an alias to \"generate\".") + _populate_generate_arguments(genparser2) + + infoparser = subparsers.add_parser( + "info", + help="Print some information from given treesum digest file", + description="""Print some informations from given treesum digest files +to stdout.""" + ) + _populate_info_arguments(infoparser) + + hparser = subparsers.add_parser( + "help", + help="Show this help message or a subcommand's help and exit", + description="Show this help message or a subcommand's help and exit.") + hparser.add_argument("help_command", nargs='?', metavar="COMMAND") + + vparser = subparsers.add_parser( + "version", + help="Show the program's version number and exit", + description="Show the program's version number and exit.") + + # Parse leniently to just check for "version" and/or help + opts, _dummy = parser.parse_known_args(args=argv) + + if opts.subcommand == "version": + print("%s (rv:%s)" % (__version__, __revision__), + file=sys.stdout) + return 0 + if opts.subcommand == "help": + if not opts.help_command: + parser.print_help() + else: + if opts.help_command == "generate": + genparser.print_help() + elif opts.help_command == "gen": + genparser2.print_help() + elif opts.help_command == "info": + infoparser.print_help() + elif opts.help_command == "version": + vparser.print_help() + elif opts.help_command == "help": + hparser.print_help() + else: + parser.print_help() + return 0 + + # Reparse strictly + opts = parser.parse_args(args=argv) + + # Minimal logging -- just for debugging - not for more "normal" use + logging.basicConfig( + level=logging.DEBUG if opts.debug else logging.WARNING, + stream=sys.stderr, + format="[%(asctime)s][%(levelname)s][%(process)d:%(name)s] %(message)s" + ) + logging.captureWarnings(True) + + return treesum(opts) + + +FollowSymlinkConfig = collections.namedtuple("FollowSymlinkConfig", + ["command_line", + "directory", + "file"]) + + +class SymlinkAction(argparse.Action): + + """`type' is fixed here. + `dest' is a tuple with three items: + + 1. follow symlinks on the command line + 2. follow directory symlinks while walking + 3. follow file symlinks while walking (not yet implemented) + + """ + + def __init__(self, *args, **kwargs): + if "nargs" in kwargs: + raise ValueError("`nargs' not allowed") + if "type" in kwargs: + raise ValueError("`type' not allowed") + c = kwargs.get("const", None) + if c is None: + raise ValueError("a const value is needed") + if (not isinstance(c, FollowSymlinkConfig) + and c not in ("follow-directory-symlinks", + "no-follow-directory-symlinks", + "follow-file-symlinks", + "no-follow-file-symlinks")): + raise ValueError( + "invalid value for the `const' configuration value") + default = kwargs.get("default", None) + if (default is not None + and not isinstance(default, FollowSymlinkConfig)): + raise TypeError("invalid type for `default'") + kwargs["nargs"] = 0 + super(SymlinkAction, self).__init__(*args, **kwargs) + + def __call__(self, parser, namespace, values, option_string=None): + curval = getattr(namespace, self.dest, None) + if curval is None: + curval = FollowSymlinkConfig(False, False, True) + if isinstance(self.const, FollowSymlinkConfig): + curval = self.const + else: + if self.const == "follow-directory-symlinks": + curval = FollowSymlinkConfig( + curval.command_line, True, curval.file) + elif self.const == "no-follow-directory-symlinks": + curval = FollowSymlinkConfig( + curval.command_line, False, curval.file) + elif self.const == "follow-file-symlinks": + curval = FollowSymlinkConfig( + curval.command_line, curval.directory, True) + elif self.const == "no-follow-file-symlinks": + curval = FollowSymlinkConfig( + curval.command_line, curval.directory, False) + else: + assert False, "Implementation error: not yet implemented" + + # Not following symlinks to files is not yet supported: reset to True +# if not curval.file: +# curval = FollowSymlinkConfig( +# curval.command_line, curval.directory, True) +# logging.warning("Coercing options to `follow-file-symlinks'") + setattr(namespace, self.dest, curval) + + +def gen_generate_opts(directories=[], + algorithm=util.default_algotag(), + append_output=False, + base64=False, + comment=[], + follow_symlinks=FollowSymlinkConfig(False, False, False), + full_mode=False, + generator="normal", + logical=None, + minimal=None, + mode=False, + mmap=None, + mtime=False, + output=None, + print_size=False, + size_only=False, + utf8=False): + if not isinstance(follow_symlinks, FollowSymlinkConfig): + raise TypeError("`follow_symlinks' must be a FollowSymlinkConfig") + # Not following symlinks to files is not yet supported: reset to True +# if not follow_symlinks.file: +# follow_symlinks = follow_symlinks._make([follow_symlinks.command_line, +# follow_symlinks.directory, +# True]) +# logging.warning("Coercing to follow-symlinks-file") + opts = argparse.Namespace( + directories=directories, + algorithm=util.argv2algo(algorithm), + append_output=append_output, + base64=base64, + comment=comment, + follow_symlinks=follow_symlinks, + generator=generator, + logical=logical, + minimal=minimal, + mmap=mmap, + metadata_full_mode=full_mode, + metadata_mode=mode, + metadata_mtime=mtime, + output=output, + print_size=print_size, + size_only=size_only, + utf8=utf8) + return opts + + +def gen_info_opts(digest_files=[], last=False): + opts = argparse.Namespace( + digest_files=digest_files, + print_only_last_block=last) + return opts + + +def treesum(opts): + # XXX TBD: opts.check and opts.checklist (as in shasum.py) + if opts.subcommand in ("generate", "gen"): + return generate_treesum(opts) + elif opts.subcommand == "info": + return print_treesum_digestfile_infos(opts) + else: + raise RuntimeError( + "command `{}' not yet handled".format(opts.subcommand)) + + +def generate_treesum(opts): + # Provide defaults + if not opts.algorithm: + opts.algorithm = util.argv2algo(util.default_algotag()) + if not opts.directories: + opts.directories.append(".") + + if opts.output is None or opts.output == "-": + if hasattr(sys.stdout, "buffer"): + out_cm = cm.nullcontext(sys.stdout.buffer) + else: + out_cm = cm.nullcontext(sys.stdout) + else: + if opts.append_output: + out_cm = open(opts.output, "ab") + else: + out_cm = open(opts.output, "wb") + out_cm = CRC32Output(out_cm) + + with out_cm as outfp: + for d in opts.directories: + V1DirectoryTreesumGenerator( + opts.algorithm, opts.mmap, opts.base64, + opts.follow_symlinks, + opts.generator, + opts.metadata_mode, + opts.metadata_full_mode, + opts.metadata_mtime, + opts.size_only, + opts.print_size, + opts.utf8, + minimal=opts.minimal).generate( + outfp, d, comment=opts.comment) + + +class V1DirectoryTreesumGenerator(object): + + def __init__(self, algorithm, use_mmap, use_base64, + follow_symlinks, + with_generator, + with_metadata_mode, with_metadata_full_mode, + with_metadata_mtime, size_only, print_size, utf8_mode, + minimal=None,): + super(V1DirectoryTreesumGenerator, self).__init__() + self._algorithm = algorithm + self._use_mmap = use_mmap + self._use_base64 = use_base64 + self._follow_symlinks = follow_symlinks + self._with_generator = with_generator + self._with_metadata_mode = with_metadata_mode + self._with_metadata_full_mode = with_metadata_full_mode + self._with_metadata_mtime = with_metadata_mtime + self._size_only = size_only + self._print_size = print_size + self._utf8_mode = utf8_mode + self._minimal = minimal + + def generate(self, outfp, root, comment=None): + """ + + :param outfp: a *binary* file with a "write()" and a "flush()" method + + """ + self._outfp = outfp + self._outfp.resetdigest() + self._outfp.write(format_bsd_line("VERSION", "1", None, False)) + self._outfp.write(format_bsd_line( + "FSENCODING", util.n(walk.getfsencoding().upper()), None, False)) + self._outfp.flush() + + if self._with_generator == "none": + pass # do nothing + elif self._with_generator == "normal": + self._outfp.write(format_bsd_line( + "GENERATOR", None, b"PY2" if util.PY2 else b"PY3", False)) + elif self._with_generator == "full": + import platform + info = "%s %s, %s" % (platform.python_implementation(), + platform.python_version(), + platform.platform()) + self._outfp.write(format_bsd_line( + "GENERATOR", None, info.encode("utf-8"), False)) + else: + raise NotImplementedError( + "not implemented: %s" % (self._with_generator,)) + + # + # Note: Given non-default flags that are relevant for + # directory traversal. + # + flags = [] + if self._with_metadata_full_mode: + flags.append("with-metadata-fullmode") + elif self._with_metadata_mode: + flags.append("with-metadata-mode") + if self._with_metadata_mtime: + flags.append("with-metadata-mtime") + flags.append("follow-symlinks-commandline" + if self._follow_symlinks.command_line + else "no-follow-symlinks-commandline") + flags.append("follow-symlinks-directory" + if self._follow_symlinks.directory + else "no-follow-symlinks-directory") + flags.append("follow-symlinks-file" + if self._follow_symlinks.file + else "no-follow-symlinks-file") + if self._size_only: + flags.append("size-only") + flags.append("utf8-encoding" if self._utf8_mode else "fs-encoding") + if self._print_size: + flags.append("print-size") + flags.sort() + self._outfp.write( + format_bsd_line("FLAGS", ",".join(flags), None, False)) + + if self._minimal is None: + # Write execution timestamps in POSIX epoch and ISO format + ts = int(time.time()) + self._outfp.write(format_bsd_line("TIMESTAMP", ts, None, False)) + ts = (datetime.datetime.utcfromtimestamp(ts)).isoformat("T") + self._outfp.write(format_bsd_line("ISOTIMESTAMP", ts, None, False)) + + if comment: + for line in comment: + self._outfp.write( + format_bsd_line("COMMENT", None, line, False)) + + if self._minimal is not None: + self._outfp.write(format_bsd_line( + "ROOT", + None, + (walk.WalkDirEntry.alt_u8(self._minimal) + if self._minimal else b""), + False)) + else: + self._outfp.write(format_bsd_line( + "ROOT", None, walk.WalkDirEntry.alt_u8(root), False)) + self._outfp.flush() + + if not self._follow_symlinks.command_line and os.path.islink(root): + linktgt = walk.WalkDirEntry.from_readlink(os.readlink(root)) + linkdgst = self._algorithm[0]() + linkdgst.update( + util.interpolate_bytes( + b"%d:%s,", len(linktgt.fspath), linktgt.fspath)) + dir_dgst = self._algorithm[0]() + dir_dgst.update(b"1:L,") + dir_dgst.update( + util.interpolate_bytes( + b"%d:%s,", len(linkdgst.digest()), linkdgst.digest())) + if self._size_only: + self._outfp.write( + format_bsd_line( + "SIZE", + None, + "./@/", + False, + 0)) + else: + self._outfp.write( + format_bsd_line( + self._algorithm[1], + dir_dgst.digest(), + "./@/", + self._use_base64)) + self._outfp.flush() + self._outfp.write(format_bsd_line( + "CRC32", self._outfp.hexcrcdigest(), None, False)) + return + + self._generate(os.path.normpath(root), tuple()) + self._outfp.write(format_bsd_line( + "CRC32", self._outfp.hexcrcdigest(), None, False)) + + def _generate(self, root, top): + logging.debug("Handling %s/%r", root, top) + path = os.path.join(root, *top) if top else root + with walk.ScanDir(path) as dirscan: + fsobjects = list(dirscan) + if self._utf8_mode: + fsobjects.sort(key=walk.WalkDirEntry.sort_key_u8) + else: + fsobjects.sort(key=walk.WalkDirEntry.sort_key_fs) + dir_dgst = self._algorithm[0]() + dir_size = 0 + dir_tainted = False + for fso in fsobjects: + if fso.is_dir: + if fso.is_symlink and not self._follow_symlinks.directory: + linktgt = walk.WalkDirEntry.from_readlink( + os.readlink(fso.path)) + # linktgt = util.fsencode(os.readlink(fso.path))) + linkdgst = self._algorithm[0]() + if self._utf8_mode: + if linktgt.u8path is None: + dir_tainted = True + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.alt_u8path), + linktgt.alt_u8path)) + else: + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.u8path), + linktgt.u8path)) + if fso.u8name is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:S,%d:%s,", + len(fso.alt_u8name), + fso.alt_u8name)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:S,%d:%s,", len(fso.u8name), fso.u8name)) + else: + if linktgt.fspath is None: + dir_tainted = True + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.alt_fspath), + linktgt.alt_fspath)) + else: + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.fspath), + linktgt.fspath)) + if fso.fsname is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:S,%d:%s,", + len(fso.alt_fsname), + fso.alt_fsname)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:S,%d:%s,", len(fso.fsname), fso.fsname)) + # + # - no mtime and no mode for symlinks + # - also does not count for dir_size + # + dir_dgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linkdgst.digest()), linkdgst.digest())) + opath = join_output_path(top, fso.name) + if self._utf8_mode: + opath = walk.WalkDirEntry.alt_u8(opath) + else: + opath = walk.WalkDirEntry.alt_fs(opath) + if self._size_only: + self._outfp.write(format_bsd_line( + "SIZE", None, + util.interpolate_bytes(b"%s/./@/", opath), + False, 0)) + else: + self._outfp.write(format_bsd_line( + self._algorithm[1], + linkdgst.digest(), + util.interpolate_bytes(b"%s/./@/", opath), + self._use_base64)) + self._outfp.flush() + else: + # + # Follow the symlink to dir or handle a "real" directory + # + + # Get subdir data from recursing into it + sub_dir_dgst, sub_dir_size = self._generate( + root, top + (fso.name, )) + + dir_size += sub_dir_size + if self._utf8_mode: + if fso.u8name is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:d,%d:%s,", + len(fso.alt_u8name), + fso.alt_u8name)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:d,%d:%s,", len(fso.u8name), fso.u8name)) + else: + if fso.fsname is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:d,%d:%s,", + len(fso.alt_fsname), + fso.alt_fsname)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:d,%d:%s,", len(fso.fsname), fso.fsname)) + dir_dgst.update(util.interpolate_bytes( + b"%d:%s,", len(sub_dir_dgst), sub_dir_dgst)) + if self._with_metadata_full_mode: + modestr = util.b(normalized_mode_str(fso.stat.st_mode)) + dir_dgst.update(util.interpolate_bytes( + b"8:fullmode,%d:%s,", len(modestr), modestr)) + elif self._with_metadata_mode: + modestr = util.b(normalized_compatible_mode_str( + fso.stat.st_mode)) + dir_dgst.update(util.interpolate_bytes( + b"4:mode,%d:%s,", len(modestr), modestr)) + else: + if fso.is_symlink and not self._follow_symlinks.file: + linktgt = walk.WalkDirEntry.from_readlink( + os.readlink(fso.path)) + # linktgt = util.fsencode(os.readlink(fso.path))) + linkdgst = self._algorithm[0]() + if self._utf8_mode: + if linktgt.u8path is None: + dir_tainted = True + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.alt_u8path), + linktgt.alt_u8path)) + else: + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.u8path), + linktgt.u8path)) + if fso.u8name is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:F,%d:%s,", + len(fso.alt_u8name), + fso.alt_u8name)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:F,%d:%s,", len(fso.u8name), fso.u8name)) + else: + if linktgt.fspath is None: + dir_tainted = True + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.alt_fspath), + linktgt.alt_fspath)) + else: + linkdgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linktgt.fspath), + linktgt.fspath)) + if fso.fsname is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:F,%d:%s,", + len(fso.alt_fsname), + fso.alt_fsname)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:F,%d:%s,", len(fso.fsname), fso.fsname)) + # + # - no mtime and no mode for symlinks + # - also does not count for dir_size + # + dir_dgst.update(util.interpolate_bytes( + b"%d:%s,", + len(linkdgst.digest()), linkdgst.digest())) + opath = join_output_path(top, fso.name) + if self._utf8_mode: + opath = walk.WalkDirEntry.alt_u8(opath) + else: + opath = walk.WalkDirEntry.alt_fs(opath) + if self._size_only: + self._outfp.write(format_bsd_line( + "SIZE", None, + util.interpolate_bytes(b"%s/./@", opath), + False, 0)) + else: + self._outfp.write(format_bsd_line( + self._algorithm[1], + linkdgst.digest(), + util.interpolate_bytes(b"%s/./@", opath), + self._use_base64)) + self._outfp.flush() + else: + # + # Follow the symlink to file or handle a "real" file + # + + if self._utf8_mode: + if fso.u8name is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:f,%d:%s,", + len(fso.alt_u8name), + fso.alt_u8name)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:f,%d:%s,", len(fso.u8name), fso.u8name)) + else: + if fso.fsname is None: + dir_tainted = True + dir_dgst.update(util.interpolate_bytes( + b"1:f,%d:%s,", + len(fso.alt_fsname), + fso.alt_fsname)) + else: + dir_dgst.update(util.interpolate_bytes( + b"1:f,%d:%s,", len(fso.fsname), fso.fsname)) + dir_size += fso.stat.st_size + if self._with_metadata_mtime: + mtime = datetime.datetime.utcfromtimestamp( + int(fso.stat.st_mtime)) + mtime = util.b(mtime.isoformat("T") + "Z") + dir_dgst.update(util.interpolate_bytes( + b"5:mtime,%d:%s,", len(mtime), mtime)) + if self._with_metadata_full_mode: + modestr = util.b(normalized_mode_str(fso.stat.st_mode)) + dir_dgst.update(util.interpolate_bytes( + b"8:fullmode,%d:%s,", len(modestr), modestr)) + elif self._with_metadata_mode: + modestr = util.b(normalized_compatible_mode_str( + fso.stat.st_mode)) + dir_dgst.update(util.interpolate_bytes( + b"4:mode,%d:%s,", len(modestr), modestr)) + if not self._size_only: + dgst = digest.compute_digest_file( + self._algorithm[0], + fso.path, + use_mmap=self._use_mmap) + dir_dgst.update(util.interpolate_bytes( + b"%d:%s,", len(dgst), dgst)) + opath = join_output_path(top, fso.name) + if self._utf8_mode: + opath = walk.WalkDirEntry.alt_u8(opath) + else: + opath = walk.WalkDirEntry.alt_fs(opath) + if self._size_only: + self._outfp.write(format_bsd_line( + "SIZE", None, opath, False, fso.stat.st_size)) + else: + if self._print_size: + self._outfp.write(format_bsd_line( + self._algorithm[1], + dgst, opath, + self._use_base64, + fso.stat.st_size)) + else: + self._outfp.write(format_bsd_line( + self._algorithm[1], dgst, opath, + self._use_base64)) + self._outfp.flush() + opath = join_output_path(top, None) + if opath: + if self._utf8_mode: + opath = walk.WalkDirEntry.alt_u8(opath) + else: + opath = walk.WalkDirEntry.alt_fs(opath) + if self._size_only: + self._outfp.write(format_bsd_line( + "SIZE", None, opath, False, dir_size)) + else: + if dir_tainted: + # + # IMPORTANT: Print errors BEFORE the associated digest line. + # Otherwise the "info" command has a problem. + # + self._outfp.write(format_bsd_line( + b"ERROR", None, b"directory is tainted", False, None)) + logging.error("Directory has filename problems: %r", opath) + if self._print_size: + self._outfp.write(format_bsd_line( + self._algorithm[1], dir_dgst.digest(), opath, + self._use_base64, dir_size)) + else: + self._outfp.write(format_bsd_line( + self._algorithm[1], dir_dgst.digest(), opath, + self._use_base64)) + self._outfp.flush() + return (dir_dgst.digest(), dir_size) + + +def join_output_path(top, name): + if name is None: + # a path for a directory is to be computed + if top: + if isinstance(top[0], bytes): + return b"/".join(top) + b"/" + else: + return u"/".join(top) + u"/" + else: + return b"" + else: + # a path for a normal file is to be computed + if top: + if isinstance(name, bytes): + return b"/".join(top) + b"/" + name + else: + return u"/".join(top) + u"/" + name + else: + return name + + +class CRC32Output(object): + + """Wrapper for a minimal binary file contextmanager that calculates + the CRC32 of the written bytes on the fly. + + Also acts as context manager proxy for the given context manager. + + """ + + __slots__ = ("_fp_cm", "_fp", "_crc32") + + def __init__(self, fp_cm): + self._fp_cm = fp_cm + self._fp = None + self.resetdigest() + + def __enter__(self): + assert self._fp is None + self._fp = self._fp_cm.__enter__() + return self + + def __exit__(self, *args): + rv = self._fp_cm.__exit__(*args) + self._fp = None + return rv + + def write(self, what): + self._fp.write(what) + self._crc32 = zlib.crc32(what, self._crc32) + + def flush(self): + self._fp.flush() + + def resetdigest(self): + """Reset the current CRC digest""" + self._crc32 = zlib.crc32(b"") + + def hexcrcdigest(self): + """ + + :rtype: str + + """ + return (hex(self.crcdigest())[2:]).upper() + + def crcdigest(self): + """ + + :rtype: int + + """ + if util.PY2: + if self._crc32 < 0: + # Return the bitpattern as unsigned 32-bit number + return (~self._crc32 ^ 0xFFFFFFFF) + else: + return self._crc32 + else: + return self._crc32 + + +def normalized_compatible_mode_str(mode): + # XXX FIXME: Windows and "executable" + modebits = stat.S_IMODE(mode) + modestr = "%o" % (modebits,) + if not modestr.startswith("0"): + modestr = "0" + modestr + return modestr + + +def normalized_mode_str(mode): + modestr = "%o" % (mode,) + if not modestr.startswith("0"): + modestr = "0" + modestr + return modestr + + +def format_bsd_line(what, value, filename, use_base64, size=None): + ls = util.b(os.linesep) + if not isinstance(what, bytes): + what = what.encode("ascii") + if what == b"TIMESTAMP": + assert filename is None + return util.interpolate_bytes(b"TIMESTAMP = %d%s", value, ls) + if what in (b"FSENCODING", b"ISOTIMESTAMP", b"FLAGS", b"VERSION", + b"CRC32"): + assert filename is None + return util.interpolate_bytes(b"%s = %s%s", what, util.b(value), ls) + assert filename is not None + if what in (b"COMMENT", b"ERROR", b"GENERATOR"): + return util.interpolate_bytes( + b"%s (%s)%s", what, util.b(filename, "utf-8"), ls) + if not isinstance(filename, bytes): + filename = util.fsencode(filename) + if what == b"SIZE": + return util.interpolate_bytes(b"SIZE (%s) = %d%s", filename, size, ls) + if value is None: + return util.interpolate_bytes(b"%s (%s)%s", what, filename, ls) + if use_base64: + value = base64.b64encode(value) + else: + value = binascii.hexlify(value) + if filename != b"./@/": + filename = util.normalize_filename(filename, True) + if size is None: + return util.interpolate_bytes( + b"%s (%s) = %s%s", what, filename, value, ls) + else: + return util.interpolate_bytes( + b"%s (%s) = %s,%d%s", what, filename, value, size, ls) + + +class TreesumReader(object): + + """Reader to read and/or verify treesum digest files. + + Supports the iterator and context manager protocol. + + """ + + PATTERN0 = re.compile(br"\A[ \t]*\r?\n\Z") # empty lines + PATTERN1 = re.compile(br"\A(VERSION|FSENCODING|FLAGS|TIMESTAMP|ISOTIMESTAMP|CRC32)[ \t]*=[ \t]*([^ \t]+)[ \t]*\r?\n\Z") # noqa: E501 line too long + PATTERN2 = re.compile(br"\A(ROOT|COMMENT|ERROR|GENERATOR)[ \t]*\((.*)\)[ \t]*\r?\n\Z") # noqa: E501 line too long + PATTERN3 = re.compile(br"\ASIZE[ \t]*\((.*)\)[ \t]*=[ \t]*(\d+)[ \t]*\r?\n\Z") # noqa: E501 line too long + PATTERN4 = re.compile(br"\A([A-Za-z0-9_-]+)[ \t]*\((.*)\)[ \t]*=[ \t]*([A-Za-z0-9=+/]+)(,(\d+))?[ \t]*\r?\n\Z") # noqa: E501 line too long + + def __init__(self, _fp, _filename, _own_fp): + self._fp = _fp + self._own_fp = _own_fp + self._filename = _filename + self._line_no = 0 + self._reset_crc() + self._expect_crc = None # NOTE: tristate: None is different from False + self._current_algo_name = self._current_algo_digest_size = None + + @classmethod + def from_path(cls_, path): + """Open file at `path` and return a reader that owns the file object""" + return cls_(open(path, "rb"), path, True) + + @classmethod + def from_binary_buffer(cls_, binary_fp, filename): + return cls_(binary_fp, filename, False) + + def __enter__(self): + return self + + def __exit__(self, *args): + self.close() + + def close(self): + if self._fp is not None: + try: + if self._own_fp: + self._fp.close() + finally: + self._fp = None + + def __iter__(self): + return self + + def __next__(self): + rec = self.read_record() + if rec is None: + raise StopIteration() + return rec + + if util.PY2: + next = __next__ + + def all_records(self): + """Iterator over all remaining records""" + while True: + rec = self.read_record() + if rec is None: + return + yield rec + + def read_record(self): + """Read and parse the "next" line. + + :returns: `None` at EOF or the parsed contents of the line + :rtype: tuple or None + + """ + # Loop to skip empty lines + while True: + line = self._get_next_line() + if not line: + # + # Skip for empty files at the very beginning. + # Check only after the first VERSION line. + # + if self._expect_crc is not None: + if self._expect_crc: + logging.warning("CRC32 is missing at EOF") + return None + if not self.PATTERN0.search(line): + break + self._update_crc(line) + # + # At the beginning transparently skip an eventually embedded signify + # signature + # + if self._line_no == 1: + if line.startswith(b"untrusted comment: "): + line = self._get_next_line() + if not line.endswith(b"\n"): + raise binascii.Error("No valid signify signature value") + # Try to decode for an early error check + base64.b64decode(line[:-1]) + mo = self.PATTERN1.search(line) + if mo: + if mo.group(1) == b"VERSION": + if self._expect_crc: + logging.warning("CRC32 missing before line %d", + self._line_no) + self._reset_crc() + self._expect_crc = True + self._update_crc(line) + return ("VERSION", util.n(mo.group(2))) + if mo.group(1) == b"CRC32": + # TODO: check + if self._expect_crc is None: + logging.warning("Lone CRC32 before VERSION in line %d", + self._line_no) + else: + if self._expect_crc: + if (self._hex_crc() + != mo.group(2).decode("latin1").upper()): + logging.warning( + "CRC32 mismatch in line %d:" + " expected: %s, given: %s", + self._line_no, + self._hex_crc(), + mo.group(2).decode("latin1").upper()) + else: + logging.warning("CRC32 before VERSION in line %d", + self._line_no) + # Do not update the CRC here but reset the state + self._expect_crc = False + return ("CRC32", util.n(mo.group(2))) + else: + self._update_crc(line) + return (util.n(mo.group(1)), util.n(mo.group(2))) + else: + mo = self.PATTERN2.search(line) + if mo: + self._update_crc(line) + if mo.group(1) in (b"COMMENT", b"ERROR", b"GENERATOR"): + return (util.u(mo.group(1)), util.u(mo.group(2), "utf-8")) + elif mo.group(1) == b"ROOT": + return ("ROOT", mo.group(2)) + assert False, line + else: + mo = self.PATTERN3.search(line) + if mo: + self._update_crc(line) + return ("SIZE", mo.group(1), int(util.n(mo.group(2)), 10)) + else: + mo = self.PATTERN4.search(line) + if mo: + self._update_crc(line) + algo_name = util.n(mo.group(1)) + if (len(mo.group(3)) == + 2 * self._get_digest_size(algo_name)): + # hex + digest = binascii.unhexlify(mo.group(3)) + else: + # base64 + digest = base64.b64decode(mo.group(3)) + if mo.group(4): + size = int(util.n(mo.group(5)), 10) + else: + size = None + return (algo_name, mo.group(2), digest, size) + else: + assert False, line + return line + + def _get_next_line(self): + line = self._fp.readline(4096) # along PATH_MAX on Linux + if line: + self._line_no += 1 + return line + + def _reset_crc(self): + self._crc32 = zlib.crc32(b"") + + def _update_crc(self, data): + self._crc32 = zlib.crc32(data, self._crc32) + + def _hex_crc(self): + return (hex(self._get_crc())[2:]).upper() + + def _get_crc(self): + """Get the current CRC always as positive number with the same bit# + pattern because Python2 yields negative numbers also. + + :return: The current CRC value as positive number on all Python + versions + :rtype: int + + """ + if util.PY2: + if self._crc32 < 0: + # Return the bitpattern as unsigned 32-bit number + return (~self._crc32 ^ 0xFFFFFFFF) + else: + return self._crc32 + else: + return self._crc32 + + def _get_digest_size(self, algo_name): + if self._current_algo_name == algo_name: + return self._current_algo_digest_size + h = util.algotag2algotype(algo_name)() + self._current_algo_name = algo_name + self._current_algo_digest_size = h.digest_size + return self._current_algo_digest_size + + +def print_treesum_digestfile_infos(opts): + print_infos_for_digestfile(opts.digest_files, opts.print_only_last_block) + + +def print_infos_for_digestfile(digest_files, print_only_last_block=True): + for fn in digest_files: + if fn == "-": + if util.PY2: + reader = TreesumReader.from_binary_buffer(sys.stdin) + else: + reader = TreesumReader.from_binary_buffer(sys.stdin.buffer) + else: + reader = TreesumReader.from_path(fn) + + with reader: + root = generator = flags = fsencoding = algorithm = digest \ + = size = None + errors = set() + comments = [] + in_block = False + block_no = 0 + for record in reader: + if record[0] == "VERSION": + assert record[1] == "1" + # start a new block + in_block = True + block_no += 1 + root = flags = algorithm = digest = size = None + comments = [] + elif record[0] == "GENERATOR": + generator = record[1] + elif record[0] == "FSENCODING": + fsencoding = record[1] + elif record[0] == "FLAGS": + flags = record[1] + elif record[0] == "ROOT": + root = record[1] + elif record[0] == "COMMENT": + comments.append(record[1]) + elif record[0] == "ERROR": + errors.add(record[1]) + elif record[0] in ("TIMESTAMP", "ISOTIMESTAMP"): + pass + elif record[0] == "CRC32": + pass + # in_block = False + else: + if not in_block: + continue + # digest line or size line + if not record[1] or record[1] == b"./@/": + if record[0] == "SIZE": + algorithm = "SIZE" + digest = None + size = record[2] + else: + algorithm = record[0] + digest = record[2] + size = record[3] + if not print_only_last_block: + print_block_data( + block_no, + root, generator, fsencoding, flags, comments, + errors, algorithm, digest, size) + root = generator = flags = fsencoding = algorithm \ + = digest = size = None + errors = set() + comments = [] + in_block = False + if print_only_last_block: + if not in_block: + if digest is not None or size is not None: + print_block_data( + block_no, + root, generator, fsencoding, flags, comments, errors, + algorithm, digest, size) + else: + logging.warning("missing block end") + + +def print_block_data(block_no, tag, generator, fsencoding, flags, comments, + errors, algorithm, digest, size): + digeststr = util.n(binascii.hexlify(digest)) if digest else "<no digest>" + sizestr = str(size) if size is not None else "<no size>" + print("BLOCK No %d:" % (block_no,)) + print(" Tag:", tag) + print(" FS-Encoding:", fsencoding) + if generator: + print(" Generator:", generator) + print(" Flags:", flags if flags else "<none>") + if comments: + print(" Comments:", comments) + print(" Algorithm:", algorithm) + if algorithm != "SIZE": + print(" Digest:", digeststr) + print(" Size:", sizestr) + print(" Errors:", errors if errors else "<none>") + + +if __name__ == "__main__": + sys.exit(main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/util/__init__.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,325 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Utility package. + +""" + +from __future__ import print_function, absolute_import + + +__all__ = ["PY2", + "PY35", + "n", "b", "u", + "normalize_filename", + "argv2algo", + "algotag2algotype", + "get_blake2b", + "get_blake2b_256", + "get_blake2s", + "default_algotag", + "fsencode", + "interpolate_bytes", + ] + + +import argparse +import hashlib +import os +import sys + + +PY2 = sys.version_info[0] < 3 +PY35 = sys.version_info[:2] >= (3, 5) + + +if PY2: + + def n(s, encoding="ascii"): + """Convert `s` to the native string implementation""" + if isinstance(s, unicode): # noqa: F821 undefined name 'unicode' + return s.encode(encoding) + return s + + def b(s, encoding="ascii"): + """Convert `s` to bytes""" + if isinstance(s, unicode): # noqa: F821 undefined name 'unicode' + return s.encode(encoding) + return s + + def u(s, encoding="ascii"): + """Convert `s` to a unicode string""" + if isinstance(s, str): + return s.decode(encoding) + return s + +else: + + def n(s, encoding="ascii"): + """Convert `s` to the native string implementation""" + if isinstance(s, (bytes, bytearray)): + return s.decode(encoding) + return s + + def b(s, encoding="ascii"): + """Convert `s` to bytes""" + if isinstance(s, str): + return s.encode(encoding) + return s + + u = n + + +def default_algotag(): + """Determine the "best" default algorithm. + + Depend on availability in :mod:`hashlib`. + + Prefer BLAKE2b-256, SHA256 or SHA1 -- in this order. + + Does not consider :mod:`pyblake2` if it is available eventually. + + """ + if "blake2b" in hashlib.algorithms_available: + return "BLAKE2b-256" + if "sha256" in hashlib.algorithms_available: + return "SHA256" + return "SHA1" + + +def get_blake2b(): + """Get the factory for blake2b""" + try: + return hashlib.blake2b + except AttributeError: + import pyblake2 + return pyblake2.blake2b + + +def get_blake2b_256(): + """Get the factory for blake2b-256""" + + try: + hashlib.blake2b + except AttributeError: + import pyblake2 + + def _get_blake(): + return pyblake2.blake2b(digest_size=32) + + else: + + def _get_blake(): + return hashlib.blake2b(digest_size=32) + + return _get_blake + + +def get_blake2s(): + """Get the factory for blake2s""" + try: + return hashlib.blake2s + except AttributeError: + import pyblake2 + return pyblake2.blake2s + + +def get_crc(name): + """Get the factory for a CRC""" + + from ..crcmod.predefined import PredefinedCrc + + def _crc_type(): + return PredefinedCrc(name) + + return _crc_type + + +def argv2algo(s): + """Convert a command line algorithm specifier into a tuple with the + type/factory of the digest and the algorithms tag for output purposes. + + :param str s: the specifier from the command line; should include all + algorithm tags also (for proper round-tripping) + :return: the internal digest specification + :rtype: a tuple (digest_type_or_factory, name_in_output) + :raises argparse.ArgumentTypeError: for unrecognized algorithms or names + + String comparisons are done case-insensitively. + + """ + s = s.lower() + if s in ("1", "sha1"): + return (hashlib.sha1, "SHA1") + elif s in ("224", "sha224"): + return (hashlib.sha224, "SHA224") + elif s in ("256", "sha256"): + return (hashlib.sha256, "SHA256") + elif s in ("384", "sha384"): + return (hashlib.sha384, "SHA384") + elif s in ("512", "sha512"): + return (hashlib.sha512, "SHA512") + elif s in ("3-224", "sha3-224"): + return (hashlib.sha3_224, "SHA3-224") + elif s in ("3-256", "sha3-256"): + return (hashlib.sha3_256, "SHA3-256") + elif s in ("3-384", "sha3-384"): + return (hashlib.sha3_384, "SHA3-384") + elif s in ("3", "3-512", "sha3-512"): + return (hashlib.sha3_512, "SHA3-512") + elif s in ("blake2b", "blake2b-512", "blake2", "blake2-512"): + return (get_blake2b(), "BLAKE2b") + elif s in ("blake2s", "blake2s-256"): + return (get_blake2s(), "BLAKE2s") + elif s in ("blake2-256", "blake2b-256"): + return (get_blake2b_256(), "BLAKE2b-256") + elif s == "md5": + return (hashlib.md5, "MD5") + elif s in ("crc24", "crc-24", + "crc24-openpgp", "crc-24-openpgp"): + return (get_crc("crc-24"), "CRC-24") + elif s in ("crc32", "crc-32", + "crc32-pkzip", "crc-32-pkzip", + "crc32-iso", "crc-32-iso", + "crc32-iso-hdlc", "crc-32-iso-hdlc"): + return (get_crc("crc-32"), "CRC-32-ISO") + elif s in ("crc32-posix", "crc-32-posix", + "crc32-cksum", "crc-32-cksum", + "posix"): + return (get_crc("posix"), "CRC-32-POSIX") + elif s in ("crc64", "crc-64", + "crc64-iso", "crc-64-iso"): + return (get_crc("crc-64"), "CRC-64-ISO") + elif s in ("crc64-2", "crc-64-2", + "crc64-iso-2", "crc-64-iso-2", + "crc64-mcrc64", "crc-64-mcrc64"): + return (get_crc("crc-64-2"), "CRC-64-ISO-2") + elif s in ("crc64-ecma", "crc-64-ecma"): + return (get_crc("crc-64-ecma"), "CRC-64-ECMA") + elif s in ("crc64-xz", "crc-64-xz", + "crc64-go-ecma", "crc-64-go-ecma"): + return (get_crc("crc-64-xz"), "CRC-64-XZ") + elif s in ("crc64-go", "crc-64-go", + "crc64-go-iso", "crc-64-go-iso"): + return (get_crc("crc-64-go"), "CRC-64-GO-ISO") + elif s in ("crc64-redis", "crc-64-redis"): + return (get_crc("crc-64-redis"), "CRC-64-REDIS") + else: + raise argparse.ArgumentTypeError( + "`{}' is not a recognized algorithm".format(s)) + + +def algotag2algotype(s): + """Convert the algorithm specifier in a BSD-style digest file to the + type/factory of the corresponding algorithm. + + :param str s: the tag (i.e. normalized name) or the algorithm + :return: the digest type or factory for `s` + :raises ValueError: on unknown and/or unhandled algorithms + + All string comparisons are case-sensitive. + + """ + if s == "SHA1": + return hashlib.sha1 + elif s == "SHA224": + return hashlib.sha224 + elif s == "SHA256": + return hashlib.sha256 + elif s == "SHA384": + return hashlib.sha384 + elif s == "SHA512": + return hashlib.sha512 + elif s == "SHA3-224": + return hashlib.sha3_224 + elif s == "SHA3-256": + return hashlib.sha3_256 + elif s == "SHA3-384": + return hashlib.sha3_384 + elif s == "SHA3-512": + return hashlib.sha3_512 + elif s in ("BLAKE2b", "BLAKE2b-512", "BLAKE2b512"): # compat for openssl + return get_blake2b() + elif s in ("BLAKE2s", "BLAKE2s-256", "BLAKE2s256"): # compat for openssl + return get_blake2s() + elif s in ("BLAKE2b-256", "BLAKE2b256"): # also compat for openssl dgst + return get_blake2b_256() + elif s == "MD5": + return hashlib.md5 + elif s == "CRC-24": + return get_crc("crc-24") + elif s == "CRC-32-ISO": + return get_crc("crc-32") + elif s == "CRC-32-POSIX": + return get_crc("posix") + elif s == "CRC-64-ISO": + return get_crc("crc-64") + elif s == "CRC-64-ISO-2": + return get_crc("crc-64-2") + elif s == "CRC-64-ECMA": + return get_crc("crc-64-ecma") + elif s == "CRC-64-XZ": + return get_crc("crc-64-xz") + elif s == "CRC-64-GO-ISO": + return get_crc("crc-64-go") + elif s == "CRC-64-REDIS": + return get_crc("crc-64-redis") + else: + raise ValueError("unknown algorithm: {}".format(s)) + + +def normalize_filename(filename, strip_leading_dot_slash=False): + if isinstance(filename, bytes): + filename = filename.replace(b"\\", b"/") + if strip_leading_dot_slash: + while filename.startswith(b"./"): + filename = filename[2:] + else: + filename = filename.replace(u"\\", u"/") + if strip_leading_dot_slash: + while filename.startswith(u"./"): + filename = filename[2:] + return filename + + +def fsencode(what): + """A somewhat compatibility function for :func:`os.fsencode`. + + If `what` is of type :class:`bytes` no :func:`os.fsencode` is required. + + """ + if isinstance(what, bytes): + return what + return os.fsencode(what) + + +def interpolate_bytes(formatstr, *values): + """Interpolate byte strings also on Python 3.4. + + :param bytes formatstr: + :param values: params for interpolation: may *not* contain Unicode strings + :rvalue: the formatted octet + :rtype: bytes + + """ + assert isinstance(formatstr, bytes) + # Python 3.5+ or Python2 know how to interpolate byte strings + if PY35 or PY2: + return formatstr % values + # Workaround with a Latin-1 dance + tformatstr = formatstr.decode("latin1") + tvalues = [] + for v in values: + if PY2: + if isinstance(v, unicode): # noqa: F821 undefined name 'unicode' + assert False + else: + if isinstance(v, str): + assert False + if isinstance(v, bytes): + tvalues.append(v.decode("latin1")) + else: + tvalues.append(v) + return (tformatstr % tuple(tvalues)).encode("latin1")
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/util/cm.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,35 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Context manager extensions and compatibility. + +""" + +from __future__ import print_function, absolute_import + + +__all__ = ["nullcontext"] + + +try: + from contextlib import nullcontext +except ImportError: + class nullcontext(object): + + """Compatibility implementation for systems that are missing yet + a standard :class:`contextlib.nullcontext`. + + """ + + __slots__ = ("thing", ) + + def __init__(self, thing=None): + self.thing = thing + + def __enter__(self): + return self.thing + + def __exit__(self, *args, **kwds): + pass
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/util/constants.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Common constants and compatibility definitions. + +""" + +from __future__ import print_function, absolute_import + + +__all__ = ["PATH_TYPES", + "READ_CHUNK_SIZE", + "MAX_AUTO_MAP_SIZE", + "MAP_WINDOW_SIZE" + ] + + +try: + import pathlib +except ImportError: + pathlib = None + +from . import PY2 + + +if PY2: + PATH_TYPES = (unicode, str) # noqa: F821 (undefined name 'unicode') +else: + if pathlib: + PATH_TYPES = (str, bytes, pathlib.Path) + else: + PATH_TYPES = (str, bytes) + +READ_CHUNK_SIZE = 2 * 1024 * 1024 # like BUFSIZE_MAX on FreeBSD +MAX_AUTO_MAP_SIZE = 8 * 1024 * 1024 +MAP_WINDOW_SIZE = MAX_AUTO_MAP_SIZE # do not totally trash memory on big files
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/util/digest.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,175 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Utility sub-module to implement a file and stream digest computations. + +""" + +from __future__ import print_function, absolute_import + + +__all__ = ["compute_digest_file", "compute_digest_stream"] + + +import errno +import io +import os +try: + import mmap +except ImportError: + mmap = None +import stat + +from . import PY2 +from . import constants + + +def compute_digest_file(hashobj, path, use_mmap=None): + """Compute the digest for a file with a filename of an open fd. + + :param hashobj: a :mod:`hashlib` compatible hash algorithm type or factory + :param path: filename within the filesystem or a file descriptor opened in + binary mode (also a socket or pipe) + :param use_mmap: Use the :mod:`mmap` module if available. + If `None` determine automatically. + :type use_mmap: bool or None + :return: the digest in binary form + :rtype: bytes + + If a file descriptor is given is must support :func:`os.read`. + + """ + h = hashobj() + if isinstance(path, constants.PATH_TYPES): + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) \ + | getattr(os, "O_SEQUENTIAL", 0) | getattr(os, "O_NOCTTY", 0) + fd = os.open(path, flags) + own_fd = True + else: + fd = path + own_fd = False + try: + try: + st = os.fstat(fd) + except TypeError: + # + # "fd" is most probably a Python socket object. + # (a pipe typically supports fstat) + # + use_mmap = False + else: + if stat.S_ISREG(st[stat.ST_MODE]): + filesize = st[stat.ST_SIZE] + if (use_mmap is None) \ + and (filesize > constants.MAX_AUTO_MAP_SIZE): + # + # This is borrowed from FreeBSD's cp(1) implementation: + # Mmap and process if less than 8M (the limit is + # so we don't totally trash memory on big files. + # This is really a minor hack, but it wins some + # CPU back. Some filesystems, such as smbnetfs, + # don't support mmap, so this is a best-effort + # attempt. + # + use_mmap = False + else: + use_mmap = False + if use_mmap is None: + use_mmap = True + if mmap is None or not use_mmap: + # No mmap available or wanted -> use traditional low-level file IO + fadvise = getattr(os, "posix_fadvise", None) + if fadvise: + fadvise(fd, 0, 0, os.POSIX_FADV_SEQUENTIAL) + if not PY2: + fileobj = io.FileIO(fd, mode="r", closefd=False) + buf = bytearray(constants.READ_CHUNK_SIZE) + with memoryview(buf) as full_view: + while True: + try: + n = fileobj.readinto(buf) + except OSError as e: + if e.errno not in (errno.EAGAIN, + errno.EWOULDBLOCK, + errno.EINTR): + raise + else: + if n == 0: + break + if n == constants.READ_CHUNK_SIZE: + h.update(buf) + else: + with full_view[:n] as partial_view: + h.update(partial_view) + else: + while True: + try: + buf = os.read(fd, constants.READ_CHUNK_SIZE) + except OSError as e: + if e.errno not in (errno.EAGAIN, + errno.EWOULDBLOCK, + errno.EINTR): + raise + else: + if len(buf) == 0: + break + h.update(buf) + else: + # + # Use mmap + # + # NOTE: On Windows mmapped files with length 0 are not supported. + # So ensure to not call mmap.mmap() if the file size is 0. + # + madvise = getattr(mmap.mmap, "madvise", None) + if filesize <= constants.MAP_WINDOW_SIZE: + mapsize = filesize + else: + mapsize = constants.MAP_WINDOW_SIZE + mapoffset = 0 + rest = filesize + while rest > 0: + m = mmap.mmap(fd, + mapsize, + access=mmap.ACCESS_READ, + offset=mapoffset) + if madvise: + madvise(m, mmap.MADV_SEQUENTIAL) + try: + h.update(m) + finally: + m.close() + rest -= mapsize + mapoffset += mapsize + if rest < mapsize: + mapsize = rest + finally: + if own_fd: + os.close(fd) + return h.digest() + + +def compute_digest_stream(hashobj, instream): + """Compute the digest for a given byte string `instream`. + + :param hashobj: a :mod:`hashlib` compatible hash algorithm type or factory + :param instream: a bytes input stream to read the data to be hashed from + :return: the digest in binary form + :rtype: bytes + + """ + h = hashobj() + while True: + try: + buf = instream.read(constants.READ_CHUNK_SIZE) + except OSError as e: + if e.errno not in (errno.EAGAIN, errno.EWOULDBLOCK, errno.EINTR): + raise + else: + if buf is not None: + if len(buf) == 0: + break + h.update(buf) + return h.digest()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cutils/util/walk.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,402 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Utility sub-module to implement a heavily customized :func:`os.walk`. + +""" + +from __future__ import print_function, absolute_import + + +__all__ = ["WalkDirEntry", "ScanDir", "getfsencoding"] + + +import os +try: + from os import scandir +except ImportError: + try: + from scandir import scandir + except ImportError: + scandir = None +import sys + +from . import PY2 + + +_notset = object() + + +_FSENCODING = sys.getfilesystemencoding() + + +if PY2: + + def _unix_path(s): + if isinstance(s, bytes): + return s.replace(b"\\", b"/") + return s.replace(u"\\", u"/") + +else: + + def _unix_path(s): + return s.replace("\\", "/") + + +class WalkDirEntry(object): + + """A :class:`os.DirEntry` alike to be used in :func:`walk` and for + its results. + + """ + + __slots__ = ("_name", "_path", # encoded as given in the ctor + "_is_symlink", "_is_dir", "_stat_result", + "_alt_fsname", "_alt_u8name") + + def __init__(self, name, path): + self._name = name # the name as given in the constructor + """The name exactly as given in the ctor""" + self._path = _unix_path(path) + """The path as given in the ctor -- but normalized to have slashes""" + self._is_symlink = self._is_dir = self._stat_result = None + self._alt_fsname = self._alt_u8name = _notset + + @property + def name(self): + """The original name exactly as given in the ctor""" + return self._name + + @property + def path(self): + """The original path exactly as given in the ctor.""" + return self._path + + @property + def fsname(self): + """The name as bytes for the filesystem. + + :rtype: bytes or None + + """ + if PY2: + if isinstance(self._name, bytes): + return self._name + try: + return self._name.encode(_FSENCODING, "strict") + except UnicodeError: + return None + else: + return os.fsencode(self._name) + + @property + def alt_fsname(self): + """Alternative and "escaped" filesystem name -- always bytes. + + :rtype: bytes + + """ + if self._alt_fsname is _notset: + self._alt_fsname = WalkDirEntry.alt_fs(self._name) + return self._alt_fsname + + @property + def fspath(self): + """Always bytes. + + :rtype: bytes or None + + """ + if PY2: + if isinstance(self._path, bytes): + return self._path + try: + return self._path.encode(_FSENCODING, "strict") + except UnicodeError: + return None + else: + return os.fsencode(self._path) + + @property + def alt_fspath(self): + """Alternative and "escaped" filesystem path -- always bytes. + + :rtype: bytes + + """ + return WalkDirEntry.alt_fs(self._path) + + @staticmethod + def alt_fs(what): + if PY2: + if isinstance(what, bytes): + return what + return what.encode(_FSENCODING, "backslashreplace") + else: + return os.fsencode(what) + + @property + def uname(self): + """Always "real", strictly encoded Unicode or `None` if this is not + possible. + + :rtype: text or None + + """ + if PY2: + if isinstance(self._name, bytes): + try: + return self._name.decode(_FSENCODING, "strict") + except UnicodeError: + return None + else: + return self._name + else: + try: + self._name.encode("utf-8", "strict") + except UnicodeError: + return None + return self._name + + @property + def upath(self): + """Always "real", strictly encoded Unicode or `None` if this is not + possible. + + :rtype: text or None + + """ + if PY2: + if isinstance(self._path, bytes): + try: + return self._path.decode(_FSENCODING, "strict") + except UnicodeError: + return None + else: + return self._path + else: + try: + self._path.encode("utf-8", "strict") + except UnicodeError: + return None + return self._path + + @property + def u8name(self): + """`.uname` as UTF-8 or `None` (as strict as `uname`)""" + n = self.uname + return n if n is None else n.encode("utf-8", "strict") + + @property + def u8path(self): + """`.upath` as UTF-8 or `None` (as strict as `upath`""" + p = self.upath + return p if p is None else p.encode("utf-8", "strict") + + @property + def alt_u8name(self): + if self._alt_u8name is _notset: + self._alt_u8name = WalkDirEntry.alt_u8(self._name) + return self._alt_u8name + + @property + def alt_u8path(self): + return WalkDirEntry.alt_u8(self._path) + + @staticmethod + def alt_u8(what): + if PY2: + if isinstance(what, bytes): + try: + return (what.decode(_FSENCODING, "strict") + .encode("utf-8", "strict")) + except UnicodeError: + return (WalkDirEntry.surrogate_decode(what) + .encode("ascii", "backslashreplace")) + else: + return what.encode("ascii", "backslashreplace") + else: + return what.encode("utf-8", "backslashreplace") + + @property + def is_symlink(self): + return self._is_symlink + + @property + def is_dir(self): + return self._is_dir + + @property + def stat(self): + return self._stat_result + + def __repr__(self): + tag = "" + if self._is_symlink: + tag += "l" + if self._is_dir: + tag += "d" + if tag: + return "<WalkDirEntry %r (%s)>" % (self._name, tag) + return "<WalkDirEntry %r>" % (self._name,) + + @classmethod + def from_direntry(cls_, entry): + w = cls_(entry.name, entry.path) + try: + w._is_dir = entry.is_dir(follow_symlinks=True) + except OSError: + # + # If is_dir() raises an OSError, consider that the entry + # is not a directory, same behaviour than os.path.isdir(). + # + w._is_dir = False + try: + w._is_symlink = entry.is_symlink() + except OSError: + # + # If is_symlink() raises an OSError, consider that the entry + # is not a symbolic link, same behaviour than os.path.islink(). + # + w._is_symlink = False + # Do not supress errors here and (consistently) follow symlinks + w._stat_result = entry.stat(follow_symlinks=True) + return w + + @classmethod + def from_path_name(cls_, path, name, _do_stat=True): + """`_nostat` is to be used only for testing purposes""" + w = cls_(name, os.path.join(path, name)) + try: + w._is_dir = os.path.isdir(w._path) + except OSError: + # + # If is_dir() raises an OSError, consider that the entry + # is not a directory, same behaviour than os.path.isdir(). + # + w._is_dir = False + try: + w._is_symlink = os.path.islink(w._path) + except OSError: + # + # If is_symlink() raises an OSError, consider that the entry + # is not a symbolic link, same behaviour than os.path.islink(). + # + w._is_symlink = False + if _do_stat: + w._stat_result = os.stat(w._path) + return w + + @classmethod + def from_readlink(cls_, path): + w = cls_(os.path.basename(path), path) + return w + + @staticmethod + def sort_key_fs(entry): + return entry.alt_fsname # because it should never throw + + @staticmethod + def sort_key_u8(entry): + return entry.alt_u8name # because it should never throw + + if PY2: + + @staticmethod + def surrogate_decode(what): + """Decode the bytes object `what` using surrogates from :pep:`383` + for all non-ASCII octets. + + """ + uwhat = [] + assert isinstance(what, bytes) + for ch in what: + chcode = ord(ch) + if chcode <= 0x7f: + uwhat.append(unichr(chcode)) # noqa: F821 unichr + else: + uwhat.append(unichr(0xDC00 + chcode)) # noqa: F821 unichr + return u"".join(uwhat) + + +if scandir: + + class ScanDir(object): + + """An :func:`os.scandir` wrapper that is always an iterator and + a context manager. + + """ + + __slots__ = ("_scandir_it", ) + + def __init__(self, path): + super(ScanDir, self).__init__() + self._scandir_it = scandir(path) + + def __iter__(self): + return self + + def __next__(self): + if self._scandir_it is None: + raise StopIteration("closed") + return WalkDirEntry.from_direntry(next(self._scandir_it)) + + if PY2: + next = __next__ + + def __enter__(self): + return self + + def __exit__(self, *args, **kwds): + self.close() + + def close(self): + if self._scandir_it is not None: + if hasattr(self._scandir_it, "close"): + self._scandir_it.close() + self._scandir_it = None + +else: + + class ScanDir(object): + + """An :func:`os.scandir` wrapper that is always an iterator and + a context manager. + + """ + + __slots__ = ("_listdir_it", "_path") + + def __init__(self, path): + super(ScanDir, self).__init__() + self._listdir_it = iter(os.listdir(path)) + self._path = path + + def __iter__(self): + return self + + def __next__(self): + if self._listdir_it is None: + raise StopIteration("closed") + return WalkDirEntry.from_path_name(self._path, + next(self._listdir_it)) + + if PY2: + next = __next__ + + def __enter__(self): + return self + + def __exit__(self, *args, **kwds): + pass + + def close(self): + pass + + +def getfsencoding(): + """Return the stored _FSENCODING of this module""" + return _FSENCODING
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/docs/notes.rst Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,81 @@ +.. -*- coding: utf-8; indent-tabs-mode: nil; -*- + + +In a aggregated directory checksum at the end of a block: +A path equals + + ``./@/`` + + Symlink to directory + + ``./@`` + + Symlink to other filesystem object + +Other paths that *end* with: + + ``/./@/`` + + Symlink to a directory + + ``/./@` + + Symlink to other filesystem object + + +Fields: + + ``FSENCODING`` + + The result of :func:`os.getfilesystemencoding` that is in effect when + generating treesum digests (process environment) + + ``ERROR`` + + Errors are reported: + + - For directories if the one of filenames has a filename problem + + ``FLAGS`` + + Some flags are always printed. + + - no file modes and no file mtime is taken into account for digesting + - no file size is printed + + Contains: + + - ``with-metadata-fullmode``: if a file mode is used completely as + given by the OS for digest computation + - ``with-metadata-mode``: if just the "portable" file modes are used + for digest computation + - ``with-metadata-mtime``: if the mtime as ISO-String and truncated to + seconds is used in digest computation + - ``print-size``: if the filesize is to to be printed also -- does not + change digests + - ``fs-encoding``: the filenames are given in FS encoding + - ``utf8-encoding``: the filenames are given in UTF-8 encoding + - ``follow-symlinks-XXX`` and ``no-follow-symlinkx-XXX``: if symlinks are followed on the command + line, while directory walking or for files + - ``size-only``: if no digest is to be computed and only a file's size + is printed and the accumulated file sizes for a directory tree + + ``ROOT`` + + Always printed in UTF-8 encoding + + ``COMMENT`` + + Always printed in UTF-8 encoding + + +Offene Fragen +============= + +VFAT und Encoding +----------------- + +Verfolgung von Encoding-Einstellungen über Mount-Punkte hinweg -- gerade +für externe Medien mit VFAT/MSDOS-FS. + + Erst einmal zurückgestellt
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/dos2unix.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Pure Python implementation of `dos2unix`. + +""" + +from __future__ import absolute_import + +import sys + +import cutils.dos2unix + + +sys.exit(cutils.dos2unix.main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/pyproject.toml Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,4 @@ +# Pure-Python-Build +[build-system] +requires = ["setuptools>=43.0", "wheel>=0.33"] +build-backend = "setuptools.build_meta"
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.cfg Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,59 @@ +[egg_info] +tag_build = +tag_date = 0 +tag_svn_revision = 0 + +[bdist_wheel] +universal = 1 + +[sdist] +formats = gztar + +[metadata] +name = py-cutils +version = attr: cutils.__version__ +description = Pure Python implementation of some coreutils with some extensions +author = Franz Glasner +author_email = fzglas.hg@dom66.de +license = BSD 3-Clause "New" or "Revised" License +url = https://pypi.dom66.de/simple/py-cutils/ +download_url = https://pypi.dom66.de/simple/py-cutils/ +license_files = LICENSE.txt +long_description = file: README.txt +long_description_content_type = text/x-rst +platforms = any +classifiers = + Development Status :: 5 - Production/Stable + Environment :: Console + Intended Audience :: Developers + Intended Audience :: End Users/Desktop + Intended Audience :: System Administrators + License :: OSI Approved :: BSD License + Operating System :: OS Independent + Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 + Topic :: System + Topic :: Utilities + +[options] +include_package_data = False +zip_safe = True +python_requires = >=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.* +packages = + cutils + cutils.util + cutils.crcmod + cutils.crcmod.python2 + cutils.crcmod.python3 + +[options.entry_points] +console_scripts = + py-dos2unix = cutils.dos2unix:main + py-shasum = cutils.shasum:main + py-treesum = cutils.treesum:main + + +[flake8] +exclude = + # Ignore the vendored crcmod2/crcmod sub-package + cutils/crcmod
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/setup.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,6 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +from setuptools import setup + +setup()
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/shasum.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Pure Python implementation of `shasum`. + +""" + +from __future__ import absolute_import + +import sys + +import cutils.shasum + + +sys.exit(cutils.shasum.main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/_test_setup.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +r"""Unit tests + +""" + +import os +import sys + + +TESTDIR = os.path.normpath(os.path.abspath(os.path.dirname(__file__))) +DATADIR = os.path.join(TESTDIR, "data") + + +sys.path.insert(0, os.path.join(TESTDIR, "..")) + +del os +del sys
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_crcmod.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,16 @@ +# -*- coding: utf-8 -*- +r"""Unit tests for the vendored crcmod2 package. + +""" + +from __future__ import absolute_import, print_function + +import unittest + +import _test_setup + +import cutils.crcmod.test + + +if __name__ == "__main__": + unittest.main(module=cutils.crcmod.test)
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_shasum.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +r"""Unit tests + +""" + +from __future__ import absolute_import, print_function + +import io +import os +import sys +import unittest +try: + from StringIO import StringIO +except ImportError: + StringIO = None + +from _test_setup import DATADIR + +from cutils import shasum + + +PY2 = sys.version_info[0] <= 2 + + +def _memfile(): + if StringIO: + return StringIO() + else: + return io.StringIO() + + +class ChangedDir(object): + + """Context manager to temporarily change the directory""" + + def __init__(self, path): + self._new_dir = path + self._prev_dir = None + + def __enter__(self): + self._prev_dir = os.getcwd() + os.chdir(self._new_dir) + + def __exit__(self, *args, **kwds): + if self._prev_dir is not None: + os.chdir(self._prev_dir) + self._prev_dir = None + + +class SignifyTests(unittest.TestCase): + + def test_empty(self): + destfile = _memfile() + opts = shasum.gen_opts(algorithm="SHA256", + dest=destfile, + files=[os.path.join(DATADIR, "empty")]) + shasum.shasum(opts) + self.assertTrue( + destfile.getvalue().startswith( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 *")) + + def test_empty_with_name(self): + destfile = _memfile() + with ChangedDir(DATADIR): + opts = shasum.gen_opts(algorithm="SHA256", + dest=destfile, + files=["empty"]) + shasum.shasum(opts) + self.assertEqual( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 *empty\n", + destfile.getvalue()) + + def test_empty_with_name_bsd(self): + destfile = _memfile() + with ChangedDir(DATADIR): + opts = shasum.gen_opts(algorithm="SHA512", + dest=destfile, + files=["empty"], + bsd=True) + shasum.shasum(opts) + self.assertEqual( + "SHA512 (empty) = cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e\n", + destfile.getvalue()) + + def test_empty_mmap(self): + destfile = _memfile() + opts = shasum.gen_opts(algorithm="SHA256", + dest=destfile, + files=[os.path.join(DATADIR, "empty")], + mmap=True) + shasum.shasum(opts) + self.assertTrue( + destfile.getvalue().startswith( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 *")) + + def test_empty_no_mmap(self): + destfile = _memfile() + opts = shasum.gen_opts(algorithm="SHA256", + dest=destfile, + files=[os.path.join(DATADIR, "empty")], + mmap=False) + shasum.shasum(opts) + self.assertTrue( + destfile.getvalue().startswith( + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 *")) + + +if __name__ == "__main__": + sys.exit(unittest.main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/tests/test_walk.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,200 @@ +# -*- coding: utf-8 -*- +r"""Unit tests + +""" + +from __future__ import absolute_import, print_function + +import sys +import unittest + +import _test_setup # noqa: F401 imported but unused + +from cutils.util import walk + + +class SurrogateEscapeTests(unittest.TestCase): + + @unittest.skipIf(sys.version_info[0] >= 3, "Skip on Python3") + def test_simple_escape(self): + w = b"\xc4" + + d = walk.WalkDirEntry.surrogate_decode(w) + self.assertEqual(u"\udcc4", d) + + @unittest.skipIf(sys.version_info[0] >= 3, "Skip on Python3") + def test_no_escape_min(self): + w = b"\x00" + + d = walk.WalkDirEntry.surrogate_decode(w) + self.assertEqual(u"\x00", d) + + @unittest.skipIf(sys.version_info[0] >= 3, "Skip on Python3") + def test_no_escape_max(self): + w = b"\x7f" + + d = walk.WalkDirEntry.surrogate_decode(w) + self.assertEqual(u"\x7f", d) + + @unittest.skipIf(sys.version_info[0] >= 3, "Skip on Python3") + def test_escape_min(self): + w = b"\x80" + + d = walk.WalkDirEntry.surrogate_decode(w) + self.assertEqual(u"\udc80", d) + + @unittest.skipIf(sys.version_info[0] >= 3, "Skip on Python3") + def test_escape_max(self): + w = b"\xff" + + d = walk.WalkDirEntry.surrogate_decode(w) + self.assertEqual(u"\udcff", d) + + @unittest.skipIf(sys.version_info[0] >= 3, "Skip on Python3") + def test_complex(self): + w = b"abc\xc4d\x80\x81\xffefg" + d = walk.WalkDirEntry.surrogate_decode(w) + self.assertEqual(u"abc\udcc4d\udc80\udc81\udcffefg", d) + + +class WalkDirEntryTests(unittest.TestCase): + + def setUp(self): + self._orig_fsencoding = walk._FSENCODING + walk._FSENCODING = "ascii" + + def tearDown(self): + walk._FSENCODING = self._orig_fsencoding + + def test_ascii(self): + entry = walk.WalkDirEntry.from_path_name("tests", "_test_setup.py") + self.assertEqual("_test_setup.py", entry.name) + self.assertEqual("tests/_test_setup.py", entry.path) + self.assertEqual(u"_test_setup.py", entry.uname) + self.assertEqual(u"tests/_test_setup.py", entry.upath) + self.assertEqual(b"_test_setup.py", entry.u8name) + self.assertEqual(b"tests/_test_setup.py", entry.u8path) + self.assertEqual(b"_test_setup.py", entry.alt_u8name) + self.assertEqual(b"tests/_test_setup.py", entry.alt_u8path) + self.assertEqual(b"_test_setup.py", entry.alt_fsname) + self.assertEqual(b"tests/_test_setup.py", entry.alt_fspath) + + @unittest.skipIf(sys.version_info[0] < 3, "Skip on Python2") + def test_with_surrogate_escaped_name(self): + # instantiate with a surrogate escaped path from PEP 383 + entry = walk.WalkDirEntry.from_path_name( + "tests", "test-\udcc4", _do_stat=False) + self.assertEqual("test-\udcc4", entry.name) + self.assertEqual("tests/test-\udcc4", entry.path) + self.assertEqual(b"test-\xc4", entry.fsname) + self.assertEqual(b"tests/test-\xc4", entry.fspath) + self.assertEqual(b"test-\xc4", entry.alt_fsname) + self.assertEqual(b"tests/test-\xc4", entry.alt_fspath) + + self.assertIsNone(entry.uname) + self.assertIsNone(entry.upath) + self.assertIsNone(entry.u8name) + self.assertIsNone(entry.u8path) + + self.assertEqual(b"test-\\udcc4", entry.alt_u8name) + self.assertEqual(b"tests/test-\\udcc4", entry.alt_u8path) + + @unittest.skipIf(sys.version_info[0] < 3, "Skip on Python2") + def test_with_surrogate_escaped_path(self): + # instantiate with a surrogate escaped path from PEP 383 + entry = walk.WalkDirEntry.from_path_name( + "tests\udcc5", "test", _do_stat=False) + self.assertEqual("test", entry.name) + self.assertEqual("tests\udcc5/test", entry.path) + self.assertEqual(b"test", entry.fsname) + self.assertEqual(b"tests\xc5/test", entry.fspath) + self.assertEqual(b"test", entry.alt_fsname) + self.assertEqual(b"tests\xc5/test", entry.alt_fspath) + + self.assertEqual("test", entry.uname) + self.assertIsNone(entry.upath) + self.assertEqual(b"test", entry.u8name) + self.assertIsNone(entry.u8path) + + self.assertEqual(b"test", entry.alt_u8name) + self.assertEqual(b"tests\\udcc5/test", entry.alt_u8path) + + @unittest.skipIf(sys.version_info[0] > 2, "Skip on Python3") + def test_py2_with_non_fsdecodable_name(self): + entry = walk.WalkDirEntry.from_path_name( + b"tests", b"test-\xc4", _do_stat=False) + self.assertEqual(b"test-\xc4", entry.name) + self.assertEqual(b"tests/test-\xc4", entry.path) + self.assertEqual(b"test-\xc4", entry.fsname) + self.assertEqual(b"tests/test-\xc4", entry.fspath) + self.assertEqual(b"test-\xc4", entry.alt_fsname) + self.assertEqual(b"tests/test-\xc4", entry.alt_fspath) + + self.assertIsNone(entry.uname) + self.assertIsNone(entry.upath) + self.assertIsNone(entry.u8name) + self.assertIsNone(entry.u8path) + + self.assertEqual(b"test-\\udcc4", entry.alt_u8name) + self.assertEqual(b"tests/test-\\udcc4", entry.alt_u8path) + + @unittest.skipIf(sys.version_info[0] > 2, "Skip on Python3") + def test_py2_with_non_fsdecodable_path(self): + entry = walk.WalkDirEntry.from_path_name( + b"tests\xc5", b"test", _do_stat=False) + self.assertEqual(b"test", entry.name) + self.assertEqual(b"tests\xc5/test", entry.path) + self.assertEqual(b"test", entry.fsname) + self.assertEqual(b"tests\xc5/test", entry.fspath) + self.assertEqual(b"test", entry.alt_fsname) + self.assertEqual(b"tests\xc5/test", entry.alt_fspath) + + self.assertEqual(b"test", entry.uname) + self.assertIsNone(entry.upath) + self.assertEqual(b"test", entry.u8name) + self.assertIsNone(entry.u8path) + + self.assertEqual(b"test", entry.alt_u8name) + self.assertEqual(b"tests\\udcc5/test", entry.alt_u8path) + + @unittest.skipIf(sys.version_info[0] > 2, "Skip on Python3") + def test_py2_with_non_fsencodable_unicode_name(self): + entry = walk.WalkDirEntry.from_path_name( + u"tests", u"test-\xc4", _do_stat=False) + self.assertEqual(u"test-\xc4", entry.name) + self.assertEqual(u"tests/test-\xc4", entry.path) + self.assertIsNone(entry.fsname) + self.assertIsNone(entry.fspath) + self.assertEqual(b"test-\\xc4", entry.alt_fsname) + self.assertEqual(b"tests/test-\\xc4", entry.alt_fspath) + + self.assertEqual(u"test-\xc4", entry.uname) + self.assertEqual(u"tests/test-\xc4", entry.upath) + self.assertEqual(b"test-\xc3\x84", entry.u8name) + self.assertEqual(b"tests/test-\xc3\x84", entry.u8path) + + self.assertEqual(b"test-\\xc4", entry.alt_u8name) + self.assertEqual(b"tests/test-\\xc4", entry.alt_u8path) + + @unittest.skipIf(sys.version_info[0] > 2, "Skip on Python3") + def test_py2_with_non_fsencodable_unicode_path(self): + entry = walk.WalkDirEntry.from_path_name( + u"tests\xc5", u"test", _do_stat=False) + self.assertEqual(u"test", entry.name) + self.assertEqual(u"tests\xc5/test", entry.path) + self.assertEqual(b"test", entry.fsname) + self.assertIsNone(entry.fspath) + self.assertEqual(b"test", entry.alt_fsname) + self.assertEqual(b"tests\\xc5/test", entry.alt_fspath) + + self.assertEqual(u"test", entry.uname) + self.assertEqual(u"tests\xc5/test", entry.upath) + self.assertEqual(b"test", entry.u8name) + self.assertEqual(b"tests\xc3\x85/test", entry.u8path) + + self.assertEqual(b"test", entry.alt_u8name) + self.assertEqual(b"tests\\xc5/test", entry.alt_u8path) + + +if __name__ == "__main__": + sys.exit(unittest.main())
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/treesum.py Fri Feb 07 12:38:14 2025 +0100 @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# :- +# :Copyright: (c) 2020-2025 Franz Glasner +# :License: BSD-3-Clause +# :- +r"""Pure Python implementation of a directory tree checksum. + +""" + +from __future__ import absolute_import + +import sys + +import cutils.treesum + + +sys.exit(cutils.treesum.main())