Mercurial > hgrepos > Python2 > PyMuPDF
diff mupdf-source/thirdparty/zxing-cpp/core/src/TextDecoder.cpp @ 2:b50eed0cc0ef upstream
ADD: MuPDF v1.26.7: the MuPDF source as downloaded by a default build of PyMuPDF 1.26.4.
The directory name has changed: no version number in the expanded directory now.
| author | Franz Glasner <fzglas.hg@dom66.de> |
|---|---|
| date | Mon, 15 Sep 2025 11:43:07 +0200 |
| parents | |
| children |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mupdf-source/thirdparty/zxing-cpp/core/src/TextDecoder.cpp Mon Sep 15 11:43:07 2025 +0200 @@ -0,0 +1,235 @@ +/* +* Copyright 2016 Nu-book Inc. +* Copyright 2022 gitlost +*/ +// SPDX-License-Identifier: Apache-2.0 + +#include "TextDecoder.h" + +#include "CharacterSet.h" +#include "ECI.h" +#include "Utf.h" +#include "ZXAlgorithms.h" +#include "libzueci/zueci.h" + +#include <cassert> +#include <stdexcept> + +namespace ZXing { + +void TextDecoder::Append(std::string& str, const uint8_t* bytes, size_t length, CharacterSet charset, bool sjisASCII) +{ + int eci = ToInt(ToECI(charset)); + const size_t str_len = str.length(); + const int bytes_len = narrow_cast<int>(length); + constexpr unsigned int replacement = 0xFFFD; + const unsigned int flags = ZUECI_FLAG_SB_STRAIGHT_THRU | (sjisASCII ? ZUECI_FLAG_SJIS_STRAIGHT_THRU : 0); + int utf8_len; + + if (eci == -1) + eci = 899; // Binary + + int error_number = zueci_dest_len_utf8(eci, bytes, bytes_len, replacement, flags, &utf8_len); + if (error_number >= ZUECI_ERROR) + throw std::runtime_error("zueci_dest_len_utf8 failed"); + + str.resize(str_len + utf8_len); // Precise length + unsigned char *utf8_buf = reinterpret_cast<unsigned char *>(str.data()) + str_len; + + error_number = zueci_eci_to_utf8(eci, bytes, bytes_len, replacement, flags, utf8_buf, &utf8_len); + if (error_number >= ZUECI_ERROR) { + str.resize(str_len); + throw std::runtime_error("zueci_eci_to_utf8 failed"); + } + assert(str.length() == str_len + utf8_len); +} + +void TextDecoder::Append(std::wstring& str, const uint8_t* bytes, size_t length, CharacterSet charset) +{ + std::string u8str; + Append(u8str, bytes, length, charset); + str.append(FromUtf8(u8str)); +} + +/** +* @param bytes bytes encoding a string, whose encoding should be guessed +* @return name of guessed encoding; at the moment will only guess one of: +* {@link #SHIFT_JIS}, {@link #UTF8}, {@link #ISO88591}, or the platform +* default encoding if none of these can possibly be correct +*/ +CharacterSet +TextDecoder::GuessEncoding(const uint8_t* bytes, size_t length, CharacterSet fallback) +{ + // For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS, + // which should be by far the most common encodings. + bool canBeISO88591 = true; + bool canBeShiftJIS = true; + bool canBeUTF8 = true; + int utf8BytesLeft = 0; + //int utf8LowChars = 0; + int utf2BytesChars = 0; + int utf3BytesChars = 0; + int utf4BytesChars = 0; + int sjisBytesLeft = 0; + //int sjisLowChars = 0; + int sjisKatakanaChars = 0; + //int sjisDoubleBytesChars = 0; + int sjisCurKatakanaWordLength = 0; + int sjisCurDoubleBytesWordLength = 0; + int sjisMaxKatakanaWordLength = 0; + int sjisMaxDoubleBytesWordLength = 0; + //int isoLowChars = 0; + //int isoHighChars = 0; + int isoHighOther = 0; + + bool utf8bom = length > 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF; + + for (size_t i = 0; i < length && (canBeISO88591 || canBeShiftJIS || canBeUTF8); ++i) + { + int value = bytes[i]; + + // UTF-8 stuff + if (canBeUTF8) { + if (utf8BytesLeft > 0) { + if ((value & 0x80) == 0) { + canBeUTF8 = false; + } + else { + utf8BytesLeft--; + } + } + else if ((value & 0x80) != 0) { + if ((value & 0x40) == 0) { + canBeUTF8 = false; + } + else { + utf8BytesLeft++; + if ((value & 0x20) == 0) { + utf2BytesChars++; + } + else { + utf8BytesLeft++; + if ((value & 0x10) == 0) { + utf3BytesChars++; + } + else { + utf8BytesLeft++; + if ((value & 0x08) == 0) { + utf4BytesChars++; + } + else { + canBeUTF8 = false; + } + } + } + } + } //else { + //utf8LowChars++; + //} + } + + // ISO-8859-1 stuff + if (canBeISO88591) { + if (value > 0x7F && value < 0xA0) { + canBeISO88591 = false; + } + else if (value > 0x9F) { + if (value < 0xC0 || value == 0xD7 || value == 0xF7) { + isoHighOther++; + } //else { + //isoHighChars++; + //} + } //else { + //isoLowChars++; + //} + } + + // Shift_JIS stuff + if (canBeShiftJIS) { + if (sjisBytesLeft > 0) { + if (value < 0x40 || value == 0x7F || value > 0xFC) { + canBeShiftJIS = false; + } + else { + sjisBytesLeft--; + } + } + else if (value == 0x80 || value == 0xA0 || value > 0xEF) { + canBeShiftJIS = false; + } + else if (value < 0x20 && value != 0xa && value != 0xd) { + canBeShiftJIS = false; // use non-printable ASCII as indication for binary content + } + else if (value > 0xA0 && value < 0xE0) { + sjisKatakanaChars++; + sjisCurDoubleBytesWordLength = 0; + sjisCurKatakanaWordLength++; + if (sjisCurKatakanaWordLength > sjisMaxKatakanaWordLength) { + sjisMaxKatakanaWordLength = sjisCurKatakanaWordLength; + } + } + else if (value > 0x7F) { + sjisBytesLeft++; + //sjisDoubleBytesChars++; + sjisCurKatakanaWordLength = 0; + sjisCurDoubleBytesWordLength++; + if (sjisCurDoubleBytesWordLength > sjisMaxDoubleBytesWordLength) { + sjisMaxDoubleBytesWordLength = sjisCurDoubleBytesWordLength; + } + } + else { + //sjisLowChars++; + sjisCurKatakanaWordLength = 0; + sjisCurDoubleBytesWordLength = 0; + } + } + } + + if (canBeUTF8 && utf8BytesLeft > 0) { + canBeUTF8 = false; + } + if (canBeShiftJIS && sjisBytesLeft > 0) { + canBeShiftJIS = false; + } + + // Easy -- if there is BOM or at least 1 valid not-single byte character (and no evidence it can't be UTF-8), done + if (canBeUTF8 && (utf8bom || utf2BytesChars + utf3BytesChars + utf4BytesChars > 0)) { + return CharacterSet::UTF8; + } + + bool assumeShiftJIS = fallback == CharacterSet::Shift_JIS || fallback == CharacterSet::EUC_JP; + // Easy -- if assuming Shift_JIS or at least 3 valid consecutive not-ascii characters (and no evidence it can't be), done + if (canBeShiftJIS && (assumeShiftJIS || sjisMaxKatakanaWordLength >= 3 || sjisMaxDoubleBytesWordLength >= 3)) { + return CharacterSet::Shift_JIS; + } + // Distinguishing Shift_JIS and ISO-8859-1 can be a little tough for short words. The crude heuristic is: + // - If we saw + // - only two consecutive katakana chars in the whole text, or + // - at least 10% of bytes that could be "upper" not-alphanumeric Latin1, + // - then we conclude Shift_JIS, else ISO-8859-1 + if (canBeISO88591 && canBeShiftJIS) { + return (sjisMaxKatakanaWordLength == 2 && sjisKatakanaChars == 2) || isoHighOther * 10 >= (int)length + ? CharacterSet::Shift_JIS : CharacterSet::ISO8859_1; + } + + // Otherwise, try in order ISO-8859-1, Shift JIS, UTF-8 and fall back to default platform encoding + if (canBeISO88591) { + return CharacterSet::ISO8859_1; + } + if (canBeShiftJIS) { + return CharacterSet::Shift_JIS; + } + if (canBeUTF8) { + return CharacterSet::UTF8; + } + // Otherwise, we take a wild guess with platform encoding + return fallback; +} + +CharacterSet +TextDecoder::DefaultEncoding() +{ + return CharacterSet::ISO8859_1; +} + +} // ZXing
