diff mupdf-source/thirdparty/zxing-cpp/core/src/TextDecoder.cpp @ 2:b50eed0cc0ef upstream

ADD: MuPDF v1.26.7: the MuPDF source as downloaded by a default build of PyMuPDF 1.26.4. The directory name has changed: no version number in the expanded directory now.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 15 Sep 2025 11:43:07 +0200
parents
children
line wrap: on
line diff
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/mupdf-source/thirdparty/zxing-cpp/core/src/TextDecoder.cpp	Mon Sep 15 11:43:07 2025 +0200
@@ -0,0 +1,235 @@
+/*
+* Copyright 2016 Nu-book Inc.
+* Copyright 2022 gitlost
+*/
+// SPDX-License-Identifier: Apache-2.0
+
+#include "TextDecoder.h"
+
+#include "CharacterSet.h"
+#include "ECI.h"
+#include "Utf.h"
+#include "ZXAlgorithms.h"
+#include "libzueci/zueci.h"
+
+#include <cassert>
+#include <stdexcept>
+
+namespace ZXing {
+
+void TextDecoder::Append(std::string& str, const uint8_t* bytes, size_t length, CharacterSet charset, bool sjisASCII)
+{
+	int eci = ToInt(ToECI(charset));
+	const size_t str_len = str.length();
+	const int bytes_len = narrow_cast<int>(length);
+	constexpr unsigned int replacement = 0xFFFD;
+	const unsigned int flags = ZUECI_FLAG_SB_STRAIGHT_THRU | (sjisASCII ? ZUECI_FLAG_SJIS_STRAIGHT_THRU : 0);
+	int utf8_len;
+
+	if (eci == -1)
+		eci = 899; // Binary
+
+	int error_number = zueci_dest_len_utf8(eci, bytes, bytes_len, replacement, flags, &utf8_len);
+	if (error_number >= ZUECI_ERROR)
+		throw std::runtime_error("zueci_dest_len_utf8 failed");
+
+	str.resize(str_len + utf8_len); // Precise length
+	unsigned char *utf8_buf = reinterpret_cast<unsigned char *>(str.data()) + str_len;
+
+	error_number = zueci_eci_to_utf8(eci, bytes, bytes_len, replacement, flags, utf8_buf, &utf8_len);
+	if (error_number >= ZUECI_ERROR) {
+		str.resize(str_len);
+		throw std::runtime_error("zueci_eci_to_utf8 failed");
+	}
+	assert(str.length() == str_len + utf8_len);
+}
+
+void TextDecoder::Append(std::wstring& str, const uint8_t* bytes, size_t length, CharacterSet charset)
+{
+	std::string u8str;
+	Append(u8str, bytes, length, charset);
+	str.append(FromUtf8(u8str));
+}
+
+/**
+* @param bytes bytes encoding a string, whose encoding should be guessed
+* @return name of guessed encoding; at the moment will only guess one of:
+*  {@link #SHIFT_JIS}, {@link #UTF8}, {@link #ISO88591}, or the platform
+*  default encoding if none of these can possibly be correct
+*/
+CharacterSet
+TextDecoder::GuessEncoding(const uint8_t* bytes, size_t length, CharacterSet fallback)
+{
+	// For now, merely tries to distinguish ISO-8859-1, UTF-8 and Shift_JIS,
+	// which should be by far the most common encodings.
+	bool canBeISO88591 = true;
+	bool canBeShiftJIS = true;
+	bool canBeUTF8 = true;
+	int utf8BytesLeft = 0;
+	//int utf8LowChars = 0;
+	int utf2BytesChars = 0;
+	int utf3BytesChars = 0;
+	int utf4BytesChars = 0;
+	int sjisBytesLeft = 0;
+	//int sjisLowChars = 0;
+	int sjisKatakanaChars = 0;
+	//int sjisDoubleBytesChars = 0;
+	int sjisCurKatakanaWordLength = 0;
+	int sjisCurDoubleBytesWordLength = 0;
+	int sjisMaxKatakanaWordLength = 0;
+	int sjisMaxDoubleBytesWordLength = 0;
+	//int isoLowChars = 0;
+	//int isoHighChars = 0;
+	int isoHighOther = 0;
+
+	bool utf8bom = length > 3 && bytes[0] == 0xEF && bytes[1] == 0xBB && bytes[2] == 0xBF;
+
+	for (size_t i = 0; i < length && (canBeISO88591 || canBeShiftJIS || canBeUTF8); ++i)
+	{
+		int value = bytes[i];
+
+		// UTF-8 stuff
+		if (canBeUTF8) {
+			if (utf8BytesLeft > 0) {
+				if ((value & 0x80) == 0) {
+					canBeUTF8 = false;
+				}
+				else {
+					utf8BytesLeft--;
+				}
+			}
+			else if ((value & 0x80) != 0) {
+				if ((value & 0x40) == 0) {
+					canBeUTF8 = false;
+				}
+				else {
+					utf8BytesLeft++;
+					if ((value & 0x20) == 0) {
+						utf2BytesChars++;
+					}
+					else {
+						utf8BytesLeft++;
+						if ((value & 0x10) == 0) {
+							utf3BytesChars++;
+						}
+						else {
+							utf8BytesLeft++;
+							if ((value & 0x08) == 0) {
+								utf4BytesChars++;
+							}
+							else {
+								canBeUTF8 = false;
+							}
+						}
+					}
+				}
+			} //else {
+			  //utf8LowChars++;
+			  //}
+		}
+
+		// ISO-8859-1 stuff
+		if (canBeISO88591) {
+			if (value > 0x7F && value < 0xA0) {
+				canBeISO88591 = false;
+			}
+			else if (value > 0x9F) {
+				if (value < 0xC0 || value == 0xD7 || value == 0xF7) {
+					isoHighOther++;
+				} //else {
+				  //isoHighChars++;
+				  //}
+			} //else {
+			  //isoLowChars++;
+			  //}
+		}
+
+		// Shift_JIS stuff
+		if (canBeShiftJIS) {
+			if (sjisBytesLeft > 0) {
+				if (value < 0x40 || value == 0x7F || value > 0xFC) {
+					canBeShiftJIS = false;
+				}
+				else {
+					sjisBytesLeft--;
+				}
+			}
+			else if (value == 0x80 || value == 0xA0 || value > 0xEF) {
+				canBeShiftJIS = false;
+			}
+			else if (value < 0x20 && value != 0xa && value != 0xd) {
+				canBeShiftJIS = false; // use non-printable ASCII as indication for binary content
+			}
+			else if (value > 0xA0 && value < 0xE0) {
+				sjisKatakanaChars++;
+				sjisCurDoubleBytesWordLength = 0;
+				sjisCurKatakanaWordLength++;
+				if (sjisCurKatakanaWordLength > sjisMaxKatakanaWordLength) {
+					sjisMaxKatakanaWordLength = sjisCurKatakanaWordLength;
+				}
+			}
+			else if (value > 0x7F) {
+				sjisBytesLeft++;
+				//sjisDoubleBytesChars++;
+				sjisCurKatakanaWordLength = 0;
+				sjisCurDoubleBytesWordLength++;
+				if (sjisCurDoubleBytesWordLength > sjisMaxDoubleBytesWordLength) {
+					sjisMaxDoubleBytesWordLength = sjisCurDoubleBytesWordLength;
+				}
+			}
+			else {
+				//sjisLowChars++;
+				sjisCurKatakanaWordLength = 0;
+				sjisCurDoubleBytesWordLength = 0;
+			}
+		}
+	}
+
+	if (canBeUTF8 && utf8BytesLeft > 0) {
+		canBeUTF8 = false;
+	}
+	if (canBeShiftJIS && sjisBytesLeft > 0) {
+		canBeShiftJIS = false;
+	}
+
+	// Easy -- if there is BOM or at least 1 valid not-single byte character (and no evidence it can't be UTF-8), done
+	if (canBeUTF8 && (utf8bom || utf2BytesChars + utf3BytesChars + utf4BytesChars > 0)) {
+		return CharacterSet::UTF8;
+	}
+
+	bool assumeShiftJIS = fallback == CharacterSet::Shift_JIS || fallback == CharacterSet::EUC_JP;
+	// Easy -- if assuming Shift_JIS or at least 3 valid consecutive not-ascii characters (and no evidence it can't be), done
+	if (canBeShiftJIS && (assumeShiftJIS || sjisMaxKatakanaWordLength >= 3 || sjisMaxDoubleBytesWordLength >= 3)) {
+		return CharacterSet::Shift_JIS;
+	}
+	// Distinguishing Shift_JIS and ISO-8859-1 can be a little tough for short words. The crude heuristic is:
+	// - If we saw
+	//   - only two consecutive katakana chars in the whole text, or
+	//   - at least 10% of bytes that could be "upper" not-alphanumeric Latin1,
+	// - then we conclude Shift_JIS, else ISO-8859-1
+	if (canBeISO88591 && canBeShiftJIS) {
+		return (sjisMaxKatakanaWordLength == 2 && sjisKatakanaChars == 2) || isoHighOther * 10 >= (int)length
+			? CharacterSet::Shift_JIS : CharacterSet::ISO8859_1;
+	}
+
+	// Otherwise, try in order ISO-8859-1, Shift JIS, UTF-8 and fall back to default platform encoding
+	if (canBeISO88591) {
+		return CharacterSet::ISO8859_1;
+	}
+	if (canBeShiftJIS) {
+		return CharacterSet::Shift_JIS;
+	}
+	if (canBeUTF8) {
+		return CharacterSet::UTF8;
+	}
+	// Otherwise, we take a wild guess with platform encoding
+	return fallback;
+}
+
+CharacterSet
+TextDecoder::DefaultEncoding()
+{
+	return CharacterSet::ISO8859_1;
+}
+
+} // ZXing