Mercurial > hgrepos > Python2 > PyMuPDF

comparison mupdf-source/thirdparty/tesseract/src/training/unicharset_extractor.cpp @ 2:b50eed0cc0ef upstream

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
ADD: MuPDF v1.26.7: the MuPDF source as downloaded by a default build of PyMuPDF 1.26.4. The directory name has changed: no version number in the expanded directory now.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 15 Sep 2025 11:43:07 +0200
parents
children
comparison
equal deleted inserted replaced
1:1d09e1dec1d9 2:b50eed0cc0ef
1 ///////////////////////////////////////////////////////////////////////
2 // File: unicharset_extractor.cpp
3 // Description: Unicode character/ligature set extractor.
4 // Author: Thomas Kielbus
5 //
6 // (C) Copyright 2006, Google Inc.
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
17 ///////////////////////////////////////////////////////////////////////
18
19 // Given a list of box files or text files on the command line, this program
20 // normalizes the text according to command-line options and generates
21 // a unicharset.
22
23 #include <cstdlib>
24 #include <filesystem>
25 #include "boxread.h"
26 #include "commandlineflags.h"
27 #include "commontraining.h" // CheckSharedLibraryVersion
28 #include "lang_model_helpers.h"
29 #include "normstrngs.h"
30 #include "unicharset.h"
31 #include "unicharset_training_utils.h"
32
33 using namespace tesseract;
34
35 static STRING_PARAM_FLAG(output_unicharset, "unicharset", "Output file path");
36 static INT_PARAM_FLAG(norm_mode, 1,
37 "Normalization mode: 1=Combine graphemes, "
38 "2=Split graphemes, 3=Pure unicode");
39
40 namespace tesseract {
41
42 // Helper normalizes and segments the given strings according to norm_mode, and
43 // adds the segmented parts to unicharset.
44 static void AddStringsToUnicharset(const std::vector<std::string> &strings, int norm_mode,
45 UNICHARSET *unicharset) {
46 for (const auto &string : strings) {
47 std::vector<std::string> normalized;
48 if (NormalizeCleanAndSegmentUTF8(UnicodeNormMode::kNFC, OCRNorm::kNone,
49 static_cast<GraphemeNormMode>(norm_mode),
50 /*report_errors*/ true, string.c_str(), &normalized)) {
51 for (const std::string &normed : normalized) {
52 // normed is a UTF-8 encoded string
53 if (normed.empty() || IsUTF8Whitespace(normed.c_str())) {
54 continue;
55 }
56 unicharset->unichar_insert(normed.c_str());
57 }
58 } else {
59 tprintf("Normalization failed for string '%s'\n", string.c_str());
60 }
61 }
62 }
63
64 static int Main(int argc, char **argv) {
65 UNICHARSET unicharset;
66 // Load input files
67 for (int arg = 1; arg < argc; ++arg) {
68 std::filesystem::path filePath = argv[arg];
69 std::string file_data = tesseract::ReadFile(argv[arg]);
70 if (file_data.empty()) {
71 continue;
72 }
73 std::vector<std::string> texts;
74 if (filePath.extension() == ".box") {
75 tprintf("Extracting unicharset from box file %s\n", argv[arg]);
76 bool res = ReadMemBoxes(-1, /*skip_blanks*/ true, &file_data[0],
77 /*continue_on_failure*/ false, /*boxes*/ nullptr, &texts,
78 /*box_texts*/ nullptr, /*pages*/ nullptr);
79 if (!res) {
80 tprintf("Cannot read box data from '%s'\n", argv[arg]);
81 return EXIT_FAILURE;
82 }
83 } else {
84 tprintf("Extracting unicharset from plain text file %s\n", argv[arg]);
85 texts.clear();
86 texts = split(file_data, '\n');
87 }
88 AddStringsToUnicharset(texts, FLAGS_norm_mode, &unicharset);
89 }
90 SetupBasicProperties(/*report_errors*/ true, /*decompose*/ false, &unicharset);
91 // Write unicharset file.
92 if (unicharset.save_to_file(FLAGS_output_unicharset.c_str())) {
93 tprintf("Wrote unicharset file %s\n", FLAGS_output_unicharset.c_str());
94 } else {
95 tprintf("Cannot save unicharset file %s\n", FLAGS_output_unicharset.c_str());
96 return EXIT_FAILURE;
97 }
98 return EXIT_SUCCESS;
99 }
100
101 } // namespace tesseract
102
103 int main(int argc, char **argv) {
104 tesseract::CheckSharedLibraryVersion();
105 if (argc > 1) {
106 tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
107 }
108 if (argc < 2) {
109 tprintf(
110 "Usage: %s [--output_unicharset filename] [--norm_mode mode]"
111 " box_or_text_file [...]\n",
112 argv[0]);
113 tprintf("Where mode means:\n");
114 tprintf(" 1=combine graphemes (use for Latin and other simple scripts)\n");
115 tprintf(" 2=split graphemes (use for Indic/Khmer/Myanmar)\n");
116 tprintf(" 3=pure unicode (use for Arabic/Hebrew/Thai/Tibetan)\n");
117 tprintf("Reads box or plain text files to extract the unicharset.\n");
118 return EXIT_FAILURE;
119 }
120 return tesseract::Main(argc, argv);
121 }