Mercurial > hgrepos > Python2 > PyMuPDF

comparison mupdf-source/thirdparty/tesseract/src/training/combine_lang_model.cpp @ 2:b50eed0cc0ef upstream

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
ADD: MuPDF v1.26.7: the MuPDF source as downloaded by a default build of PyMuPDF 1.26.4. The directory name has changed: no version number in the expanded directory now.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 15 Sep 2025 11:43:07 +0200
parents
children
comparison
equal deleted inserted replaced
1:1d09e1dec1d9 2:b50eed0cc0ef
1 // Copyright 2017 Google Inc. All Rights Reserved.
2 // Author: rays@google.com (Ray Smith)
3 // Purpose: Program to generate a traineddata file that can be used to train an
4 // LSTM-based neural network model from a unicharset and an optional
5 // set of wordlists. Eliminates the need to run
6 // set_unicharset_properties, wordlist2dawg, some non-existent binary
7 // to generate the recoder, and finally combine_tessdata.
8
9 // Licensed under the Apache License, Version 2.0 (the "License");
10 // you may not use this file except in compliance with the License.
11 // You may obtain a copy of the License at
12 // http://www.apache.org/licenses/LICENSE-2.0
13 // Unless required by applicable law or agreed to in writing, software
14 // distributed under the License is distributed on an "AS IS" BASIS,
15 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16 // See the License for the specific language governing permissions and
17 // limitations under the License.
18
19 #include "commandlineflags.h"
20 #include "commontraining.h" // CheckSharedLibraryVersion
21 #include "lang_model_helpers.h"
22 #include "tesserrstream.h" // for tesserr
23 #include "tprintf.h"
24 #include "unicharset_training_utils.h"
25
26 using namespace tesseract;
27
28 static STRING_PARAM_FLAG(input_unicharset, "",
29 "Filename with unicharset to complete and use in encoding");
30 static STRING_PARAM_FLAG(script_dir, "", "Directory name for input script unicharsets");
31 static STRING_PARAM_FLAG(words, "", "File listing words to use for the system dictionary");
32 static STRING_PARAM_FLAG(puncs, "", "File listing punctuation patterns");
33 static STRING_PARAM_FLAG(numbers, "", "File listing number patterns");
34 static STRING_PARAM_FLAG(output_dir, "", "Root directory for output files");
35 static STRING_PARAM_FLAG(version_str, "", "Version string to add to traineddata file");
36 static STRING_PARAM_FLAG(lang, "", "Name of language being processed");
37 static BOOL_PARAM_FLAG(lang_is_rtl, false, "True if lang being processed is written right-to-left");
38 static BOOL_PARAM_FLAG(pass_through_recoder, false,
39 "If true, the recoder is a simple pass-through of the "
40 "unicharset. Otherwise, potentially a compression of it");
41
42 int main(int argc, char **argv) {
43 // Sets properties on the input unicharset file, and writes:
44 // rootdir/lang/lang.charset_size=ddd.txt
45 // rootdir/lang/lang.traineddata
46 // rootdir/lang/lang.unicharset
47 // If the 3 word lists are provided, the dawgs are also added
48 // to the traineddata file.
49 // The output unicharset and charset_size files are just for
50 // human readability.
51 tesseract::CheckSharedLibraryVersion();
52 tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
53
54 // If these reads fail, we get a warning message and an empty list of words.
55 std::vector<std::string> words = split(tesseract::ReadFile(FLAGS_words.c_str()), '\n');
56 std::vector<std::string> puncs = split(tesseract::ReadFile(FLAGS_puncs.c_str()), '\n');
57 std::vector<std::string> numbers = split(tesseract::ReadFile(FLAGS_numbers.c_str()), '\n');
58 // Load the input unicharset
59 UNICHARSET unicharset;
60 if (!unicharset.load_from_file(FLAGS_input_unicharset.c_str(), false)) {
61 tprintf("Failed to load unicharset from %s\n", FLAGS_input_unicharset.c_str());
62 return EXIT_FAILURE;
63 }
64 tesserr << "Loaded unicharset of size " << unicharset.size()
65 << " from file " << FLAGS_input_unicharset.c_str() << '\n';
66
67 // Set unichar properties
68 tprintf("Setting unichar properties\n");
69 tesseract::SetupBasicProperties(/*report_errors*/ true,
70 /*decompose (NFD)*/ false, &unicharset);
71 tprintf("Setting script properties\n");
72 tesseract::SetScriptProperties(FLAGS_script_dir.c_str(), &unicharset);
73 // Combine everything into a traineddata file.
74 return tesseract::CombineLangModel(unicharset, FLAGS_script_dir.c_str(),
75 FLAGS_version_str.c_str(), FLAGS_output_dir.c_str(),
76 FLAGS_lang.c_str(), FLAGS_pass_through_recoder, words, puncs,
77 numbers, FLAGS_lang_is_rtl, /*reader*/ nullptr,
78 /*writer*/ nullptr);
79 }