Mercurial > hgrepos > Python2 > PyMuPDF

comparison mupdf-source/thirdparty/tesseract/src/training/unicharset/lstmtester.cpp @ 2:b50eed0cc0ef upstream

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
ADD: MuPDF v1.26.7: the MuPDF source as downloaded by a default build of PyMuPDF 1.26.4. The directory name has changed: no version number in the expanded directory now.
author Franz Glasner <fzglas.hg@dom66.de>
date Mon, 15 Sep 2025 11:43:07 +0200
parents
children
comparison
equal deleted inserted replaced
1:1d09e1dec1d9 2:b50eed0cc0ef
1 ///////////////////////////////////////////////////////////////////////
2 // File: lstmtester.cpp
3 // Description: Top-level line evaluation class for LSTM-based networks.
4 // Author: Ray Smith
5 //
6 // (C) Copyright 2016, Google Inc.
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 ///////////////////////////////////////////////////////////////////////
17
18 #include "lstmtester.h"
19 #include <iomanip> // for std::setprecision
20 #include <thread> // for std::thread
21 #include "fileio.h" // for LoadFileLinesToStrings
22
23 namespace tesseract {
24
25 LSTMTester::LSTMTester(int64_t max_memory) : test_data_(max_memory) {}
26
27 // Loads a set of lstmf files that were created using the lstm.train config to
28 // tesseract into memory ready for testing. Returns false if nothing was
29 // loaded. The arg is a filename of a file that lists the filenames.
30 bool LSTMTester::LoadAllEvalData(const char *filenames_file) {
31 std::vector<std::string> filenames;
32 if (!LoadFileLinesToStrings(filenames_file, &filenames)) {
33 tprintf("Failed to load list of eval filenames from %s\n", filenames_file);
34 return false;
35 }
36 return LoadAllEvalData(filenames);
37 }
38
39 // Loads a set of lstmf files that were created using the lstm.train config to
40 // tesseract into memory ready for testing. Returns false if nothing was
41 // loaded.
42 bool LSTMTester::LoadAllEvalData(const std::vector<std::string> &filenames) {
43 test_data_.Clear();
44 bool result = test_data_.LoadDocuments(filenames, CS_SEQUENTIAL, nullptr);
45 total_pages_ = test_data_.TotalPages();
46 return result;
47 }
48
49 // Runs an evaluation asynchronously on the stored data and returns a string
50 // describing the results of the previous test.
51 std::string LSTMTester::RunEvalAsync(int iteration, const double *training_errors,
52 const TessdataManager &model_mgr, int training_stage) {
53 std::string result;
54 if (total_pages_ == 0) {
55 result += "No test data at iteration " + std::to_string(iteration);
56 return result;
57 }
58 if (!LockIfNotRunning()) {
59 result += "Previous test incomplete, skipping test at iteration " + std::to_string(iteration);
60 return result;
61 }
62 // Save the args.
63 std::string prev_result = test_result_;
64 test_result_ = "";
65 if (training_errors != nullptr) {
66 test_iteration_ = iteration;
67 test_training_errors_ = training_errors;
68 test_model_mgr_ = model_mgr;
69 test_training_stage_ = training_stage;
70 std::thread t(&LSTMTester::ThreadFunc, this);
71 t.detach();
72 } else {
73 UnlockRunning();
74 }
75 return prev_result;
76 }
77
78 // Runs an evaluation synchronously on the stored data and returns a string
79 // describing the results.
80 std::string LSTMTester::RunEvalSync(int iteration, const double *training_errors,
81 const TessdataManager &model_mgr, int training_stage,
82 int verbosity) {
83 LSTMTrainer trainer;
84 trainer.InitCharSet(model_mgr);
85 TFile fp;
86 if (!model_mgr.GetComponent(TESSDATA_LSTM, &fp) || !trainer.DeSerialize(&model_mgr, &fp)) {
87 return "Deserialize failed";
88 }
89 int eval_iteration = 0;
90 double char_error = 0.0;
91 double word_error = 0.0;
92 int error_count = 0;
93 while (error_count < total_pages_) {
94 const ImageData *trainingdata = test_data_.GetPageBySerial(eval_iteration);
95 trainer.SetIteration(++eval_iteration);
96 NetworkIO fwd_outputs, targets;
97 Trainability result = trainer.PrepareForBackward(trainingdata, &fwd_outputs, &targets);
98 if (result != UNENCODABLE) {
99 char_error += trainer.NewSingleError(tesseract::ET_CHAR_ERROR);
100 word_error += trainer.NewSingleError(tesseract::ET_WORD_RECERR);
101 ++error_count;
102 if (verbosity > 1 || (verbosity > 0 && result != PERFECT)) {
103 tprintf("Truth:%s\n", trainingdata->transcription().c_str());
104 std::vector<int> ocr_labels;
105 std::vector<int> xcoords;
106 trainer.LabelsFromOutputs(fwd_outputs, &ocr_labels, &xcoords);
107 std::string ocr_text = trainer.DecodeLabels(ocr_labels);
108 tprintf("OCR :%s\n", ocr_text.c_str());
109 if (verbosity > 2 || (verbosity > 1 && result != PERFECT)) {
110 tprintf("Line BCER=%f, BWER=%f\n\n",
111 trainer.NewSingleError(tesseract::ET_CHAR_ERROR),
112 trainer.NewSingleError(tesseract::ET_WORD_RECERR));
113 }
114 }
115 }
116 }
117 char_error *= 100.0 / total_pages_;
118 word_error *= 100.0 / total_pages_;
119 std::stringstream result;
120 result.imbue(std::locale::classic());
121 result << std::fixed << std::setprecision(3);
122 if (iteration != 0 || training_stage != 0) {
123 result << "At iteration " << iteration
124 << ", stage " << training_stage << ", ";
125 }
126 result << "BCER eval=" << char_error << ", BWER eval=" << word_error;
127 return result.str();
128 }
129
130 // Helper thread function for RunEvalAsync.
131 // LockIfNotRunning must have returned true before calling ThreadFunc, and
132 // it will call UnlockRunning to release the lock after RunEvalSync completes.
133 void LSTMTester::ThreadFunc() {
134 test_result_ =
135 RunEvalSync(test_iteration_, test_training_errors_, test_model_mgr_, test_training_stage_,
136 /*verbosity*/ 0);
137 UnlockRunning();
138 }
139
140 // Returns true if there is currently nothing running, and takes the lock
141 // if there is nothing running.
142 bool LSTMTester::LockIfNotRunning() {
143 std::lock_guard<std::mutex> lock(running_mutex_);
144 if (async_running_) {
145 return false;
146 }
147 async_running_ = true;
148 return true;
149 }
150
151 // Releases the running lock.
152 void LSTMTester::UnlockRunning() {
153 std::lock_guard<std::mutex> lock(running_mutex_);
154 async_running_ = false;
155 }
156
157 } // namespace tesseract