From 509a33bd1fceff8665a0543743021f70b0a334a1 Mon Sep 17 00:00:00 2001 From: andyjpaddle Date: Sat, 2 Apr 2022 02:34:47 +0000 Subject: [PATCH 01/21] fix sar bug --- ppocr/modeling/heads/rec_sar_head.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ppocr/modeling/heads/rec_sar_head.py b/ppocr/modeling/heads/rec_sar_head.py index a46cce7de2..3b76742687 100644 --- a/ppocr/modeling/heads/rec_sar_head.py +++ b/ppocr/modeling/heads/rec_sar_head.py @@ -216,7 +216,7 @@ class ParallelSARDecoder(BaseDecoder): self.pred_dropout = nn.Dropout(pred_dropout) pred_num_classes = self.num_classes - 1 if pred_concat: - fc_in_channel = decoder_rnn_out_size + d_model + d_enc + fc_in_channel = decoder_rnn_out_size + d_model + encoder_rnn_out_size else: fc_in_channel = d_model self.prediction = nn.Linear(fc_in_channel, pred_num_classes) From 0527b02a72ea207aedf7f1aefbefd48ac2259f1d Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Mon, 4 Apr 2022 06:24:18 +0000 Subject: [PATCH 02/21] repleace # with // --- deploy/cpp_infer/external-cmake/auto-log.cmake | 1 + deploy/cpp_infer/src/main.cpp | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/deploy/cpp_infer/external-cmake/auto-log.cmake b/deploy/cpp_infer/external-cmake/auto-log.cmake index becbff0f45..c998b3b145 100644 --- a/deploy/cpp_infer/external-cmake/auto-log.cmake +++ b/deploy/cpp_infer/external-cmake/auto-log.cmake @@ -6,6 +6,7 @@ set(FETCHCONTENT_BASE_DIR "${CMAKE_CURRENT_BINARY_DIR}/third-party") FetchContent_Declare( extern_Autolog PREFIX autolog + # If you don't have access to github, replace it with https://gitee.com/Double_V/AutoLog GIT_REPOSITORY https://github.com/LDOUBLEV/AutoLog.git GIT_TAG main ) diff --git a/deploy/cpp_infer/src/main.cpp b/deploy/cpp_infer/src/main.cpp index b7b0b2486b..efc1e50ce9 100644 --- a/deploy/cpp_infer/src/main.cpp +++ b/deploy/cpp_infer/src/main.cpp @@ -107,7 +107,7 @@ int main_det(std::vector cv_all_img_names) { std::vector det_times; det.Run(srcimg, boxes, &det_times); - //// visualization + // visualization if (FLAGS_visualize) { std::string file_name = Utility::basename(cv_all_img_names[i]); Utility::VisualizeBboxes(srcimg, boxes, FLAGS_output + "/" + file_name); @@ -223,7 +223,7 @@ int main_system(std::vector cv_all_img_names) { << cv_all_img_names[i] << endl; exit(1); } -#det + // det std::vector>> boxes; std::vector det_times; std::vector rec_times; @@ -237,7 +237,7 @@ int main_system(std::vector cv_all_img_names) { time_info_det[1] += det_times[1]; time_info_det[2] += det_times[2]; -#rec + // rec std::vector img_list; for (int j = 0; j < boxes.size(); j++) { cv::Mat crop_img; From 22b57e6e85739ea2bf875cd2a9a348a7e3d7ec63 Mon Sep 17 00:00:00 2001 From: LDOUBLEV Date: Thu, 7 Apr 2022 08:26:20 +0000 Subject: [PATCH 03/21] fix rec pact hang --- deploy/slim/quantization/export_model.py | 1 + ppocr/modeling/heads/rec_ctc_head.py | 3 ++- .../train_infer_python.txt | 8 +++---- .../det_mv3_east_v2.0/train_infer_python.txt | 4 ++-- test_tipc/prepare.sh | 8 +++++++ tools/infer/utility.py | 22 ++++++++++++++++--- 6 files changed, 36 insertions(+), 10 deletions(-) diff --git a/deploy/slim/quantization/export_model.py b/deploy/slim/quantization/export_model.py index 34cf80f5e5..bbd291c334 100755 --- a/deploy/slim/quantization/export_model.py +++ b/deploy/slim/quantization/export_model.py @@ -127,6 +127,7 @@ def main(): arch_config = config["Architecture"] if arch_config["algorithm"] in ["Distillation", ]: # distillation model for idx, name in enumerate(model.model_name_list): + model.model_list[idx].eval() sub_model_save_path = os.path.join(save_path, name, "inference") export_single_model(quanter, model.model_list[idx], infer_shape, sub_model_save_path, logger) diff --git a/ppocr/modeling/heads/rec_ctc_head.py b/ppocr/modeling/heads/rec_ctc_head.py index 35d33d5f56..5cc4f9d8a4 100755 --- a/ppocr/modeling/heads/rec_ctc_head.py +++ b/ppocr/modeling/heads/rec_ctc_head.py @@ -80,8 +80,9 @@ class CTCHead(nn.Layer): result = (x, predicts) else: result = predicts - + print("self.training: ", self.training) if not self.training: + print("self.training, ", self.training) predicts = F.softmax(predicts, axis=2) result = predicts diff --git a/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt b/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt index 03d749f557..cfe0947fe0 100644 --- a/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt +++ b/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt @@ -1,13 +1,13 @@ ===========================train_params=========================== -model_name:PPOCRv2_ocr_rec_pact +model_name:ch_PPOCRv2_rec_PACT python:python3.7 -gpu_list:0|0,1 +gpu_list:6 Global.use_gpu:True|True Global.auto_cast:fp32 -Global.epoch_num:lite_train_lite_infer=3|whole_train_whole_infer=300 +Global.epoch_num:lite_train_lite_infer=6|whole_train_whole_infer=300 Global.save_model_dir:./output/ Train.loader.batch_size_per_card:lite_train_lite_infer=16|whole_train_whole_infer=128 -Global.pretrained_model:null +Global.pretrained_model:pretrain_models/ch_PP-OCRv2_rec_train/best_accuracy train_model_name:latest train_infer_img_dir:./inference/rec_inference null:null diff --git a/test_tipc/configs/det_mv3_east_v2.0/train_infer_python.txt b/test_tipc/configs/det_mv3_east_v2.0/train_infer_python.txt index 0603fa10a6..5634297973 100644 --- a/test_tipc/configs/det_mv3_east_v2.0/train_infer_python.txt +++ b/test_tipc/configs/det_mv3_east_v2.0/train_infer_python.txt @@ -1,13 +1,13 @@ ===========================train_params=========================== model_name:det_mv3_east_v2.0 python:python3.7 -gpu_list:0 +gpu_list:0|0,1 Global.use_gpu:True|True Global.auto_cast:fp32 Global.epoch_num:lite_train_lite_infer=1|whole_train_whole_infer=500 Global.save_model_dir:./output/ Train.loader.batch_size_per_card:lite_train_lite_infer=2|whole_train_whole_infer=4 -Global.pretrained_model:null +Global.pretrained_model:./pretrain_models/det_mv3_east_v2.0_train/best_accuracy train_model_name:latest train_infer_img_dir:./train_data/icdar2015/text_localization/ch4_test_images/ null:null diff --git a/test_tipc/prepare.sh b/test_tipc/prepare.sh index 31433884a6..6a8983009e 100644 --- a/test_tipc/prepare.sh +++ b/test_tipc/prepare.sh @@ -64,6 +64,10 @@ if [ ${MODE} = "lite_train_lite_infer" ];then wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_server_v2.0_det_train.tar --no-check-certificate cd ./pretrain_models/ && tar xf ch_ppocr_server_v2.0_det_train.tar && cd ../ fi + if [ ${model_name} == "ch_PPOCRv2_rec" ] || [ ${model_name} == "ch_PPOCRv2_rec_PACT" ]; then + wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/PP-OCRv2/chinese/ch_PP-OCRv2_rec_train.tar --no-check-certificate + cd ./pretrain_models/ && tar xf ch_PP-OCRv2_rec_train.tar && cd ../ + fi if [ ${model_name} == "det_r18_db_v2_0" ]; then wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/pretrained/ResNet18_vd_pretrained.pdparams --no-check-certificate fi @@ -91,6 +95,10 @@ if [ ${MODE} = "lite_train_lite_infer" ];then wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/ch/ch_ppocr_mobile_v2.0_rec_train.tar --no-check-certificate cd ./pretrain_models/ && tar xf ch_ppocr_mobile_v2.0_rec_train.tar && cd ../ fi + if [ ${model_name} == "det_mv3_east_v2.0" ]; then + wget -nc -P ./pretrain_models/ https://paddleocr.bj.bcebos.com/dygraph_v2.0/en/det_mv3_east_v2.0_train.tar --no-check-certificate + cd ./pretrain_models/ && tar xf det_mv3_east_v2.0_train.tar && cd ../ + fi elif [ ${MODE} = "whole_train_whole_infer" ];then wget -nc -P ./pretrain_models/ https://paddle-imagenet-models-name.bj.bcebos.com/dygraph/MobileNetV3_large_x0_5_pretrained.pdparams --no-check-certificate diff --git a/tools/infer/utility.py b/tools/infer/utility.py index 80abba67b2..1b282a0112 100644 --- a/tools/infer/utility.py +++ b/tools/infer/utility.py @@ -312,12 +312,28 @@ def create_predictor(args, mode, logger): input_names = predictor.get_input_names() for name in input_names: input_tensor = predictor.get_input_handle(name) - output_names = predictor.get_output_names() - output_tensors = [] + output_tensors = get_output_tensors(args, mode, predictor) + #output_names = predictor.get_output_names() + #output_tensors = [] + #print("output names", output_names) + #for output_name in output_names: + # output_tensor = predictor.get_output_handle(output_name) + # output_tensors.append(output_tensor) + return predictor, input_tensor, output_tensors, config + + +def get_output_tensors(args, mode, predictor): + output_names = predictor.get_output_names() + output_tensors = [] + if mode == "rec" and args.rec_algorithm == "CRNN": + output_name = 'softmax_0.tmp_0' + if output_name in output_names: + return [predictor.get_output_handle(output_name)] + else: for output_name in output_names: output_tensor = predictor.get_output_handle(output_name) output_tensors.append(output_tensor) - return predictor, input_tensor, output_tensors, config + return output_tensors def get_infer_gpuid(): From a8c8f485cb82efc9235438f3e66c80ddffb0dd6a Mon Sep 17 00:00:00 2001 From: LDOUBLEV Date: Thu, 7 Apr 2022 08:30:17 +0000 Subject: [PATCH 04/21] delete debug --- ppocr/modeling/heads/rec_ctc_head.py | 2 -- ppocr/postprocess/rec_postprocess.py | 3 ++- test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt | 2 +- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/ppocr/modeling/heads/rec_ctc_head.py b/ppocr/modeling/heads/rec_ctc_head.py index 5cc4f9d8a4..6c1cf06596 100755 --- a/ppocr/modeling/heads/rec_ctc_head.py +++ b/ppocr/modeling/heads/rec_ctc_head.py @@ -80,9 +80,7 @@ class CTCHead(nn.Layer): result = (x, predicts) else: result = predicts - print("self.training: ", self.training) if not self.training: - print("self.training, ", self.training) predicts = F.softmax(predicts, axis=2) result = predicts diff --git a/ppocr/postprocess/rec_postprocess.py b/ppocr/postprocess/rec_postprocess.py index de771acca8..0ffe917b0e 100644 --- a/ppocr/postprocess/rec_postprocess.py +++ b/ppocr/postprocess/rec_postprocess.py @@ -89,10 +89,11 @@ class CTCLabelDecode(BaseRecLabelDecode): use_space_char) def __call__(self, preds, label=None, *args, **kwargs): - if isinstance(preds, tuple): + if isinstance(preds, tuple) or isinstance(preds, list): preds = preds[-1] if isinstance(preds, paddle.Tensor): preds = preds.numpy() + print(preds.shape) preds_idx = preds.argmax(axis=2) preds_prob = preds.max(axis=2) text = self.decode(preds_idx, preds_prob, is_remove_duplicate=True) diff --git a/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt b/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt index cfe0947fe0..98c125229d 100644 --- a/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt +++ b/test_tipc/configs/ch_PP-OCRv2_rec_PACT/train_infer_python.txt @@ -1,7 +1,7 @@ ===========================train_params=========================== model_name:ch_PPOCRv2_rec_PACT python:python3.7 -gpu_list:6 +gpu_list:0|0,1 Global.use_gpu:True|True Global.auto_cast:fp32 Global.epoch_num:lite_train_lite_infer=6|whole_train_whole_infer=300 From 5efa071c075a0423920d2a2470d560b3035abd3a Mon Sep 17 00:00:00 2001 From: LDOUBLEV Date: Thu, 7 Apr 2022 08:31:37 +0000 Subject: [PATCH 05/21] delete debug --- ppocr/postprocess/rec_postprocess.py | 1 - tools/infer/utility.py | 6 ------ 2 files changed, 7 deletions(-) diff --git a/ppocr/postprocess/rec_postprocess.py b/ppocr/postprocess/rec_postprocess.py index 0ffe917b0e..3bc7bcdf9b 100644 --- a/ppocr/postprocess/rec_postprocess.py +++ b/ppocr/postprocess/rec_postprocess.py @@ -93,7 +93,6 @@ class CTCLabelDecode(BaseRecLabelDecode): preds = preds[-1] if isinstance(preds, paddle.Tensor): preds = preds.numpy() - print(preds.shape) preds_idx = preds.argmax(axis=2) preds_prob = preds.max(axis=2) text = self.decode(preds_idx, preds_prob, is_remove_duplicate=True) diff --git a/tools/infer/utility.py b/tools/infer/utility.py index 1b282a0112..fd2605d963 100644 --- a/tools/infer/utility.py +++ b/tools/infer/utility.py @@ -313,12 +313,6 @@ def create_predictor(args, mode, logger): for name in input_names: input_tensor = predictor.get_input_handle(name) output_tensors = get_output_tensors(args, mode, predictor) - #output_names = predictor.get_output_names() - #output_tensors = [] - #print("output names", output_names) - #for output_name in output_names: - # output_tensor = predictor.get_output_handle(output_name) - # output_tensors.append(output_tensor) return predictor, input_tensor, output_tensors, config From 30589045760b2bdb6561e4aeeccdd0978f79ae4e Mon Sep 17 00:00:00 2001 From: andyjpaddle Date: Thu, 7 Apr 2022 09:30:24 +0000 Subject: [PATCH 06/21] fix sar infer benchmark --- test_tipc/configs/rec_r31_sar/train_infer_python.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_tipc/configs/rec_r31_sar/train_infer_python.txt b/test_tipc/configs/rec_r31_sar/train_infer_python.txt index c5018500f9..1a32a3d507 100644 --- a/test_tipc/configs/rec_r31_sar/train_infer_python.txt +++ b/test_tipc/configs/rec_r31_sar/train_infer_python.txt @@ -50,4 +50,4 @@ inference:tools/infer/predict_rec.py --rec_char_dict_path=./ppocr/utils/dict90.t --benchmark:True null:null ===========================infer_benchmark_params========================== -random_infer_input:[{float32,[3,48,48,160]}] +random_infer_input:[{float32,[3,48,160]}] From 3c6f9f8174f7ccf0ba756aee6452aed54bd828eb Mon Sep 17 00:00:00 2001 From: Bin Lu Date: Thu, 7 Apr 2022 19:38:47 +0800 Subject: [PATCH 07/21] Upate VQA Readme (#5901) * update VQA README.md and add en version * restore ppstructure/README.md --- ppstructure/vqa/README.md | 251 +++++++++-------- ppstructure/vqa/README_ch.md | 257 ++++++++++++++++++ .../eval_with_label_end2end.py | 0 .../vqa/{helper => tools}/trans_xfun_data.py | 0 4 files changed, 388 insertions(+), 120 deletions(-) create mode 100644 ppstructure/vqa/README_ch.md rename ppstructure/vqa/{helper => tools}/eval_with_label_end2end.py (100%) rename ppstructure/vqa/{helper => tools}/trans_xfun_data.py (100%) diff --git a/ppstructure/vqa/README.md b/ppstructure/vqa/README.md index f142778506..a2117c9e06 100644 --- a/ppstructure/vqa/README.md +++ b/ppstructure/vqa/README.md @@ -1,67 +1,68 @@ -- [文档视觉问答（DOC-VQA）](#文档视觉问答doc-vqa) - - [1. 简介](#1-简介) - - [2. 性能](#2-性能) - - [3. 效果演示](#3-效果演示) +English | [简体中文](README_ch.md) + +- [Document Visual Question Answering (Doc-VQA)](#Document-Visual-Question-Answering) + - [1. Introduction](#1-Introduction) + - [2. Performance](#2-performance) + - [3. Effect demo](#3-Effect-demo) - [3.1 SER](#31-ser) - [3.2 RE](#32-re) - - [4. 安装](#4-安装) - - [4.1 安装依赖](#41-安装依赖) - - [4.2 安装PaddleOCR（包含 PP-OCR 和 VQA）](#42-安装paddleocr包含-pp-ocr-和-vqa) - - [5. 使用](#5-使用) - - [5.1 数据和预训练模型准备](#51-数据和预训练模型准备) + - [4. Install](#4-Install) + - [4.1 Installation dependencies](#41-Install-dependencies) + - [4.2 Install PaddleOCR](#42-Install-PaddleOCR) + - [5. Usage](#5-Usage) + - [5.1 Data and Model Preparation](#51-Data-and-Model-Preparation) - [5.2 SER](#52-ser) - [5.3 RE](#53-re) - - [6. 参考链接](#6-参考链接) + - [6. Reference](#6-Reference-Links) + +# Document Visual Question Answering + +## 1 Introduction + +VQA refers to visual question answering, which mainly asks and answers image content. DOC-VQA is one of the VQA tasks. DOC-VQA mainly asks questions about the text content of text images. + +The DOC-VQA algorithm in PP-Structure is developed based on the PaddleNLP natural language processing algorithm library. + +The main features are as follows: + +- Integrate [LayoutXLM](https://arxiv.org/pdf/2104.08836.pdf) model and PP-OCR prediction engine. +- Supports Semantic Entity Recognition (SER) and Relation Extraction (RE) tasks based on multimodal methods. Based on the SER task, the text recognition and classification in the image can be completed; based on the RE task, the relationship extraction of the text content in the image can be completed, such as judging the problem pair (pair). +- Supports custom training for SER tasks and RE tasks. +- Supports end-to-end system prediction and evaluation of OCR+SER. +- Supports end-to-end system prediction of OCR+SER+RE. -# 文档视觉问答（DOC-VQA） +This project is an open source implementation of [LayoutXLM: Multimodal Pre-training for Multilingual Visually-rich Document Understanding](https://arxiv.org/pdf/2104.08836.pdf) on Paddle 2.2, +Included fine-tuning code on [XFUND dataset](https://github.com/doc-analysis/XFUND). -## 1. 简介 +## 2. Performance -VQA指视觉问答，主要针对图像内容进行提问和回答,DOC-VQA是VQA任务中的一种，DOC-VQA主要针对文本图像的文字内容提出问题。 +We evaluate the algorithm on the Chinese dataset of [XFUND](https://github.com/doc-analysis/XFUND), and the performance is as follows -PP-Structure 里的 DOC-VQA算法基于PaddleNLP自然语言处理算法库进行开发。 - -主要特性如下： - -- 集成[LayoutXLM](https://arxiv.org/pdf/2104.08836.pdf)模型以及PP-OCR预测引擎。 -- 支持基于多模态方法的语义实体识别 (Semantic Entity Recognition, SER) 以及关系抽取 (Relation Extraction, RE) 任务。基于 SER 任务，可以完成对图像中的文本识别与分类；基于 RE 任务，可以完成对图象中的文本内容的关系提取，如判断问题对(pair)。 -- 支持SER任务和RE任务的自定义训练。 -- 支持OCR+SER的端到端系统预测与评估。 -- 支持OCR+SER+RE的端到端系统预测。 - - -本项目是 [LayoutXLM: Multimodal Pre-training for Multilingual Visually-rich Document Understanding](https://arxiv.org/pdf/2104.08836.pdf) 在 Paddle 2.2上的开源实现， -包含了在 [XFUND数据集](https://github.com/doc-analysis/XFUND) 上的微调代码。 - -## 2. 性能 - -我们在 [XFUN](https://github.com/doc-analysis/XFUND) 的中文数据集上对算法进行了评估，性能如下 - -| 模型 | 任务 | hmean | 模型下载地址 | +| Model | Task | hmean | Model download address | |:---:|:---:|:---:| :---:| -| LayoutXLM | SER | 0.9038 | [链接](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutXLM_xfun_zh.tar) | -| LayoutXLM | RE | 0.7483 | [链接](https://paddleocr.bj.bcebos.com/pplayout/re_LayoutXLM_xfun_zh.tar) | -| LayoutLMv2 | SER | 0.8544 | [链接](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutLMv2_xfun_zh.tar) -| LayoutLMv2 | RE | 0.6777 | [链接](https://paddleocr.bj.bcebos.com/pplayout/re_LayoutLMv2_xfun_zh.tar) | -| LayoutLM | SER | 0.7731 | [链接](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutLM_xfun_zh.tar) | +| LayoutXLM | SER | 0.9038 | [link](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutXLM_xfun_zh.tar) | +| LayoutXLM | RE | 0.7483 | [link](https://paddleocr.bj.bcebos.com/pplayout/re_LayoutXLM_xfun_zh.tar) | +| LayoutLMv2 | SER | 0.8544 | [link](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutLMv2_xfun_zh.tar) +| LayoutLMv2 | RE | 0.6777 | [link](https://paddleocr.bj.bcebos.com/pplayout/re_LayoutLMv2_xfun_zh.tar) | +| LayoutLM | SER | 0.7731 | [link](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutLM_xfun_zh.tar) | -## 3. 效果演示 +## 3. Effect demo -**注意：** 测试图片来源于XFUN数据集。 +**Note:** The test images are from the XFUND dataset. ### 3.1 SER ![](../../doc/vqa/result_ser/zh_val_0_ser.jpg) | ![](../../doc/vqa/result_ser/zh_val_42_ser.jpg) ---|--- -图中不同颜色的框表示不同的类别，对于XFUN数据集，有`QUESTION`, `ANSWER`, `HEADER` 3种类别 +Boxes with different colors in the figure represent different categories. For the XFUND dataset, there are 3 categories: `QUESTION`, `ANSWER`, `HEADER` -* 深紫色：HEADER -* 浅紫色：QUESTION -* 军绿色：ANSWER +* Dark purple: HEADER +* Light purple: QUESTION +* Army Green: ANSWER -在OCR检测框的左上方也标出了对应的类别和OCR识别结果。 +The corresponding categories and OCR recognition results are also marked on the upper left of the OCR detection frame. ### 3.2 RE @@ -69,175 +70,185 @@ PP-Structure 里的 DOC-VQA算法基于PaddleNLP自然语言处理算法库进 ---|--- -图中红色框表示问题，蓝色框表示答案，问题和答案之间使用绿色线连接。在OCR检测框的左上方也标出了对应的类别和OCR识别结果。 +The red box in the figure represents the question, the blue box represents the answer, and the question and the answer are connected by a green line. The corresponding categories and OCR recognition results are also marked on the upper left of the OCR detection frame. -## 4. 安装 +## 4. Install -### 4.1 安装依赖 +### 4.1 Install dependencies -- **（1) 安装PaddlePaddle** +- **(1) Install PaddlePaddle** ```bash python3 -m pip install --upgrade pip -# GPU安装 +# GPU installation python3 -m pip install "paddlepaddle-gpu>=2.2" -i https://mirror.baidu.com/pypi/simple -# CPU安装 +# CPU installation python3 -m pip install "paddlepaddle>=2.2" -i https://mirror.baidu.com/pypi/simple -``` -更多需求，请参照[安装文档](https://www.paddlepaddle.org.cn/install/quick)中的说明进行操作。 +```` +For more requirements, please refer to the instructions in [Installation Documentation](https://www.paddlepaddle.org.cn/install/quick). -### 4.2 安装PaddleOCR（包含 PP-OCR 和 VQA） +### 4.2 Install PaddleOCR -- **（1）pip快速安装PaddleOCR whl包（仅预测）** +- **(1) pip install PaddleOCR whl package quickly (prediction only)** ```bash python3 -m pip install paddleocr -``` +```` -- **（2）下载VQA源码（预测+训练）** +- **(2) Download VQA source code (prediction + training)** ```bash -【推荐】git clone https://github.com/PaddlePaddle/PaddleOCR +[Recommended] git clone https://github.com/PaddlePaddle/PaddleOCR -# 如果因为网络问题无法pull成功，也可选择使用码云上的托管： +# If the pull cannot be successful due to network problems, you can also choose to use the hosting on the code cloud: git clone https://gitee.com/paddlepaddle/PaddleOCR -# 注：码云托管代码可能无法实时同步本github项目更新，存在3~5天延时，请优先使用推荐方式。 -``` +# Note: Code cloud hosting code may not be able to synchronize the update of this github project in real time, there is a delay of 3 to 5 days, please use the recommended method first. +```` -- **（3）安装VQA的`requirements`** +- **(3) Install VQA's `requirements`** ```bash python3 -m pip install -r ppstructure/vqa/requirements.txt -``` +```` -## 5. 使用 +## 5. Usage -### 5.1 数据和预训练模型准备 +### 5.1 Data and Model Preparation -如果希望直接体验预测过程，可以下载我们提供的预训练模型，跳过训练过程，直接预测即可。 +If you want to experience the prediction process directly, you can download the pre-training model provided by us, skip the training process, and just predict directly. -* 下载处理好的数据集 +* Download the processed dataset -处理好的XFUN中文数据集下载地址：[https://paddleocr.bj.bcebos.com/dataset/XFUND.tar](https://paddleocr.bj.bcebos.com/dataset/XFUND.tar)。 +The download address of the processed XFUND Chinese dataset: [https://paddleocr.bj.bcebos.com/dataset/XFUND.tar](https://paddleocr.bj.bcebos.com/dataset/XFUND.tar). -下载并解压该数据集，解压后将数据集放置在当前目录下。 +Download and unzip the dataset, and place the dataset in the current directory after unzipping. ```shell wget https://paddleocr.bj.bcebos.com/dataset/XFUND.tar -``` +```` -* 转换数据集 +* Convert the dataset -若需进行其他XFUN数据集的训练，可使用下面的命令进行数据集的转换 +If you need to train other XFUND datasets, you can use the following commands to convert the datasets ```bash -python3 ppstructure/vqa/helper/trans_xfun_data.py --ori_gt_path=path/to/json_path --output_path=path/to/save_path -``` +python3 ppstructure/vqa/tools/trans_xfun_data.py --ori_gt_path=path/to/json_path --output_path=path/to/save_path +```` + +* Download the pretrained models +```bash +mkdir pretrain && cd pretrain +#download the SER model +wget https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutXLM_xfun_zh.tar && tar -xvf ser_LayoutXLM_xfun_zh.tar +#download the RE model +wget https://paddleocr.bj.bcebos.com/pplayout/re_LayoutXLM_xfun_zh.tar && tar -xvf re_LayoutXLM_xfun_zh.tar +cd ../ +```` ### 5.2 SER -启动训练之前，需要修改下面的四个字段 +Before starting training, you need to modify the following four fields -1. `Train.dataset.data_dir`：指向训练集图片存放目录 -2. `Train.dataset.label_file_list`：指向训练集标注文件 -3. `Eval.dataset.data_dir`：指指向验证集图片存放目录 -4. `Eval.dataset.label_file_list`：指向验证集标注文件 +1. `Train.dataset.data_dir`: point to the directory where the training set images are stored +2. `Train.dataset.label_file_list`: point to the training set label file +3. `Eval.dataset.data_dir`: refers to the directory where the validation set images are stored +4. `Eval.dataset.label_file_list`: point to the validation set label file -* 启动训练 +* start training ```shell CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/ser/layoutxlm.yml -``` +```` -最终会打印出`precision`, `recall`, `hmean`等指标。 -在`./output/ser_layoutxlm/`文件夹中会保存训练日志，最优的模型和最新epoch的模型。 +Finally, `precision`, `recall`, `hmean` and other indicators will be printed. +In the `./output/ser_layoutxlm/` folder will save the training log, the optimal model and the model for the latest epoch. -* 恢复训练 +* resume training -恢复训练需要将之前训练好的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 +To resume training, assign the folder path of the previously trained model to the `Architecture.Backbone.checkpoints` field. ```shell CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/ser/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir -``` +```` -* 评估 +* evaluate -评估需要将待评估的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 +Evaluation requires assigning the folder path of the model to be evaluated to the `Architecture.Backbone.checkpoints` field. ```shell CUDA_VISIBLE_DEVICES=0 python3 tools/eval.py -c configs/vqa/ser/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir -``` -最终会打印出`precision`, `recall`, `hmean`等指标 +```` +Finally, `precision`, `recall`, `hmean` and other indicators will be printed -* 使用`OCR引擎 + SER`串联预测 +* Use `OCR engine + SER` tandem prediction -使用如下命令即可完成`OCR引擎 + SER`的串联预测 +Use the following command to complete the series prediction of `OCR engine + SER`, taking the pretrained SER model as an example: ```shell -CUDA_VISIBLE_DEVICES=0 python3 tools/infer_vqa_token_ser.py -c configs/vqa/ser/layoutxlm.yml -o Architecture.Backbone.checkpoints=ser_LayoutXLM_xfun_zh/ Global.infer_img=doc/vqa/input/zh_val_42.jpg -``` +CUDA_VISIBLE_DEVICES=0 python3 tools/infer_vqa_token_ser.py -c configs/vqa/ser/layoutxlm.yml -o Architecture.Backbone.checkpoints=pretrain/ser_LayoutXLM_xfun_zh/Global.infer_img=doc/vqa/input/zh_val_42.jpg +```` -最终会在`config.Global.save_res_path`字段所配置的目录下保存预测结果可视化图像以及预测结果文本文件，预测结果文本文件名为`infer_results.txt`。 +Finally, the prediction result visualization image and the prediction result text file will be saved in the directory configured by the `config.Global.save_res_path` field. The prediction result text file is named `infer_results.txt`. -* 对`OCR引擎 + SER`预测系统进行端到端评估 +* End-to-end evaluation of `OCR engine + SER` prediction system -首先使用 `tools/infer_vqa_token_ser.py` 脚本完成数据集的预测，然后使用下面的命令进行评估。 +First use the `tools/infer_vqa_token_ser.py` script to complete the prediction of the dataset, then use the following command to evaluate. ```shell export CUDA_VISIBLE_DEVICES=0 -python3 helper/eval_with_label_end2end.py --gt_json_path XFUND/zh_val/xfun_normalize_val.json --pred_json_path output_res/infer_results.txt -``` +python3 tools/eval_with_label_end2end.py --gt_json_path XFUND/zh_val/xfun_normalize_val.json --pred_json_path output_res/infer_results.txt +```` ### 5.3 RE -* 启动训练 +* start training -启动训练之前，需要修改下面的四个字段 +Before starting training, you need to modify the following four fields -1. `Train.dataset.data_dir`：指向训练集图片存放目录 -2. `Train.dataset.label_file_list`：指向训练集标注文件 -3. `Eval.dataset.data_dir`：指指向验证集图片存放目录 -4. `Eval.dataset.label_file_list`：指向验证集标注文件 +1. `Train.dataset.data_dir`: point to the directory where the training set images are stored +2. `Train.dataset.label_file_list`: point to the training set label file +3. `Eval.dataset.data_dir`: refers to the directory where the validation set images are stored +4. `Eval.dataset.label_file_list`: point to the validation set label file ```shell CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/re/layoutxlm.yml -``` +```` -最终会打印出`precision`, `recall`, `hmean`等指标。 -在`./output/re_layoutxlm/`文件夹中会保存训练日志，最优的模型和最新epoch的模型。 +Finally, `precision`, `recall`, `hmean` and other indicators will be printed. +In the `./output/re_layoutxlm/` folder will save the training log, the optimal model and the model for the latest epoch. -* 恢复训练 +* resume training -恢复训练需要将之前训练好的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 +To resume training, assign the folder path of the previously trained model to the `Architecture.Backbone.checkpoints` field. ```shell CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/re/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir -``` +```` -* 评估 +* evaluate -评估需要将待评估的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 +Evaluation requires assigning the folder path of the model to be evaluated to the `Architecture.Backbone.checkpoints` field. ```shell CUDA_VISIBLE_DEVICES=0 python3 tools/eval.py -c configs/vqa/re/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir -``` -最终会打印出`precision`, `recall`, `hmean`等指标 +```` +Finally, `precision`, `recall`, `hmean` and other indicators will be printed -* 使用`OCR引擎 + SER + RE`串联预测 +* Use `OCR engine + SER + RE` tandem prediction -使用如下命令即可完成`OCR引擎 + SER + RE`的串联预测 +Use the following command to complete the series prediction of `OCR engine + SER + RE`, taking the pretrained SER and RE models as an example: ```shell export CUDA_VISIBLE_DEVICES=0 -python3 tools/infer_vqa_token_ser_re.py -c configs/vqa/re/layoutxlm.yml -o Architecture.Backbone.checkpoints=re_LayoutXLM_xfun_zh/ Global.infer_img=doc/vqa/input/zh_val_21.jpg -c_ser configs/vqa/ser/layoutxlm.yml -o_ser Architecture.Backbone.checkpoints=ser_LayoutXLM_xfun_zh/ -``` +python3 tools/infer_vqa_token_ser_re.py -c configs/vqa/re/layoutxlm.yml -o Architecture.Backbone.checkpoints=pretrain/re_LayoutXLM_xfun_zh/Global.infer_img=doc/vqa/input/zh_val_21.jpg -c_ser configs/vqa/ser/layoutxlm. yml -o_ser Architecture.Backbone.checkpoints=pretrain/ser_LayoutXLM_xfun_zh/ +```` -最终会在`config.Global.save_res_path`字段所配置的目录下保存预测结果可视化图像以及预测结果文本文件，预测结果文本文件名为`infer_results.txt`。 +Finally, the prediction result visualization image and the prediction result text file will be saved in the directory configured by the `config.Global.save_res_path` field. The prediction result text file is named `infer_results.txt`. -## 6. 参考链接 +## 6. Reference Links - LayoutXLM: Multimodal Pre-training for Multilingual Visually-rich Document Understanding, https://arxiv.org/pdf/2104.08836.pdf - microsoft/unilm/layoutxlm, https://github.com/microsoft/unilm/tree/master/layoutxlm diff --git a/ppstructure/vqa/README_ch.md b/ppstructure/vqa/README_ch.md new file mode 100644 index 0000000000..ff513f8f7d --- /dev/null +++ b/ppstructure/vqa/README_ch.md @@ -0,0 +1,257 @@ +[English](README.md) | 简体中文 + +- [文档视觉问答（DOC-VQA）](#文档视觉问答doc-vqa) + - [1. 简介](#1-简介) + - [2. 性能](#2-性能) + - [3. 效果演示](#3-效果演示) + - [3.1 SER](#31-ser) + - [3.2 RE](#32-re) + - [4. 安装](#4-安装) + - [4.1 安装依赖](#41-安装依赖) + - [4.2 安装PaddleOCR（包含 PP-OCR 和 VQA）](#42-安装paddleocr包含-pp-ocr-和-vqa) + - [5. 使用](#5-使用) + - [5.1 数据和预训练模型准备](#51-数据和预训练模型准备) + - [5.2 SER](#52-ser) + - [5.3 RE](#53-re) + - [6. 参考链接](#6-参考链接) + +# 文档视觉问答（DOC-VQA） + +## 1. 简介 + +VQA指视觉问答，主要针对图像内容进行提问和回答,DOC-VQA是VQA任务中的一种，DOC-VQA主要针对文本图像的文字内容提出问题。 + +PP-Structure 里的 DOC-VQA算法基于PaddleNLP自然语言处理算法库进行开发。 + +主要特性如下： + +- 集成[LayoutXLM](https://arxiv.org/pdf/2104.08836.pdf)模型以及PP-OCR预测引擎。 +- 支持基于多模态方法的语义实体识别 (Semantic Entity Recognition, SER) 以及关系抽取 (Relation Extraction, RE) 任务。基于 SER 任务，可以完成对图像中的文本识别与分类；基于 RE 任务，可以完成对图象中的文本内容的关系提取，如判断问题对(pair)。 +- 支持SER任务和RE任务的自定义训练。 +- 支持OCR+SER的端到端系统预测与评估。 +- 支持OCR+SER+RE的端到端系统预测。 + +本项目是 [LayoutXLM: Multimodal Pre-training for Multilingual Visually-rich Document Understanding](https://arxiv.org/pdf/2104.08836.pdf) 在 Paddle 2.2上的开源实现， +包含了在 [XFUND数据集](https://github.com/doc-analysis/XFUND) 上的微调代码。 + +## 2. 性能 + +我们在 [XFUND](https://github.com/doc-analysis/XFUND) 的中文数据集上对算法进行了评估，性能如下 + +| 模型 | 任务 | hmean | 模型下载地址 | +|:---:|:---:|:---:| :---:| +| LayoutXLM | SER | 0.9038 | [链接](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutXLM_xfun_zh.tar) | +| LayoutXLM | RE | 0.7483 | [链接](https://paddleocr.bj.bcebos.com/pplayout/re_LayoutXLM_xfun_zh.tar) | +| LayoutLMv2 | SER | 0.8544 | [链接](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutLMv2_xfun_zh.tar) +| LayoutLMv2 | RE | 0.6777 | [链接](https://paddleocr.bj.bcebos.com/pplayout/re_LayoutLMv2_xfun_zh.tar) | +| LayoutLM | SER | 0.7731 | [链接](https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutLM_xfun_zh.tar) | + +## 3. 效果演示 + +**注意：** 测试图片来源于XFUND数据集。 + +### 3.1 SER + +![](../../doc/vqa/result_ser/zh_val_0_ser.jpg) | ![](../../doc/vqa/result_ser/zh_val_42_ser.jpg) +---|--- + +图中不同颜色的框表示不同的类别，对于XFUND数据集，有`QUESTION`, `ANSWER`, `HEADER` 3种类别 + +* 深紫色：HEADER +* 浅紫色：QUESTION +* 军绿色：ANSWER + +在OCR检测框的左上方也标出了对应的类别和OCR识别结果。 + +### 3.2 RE + +![](../../doc/vqa/result_re/zh_val_21_re.jpg) | ![](../../doc/vqa/result_re/zh_val_40_re.jpg) +---|--- + + +图中红色框表示问题，蓝色框表示答案，问题和答案之间使用绿色线连接。在OCR检测框的左上方也标出了对应的类别和OCR识别结果。 + +## 4. 安装 + +### 4.1 安装依赖 + +- **（1) 安装PaddlePaddle** + +```bash +python3 -m pip install --upgrade pip + +# GPU安装 +python3 -m pip install "paddlepaddle-gpu>=2.2" -i https://mirror.baidu.com/pypi/simple + +# CPU安装 +python3 -m pip install "paddlepaddle>=2.2" -i https://mirror.baidu.com/pypi/simple + +``` +更多需求，请参照[安装文档](https://www.paddlepaddle.org.cn/install/quick)中的说明进行操作。 + +### 4.2 安装PaddleOCR（包含 PP-OCR 和 VQA） + +- **（1）pip快速安装PaddleOCR whl包（仅预测）** + +```bash +python3 -m pip install paddleocr +``` + +- **（2）下载VQA源码（预测+训练）** + +```bash +【推荐】git clone https://github.com/PaddlePaddle/PaddleOCR + +# 如果因为网络问题无法pull成功，也可选择使用码云上的托管： +git clone https://gitee.com/paddlepaddle/PaddleOCR + +# 注：码云托管代码可能无法实时同步本github项目更新，存在3~5天延时，请优先使用推荐方式。 +``` + +- **（3）安装VQA的`requirements`** + +```bash +python3 -m pip install -r ppstructure/vqa/requirements.txt +``` + +## 5. 使用 + +### 5.1 数据和预训练模型准备 + +如果希望直接体验预测过程，可以下载我们提供的预训练模型，跳过训练过程，直接预测即可。 + +* 下载处理好的数据集 + +处理好的XFUND中文数据集下载地址：[https://paddleocr.bj.bcebos.com/dataset/XFUND.tar](https://paddleocr.bj.bcebos.com/dataset/XFUND.tar)。 + + +下载并解压该数据集，解压后将数据集放置在当前目录下。 + +```shell +wget https://paddleocr.bj.bcebos.com/dataset/XFUND.tar +``` + +* 转换数据集 + +若需进行其他XFUND数据集的训练，可使用下面的命令进行数据集的转换 + +```bash +python3 ppstructure/vqa/tools/trans_xfun_data.py --ori_gt_path=path/to/json_path --output_path=path/to/save_path +``` + +* 下载预训练模型 +```bash +mkdir pretrain && cd pretrain +#下载SER模型 +wget https://paddleocr.bj.bcebos.com/pplayout/ser_LayoutXLM_xfun_zh.tar && tar -xvf ser_LayoutXLM_xfun_zh.tar +#下载RE模型 +wget https://paddleocr.bj.bcebos.com/pplayout/re_LayoutXLM_xfun_zh.tar && tar -xvf re_LayoutXLM_xfun_zh.tar +cd ../ +``` + +### 5.2 SER + +启动训练之前，需要修改下面的四个字段 + +1. `Train.dataset.data_dir`：指向训练集图片存放目录 +2. `Train.dataset.label_file_list`：指向训练集标注文件 +3. `Eval.dataset.data_dir`：指指向验证集图片存放目录 +4. `Eval.dataset.label_file_list`：指向验证集标注文件 + +* 启动训练 +```shell +CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/ser/layoutxlm.yml +``` + +最终会打印出`precision`, `recall`, `hmean`等指标。 +在`./output/ser_layoutxlm/`文件夹中会保存训练日志，最优的模型和最新epoch的模型。 + +* 恢复训练 + +恢复训练需要将之前训练好的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 + +```shell +CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/ser/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir +``` + +* 评估 + +评估需要将待评估的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 + +```shell +CUDA_VISIBLE_DEVICES=0 python3 tools/eval.py -c configs/vqa/ser/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir +``` +最终会打印出`precision`, `recall`, `hmean`等指标 + +* 使用`OCR引擎 + SER`串联预测 + +使用如下命令即可完成`OCR引擎 + SER`的串联预测, 以SER预训练模型为例: +```shell +CUDA_VISIBLE_DEVICES=0 python3 tools/infer_vqa_token_ser.py -c configs/vqa/ser/layoutxlm.yml -o Architecture.Backbone.checkpoints=pretrain/ser_LayoutXLM_xfun_zh/ Global.infer_img=doc/vqa/input/zh_val_42.jpg +``` + +最终会在`config.Global.save_res_path`字段所配置的目录下保存预测结果可视化图像以及预测结果文本文件，预测结果文本文件名为`infer_results.txt`。 + +* 对`OCR引擎 + SER`预测系统进行端到端评估 + +首先使用 `tools/infer_vqa_token_ser.py` 脚本完成数据集的预测，然后使用下面的命令进行评估。 + +```shell +export CUDA_VISIBLE_DEVICES=0 +python3 tools/eval_with_label_end2end.py --gt_json_path XFUND/zh_val/xfun_normalize_val.json --pred_json_path output_res/infer_results.txt +``` + +### 5.3 RE + +* 启动训练 + +启动训练之前，需要修改下面的四个字段 + +1. `Train.dataset.data_dir`：指向训练集图片存放目录 +2. `Train.dataset.label_file_list`：指向训练集标注文件 +3. `Eval.dataset.data_dir`：指指向验证集图片存放目录 +4. `Eval.dataset.label_file_list`：指向验证集标注文件 + +```shell +CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/re/layoutxlm.yml +``` + +最终会打印出`precision`, `recall`, `hmean`等指标。 +在`./output/re_layoutxlm/`文件夹中会保存训练日志，最优的模型和最新epoch的模型。 + +* 恢复训练 + +恢复训练需要将之前训练好的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 + +```shell +CUDA_VISIBLE_DEVICES=0 python3 tools/train.py -c configs/vqa/re/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir +``` + +* 评估 + +评估需要将待评估的模型所在文件夹路径赋值给 `Architecture.Backbone.checkpoints` 字段。 + +```shell +CUDA_VISIBLE_DEVICES=0 python3 tools/eval.py -c configs/vqa/re/layoutxlm.yml -o Architecture.Backbone.checkpoints=path/to/model_dir +``` +最终会打印出`precision`, `recall`, `hmean`等指标 + +* 使用`OCR引擎 + SER + RE`串联预测 + +使用如下命令即可完成`OCR引擎 + SER + RE`的串联预测, 以预训练SER和RE模型为例： +```shell +export CUDA_VISIBLE_DEVICES=0 +python3 tools/infer_vqa_token_ser_re.py -c configs/vqa/re/layoutxlm.yml -o Architecture.Backbone.checkpoints=pretrain/re_LayoutXLM_xfun_zh/ Global.infer_img=doc/vqa/input/zh_val_21.jpg -c_ser configs/vqa/ser/layoutxlm.yml -o_ser Architecture.Backbone.checkpoints=pretrain/ser_LayoutXLM_xfun_zh/ +``` + +最终会在`config.Global.save_res_path`字段所配置的目录下保存预测结果可视化图像以及预测结果文本文件，预测结果文本文件名为`infer_results.txt`。 + +## 6. 参考链接 + +- LayoutXLM: Multimodal Pre-training for Multilingual Visually-rich Document Understanding, https://arxiv.org/pdf/2104.08836.pdf +- microsoft/unilm/layoutxlm, https://github.com/microsoft/unilm/tree/master/layoutxlm +- XFUND dataset, https://github.com/doc-analysis/XFUND + +## License + +The content of this project itself is licensed under the [Attribution-NonCommercial-ShareAlike 4.0 International (CC BY-NC-SA 4.0)](https://creativecommons.org/licenses/by-nc-sa/4.0/) diff --git a/ppstructure/vqa/helper/eval_with_label_end2end.py b/ppstructure/vqa/tools/eval_with_label_end2end.py similarity index 100% rename from ppstructure/vqa/helper/eval_with_label_end2end.py rename to ppstructure/vqa/tools/eval_with_label_end2end.py diff --git a/ppstructure/vqa/helper/trans_xfun_data.py b/ppstructure/vqa/tools/trans_xfun_data.py similarity index 100% rename from ppstructure/vqa/helper/trans_xfun_data.py rename to ppstructure/vqa/tools/trans_xfun_data.py From 3bf40c7628cacb52498bd2830e7e75498e29e07c Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Fri, 8 Apr 2022 08:00:13 +0000 Subject: [PATCH 08/21] rm rec_char_type --- deploy/hubserving/ocr_rec/params.py | 1 - deploy/hubserving/ocr_system/params.py | 1 - deploy/hubserving/readme.md | 2 +- deploy/hubserving/readme_en.md | 2 +- deploy/hubserving/structure_table/params.py | 1 - doc/doc_ch/inference_ppocr.md | 15 +++++----- doc/doc_ch/quickstart.md | 25 +++++++++-------- doc/doc_ch/recognition.md | 31 +++++++++++---------- doc/doc_ch/serving_inference.md | 1 - doc/doc_ch/whl.md | 1 - doc/doc_en/inference_en.md | 4 +-- doc/doc_en/inference_ppocr_en.md | 15 +++++----- doc/doc_en/quickstart_en.md | 21 +++++++------- doc/doc_en/recognition_en.md | 4 +-- doc/doc_en/whl_en.md | 1 - ppstructure/table/README.md | 2 +- ppstructure/table/README_ch.md | 2 +- ppstructure/table/predict_structure.py | 5 ++-- ppstructure/utility.py | 1 - 19 files changed, 67 insertions(+), 68 deletions(-) diff --git a/deploy/hubserving/ocr_rec/params.py b/deploy/hubserving/ocr_rec/params.py index 70b50dd4d6..227a2b20ad 100644 --- a/deploy/hubserving/ocr_rec/params.py +++ b/deploy/hubserving/ocr_rec/params.py @@ -29,7 +29,6 @@ def read_params(): cfg.rec_model_dir = "./inference/ch_PP-OCRv2_rec_infer/" cfg.rec_image_shape = "3, 32, 320" - cfg.rec_char_type = 'ch' cfg.rec_batch_num = 30 cfg.max_text_length = 25 diff --git a/deploy/hubserving/ocr_system/params.py b/deploy/hubserving/ocr_system/params.py index 6d74294438..ca833831f8 100755 --- a/deploy/hubserving/ocr_system/params.py +++ b/deploy/hubserving/ocr_system/params.py @@ -47,7 +47,6 @@ def read_params(): cfg.rec_model_dir = "./inference/ch_PP-OCRv2_rec_infer/" cfg.rec_image_shape = "3, 32, 320" - cfg.rec_char_type = 'ch' cfg.rec_batch_num = 30 cfg.max_text_length = 25 diff --git a/deploy/hubserving/readme.md b/deploy/hubserving/readme.md index 22699d7122..ab6dbeff74 100755 --- a/deploy/hubserving/readme.md +++ b/deploy/hubserving/readme.md @@ -188,7 +188,7 @@ hub serving start -c deploy/hubserving/ocr_system/config.json - **output**：可视化结果保存路径，默认为`./hubserving_result` 访问示例： -```python tools/test_hubserving.py --server_url=http://127.0.0.1:8868/predict/ocr_system --image_dir./doc/imgs/ --visualize=false``` +```python tools/test_hubserving.py --server_url=http://127.0.0.1:8868/predict/ocr_system --image_dir=./doc/imgs/ --visualize=false``` ## 4. 返回结果格式说明 返回结果为列表（list），列表中的每一项为词典（dict），词典一共可能包含3种字段，信息如下： diff --git a/deploy/hubserving/readme_en.md b/deploy/hubserving/readme_en.md index b32e6aa822..8b99796a25 100755 --- a/deploy/hubserving/readme_en.md +++ b/deploy/hubserving/readme_en.md @@ -196,7 +196,7 @@ For example, if using the configuration file to start the text angle classificat **Eg.** ```shell -python tools/test_hubserving.py --server_url=http://127.0.0.1:8868/predict/ocr_system --image_dir./doc/imgs/ --visualize=false` +python tools/test_hubserving.py --server_url=http://127.0.0.1:8868/predict/ocr_system --image_dir=./doc/imgs/ --visualize=false` ``` ## 4. Returned result format diff --git a/deploy/hubserving/structure_table/params.py b/deploy/hubserving/structure_table/params.py index cc1a73687b..9632c2f70b 100755 --- a/deploy/hubserving/structure_table/params.py +++ b/deploy/hubserving/structure_table/params.py @@ -25,7 +25,6 @@ def read_params(): # params for table structure model cfg.table_max_len = 488 cfg.table_model_dir = './inference/en_ppocr_mobile_v2.0_table_structure_infer/' - cfg.table_char_type = 'en' cfg.table_char_dict_path = './ppocr/utils/dict/table_structure_dict.txt' cfg.show_log = False return cfg diff --git a/doc/doc_ch/inference_ppocr.md b/doc/doc_ch/inference_ppocr.md index 3e46f17d3a..5fb3811eb4 100644 --- a/doc/doc_ch/inference_ppocr.md +++ b/doc/doc_ch/inference_ppocr.md @@ -3,12 +3,13 @@ 本文介绍针对PP-OCR模型库的Python推理引擎使用方法，内容依次为文本检测、文本识别、方向分类器以及三者串联在CPU、GPU上的预测方法。 -- [1. 文本检测模型推理](#文本检测模型推理) -- [2. 文本识别模型推理](#文本识别模型推理) - - [2.1 超轻量中文识别模型推理](#超轻量中文识别模型推理) - - [2.2 多语言模型的推理](#多语言模型的推理) -- [3. 方向分类模型推理](#方向分类模型推理) -- [4. 文本检测、方向分类和文字识别串联推理](#文本检测、方向分类和文字识别串联推理) +- [基于Python引擎的PP-OCR模型库推理](#基于python引擎的pp-ocr模型库推理) + - [1. 文本检测模型推理](#1-文本检测模型推理) + - [2. 文本识别模型推理](#2-文本识别模型推理) + - [2.1 超轻量中文识别模型推理](#21-超轻量中文识别模型推理) + - [2.2 多语言模型的推理](#22-多语言模型的推理) + - [3. 方向分类模型推理](#3-方向分类模型推理) + - [4. 文本检测、方向分类和文字识别串联推理](#4-文本检测方向分类和文字识别串联推理) @@ -82,7 +83,7 @@ Predicts of ./doc/imgs_words/ch/word_4.jpg:('实力活力', 0.98458153) 如果您需要预测的是其他语言模型，可以在[此链接](./models_list.md#%E5%A4%9A%E8%AF%AD%E8%A8%80%E8%AF%86%E5%88%AB%E6%A8%A1%E5%9E%8B)中找到对应语言的inference模型，在使用inference模型预测时，需要通过`--rec_char_dict_path`指定使用的字典路径, 同时为了得到正确的可视化结果，需要通过 `--vis_font_path` 指定可视化的字体路径，`doc/fonts/` 路径下有默认提供的小语种字体，例如韩文识别： ``` wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/multilingual/korean_mobile_v2.0_rec_infer.tar -python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/korean/1.jpg" --rec_model_dir="./your inference model" --rec_char_type="korean" --rec_char_dict_path="ppocr/utils/dict/korean_dict.txt" --vis_font_path="doc/fonts/korean.ttf" +python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/korean/1.jpg" --rec_model_dir="./your inference model" --rec_char_dict_path="ppocr/utils/dict/korean_dict.txt" --vis_font_path="doc/fonts/korean.ttf" ``` ![](../imgs_words/korean/1.jpg) diff --git a/doc/doc_ch/quickstart.md b/doc/doc_ch/quickstart.md index 1e0d914140..49b0e7743e 100644 --- a/doc/doc_ch/quickstart.md +++ b/doc/doc_ch/quickstart.md @@ -1,14 +1,15 @@ # PaddleOCR快速开始 -- [1. 安装PaddleOCR whl包](#1) -- [2. 便捷使用](#2) - - [2.1 命令行使用](#21) - - [2.1.1 中英文模型](#211) - - [2.1.2 多语言模型](#212) - - [2.1.3 版面分析](#213) - - [2.2 Python脚本使用](#22) - - [2.2.1 中英文与多语言使用](#221) - - [2.2.2 版面分析](#222) +- [PaddleOCR快速开始](#paddleocr快速开始) + - [1. 安装PaddleOCR whl包](#1-安装paddleocr-whl包) + - [2. 便捷使用](#2-便捷使用) + - [2.1 命令行使用](#21-命令行使用) + - [2.1.1 中英文模型](#211-中英文模型) + - [2.1.2 多语言模型](#212-多语言模型) + - [2.1.3 版面分析](#213-版面分析) + - [2.2 Python脚本使用](#22-python脚本使用) + - [2.2.1 中英文与多语言使用](#221-中英文与多语言使用) + - [2.2.2 版面分析](#222-版面分析) @@ -165,8 +166,8 @@ paddleocr --image_dir=./table/1.png --type=structure /output/table/1/ └─ res.txt └─ [454, 360, 824, 658].xlsx 表格识别结果 - └─ [16, 2, 828, 305].jpg 被裁剪出的图片区域 - └─ [17, 361, 404, 711].xlsx 表格识别结果 + └─ [16, 2, 828, 305].jpg 被裁剪出的图片区域 + └─ [17, 361, 404, 711].xlsx 表格识别结果 ``` - **参数说明** @@ -176,7 +177,7 @@ paddleocr --image_dir=./table/1.png --type=structure | output | excel和识别结果保存的地址 | ./output/table | | table_max_len | 表格结构模型预测时，图像的长边resize尺度 | 488 | | table_model_dir | 表格结构模型 inference 模型地址 | None | - | table_char_type | 表格结构模型所用字典地址 | ../ppocr/utils/dict/table_structure_dict.txt | + | table_char_dict_path | 表格结构模型所用字典地址 | ../ppocr/utils/dict/table_structure_dict.txt | 大部分参数和paddleocr whl包保持一致，见 [whl包文档](./whl.md) diff --git a/doc/doc_ch/recognition.md b/doc/doc_ch/recognition.md index f030b0ee53..e7e7842273 100644 --- a/doc/doc_ch/recognition.md +++ b/doc/doc_ch/recognition.md @@ -2,19 +2,20 @@ 本文提供了PaddleOCR文本识别任务的全流程指南，包括数据准备、模型训练、调优、评估、预测，各个阶段的详细说明： -- [1 数据准备](#数据准备) - - [1.1 自定义数据集](#自定义数据集) - - [1.2 数据下载](#数据下载) - - [1.3 字典](#字典) - - [1.4 支持空格](#支持空格) -- [2 启动训练](#启动训练) - - [2.1 数据增强](#数据增强) - - [2.2 通用模型训练](#通用模型训练) - - [2.3 多语言模型训练](#多语言模型训练) - - [2.4 知识蒸馏训练](#知识蒸馏训练) -- [3 评估](#评估) -- [4 预测](#预测) -- [5 转Inference模型测试](#Inference) +- [文字识别](#文字识别) + - [1. 数据准备](#1-数据准备) + - [1.1 自定义数据集](#11-自定义数据集) + - [1.2 数据下载](#12-数据下载) + - [1.3 字典](#13-字典) + - [1.4 添加空格类别](#14-添加空格类别) + - [2. 启动训练](#2-启动训练) + - [2.1 数据增强](#21-数据增强) + - [2.2 通用模型训练](#22-通用模型训练) + - [2.3 多语言模型训练](#23-多语言模型训练) + - [2.4 知识蒸馏训练](#24-知识蒸馏训练) + - [3 评估](#3-评估) + - [4 预测](#4-预测) + - [5. 转Inference模型测试](#5-转inference模型测试) @@ -474,8 +475,8 @@ python3 tools/export_model.py -c configs/rec/ch_ppocr_v2.0/rec_chinese_lite_trai - 自定义模型推理 - 如果训练时修改了文本的字典，在使用inference模型预测时，需要通过`--rec_char_dict_path`指定使用的字典路径，并且设置 `rec_char_type=ch` + 如果训练时修改了文本的字典，在使用inference模型预测时，需要通过`--rec_char_dict_path`指定使用的字典路径 ``` - python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words_en/word_336.png" --rec_model_dir="./your inference model" --rec_image_shape="3, 32, 100" --rec_char_type="ch" --rec_char_dict_path="your text dict path" + python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words_en/word_336.png" --rec_model_dir="./your inference model" --rec_image_shape="3, 32, 100" --rec_char_dict_path="your text dict path" ``` diff --git a/doc/doc_ch/serving_inference.md b/doc/doc_ch/serving_inference.md index fea5a24546..30ea7ee7c1 100644 --- a/doc/doc_ch/serving_inference.md +++ b/doc/doc_ch/serving_inference.md @@ -98,7 +98,6 @@ def read_params(): cfg.rec_model_dir = "./ocr_rec_server/" # 识别算法模型路径 cfg.rec_image_shape = "3, 32, 320" - cfg.rec_char_type = 'ch' cfg.rec_batch_num = 30 cfg.max_text_length = 25 diff --git a/doc/doc_ch/whl.md b/doc/doc_ch/whl.md index 2d524b83d7..b2eb4ba17c 100644 --- a/doc/doc_ch/whl.md +++ b/doc/doc_ch/whl.md @@ -401,7 +401,6 @@ im_show.save('result.jpg') | rec_algorithm | 使用的识别算法类型 | CRNN | | rec_model_dir | 识别模型所在文件夹。传参方式有两种，1. None: 自动下载内置模型到 `~/.paddleocr/rec`；2.自己转换好的inference模型路径，模型路径下必须包含model和params文件 | None | | rec_image_shape | 识别算法的输入图片尺寸 | "3,32,320" | -| rec_char_type | 识别算法的字符类型，中英文(ch)、英文(en)、法语(french)、德语(german)、韩语(korean)、日语(japan) | ch | | rec_batch_num | 进行识别时，同时前向的图片数 | 30 | | max_text_length | 识别算法能识别的最大文字长度 | 25 | | rec_char_dict_path | 识别模型字典路径，当rec_model_dir使用方式2传参时需要修改为自己的字典路径 | ./ppocr/utils/ppocr_keys_v1.txt | diff --git a/doc/doc_en/inference_en.md b/doc/doc_en/inference_en.md index a8a96e30f0..d1233780d8 100755 --- a/doc/doc_en/inference_en.md +++ b/doc/doc_en/inference_en.md @@ -296,7 +296,7 @@ Predicts of ./doc/imgs_words_en/word_336.png:('super', 0.9999073) - The image resolution used in training is different: the image resolution used in training the above model is [3，32，100], while during our Chinese model training, in order to ensure the recognition effect of long text, the image resolution used in training is [3, 32, 320]. The default shape parameter of the inference stage is the image resolution used in training phase, that is [3, 32, 320]. Therefore, when running inference of the above English model here, you need to set the shape of the recognition image through the parameter `rec_image_shape`. -- Character list: the experiment in the DTRB paper is only for 26 lowercase English characters and 10 numbers, a total of 36 characters. All upper and lower case characters are converted to lower case characters, and characters not in the above list are ignored and considered as spaces. Therefore, no characters dictionary file is used here, but a dictionary is generated by the below command. Therefore, the parameter `rec_char_type` needs to be set during inference, which is specified as "en" in English. +- Character list: the experiment in the DTRB paper is only for 26 lowercase English characters and 10 numbers, a total of 36 characters. All upper and lower case characters are converted to lower case characters, and characters not in the above list are ignored and considered as spaces. Therefore, no characters dictionary file is used here, but a dictionary is generated by the below command. ``` self.character_str = "0123456789abcdefghijklmnopqrstuvwxyz" @@ -320,7 +320,7 @@ python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words_en/word_336.png ### 3.4 Text Recognition Model Inference Using Custom Characters Dictionary -If the text dictionary is modified during training, when using the inference model to predict, you need to specify the dictionary path used by `--rec_char_dict_path`, and set `rec_char_type=ch` +If the text dictionary is modified during training, when using the inference model to predict, you need to specify the dictionary path used by `--rec_char_dict_path` ``` python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words_en/word_336.png" --rec_model_dir="./your inference model" --rec_image_shape="3, 32, 100" --rec_char_dict_path="your text dict path" diff --git a/doc/doc_en/inference_ppocr_en.md b/doc/doc_en/inference_ppocr_en.md index 21f4c64d6d..8dc30d3106 100755 --- a/doc/doc_en/inference_ppocr_en.md +++ b/doc/doc_en/inference_ppocr_en.md @@ -4,12 +4,13 @@ This article introduces the use of the Python inference engine for the PP-OCR model library. The content is in order of text detection, text recognition, direction classifier and the prediction method of the three in series on the CPU and GPU. -- [Text Detection Model Inference](#DETECTION_MODEL_INFERENCE) -- [Text Recognition Model Inference](#RECOGNITION_MODEL_INFERENCE) - - [1. Lightweight Chinese Recognition Model Inference](#LIGHTWEIGHT_RECOGNITION) - - [2. Multilingual Model Inference](#MULTILINGUAL_MODEL_INFERENCE) -- [Angle Classification Model Inference](#ANGLE_CLASS_MODEL_INFERENCE) -- [Text Detection Angle Classification and Recognition Inference Concatenation](#CONCATENATION) +- [Python Inference for PP-OCR Model Zoo](#python-inference-for-pp-ocr-model-zoo) + - [Text Detection Model Inference](#text-detection-model-inference) + - [Text Recognition Model Inference](#text-recognition-model-inference) + - [1. Lightweight Chinese Recognition Model Inference](#1-lightweight-chinese-recognition-model-inference) + - [2. Multilingual Model Inference](#2-multilingual-model-inference) + - [Angle Classification Model Inference](#angle-classification-model-inference) + - [Text Detection Angle Classification and Recognition Inference Concatenation](#text-detection-angle-classification-and-recognition-inference-concatenation) @@ -82,7 +83,7 @@ You need to specify the visual font path through `--vis_font_path`. There are sm ``` wget wget https://paddleocr.bj.bcebos.com/dygraph_v2.0/multilingual/korean_mobile_v2.0_rec_infer.tar -python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/korean/1.jpg" --rec_model_dir="./your inference model" --rec_char_type="korean" --rec_char_dict_path="ppocr/utils/dict/korean_dict.txt" --vis_font_path="doc/fonts/korean.ttf" +python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words/korean/1.jpg" --rec_model_dir="./your inference model" --rec_char_dict_path="ppocr/utils/dict/korean_dict.txt" --vis_font_path="doc/fonts/korean.ttf" ``` ![](../imgs_words/korean/1.jpg) diff --git a/doc/doc_en/quickstart_en.md b/doc/doc_en/quickstart_en.md index 240a4ba11f..f7f65089a7 100644 --- a/doc/doc_en/quickstart_en.md +++ b/doc/doc_en/quickstart_en.md @@ -1,15 +1,16 @@ # PaddleOCR Quick Start -+ [1. Install PaddleOCR Whl Package](#1-install-paddleocr-whl-package) -* [2. Easy-to-Use](#2-easy-to-use) - + [2.1 Use by Command Line](#21-use-by-command-line) - - [2.1.1 English and Chinese Model](#211-english-and-chinese-model) - - [2.1.2 Multi-language Model](#212-multi-language-model) - - [2.1.3 Layout Analysis](#213-layoutAnalysis) - + [2.2 Use by Code](#22-use-by-code) - - [2.2.1 Chinese & English Model and Multilingual Model](#221-chinese---english-model-and-multilingual-model) - - [2.2.2 Layout Analysis](#222-layoutAnalysis) +- [PaddleOCR Quick Start](#paddleocr-quick-start) + - [1. Install PaddleOCR Whl Package](#1-install-paddleocr-whl-package) + - [2. Easy-to-Use](#2-easy-to-use) + - [2.1 Use by Command Line](#21-use-by-command-line) + - [2.1.1 Chinese and English Model](#211-chinese-and-english-model) + - [2.1.2 Multi-language Model](#212-multi-language-model) + - [2.1.3 Layout Analysis](#213-layout-analysis) + - [2.2 Use by Code](#22-use-by-code) + - [2.2.1 Chinese & English Model and Multilingual Model](#221-chinese--english-model-and-multilingual-model) + - [2.2.2 Layout Analysis](#222-layout-analysis) @@ -171,7 +172,7 @@ paddleocr --image_dir=../doc/table/1.png --type=structure | output | The path where excel and recognition results are saved | ./output/table | | table_max_len | The long side of the image is resized in table structure model | 488 | | table_model_dir | inference model path of table structure model | None | - | table_char_type | dict path of table structure model | ../ppocr/utils/dict/table_structure_dict.txt | + | table_char_dict_path | dict path of table structure model | ../ppocr/utils/dict/table_structure_dict.txt | diff --git a/doc/doc_en/recognition_en.md b/doc/doc_en/recognition_en.md index 20f4b9457b..c3700070b9 100644 --- a/doc/doc_en/recognition_en.md +++ b/doc/doc_en/recognition_en.md @@ -470,8 +470,8 @@ inference/det_db/ - Text recognition model Inference using custom characters dictionary - If the text dictionary is modified during training, when using the inference model to predict, you need to specify the dictionary path used by `--rec_char_dict_path`, and set `rec_char_type=ch` + If the text dictionary is modified during training, when using the inference model to predict, you need to specify the dictionary path used by `--rec_char_dict_path` ``` - python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words_en/word_336.png" --rec_model_dir="./your inference model" --rec_image_shape="3, 32, 100" --rec_char_type="ch" --rec_char_dict_path="your text dict path" + python3 tools/infer/predict_rec.py --image_dir="./doc/imgs_words_en/word_336.png" --rec_model_dir="./your inference model" --rec_image_shape="3, 32, 100" --rec_char_dict_path="your text dict path" ``` diff --git a/doc/doc_en/whl_en.md b/doc/doc_en/whl_en.md index 2671fbb9f0..35b2b1798a 100644 --- a/doc/doc_en/whl_en.md +++ b/doc/doc_en/whl_en.md @@ -348,7 +348,6 @@ im_show.save('result.jpg') | rec_algorithm | Type of recognition algorithm selected | CRNN | | rec_model_dir | the text recognition inference model folder. There are two ways to transfer parameters, 1. None: Automatically download the built-in model to `~/.paddleocr/rec`; 2. The path of the inference model converted by yourself, the model and params files must be included in the model path | None | | rec_image_shape | image shape of recognition algorithm | "3,32,320" | -| rec_char_type | Character type of recognition algorithm, Chinese (ch) or English (en) | ch | | rec_batch_num | When performing recognition, the batchsize of forward images | 30 | | max_text_length | The maximum text length that the recognition algorithm can recognize | 25 | | rec_char_dict_path | the alphabet path which needs to be modified to your own path when `rec_model_Name` use mode 2 | ./ppocr/utils/ppocr_keys_v1.txt | diff --git a/ppstructure/table/README.md b/ppstructure/table/README.md index 6137cfaef6..65d2cd22b6 100644 --- a/ppstructure/table/README.md +++ b/ppstructure/table/README.md @@ -117,7 +117,7 @@ teds: 93.32 ```python cd PaddleOCR/ppstructure -python3 table/predict_table.py --det_model_dir=path/to/det_model_dir --rec_model_dir=path/to/rec_model_dir --table_model_dir=path/to/table_model_dir --image_dir=../doc/table/1.png --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt --rec_char_type=EN --det_limit_side_len=736 --det_limit_type=min --output ../output/table +python3 table/predict_table.py --det_model_dir=path/to/det_model_dir --rec_model_dir=path/to/rec_model_dir --table_model_dir=path/to/table_model_dir --image_dir=../doc/table/1.png --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt --det_limit_side_len=736 --det_limit_type=min --output ../output/table ``` After running, the excel sheet of each picture will be saved in the directory specified by the output field diff --git a/ppstructure/table/README_ch.md b/ppstructure/table/README_ch.md index 39081995e6..4a617eeb46 100644 --- a/ppstructure/table/README_ch.md +++ b/ppstructure/table/README_ch.md @@ -117,7 +117,7 @@ teds: 93.32 ```python cd PaddleOCR/ppstructure -python3 table/predict_table.py --det_model_dir=path/to/det_model_dir --rec_model_dir=path/to/rec_model_dir --table_model_dir=path/to/table_model_dir --image_dir=../doc/table/1.png --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt --rec_char_type=EN --det_limit_side_len=736 --det_limit_type=min --output ../output/table +python3 table/predict_table.py --det_model_dir=path/to/det_model_dir --rec_model_dir=path/to/rec_model_dir --table_model_dir=path/to/table_model_dir --image_dir=../doc/table/1.png --rec_char_dict_path=../ppocr/utils/dict/table_dict.txt --table_char_dict_path=../ppocr/utils/dict/table_structure_dict.txt --det_limit_side_len=736 --det_limit_type=min --output ../output/table ``` Reference diff --git a/ppstructure/table/predict_structure.py b/ppstructure/table/predict_structure.py index fc85327b3a..0179c614ae 100755 --- a/ppstructure/table/predict_structure.py +++ b/ppstructure/table/predict_structure.py @@ -58,7 +58,6 @@ class TableStructurer(object): }] postprocess_params = { 'name': 'TableLabelDecode', - "character_type": args.table_char_type, "character_dict_path": args.table_char_dict_path, } @@ -104,7 +103,9 @@ class TableStructurer(object): res_loc_final.append([left, top, right, bottom]) structure_str_list = structure_str_list[0][:-1] - structure_str_list = ['', '', ''] + structure_str_list + ['
', '', ''] + structure_str_list = [ + '', '', '' + ] + structure_str_list + ['
', '', ''] elapse = time.time() - starttime return (structure_str_list, res_loc_final), elapse diff --git a/ppstructure/utility.py b/ppstructure/utility.py index 10d9f71a7c..081a5f6ae3 100644 --- a/ppstructure/utility.py +++ b/ppstructure/utility.py @@ -26,7 +26,6 @@ def init_args(): # params for table structure parser.add_argument("--table_max_len", type=int, default=488) parser.add_argument("--table_model_dir", type=str) - parser.add_argument("--table_char_type", type=str, default='en') parser.add_argument( "--table_char_dict_path", type=str, From cc5b6381faeaa986df4038436f23ddc65bbc97fa Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Fri, 8 Apr 2022 08:10:54 +0000 Subject: [PATCH 09/21] change rec_batch_num from 30 to 6 in hubserving --- deploy/hubserving/ocr_rec/params.py | 2 +- deploy/hubserving/ocr_system/params.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/hubserving/ocr_rec/params.py b/deploy/hubserving/ocr_rec/params.py index 227a2b20ad..09bdeeb3c6 100644 --- a/deploy/hubserving/ocr_rec/params.py +++ b/deploy/hubserving/ocr_rec/params.py @@ -29,7 +29,7 @@ def read_params(): cfg.rec_model_dir = "./inference/ch_PP-OCRv2_rec_infer/" cfg.rec_image_shape = "3, 32, 320" - cfg.rec_batch_num = 30 + cfg.rec_batch_num = 6 cfg.max_text_length = 25 cfg.rec_char_dict_path = "./ppocr/utils/ppocr_keys_v1.txt" diff --git a/deploy/hubserving/ocr_system/params.py b/deploy/hubserving/ocr_system/params.py index ca833831f8..9972a3ded8 100755 --- a/deploy/hubserving/ocr_system/params.py +++ b/deploy/hubserving/ocr_system/params.py @@ -47,7 +47,7 @@ def read_params(): cfg.rec_model_dir = "./inference/ch_PP-OCRv2_rec_infer/" cfg.rec_image_shape = "3, 32, 320" - cfg.rec_batch_num = 30 + cfg.rec_batch_num = 6 cfg.max_text_length = 25 cfg.rec_char_dict_path = "./ppocr/utils/ppocr_keys_v1.txt" From 02a892aa755093be1466c1ec74e0fc94d55d89fe Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Sat, 9 Apr 2022 02:19:01 +0000 Subject: [PATCH 10/21] Keep the parameter names the same as python --- deploy/cpp_infer/include/ocr_det.h | 6 ++-- deploy/cpp_infer/include/postprocess_op.h | 2 +- deploy/cpp_infer/{include => src}/clipper.cpp | 0 deploy/cpp_infer/src/main.cpp | 31 +++++++++++-------- deploy/cpp_infer/src/ocr_det.cpp | 2 +- deploy/cpp_infer/src/postprocess_op.cpp | 15 +++++---- 6 files changed, 30 insertions(+), 26 deletions(-) rename deploy/cpp_infer/{include => src}/clipper.cpp (100%) diff --git a/deploy/cpp_infer/include/ocr_det.h b/deploy/cpp_infer/include/ocr_det.h index 6e4086fbaa..e2f6d8c941 100644 --- a/deploy/cpp_infer/include/ocr_det.h +++ b/deploy/cpp_infer/include/ocr_det.h @@ -45,7 +45,7 @@ public: const double &det_db_thresh, const double &det_db_box_thresh, const double &det_db_unclip_ratio, - const bool &use_polygon_score, const bool &use_dilation, + const bool &det_db_score_mode, const bool &use_dilation, const bool &use_tensorrt, const std::string &precision) { this->use_gpu_ = use_gpu; this->gpu_id_ = gpu_id; @@ -58,7 +58,7 @@ public: this->det_db_thresh_ = det_db_thresh; this->det_db_box_thresh_ = det_db_box_thresh; this->det_db_unclip_ratio_ = det_db_unclip_ratio; - this->use_polygon_score_ = use_polygon_score; + this->det_db_score_mode_ = det_db_score_mode; this->use_dilation_ = use_dilation; this->use_tensorrt_ = use_tensorrt; @@ -88,7 +88,7 @@ private: double det_db_thresh_ = 0.3; double det_db_box_thresh_ = 0.5; double det_db_unclip_ratio_ = 2.0; - bool use_polygon_score_ = false; + std::string det_db_score_mode_ = "slow"; bool use_dilation_ = false; bool visualize_ = true; diff --git a/deploy/cpp_infer/include/postprocess_op.h b/deploy/cpp_infer/include/postprocess_op.h index b384b79b30..4a98b151bd 100644 --- a/deploy/cpp_infer/include/postprocess_op.h +++ b/deploy/cpp_infer/include/postprocess_op.h @@ -56,7 +56,7 @@ public: std::vector>> BoxesFromBitmap(const cv::Mat pred, const cv::Mat bitmap, const float &box_thresh, const float &det_db_unclip_ratio, - const bool &use_polygon_score); + const std::string &det_db_score_mode); std::vector>> FilterTagDetRes(std::vector>> boxes, diff --git a/deploy/cpp_infer/include/clipper.cpp b/deploy/cpp_infer/src/clipper.cpp similarity index 100% rename from deploy/cpp_infer/include/clipper.cpp rename to deploy/cpp_infer/src/clipper.cpp diff --git a/deploy/cpp_infer/src/main.cpp b/deploy/cpp_infer/src/main.cpp index efc1e50ce9..aebb0cb32d 100644 --- a/deploy/cpp_infer/src/main.cpp +++ b/deploy/cpp_infer/src/main.cpp @@ -36,25 +36,26 @@ #include "auto_log/autolog.h" #include +// common args DEFINE_bool(use_gpu, false, "Infering with GPU or CPU."); +DEFINE_bool(use_tensorrt, false, "Whether use tensorrt."); DEFINE_int32(gpu_id, 0, "Device id of GPU to execute."); DEFINE_int32(gpu_mem, 4000, "GPU id when infering with GPU."); DEFINE_int32(cpu_threads, 10, "Num of threads with CPU."); DEFINE_bool(enable_mkldnn, false, "Whether use mkldnn with CPU."); -DEFINE_bool(use_tensorrt, false, "Whether use tensorrt."); DEFINE_string(precision, "fp32", "Precision be one of fp32/fp16/int8"); DEFINE_bool(benchmark, false, "Whether use benchmark."); DEFINE_string(output, "./output/", "Save benchmark log path."); -// detection related DEFINE_string(image_dir, "", "Dir of input image."); +DEFINE_bool(visualize, true, "Whether show the detection results."); +// detection related DEFINE_string(det_model_dir, "", "Path of det inference model."); DEFINE_int32(max_side_len, 960, "max_side_len of input image."); DEFINE_double(det_db_thresh, 0.3, "Threshold of det_db_thresh."); DEFINE_double(det_db_box_thresh, 0.6, "Threshold of det_db_box_thresh."); DEFINE_double(det_db_unclip_ratio, 1.5, "Threshold of det_db_unclip_ratio."); -DEFINE_bool(use_polygon_score, false, "Whether use polygon score."); DEFINE_bool(use_dilation, false, "Whether use the dilation on output map."); -DEFINE_bool(visualize, true, "Whether show the detection results."); +DEFINE_bool(det_db_score_mode, false, "Whether use polygon score."); // classification related DEFINE_bool(use_angle_cls, false, "Whether use use_angle_cls."); DEFINE_string(cls_model_dir, "", "Path of cls inference model."); @@ -85,7 +86,7 @@ int main_det(std::vector cv_all_img_names) { FLAGS_gpu_mem, FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_max_side_len, FLAGS_det_db_thresh, FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio, - FLAGS_use_polygon_score, FLAGS_use_dilation, + FLAGS_det_db_score_mode, FLAGS_use_dilation, FLAGS_use_tensorrt, FLAGS_precision); if (!PathExists(FLAGS_output)) { @@ -117,13 +118,21 @@ int main_det(std::vector cv_all_img_names) { time_info[2] += det_times[2]; if (FLAGS_benchmark) { - cout << cv_all_img_names[i] << '\t'; + cout << cv_all_img_names[i] << "\t["; for (int n = 0; n < boxes.size(); n++) { + cout << '['; for (int m = 0; m < boxes[n].size(); m++) { - cout << boxes[n][m][0] << ' ' << boxes[n][m][1] << ' '; + cout << '[' << boxes[n][m][0] << ',' << boxes[n][m][1] << "]"; + if (m != boxes[n].size() - 1) { + cout << ','; + } + } + cout << ']'; + if (n != boxes.size() - 1) { + cout << ','; } } - cout << endl; + cout << ']' << endl; } } @@ -140,8 +149,6 @@ int main_rec(std::vector cv_all_img_names) { std::vector time_info = {0, 0, 0}; std::string rec_char_dict_path = FLAGS_rec_char_dict_path; - if (FLAGS_benchmark) - rec_char_dict_path = FLAGS_rec_char_dict_path.substr(6); cout << "label file: " << rec_char_dict_path << endl; CRNNRecognizer rec(FLAGS_rec_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, @@ -194,7 +201,7 @@ int main_system(std::vector cv_all_img_names) { FLAGS_gpu_mem, FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_max_side_len, FLAGS_det_db_thresh, FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio, - FLAGS_use_polygon_score, FLAGS_use_dilation, + FLAGS_det_db_score_mode, FLAGS_use_dilation, FLAGS_use_tensorrt, FLAGS_precision); Classifier *cls = nullptr; @@ -205,8 +212,6 @@ int main_system(std::vector cv_all_img_names) { } std::string rec_char_dict_path = FLAGS_rec_char_dict_path; - if (FLAGS_benchmark) - rec_char_dict_path = FLAGS_rec_char_dict_path.substr(6); cout << "label file: " << rec_char_dict_path << endl; CRNNRecognizer rec(FLAGS_rec_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, diff --git a/deploy/cpp_infer/src/ocr_det.cpp b/deploy/cpp_infer/src/ocr_det.cpp index d72dc40cdd..ff7164ca24 100644 --- a/deploy/cpp_infer/src/ocr_det.cpp +++ b/deploy/cpp_infer/src/ocr_det.cpp @@ -161,7 +161,7 @@ void DBDetector::Run(cv::Mat &img, boxes = post_processor_.BoxesFromBitmap( pred_map, bit_map, this->det_db_box_thresh_, this->det_db_unclip_ratio_, - this->use_polygon_score_); + this->det_db_score_mode_); boxes = post_processor_.FilterTagDetRes(boxes, ratio_h, ratio_w, srcimg); auto postprocess_end = std::chrono::steady_clock::now(); diff --git a/deploy/cpp_infer/src/postprocess_op.cpp b/deploy/cpp_infer/src/postprocess_op.cpp index c398557204..5374fb1a4e 100644 --- a/deploy/cpp_infer/src/postprocess_op.cpp +++ b/deploy/cpp_infer/src/postprocess_op.cpp @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include #include -#include namespace PaddleOCR { @@ -187,23 +187,22 @@ float PostProcessor::PolygonScoreAcc(std::vector contour, cv::Mat mask; mask = cv::Mat::zeros(ymax - ymin + 1, xmax - xmin + 1, CV_8UC1); + cv::Point *rook_point = new cv::Point[contour.size()]; - cv::Point* rook_point = new cv::Point[contour.size()]; - for (int i = 0; i < contour.size(); ++i) { rook_point[i] = cv::Point(int(box_x[i]) - xmin, int(box_y[i]) - ymin); } const cv::Point *ppt[1] = {rook_point}; int npt[] = {int(contour.size())}; - cv::fillPoly(mask, ppt, npt, 1, cv::Scalar(1)); cv::Mat croppedImg; - pred(cv::Rect(xmin, ymin, xmax - xmin + 1, ymax - ymin + 1)).copyTo(croppedImg); + pred(cv::Rect(xmin, ymin, xmax - xmin + 1, ymax - ymin + 1)) + .copyTo(croppedImg); float score = cv::mean(croppedImg, mask)[0]; - delete []rook_point; + delete[] rook_point; return score; } @@ -247,7 +246,7 @@ float PostProcessor::BoxScoreFast(std::vector> box_array, std::vector>> PostProcessor::BoxesFromBitmap( const cv::Mat pred, const cv::Mat bitmap, const float &box_thresh, - const float &det_db_unclip_ratio, const bool &use_polygon_score) { + const float &det_db_unclip_ratio, const std::string &det_db_score_mode) { const int min_size = 3; const int max_candidates = 1000; @@ -281,7 +280,7 @@ std::vector>> PostProcessor::BoxesFromBitmap( } float score; - if (use_polygon_score) + if (det_db_score_mode == "slow") /* compute using polygon*/ score = PolygonScoreAcc(contours[_i], pred); else From 15a22fa05a63fad3374d0a0af0e0e4ec6e863add Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Sat, 9 Apr 2022 03:07:13 +0000 Subject: [PATCH 11/21] fix bug --- deploy/cpp_infer/include/ocr_det.h | 5 +++-- deploy/cpp_infer/readme.md | 2 +- deploy/cpp_infer/readme_en.md | 2 +- deploy/cpp_infer/src/main.cpp | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/deploy/cpp_infer/include/ocr_det.h b/deploy/cpp_infer/include/ocr_det.h index e2f6d8c941..2439249534 100644 --- a/deploy/cpp_infer/include/ocr_det.h +++ b/deploy/cpp_infer/include/ocr_det.h @@ -45,8 +45,9 @@ public: const double &det_db_thresh, const double &det_db_box_thresh, const double &det_db_unclip_ratio, - const bool &det_db_score_mode, const bool &use_dilation, - const bool &use_tensorrt, const std::string &precision) { + const std::string &det_db_score_mode, + const bool &use_dilation, const bool &use_tensorrt, + const std::string &precision) { this->use_gpu_ = use_gpu; this->gpu_id_ = gpu_id; this->gpu_mem_ = gpu_mem; diff --git a/deploy/cpp_infer/readme.md b/deploy/cpp_infer/readme.md index e710488102..d269cc9e8f 100644 --- a/deploy/cpp_infer/readme.md +++ b/deploy/cpp_infer/readme.md @@ -267,7 +267,7 @@ CUDNN_LIB_DIR=/your_cudnn_lib_dir |det_db_thresh|float|0.3|用于过滤DB预测的二值化图像，设置为0.-0.3对结果影响不明显| |det_db_box_thresh|float|0.5|DB后处理过滤box的阈值，如果检测存在漏框情况，可酌情减小| |det_db_unclip_ratio|float|1.6|表示文本框的紧致程度，越小则文本框更靠近文本| -|use_polygon_score|bool|false|是否使用多边形框计算bbox score，false表示使用矩形框计算。矩形框计算速度更快，多边形框对弯曲文本区域计算更准确。| +|det_db_score_mode|string|slow|slow:使用多边形框计算bbox score，fast:使用矩形框计算。矩形框计算速度更快，多边形框对弯曲文本区域计算更准确。| |visualize|bool|true|是否对结果进行可视化，为1时，预测结果会保存在`output`字段指定的文件夹下和输入图像同名的图像上。| - 方向分类器相关 diff --git a/deploy/cpp_infer/readme_en.md b/deploy/cpp_infer/readme_en.md index 61d6509539..b7687a9f65 100644 --- a/deploy/cpp_infer/readme_en.md +++ b/deploy/cpp_infer/readme_en.md @@ -260,7 +260,7 @@ More parameters are as follows, |det_db_thresh|float|0.3|Used to filter the binarized image of DB prediction, setting 0.-0.3 has no obvious effect on the result| |det_db_box_thresh|float|0.5|DB post-processing filter box threshold, if there is a missing box detected, it can be reduced as appropriate| |det_db_unclip_ratio|float|1.6|Indicates the compactness of the text box, the smaller the value, the closer the text box to the text| -|use_polygon_score|bool|false|Whether to use polygon box to calculate bbox score, false means to use rectangle box to calculate. Use rectangular box to calculate faster, and polygonal box more accurate for curved text area.| +|det_db_score_mode|string|slow| slow: use polygon box to calculate bbox score, fast: use rectangle box to calculate. Use rectangular box to calculate faster, and polygonal box more accurate for curved text area.| |visualize|bool|true|Whether to visualize the results，when it is set as true, the prediction results will be saved in the folder specified by the `output` field on an image with the same name as the input image.| - Classifier related parameters diff --git a/deploy/cpp_infer/src/main.cpp b/deploy/cpp_infer/src/main.cpp index aebb0cb32d..046a68a969 100644 --- a/deploy/cpp_infer/src/main.cpp +++ b/deploy/cpp_infer/src/main.cpp @@ -55,7 +55,7 @@ DEFINE_double(det_db_thresh, 0.3, "Threshold of det_db_thresh."); DEFINE_double(det_db_box_thresh, 0.6, "Threshold of det_db_box_thresh."); DEFINE_double(det_db_unclip_ratio, 1.5, "Threshold of det_db_unclip_ratio."); DEFINE_bool(use_dilation, false, "Whether use the dilation on output map."); -DEFINE_bool(det_db_score_mode, false, "Whether use polygon score."); +DEFINE_string(det_db_score_mode, "slow", "Whether use polygon score."); // classification related DEFINE_bool(use_angle_cls, false, "Whether use use_angle_cls."); DEFINE_string(cls_model_dir, "", "Path of cls inference model."); From f889907c1c06e9c70b81d11a997bae1ef4dbb57c Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Sat, 9 Apr 2022 03:42:36 +0000 Subject: [PATCH 12/21] move args to args.h --- deploy/cpp_infer/include/args.h | 46 +++++++++++++++++++++++++++++++++ deploy/cpp_infer/src/args.cpp | 45 ++++++++++++++++++++++++++++++++ deploy/cpp_infer/src/main.cpp | 32 +---------------------- 3 files changed, 92 insertions(+), 31 deletions(-) create mode 100644 deploy/cpp_infer/include/args.h create mode 100644 deploy/cpp_infer/src/args.cpp diff --git a/deploy/cpp_infer/include/args.h b/deploy/cpp_infer/include/args.h new file mode 100644 index 0000000000..4f87c0fa5f --- /dev/null +++ b/deploy/cpp_infer/include/args.h @@ -0,0 +1,46 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +// common args +DECLARE_bool(use_gpu); +DECLARE_bool(use_tensorrt); +DECLARE_int32(gpu_id); +DECLARE_int32(gpu_mem); +DECLARE_int32(cpu_threads); +DECLARE_bool(enable_mkldnn); +DECLARE_string(precision); +DECLARE_bool(benchmark); +DECLARE_string(output); +DECLARE_string(image_dir); +DECLARE_bool(visualize); +// detection related +DECLARE_string(det_model_dir); +DECLARE_int32(max_side_len); +DECLARE_double(det_db_thresh); +DECLARE_double(det_db_box_thresh); +DECLARE_double(det_db_unclip_ratio); +DECLARE_bool(use_dilation); +DECLARE_string(det_db_score_mode); +// classification related +DECLARE_bool(use_angle_cls); +DECLARE_string(cls_model_dir); +DECLARE_double(cls_thresh); +// recognition related +DECLARE_string(rec_model_dir); +DECLARE_int32(rec_batch_num); +DECLARE_string(rec_char_dict_path); \ No newline at end of file diff --git a/deploy/cpp_infer/src/args.cpp b/deploy/cpp_infer/src/args.cpp new file mode 100644 index 0000000000..539968eb1a --- /dev/null +++ b/deploy/cpp_infer/src/args.cpp @@ -0,0 +1,45 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include + +// common args +DEFINE_bool(use_gpu, false, "Infering with GPU or CPU."); +DEFINE_bool(use_tensorrt, false, "Whether use tensorrt."); +DEFINE_int32(gpu_id, 0, "Device id of GPU to execute."); +DEFINE_int32(gpu_mem, 4000, "GPU id when infering with GPU."); +DEFINE_int32(cpu_threads, 10, "Num of threads with CPU."); +DEFINE_bool(enable_mkldnn, false, "Whether use mkldnn with CPU."); +DEFINE_string(precision, "fp32", "Precision be one of fp32/fp16/int8"); +DEFINE_bool(benchmark, false, "Whether use benchmark."); +DEFINE_string(output, "./output/", "Save benchmark log path."); +DEFINE_string(image_dir, "", "Dir of input image."); +DEFINE_bool(visualize, true, "Whether show the detection results."); +// detection related +DEFINE_string(det_model_dir, "", "Path of det inference model."); +DEFINE_int32(max_side_len, 960, "max_side_len of input image."); +DEFINE_double(det_db_thresh, 0.3, "Threshold of det_db_thresh."); +DEFINE_double(det_db_box_thresh, 0.6, "Threshold of det_db_box_thresh."); +DEFINE_double(det_db_unclip_ratio, 1.5, "Threshold of det_db_unclip_ratio."); +DEFINE_bool(use_dilation, false, "Whether use the dilation on output map."); +DEFINE_string(det_db_score_mode, "slow", "Whether use polygon score."); +// classification related +DEFINE_bool(use_angle_cls, false, "Whether use use_angle_cls."); +DEFINE_string(cls_model_dir, "", "Path of cls inference model."); +DEFINE_double(cls_thresh, 0.9, "Threshold of cls_thresh."); +// recognition related +DEFINE_string(rec_model_dir, "", "Path of rec inference model."); +DEFINE_int32(rec_batch_num, 6, "rec_batch_num."); +DEFINE_string(rec_char_dict_path, "../../ppocr/utils/ppocr_keys_v1.txt", + "Path of dictionary."); \ No newline at end of file diff --git a/deploy/cpp_infer/src/main.cpp b/deploy/cpp_infer/src/main.cpp index 046a68a969..27c7bde09c 100644 --- a/deploy/cpp_infer/src/main.cpp +++ b/deploy/cpp_infer/src/main.cpp @@ -27,6 +27,7 @@ #include #include +#include #include #include #include @@ -34,37 +35,6 @@ #include #include "auto_log/autolog.h" -#include - -// common args -DEFINE_bool(use_gpu, false, "Infering with GPU or CPU."); -DEFINE_bool(use_tensorrt, false, "Whether use tensorrt."); -DEFINE_int32(gpu_id, 0, "Device id of GPU to execute."); -DEFINE_int32(gpu_mem, 4000, "GPU id when infering with GPU."); -DEFINE_int32(cpu_threads, 10, "Num of threads with CPU."); -DEFINE_bool(enable_mkldnn, false, "Whether use mkldnn with CPU."); -DEFINE_string(precision, "fp32", "Precision be one of fp32/fp16/int8"); -DEFINE_bool(benchmark, false, "Whether use benchmark."); -DEFINE_string(output, "./output/", "Save benchmark log path."); -DEFINE_string(image_dir, "", "Dir of input image."); -DEFINE_bool(visualize, true, "Whether show the detection results."); -// detection related -DEFINE_string(det_model_dir, "", "Path of det inference model."); -DEFINE_int32(max_side_len, 960, "max_side_len of input image."); -DEFINE_double(det_db_thresh, 0.3, "Threshold of det_db_thresh."); -DEFINE_double(det_db_box_thresh, 0.6, "Threshold of det_db_box_thresh."); -DEFINE_double(det_db_unclip_ratio, 1.5, "Threshold of det_db_unclip_ratio."); -DEFINE_bool(use_dilation, false, "Whether use the dilation on output map."); -DEFINE_string(det_db_score_mode, "slow", "Whether use polygon score."); -// classification related -DEFINE_bool(use_angle_cls, false, "Whether use use_angle_cls."); -DEFINE_string(cls_model_dir, "", "Path of cls inference model."); -DEFINE_double(cls_thresh, 0.9, "Threshold of cls_thresh."); -// recognition related -DEFINE_string(rec_model_dir, "", "Path of rec inference model."); -DEFINE_int32(rec_batch_num, 6, "rec_batch_num."); -DEFINE_string(rec_char_dict_path, "../../ppocr/utils/ppocr_keys_v1.txt", - "Path of dictionary."); using namespace std; using namespace cv; From 41c602546223bda871eac4c81fc51629043cfc45 Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Sun, 10 Apr 2022 13:28:27 +0000 Subject: [PATCH 13/21] Refactor the code to keep the logic consistent with the whl package --- deploy/cpp_infer/imgs/cpp_infer_pred_12.png | Bin 26798 -> 0 bytes deploy/cpp_infer/include/args.h | 10 +- deploy/cpp_infer/include/ocr_cls.h | 12 +- deploy/cpp_infer/include/ocr_det.h | 2 +- deploy/cpp_infer/include/ocr_rec.h | 2 +- deploy/cpp_infer/include/paddleocr.h | 67 +++++ deploy/cpp_infer/include/utility.h | 19 +- deploy/cpp_infer/readme.md | 122 +++++--- deploy/cpp_infer/readme_en.md | 128 ++++++--- deploy/cpp_infer/src/args.cpp | 13 +- deploy/cpp_infer/src/main.cpp | 295 ++------------------ deploy/cpp_infer/src/ocr_cls.cpp | 128 +++++---- deploy/cpp_infer/src/ocr_det.cpp | 9 +- deploy/cpp_infer/src/ocr_rec.cpp | 16 +- deploy/cpp_infer/src/paddleocr.cpp | 225 +++++++++++++++ deploy/cpp_infer/src/utility.cpp | 53 +++- 16 files changed, 681 insertions(+), 420 deletions(-) delete mode 100644 deploy/cpp_infer/imgs/cpp_infer_pred_12.png create mode 100644 deploy/cpp_infer/include/paddleocr.h create mode 100644 deploy/cpp_infer/src/paddleocr.cpp diff --git a/deploy/cpp_infer/imgs/cpp_infer_pred_12.png b/deploy/cpp_infer/imgs/cpp_infer_pred_12.png deleted file mode 100644 index eb5f64e1f6c329f7ae772c50edce7fc8afcb1211..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 26798 zcmagFV~{3Y)UH{!ZQC}wY_rR@ZQHhOblFB%J>@QT*|whI^!uH2=EuZL%%6;m9Xod9 zzBATZcjmfww6dZUA{-tZ2nYzGjI_8a2ncBAzk3fDsDD=o*zJOU1Af+GV#+dNVkF8g zjuzH-<{%(6$)-j|bTSMylg7qIMw2u2v~Vt-s!>sKszw7NgENC8B%?-SBn4S|db>C% zyCXs9em&!ZFeS_vApqHj5AUUlHoq?{uEuN^O-jDbr(jS|SsQ0IaByKt_c?&0;Y?31Q53K%19HFOf++(6VMm!8Fm)4iU`cK-R%z zP}-rQ@{x>ngVuiIM$`Uihn{4L!XFG;8j%><98_IR4(gGC>f$YD;r~%QmYRc&hu{K@ zGR@+}(#b-@lEse0qRn>7#*cY>{sWnN*KM#BRp#-%aDE8J1kO3@}QHN?*R zQ%P!Qa&R7`3<<*$rBoO?Sius5KM`fxaB|S78ZbDx8Hys8Gd|elj|?6)8# zU<(yte*%XVi?)Yk`7tgUWteAS2Vwlc!^+aM{r>&kf(H)%P5~7$fd2jc-FN!^JrEoG z41RF=0||t}(|R|~77ZB$a2n94QIdY)l`JNu9jkLfMiz@X`8{e_zYoT4?1~cVy`W zlV=5t;Z-XsS;bxlFWHQl*68mFwg{o4>Cqd(E7beJv6A)VvOP_ zv|fQpgv;Xp8cCwP{ZQNeF8HRK;!L9@QTtI5Bp+<40Giz;U>h?H{Ed@BuAP}&b$8u( zQE!DG6o_b=4=3B~@@`#ChG}mw;cIZs^28Q-vWajgWcDNw8cEu81#Xv>9#)EstHs^f ze}5LS&=;5jyr^KjCg%Q}C~xd%`#txc60fc*KVrKGW(4CR5CP5|7*=Z~m<09!r-w__ zJZ7l#tS!mC8R#Nx8vomqnlaP#3Y&cSgup~|-F)Tm*NR9Bh7y{_M<489B8$%EY{1c) zDP}@)ZN{8-j1t#(O_!jbo-NOP zklbYTB`J+pShmq=^U~#u8*V1 zJI^+e67t_kt>26x&Sw~vi9<#`^!u@icpR7fU1VBwe4bBH6|9E!qhl*ZDABX$2_^;CnRXbh_6qWGbBQfy zR0&nWQ7iSDFmDVYFeN;B_o@0D0psOCv%!?Y!H{S4)XOJz`B9BsTa=BtLEfL^f>L$) z-*QQGSTafo9{ug>TC??=kLg@I5QgzTx2as$t700yeUxBQF&u7EnsEw}LSx!Tl3J`$ zpnaR)elJF8MFIWAn(nhWH9zke`EqqxzDYuXN zlb_jUO3vl8zNyC6b@iR5XeB?5S!Si$o3WKp!{1FFnxIiJIuG*K%icY$eAtNoPBi~7 zcjjPtSsjy+r4_D`uwqpkP0^O4&=>2YTeE~HP*bR+k@3pmTkmqU{L06yeR z7|5!A-}z56<3`OQ)wh~dDB;s1dJPgX>pXHW=DHi$S= zJGj@ds6O)D{E#rA!safSE6b2B+ydu1?6h2}FxkW!1MFD6>6EiNsUiB08LU?1OYLDU zBJIKQ-x;Wo{$g8fauGpuNq-N3QeE~*PiDb3$sytYFDrlHz^UVE*{a6v@7$=vQiY*- z#9vZKs=ApldtiScv9pTG%_6)c5UXzC^v1Z&D%&a?(b?&iMwi3NA{|u|-RXp(ztc+) zcKyX1iPCS{`@rYzlCjnk_UvM--&kqd&o6`20;&%r#;KEK(U>Xd*j%M$Qh!xg7^+; zHZfuz6|Pf!J-`r8p#N8<22E9Vle1!fayC1Bx}}ZuGuzcQ7*9#L5|@=e`?#su5Tg+g-Q!jVAgi~do*URh~>(NvP_4#8X z*ujsg)m6;>tU{}+i@>;y&Qvly_Is`Z#valMl-`D;5Mt$z2Gc*EZgxYjQ0IxhOvayY z*2ozW7F7@tzSWfI0~CjNzF_an*5ZIsh<1@dFoA5D92zJGtL^zHyKcnQxWBXkJk5tF zY>)*4km@5wZM3@T?~*`K(ni1IXN=_5)5-B$c!5~T9d-#APP!?|p*BnM6 zH}R9p8xwuvPwC+m&vD)Ud6Y@6#04_BbrpYv!$w4fE%)v&r)y{5J)hpSz1XA|D+$=Z zp&?`y3d+YF1P0YrxZ2D1E!$3HllI7?_uBm#hL-<(70{E|3+e)GyVR-~$Cg^$Xm&e6 zI~NlfnRyH<5C;=2tX9Zh;&U02b$_AUQ9}26#~v>(Hk)XXS*v!q2W+h$kr@`lz zv`u;={pClq?NVe442RVhhHGiKnGFnkvn8xY<%|Iw=#_$M75z~h$D0LkF+q%>ShyFH=Ue2~_WQ8_ zN%vXhFRHGeSUf~RQ9nXYE`T68szJFt5re92u$M)=v&Dne8G zIN>r_@OLU+)UV-*5ru3UGj?qVC_f*BB)VU~UXy>%9v3=kvA@WyQfmm&HroUcaegNo z$_GJ9BzZ@tc4jlBVY;Ajneq1TaD?E0u(Np4r;~I>n$djP#Hb%N8@v$-hZr)Y4^3Bc zZ#*a+#mrh5o!2oeUy13@#LXrU(ed&&{g<^l%LdiJ;ad)X3D1gc%6<=6F0Qs*t{9OV zFIPChv46zje+IzKu8ehh-7RoLARIAocR7jradCgMA_bw*F165(9*h4Dhv%uO%-fHX z13y-CsXu?OG!$aOBWC@^dnOD$TjfCP+w&?IZ|Oo`Q0y}4at7;~AnHe6yCe96AAe)| z@|-teW!hJsO8j5_RR)I>;5=3h?X87C1yDu;j$TthVdScC#cEQbKK}ET|DO0?UU7m` z_Z9t4q{~i{JgWiuRxwY0NdG_PQ>t{Z(B|mgN+Zeif4R?pkz5!L7TBFWT^Rqv#rwY( z|39X5kCuczGqN(9Kjrkf=LpFVj$!HEEZY$&$!0T1G-Mk2qqQAjS42Y&#k;z z`5f4W1?a`#=qG?(7pTg>$)dUT6X>-**Y3s9Uhw^PO7xIHN$Lb}zjrBP?_mflvvQ;Z z!3-xz+?F;Y6hCTelba0KbC$$?*sNx}>M*WbgoN?^Gm&K~)cmxNKRn-bH}rX8&AAr0 z*nb8Q`s;7Z4nI>uSWf7E3*12uIbSsMleR*g@iqBZ*8W{xTEzzK-@dIsW-h8^7O++R zipK9~wlA?w8X!3Qtvm<{IwS&fK@4H2wB=Z9!5h_%4F4S2s1=(dF%S3j3LNP|5Bd4E zYh<{xr!rNfQB>(t`i)Ya&z<29boAog*d2rq8170?BiGJZ*1#F%ScJgTH=OAC;&?s1 zf$TUdMd^WzB6_4_9LU`*jWdfYf%Q@d$<4+zc3&eoGb%~rnDczR+(+`I-sNQGCJ?c0 zb#@M!x#)Ki`UFW8CIZod_7z2{ip#|`L56PBQ#iPkW=Ux`=EX4pP4iJweg|zXgD+DDrA}06)n}!n^Y^~(^QV8$(4&r>}lGm z<0E3bV-uxB?$yHpC6Ea@Jgi&}Igrfd=E$h!RYvHdq6f~wsg=amD!mO3e4?Mrmr$F0 zk0t3Zc!!Q=ABY9X(%PvWI$anHxWS%Fn{mgDm}j&*RtfH^nxN>%#;RUgPewKt#SY<> zdTGMe_sAG4AVhzPyr>{pFWx#zwbL;p3zET;c6x<+d0z7X+o5vi70UaK)oN_)--Os4 zc0t_{vt@+!7K`s|Vdm#fUakTbn{QBzI{bT3^$L}Qyy4`~$Rrw^R<&X}az!?Dx}XW| zgDOb44DB7P0cF~;z4E}kVKzJN%M(5N4!x5PGBmgd?V-O7OJDS_7|w4hQKRu9Cnia z7!~>EVc3i*qwp+FnHa$12NnuBwYYVPhz6Tu0+nU2Crk#{p9f3j$HB z5B!In=W!Y~597qE+Z@lPOM78~R~|xBz(9zv71_|TxUUrli=6txvBo3vWjz-H9p)7>3x*%6HMU&K5ep#X`V_&?0%b0aE2Dd9(kaVa=*BKOo;4+B0~Z2mkOk@6EE zVnCcsk1;DBP;@D|xFfR-GQZblLiwzBP`-|vSe)xnWbAU=LzA2kiG+yW|q+Q_uPTnz$k7Gkv#+%x{rvDHI~D~QW=uZ~WhCF7T<*$Jq@ za6{%L1)KoO_Dz}7tm4n8-MFomZ@0QQ_soK=@~xu{S$w;Sijc!2)ffJZUNBsO4~=W9 zVHR;63Nm9FlyzgAU-EpUz>{#66B0hFdTJNZm}ER&czO*~w}yt~hh|z?&QfQokHmEd zH-6~cl^E~)@WU5t%L}-QGBdKWbs3~D4QNK#;m%7Ks;7bwwHi_O!X6HA4l4sN(GA1l z)_!*9aC*MY-Z(JCaWaHpq`*dUQe3V*V*h>T*id?honYHTv{o1Eh}1}=gF$aK>nF`h zW8YSno_HvEUCX_2((mgD%vr=boEOsPO?CCfsb5$FbMLJ_MB9Tn{;Fku z-R*2IB}c50!~pXORo6(_hrls%K`kcn3j{j-I2v0?`FRY#(H*j_3V#DK*~)5Dsi1n_ z(O?KzyJIgjGd?;7q0UUYY>s%^ChMM+xhdq$EeJ_GIvDwEb+dLR!I1n0tbpuJCj_yX z!jb6C(RCDmYuD1kd|+AVi^FvGzH`E4hu!83*T0rx-@B2aS+2f z3X3QxS0}0&k8uwPkty}wo_(4C*s8>cnXQFlbAP=yYbdE{d>WonB9*_mMD1u^% zw#Zc6F=2P#iqa&#jZ;VIHBc(-Zc&d?x9;$NR5t+KE-c`T10YjWV~+ zABYdX?F1?(h8_lO0K&%!)~vcsmddQ>D~uy6&Ij(7ZPhLZZKDVxa+<|0skAY&Q`QmT zE@!#8S#8-R?JI*y6N;rLt;{F}c|i@K^kXL5rj>(<^lc>%$1sHGkoYk-L5Q=5B4temZzBY)Y=zV+YduuF(WYtBk@@?Ghk9W zbhp~2MBHlm-JJuJD5ud^YDiCPeQ5Oivh&*jgv-6EvtjM&&1ej#TdE}q`!Zei8HnM> zi{`itl#!_9cIoMTpLxn2CQQP^3}pHi zL+$Er>iX0Y;R(1&GL7TX8W*g*hg?-M&@VX3U@rPaEQF)J8}jTQx?8Q-w1!PQvm&23$ZtEnOU zBo>eObpcQj;}%s*=@zjSS{#}LW1X^o#A7JMNEUxwIZTC0n7M}i8-g&dg>2fcW0g$( zes-HsDCLB;>L%U#kG|{ZF>VPM{1#B7_6KEFmZvErF^L=`W0Vnk)F-T|ykDgbutTxi!yM}(yiHq4 z=++@I5w1nw%w-J;k75U_r7BmJBtr1Lbg~?kj3>gG+8gLPU6B!NxI&spuyWU=LNGd? zMvx2W>Dh<_nD@C!nx?H{7LRv-B(iC{fVq3DT4q&S8Zvl&!9BWhhzxDeqG2uUs*K-r z3w*8YL5O!2e!uXSU{jJ!E=BL$D}40Yf=kXU8?Z(@bpvB9=N8Z2hYQWW0+V}b#!u^M z$EzCFv{)>iP?%=2;s><6j3%>=(M)e~h+=GgQa@DO5koG4jta%Kt)tV)zpIUGEDM))IB@ z*2p109#Zi;92?tjxW7ielSS+3Mtbuu~G1Ja}^loemLNUO@Ue|C;Xs5EpQ$E8)rE!vPMkE*m>x4 z3-V;(Y_Vw5mBf}!kF&~;GuzR)i)7hZ?f4xLG$dEF`8>=Sk<;lSxIv^F#0_P2QvWl- z<`Y{9owhaQXOvSvXzHYPSPNKkH>+zyup4X_i|Y?&J)KrO(3E(x@iI=QxrWWd%ta78 zag(--p@#S_7u6!a;@;MWtEwRi`km|+;XFNl)ZmmM&9MhQy)C_kmh_Z+h+EV%3!P=)p|h`BpfhV|Ie1}u0BD-11|ILjLnt_yl1yI;sW;TA zewl%Rp&oob@A}qGIXq%X?vq<+(A1tRbLNAA7SmkPSa6$XjqKNgJB7?JQa9|$!srLl z%Zp-G%~p5(Hdf=fw|3;&Z{EkyLq;1cP+`! z`fwU_X-m4vO1;k=4a-qY=lqO<^UP%wNy>gPhL{( zynZO5yjxJX2nU|P{C$Bj-G3b%$bG1}ortXCcWu%=NJzj$rm+<0LsYy%pw`Nxub&Y==M} zJDb7+OEtWS+K+k3=eG#($$X7NwNQk9auC%=C9^XYM{3rk(iSS475C*~g%;y3M^Xo7 z1P_L--gap*=Nb8<8FVF-h68!jEFh6>T($rP(1hDysl)IBxh`OenonJgO9)!Cg=0_q zax}k4I6%2Ypp|7rFr_ZF>|z#oKK01Q<$Sr%mNIM{BiPet4unN^Lg85-4`@bT7CxPL z(e<$sq3f%qi_SVN(;F-yclk^x1bB?Mx&s)wj<(0L2JfWk^3Gdblw|>q@dRSU5QsaC?XmD?aXO88dR9VG{T0`Y$Pj z@Z=T5k!BU2&6v!k;;EQhaduveH}QR0%jVRr#e@@WhQNqxzQa&P;Soi;Nn50(>DT*W z_mhg#&kv3o&*hK8-l{VxUKOZ_m(!vQ*vx!V`j^8UkEm4-&2^$A4VtKn_jN67+5aS`) z=~UvVWB8oc7RX^)puig>5~YHg6?gmzSo7CAq=Ao#rscr{!v}?@KU9cl;8t9G7CU^$ zH26wX+$;)5wZ&iU)jUnmvSVlD%=m_rf4V=^O*15r$UuWW8Q-sRe-Zr_h$6d2UtjB` zxH7EKY-2*S&$^?e^%Vosg9-E)5+0kx8x_nswb4r0cVMZ~=6r-EjUZkR?|;bvfO@_e z@0T&aiK8+!wPkh8byt9|)flKrfF4wyreej~C=ov=$AHl~I8c~g3x!oi+mbb8jWb3+ zy|i|`DuqO#l?t;_u!p?pp})uicdIwm;SSwNpA2VCFM~^wDlmjei_8 zojq?hXu~IEobYVA(;u`h_pQ3&Vxl)x{$2o*{p`N{$Y=Eoq@_<3RZ;#{)T@ZZ=XCsq zzZ;i{aj#=F5o7LMdc;p0v)wpyJ!x;3u~+Q zMDuqe)ih=xdt|{GaE9YN>+AwHC7`tOM79;$4{hz@h_XHC5Pg88C1uX z6#jDBYj10%cKu`ky`X{XrEH5O)0WsnG>fiZX)y%_W+5bXs{-{9g z=QBwN);o$e?jYe^w=$0#`k%6nIsUdEVMc)WPPd@>dj-FhVohJF;T*D#J1JXl9l1vC zd*ynhQwH<3``S1vmS$Kh@@6DZJqqLdxLPdz3B4s^|ANO5D1MJSx3l(|8!sKYzO6FOa}BZ4EbdGYHiTiICLY476IV{SENCkRnL^3&R2C_-iI+YoLN6;*3@LuzZP z1}$iKTM;gBhBQnT5cgK<*;mtZ3RQV!XADhX%!O5{m@Ixb-u|6>%iwT@%e30Vy2EXU zieC6JvOpF_d7#N2IS=z{)6?K14WjT*|QGVwTORK}yaHKMf=LVOZ zP%u55W*{fjJ7+`Zqg^(ExZ#q!W*Bd}FcsPK;07R+x*#Bkl;7tiogFxgnZFiIikC`p zmwYi}sF7glw){ax^UM<;#k4&~b0KWV6`#jHZzf}zmTnbrxhc*}7m{j8&2xsa9#^{6 z_c#?f>;EPf=lw1^#yU|du<51_)B(T+ZgTW5azHHGpZFD&EQ5mYW0580hG#nyo3v#8 zb|?(*^e8_){^XLEdp7aTQO#f^Oew-MGhfjWf;Yw=1IT_FO!hM7=ah!0%OX_bRL!`x z&g61o`;tTQL@85@J_fy7QAxx}(4}|xQa2tOIgF^CiAcX3c@vR#(=F1ZO2X!FIRc6^ zut1rv#qLYi*H-t~?Q9IgJ_Y`G&&3HAQZLTr^g$6fgm7GtjcRBSbe_;EvlPGXVI3R6x_c^#vvQ|b3&~F7(5h8DTW2fVw z!6eFie>$px)5dSiJJN|tkHyU4vQ1!9TamjG|1P!?x1|Gn^m##CJfh&LP)UnL0)8N z$w89eNH{BQ)8OJ#g0r}dL$9cF{fpH(DHfqYU`ZJ&HGmYm2<_$ciO}va zTl4EDqb3umV1AP9rYEE236x00}f1d38e<7OSYBJS^HZkWam1Lgh>$sgh* z4)zM78AjT&C-|%&YnkGlX3OYf%U$*0(}Cfkq4+NDfP^*HDB0Nyd3p#$CNqs@+lu_0 zd&LyP9nqR&sX;zS-(+4pttg16G1sT3iP~9{C)JqVxU`(3a%R$H_67#%M5~y3SnG42 zDuc70Zq?ii2g!^Th{1G8u*Ca6{Z68wreQDOifC+);+0Y4!^oEHzcy{UbzSclQGTYZ z6Sd2kRk_jWu{bhPKVdQ-Qc_xvF2j-T{NsX_)c*C50^g`o;Qu%3G(ZITbUqM(YuNrV_#45HZ0ldhx`;6eA6&I91O886*V2L918%FNanF+AopnaP5N}y2N7vs8?2)!r z3s!Z2`}fF-AS74kK|q=Yj2-1{oo4s_uprK+Y03(3=v2pCX(Q65{W;wnF3Lo}bl`lt z<^Jdyi{`~=y(b% zbV;2BBc(tk{>5h_=?`Lm|{#=ECbCQG`(#Bto-DmLVPkeZLFb-_#Cc-xb(3 z_lras2E+HF02mTinNN7muY<#ns0uQCN+z zvK9F@xCKPBOs+b?>VA{R*GQ)!lv4zPJF9ZV@Pp4H2rZy17sgfGlL)WZdehgdJwc%* z8Ginzx$R^b*joTq{adMo8}0R5M;wcYLr2MN*5Z_X9{)n9->xYzoEj%u2}MwhWthd8 z4d#Jfh2m@lpdi07OU-_f>c3a{WRNpu31rC_8R0)oC7jtGR z{{XNB_|B7Gg;mk5*(pNyF=l0|^cD)nH460W;hKKSGh6sHW$SJy1K(m`^lE~d`mr%g zdvBP_Tdp4I#sRhV%C5i5zGTX(0Vu3y+sp&1qkj>Y^uvPHjy8&NzRU%LveQ9vj8BA8 ziI9#n>D#n)VBnFtRYqk}E(;hT5!f<=aBErXTm}E3S7EfvI3B+3WByZJTm7N2W;mFa zabPPKsz~+Ae4WlgGplT%s`|7{4c{OS-vx0vc;52)U-&-btD&j!nD=U2ySIY;5MeL7 zt69N+dUd^k_INP$mbLj9S782ae&|e`EmB(F*EGDzq4s~#SzR#|%9OD_9aJD+LMAJ$ zX89EgdEfAc)pdR1eta>9TF-n_C7fK3f6&TdSV(rt@U0cncR-h~{Sr{=vE%bR4lc&A zHcl7@G$#AVTR+HC0}SspBY^e&%NS7N7bVfB?Nia|zLDdq5NeT{=TXZ~jX9&-5agG1 zKa07*9&A@(*)1PQC-X4%49_4yms2`CuEpq%t7^%7i9*4a*|2wDNWiZ_dN5ht{}>0eB&1j?kvZsaRB2p4sHho`Nvk*eOJt5>%pn?IJ#2Sey_fv zvAfAHF8OAMl$;mL7bk`nnmQgH?e&e3u?3=Qq0gj8f34A{>%`D6)TrX$kCD#alK}x) z@Rr?d0LQ2L@1<>Yao}Ybt05Yty$l^guNcahM(kl2r8D{Gw3Q!~Iy#QA7|Q^ryG}{0 zmggEfV>;_R)Qmo_$e(u9N}AS>cMjU3%r39g=Fv#kq~eUVAZ%$l0KW;?c%)Y;W?Od{ zJt>IN`G>pqSJJPIS5jh9J7w~<^;|g=Ln6}By<(fOS%230mU~IhVWr8?rnNhm7N+6G zva{4mMfwV^z`jJDy$SR6l- ziA>UZXN_R+TO+wZB1C%&h6I_txrw!{Vl8^@vT~EvmxlG!5`9~X{2h!AIt(zg=yjj8 ztqaWJF+=RW!9`?m%gfU-6KH(U!Im<(iq=RKMJ-NXrymJEA|}(U4yk{5X7;Ba`M`7N zcW(}(i*gUDwY1+9bj8CCco|~~*-0&Q@MTWZ&x*5m!-n}2ClsiGdHCFnAdYAo z=g%)$3{FK|MKc6L#Peu)R3a3npRCE`9xH;>PLO*A z7M+3Rl)MK~EVvEPVavWgh?WVhg!@yGSiYt|=^w0#b{)Y5ULB7Q`QG%Rw~SM6eti3Q zCQbza6RK2<0Gf}S2LtQvuIa-Fl{cSqP)Ya3uu=5Y3YT^U9w$0{DSs@+)FDZBO0o=e zhg&IYo$rk8m!@p7m;WQgtF4xhW-Z79p~EH=CU!UI>dDkL>7>Wg!tyM2I)xe7DdlfU z4;}xy()f8r+1>=Mpxqu2#9nMbu$&EzzEoshy-`{=64u!ER36y_HR+ zv*f+ZBUC<_#r#7W=J@35VH-#MrNkB@K;bl2Jb%C_2C7hAX5jV%B|BE*>jeUwKORb? zboZnTbc?>~>W^lQd0Qb1C}r44n_mmD=#RMPHg#}r+ju2SCiPZi;Y_YPh`a?G(rujJ z>V&^<mx2ah&|)x+8&>+2BM%veLND2$LWDuxxaxcD0L1GgIg z())&24$A;bb$o*3b5LX;auwE2XLh8uv5HNbSLkOf9D2n5Dum=334l3UPH@5lCVH>1 zn@(b_=b2q++RMQGWEt~w$=j@nC7!Cf34S=yS6z8j3w1TVGNDTc#e#7(AUd+O1|1a+ z$RvKJ$lm_|a;V(z8;)}eDu68F1pk9e0 z5Qtv^RC&s!K21Eip78uTFs50ex)Lovsz`L*>s_rJvc!0DKc1Xp02xglrJ%@p6zL-S zS8m2m`C`cNH8?6u0LOkF5}-6>C;UFeBH}3H z!;>?>O$ZLz7zbSG46fEP5`^uw;ypXk$^cKI|NA22qrilE$XPPlzWN|;Q}@${z4)uT z`+>&$zEb@;S#xX`S&Q9f^(z{mDMS_D)~OSJ&xTv}L|VW)?bmg6Xy?re=8VKQPDAKPvLy37U1)f1CLKnX&u5Tz|r%z%N%UyGCfx*jb|Q>idVTD`?tBF~xSmyZ>5` z{Mgvv&bQ|53t2+cJ~gQ+&~_E?isMV*FK4;)KKm{#2I9mLFg0@c?DE@5RD#_yuwAf22kgZEmEuu5O}(hgy@F}zmu^o}^ElgnVsuxf?c_2TiSI z&LCNtTe)B^2`1!o1-h~Tl}V~)j23ni2*jx{q&i4a2i@T0xEp)^EF5-GJ7VAB2-7U^ z&5f|ytYXdKV0~OAsL##X1B{(pqhfWSpfWZd^bq8PSH7L^0^wOkkH9<*KPqzV+g>pH ziFa?gALfrcW?rvpuI?wVd5eOAyaUkn1@so?FVSZ80wz5$S!M+La0htp912Gv1$USs z5L^d$78>Ho@mAT4iPAsS2Pb_!Z(!yIx=Nb2zAFci@uQfLT7)!#g(~ni#3hFyR3l+J zrG}HWPIYa$&#}BD?cRjM3WQ}RyA;I%5A05l+U9Z^Ywi+(1;)+DXK|Tt&l7z8wLeVV zoa_=q5soj>BdV&{K3eHQB|jn4ehJ!9zys2)rG-KQ5D{#k-)pJc)B_BEwTE;`JW26r zRJ)z7;bpihRXLQ4_%p<_ZWQ6JP;va|TD%?pVzw}`_E=^nr#a>PQ}3&2pSHLn43d9! zVG^i`1h)e=+_xP>593-lD{|KSFcz{73Wl6iWD;+c4Im1BI2meD2T4dT9j-wv&bdM@ z^3(j=i$kkcc^_2GCiK1&8GCG%_wQfXkCTE(4QMaPv+y0PN&vZ)(E@{-JKH-Ffo?xS zq1M|7uW`y(i(p9bjeL-&*i$f-g%wr)?Q?EBy7%`ASBjcmR&bI5b3rymzUHgkmXJo4 zB_(A|xUpiV0XYuv{NUu_lAxZ1BL3L3jO^VDgF;DSvJT<5=x`#xWNE{B`Z-;9{W@Z& z`)Rq)y?nzze&uKXDNAw59T{XGGGEgpmiIj5{#Kx7; z<|^?7w^qo6`=W!Sa#OFGegrM%EfKXucIrp@C_`&@d9y4Cbw&z)SRG|R@Rhn+ zEkQA}#vDRS;pU?kF4J>|mY*qnRE_9K%%^fp@_GSj|2S9Gi+Ix3Ji5M2Ia;y%q3(;( zTmJ-V?mghICe~d+9nUGK4d~=Qk~jb9S(5KXG~onk`~!Mgt-p%K{p>j2uyZ~bEU*2Y z*N$P{9w=p$DQPvxL$_U(opAY$t{RjYSqb+5NH9W)=z?Ook2t_*3Vht==Lvj*8gN^N zmWri+i~(O=4y09T%x}$xS4bH6C%G6E#84{@_y{)-=~E^njiN!szQM>s_OW&=!4>V* zf_7WgHfL9ANI@U0C#nW?r5*3!IYNQSsTR(o*4TWP%QBm>nbfXLM7vBz%uX4T>HQB~ zluF#coGwCK)y6qnsNbes4h1~_L5T#+6a$G-8d@xrl<1^0ifB|nbDN`DyZuolFF;RF zc%OvYzwR03J^e#nxFJh#;fc#;XU8g;ak1iU?@3PniOtrP9`S>D%!p~X2>Dw81-{S6 zIG-)U48?8k2gB1%c=~eQH8$a|)>90JoL3V2B`e6@7Cx=L;M(kc}1fD@sW- zF#51^oc8w7xZnmq*tp*WN3vl5WTE&EsqA5g&nhEm3$;v~-OZw#yu9pqE@JxgHcv{- za>UW(HwkB%{BRq08^_4pwD2HL`PqA^H9V?+4ed5?mEx7>D*|Z7Rj_oh>utn&Ozb+* zn#H)rUX?}G$Yw{#ZD5Zqs}FsRtbpK1x%cYD_y$sMb!^cCoId>xDW&pkd5V`kJc}2T zR~Ttz4J}-!F7@ZYyOeTFhRf-}6U1uO+DVmfJS!0L1xXMin6?Ayc(>QvR|ATg=hyH0 z_$-!&BrH(3Eq^noh|{`v?E|3?*Jf4v;S?nddST9Mc?lsQolaza(;m|?%dfDKl`oW` z-^-DXFGKlIplhfGlDX>i&5fJ{(xzo=uGE*I59J%E!s&AAj&Xi_NF(?ysq=y$5s&# zR_|a)k|dqNP7DY^?+UIa=l+aIAbFP+bi1od{8i%0qp6DR=#*HliIf-%lxDVzLB@)@cNrir!UFT_ zD}TW>7-2~XEfGxSR7nY~$JKbz(evL?2dw0GlW#_oHC_td7P&+^ap?~?(d}u7YI7Zb zrD8(a{|s?hoi$ZC)D!e2LegeCDnxS|!7Cs70Z)rro)pdAYHm0&QhTlNXKH^+(h^hc ztq4_emN>Mbox!Ght%;S|wwxx$fZg?YtCSG!er$2T=A{j>AVIs(r?dM4odMI=Ag{~* z>rPCsKp4chR#EJS3Uz-Q`pEVNO6|Cw#OMjH>Th*nY-WQ!p=_E;Ttp72()!6Uoz`G} zC`8ART9m7F^ox9ywYg#e`3P@|2hD@HA&%8%EdNCqgciw-ZkBr_Nu#_O+t_}sFfrdX zsrAMaN%#FL8G`OT805^Fi_2Xg{d}sa)<1SGYHoS*71z7gFQ1uB-fFx~x^o6>OtKoL zSvY92i`L5UPuO?~Gf86;jN&$G+N!R}SQ(JUZZFZ;G0S92Gd`Q!vR~~DAz1$n`*edI zB_o5ong4}PF9-XeQfv^%VKEo$O5#g}y?1*#T&%Q&y=G&ha?V!o{(AJR{d-gnD-%`e z&!pF~+;w2x)F+n3MYOuif~I@JKXi_wG;6;v_8$rEGx(JG51o4~+Dyl?TdmNK4D4i1 zfl2kVT3yvn?bNgvyE?5|Kh0H%%NRj}b=FVtCblg8p6F;gco{IwN~FwbwHsp9a&toJ ze8MkQ_}p+hInAvlCwk0Bg)Z!-nvBb>Z#y%Tlq3ag+-$IbC0IzT;qmvvnI@y1B*xqO zbJRJ;bTcWk^!$v%&rlo{V;)~`;m-Ef7o^Ag!E(ZE>Q8BPtumz%y)!VjEr-f=Q`@@V z>$FSLDr*Q{Xbj)nup%CI2G+3LoL! z>bmQoIJΜJCXE!JVLkli#jdf!jz?9(ged-j2tIY8juv9QdWU?MDXJ)l zKK*Rw?G|&frCm?#VWsM4GgR!%n^!9x{y4K5oNp)OifFFec9w*jD`-zU_>M)tct^15 zw=p-U+Rl*I#g2I`~jn+yF7$+E|xCbX3SUa1*r}XILIYY{E3w^qH|G&nVlE zw_PbVWBFNc$cJQ=(&3iK3$~p06Heab@8tpxoGBKBw^y}MRPTBKV=&pymi# z19@j?nL~XzkULVeID3|B8W}}RrltL-7kzH&{D+~sg1GA$WTK&K$j;vfzQXGXlJ*eP z3Zo1eW?eO;-`{ivQ6~OSKWC15CZtSchuhPw@NAwP;{a zfo5i;jQ27%U;Mx|_Oe!dI>a9%nD0HT6o3iv%EP{?ddBSMH}o%f-yB?Ml$+XBJVk%x zo7YfwL|ma7OpM7e*ZeBfhd%LXpZJYPvoDqEYQ)wJz?OE9 zN&>C=g`1O}9NOs@MnR6ZHQ0_s;2$^lSHW?_7nQGnadYO8$#sQx>^j0$s?9C7-gbfl zlP_8e<2394Pw0fajlX=tu0~;07ZKmoi1S0Pzzb+_YPtz(#AZx3J{YqmP14nZo1LJA z2+}HkX-0u8K8f1(K52Y6#Fa)WGOOM89h)fcU@91hE)lj=`;-yn!!oTtP$+s=7wUO# zwr3MdlA}Y5S$Q$JUiM{I^JHIaT}+7pnf2A3K{-H$BxCdMyEAup8aN+K5frd+=9Bas zAgf~02Q>?rKvT~(7LbL8NbCY^TgV^jlF#GT79BQi#ktMra#G)Funs4RQ&RoJU~0V= zO!Cy&9k3VkCfHJIyUVhEot0y-&ofZ$Nl7(FUxWT(OC3UpT~9jf@YZzB9$;P8Qr+EegT@9NFaTQAF@6n zB6SdNMHa_%~>}suSW#%22k9zbcL!**We z%1FjXYa*O2?Bt%_Cb2JF9q33$KeO_;RGua;7SP^^8;d=?w#jk$K%rptTS|9Fu^^g= zUR5jLBlBNU(n?$K)ii70Kn1iEw8u<79shQzxv7`w;Hx~I?Ti?k%f1R|z4K%df<|G1 z%)fZ538ID^qJu5YHZFUQcnx5$eY1g+co^L3oDE)b3SsLd1z?>Q26AaW1X>4H|F{tq zi`nsd_Cr4exk>@u7x)}+5vGxg!rsu#KlYM+fpLz1uVgw8N^~~A#5&x;2I)hiF}KB^ zd~2^IsJw4JR4E=rw0&*hwCU0?i<;Iu{D^hXy>yGonwN#W9~m}>xnnZAORD*U^5u(@ z%5ni`adbAXN7dWop=XI!y*x{S4}yEoDa#Bp>={jzxIQt1t{VMWq0aCdaTv&dP?$QL z!16VkSSv8q1M1HZp%?yIl%;bS?o=-0BVJJHAw7Ob%(tGI?zMdn=E>%cqeD=)H<9CEd?LreCkW1hGcvWeD0zFWQC9w*xjNmhQ zTADv7r;q+zM&;Lc+yOutCH28AF2~hM3ZoxpZ2ou}gXqU+Q|VncSQeIERY#D2PZCOhN3`^h$XIp%t zBM{VoDqkMc(XXYsbh)Ev2k)+pl*Aoz)sga-mShf=Z!bPdn`oZsIYNGv5sed8t&Oz3 zCvu?-o%`xHlvLGBDhXL^JVvFrjoNN=Ly1ySr7v)6*gWA$kCke|I;lR$OXrpRhc!rewVG1VCe*SW7u9%@12iTI2f!JOzpuN7R-S~#kpl}6MOu#w` ziiSn-kT&%EDgJFW-^Bj=$FXX*CdsnHjgElBbilln!(;X$uDZI@abotjZRlZ z^ef(qAZ)MqHJ3F!SFvt$-wgvNHRYhSz{zkEFtTj&n+i><37n|P~T2Xs7@TB+h&93t$!nvwUg>Y^h-FRCnDs_9;5A+pbxJlfMM}bH`x24 zQ~#dD{9)vQo%dTGRrRDU#0Bi)raIlkossp4P$!gyqe!R5&m%NaRkx{-%xvM za}bi!FS5te!r<%?$VyH}&Yml27_iFIbz?q*?waY-3Qg?g5K%KL$ARpXCw zpXiI{mr2;%@f@la)yp{iIixxJB8{3v_5Gs=FTP==Ph#oSKUEf`kJk9vaZ)u+u^e50 z>oiq!iLo#f_J!Z}kii2dL1E+{FFfeSu6v{WB_Gz0z+Z9pnwM{Me{r_U0K&cYjs$ud zBMbo+$uV*V>*A(i1qm2P+cBHcqmx|Hu9x%aSKI*%w5s!we^S9~j9hAoC6|x`>ARc1 z0}UA4GcHtU$J*$}^T$Kcnz)|1R%V{~cCSKGmfv- zJ#VPXX)0^{@5TFuR{t2PFnj;ztN$j^KN2KnjLsHRW>@^DY{36(k0~`N|Bt%Q4Qg(; zFcDqxM&n}x+a$L4uNVX zEAn(>E8a#U*l}uNU4PuopJ27)EvRU<^?ol}xrUtnh-EYOZsj+TVedKRw#jjblZ3Kwl7M_`kJ3w;&M!zq9f2S(Ao)&Gi~)vQR>iv+%x)8|2F3ycJbS z0;V9}dA9fjPGW@4dgkT{Bx=X8aOhy(O-UYB!<0Wh?jQcNl)A=qk^r)wQcR+wNF>uY zX`KYC@GC`p9nruL?z+aSf9gpoRco29%@$Cg`QUVMW&Dk3><6>oU)ag4eaEX7fndGx zSGJts4__3=x`DC(x~lNfv&wl!hhdb^-!S37xt;maq}&z+jMPF;XJx3}xskDx{E224 z3bbLdy~0pPEu^otv*(Bz`>|)c%>Qyzei1%64!M8#U6tiMi%|-}ceG>$D^z+#d!FFyxI1 z7krGrQyjU8u=v*C$l9hNV~ULYS{#eb`SsTKc}C1>x%Ni+{YNZ7NC5b>x6Fua)5Ogs z#ZV}p{fC?s65OjhV$3YQmAfG71LR0MH8=lok)*io>jpN<;B#QtBwx+*vZ^E5tJeV_WZjH-%- z2P^q2FhCq;>kM^XH~h_K8JAB~$OKt@ki=Dn`U#*pUO;J0{KMM$twEstSHLml|973c z2)VI*RgS(a%1i2Px;q#=)ZFY8HUCN1$xfyHPK^?zRQl z=iXjEZ-B zl`&JQwo~Z-EPusMCHA#WnjB2g3#@t*Rx{D`ej~Dd3V<73&nc^R1jl*HeE#|I5iX2m zwv%zdm{C2e#YI$Z;G-E&YB+)w(qLQaK!qH4+X=sPkm$QTexYvJ*N6MMY3P=ux(F4p zPZ92lyu?92IAN}nwDEhw1q|(BeZa`slByVoOVo=qjlR`#(dHEW&pWHq%uSJ2wHNUZ z%}6pPt1IIIE!VH7Az-{pZqIss0(I@DV#pU2#v4j6D>dnk<@62kFSY5Qr! zY1|K3PtbNTmB1;gH+;o|bIgYg2}_Z}FELiX*25VLt?G713>6=r5ekVNC(S!K6q^$j z^Y{>XA2fJOG=7&_MRBOBmyv7$##mex`ZDSa!FHw(8|^_oSyGPBC()Z85@JGw9=A2N z>P)R69RZTP#xH0jwgINV&VgBCYc1*dXrc&dg^Le@v?5l<2;V0Eb`&SKwq<}m;#0ntsLZlapg9`U0GSQZipy7ZrS2}UO zn{TyXOVd*R*j#wMDNB>1d+|DUIV>(h=-G28BkkV!rR@L9V!>Q-C1!e{b9BB?d|asi zt(@u})bXVQH9d2wh~;Z^YW7g~uhS?Veq{)W`S>|r%Fuc^CC<2Y*)%~1XBm1C^Q};5 z{S6@K*$8PCI>Tt$JsnMKd1Nnp8mt&H+p z$9Z(q`U^x^f}xtjU&+`Iv&W2d_%%NO0aUokTWK)qM&OJ?GH4`V0p`U4kx*-AJK9@fyzPas0iAxynD5^O@d(oz zZNG>}g-^7*VWEg4{b`#Xp)0%nT02Bf0Ary0QC>ajt^h z|8ZAD(e08RykHS^pWYehMF>JdqvrD1uGtqY-zq(jM+2Y4%?Pgi>Hrgp;))o;F+8!Z zM~?%+0Sn!#jtV$^jH@@XW8j!})(1+9t(0i@v)#xes>#R#zH_fg; z9c8fBg63^Q;u~f%g77=1KjbC4Zf-yMvhO6Da3~w~ir@sU4nZTi7XxDNCf}BkqEMT@ zr}12wFw !=}Q2+Ox10g2m|hZWH!ryxLchtPW}IF1FOmr(Yx% za*t$U=r+TTmR!vk(5=(wRj|+^aAlL=`K5q}ZrXVjTd4L~qOayh3u7ta8Ls`CIqH?< zUWD=nKJWsJ_-%8b)Lj@GcaZdWRE$Gz&CG+naoN6;F82dbYJ^K6yo=9=$9s$$qN-?C zoatbZ?m18)5t&SI4k{yq`)O1TXo-(t^uzS) z>xv-p_lg?sgjZc!CctMN>FTM2cZ4& z3$b?KY-tM}?jK)$OKMAnU#Y+H`U995oD^^WLCJ1F9yRWBwP{XZ;E*X9U``jykg^Dhg4&U-} zJI*_jaFex2fEKV*vyk^7=Fe{PJv|FY(1XD>H{rG5ROmy4p(QP_lVr=3VC;OPmvBT_ z57=3qO%IZLPJ0Nj$k7;cAkY{)XFj%j>FByZS{0K+eC22bBQSWF2(-2z)_etaGLqhC zWiL0rzED=Sp_oboo)y7H*PbQO4`1gO8v{B@1I^W+eD6iAK7VnSrs+$_l5cBt<>z{M zx&K=*3n{KyY0ReKU#$uJKFWHiwxhcSb8E0a_ik(XA^F7LQ8>_!^M=2}DOE?$QzTYN z>RE>8Y$vwll;|v?QaX+Vp!M=JPS%#u!yf0ZQ4@f;YjiufYE;r=doKNKf1ZASxtd4o zoml<()8#FTS#IqwN{Y;2`T{8MS7gY7Q|83WjQLg)v`~P`R?#T}J@#n3Zi{IR>LAem zM4s|GNqivTK0Ny#pCIDb-5T~3E)Q-{?shH`H9aG`b^Cbzi1((NA9Wf5kBRt%-D*{N zUrnUfakIZf^?;v&o?rtrEK2a#kVTAbja?AIYJHFJKyVsHMPbD9cme_~gtQnEI61VW)J02PyUesPmZPSvc);_wu1!>PF!t2pXV(rO<`u$$`ew~g7+_r4Ia5?mP;IM0J z;}@HYnF=m*-@#PB6&0og*K|1L-cHvAsm^2fS;MtxffUmt?yNbzUC}T;));W48_v>_m8{?y743bdh-4A(nTYQ zm#O2uh5K17HSV0e{L#PI7*6qCe_t($!6y#>FHNf;Ga1_!;aE2voUEYev!hbCpa)IF zt$mdC^y`)-M?sfxCZn&EW%seSoSh#zb58CByPuqv|Yg7x=8gX1U> zC_+wWvh?y;E4T6jb>~-4+BGwB_Pl%)-SN>AlyEhZ=RMcmmO!T-*_-VK`=8nQFB3K5 z3%7#iiF#|5fv)TZGwGfya`-8M-@1&Py5z#_B2nNghA@fUNyIqV=n>qqezDZYeIiMz z+pwhEksj&=jN~T_j*Q$KD*zmwR|9SDhmP_L##%C=QYDHqPoJ||E8=nrN?b@cO2Dk)3Jx-K&Ph}B{;TRB$U6K-lHN=>y2ouqo3}A;{ zoLY-#^8dR*cAq8>wmk6PUyiMCGtEr~YSy3s-S_Qim!!%6{PoX~K>VVlBbvrp_^?f6uu zzmQn$7Fbc2?X)PCg7vf@Be}~r)U;)fO+tDE-zTX3mzX=|a!m9MjDwru?)m#2jE_(t z3{Bi?Hp>w_Ujt=>Kz_$kk{Fl0)HH>bAH*bAjEkG0UGKxebaIvt*IA&82&2Km&c?F3 zBg56L>NK=IDK^ddIzW?QDN_;V?Ad5-=;XmOfwRJYI!Og~4ebT1h8EzxA&WsbbVO7H ziRNKqLh(y6-&n;>VJ?NCI1U)aUl(Qvb!x^I?+{8cVSJQ9{vCe1UIa{4Rj?IZ3{y9p zUy&fto9A$-)8Gik4NR{6J8zTAke6a71CB&E{8SUB~F zL41L$;pCwha1~HF#T2TNZ?TcALn=r@N>&wr2DpHy0}9FdDB#Dg!+T{i2KV|$`YeZ; za(!+5Rn&ZlAkt_6h#keaZ@jH-%KZ8+JvZ!k4+Qu}mO&!kN>8!yCI_T%017rR2^Id7 z$V#T*gI{q3;1boR4DNr9$_5pa%=c4=gD2Ff*vR*GSksC%jc*F$A_+L5e#q2`oLdx8 z@m;l>A8(lVIsJDi_d4SSY4gP0wFvy{e)jn4T^anRmy}=!;U5#QEQS9>BIg71)<#=&(C!#-IY#x94`7`+229Dz5wW)Q_{# zm=i-Dm6$_vvc~6Cg1}6&;O`V)pAhI+Cp1hhA%ZzPKCQAzL&BS43EPG^i(f)|Re1*F zf12}S^jsj0t+}&*Y>Ms~G)dyqpK6)|I8-0$z++L3j-+>BacnnLPjeLMhcV^K8(m}21?lf6Pi2t1lHnkAR1k%NajIIE* za0{E`Tn%2-g;Ol~xR(P%Kabbnch#K`@}m2DauvF7=>BR0 zL87-hP3*)@<=l}UFkPGxpLFNJM#pNu9<)DSNke9S^DK^%sp zW&t_Fk=YNIkEAI+YX3}pU6^Q#ZtSX~E;z$zlCm!!vDU(7UDfQbNLpU+TDBZr%XnR81qa`?C84E`LF?sF;Q)gAN zUIMk{0GS;m^N@bz5fLLikII1aFA6vi&VANuw&x{lZVjE#b35a7nRS2VWTJW03swE> z)Kq_bkKDw+HPeCJ@@1Vc_9qJlr1E)q$%UYdxe3M}7h`E-J$PnSs6UF-J9=J&w1x(O zgCn-P7JsF-K%R7b;W2)<*xd$ zJ`xw{)zx8&8@bGDT`udX^iwVK!h2rL%%Y}iVwq^UO5DD*#b}wTrTmOfGq}-X=XR3D zB*sAaJz3V_^m?FC*>sW$jia4Z!IQXm+^fD#t(~BFV5Fk7QaRbsJ6k zqab>6Nc9AtiJ{bAhXI^pd7a(6xG#Yli90i&M<+*WZzhEsvlzX14wM`E?|yd+GqJ5E z$TcefzW`MY#}+b@1^+XG4A2g;V;u6`ktsWF9))z~pSD(7KKaHmV)7 zcI&YafsKBW$od)_UHe)NQpwYO)s7JAErrnwdSWGSBhettk|QW;3Cn5Pif%C6P6dM? zV4)OAJ@+s!24knN3hBYX#NS}ow&sGc6-UaZ(`(&X!??IiNkMF^^UC4bRHwFCc^DsX zc)5drk(ra0PjYR;32G=9Pssa#D9qj#vyoQcf4ColPWT#ReR$1gke-h1Fqkziny-#7 zc8zRn9Cz5j#pY20_tvDYuPkC5QyK3XEw-ntR~c{Qb;CVs02#VBoPyDuuDubS-7?&= zr_-dOa+$DszeEfEOwOB9vktD;SMtygSgA?-6o$2%I;oo6e=kHDuq|9L@;IExs;#F# zai!Iolz?@)V9f`OTdIlCB9VU#W!JEaY0Y zlLN-k9#0kk+$Z)W^u_Uibkb)|Jm#oAyzu&7JPJu#R{hD#T=?7ucinik(3Vr&+hphx z|6nloc%N3g1w+dU?1EU&;wno*K46lOhAe@`QKtucnh%nfTc$z9IVOgPd`s3MoV9|U zuoA`Igj+pNNC^@)N3PF2?y+3lKQh#tEd?{2%5vWu{ioY<-oII2P+dvyb!LkVUUP+y zE?vN18~j0DE(!I58fIK{j-rc+AlBF^&!Cv>=;x7`j791b$7(s}X9BuD;b2_&{@79m zR?@ssa(VuYdEi&XiS8tW@|;srWthk)jb8A9!t&CN!00JR|5WEG4ZFFgt!1RlMQ&pV z$gz+JU42hbm0P&lc~4{ry|BoP#&h@- zaG<)SyFaOYv-fURQ|9Y@cC12n6mX;RzU>l2q>}nf+oeNmyDL(MF7}ya-fF6S*>_q9 zEzTQ9!W&ljLEA`mYbp&Y)XZ2rba`hq5MIq2DIeS458wCRcDqw9toWZ{Eat&tywU7> zr&;3n-vIZ%RZEOt@8tY3c?6sB-*B-QZX@$8==HT#8%39U8y;_GONI!g+{fzg2ctMz SynTlVMovmuvgVU<@c#lG=@t?I diff --git a/deploy/cpp_infer/include/args.h b/deploy/cpp_infer/include/args.h index 4f87c0fa5f..fe5d45f6e0 100644 --- a/deploy/cpp_infer/include/args.h +++ b/deploy/cpp_infer/include/args.h @@ -27,7 +27,7 @@ DECLARE_string(precision); DECLARE_bool(benchmark); DECLARE_string(output); DECLARE_string(image_dir); -DECLARE_bool(visualize); +DECLARE_string(type); // detection related DECLARE_string(det_model_dir); DECLARE_int32(max_side_len); @@ -36,11 +36,17 @@ DECLARE_double(det_db_box_thresh); DECLARE_double(det_db_unclip_ratio); DECLARE_bool(use_dilation); DECLARE_string(det_db_score_mode); +DECLARE_bool(visualize); // classification related DECLARE_bool(use_angle_cls); DECLARE_string(cls_model_dir); DECLARE_double(cls_thresh); +DECLARE_int32(cls_batch_num); // recognition related DECLARE_string(rec_model_dir); DECLARE_int32(rec_batch_num); -DECLARE_string(rec_char_dict_path); \ No newline at end of file +DECLARE_string(rec_char_dict_path); +// forward related +DECLARE_bool(det); +DECLARE_bool(rec); +DECLARE_bool(cls); diff --git a/deploy/cpp_infer/include/ocr_cls.h b/deploy/cpp_infer/include/ocr_cls.h index 742e1f8bb0..53bb778c18 100644 --- a/deploy/cpp_infer/include/ocr_cls.h +++ b/deploy/cpp_infer/include/ocr_cls.h @@ -42,7 +42,8 @@ public: const int &gpu_id, const int &gpu_mem, const int &cpu_math_library_num_threads, const bool &use_mkldnn, const double &cls_thresh, - const bool &use_tensorrt, const std::string &precision) { + const bool &use_tensorrt, const std::string &precision, + const int &cls_batch_num) { this->use_gpu_ = use_gpu; this->gpu_id_ = gpu_id; this->gpu_mem_ = gpu_mem; @@ -52,14 +53,17 @@ public: this->cls_thresh = cls_thresh; this->use_tensorrt_ = use_tensorrt; this->precision_ = precision; + this->cls_batch_num_ = cls_batch_num; LoadModel(model_dir); } + double cls_thresh = 0.5; // Load Paddle inference model void LoadModel(const std::string &model_dir); - cv::Mat Run(cv::Mat &img); + void Run(std::vector img_list, std::vector &cls_labels, + std::vector &cls_scores, std::vector ×); private: std::shared_ptr predictor_; @@ -69,17 +73,17 @@ private: int gpu_mem_ = 4000; int cpu_math_library_num_threads_ = 4; bool use_mkldnn_ = false; - double cls_thresh = 0.5; std::vector mean_ = {0.5f, 0.5f, 0.5f}; std::vector scale_ = {1 / 0.5f, 1 / 0.5f, 1 / 0.5f}; bool is_scale_ = true; bool use_tensorrt_ = false; std::string precision_ = "fp32"; + int cls_batch_num_ = 1; // pre-process ClsResizeImg resize_op_; Normalize normalize_op_; - Permute permute_op_; + PermuteBatch permute_op_; }; // class Classifier diff --git a/deploy/cpp_infer/include/ocr_det.h b/deploy/cpp_infer/include/ocr_det.h index 2439249534..7efd4d8f0f 100644 --- a/deploy/cpp_infer/include/ocr_det.h +++ b/deploy/cpp_infer/include/ocr_det.h @@ -73,7 +73,7 @@ public: // Run predictor void Run(cv::Mat &img, std::vector>> &boxes, - std::vector *times); + std::vector ×); private: std::shared_ptr predictor_; diff --git a/deploy/cpp_infer/include/ocr_rec.h b/deploy/cpp_infer/include/ocr_rec.h index 4052553d96..a55d482bd9 100644 --- a/deploy/cpp_infer/include/ocr_rec.h +++ b/deploy/cpp_infer/include/ocr_rec.h @@ -68,7 +68,7 @@ public: void LoadModel(const std::string &model_dir); void Run(std::vector img_list, std::vector &rec_texts, - std::vector &rec_text_scores, std::vector *times); + std::vector &rec_text_scores, std::vector ×); private: std::shared_ptr predictor_; diff --git a/deploy/cpp_infer/include/paddleocr.h b/deploy/cpp_infer/include/paddleocr.h new file mode 100644 index 0000000000..499fbee317 --- /dev/null +++ b/deploy/cpp_infer/include/paddleocr.h @@ -0,0 +1,67 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "opencv2/core.hpp" +#include "opencv2/imgcodecs.hpp" +#include "opencv2/imgproc.hpp" +#include "paddle_api.h" +#include "paddle_inference_api.h" +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include +#include + +using namespace paddle_infer; + +namespace PaddleOCR { + +class PaddleOCR { +public: + explicit PaddleOCR(); + ~PaddleOCR(); + std::vector> + ocr(std::vector cv_all_img_names, bool det = true, + bool rec = true, bool cls = true); + +private: + DBDetector *detector_ = nullptr; + Classifier *classifier_ = nullptr; + CRNNRecognizer *recognizer_ = nullptr; + + void det(cv::Mat img, std::vector &ocr_results, + std::vector ×); + void rec(std::vector img_list, + std::vector &ocr_results, + std::vector ×); + void cls(std::vector img_list, + std::vector &ocr_results, + std::vector ×); + void log(std::vector &det_times, std::vector &rec_times, + std::vector &cls_times, int img_num); +}; + +} // namespace PaddleOCR diff --git a/deploy/cpp_infer/include/utility.h b/deploy/cpp_infer/include/utility.h index f0dddacdac..33e995fd02 100644 --- a/deploy/cpp_infer/include/utility.h +++ b/deploy/cpp_infer/include/utility.h @@ -32,14 +32,21 @@ namespace PaddleOCR { +struct OCRPredictResult { + std::vector> box; + std::string text; + float score = -1.0; + float cls_score; + int cls_label = -1; +}; + class Utility { public: static std::vector ReadDict(const std::string &path); - static void - VisualizeBboxes(const cv::Mat &srcimg, - const std::vector>> &boxes, - const std::string &save_path); + static void VisualizeBboxes(const cv::Mat &srcimg, + const std::vector &ocr_result, + const std::string &save_path); template inline static size_t argmax(ForwardIterator first, ForwardIterator last) { @@ -55,6 +62,10 @@ public: static std::vector argsort(const std::vector &array); static std::string basename(const std::string &filename); + + static bool PathExists(const std::string &path); + + static void print_result(const std::vector &ocr_result); }; } // namespace PaddleOCR \ No newline at end of file diff --git a/deploy/cpp_infer/readme.md b/deploy/cpp_infer/readme.md index d269cc9e8f..95b28397d0 100644 --- a/deploy/cpp_infer/readme.md +++ b/deploy/cpp_infer/readme.md @@ -9,9 +9,12 @@ - [2.1 将模型导出为inference model](#21-将模型导出为inference-model) - [2.2 编译PaddleOCR C++预测demo](#22-编译paddleocr-c预测demo) - [2.3 运行demo](#23-运行demo) - - [1. 只调用检测：](#1-只调用检测) - - [2. 只调用识别：](#2-只调用识别) - - [3. 调用串联：](#3-调用串联) + - [1. 检测+分类+识别：](#1-检测分类识别) + - [2. 检测+识别：](#2-检测识别) + - [3. 检测：](#3-检测) + - [4. 分类+识别：](#4-分类识别) + - [5. 识别：](#5-识别) + - [6. 分类：](#6-分类) - [3. FAQ](#3-faq) # 服务器端C++预测 @@ -181,6 +184,9 @@ inference/ |-- rec_rcnn | |--inference.pdiparams | |--inference.pdmodel +|-- cls +| |--inference.pdiparams +| |--inference.pdmodel ``` @@ -213,36 +219,71 @@ CUDNN_LIB_DIR=/your_cudnn_lib_dir 运行方式： ```shell -./build/ppocr [--param1] [--param2] [...] +./build/ppocr [--param1] [--param2] [...] ``` -其中，`mode`为必选参数，表示选择的功能，取值范围['det', 'rec', 'system']，分别表示调用检测、识别、检测识别串联（包括方向分类器）。具体命令如下： +具体命令如下： -##### 1. 只调用检测： +##### 1. 检测+分类+识别： ```shell -./build/ppocr det \ - --det_model_dir=inference/ch_ppocr_mobile_v2.0_det_infer \ - --image_dir=../../doc/imgs/12.jpg -``` -##### 2. 只调用识别： -```shell -./build/ppocr rec \ - --rec_model_dir=inference/ch_ppocr_mobile_v2.0_rec_infer \ - --image_dir=../../doc/imgs_words/ch/ -``` -##### 3. 调用串联： -```shell -# 不使用方向分类器 -./build/ppocr system \ - --det_model_dir=inference/ch_ppocr_mobile_v2.0_det_infer \ - --rec_model_dir=inference/ch_ppocr_mobile_v2.0_rec_infer \ - --image_dir=../../doc/imgs/12.jpg -# 使用方向分类器 -./build/ppocr system \ - --det_model_dir=inference/ch_ppocr_mobile_v2.0_det_infer \ +./build/ppocr --det_model_dir=inference/det_db \ + --rec_model_dir=inference/rec_rcnn \ + --cls_model_dir=inference/cls \ + --image_dir=../../doc/imgs/12.jpg \ --use_angle_cls=true \ - --cls_model_dir=inference/ch_ppocr_mobile_v2.0_cls_infer \ - --rec_model_dir=inference/ch_ppocr_mobile_v2.0_rec_infer \ - --image_dir=../../doc/imgs/12.jpg + --det=true \ + --rec=true \ + --cls=true \ +``` + +##### 2. 检测+识别： +```shell +./build/ppocr --det_model_dir=inference/det_db \ + --rec_model_dir=inference/rec_rcnn \ + --image_dir=../../doc/imgs/12.jpg \ + --use_angle_cls=false \ + --det=true \ + --rec=true \ + --cls=false \ +``` + +##### 3. 检测： +```shell +./build/ppocr --det_model_dir=inference/det_db \ + --image_dir=../../doc/imgs/12.jpg \ + --det=true \ + --rec=false +``` + +##### 4. 分类+识别： +```shell +./build/ppocr --rec_model_dir=inference/rec_rcnn \ + --cls_model_dir=inference/cls \ + --image_dir=../../doc/imgs_words/ch/word_1.jpg \ + --use_angle_cls=true \ + --det=false \ + --rec=true \ + --cls=true \ +``` + +##### 5. 识别： +```shell +./build/ppocr --rec_model_dir=inference/rec_rcnn \ + --image_dir=../../doc/imgs_words/ch/word_1.jpg \ + --use_angle_cls=false \ + --det=false \ + --rec=true \ + --cls=false \ +``` + +##### 6. 分类： +```shell +./build/ppocr --cls_model_dir=inference/cls \ + --cls_model_dir=inference/cls \ + --image_dir=../../doc/imgs_words/ch/word_1.jpg \ + --use_angle_cls=true \ + --det=false \ + --rec=false \ + --cls=true \ ``` 更多支持的可调节参数解释如下： @@ -258,6 +299,15 @@ CUDNN_LIB_DIR=/your_cudnn_lib_dir |enable_mkldnn|bool|true|是否使用mkldnn库| |output|str|./output|可视化结果保存的路径| +- 前向相关 + +|参数名称|类型|默认参数|意义| +| :---: | :---: | :---: | :---: | +|det|bool|true|前向是否执行文字检测| +|rec|bool|true|前向是否执行文字识别| +|cls|bool|false|前向是否执行文字方向分类| + + - 检测模型相关 |参数名称|类型|默认参数|意义| @@ -277,6 +327,7 @@ CUDNN_LIB_DIR=/your_cudnn_lib_dir |use_angle_cls|bool|false|是否使用方向分类器| |cls_model_dir|string|-|方向分类器inference model地址| |cls_thresh|float|0.9|方向分类器的得分阈值| +|cls_batch_num|int|1|方向分类器batchsize| - 识别模型相关 @@ -284,15 +335,22 @@ CUDNN_LIB_DIR=/your_cudnn_lib_dir | :---: | :---: | :---: | :---: | |rec_model_dir|string|-|识别模型inference model地址| |rec_char_dict_path|string|../../ppocr/utils/ppocr_keys_v1.txt|字典文件| +|rec_batch_num|int|6|识别模型batchsize| * PaddleOCR也支持多语言的预测，更多支持的语言和模型可以参考[识别文档](../../doc/doc_ch/recognition.md)中的多语言字典与模型部分，如果希望进行多语言预测，只需将修改`rec_char_dict_path`（字典文件路径）以及`rec_model_dir`（inference模型路径）字段即可。 最终屏幕上会输出检测结果如下。 -
- -
+```bash +predict img: ../../doc/imgs/12.jpg +../../doc/imgs/12.jpg +0 det boxes: [[79,553],[399,541],[400,573],[80,585]] rec text: 打浦路252935号 rec score: 0.933757 +1 det boxes: [[31,509],[510,488],[511,529],[33,549]] rec text: 绿洲仕格维花园公寓 rec score: 0.951745 +2 det boxes: [[181,456],[395,448],[396,480],[182,488]] rec text: 打浦路15号 rec score: 0.91956 +3 det boxes: [[43,413],[480,391],[481,428],[45,450]] rec text: 上海斯格威铂尔多大酒店 rec score: 0.915914 +The detection visualized image saved in ./output//12.jpg +``` ## 3. FAQ diff --git a/deploy/cpp_infer/readme_en.md b/deploy/cpp_infer/readme_en.md index b7687a9f65..4290fbb0fa 100644 --- a/deploy/cpp_infer/readme_en.md +++ b/deploy/cpp_infer/readme_en.md @@ -9,9 +9,12 @@ - [2.1 Export the inference model](#21-export-the-inference-model) - [2.2 Compile PaddleOCR C++ inference demo](#22-compile-paddleocr-c-inference-demo) - [Run the demo](#run-the-demo) - - [1. run det demo:](#1-run-det-demo) - - [2. run rec demo:](#2-run-rec-demo) - - [3. run system demo:](#3-run-system-demo) + - [1. det+cls+rec：](#1-detclsrec) + - [2. det+rec：](#2-detrec) + - [3. det](#3-det) + - [4. cls+rec：](#4-clsrec) + - [5. rec](#5-rec) + - [6. cls](#6-cls) - [3. FAQ](#3-faq) # Server-side C++ Inference @@ -166,6 +169,9 @@ inference/ |-- rec_rcnn | |--inference.pdiparams | |--inference.pdmodel +|-- cls +| |--inference.pdiparams +| |--inference.pdmodel ``` @@ -198,44 +204,72 @@ or the generated Paddle inference library path (`build/paddle_inference_install_ ### Run the demo Execute the built executable file: ```shell -./build/ppocr [--param1] [--param2] [...] +./build/ppocr [--param1] [--param2] [...] ``` -`mode` is a required parameter，and the valid values are - -mode value | Model used ------|------ -det | Detection only -rec | Recognition only -system | End-to-end system Specifically, -##### 1. run det demo: +##### 1. det+cls+rec： ```shell -./build/ppocr det \ - --det_model_dir=inference/ch_ppocr_mobile_v2.0_det_infer \ - --image_dir=../../doc/imgs/12.jpg -``` -##### 2. run rec demo: -```shell -./build/ppocr rec \ - --rec_model_dir=inference/ch_ppocr_mobile_v2.0_rec_infer \ - --image_dir=../../doc/imgs_words/ch/ -``` -##### 3. run system demo: -```shell -# without text direction classifier -./build/ppocr system \ - --det_model_dir=inference/ch_ppocr_mobile_v2.0_det_infer \ - --rec_model_dir=inference/ch_ppocr_mobile_v2.0_rec_infer \ - --image_dir=../../doc/imgs/12.jpg -# with text direction classifier -./build/ppocr system \ - --det_model_dir=inference/ch_ppocr_mobile_v2.0_det_infer \ +./build/ppocr --det_model_dir=inference/det_db \ + --rec_model_dir=inference/rec_rcnn \ + --cls_model_dir=inference/cls \ + --image_dir=../../doc/imgs/12.jpg \ --use_angle_cls=true \ - --cls_model_dir=inference/ch_ppocr_mobile_v2.0_cls_infer \ - --rec_model_dir=inference/ch_ppocr_mobile_v2.0_rec_infer \ - --image_dir=../../doc/imgs/12.jpg + --det=true \ + --rec=true \ + --cls=true \ +``` + +##### 2. det+rec： +```shell +./build/ppocr --det_model_dir=inference/det_db \ + --rec_model_dir=inference/rec_rcnn \ + --image_dir=../../doc/imgs/12.jpg \ + --use_angle_cls=false \ + --det=true \ + --rec=true \ + --cls=false \ +``` + +##### 3. det +```shell +./build/ppocr --det_model_dir=inference/det_db \ + --image_dir=../../doc/imgs/12.jpg \ + --det=true \ + --rec=false +``` + +##### 4. cls+rec： +```shell +./build/ppocr --rec_model_dir=inference/rec_rcnn \ + --cls_model_dir=inference/cls \ + --image_dir=../../doc/imgs_words/ch/word_1.jpg \ + --use_angle_cls=true \ + --det=false \ + --rec=true \ + --cls=true \ +``` + +##### 5. rec +```shell +./build/ppocr --rec_model_dir=inference/rec_rcnn \ + --image_dir=../../doc/imgs_words/ch/word_1.jpg \ + --use_angle_cls=false \ + --det=false \ + --rec=true \ + --cls=false \ +``` + +##### 6. cls +```shell +./build/ppocr --cls_model_dir=inference/cls \ + --cls_model_dir=inference/cls \ + --image_dir=../../doc/imgs_words/ch/word_1.jpg \ + --use_angle_cls=true \ + --det=false \ + --rec=false \ + --cls=true \ ``` More parameters are as follows, @@ -251,6 +285,16 @@ More parameters are as follows, |enable_mkldnn|bool|true|Whether to use mkdlnn library| |output|str|./output|Path where visualization results are saved| + +- forward + +|parameter|data type|default|meaning| +| :---: | :---: | :---: | :---: | +|det|bool|true|前向是否执行文字检测| +|rec|bool|true|前向是否执行文字识别| +|cls|bool|false|前向是否执行文字方向分类| + + - Detection related parameters |parameter|data type|default|meaning| @@ -270,6 +314,7 @@ More parameters are as follows, |use_angle_cls|bool|false|Whether to use the direction classifier| |cls_model_dir|string|-|Address of direction classifier inference model| |cls_thresh|float|0.9|Score threshold of the direction classifier| +|cls_batch_num|int|1|batch size of classifier| - Recognition related parameters @@ -277,15 +322,22 @@ More parameters are as follows, | --- | --- | --- | --- | |rec_model_dir|string|-|Address of recognition inference model| |rec_char_dict_path|string|../../ppocr/utils/ppocr_keys_v1.txt|dictionary file| +|rec_batch_num|int|6|batch size of recognition| * Multi-language inference is also supported in PaddleOCR, you can refer to [recognition tutorial](../../doc/doc_en/recognition_en.md) for more supported languages and models in PaddleOCR. Specifically, if you want to infer using multi-language models, you just need to modify values of `rec_char_dict_path` and `rec_model_dir`. The detection results will be shown on the screen, which is as follows. -
- -
+```bash +predict img: ../../doc/imgs/12.jpg +../../doc/imgs/12.jpg +0 det boxes: [[79,553],[399,541],[400,573],[80,585]] rec text: 打浦路252935号 rec score: 0.933757 +1 det boxes: [[31,509],[510,488],[511,529],[33,549]] rec text: 绿洲仕格维花园公寓 rec score: 0.951745 +2 det boxes: [[181,456],[395,448],[396,480],[182,488]] rec text: 打浦路15号 rec score: 0.91956 +3 det boxes: [[43,413],[480,391],[481,428],[45,450]] rec text: 上海斯格威铂尔多大酒店 rec score: 0.915914 +The detection visualized image saved in ./output//12.jpg +``` ## 3. FAQ diff --git a/deploy/cpp_infer/src/args.cpp b/deploy/cpp_infer/src/args.cpp index 539968eb1a..82cfb54ae7 100644 --- a/deploy/cpp_infer/src/args.cpp +++ b/deploy/cpp_infer/src/args.cpp @@ -25,7 +25,9 @@ DEFINE_string(precision, "fp32", "Precision be one of fp32/fp16/int8"); DEFINE_bool(benchmark, false, "Whether use benchmark."); DEFINE_string(output, "./output/", "Save benchmark log path."); DEFINE_string(image_dir, "", "Dir of input image."); -DEFINE_bool(visualize, true, "Whether show the detection results."); +DEFINE_string( + type, "ocr", + "Perform ocr or structure, the value is selected in ['ocr','structure']."); // detection related DEFINE_string(det_model_dir, "", "Path of det inference model."); DEFINE_int32(max_side_len, 960, "max_side_len of input image."); @@ -34,12 +36,19 @@ DEFINE_double(det_db_box_thresh, 0.6, "Threshold of det_db_box_thresh."); DEFINE_double(det_db_unclip_ratio, 1.5, "Threshold of det_db_unclip_ratio."); DEFINE_bool(use_dilation, false, "Whether use the dilation on output map."); DEFINE_string(det_db_score_mode, "slow", "Whether use polygon score."); +DEFINE_bool(visualize, true, "Whether show the detection results."); // classification related DEFINE_bool(use_angle_cls, false, "Whether use use_angle_cls."); DEFINE_string(cls_model_dir, "", "Path of cls inference model."); DEFINE_double(cls_thresh, 0.9, "Threshold of cls_thresh."); +DEFINE_int32(cls_batch_num, 1, "cls_batch_num."); // recognition related DEFINE_string(rec_model_dir, "", "Path of rec inference model."); DEFINE_int32(rec_batch_num, 6, "rec_batch_num."); DEFINE_string(rec_char_dict_path, "../../ppocr/utils/ppocr_keys_v1.txt", - "Path of dictionary."); \ No newline at end of file + "Path of dictionary."); + +// ocr forward related +DEFINE_bool(det, true, "Whether use det in forward."); +DEFINE_bool(rec, true, "Whether use rec in forward."); +DEFINE_bool(cls, false, "Whether use cls in forward."); \ No newline at end of file diff --git a/deploy/cpp_infer/src/main.cpp b/deploy/cpp_infer/src/main.cpp index 27c7bde09c..84bf1698a5 100644 --- a/deploy/cpp_infer/src/main.cpp +++ b/deploy/cpp_infer/src/main.cpp @@ -11,248 +11,19 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - -#include "omp.h" #include "opencv2/core.hpp" #include "opencv2/imgcodecs.hpp" #include "opencv2/imgproc.hpp" -#include -#include #include -#include -#include #include -#include -#include -#include - #include -#include -#include -#include -#include -#include +#include -#include "auto_log/autolog.h" - -using namespace std; -using namespace cv; using namespace PaddleOCR; -static bool PathExists(const std::string &path) { -#ifdef _WIN32 - struct _stat buffer; - return (_stat(path.c_str(), &buffer) == 0); -#else - struct stat buffer; - return (stat(path.c_str(), &buffer) == 0); -#endif // !_WIN32 -} - -int main_det(std::vector cv_all_img_names) { - std::vector time_info = {0, 0, 0}; - DBDetector det(FLAGS_det_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_threads, FLAGS_enable_mkldnn, - FLAGS_max_side_len, FLAGS_det_db_thresh, - FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio, - FLAGS_det_db_score_mode, FLAGS_use_dilation, - FLAGS_use_tensorrt, FLAGS_precision); - - if (!PathExists(FLAGS_output)) { - mkdir(FLAGS_output.c_str(), 0777); - } - - for (int i = 0; i < cv_all_img_names.size(); ++i) { - if (!FLAGS_benchmark) { - cout << "The predict img: " << cv_all_img_names[i] << endl; - } - - cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); - if (!srcimg.data) { - std::cerr << "[ERROR] image read failed! image path: " - << cv_all_img_names[i] << endl; - exit(1); - } - std::vector>> boxes; - std::vector det_times; - - det.Run(srcimg, boxes, &det_times); - // visualization - if (FLAGS_visualize) { - std::string file_name = Utility::basename(cv_all_img_names[i]); - Utility::VisualizeBboxes(srcimg, boxes, FLAGS_output + "/" + file_name); - } - time_info[0] += det_times[0]; - time_info[1] += det_times[1]; - time_info[2] += det_times[2]; - - if (FLAGS_benchmark) { - cout << cv_all_img_names[i] << "\t["; - for (int n = 0; n < boxes.size(); n++) { - cout << '['; - for (int m = 0; m < boxes[n].size(); m++) { - cout << '[' << boxes[n][m][0] << ',' << boxes[n][m][1] << "]"; - if (m != boxes[n].size() - 1) { - cout << ','; - } - } - cout << ']'; - if (n != boxes.size() - 1) { - cout << ','; - } - } - cout << ']' << endl; - } - } - - if (FLAGS_benchmark) { - AutoLogger autolog("ocr_det", FLAGS_use_gpu, FLAGS_use_tensorrt, - FLAGS_enable_mkldnn, FLAGS_cpu_threads, 1, "dynamic", - FLAGS_precision, time_info, cv_all_img_names.size()); - autolog.report(); - } - return 0; -} - -int main_rec(std::vector cv_all_img_names) { - std::vector time_info = {0, 0, 0}; - - std::string rec_char_dict_path = FLAGS_rec_char_dict_path; - cout << "label file: " << rec_char_dict_path << endl; - - CRNNRecognizer rec(FLAGS_rec_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_threads, FLAGS_enable_mkldnn, - rec_char_dict_path, FLAGS_use_tensorrt, FLAGS_precision, - FLAGS_rec_batch_num); - - std::vector img_list; - for (int i = 0; i < cv_all_img_names.size(); ++i) { - cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); - if (!srcimg.data) { - std::cerr << "[ERROR] image read failed! image path: " - << cv_all_img_names[i] << endl; - exit(1); - } - img_list.push_back(srcimg); - } - std::vector rec_texts(img_list.size(), ""); - std::vector rec_text_scores(img_list.size(), 0); - std::vector rec_times; - rec.Run(img_list, rec_texts, rec_text_scores, &rec_times); - // output rec results - for (int i = 0; i < rec_texts.size(); i++) { - cout << "The predict img: " << cv_all_img_names[i] << "\t" << rec_texts[i] - << "\t" << rec_text_scores[i] << endl; - } - time_info[0] += rec_times[0]; - time_info[1] += rec_times[1]; - time_info[2] += rec_times[2]; - - if (FLAGS_benchmark) { - AutoLogger autolog("ocr_rec", FLAGS_use_gpu, FLAGS_use_tensorrt, - FLAGS_enable_mkldnn, FLAGS_cpu_threads, - FLAGS_rec_batch_num, "dynamic", FLAGS_precision, - time_info, cv_all_img_names.size()); - autolog.report(); - } - return 0; -} - -int main_system(std::vector cv_all_img_names) { - std::vector time_info_det = {0, 0, 0}; - std::vector time_info_rec = {0, 0, 0}; - - if (!PathExists(FLAGS_output)) { - mkdir(FLAGS_output.c_str(), 0777); - } - - DBDetector det(FLAGS_det_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_threads, FLAGS_enable_mkldnn, - FLAGS_max_side_len, FLAGS_det_db_thresh, - FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio, - FLAGS_det_db_score_mode, FLAGS_use_dilation, - FLAGS_use_tensorrt, FLAGS_precision); - - Classifier *cls = nullptr; - if (FLAGS_use_angle_cls) { - cls = new Classifier(FLAGS_cls_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_threads, FLAGS_enable_mkldnn, - FLAGS_cls_thresh, FLAGS_use_tensorrt, FLAGS_precision); - } - - std::string rec_char_dict_path = FLAGS_rec_char_dict_path; - cout << "label file: " << rec_char_dict_path << endl; - - CRNNRecognizer rec(FLAGS_rec_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, - FLAGS_gpu_mem, FLAGS_cpu_threads, FLAGS_enable_mkldnn, - rec_char_dict_path, FLAGS_use_tensorrt, FLAGS_precision, - FLAGS_rec_batch_num); - - for (int i = 0; i < cv_all_img_names.size(); ++i) { - cout << "The predict img: " << cv_all_img_names[i] << endl; - - cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); - if (!srcimg.data) { - std::cerr << "[ERROR] image read failed! image path: " - << cv_all_img_names[i] << endl; - exit(1); - } - // det - std::vector>> boxes; - std::vector det_times; - std::vector rec_times; - - det.Run(srcimg, boxes, &det_times); - if (FLAGS_visualize) { - std::string file_name = Utility::basename(cv_all_img_names[i]); - Utility::VisualizeBboxes(srcimg, boxes, FLAGS_output + "/" + file_name); - } - time_info_det[0] += det_times[0]; - time_info_det[1] += det_times[1]; - time_info_det[2] += det_times[2]; - - // rec - std::vector img_list; - for (int j = 0; j < boxes.size(); j++) { - cv::Mat crop_img; - crop_img = Utility::GetRotateCropImage(srcimg, boxes[j]); - if (cls != nullptr) { - crop_img = cls->Run(crop_img); - } - img_list.push_back(crop_img); - } - std::vector rec_texts(img_list.size(), ""); - std::vector rec_text_scores(img_list.size(), 0); - rec.Run(img_list, rec_texts, rec_text_scores, &rec_times); - // output rec results - for (int i = 0; i < rec_texts.size(); i++) { - std::cout << i << "\t" << rec_texts[i] << "\t" << rec_text_scores[i] - << std::endl; - } - time_info_rec[0] += rec_times[0]; - time_info_rec[1] += rec_times[1]; - time_info_rec[2] += rec_times[2]; - } - - if (FLAGS_benchmark) { - AutoLogger autolog_det("ocr_det", FLAGS_use_gpu, FLAGS_use_tensorrt, - FLAGS_enable_mkldnn, FLAGS_cpu_threads, 1, "dynamic", - FLAGS_precision, time_info_det, - cv_all_img_names.size()); - AutoLogger autolog_rec("ocr_rec", FLAGS_use_gpu, FLAGS_use_tensorrt, - FLAGS_enable_mkldnn, FLAGS_cpu_threads, - FLAGS_rec_batch_num, "dynamic", FLAGS_precision, - time_info_rec, cv_all_img_names.size()); - autolog_det.report(); - std::cout << endl; - autolog_rec.report(); - } - return 0; -} - -void check_params(char *mode) { - if (strcmp(mode, "det") == 0) { +void check_params() { + if (FLAGS_det) { if (FLAGS_det_model_dir.empty() || FLAGS_image_dir.empty()) { std::cout << "Usage[det]: ./ppocr " "--det_model_dir=/PATH/TO/DET_INFERENCE_MODEL/ " @@ -260,7 +31,7 @@ void check_params(char *mode) { exit(1); } } - if (strcmp(mode, "rec") == 0) { + if (FLAGS_rec) { if (FLAGS_rec_model_dir.empty() || FLAGS_image_dir.empty()) { std::cout << "Usage[rec]: ./ppocr " "--rec_model_dir=/PATH/TO/REC_INFERENCE_MODEL/ " @@ -268,19 +39,10 @@ void check_params(char *mode) { exit(1); } } - if (strcmp(mode, "system") == 0) { - if ((FLAGS_det_model_dir.empty() || FLAGS_rec_model_dir.empty() || - FLAGS_image_dir.empty()) || - (FLAGS_use_angle_cls && FLAGS_cls_model_dir.empty())) { - std::cout << "Usage[system without angle cls]: ./ppocr " - "--det_model_dir=/PATH/TO/DET_INFERENCE_MODEL/ " - << "--rec_model_dir=/PATH/TO/REC_INFERENCE_MODEL/ " - << "--image_dir=/PATH/TO/INPUT/IMAGE/" << std::endl; - std::cout << "Usage[system with angle cls]: ./ppocr " - "--det_model_dir=/PATH/TO/DET_INFERENCE_MODEL/ " - << "--use_angle_cls=true " - << "--cls_model_dir=/PATH/TO/CLS_INFERENCE_MODEL/ " - << "--rec_model_dir=/PATH/TO/REC_INFERENCE_MODEL/ " + if (FLAGS_cls && FLAGS_use_angle_cls) { + if (FLAGS_cls_model_dir.empty() || FLAGS_image_dir.empty()) { + std::cout << "Usage[cls]: ./ppocr " + << "--cls_model_dir=/PATH/TO/REC_INFERENCE_MODEL/ " << "--image_dir=/PATH/TO/INPUT/IMAGE/" << std::endl; exit(1); } @@ -293,19 +55,11 @@ void check_params(char *mode) { } int main(int argc, char **argv) { - if (argc <= 1 || - (strcmp(argv[1], "det") != 0 && strcmp(argv[1], "rec") != 0 && - strcmp(argv[1], "system") != 0)) { - std::cout << "Please choose one mode of [det, rec, system] !" << std::endl; - return -1; - } - std::cout << "mode: " << argv[1] << endl; - // Parsing command-line google::ParseCommandLineFlags(&argc, &argv, true); - check_params(argv[1]); + check_params(); - if (!PathExists(FLAGS_image_dir)) { + if (!Utility::PathExists(FLAGS_image_dir)) { std::cerr << "[ERROR] image path not exist! image_dir: " << FLAGS_image_dir << endl; exit(1); @@ -315,13 +69,26 @@ int main(int argc, char **argv) { cv::glob(FLAGS_image_dir, cv_all_img_names); std::cout << "total images num: " << cv_all_img_names.size() << endl; - if (strcmp(argv[1], "det") == 0) { - return main_det(cv_all_img_names); - } - if (strcmp(argv[1], "rec") == 0) { - return main_rec(cv_all_img_names); - } - if (strcmp(argv[1], "system") == 0) { - return main_system(cv_all_img_names); + PaddleOCR::PaddleOCR ocr = PaddleOCR::PaddleOCR(); + + std::vector> ocr_results = + ocr.ocr(cv_all_img_names, FLAGS_det, FLAGS_rec, FLAGS_cls); + + for (int i = 0; i < cv_all_img_names.size(); ++i) { + cout << cv_all_img_names[i] << "\n"; + Utility::print_result(ocr_results[i]); + if (FLAGS_visualize && FLAGS_det) { + cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); + if (!srcimg.data) { + std::cerr << "[ERROR] image read failed! image path: " + << cv_all_img_names[i] << endl; + exit(1); + } + std::string file_name = Utility::basename(cv_all_img_names[i]); + + Utility::VisualizeBboxes(srcimg, ocr_results[i], + FLAGS_output + "/" + file_name); + } + cout << "***************************" << endl; } } diff --git a/deploy/cpp_infer/src/ocr_cls.cpp b/deploy/cpp_infer/src/ocr_cls.cpp index 3b04b6f824..674630bf1e 100644 --- a/deploy/cpp_infer/src/ocr_cls.cpp +++ b/deploy/cpp_infer/src/ocr_cls.cpp @@ -16,57 +16,84 @@ namespace PaddleOCR { -cv::Mat Classifier::Run(cv::Mat &img) { - cv::Mat src_img; - img.copyTo(src_img); - cv::Mat resize_img; +void Classifier::Run(std::vector img_list, + std::vector &cls_labels, + std::vector &cls_scores, + std::vector ×) { + std::chrono::duration preprocess_diff = + std::chrono::steady_clock::now() - std::chrono::steady_clock::now(); + std::chrono::duration inference_diff = + std::chrono::steady_clock::now() - std::chrono::steady_clock::now(); + std::chrono::duration postprocess_diff = + std::chrono::steady_clock::now() - std::chrono::steady_clock::now(); + int img_num = img_list.size(); std::vector cls_image_shape = {3, 48, 192}; - int index = 0; - float wh_ratio = float(img.cols) / float(img.rows); + for (int beg_img_no = 0; beg_img_no < img_num; + beg_img_no += this->cls_batch_num_) { + auto preprocess_start = std::chrono::steady_clock::now(); + int end_img_no = min(img_num, beg_img_no + this->cls_batch_num_); + int batch_num = end_img_no - beg_img_no; + // preprocess + std::vector norm_img_batch; + for (int ino = beg_img_no; ino < end_img_no; ino++) { + cv::Mat srcimg; + img_list[ino].copyTo(srcimg); + cv::Mat resize_img; + this->resize_op_.Run(srcimg, resize_img, this->use_tensorrt_, + cls_image_shape); - this->resize_op_.Run(img, resize_img, this->use_tensorrt_, cls_image_shape); - - this->normalize_op_.Run(&resize_img, this->mean_, this->scale_, - this->is_scale_); - - std::vector input(1 * 3 * resize_img.rows * resize_img.cols, 0.0f); - - this->permute_op_.Run(&resize_img, input.data()); - - // Inference. - auto input_names = this->predictor_->GetInputNames(); - auto input_t = this->predictor_->GetInputHandle(input_names[0]); - input_t->Reshape({1, 3, resize_img.rows, resize_img.cols}); - input_t->CopyFromCpu(input.data()); - this->predictor_->Run(); - - std::vector softmax_out; - std::vector label_out; - auto output_names = this->predictor_->GetOutputNames(); - auto softmax_out_t = this->predictor_->GetOutputHandle(output_names[0]); - auto softmax_shape_out = softmax_out_t->shape(); - - int softmax_out_num = - std::accumulate(softmax_shape_out.begin(), softmax_shape_out.end(), 1, - std::multiplies()); - - softmax_out.resize(softmax_out_num); - - softmax_out_t->CopyToCpu(softmax_out.data()); - - float score = 0; - int label = 0; - for (int i = 0; i < softmax_out_num; i++) { - if (softmax_out[i] > score) { - score = softmax_out[i]; - label = i; + this->normalize_op_.Run(&resize_img, this->mean_, this->scale_, + this->is_scale_); + norm_img_batch.push_back(resize_img); } + std::vector input(batch_num * cls_image_shape[0] * + cls_image_shape[1] * cls_image_shape[2], + 0.0f); + this->permute_op_.Run(norm_img_batch, input.data()); + auto preprocess_end = std::chrono::steady_clock::now(); + preprocess_diff += preprocess_end - preprocess_start; + + // inference. + auto input_names = this->predictor_->GetInputNames(); + auto input_t = this->predictor_->GetInputHandle(input_names[0]); + input_t->Reshape({batch_num, cls_image_shape[0], cls_image_shape[1], + cls_image_shape[2]}); + auto inference_start = std::chrono::steady_clock::now(); + input_t->CopyFromCpu(input.data()); + this->predictor_->Run(); + + std::vector predict_batch; + auto output_names = this->predictor_->GetOutputNames(); + auto output_t = this->predictor_->GetOutputHandle(output_names[0]); + auto predict_shape = output_t->shape(); + + int out_num = std::accumulate(predict_shape.begin(), predict_shape.end(), 1, + std::multiplies()); + predict_batch.resize(out_num); + + output_t->CopyToCpu(predict_batch.data()); + auto inference_end = std::chrono::steady_clock::now(); + inference_diff += inference_end - inference_start; + + // postprocess + auto postprocess_start = std::chrono::steady_clock::now(); + for (int batch_idx = 0; batch_idx < predict_shape[0]; batch_idx++) { + int label = int( + Utility::argmax(&predict_batch[batch_idx * predict_shape[1]], + &predict_batch[(batch_idx + 1) * predict_shape[1]])); + float score = float(*std::max_element( + &predict_batch[batch_idx * predict_shape[1]], + &predict_batch[(batch_idx + 1) * predict_shape[1]])); + cls_labels[beg_img_no + batch_idx] = label; + cls_scores[beg_img_no + batch_idx] = score; + } + auto postprocess_end = std::chrono::steady_clock::now(); + postprocess_diff += postprocess_end - postprocess_start; } - if (label % 2 == 1 && score > this->cls_thresh) { - cv::rotate(src_img, src_img, 1); - } - return src_img; + times.push_back(double(preprocess_diff.count() * 1000)); + times.push_back(double(inference_diff.count() * 1000)); + times.push_back(double(postprocess_diff.count() * 1000)); } void Classifier::LoadModel(const std::string &model_dir) { @@ -81,13 +108,10 @@ void Classifier::LoadModel(const std::string &model_dir) { if (this->precision_ == "fp16") { precision = paddle_infer::Config::Precision::kHalf; } - if (this->precision_ == "int8") { + if (this->precision_ == "int8") { precision = paddle_infer::Config::Precision::kInt8; - } - config.EnableTensorRtEngine( - 1 << 20, 10, 3, - precision, - false, false); + } + config.EnableTensorRtEngine(1 << 20, 10, 3, precision, false, false); } } else { config.DisableGpu(); diff --git a/deploy/cpp_infer/src/ocr_det.cpp b/deploy/cpp_infer/src/ocr_det.cpp index ff7164ca24..c08f97b55a 100644 --- a/deploy/cpp_infer/src/ocr_det.cpp +++ b/deploy/cpp_infer/src/ocr_det.cpp @@ -94,7 +94,7 @@ void DBDetector::LoadModel(const std::string &model_dir) { void DBDetector::Run(cv::Mat &img, std::vector>> &boxes, - std::vector *times) { + std::vector ×) { float ratio_h{}; float ratio_w{}; @@ -165,16 +165,15 @@ void DBDetector::Run(cv::Mat &img, boxes = post_processor_.FilterTagDetRes(boxes, ratio_h, ratio_w, srcimg); auto postprocess_end = std::chrono::steady_clock::now(); - std::cout << "Detected boxes num: " << boxes.size() << endl; std::chrono::duration preprocess_diff = preprocess_end - preprocess_start; - times->push_back(double(preprocess_diff.count() * 1000)); + times.push_back(double(preprocess_diff.count() * 1000)); std::chrono::duration inference_diff = inference_end - inference_start; - times->push_back(double(inference_diff.count() * 1000)); + times.push_back(double(inference_diff.count() * 1000)); std::chrono::duration postprocess_diff = postprocess_end - postprocess_start; - times->push_back(double(postprocess_diff.count() * 1000)); + times.push_back(double(postprocess_diff.count() * 1000)); } } // namespace PaddleOCR diff --git a/deploy/cpp_infer/src/ocr_rec.cpp b/deploy/cpp_infer/src/ocr_rec.cpp index 4c94e8f3fc..54ed3fefcb 100644 --- a/deploy/cpp_infer/src/ocr_rec.cpp +++ b/deploy/cpp_infer/src/ocr_rec.cpp @@ -19,7 +19,7 @@ namespace PaddleOCR { void CRNNRecognizer::Run(std::vector img_list, std::vector &rec_texts, std::vector &rec_text_scores, - std::vector *times) { + std::vector ×) { std::chrono::duration preprocess_diff = std::chrono::steady_clock::now() - std::chrono::steady_clock::now(); std::chrono::duration inference_diff = @@ -38,6 +38,7 @@ void CRNNRecognizer::Run(std::vector img_list, beg_img_no += this->rec_batch_num_) { auto preprocess_start = std::chrono::steady_clock::now(); int end_img_no = min(img_num, beg_img_no + this->rec_batch_num_); + int batch_num = end_img_no - beg_img_no; float max_wh_ratio = 0; for (int ino = beg_img_no; ino < end_img_no; ino++) { int h = img_list[indices[ino]].rows; @@ -45,6 +46,7 @@ void CRNNRecognizer::Run(std::vector img_list, float wh_ratio = w * 1.0 / h; max_wh_ratio = max(max_wh_ratio, wh_ratio); } + int batch_width = 0; std::vector norm_img_batch; for (int ino = beg_img_no; ino < end_img_no; ino++) { @@ -59,15 +61,14 @@ void CRNNRecognizer::Run(std::vector img_list, batch_width = max(resize_img.cols, batch_width); } - std::vector input(this->rec_batch_num_ * 3 * 32 * batch_width, 0.0f); + std::vector input(batch_num * 3 * 32 * batch_width, 0.0f); this->permute_op_.Run(norm_img_batch, input.data()); auto preprocess_end = std::chrono::steady_clock::now(); preprocess_diff += preprocess_end - preprocess_start; - // Inference. auto input_names = this->predictor_->GetInputNames(); auto input_t = this->predictor_->GetInputHandle(input_names[0]); - input_t->Reshape({this->rec_batch_num_, 3, 32, batch_width}); + input_t->Reshape({batch_num, 3, 32, batch_width}); auto inference_start = std::chrono::steady_clock::now(); input_t->CopyFromCpu(input.data()); this->predictor_->Run(); @@ -84,7 +85,6 @@ void CRNNRecognizer::Run(std::vector img_list, output_t->CopyToCpu(predict_batch.data()); auto inference_end = std::chrono::steady_clock::now(); inference_diff += inference_end - inference_start; - // ctc decode auto postprocess_start = std::chrono::steady_clock::now(); for (int m = 0; m < predict_shape[0]; m++) { @@ -120,9 +120,9 @@ void CRNNRecognizer::Run(std::vector img_list, auto postprocess_end = std::chrono::steady_clock::now(); postprocess_diff += postprocess_end - postprocess_start; } - times->push_back(double(preprocess_diff.count() * 1000)); - times->push_back(double(inference_diff.count() * 1000)); - times->push_back(double(postprocess_diff.count() * 1000)); + times.push_back(double(preprocess_diff.count() * 1000)); + times.push_back(double(inference_diff.count() * 1000)); + times.push_back(double(postprocess_diff.count() * 1000)); } void CRNNRecognizer::LoadModel(const std::string &model_dir) { diff --git a/deploy/cpp_infer/src/paddleocr.cpp b/deploy/cpp_infer/src/paddleocr.cpp new file mode 100644 index 0000000000..3ea6e78d05 --- /dev/null +++ b/deploy/cpp_infer/src/paddleocr.cpp @@ -0,0 +1,225 @@ +// Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include + +#include "auto_log/autolog.h" +#include +#include + +namespace PaddleOCR { + +PaddleOCR::PaddleOCR() { + if (FLAGS_det) { + this->detector_ = new DBDetector( + FLAGS_det_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, FLAGS_gpu_mem, + FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_max_side_len, + FLAGS_det_db_thresh, FLAGS_det_db_box_thresh, FLAGS_det_db_unclip_ratio, + FLAGS_det_db_score_mode, FLAGS_use_dilation, FLAGS_use_tensorrt, + FLAGS_precision); + } + + if (FLAGS_cls && FLAGS_use_angle_cls) { + this->classifier_ = new Classifier( + FLAGS_cls_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, FLAGS_gpu_mem, + FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_cls_thresh, + FLAGS_use_tensorrt, FLAGS_precision, FLAGS_cls_batch_num); + } + if (FLAGS_rec) { + this->recognizer_ = new CRNNRecognizer( + FLAGS_rec_model_dir, FLAGS_use_gpu, FLAGS_gpu_id, FLAGS_gpu_mem, + FLAGS_cpu_threads, FLAGS_enable_mkldnn, FLAGS_rec_char_dict_path, + FLAGS_use_tensorrt, FLAGS_precision, FLAGS_rec_batch_num); + } +}; + +void PaddleOCR::det(cv::Mat img, std::vector &ocr_results, + std::vector ×) { + std::vector>> boxes; + std::vector det_times; + + this->detector_->Run(img, boxes, det_times); + + for (int i = 0; i < boxes.size(); i++) { + OCRPredictResult res; + res.box = boxes[i]; + ocr_results.push_back(res); + } + + times[0] += det_times[0]; + times[1] += det_times[1]; + times[2] += det_times[2]; +} + +void PaddleOCR::rec(std::vector img_list, + std::vector &ocr_results, + std::vector ×) { + std::vector rec_texts(img_list.size(), ""); + std::vector rec_text_scores(img_list.size(), 0); + std::vector rec_times; + this->recognizer_->Run(img_list, rec_texts, rec_text_scores, rec_times); + // output rec results + for (int i = 0; i < rec_texts.size(); i++) { + ocr_results[i].text = rec_texts[i]; + ocr_results[i].score = rec_text_scores[i]; + } + times[0] += rec_times[0]; + times[1] += rec_times[1]; + times[2] += rec_times[2]; +} + +void PaddleOCR::cls(std::vector img_list, + std::vector &ocr_results, + std::vector ×) { + std::vector cls_labels(img_list.size(), 0); + std::vector cls_scores(img_list.size(), 0); + std::vector cls_times; + this->classifier_->Run(img_list, cls_labels, cls_scores, cls_times); + // output cls results + for (int i = 0; i < cls_labels.size(); i++) { + ocr_results[i].cls_label = cls_labels[i]; + ocr_results[i].cls_score = cls_scores[i]; + } + times[0] += cls_times[0]; + times[1] += cls_times[1]; + times[2] += cls_times[2]; +} + +std::vector> +PaddleOCR::ocr(std::vector cv_all_img_names, bool det, bool rec, + bool cls) { + std::vector time_info_det = {0, 0, 0}; + std::vector time_info_rec = {0, 0, 0}; + std::vector time_info_cls = {0, 0, 0}; + std::vector> ocr_results; + + if (!det) { + std::vector ocr_result; + // read image + std::vector img_list; + for (int i = 0; i < cv_all_img_names.size(); ++i) { + cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); + if (!srcimg.data) { + std::cerr << "[ERROR] image read failed! image path: " + << cv_all_img_names[i] << endl; + exit(1); + } + img_list.push_back(srcimg); + OCRPredictResult res; + ocr_result.push_back(res); + } + if (cls && this->classifier_ != nullptr) { + this->cls(img_list, ocr_result, time_info_cls); + for (int i = 0; i < img_list.size(); i++) { + if (ocr_result[i].cls_label % 2 == 1 && + ocr_result[i].cls_score > this->classifier_->cls_thresh) { + cv::rotate(img_list[i], img_list[i], 1); + } + } + } + if (rec) { + this->rec(img_list, ocr_result, time_info_rec); + } + for (int i = 0; i < cv_all_img_names.size(); ++i) { + std::vector ocr_result_tmp; + ocr_result_tmp.push_back(ocr_result[i]); + ocr_results.push_back(ocr_result_tmp); + } + } else { + if (!Utility::PathExists(FLAGS_output) && FLAGS_det) { + mkdir(FLAGS_output.c_str(), 0777); + } + + for (int i = 0; i < cv_all_img_names.size(); ++i) { + std::vector ocr_result; + cout << "predict img: " << cv_all_img_names[i] << endl; + + cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); + if (!srcimg.data) { + std::cerr << "[ERROR] image read failed! image path: " + << cv_all_img_names[i] << endl; + exit(1); + } + // det + this->det(srcimg, ocr_result, time_info_det); + // crop image + std::vector img_list; + for (int j = 0; j < ocr_result.size(); j++) { + cv::Mat crop_img; + crop_img = Utility::GetRotateCropImage(srcimg, ocr_result[j].box); + img_list.push_back(crop_img); + } + + // cls + if (cls && this->classifier_ != nullptr) { + this->cls(img_list, ocr_result, time_info_cls); + for (int i = 0; i < img_list.size(); i++) { + if (ocr_result[i].cls_label % 2 == 1 && + ocr_result[i].cls_score > this->classifier_->cls_thresh) { + cv::rotate(img_list[i], img_list[i], 1); + } + } + } + // rec + if (rec) { + this->rec(img_list, ocr_result, time_info_rec); + } + ocr_results.push_back(ocr_result); + } + } + if (FLAGS_benchmark) { + this->log(time_info_det, time_info_rec, time_info_cls, + cv_all_img_names.size()); + } + return ocr_results; +} // namespace PaddleOCR + +void PaddleOCR::log(std::vector &det_times, + std::vector &rec_times, + std::vector &cls_times, int img_num) { + if (det_times[0] + det_times[1] + det_times[2] > 0) { + AutoLogger autolog_det("ocr_det", FLAGS_use_gpu, FLAGS_use_tensorrt, + FLAGS_enable_mkldnn, FLAGS_cpu_threads, 1, "dynamic", + FLAGS_precision, det_times, img_num); + autolog_det.report(); + } + if (rec_times[0] + rec_times[1] + rec_times[2] > 0) { + AutoLogger autolog_rec("ocr_rec", FLAGS_use_gpu, FLAGS_use_tensorrt, + FLAGS_enable_mkldnn, FLAGS_cpu_threads, + FLAGS_rec_batch_num, "dynamic", FLAGS_precision, + rec_times, img_num); + autolog_rec.report(); + } + if (cls_times[0] + cls_times[1] + cls_times[2] > 0) { + AutoLogger autolog_cls("ocr_cls", FLAGS_use_gpu, FLAGS_use_tensorrt, + FLAGS_enable_mkldnn, FLAGS_cpu_threads, + FLAGS_cls_batch_num, "dynamic", FLAGS_precision, + cls_times, img_num); + autolog_cls.report(); + } +} +PaddleOCR::~PaddleOCR() { + if (this->detector_ != nullptr) { + delete this->detector_; + } + if (this->classifier_ != nullptr) { + delete this->classifier_; + } + if (this->recognizer_ != nullptr) { + delete this->recognizer_; + } +}; + +} // namespace PaddleOCR diff --git a/deploy/cpp_infer/src/utility.cpp b/deploy/cpp_infer/src/utility.cpp index 034df07804..339e992daa 100644 --- a/deploy/cpp_infer/src/utility.cpp +++ b/deploy/cpp_infer/src/utility.cpp @@ -38,16 +38,16 @@ std::vector Utility::ReadDict(const std::string &path) { return m_vec; } -void Utility::VisualizeBboxes( - const cv::Mat &srcimg, - const std::vector>> &boxes, - const std::string &save_path) { +void Utility::VisualizeBboxes(const cv::Mat &srcimg, + const std::vector &ocr_result, + const std::string &save_path) { cv::Mat img_vis; srcimg.copyTo(img_vis); - for (int n = 0; n < boxes.size(); n++) { + for (int n = 0; n < ocr_result.size(); n++) { cv::Point rook_points[4]; - for (int m = 0; m < boxes[n].size(); m++) { - rook_points[m] = cv::Point(int(boxes[n][m][0]), int(boxes[n][m][1])); + for (int m = 0; m < ocr_result[n].box.size(); m++) { + rook_points[m] = + cv::Point(int(ocr_result[n].box[m][0]), int(ocr_result[n].box[m][1])); } const cv::Point *ppt[1] = {rook_points}; @@ -196,4 +196,43 @@ std::string Utility::basename(const std::string &filename) { return filename.substr(index + 1, len - index); } +bool Utility::PathExists(const std::string &path) { +#ifdef _WIN32 + struct _stat buffer; + return (_stat(path.c_str(), &buffer) == 0); +#else + struct stat buffer; + return (stat(path.c_str(), &buffer) == 0); +#endif // !_WIN32 +} + +void Utility::print_result(const std::vector &ocr_result) { + for (int i = 0; i < ocr_result.size(); i++) { + std::cout << i << "\t"; + // det + std::vector> boxes = ocr_result[i].box; + if (boxes.size() > 0) { + std::cout << "det boxes: ["; + for (int n = 0; n < boxes.size(); n++) { + std::cout << '[' << boxes[n][0] << ',' << boxes[n][1] << "]"; + if (n != boxes.size() - 1) { + std::cout << ','; + } + } + std::cout << "] "; + } + // rec + if (ocr_result[i].score != -1.0) { + std::cout << "rec text: " << ocr_result[i].text + << " rec score: " << ocr_result[i].score << " "; + } + + // cls + if (ocr_result[i].cls_label != -1) { + std::cout << "cls label: " << ocr_result[i].cls_label + << " cls score: " << ocr_result[i].cls_score; + } + std::cout << std::endl; + } +} } // namespace PaddleOCR \ No newline at end of file From 1c589d6a3a5a53d53a1137a3847816b51d2881dd Mon Sep 17 00:00:00 2001 From: LDOUBLEV Date: Mon, 11 Apr 2022 05:13:56 +0000 Subject: [PATCH 14/21] fix slim export model --- deploy/slim/quantization/export_model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/deploy/slim/quantization/export_model.py b/deploy/slim/quantization/export_model.py index bbd291c334..822fd5da4c 100755 --- a/deploy/slim/quantization/export_model.py +++ b/deploy/slim/quantization/export_model.py @@ -133,6 +133,7 @@ def main(): sub_model_save_path, logger) else: save_path = os.path.join(save_path, "inference") + model.eval() export_single_model(quanter, model, infer_shape, save_path, logger) From 202fa428544b1bfef7d11a5486a3f3aed713f95d Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Mon, 11 Apr 2022 07:24:17 +0000 Subject: [PATCH 15/21] adatator tipc --- deploy/cpp_infer/src/main.cpp | 35 ++++++++++++------- deploy/cpp_infer/src/paddleocr.cpp | 4 ++- ..._normal_normal_infer_cpp_linux_gpu_cpu.txt | 10 ++++-- test_tipc/test_inference_cpp.sh | 22 ++++++++++-- 4 files changed, 52 insertions(+), 19 deletions(-) diff --git a/deploy/cpp_infer/src/main.cpp b/deploy/cpp_infer/src/main.cpp index 84bf1698a5..66ac795f59 100644 --- a/deploy/cpp_infer/src/main.cpp +++ b/deploy/cpp_infer/src/main.cpp @@ -75,20 +75,31 @@ int main(int argc, char **argv) { ocr.ocr(cv_all_img_names, FLAGS_det, FLAGS_rec, FLAGS_cls); for (int i = 0; i < cv_all_img_names.size(); ++i) { - cout << cv_all_img_names[i] << "\n"; - Utility::print_result(ocr_results[i]); - if (FLAGS_visualize && FLAGS_det) { - cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); - if (!srcimg.data) { - std::cerr << "[ERROR] image read failed! image path: " - << cv_all_img_names[i] << endl; - exit(1); + if (FLAGS_benchmark) { + cout << cv_all_img_names[i] << '\t'; + for (int n = 0; n < ocr_results[i].size(); n++) { + for (int m = 0; m < ocr_results[i][n].box.size(); m++) { + cout << ocr_results[i][n].box[m][0] << ' ' + << ocr_results[i][n].box[m][1] << ' '; + } } - std::string file_name = Utility::basename(cv_all_img_names[i]); + cout << endl; + } else { + cout << cv_all_img_names[i] << "\n"; + Utility::print_result(ocr_results[i]); + if (FLAGS_visualize && FLAGS_det) { + cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); + if (!srcimg.data) { + std::cerr << "[ERROR] image read failed! image path: " + << cv_all_img_names[i] << endl; + exit(1); + } + std::string file_name = Utility::basename(cv_all_img_names[i]); - Utility::VisualizeBboxes(srcimg, ocr_results[i], - FLAGS_output + "/" + file_name); + Utility::VisualizeBboxes(srcimg, ocr_results[i], + FLAGS_output + "/" + file_name); + } + cout << "***************************" << endl; } - cout << "***************************" << endl; } } diff --git a/deploy/cpp_infer/src/paddleocr.cpp b/deploy/cpp_infer/src/paddleocr.cpp index 3ea6e78d05..861461a01b 100644 --- a/deploy/cpp_infer/src/paddleocr.cpp +++ b/deploy/cpp_infer/src/paddleocr.cpp @@ -144,7 +144,9 @@ PaddleOCR::ocr(std::vector cv_all_img_names, bool det, bool rec, for (int i = 0; i < cv_all_img_names.size(); ++i) { std::vector ocr_result; - cout << "predict img: " << cv_all_img_names[i] << endl; + if (!FLAGS_benchmark) { + cout << "predict img: " << cv_all_img_names[i] << endl; + } cv::Mat srcimg = cv::imread(cv_all_img_names[i], cv::IMREAD_COLOR); if (!srcimg.data) { diff --git a/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt b/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt index 0b41398769..62993c6a72 100644 --- a/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt +++ b/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt @@ -3,8 +3,8 @@ model_name:ocr_det use_opencv:True infer_model:./inference/ch_ppocr_mobile_v2.0_det_infer/ infer_quant:False -inference:./deploy/cpp_infer/build/ppocr det ---use_gpu:True|False +inference:./deploy/cpp_infer/build/ppocr +--use_gpu:False --enable_mkldnn:True|False --cpu_threads:1|6 --rec_batch_num:1 @@ -13,4 +13,8 @@ inference:./deploy/cpp_infer/build/ppocr det --det_model_dir: --image_dir:./inference/ch_det_data_50/all-sum-510/ null:null ---benchmark:True \ No newline at end of file +--benchmark:True +--det:True +--rec:False +--cls:False +--use_angle_cls:False \ No newline at end of file diff --git a/test_tipc/test_inference_cpp.sh b/test_tipc/test_inference_cpp.sh index 257200fb10..9885e39372 100644 --- a/test_tipc/test_inference_cpp.sh +++ b/test_tipc/test_inference_cpp.sh @@ -2,7 +2,7 @@ source test_tipc/common_func.sh FILENAME=$1 -dataline=$(awk 'NR==1, NR==16{print}' $FILENAME) +dataline=$(awk 'NR==1, NR==20{print}' $FILENAME) # parser params IFS=$'\n' @@ -34,6 +34,14 @@ cpp_infer_key1=$(func_parser_key "${lines[14]}") cpp_infer_value1=$(func_parser_value "${lines[14]}") cpp_benchmark_key=$(func_parser_key "${lines[15]}") cpp_benchmark_value=$(func_parser_value "${lines[15]}") +cpp_det_key=$(func_parser_key "${lines[16]}") +cpp_det_value=$(func_parser_value "${lines[16]}") +cpp_rec_key=$(func_parser_key "${lines[17]}") +cpp_rec_value=$(func_parser_value "${lines[17]}") +cpp_cls_key=$(func_parser_key "${lines[18]}") +cpp_cls_value=$(func_parser_value "${lines[18]}") +cpp_use_angle_cls_key=$(func_parser_key "${lines[19]}") +cpp_use_angle_cls_value=$(func_parser_value "${lines[19]}") LOG_PATH="./test_tipc/output" mkdir -p ${LOG_PATH} @@ -68,7 +76,11 @@ function func_cpp_inference(){ set_cpu_threads=$(func_set_params "${cpp_cpu_threads_key}" "${threads}") set_model_dir=$(func_set_params "${cpp_infer_model_key}" "${_model_dir}") set_infer_params1=$(func_set_params "${cpp_infer_key1}" "${cpp_infer_value1}") - command="${_script} ${cpp_use_gpu_key}=${use_gpu} ${set_mkldnn} ${set_cpu_threads} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_infer_params1} > ${_save_log_path} 2>&1 " + set_det=$(func_set_params "${cpp_det_key}" "${cpp_det_value}") + set_rec=$(func_set_params "${cpp_rec_key}" "${cpp_rec_value}") + set_cls=$(func_set_params "${cpp_cls_key}" "${cpp_cls_value}") + set_use_angle_cls=$(func_set_params "${cpp_use_angle_cls_key}" "${cpp_use_angle_cls_value}") + command="${_script} ${cpp_use_gpu_key}=${use_gpu} ${set_mkldnn} ${set_cpu_threads} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_det} ${set_rec} ${set_cls} ${set_use_angle_cls} ${set_infer_params1} > ${_save_log_path} 2>&1 " eval $command last_status=${PIPESTATUS[0]} eval "cat ${_save_log_path}" @@ -97,7 +109,11 @@ function func_cpp_inference(){ set_precision=$(func_set_params "${cpp_precision_key}" "${precision}") set_model_dir=$(func_set_params "${cpp_infer_model_key}" "${_model_dir}") set_infer_params1=$(func_set_params "${cpp_infer_key1}" "${cpp_infer_value1}") - command="${_script} ${cpp_use_gpu_key}=${use_gpu} ${set_tensorrt} ${set_precision} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_infer_params1} > ${_save_log_path} 2>&1 " + set_det=$(func_set_params "${cpp_det_key}" "${cpp_det_value}") + set_rec=$(func_set_params "${cpp_rec_key}" "${cpp_rec_value}") + set_cls=$(func_set_params "${cpp_cls_key}" "${cpp_cls_value}") + set_use_angle_cls=$(func_set_params "${cpp_use_angle_cls_key}" "${cpp_use_angle_cls_value}") + command="${_script} ${cpp_use_gpu_key}=${use_gpu} ${set_tensorrt} ${set_precision} ${set_model_dir} ${set_batchsize} ${set_infer_data} ${set_benchmark} ${set_det} ${set_rec} ${set_cls} ${set_use_angle_cls} ${set_infer_params1} > ${_save_log_path} 2>&1 " eval $command last_status=${PIPESTATUS[0]} eval "cat ${_save_log_path}" From 34587a3fbcf35f2b079f423254025de8395427c8 Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Mon, 11 Apr 2022 08:19:05 +0000 Subject: [PATCH 16/21] rm post_processor_ --- deploy/cpp_infer/include/ocr_rec.h | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deploy/cpp_infer/include/ocr_rec.h b/deploy/cpp_infer/include/ocr_rec.h index a55d482bd9..f634b6dc84 100644 --- a/deploy/cpp_infer/include/ocr_rec.h +++ b/deploy/cpp_infer/include/ocr_rec.h @@ -30,7 +30,6 @@ #include #include -#include #include #include @@ -93,9 +92,6 @@ private: Normalize normalize_op_; PermuteBatch permute_op_; - // post-process - PostProcessor post_processor_; - }; // class CrnnRecognizer } // namespace PaddleOCR From b62e6954adfab0edb4a3fcd9835f147c89111806 Mon Sep 17 00:00:00 2001 From: LDOUBLEV Date: Tue, 12 Apr 2022 14:00:07 +0800 Subject: [PATCH 17/21] fix rec output name error --- tools/infer/utility.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tools/infer/utility.py b/tools/infer/utility.py index fd2605d963..b16aecd496 100644 --- a/tools/infer/utility.py +++ b/tools/infer/utility.py @@ -323,6 +323,10 @@ def get_output_tensors(args, mode, predictor): output_name = 'softmax_0.tmp_0' if output_name in output_names: return [predictor.get_output_handle(output_name)] + else: + for output_name in output_names: + output_tensor = predictor.get_output_handle(output_name) + output_tensors.append(output_tensor) else: for output_name in output_names: output_tensor = predictor.get_output_handle(output_name) From 0f5fb9bd351b44fedede30094d25040f6ee4b3a9 Mon Sep 17 00:00:00 2001 From: WenmuZhou Date: Tue, 12 Apr 2022 19:39:50 +0800 Subject: [PATCH 18/21] rm to tensor --- ppocr/modeling/heads/det_fce_head.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ppocr/modeling/heads/det_fce_head.py b/ppocr/modeling/heads/det_fce_head.py index 5d5e9205f7..9503989f58 100644 --- a/ppocr/modeling/heads/det_fce_head.py +++ b/ppocr/modeling/heads/det_fce_head.py @@ -63,7 +63,7 @@ class FCEHead(nn.Layer): weight_attr=ParamAttr( name='cls_weights', initializer=Normal( - mean=paddle.to_tensor(0.), std=paddle.to_tensor(0.01))), + mean=0., std=0.01)), bias_attr=True) self.out_conv_reg = nn.Conv2D( in_channels=self.in_channels, @@ -75,7 +75,7 @@ class FCEHead(nn.Layer): weight_attr=ParamAttr( name='reg_weights', initializer=Normal( - mean=paddle.to_tensor(0.), std=paddle.to_tensor(0.01))), + mean=0., std=0.01)), bias_attr=True) def forward(self, feats, targets=None): From 7cb52324254f6b91dc9e71e4d45404597bc192a7 Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Tue, 12 Apr 2022 11:44:07 +0000 Subject: [PATCH 19/21] fix --- deploy/cpp_infer/include/ocr_cls.h | 2 +- .../model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/cpp_infer/include/ocr_cls.h b/deploy/cpp_infer/include/ocr_cls.h index 53bb778c18..f5429a7c5b 100644 --- a/deploy/cpp_infer/include/ocr_cls.h +++ b/deploy/cpp_infer/include/ocr_cls.h @@ -57,7 +57,7 @@ public: LoadModel(model_dir); } - double cls_thresh = 0.5; + double cls_thresh = 0.9; // Load Paddle inference model void LoadModel(const std::string &model_dir); diff --git a/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt b/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt index 62993c6a72..d0ae17ccb5 100644 --- a/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt +++ b/test_tipc/configs/ch_ppocr_mobile_v2.0_det/model_linux_gpu_normal_normal_infer_cpp_linux_gpu_cpu.txt @@ -4,7 +4,7 @@ use_opencv:True infer_model:./inference/ch_ppocr_mobile_v2.0_det_infer/ infer_quant:False inference:./deploy/cpp_infer/build/ppocr ---use_gpu:False +--use_gpu:True|False --enable_mkldnn:True|False --cpu_threads:1|6 --rec_batch_num:1 From d50addb819c928c74ce99ce284821803a694f025 Mon Sep 17 00:00:00 2001 From: andyjpaddle Date: Tue, 12 Apr 2022 13:03:51 +0000 Subject: [PATCH 20/21] fix rare trt shape error --- .../rec_r34_vd_tps_bilstm_att_v2.0/train_infer_python.txt | 2 +- tools/infer/predict_rec.py | 4 ++++ tools/infer/utility.py | 4 ++++ 3 files changed, 9 insertions(+), 1 deletion(-) diff --git a/test_tipc/configs/rec_r34_vd_tps_bilstm_att_v2.0/train_infer_python.txt b/test_tipc/configs/rec_r34_vd_tps_bilstm_att_v2.0/train_infer_python.txt index 9cee5d0b7d..55e937881b 100644 --- a/test_tipc/configs/rec_r34_vd_tps_bilstm_att_v2.0/train_infer_python.txt +++ b/test_tipc/configs/rec_r34_vd_tps_bilstm_att_v2.0/train_infer_python.txt @@ -37,7 +37,7 @@ export2:null train_model:./inference/rec_r34_vd_tps_bilstm_att_v2.0_train/best_accuracy infer_export:tools/export_model.py -c test_tipc/configs/rec_r34_vd_tps_bilstm_att_v2.0/rec_r34_vd_tps_bilstm_att.yml -o infer_quant:False -inference:tools/infer/predict_rec.py --rec_char_dict_path=./ppocr/utils/ic15_dict.txt --rec_image_shape="3,32,100" --rec_algorithm="RARE" +inference:tools/infer/predict_rec.py --rec_char_dict_path=./ppocr/utils/ic15_dict.txt --rec_image_shape="3,32,100" --rec_algorithm="RARE" --min_subgraph_size=5 --use_gpu:True|False --enable_mkldnn:True|False --cpu_threads:1|6 diff --git a/tools/infer/predict_rec.py b/tools/infer/predict_rec.py index eebb2b3ba4..c5aacb0600 100755 --- a/tools/infer/predict_rec.py +++ b/tools/infer/predict_rec.py @@ -119,6 +119,10 @@ class TextRecognizer(object): resized_w = imgW else: resized_w = int(math.ceil(imgH * ratio)) + if self.rec_algorithm == 'RARE': + if resized_w > self.rec_image_shape[2]: + resized_w = self.rec_image_shape[2] + imgW = self.rec_image_shape[2] resized_image = cv2.resize(img, (resized_w, imgH)) resized_image = resized_image.astype('float32') resized_image = resized_image.transpose((2, 0, 1)) / 255 diff --git a/tools/infer/utility.py b/tools/infer/utility.py index fd2605d963..b16aecd496 100644 --- a/tools/infer/utility.py +++ b/tools/infer/utility.py @@ -323,6 +323,10 @@ def get_output_tensors(args, mode, predictor): output_name = 'softmax_0.tmp_0' if output_name in output_names: return [predictor.get_output_handle(output_name)] + else: + for output_name in output_names: + output_tensor = predictor.get_output_handle(output_name) + output_tensors.append(output_tensor) else: for output_name in output_names: output_tensor = predictor.get_output_handle(output_name) From 8aea1121452f2cef4117b0d84d44df5faf195728 Mon Sep 17 00:00:00 2001 From: andyjpaddle Date: Wed, 13 Apr 2022 07:06:12 +0000 Subject: [PATCH 21/21] fix srn trt shape error --- .../configs/rec_r50_fpn_vd_none_srn/train_infer_python.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test_tipc/configs/rec_r50_fpn_vd_none_srn/train_infer_python.txt b/test_tipc/configs/rec_r50_fpn_vd_none_srn/train_infer_python.txt index 187c1cc13a..4877512b68 100644 --- a/test_tipc/configs/rec_r50_fpn_vd_none_srn/train_infer_python.txt +++ b/test_tipc/configs/rec_r50_fpn_vd_none_srn/train_infer_python.txt @@ -37,7 +37,7 @@ export2:null train_model:./inference/rec_r50_vd_srn_train/best_accuracy infer_export:tools/export_model.py -c test_tipc/configs/rec_r50_fpn_vd_none_srn/rec_r50_fpn_srn.yml -o infer_quant:False -inference:tools/infer/predict_rec.py --rec_char_dict_path=./ppocr/utils/ic15_dict.txt --rec_image_shape="1,64,256" --rec_algorithm="SRN" --use_space_char=False +inference:tools/infer/predict_rec.py --rec_char_dict_path=./ppocr/utils/ic15_dict.txt --rec_image_shape="1,64,256" --rec_algorithm="SRN" --use_space_char=False --min_subgraph_size=3 --use_gpu:True|False --enable_mkldnn:True|False --cpu_threads:1|6