From f01dbb5648a97af6a3160e5653affe5a0bb3dc4e Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Mon, 20 Dec 2021 14:23:36 +0000 Subject: [PATCH 1/7] add LayoutLM ser --- ppstructure/vqa/README.md | 16 +++++++---- ppstructure/vqa/eval_ser.py | 47 +++++++++++++++++++++--------- ppstructure/vqa/infer_re.py | 9 ++++-- ppstructure/vqa/infer_ser.py | 47 +++++++++++++++++++----------- ppstructure/vqa/infer_ser_e2e.py | 49 +++++++++++++++++++++----------- ppstructure/vqa/losses.py | 35 +++++++++++++++++++++++ ppstructure/vqa/train_ser.py | 48 ++++++++++++++++++++++--------- ppstructure/vqa/utils.py | 2 ++ 8 files changed, 185 insertions(+), 68 deletions(-) create mode 100644 ppstructure/vqa/losses.py diff --git a/ppstructure/vqa/README.md b/ppstructure/vqa/README.md index 2216950e58..51a616c439 100644 --- a/ppstructure/vqa/README.md +++ b/ppstructure/vqa/README.md @@ -18,12 +18,13 @@ PP-Structure 里的 DOC-VQA算法基于PaddleNLP自然语言处理算法库进 ## 1 性能 -我们在 [XFUN](https://github.com/doc-analysis/XFUND) 评估数据集上对算法进行了评估，性能如下 +我们在 [XFUN](https://github.com/doc-analysis/XFUND) 的中文数据集上对算法进行了评估，性能如下 -|任务| f1 | 模型下载地址| -|:---:|:---:| :---:| -|SER|0.9056| [链接](https://paddleocr.bj.bcebos.com/pplayout/PP-Layout_v1.0_ser_pretrained.tar)| -|RE|0.7113| [链接](https://paddleocr.bj.bcebos.com/pplayout/PP-Layout_v1.0_re_pretrained.tar)| +| 模型 | 任务 | f1 | 模型下载地址 | +|:---:|:---:|:---:| :---:| +| LayoutXLM | RE | 0.7113 | [链接](https://paddleocr.bj.bcebos.com/pplayout/PP-Layout_v1.0_re_pretrained.tar) | +| LayoutXLM | SER | 0.9056 | [链接](https://paddleocr.bj.bcebos.com/pplayout/PP-Layout_v1.0_ser_pretrained.tar) | +| LayoutLM | SER | 0.78 | [链接](https://paddleocr.bj.bcebos.com/pplayout/LayoutLM_ser_pretrained.tar) | @@ -135,6 +136,7 @@ wget https://paddleocr.bj.bcebos.com/dataset/XFUND.tar ```shell python3.7 train_ser.py \ --model_name_or_path "layoutxlm-base-uncased" \ + --ser_model_type "LayoutLM" \ --train_data_dir "XFUND/zh_train/image" \ --train_label_path "XFUND/zh_train/xfun_normalize_train.json" \ --eval_data_dir "XFUND/zh_val/image" \ @@ -155,6 +157,7 @@ python3.7 train_ser.py \ ```shell python3.7 train_ser.py \ --model_name_or_path "model_path" \ + --ser_model_type "LayoutXLM" \ --train_data_dir "XFUND/zh_train/image" \ --train_label_path "XFUND/zh_train/xfun_normalize_train.json" \ --eval_data_dir "XFUND/zh_val/image" \ @@ -175,6 +178,7 @@ python3.7 train_ser.py \ export CUDA_VISIBLE_DEVICES=0 python3 eval_ser.py \ --model_name_or_path "PP-Layout_v1.0_ser_pretrained/" \ + --ser_model_type "LayoutXLM" \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ --per_gpu_eval_batch_size 8 \ @@ -190,6 +194,7 @@ python3 eval_ser.py \ export CUDA_VISIBLE_DEVICES=0 python3.7 infer_ser.py \ --model_name_or_path "./PP-Layout_v1.0_ser_pretrained/" \ + --ser_model_type "LayoutXLM" \ --output_dir "output_res/" \ --infer_imgs "XFUND/zh_val/image/" \ --ocr_json_path "XFUND/zh_val/xfun_normalize_val.json" @@ -203,6 +208,7 @@ python3.7 infer_ser.py \ export CUDA_VISIBLE_DEVICES=0 python3.7 infer_ser_e2e.py \ --model_name_or_path "./output/PP-Layout_v1.0_ser_pretrained/" \ + --ser_model_type "LayoutXLM" \ --max_seq_length 512 \ --output_dir "output_res_e2e/" \ --infer_imgs "images/input/zh_val_0.jpg" diff --git a/ppstructure/vqa/eval_ser.py b/ppstructure/vqa/eval_ser.py index acf37452a4..52eeb8a1da 100644 --- a/ppstructure/vqa/eval_ser.py +++ b/ppstructure/vqa/eval_ser.py @@ -29,11 +29,21 @@ import paddle import numpy as np from seqeval.metrics import classification_report, f1_score, precision_score, recall_score from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer, LayoutXLMForTokenClassification +from paddlenlp.transformers import LayoutLMModel, LayoutLMTokenizer, LayoutLMForTokenClassification + from xfun import XFUNDataset +from losses import SERLoss from utils import parse_args, get_bio_label_maps, print_arguments from ppocr.utils.logging import get_logger +MODELS = { + 'LayoutXLM': + (LayoutXLMTokenizer, LayoutXLMModel, LayoutXLMForTokenClassification), + 'LayoutLM': + (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification) +} + def eval(args): logger = get_logger() @@ -42,9 +52,9 @@ def eval(args): label2id_map, id2label_map = get_bio_label_maps(args.label_map_path) pad_token_label_id = paddle.nn.CrossEntropyLoss().ignore_index - tokenizer = LayoutXLMTokenizer.from_pretrained(args.model_name_or_path) - model = LayoutXLMForTokenClassification.from_pretrained( - args.model_name_or_path) + tokenizer_class, base_model_class, model_class = MODELS[args.ser_model_type] + tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path) + model = model_class.from_pretrained(args.model_name_or_path) eval_dataset = XFUNDataset( tokenizer, @@ -65,8 +75,11 @@ def eval(args): use_shared_memory=True, collate_fn=None, ) - results, _ = evaluate(args, model, tokenizer, eval_dataloader, label2id_map, - id2label_map, pad_token_label_id, logger) + loss_class = SERLoss(len(label2id_map)) + + results, _ = evaluate(args, model, tokenizer, loss_class, eval_dataloader, + label2id_map, id2label_map, pad_token_label_id, + logger) logger.info(results) @@ -74,6 +87,7 @@ def eval(args): def evaluate(args, model, tokenizer, + loss_class, eval_dataloader, label2id_map, id2label_map, @@ -88,24 +102,29 @@ def evaluate(args, model.eval() for idx, batch in enumerate(eval_dataloader): with paddle.no_grad(): + if args.ser_model_type == 'LayoutLM': + if 'image' in batch: + batch.pop('image') + labels = batch.pop('labels') outputs = model(**batch) - tmp_eval_loss, logits = outputs[:2] + if args.ser_model_type == 'LayoutXLM': + outputs = outputs[0] + loss = loss_class(labels, outputs, batch['attention_mask']) - tmp_eval_loss = tmp_eval_loss.mean() + loss = loss.mean() if paddle.distributed.get_rank() == 0: logger.info("[Eval]process: {}/{}, loss: {:.5f}".format( - idx, len(eval_dataloader), tmp_eval_loss.numpy()[0])) + idx, len(eval_dataloader), loss.numpy()[0])) - eval_loss += tmp_eval_loss.item() + eval_loss += loss.item() nb_eval_steps += 1 if preds is None: - preds = logits.numpy() - out_label_ids = batch["labels"].numpy() + preds = outputs.numpy() + out_label_ids = labels.numpy() else: - preds = np.append(preds, logits.numpy(), axis=0) - out_label_ids = np.append( - out_label_ids, batch["labels"].numpy(), axis=0) + preds = np.append(preds, outputs.numpy(), axis=0) + out_label_ids = np.append(out_label_ids, labels.numpy(), axis=0) eval_loss = eval_loss / nb_eval_steps preds = np.argmax(preds, axis=2) diff --git a/ppstructure/vqa/infer_re.py b/ppstructure/vqa/infer_re.py index 139ade9aae..2ffa60f5d6 100644 --- a/ppstructure/vqa/infer_re.py +++ b/ppstructure/vqa/infer_re.py @@ -56,7 +56,11 @@ def infer(args): ocr_info_list = load_ocr(args.eval_data_dir, args.eval_label_path) for idx, batch in enumerate(eval_dataloader): - logger.info("[Infer] process: {}/{}".format(idx, len(eval_dataloader))) + save_img_path = os.path.join( + args.output_dir, + os.path.splitext(os.path.basename(img_path))[0] + "_re.jpg") + logger.info("[Infer] process: {}/{}, save_result to {}".format( + idx, len(eval_dataloader), save_img_path)) with paddle.no_grad(): outputs = model(**batch) pred_relations = outputs['pred_relations'] @@ -85,8 +89,7 @@ def infer(args): img = cv2.imread(image_path) img_show = draw_re_results(img, result) - save_path = os.path.join(args.output_dir, os.path.basename(image_path)) - cv2.imwrite(save_path, img_show) + cv2.imwrite(save_img_path, img_show) def load_ocr(img_folder, json_path): diff --git a/ppstructure/vqa/infer_ser.py b/ppstructure/vqa/infer_ser.py index 3c7ad03aa3..2b715d7b92 100644 --- a/ppstructure/vqa/infer_ser.py +++ b/ppstructure/vqa/infer_ser.py @@ -24,6 +24,14 @@ import paddle # relative reference from utils import parse_args, get_image_file_list, draw_ser_results, get_bio_label_maps from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer, LayoutXLMForTokenClassification +from paddlenlp.transformers import LayoutLMModel, LayoutLMTokenizer, LayoutLMForTokenClassification + +MODELS = { + 'LayoutXLM': + (LayoutXLMTokenizer, LayoutXLMModel, LayoutXLMForTokenClassification), + 'LayoutLM': + (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification) +} def pad_sentences(tokenizer, @@ -217,10 +225,10 @@ def infer(args): os.makedirs(args.output_dir, exist_ok=True) # init token and model - tokenizer = LayoutXLMTokenizer.from_pretrained(args.model_name_or_path) - # model = LayoutXLMModel.from_pretrained(args.model_name_or_path) - model = LayoutXLMForTokenClassification.from_pretrained( - args.model_name_or_path) + tokenizer_class, base_model_class, model_class = MODELS[args.ser_model_type] + tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path) + model = model_class.from_pretrained(args.model_name_or_path) + model.eval() # load ocr results json @@ -240,7 +248,10 @@ def infer(args): "w", encoding='utf-8') as fout: for idx, img_path in enumerate(infer_imgs): - print("process: [{}/{}]".format(idx, len(infer_imgs), img_path)) + save_img_path = os.path.join(args.output_dir, + os.path.basename(img_path)) + print("process: [{}/{}], save_result to {}".format( + idx, len(infer_imgs), save_img_path)) img = cv2.imread(img_path) @@ -250,15 +261,21 @@ def infer(args): ori_img=img, ocr_info=ocr_info, max_seq_len=args.max_seq_length) + if args.ser_model_type == 'LayoutLM': + preds = model( + input_ids=inputs["input_ids"], + bbox=inputs["bbox"], + token_type_ids=inputs["token_type_ids"], + attention_mask=inputs["attention_mask"]) + elif args.ser_model_type == 'LayoutXLM': + preds = model( + input_ids=inputs["input_ids"], + bbox=inputs["bbox"], + image=inputs["image"], + token_type_ids=inputs["token_type_ids"], + attention_mask=inputs["attention_mask"]) + preds = preds[0] - outputs = model( - input_ids=inputs["input_ids"], - bbox=inputs["bbox"], - image=inputs["image"], - token_type_ids=inputs["token_type_ids"], - attention_mask=inputs["attention_mask"]) - - preds = outputs[0] preds = postprocess(inputs["attention_mask"], preds, args.label_map_path) ocr_info = merge_preds_list_with_ocr_info( @@ -271,9 +288,7 @@ def infer(args): }, ensure_ascii=False) + "\n") img_res = draw_ser_results(img, ocr_info) - cv2.imwrite( - os.path.join(args.output_dir, os.path.basename(img_path)), - img_res) + cv2.imwrite(save_img_path, img_res) return diff --git a/ppstructure/vqa/infer_ser_e2e.py b/ppstructure/vqa/infer_ser_e2e.py index a93a98ff6d..05a029822b 100644 --- a/ppstructure/vqa/infer_ser_e2e.py +++ b/ppstructure/vqa/infer_ser_e2e.py @@ -22,12 +22,20 @@ from PIL import Image import paddle from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer, LayoutXLMForTokenClassification +from paddlenlp.transformers import LayoutLMModel, LayoutLMTokenizer, LayoutLMForTokenClassification # relative reference from utils import parse_args, get_image_file_list, draw_ser_results, get_bio_label_maps from utils import pad_sentences, split_page, preprocess, postprocess, merge_preds_list_with_ocr_info +MODELS = { + 'LayoutXLM': + (LayoutXLMTokenizer, LayoutXLMModel, LayoutXLMForTokenClassification), + 'LayoutLM': + (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification) +} + def trans_poly_to_bbox(poly): x1 = np.min([p[0] for p in poly]) @@ -50,14 +58,15 @@ def parse_ocr_info_for_ser(ocr_result): class SerPredictor(object): def __init__(self, args): - + self.args = args self.max_seq_length = args.max_seq_length # init ser token and model - self.tokenizer = LayoutXLMTokenizer.from_pretrained( - args.model_name_or_path) - self.model = LayoutXLMForTokenClassification.from_pretrained( + tokenizer_class, base_model_class, model_class = MODELS[ + args.ser_model_type] + self.tokenizer = tokenizer_class.from_pretrained( args.model_name_or_path) + self.model = model_class.from_pretrained(args.model_name_or_path) self.model.eval() # init ocr_engine @@ -89,14 +98,21 @@ class SerPredictor(object): ocr_info=ocr_info, max_seq_len=self.max_seq_length) - outputs = self.model( - input_ids=inputs["input_ids"], - bbox=inputs["bbox"], - image=inputs["image"], - token_type_ids=inputs["token_type_ids"], - attention_mask=inputs["attention_mask"]) + if args.ser_model_type == 'LayoutLM': + preds = self.model( + input_ids=inputs["input_ids"], + bbox=inputs["bbox"], + token_type_ids=inputs["token_type_ids"], + attention_mask=inputs["attention_mask"]) + elif args.ser_model_type == 'LayoutXLM': + preds = self.model( + input_ids=inputs["input_ids"], + bbox=inputs["bbox"], + image=inputs["image"], + token_type_ids=inputs["token_type_ids"], + attention_mask=inputs["attention_mask"]) + preds = preds[0] - preds = outputs[0] preds = postprocess(inputs["attention_mask"], preds, self.id2label_map) ocr_info = merge_preds_list_with_ocr_info( ocr_info, inputs["segment_offset_id"], preds, @@ -118,7 +134,11 @@ if __name__ == "__main__": "w", encoding='utf-8') as fout: for idx, img_path in enumerate(infer_imgs): - print("process: [{}/{}], {}".format(idx, len(infer_imgs), img_path)) + save_img_path = os.path.join( + args.output_dir, + os.path.splitext(os.path.basename(img_path))[0] + "_ser.jpg") + print("process: [{}/{}], save_result to {}".format( + idx, len(infer_imgs), save_img_path)) img = cv2.imread(img_path) @@ -129,7 +149,4 @@ if __name__ == "__main__": }, ensure_ascii=False) + "\n") img_res = draw_ser_results(img, result) - cv2.imwrite( - os.path.join(args.output_dir, - os.path.splitext(os.path.basename(img_path))[0] + - "_ser.jpg"), img_res) + cv2.imwrite(save_img_path, img_res) diff --git a/ppstructure/vqa/losses.py b/ppstructure/vqa/losses.py new file mode 100644 index 0000000000..e8dad01c31 --- /dev/null +++ b/ppstructure/vqa/losses.py @@ -0,0 +1,35 @@ +# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from paddle import nn + + +class SERLoss(nn.Layer): + def __init__(self, num_classes): + super().__init__() + self.loss_class = nn.CrossEntropyLoss() + self.num_classes = num_classes + self.ignore_index = self.loss_class.ignore_index + + def forward(self, labels, outputs, attention_mask): + if attention_mask is not None: + active_loss = attention_mask.reshape([-1, ]) == 1 + active_outputs = outputs.reshape( + [-1, self.num_classes])[active_loss] + active_labels = labels.reshape([-1, ])[active_loss] + loss = self.loss_class(active_outputs, active_labels) + else: + loss = self.loss_class( + outputs.reshape([-1, self.num_classes]), labels.reshape([-1, ])) + return loss diff --git a/ppstructure/vqa/train_ser.py b/ppstructure/vqa/train_ser.py index 6791cea89a..2670ef9eea 100644 --- a/ppstructure/vqa/train_ser.py +++ b/ppstructure/vqa/train_ser.py @@ -29,11 +29,21 @@ import paddle import numpy as np from seqeval.metrics import classification_report, f1_score, precision_score, recall_score from paddlenlp.transformers import LayoutXLMModel, LayoutXLMTokenizer, LayoutXLMForTokenClassification +from paddlenlp.transformers import LayoutLMModel, LayoutLMTokenizer, LayoutLMForTokenClassification + from xfun import XFUNDataset from utils import parse_args, get_bio_label_maps, print_arguments, set_seed from eval_ser import evaluate +from losses import SERLoss from ppocr.utils.logging import get_logger +MODELS = { + 'LayoutXLM': + (LayoutXLMTokenizer, LayoutXLMModel, LayoutXLMForTokenClassification), + 'LayoutLM': + (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification) +} + def train(args): os.makedirs(args.output_dir, exist_ok=True) @@ -44,22 +54,24 @@ def train(args): print_arguments(args, logger) label2id_map, id2label_map = get_bio_label_maps(args.label_map_path) - pad_token_label_id = paddle.nn.CrossEntropyLoss().ignore_index + loss_class = SERLoss(len(label2id_map)) + + pad_token_label_id = loss_class.ignore_index # dist mode if distributed: paddle.distributed.init_parallel_env() - tokenizer = LayoutXLMTokenizer.from_pretrained(args.model_name_or_path) + tokenizer_class, base_model_class, model_class = MODELS[args.ser_model_type] + tokenizer = tokenizer_class.from_pretrained(args.model_name_or_path) if not args.resume: - model = LayoutXLMModel.from_pretrained(args.model_name_or_path) - model = LayoutXLMForTokenClassification( - model, num_classes=len(label2id_map), dropout=None) + base_model = base_model_class.from_pretrained(args.model_name_or_path) + model = model_class( + base_model, num_classes=len(label2id_map), dropout=None) logger.info('train from scratch') else: logger.info('resume from {}'.format(args.model_name_or_path)) - model = LayoutXLMForTokenClassification.from_pretrained( - args.model_name_or_path) + model = model_class.from_pretrained(args.model_name_or_path) # dist mode if distributed: @@ -153,12 +165,19 @@ def train(args): for step, batch in enumerate(train_dataloader): train_reader_cost += time.time() - reader_start + if args.ser_model_type == 'LayoutLM': + if 'image' in batch: + batch.pop('image') + labels = batch.pop('labels') + train_start = time.time() outputs = model(**batch) train_run_cost += time.time() - train_start + if args.ser_model_type == 'LayoutXLM': + outputs = outputs[0] + loss = loss_class(labels, outputs, batch['attention_mask']) # model outputs are always tuple in ppnlp (see doc) - loss = outputs[0] loss = loss.mean() loss.backward() tr_loss += loss.item() @@ -166,7 +185,7 @@ def train(args): lr_scheduler.step() # Update learning rate schedule optimizer.clear_grad() global_step += 1 - total_samples += batch['image'].shape[0] + total_samples += batch['input_ids'].shape[0] if rank == 0 and step % print_step == 0: logger.info( @@ -186,9 +205,9 @@ def train(args): if rank == 0 and args.eval_steps > 0 and global_step % args.eval_steps == 0 and args.evaluate_during_training: # Log metrics # Only evaluate when single GPU otherwise metrics may not average well - results, _ = evaluate(args, model, tokenizer, eval_dataloader, - label2id_map, id2label_map, - pad_token_label_id, logger) + results, _ = evaluate(args, model, tokenizer, loss_class, + eval_dataloader, label2id_map, + id2label_map, pad_token_label_id, logger) if best_metrics is None or results["f1"] >= best_metrics["f1"]: best_metrics = copy.deepcopy(results) @@ -201,7 +220,8 @@ def train(args): tokenizer.save_pretrained(output_dir) paddle.save(args, os.path.join(output_dir, "training_args.bin")) - logger.info("Saving model checkpoint to %s", output_dir) + logger.info("Saving model checkpoint to {}".format( + output_dir)) logger.info("[epoch {}/{}][iter: {}/{}] results: {}".format( epoch_id, args.num_train_epochs, step, @@ -219,7 +239,7 @@ def train(args): model.save_pretrained(output_dir) tokenizer.save_pretrained(output_dir) paddle.save(args, os.path.join(output_dir, "training_args.bin")) - logger.info("Saving model checkpoint to %s", output_dir) + logger.info("Saving model checkpoint to {}".format(output_dir)) return global_step, tr_loss / global_step diff --git a/ppstructure/vqa/utils.py b/ppstructure/vqa/utils.py index 414317c0d1..b9f2edc860 100644 --- a/ppstructure/vqa/utils.py +++ b/ppstructure/vqa/utils.py @@ -350,6 +350,8 @@ def parse_args(): # yapf: disable parser.add_argument("--model_name_or_path", default=None, type=str, required=True,) + parser.add_argument("--ser_model_type", + default='LayoutXLM', type=str) parser.add_argument("--re_model_name_or_path", default=None, type=str, required=False,) parser.add_argument("--train_data_dir", default=None, From 9131c4a7acc12ba79c7a60aef72a94abb5d730eb Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Mon, 20 Dec 2021 14:38:49 +0000 Subject: [PATCH 2/7] add LayoutLM ser --- ppstructure/vqa/README.md | 10 ++--- ppstructure/vqa/infer.sh | 61 +++++++++++++++++++++++++++++ ppstructure/vqa/infer_re.py | 10 ++--- ppstructure/vqa/infer_ser_e2e.py | 4 +- ppstructure/vqa/infer_ser_re_e2e.py | 11 +++--- 5 files changed, 79 insertions(+), 17 deletions(-) create mode 100644 ppstructure/vqa/infer.sh diff --git a/ppstructure/vqa/README.md b/ppstructure/vqa/README.md index 51a616c439..975139c793 100644 --- a/ppstructure/vqa/README.md +++ b/ppstructure/vqa/README.md @@ -195,7 +195,7 @@ export CUDA_VISIBLE_DEVICES=0 python3.7 infer_ser.py \ --model_name_or_path "./PP-Layout_v1.0_ser_pretrained/" \ --ser_model_type "LayoutXLM" \ - --output_dir "output_res/" \ + --output_dir "output/ser/" \ --infer_imgs "XFUND/zh_val/image/" \ --ocr_json_path "XFUND/zh_val/xfun_normalize_val.json" ``` @@ -210,7 +210,7 @@ python3.7 infer_ser_e2e.py \ --model_name_or_path "./output/PP-Layout_v1.0_ser_pretrained/" \ --ser_model_type "LayoutXLM" \ --max_seq_length 512 \ - --output_dir "output_res_e2e/" \ + --output_dir "output/ser_e2e/" \ --infer_imgs "images/input/zh_val_0.jpg" ``` @@ -284,7 +284,7 @@ python3 eval_re.py \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ --label_map_path 'labels/labels_ser.txt' \ - --output_dir "output/re_test/" \ + --output_dir "output/re/" \ --per_gpu_eval_batch_size 8 \ --num_workers 8 \ --seed 2048 @@ -302,7 +302,7 @@ python3 infer_re.py \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ --label_map_path 'labels/labels_ser.txt' \ - --output_dir "output_res" \ + --output_dir "output/re/" \ --per_gpu_eval_batch_size 1 \ --seed 2048 ``` @@ -317,7 +317,7 @@ python3.7 infer_ser_re_e2e.py \ --model_name_or_path "./PP-Layout_v1.0_ser_pretrained/" \ --re_model_name_or_path "./PP-Layout_v1.0_re_pretrained/" \ --max_seq_length 512 \ - --output_dir "output_ser_re_e2e_train/" \ + --output_dir "output/ser_re_e2e/" \ --infer_imgs "images/input/zh_val_21.jpg" ``` diff --git a/ppstructure/vqa/infer.sh b/ppstructure/vqa/infer.sh new file mode 100644 index 0000000000..2cd1cea447 --- /dev/null +++ b/ppstructure/vqa/infer.sh @@ -0,0 +1,61 @@ +export CUDA_VISIBLE_DEVICES=6 +# python3.7 infer_ser_e2e.py \ +# --model_name_or_path "output/ser_distributed/best_model" \ +# --max_seq_length 512 \ +# --output_dir "output_res_e2e/" \ +# --infer_imgs "/ssd1/zhoujun20/VQA/data/XFUN_v1.0_data/zh.val/zh_val_0.jpg" + + +# python3.7 infer_ser_re_e2e.py \ +# --model_name_or_path "output/ser_distributed/best_model" \ +# --re_model_name_or_path "output/re_test/best_model" \ +# --max_seq_length 512 \ +# --output_dir "output_ser_re_e2e_train/" \ +# --infer_imgs "images/input/zh_val_21.jpg" + +# python3.7 infer_ser.py \ +# --model_name_or_path "output/ser_LayoutLM/best_model" \ +# --ser_model_type "LayoutLM" \ +# --output_dir "ser_LayoutLM/" \ +# --infer_imgs "images/input/zh_val_21.jpg" \ +# --ocr_json_path "/ssd1/zhoujun20/VQA/data/XFUN_v1.0_data/xfun_normalize_val.json" + +python3.7 infer_ser.py \ + --model_name_or_path "output/ser_new/best_model" \ + --ser_model_type "LayoutXLM" \ + --output_dir "ser_new/" \ + --infer_imgs "images/input/zh_val_21.jpg" \ + --ocr_json_path "/ssd1/zhoujun20/VQA/data/XFUN_v1.0_data/xfun_normalize_val.json" + +# python3.7 infer_ser_e2e.py \ +# --model_name_or_path "output/ser_new/best_model" \ +# --ser_model_type "LayoutXLM" \ +# --max_seq_length 512 \ +# --output_dir "output/ser_new/" \ +# --infer_imgs "images/input/zh_val_0.jpg" + + +# python3.7 infer_ser_e2e.py \ +# --model_name_or_path "output/ser_LayoutLM/best_model" \ +# --ser_model_type "LayoutLM" \ +# --max_seq_length 512 \ +# --output_dir "output/ser_LayoutLM/" \ +# --infer_imgs "images/input/zh_val_0.jpg" + +# python3 infer_re.py \ +# --model_name_or_path "/ssd1/zhoujun20/VQA/PaddleOCR/ppstructure/vqa/output/re_test/best_model/" \ +# --max_seq_length 512 \ +# --eval_data_dir "/ssd1/zhoujun20/VQA/data/XFUN_v1.0_data/zh.val" \ +# --eval_label_path "/ssd1/zhoujun20/VQA/data/XFUN_v1.0_data/xfun_normalize_val.json" \ +# --label_map_path 'labels/labels_ser.txt' \ +# --output_dir "output_res" \ +# --per_gpu_eval_batch_size 1 \ +# --seed 2048 + +# python3.7 infer_ser_re_e2e.py \ +# --model_name_or_path "output/ser_LayoutLM/best_model" \ +# --ser_model_type "LayoutLM" \ +# --re_model_name_or_path "output/re_new/best_model" \ +# --max_seq_length 512 \ +# --output_dir "output_ser_re_e2e/" \ +# --infer_imgs "images/input/zh_val_21.jpg" \ No newline at end of file diff --git a/ppstructure/vqa/infer_re.py b/ppstructure/vqa/infer_re.py index 2ffa60f5d6..98c61bacce 100644 --- a/ppstructure/vqa/infer_re.py +++ b/ppstructure/vqa/infer_re.py @@ -56,19 +56,19 @@ def infer(args): ocr_info_list = load_ocr(args.eval_data_dir, args.eval_label_path) for idx, batch in enumerate(eval_dataloader): + ocr_info = ocr_info_list[idx] + image_path = ocr_info['image_path'] + ocr_info = ocr_info['ocr_info'] + save_img_path = os.path.join( args.output_dir, - os.path.splitext(os.path.basename(img_path))[0] + "_re.jpg") + os.path.splitext(os.path.basename(image_path))[0] + "_re.jpg") logger.info("[Infer] process: {}/{}, save_result to {}".format( idx, len(eval_dataloader), save_img_path)) with paddle.no_grad(): outputs = model(**batch) pred_relations = outputs['pred_relations'] - ocr_info = ocr_info_list[idx] - image_path = ocr_info['image_path'] - ocr_info = ocr_info['ocr_info'] - # 根据entity里的信息，做token解码后去过滤不要的ocr_info ocr_info = filter_bg_by_txt(ocr_info, batch, tokenizer) diff --git a/ppstructure/vqa/infer_ser_e2e.py b/ppstructure/vqa/infer_ser_e2e.py index 05a029822b..bceb3434b5 100644 --- a/ppstructure/vqa/infer_ser_e2e.py +++ b/ppstructure/vqa/infer_ser_e2e.py @@ -98,13 +98,13 @@ class SerPredictor(object): ocr_info=ocr_info, max_seq_len=self.max_seq_length) - if args.ser_model_type == 'LayoutLM': + if self.args.ser_model_type == 'LayoutLM': preds = self.model( input_ids=inputs["input_ids"], bbox=inputs["bbox"], token_type_ids=inputs["token_type_ids"], attention_mask=inputs["attention_mask"]) - elif args.ser_model_type == 'LayoutXLM': + elif self.args.ser_model_type == 'LayoutXLM': preds = self.model( input_ids=inputs["input_ids"], bbox=inputs["bbox"], diff --git a/ppstructure/vqa/infer_ser_re_e2e.py b/ppstructure/vqa/infer_ser_re_e2e.py index 23737406d1..a6316b625c 100644 --- a/ppstructure/vqa/infer_ser_re_e2e.py +++ b/ppstructure/vqa/infer_ser_re_e2e.py @@ -117,7 +117,11 @@ if __name__ == "__main__": "w", encoding='utf-8') as fout: for idx, img_path in enumerate(infer_imgs): - print("process: [{}/{}], {}".format(idx, len(infer_imgs), img_path)) + save_img_path = os.path.join( + args.output_dir, + os.path.splitext(os.path.basename(img_path))[0] + "_re.jpg") + print("process: [{}/{}], save_result to {}".format( + idx, len(infer_imgs), save_img_path)) img = cv2.imread(img_path) @@ -128,7 +132,4 @@ if __name__ == "__main__": }, ensure_ascii=False) + "\n") img_res = draw_re_results(img, result) - cv2.imwrite( - os.path.join(args.output_dir, - os.path.splitext(os.path.basename(img_path))[0] + - "_re.jpg"), img_res) + cv2.imwrite(save_img_path, img_res) From 11d6814f365b2e881adbdedc9865f86dae3022a9 Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Mon, 20 Dec 2021 14:47:43 +0000 Subject: [PATCH 3/7] rm _ --- ppstructure/vqa/infer_re.py | 2 +- ppstructure/vqa/infer_ser.py | 2 +- ppstructure/vqa/infer_ser_e2e.py | 2 +- ppstructure/vqa/infer_ser_re_e2e.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ppstructure/vqa/infer_re.py b/ppstructure/vqa/infer_re.py index 98c61bacce..7937700a78 100644 --- a/ppstructure/vqa/infer_re.py +++ b/ppstructure/vqa/infer_re.py @@ -63,7 +63,7 @@ def infer(args): save_img_path = os.path.join( args.output_dir, os.path.splitext(os.path.basename(image_path))[0] + "_re.jpg") - logger.info("[Infer] process: {}/{}, save_result to {}".format( + logger.info("[Infer] process: {}/{}, save result to {}".format( idx, len(eval_dataloader), save_img_path)) with paddle.no_grad(): outputs = model(**batch) diff --git a/ppstructure/vqa/infer_ser.py b/ppstructure/vqa/infer_ser.py index 2b715d7b92..7994b5449a 100644 --- a/ppstructure/vqa/infer_ser.py +++ b/ppstructure/vqa/infer_ser.py @@ -250,7 +250,7 @@ def infer(args): for idx, img_path in enumerate(infer_imgs): save_img_path = os.path.join(args.output_dir, os.path.basename(img_path)) - print("process: [{}/{}], save_result to {}".format( + print("process: [{}/{}], save result to {}".format( idx, len(infer_imgs), save_img_path)) img = cv2.imread(img_path) diff --git a/ppstructure/vqa/infer_ser_e2e.py b/ppstructure/vqa/infer_ser_e2e.py index bceb3434b5..6bb0247501 100644 --- a/ppstructure/vqa/infer_ser_e2e.py +++ b/ppstructure/vqa/infer_ser_e2e.py @@ -137,7 +137,7 @@ if __name__ == "__main__": save_img_path = os.path.join( args.output_dir, os.path.splitext(os.path.basename(img_path))[0] + "_ser.jpg") - print("process: [{}/{}], save_result to {}".format( + print("process: [{}/{}], save result to {}".format( idx, len(infer_imgs), save_img_path)) img = cv2.imread(img_path) diff --git a/ppstructure/vqa/infer_ser_re_e2e.py b/ppstructure/vqa/infer_ser_re_e2e.py index a6316b625c..32d8850a16 100644 --- a/ppstructure/vqa/infer_ser_re_e2e.py +++ b/ppstructure/vqa/infer_ser_re_e2e.py @@ -120,7 +120,7 @@ if __name__ == "__main__": save_img_path = os.path.join( args.output_dir, os.path.splitext(os.path.basename(img_path))[0] + "_re.jpg") - print("process: [{}/{}], save_result to {}".format( + print("process: [{}/{}], save result to {}".format( idx, len(infer_imgs), save_img_path)) img = cv2.imread(img_path) From 113c9bb565b89806e05160c2c275efda8268fe61 Mon Sep 17 00:00:00 2001 From: zhoujun <572459439@qq.com> Date: Mon, 20 Dec 2021 17:57:32 -0600 Subject: [PATCH 4/7] Update README.md --- ppstructure/vqa/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ppstructure/vqa/README.md b/ppstructure/vqa/README.md index 975139c793..35aa1d2608 100644 --- a/ppstructure/vqa/README.md +++ b/ppstructure/vqa/README.md @@ -136,7 +136,7 @@ wget https://paddleocr.bj.bcebos.com/dataset/XFUND.tar ```shell python3.7 train_ser.py \ --model_name_or_path "layoutxlm-base-uncased" \ - --ser_model_type "LayoutLM" \ + --ser_model_type "LayoutXLM" \ --train_data_dir "XFUND/zh_train/image" \ --train_label_path "XFUND/zh_train/xfun_normalize_train.json" \ --eval_data_dir "XFUND/zh_val/image" \ @@ -316,6 +316,7 @@ export CUDA_VISIBLE_DEVICES=0 python3.7 infer_ser_re_e2e.py \ --model_name_or_path "./PP-Layout_v1.0_ser_pretrained/" \ --re_model_name_or_path "./PP-Layout_v1.0_re_pretrained/" \ + --ser_model_type "LayoutXLM" \ --max_seq_length 512 \ --output_dir "output/ser_re_e2e/" \ --infer_imgs "images/input/zh_val_21.jpg" From 1e9fe08f173499d2e293df4f32d8f000aec14b37 Mon Sep 17 00:00:00 2001 From: zhoujun <572459439@qq.com> Date: Mon, 20 Dec 2021 17:59:39 -0600 Subject: [PATCH 5/7] Update README.md --- ppstructure/vqa/README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/ppstructure/vqa/README.md b/ppstructure/vqa/README.md index 35aa1d2608..708f0ea4cf 100644 --- a/ppstructure/vqa/README.md +++ b/ppstructure/vqa/README.md @@ -193,7 +193,7 @@ python3 eval_ser.py \ ```shell export CUDA_VISIBLE_DEVICES=0 python3.7 infer_ser.py \ - --model_name_or_path "./PP-Layout_v1.0_ser_pretrained/" \ + --model_name_or_path "PP-Layout_v1.0_ser_pretrained/" \ --ser_model_type "LayoutXLM" \ --output_dir "output/ser/" \ --infer_imgs "XFUND/zh_val/image/" \ @@ -207,7 +207,7 @@ python3.7 infer_ser.py \ ```shell export CUDA_VISIBLE_DEVICES=0 python3.7 infer_ser_e2e.py \ - --model_name_or_path "./output/PP-Layout_v1.0_ser_pretrained/" \ + --model_name_or_path "PP-Layout_v1.0_ser_pretrained/" \ --ser_model_type "LayoutXLM" \ --max_seq_length 512 \ --output_dir "output/ser_e2e/" \ @@ -279,7 +279,7 @@ python3 train_re.py \ ```shell export CUDA_VISIBLE_DEVICES=0 python3 eval_re.py \ - --model_name_or_path "output/check/checkpoint-best" \ + --model_name_or_path "PP-Layout_v1.0_re_pretrained/" \ --max_seq_length 512 \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ @@ -297,7 +297,7 @@ python3 eval_re.py \ ```shell export CUDA_VISIBLE_DEVICES=0 python3 infer_re.py \ - --model_name_or_path "./PP-Layout_v1.0_re_pretrained/" \ + --model_name_or_path "PP-Layout_v1.0_re_pretrained/" \ --max_seq_length 512 \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ @@ -314,8 +314,8 @@ python3 infer_re.py \ ```shell export CUDA_VISIBLE_DEVICES=0 python3.7 infer_ser_re_e2e.py \ - --model_name_or_path "./PP-Layout_v1.0_ser_pretrained/" \ - --re_model_name_or_path "./PP-Layout_v1.0_re_pretrained/" \ + --model_name_or_path "PP-Layout_v1.0_ser_pretrained/" \ + --re_model_name_or_path "PP-Layout_v1.0_re_pretrained/" \ --ser_model_type "LayoutXLM" \ --max_seq_length 512 \ --output_dir "output/ser_re_e2e/" \ From 5feb969ec4b6285baa441e9211814cbe648ddc52 Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Wed, 22 Dec 2021 06:31:21 +0000 Subject: [PATCH 6/7] update readme --- ppstructure/vqa/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ppstructure/vqa/README.md b/ppstructure/vqa/README.md index 708f0ea4cf..4cf2432f40 100644 --- a/ppstructure/vqa/README.md +++ b/ppstructure/vqa/README.md @@ -234,7 +234,7 @@ python3 train_re.py \ --train_label_path "XFUND/zh_train/xfun_normalize_train.json" \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ - --label_map_path 'labels/labels_ser.txt' \ + --label_map_path "labels/labels_ser.txt" \ --num_train_epochs 200 \ --eval_steps 10 \ --output_dir "output/re/" \ @@ -258,7 +258,7 @@ python3 train_re.py \ --train_label_path "XFUND/zh_train/xfun_normalize_train.json" \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ - --label_map_path 'labels/labels_ser.txt' \ + --label_map_path "labels/labels_ser.txt" \ --num_train_epochs 2 \ --eval_steps 10 \ --output_dir "output/re/" \ @@ -283,7 +283,7 @@ python3 eval_re.py \ --max_seq_length 512 \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ - --label_map_path 'labels/labels_ser.txt' \ + --label_map_path "labels/labels_ser.txt" \ --output_dir "output/re/" \ --per_gpu_eval_batch_size 8 \ --num_workers 8 \ @@ -301,7 +301,7 @@ python3 infer_re.py \ --max_seq_length 512 \ --eval_data_dir "XFUND/zh_val/image" \ --eval_label_path "XFUND/zh_val/xfun_normalize_val.json" \ - --label_map_path 'labels/labels_ser.txt' \ + --label_map_path "labels/labels_ser.txt" \ --output_dir "output/re/" \ --per_gpu_eval_batch_size 1 \ --seed 2048 From 46efa8a127df51789b356729749dbd7866c0c351 Mon Sep 17 00:00:00 2001 From: WenmuZhou <572459439@qq.com> Date: Wed, 22 Dec 2021 06:49:41 +0000 Subject: [PATCH 7/7] rm unused code --- ppstructure/vqa/infer_ser_e2e.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/ppstructure/vqa/infer_ser_e2e.py b/ppstructure/vqa/infer_ser_e2e.py index a90680dafe..33fe4dbb5e 100644 --- a/ppstructure/vqa/infer_ser_e2e.py +++ b/ppstructure/vqa/infer_ser_e2e.py @@ -40,13 +40,6 @@ MODELS = { (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification) } -MODELS = { - 'LayoutXLM': - (LayoutXLMTokenizer, LayoutXLMModel, LayoutXLMForTokenClassification), - 'LayoutLM': - (LayoutLMTokenizer, LayoutLMModel, LayoutLMForTokenClassification) -} - def trans_poly_to_bbox(poly): x1 = np.min([p[0] for p in poly])