update can data loading method and tipc configs, revert precommit config

ppocr/data/__init__.py

@@ -37,7 +37,6 @@ from ppocr.data.simple_dataset import SimpleDataSet
 from ppocr.data.lmdb_dataset import LMDBDataSet, LMDBDataSetSR
 from ppocr.data.pgnet_dataset import PGDataSet
 from ppocr.data.pubtab_dataset import PubTabDataSet
-from ppocr.data.hmer_dataset import HMERDataSet
 
 __all__ = ['build_dataloader', 'transform', 'create_operators']
 
@@ -56,7 +55,7 @@ def build_dataloader(config, mode, device, logger, seed=None):
 
     support_dict = [
         'SimpleDataSet', 'LMDBDataSet', 'PGDataSet', 'PubTabDataSet',
-        'LMDBDataSetSR', 'HMERDataSet'
+        'LMDBDataSetSR'
     ]
     module_name = config[mode]['dataset']['name']
     assert module_name in support_dict, Exception(

ppocr/data/collate_fn.py

@@ -95,8 +95,8 @@ class DyMaskCollator(object):
                 1] > max_height else max_height
             max_width = item[0].shape[2] if item[0].shape[
                 2] > max_width else max_width
-            max_length = item[1].shape[0] if item[1].shape[
-                0] > max_length else max_length
+            max_length = len(item[1]) if len(item[
+                1]) > max_length else max_length
             proper_items.append(item)
 
         images, image_masks = np.zeros(
@@ -111,7 +111,7 @@ class DyMaskCollator(object):
             _, h, w = proper_items[i][0].shape
             images[i][:, :h, :w] = proper_items[i][0]
             image_masks[i][:, :h, :w] = 1
-            l = proper_items[i][1].shape[0]
+            l = len(proper_items[i][1])
             labels[i][:l] = proper_items[i][1]
             label_masks[i][:l] = 1
 

ppocr/data/hmer_dataset.py

@@ -1,99 +0,0 @@
-# copyright (c) 2020 PaddlePaddle Authors. All Rights Reserve.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-import os, json, random, traceback
-import numpy as np
-
-from PIL import Image
-from paddle.io import Dataset
-
-from .imaug import transform, create_operators
-
-
-class HMERDataSet(Dataset):
-    def __init__(self, config, mode, logger, seed=None):
-        super(HMERDataSet, self).__init__()
-
-        self.logger = logger
-        self.seed = seed
-        self.mode = mode
-
-        global_config = config['Global']
-        dataset_config = config[mode]['dataset']
-        self.data_dir = config[mode]['dataset']['data_dir']
-
-        label_file_list = dataset_config['label_file_list']
-        data_source_num = len(label_file_list)
-        ratio_list = dataset_config.get("ratio_list", [1.0])
-
-        self.data_lines, self.labels = self.get_image_info_list(label_file_list,
-                                                                ratio_list)
-        self.data_idx_order_list = list(range(len(self.data_lines)))
-        if self.mode == "train" and self.do_shuffle:
-            self.shuffle_data_random()
-
-        if isinstance(ratio_list, (float, int)):
-            ratio_list = [float(ratio_list)] * int(data_source_num)
-
-        assert len(
-            ratio_list
-        ) == data_source_num, "The length of ratio_list should be the same as the file_list."
-
-        self.ops = create_operators(dataset_config['transforms'], global_config)
-        self.need_reset = True in [x < 1 for x in ratio_list]
-
-    def get_image_info_list(self, file_list, ratio_list):
-        if isinstance(file_list, str):
-            file_list = [file_list]
-        labels = {}
-        for idx, file in enumerate(file_list):
-            with open(file, "r") as f:
-                lines = json.load(f)
-                labels.update(lines)
-        data_lines = [name for name in labels.keys()]
-        return data_lines, labels
-
-    def shuffle_data_random(self):
-        random.seed(self.seed)
-        random.shuffle(self.data_lines)
-        return
-
-    def __len__(self):
-        return len(self.data_idx_order_list)
-
-    def __getitem__(self, idx):
-        file_idx = self.data_idx_order_list[idx]
-        data_name = self.data_lines[file_idx]
-        try:
-            file_name = data_name + '.jpg'
-            img_path = os.path.join(self.data_dir, file_name)
-            if not os.path.exists(img_path):
-                raise Exception("{} does not exist!".format(img_path))
-            with open(img_path, 'rb') as f:
-                img = f.read()
-
-            label = self.labels.get(data_name).split()
-            label = np.array([int(item) for item in label])
-
-            data = {'image': img, 'label': label}
-            outs = transform(data, self.ops)
-        except:
-            self.logger.error(
-                "When parsing line {}, error happened with msg: {}".format(
-                    file_name, traceback.format_exc()))
-            outs = None
-        if outs is None:
-            # during evaluation, we should fix the idx to get same results for many times of evaluation.
-            rnd_idx = np.random.randint(self.__len__())
-            return self.__getitem__(rnd_idx)
-        return outs

ppocr/data/imaug/label_ops.py

@@ -1476,4 +1476,33 @@ class CTLabelEncode(object):
 
         data['polys'] = boxes
         data['texts'] = txts
-        return data
+        return data
+
+
+class SeqLabelEncode(BaseRecLabelEncode):
+    def __init__(self,
+                 character_dict_path,
+                 max_text_length=100,
+                 use_space_char=False,
+                 lower=True,
+                 **kwargs):
+        super(SeqLabelEncode, self).__init__(
+            max_text_length, character_dict_path, use_space_char, lower)
+
+    def encode(self, text_seq):
+        text_seq_encoded = []
+        for text in text_seq:
+            if text not in self.character:
+                continue
+            text_seq_encoded.append(self.dict.get(text))
+        if len(text_seq_encoded) == 0:
+            return None
+        return text_seq_encoded
+
+    def __call__(self, data):
+        label = data['label']
+        if isinstance(label, str):
+            label = label.strip().split()
+        label.append(self.end_str)
+        data['label'] = self.encode(label)
+        return data