fix: support accented characters in word segmentation for return_word… (#17201)

* fix: support accented characters in word segmentation for return_word_box

Fixes #17156

The word segmentation in get_word_info() was using [a-zA-Z0-9] regex which
only matched ASCII letters and digits. This caused words with accented
characters (ä, ö, ü, é, à, etc.) to be incorrectly split into separate
segments.

Changed to use \w with re.UNICODE flag which properly matches:
- All Unicode letter characters (including accented/diacritic characters)
- Digits from all scripts
- Excludes underscore (which \w includes but we want as splitter)

This fix enables proper word grouping for German, French, Polish, and
other languages with accented characters while maintaining backward
compatibility with existing ASCII text processing.

Example: 'Grüßen' now stays as one word instead of ['Gr', 'üß', 'en']

* fix: resolve pytest warning by using assert instead of return

ppocr/postprocess/rec_postprocess.py

@@ -18,35 +18,6 @@ import paddle
 from paddle.nn import functional as F
 import re
 import json
-import unicodedata
-
-
-def is_latin_char(char):
-    """
-    Check if a character is a Latin letter (including accented characters).
-    This will properly categorize accented characters like é, è, à, ç, etc.
-    """
-    try:
-        # Get the Unicode category
-        category = unicodedata.category(char)
-        # Lu = Letter, uppercase
-        # Ll = Letter, lowercase
-        # Lt = Letter, titlecase
-        # Lm = Letter, modifier (includes some Latin extended characters)
-        if not category.startswith("L"):
-            return False
-
-        # Check if the character name starts with LATIN
-        # This covers all Latin-based characters including:
-        # - LATIN SMALL LETTER E WITH ACUTE (é)
-        # - LATIN SMALL LETTER A WITH GRAVE (à)
-        # - LATIN SMALL LETTER C WITH CEDILLA (ç)
-        # - LATIN SMALL LETTER E WITH CIRCUMFLEX (ê)
-        # - etc.
-        char_name = unicodedata.name(char, "")
-        return char_name.startswith("LATIN")
-    except ValueError:
-        return False
 
 
 class BaseRecLabelDecode(object):
@@ -124,8 +95,9 @@ class BaseRecLabelDecode(object):
         for c_i, char in enumerate(text):
             if "\u4e00" <= char <= "\u9fff":
                 c_state = "cn"
-            # Modified condition to include accented characters used in French and other Latin-based languages
-            elif bool(re.search("[a-zA-Z0-9]", char)) or is_latin_char(char):
+            # Use \w with UNICODE flag to match letters (including accented chars like ä, ö, ü, é, etc.) and digits
+            # Exclude underscore since \w includes it but we want to treat it as splitter
+            elif bool(re.search(r"[\w]", char, re.UNICODE)) and char != "_":
                 c_state = "en&num"
             else:
                 c_state = "splitter"

tests/test_french_accents.py

@@ -129,9 +129,8 @@ def test_french_word_grouping():
         print("Some tests FAILED. Please review the output above.")
     print("=" * 70)
 
-    return all_passed
+    assert all_passed, "Some French accent tests failed"
 
 
 if __name__ == "__main__":
-    success = test_french_word_grouping()
-    sys.exit(0 if success else 1)
+    test_french_word_grouping()

tests/test_rec_postprocess.py

@@ -0,0 +1,107 @@
+import os
+import sys
+
+import numpy as np
+import pytest
+
+current_dir = os.path.dirname(os.path.abspath(__file__))
+sys.path.append(os.path.abspath(os.path.join(current_dir, "..")))
+
+from ppocr.postprocess.rec_postprocess import BaseRecLabelDecode
+
+
+class TestBaseRecLabelDecode:
+    """Tests for BaseRecLabelDecode.get_word_info() method."""
+
+    @pytest.fixture
+    def decoder(self):
+        """Create a BaseRecLabelDecode instance for testing."""
+        return BaseRecLabelDecode()
+
+    def test_get_word_info_with_german_accented_chars(self, decoder):
+        """Test that German words with accented characters are not split."""
+        text = "Grüßen"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 1, "German word should not be split"
+        assert "".join(word_list[0]) == "Grüßen"
+        assert state_list[0] == "en&num"
+
+    def test_get_word_info_with_longer_german_word(self, decoder):
+        """Test longer German words with umlauts remain intact."""
+        text = "ungewöhnlichen"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 1, "German word should not be split"
+        assert "".join(word_list[0]) == "ungewöhnlichen"
+        assert state_list[0] == "en&num"
+
+    def test_get_word_info_with_french_accented_chars(self, decoder):
+        """Test French words with accented characters."""
+        text = "café"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 1, "French word should not be split"
+        assert "".join(word_list[0]) == "café"
+
+    def test_get_word_info_underscore_as_splitter(self, decoder):
+        """Test that underscores are treated as word splitters."""
+        text = "hello_world"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 2, "Underscore should split words"
+        assert "".join(word_list[0]) == "hello"
+        assert "".join(word_list[1]) == "world"
+
+    def test_get_word_info_with_mixed_content(self, decoder):
+        """Test mixed content with spaces and accented characters."""
+        text = "Grüßen Sie"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 2, "Should have two words separated by space"
+        assert "".join(word_list[0]) == "Grüßen"
+        assert "".join(word_list[1]) == "Sie"
+
+    def test_get_word_info_with_french_apostrophe(self, decoder):
+        """Test French words with apostrophes like n'êtes."""
+        text = "n'êtes"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        # Apostrophe should keep words connected in French context
+        assert len(word_list) == 1, "French apostrophe should connect words"
+        assert "".join(word_list[0]) == "n'êtes"
+
+    def test_get_word_info_with_ascii_only(self, decoder):
+        """Test backward compatibility with ASCII-only text."""
+        text = "hello world"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 2
+        assert "".join(word_list[0]) == "hello"
+        assert "".join(word_list[1]) == "world"
+
+    def test_get_word_info_with_numbers(self, decoder):
+        """Test that numbers are properly handled."""
+        text = "VGG-16"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 1, "Hyphenated word-number should stay together"
+        assert "".join(word_list[0]) == "VGG-16"
+
+    def test_get_word_info_with_floating_point(self, decoder):
+        """Test floating point numbers stay together."""
+        text = "price 3.14"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 2
+        assert "".join(word_list[0]) == "price"
+        assert "".join(word_list[1]) == "3.14"
+
+    def test_get_word_info_with_chinese(self, decoder):
+        """Test Chinese characters are properly grouped."""
+        text = "你好啊"
+        selection = np.ones(len(text), dtype=bool)
+        word_list, _, state_list = decoder.get_word_info(text, selection)
+        assert len(word_list) == 1
+        assert "".join(word_list[0]) == "你好啊"
+        assert state_list[0] == "cn"