fix: estimate emoji token cost above the plain text rate (#10216)

* fix: estimate emoji token cost above the plain text rate

Emoji, variation selector and zero width joiner characters fell into the
plain text bucket at 0.3 tokens per character. Real BPE tokenizers charge
about 3 tokens per emoji character, so an emoji heavy context was
underestimated by an order of magnitude and the post compression recheck
let requests through above the model limit.

Fixes #10208

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

* test(agent): bound emoji token estimate to catch overestimate regressions

* Update token_counter.py

* Update token_counter.py

---------

Co-authored-by: Jamal Kamaladdin <jamalkamaladdin@users.noreply.github.com>
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
Co-authored-by: Soulter <37870767+Soulter@users.noreply.github.com>

astrbot/core/agent/context/token_counter.py

@@ -1,4 +1,5 @@
 import json
+import re
 from typing import Protocol, runtime_checkable
 
 from ..message import AudioURLPart, ImageURLPart, Message, TextPart, ThinkPart
@@ -34,6 +35,15 @@ class TokenCounter(Protocol):
 IMAGE_TOKEN_ESTIMATE = 765
 AUDIO_TOKEN_ESTIMATE = 500
 
+# An emoji costs about 2 tokens per character under common BPE tokenizers, and
+# flag or zero width joiner sequences cost more. The plain text rate of 0.3
+# underestimates an emoji heavy context by an order of magnitude.
+EMOJI_TOKEN_ESTIMATE = 2.0
+
+# Emoji blocks, the miscellaneous symbols and dingbats block, the variation
+# selectors and the zero width joiner that build flag and multi person emoji.
+EMOJI_PATTERN = re.compile("[\u200d\u2600-\u27bf\ufe00-\ufe0f\U0001f000-\U0001faff]")
+
 
 class EstimateTokenCounter:
     """Estimate token counter implementation.
@@ -74,5 +84,8 @@ class EstimateTokenCounter:
 
     def _estimate_tokens(self, text: str) -> int:
         chinese_count = len([c for c in text if "\u4e00" <= c <= "\u9fff"])
-        other_count = len(text) - chinese_count
-        return int(chinese_count * 0.6 + other_count * 0.3)
+        emoji_count = len(EMOJI_PATTERN.findall(text))
+        other_count = len(text) - chinese_count - emoji_count
+        return int(
+            chinese_count * 0.6 + emoji_count * EMOJI_TOKEN_ESTIMATE + other_count * 0.3
+        )

tests/agent/test_token_counter.py

@@ -102,6 +102,31 @@ class TestMultimodalCounting:
         assert tokens == IMAGE_TOKEN_ESTIMATE * 3
 
 
+class TestEmojiCounting:
+    def test_emoji_costs_more_than_ascii(self):
+        """一个 emoji 的 token 开销远高于一个 ASCII 字符。"""
+        emoji = counter.count_tokens([_msg("user", "🔥" * 1000)])
+        ascii_text = counter.count_tokens([_msg("user", "a" * 1000)])
+        assert emoji > ascii_text * 5
+
+    def test_emoji_estimate_close_to_real_usage(self):
+        """50000 个 🔥 的真实 prompt_tokens 实测为 150082。"""
+        tokens = counter.count_tokens([_msg("user", "🔥" * 50_000)])
+        assert 100_000 <= tokens <= 170_000
+
+    def test_flag_and_zwj_sequences_counted(self):
+        """国旗与 ZWJ 组合的 token 开销同样高于普通字符。"""
+        flags = counter.count_tokens([_msg("user", "🇨🇳" * 500)])
+        family = counter.count_tokens([_msg("user", "👨‍👩‍👧‍👦" * 500)])
+        assert flags > counter.count_tokens([_msg("user", "ab" * 500)]) * 5
+        assert family > counter.count_tokens([_msg("user", "abcdefg" * 500)]) * 2
+
+    def test_plain_text_estimate_unchanged(self):
+        """纯文本与中文的估算值不受 emoji 分支影响。"""
+        assert counter.count_tokens([_msg("user", "a" * 100)]) == 30
+        assert counter.count_tokens([_msg("user", "你" * 100)]) == 60
+
+
 class TestTrustedUsage:
     def test_trusted_overrides(self):
         """如果 API 返回了 token 数，直接用它不做估算。"""