diff --git a/src/backend/bisheng/common/utils/simhash_utils.py b/src/backend/bisheng/common/utils/simhash_utils.py index e5ea5348f..8ff2c3121 100644 --- a/src/backend/bisheng/common/utils/simhash_utils.py +++ b/src/backend/bisheng/common/utils/simhash_utils.py @@ -8,11 +8,25 @@ def compute_simhash_64_hex(text: str) -> str: """Compute a 64-bit SimHash of *text*, return as 16-char lowercase hex. Uses jieba for CJK-aware tokenization. Empty/whitespace text produces "0" * 16. + + jieba.lcut splits Latin-script runs word-by-word but emits every space + character between them as its own token (e.g. "The quick brown" -> + ["The", " ", "quick", " ", "brown"]). Simhash() takes an unweighted list, + so a repeated token's hash gets counted once per occurrence — in English/ + code/config-heavy text, space tokens can be close to half the list, + letting that single low-information token's hash dominate the bit vote + and drown out the real content. Two unrelated documents with a similar + word/space ratio can then converge on the same or a near-identical + fingerprint regardless of what they actually say (verified: two + unrelated English sentences hashed identically before this filter, and + diverged to 58% similarity after it). Dropping whitespace-only tokens + removes that dominant no-signal feature without changing behavior for + CJK-only text (jieba doesn't emit space tokens between CJK characters). """ text = (text or "").strip() if not text: return "0" * 16 - tokens = jieba.lcut(text) + tokens = [token for token in jieba.lcut(text) if token.strip()] sh = Simhash(tokens, f=64) return f"{sh.value:016x}" diff --git a/src/backend/test/knowledge/test_simhash_utils.py b/src/backend/test/knowledge/test_simhash_utils.py index 280c6c0d2..4f9b4fa5a 100644 --- a/src/backend/test/knowledge/test_simhash_utils.py +++ b/src/backend/test/knowledge/test_simhash_utils.py @@ -48,3 +48,26 @@ def test_similarity_unrelated_text_low(): h_b = compute_simhash_64_hex("足球运动员转会市场的经济学分析与球队预算管理") # Just assert it's strictly less than 1 — exact value depends on tokenization assert similarity(h_a, h_b) < 1.0 + + +def test_similarity_unrelated_latin_text_not_1(): + """Regression test for the whitespace-token collision bug. + + jieba.lcut splits Latin-script runs word-by-word but emits every space + between words as its own token. Before filtering those out, + Simhash(tokens, f=64) — an unweighted list where a repeated token's hash + is counted once per occurrence — let the space token's hash dominate the + bit vote for any text with a lot of English/code content, so unrelated + English/Latin sentences with a similar word/space ratio collapsed onto + the exact same 64-bit fingerprint (similarity == 1.0) regardless of what + they actually said. + """ + h_a = compute_simhash_64_hex( + "The quick brown fox jumps over the lazy dog near the riverbank " + "while birds sing in the morning sun." + ) + h_b = compute_simhash_64_hex( + "Lorem ipsum dolor sit amet consectetur adipiscing elit sed do " + "eiusmod tempor incididunt ut labore." + ) + assert similarity(h_a, h_b) < 1.0