mirror of
https://github.com/Tencent/WeKnora.git
synced 2026-08-29 02:04:30 +08:00
test(chunks): pin exact-overlap contracts for trusted merges
- Overlaps wider than the 400-rune text window must be trimmed once. - Adjacent chunks built from one repeating row must keep every row. - A length-preserving text mismatch must fall back and keep both bodies.
This commit is contained in:
@@ -55,6 +55,52 @@ func TestGroupAndMerge_TrustedOverlapTrimsAndExtendsRange(t *testing.T) {
|
||||
assert.ElementsMatch(t, []string{"c2"}, results[0].SubChunkID)
|
||||
}
|
||||
|
||||
func TestGroupAndMerge_TrustedOverlapBeyondTextSearchWindow(t *testing.T) {
|
||||
// A real overlap wider than the text matcher's 400-rune window: the position
|
||||
// path knows the exact amount, so the overlap must not survive twice.
|
||||
overlap := rangeText('o', 500)
|
||||
first := trustedResult("c1", 1, 0, 1000, rangeText('a', 500)+overlap)
|
||||
second := trustedResult("c2", 2, 500, 1500, overlap+rangeText('b', 500))
|
||||
|
||||
results := mergeGrouped(docChunks(first, second))
|
||||
|
||||
require.Len(t, results, 1)
|
||||
assert.Equal(t, rangeText('a', 500)+overlap+rangeText('b', 500), results[0].Content)
|
||||
assert.Equal(t, 1500, results[0].EndAt)
|
||||
}
|
||||
|
||||
func TestGroupAndMerge_TrustedAdjacentRepeatedTextKeepsEveryRow(t *testing.T) {
|
||||
// Adjacent trusted chunks (positional overlap 0) made of one repeating row:
|
||||
// searching for the longest suffix match would mistake the repetition for an
|
||||
// overlap and drop rows, so the exact-overlap path must concatenate verbatim.
|
||||
row := "| cell | cell |\n"
|
||||
firstBody := rangeText('x', 100) + strings.Repeat(row, 2)
|
||||
secondBody := strings.Repeat(row, 3) + rangeText('y', 60)
|
||||
first := trustedResult("c1", 1, 0, runeLen(firstBody), firstBody)
|
||||
second := trustedResult("c2", 2, runeLen(firstBody), runeLen(firstBody)+runeLen(secondBody), secondBody)
|
||||
|
||||
results := mergeGrouped(docChunks(first, second))
|
||||
|
||||
require.Len(t, results, 1)
|
||||
assert.Equal(t, firstBody+secondBody, results[0].Content)
|
||||
assert.Equal(t, 5, strings.Count(results[0].Content, row), "no repeated row may be dropped")
|
||||
}
|
||||
|
||||
func TestGroupAndMerge_TrustedOverlapTextMismatchFallsBackToTextMatch(t *testing.T) {
|
||||
// Length invariant holds but the bodies disagree inside the overlap window
|
||||
// (HTML entities, synthetic headers): the exact path must refuse and the text
|
||||
// fallback must keep both bodies rather than trim on coordinates.
|
||||
first := trustedResult("c1", 1, 0, 100, rangeText('a', 80)+rangeText('b', 20))
|
||||
second := trustedResult("c2", 2, 80, 200, rangeText('c', 20)+rangeText('d', 100))
|
||||
|
||||
results := mergeGrouped(docChunks(first, second))
|
||||
|
||||
require.Len(t, results, 1)
|
||||
assert.Contains(t, results[0].Content, rangeText('b', 20))
|
||||
assert.Contains(t, results[0].Content, rangeText('c', 20))
|
||||
assert.Contains(t, results[0].Content, rangeText('d', 100))
|
||||
}
|
||||
|
||||
func TestGroupAndMerge_TrustedAdjacentJoinsSeamlessly(t *testing.T) {
|
||||
// Adjacent trusted chunks are contiguous in the original document: the
|
||||
// merged body must be the exact concatenation, with no separator inserted.
|
||||
|
||||
@@ -58,6 +58,50 @@ func TestAppendWithOverlap_NoOverlap(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendWithExactOverlap_TrimsKnownOverlap(t *testing.T) {
|
||||
overlap := "shared boundary text"
|
||||
got, ok := AppendWithExactOverlap("before "+overlap, overlap+" after", len([]rune(overlap)))
|
||||
if !ok {
|
||||
t.Fatal("exact overlap should be accepted")
|
||||
}
|
||||
want := "before " + overlap + " after"
|
||||
if got != want {
|
||||
t.Fatalf("exact overlap mismatch:\n got=%q\nwant=%q", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendWithExactOverlap_ZeroOverlapConcatenatesRepeatedText(t *testing.T) {
|
||||
// 首尾相接的两段都由同一周期性文本组成(表格行 / 日志行)。重叠量为 0 时
|
||||
// 必须原样拼接,不能去搜索后缀匹配,否则会吃掉 next 开头的重复行。
|
||||
row := "| cell | cell |\n"
|
||||
acc := "前言\n" + row + row
|
||||
next := row + row + row + "结尾\n"
|
||||
got, ok := AppendWithExactOverlap(acc, next, 0)
|
||||
if !ok {
|
||||
t.Fatal("zero overlap should be accepted")
|
||||
}
|
||||
if got != acc+next {
|
||||
t.Fatalf("zero overlap must concatenate verbatim:\n got=%q\nwant=%q", got, acc+next)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendWithExactOverlap_RejectsMismatchedOverlap(t *testing.T) {
|
||||
// 长度不变式成立但文本已经不同(HTML 实体、补写表头等),必须拒绝,
|
||||
// 由调用方回退到按文本匹配。
|
||||
if _, ok := AppendWithExactOverlap("abcdefghijkl", "XYZdefghijkl", 6); ok {
|
||||
t.Fatal("mismatched overlap should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAppendWithExactOverlap_RejectsOverlapLongerThanBodies(t *testing.T) {
|
||||
if _, ok := AppendWithExactOverlap("short", "shorter", 99); ok {
|
||||
t.Fatal("overlap exceeding body length should be rejected")
|
||||
}
|
||||
if _, ok := AppendWithExactOverlap("short", "shorter", -1); ok {
|
||||
t.Fatal("negative overlap should be rejected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinChunkContentUsesCurrentTextInsteadOfSourceOffsets(t *testing.T) {
|
||||
first := "first edited body with no original overlap"
|
||||
second := "second independently edited body"
|
||||
|
||||
Reference in New Issue
Block a user