test(chunks): pin exact-overlap contracts for trusted merges

- Overlaps wider than the 400-rune text window must be trimmed once.
- Adjacent chunks built from one repeating row must keep every row.
- A length-preserving text mismatch must fall back and keep both bodies.
This commit is contained in:
wizardchen
2026-08-06 12:19:20 +08:00
committed by lyingbug
parent c9dd5a574e
commit dfe143818c
2 changed files with 90 additions and 0 deletions
@@ -55,6 +55,52 @@ func TestGroupAndMerge_TrustedOverlapTrimsAndExtendsRange(t *testing.T) {
assert.ElementsMatch(t, []string{"c2"}, results[0].SubChunkID)
}
func TestGroupAndMerge_TrustedOverlapBeyondTextSearchWindow(t *testing.T) {
// A real overlap wider than the text matcher's 400-rune window: the position
// path knows the exact amount, so the overlap must not survive twice.
overlap := rangeText('o', 500)
first := trustedResult("c1", 1, 0, 1000, rangeText('a', 500)+overlap)
second := trustedResult("c2", 2, 500, 1500, overlap+rangeText('b', 500))
results := mergeGrouped(docChunks(first, second))
require.Len(t, results, 1)
assert.Equal(t, rangeText('a', 500)+overlap+rangeText('b', 500), results[0].Content)
assert.Equal(t, 1500, results[0].EndAt)
}
func TestGroupAndMerge_TrustedAdjacentRepeatedTextKeepsEveryRow(t *testing.T) {
// Adjacent trusted chunks (positional overlap 0) made of one repeating row:
// searching for the longest suffix match would mistake the repetition for an
// overlap and drop rows, so the exact-overlap path must concatenate verbatim.
row := "| cell | cell |\n"
firstBody := rangeText('x', 100) + strings.Repeat(row, 2)
secondBody := strings.Repeat(row, 3) + rangeText('y', 60)
first := trustedResult("c1", 1, 0, runeLen(firstBody), firstBody)
second := trustedResult("c2", 2, runeLen(firstBody), runeLen(firstBody)+runeLen(secondBody), secondBody)
results := mergeGrouped(docChunks(first, second))
require.Len(t, results, 1)
assert.Equal(t, firstBody+secondBody, results[0].Content)
assert.Equal(t, 5, strings.Count(results[0].Content, row), "no repeated row may be dropped")
}
func TestGroupAndMerge_TrustedOverlapTextMismatchFallsBackToTextMatch(t *testing.T) {
// Length invariant holds but the bodies disagree inside the overlap window
// (HTML entities, synthetic headers): the exact path must refuse and the text
// fallback must keep both bodies rather than trim on coordinates.
first := trustedResult("c1", 1, 0, 100, rangeText('a', 80)+rangeText('b', 20))
second := trustedResult("c2", 2, 80, 200, rangeText('c', 20)+rangeText('d', 100))
results := mergeGrouped(docChunks(first, second))
require.Len(t, results, 1)
assert.Contains(t, results[0].Content, rangeText('b', 20))
assert.Contains(t, results[0].Content, rangeText('c', 20))
assert.Contains(t, results[0].Content, rangeText('d', 100))
}
func TestGroupAndMerge_TrustedAdjacentJoinsSeamlessly(t *testing.T) {
// Adjacent trusted chunks are contiguous in the original document: the
// merged body must be the exact concatenation, with no separator inserted.
+44
View File
@@ -58,6 +58,50 @@ func TestAppendWithOverlap_NoOverlap(t *testing.T) {
}
}
func TestAppendWithExactOverlap_TrimsKnownOverlap(t *testing.T) {
overlap := "shared boundary text"
got, ok := AppendWithExactOverlap("before "+overlap, overlap+" after", len([]rune(overlap)))
if !ok {
t.Fatal("exact overlap should be accepted")
}
want := "before " + overlap + " after"
if got != want {
t.Fatalf("exact overlap mismatch:\n got=%q\nwant=%q", got, want)
}
}
func TestAppendWithExactOverlap_ZeroOverlapConcatenatesRepeatedText(t *testing.T) {
// 首尾相接的两段都由同一周期性文本组成(表格行 / 日志行)。重叠量为 0 时
// 必须原样拼接,不能去搜索后缀匹配,否则会吃掉 next 开头的重复行。
row := "| cell | cell |\n"
acc := "前言\n" + row + row
next := row + row + row + "结尾\n"
got, ok := AppendWithExactOverlap(acc, next, 0)
if !ok {
t.Fatal("zero overlap should be accepted")
}
if got != acc+next {
t.Fatalf("zero overlap must concatenate verbatim:\n got=%q\nwant=%q", got, acc+next)
}
}
func TestAppendWithExactOverlap_RejectsMismatchedOverlap(t *testing.T) {
// 长度不变式成立但文本已经不同(HTML 实体、补写表头等),必须拒绝,
// 由调用方回退到按文本匹配。
if _, ok := AppendWithExactOverlap("abcdefghijkl", "XYZdefghijkl", 6); ok {
t.Fatal("mismatched overlap should be rejected")
}
}
func TestAppendWithExactOverlap_RejectsOverlapLongerThanBodies(t *testing.T) {
if _, ok := AppendWithExactOverlap("short", "shorter", 99); ok {
t.Fatal("overlap exceeding body length should be rejected")
}
if _, ok := AppendWithExactOverlap("short", "shorter", -1); ok {
t.Fatal("negative overlap should be rejected")
}
}
func TestJoinChunkContentUsesCurrentTextInsteadOfSourceOffsets(t *testing.T) {
first := "first edited body with no original overlap"
second := "second independently edited body"