From 473c8726bbdd743040e94f4a13b592091efd35c8 Mon Sep 17 00:00:00 2001 From: neoczr Date: Fri, 18 Sep 2026 21:10:17 +0800 Subject: [PATCH] =?UTF-8?q?fix(chunking):=20=E4=BF=AE=E5=A4=8D=E5=86=85?= =?UTF-8?q?=E8=81=94=20HTML=20=E8=A1=A8=E6=A0=BC=E4=B8=8E=E4=BF=9D?= =?UTF-8?q?=E6=8A=A4=E6=A8=A1=E5=BC=8F=E5=AF=BC=E8=87=B4=E7=9A=84=E8=B6=85?= =?UTF-8?q?=E9=95=BF=20chunk=20(#3404)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(ocr): normalize inline HTML tables in VLM/OCR sanitizer The VLM/OCR path left raw HTML blocks in the body: looksLikeHTML does not match markdown text with an embedded table, so ocrHTMLToMarkdown (which also lacked the table plugin) never ran. The chunker then split the raw markup mid-row, tripping the 7500-char hard limit and invalidating the heading layer for the whole document. Reuse docparser.NormalizeHTMLTables (already used by PaddleOCR-VL) instead: export it and call it unconditionally from sanitizeOCRText, since it is a safe no-op when no
is present. * fix(ocr): keep merged-cell HTML tables chunkable - Narrow span detection to values >1 so rowspan=1/colspan=1 tables still convert to GFM instead of falling back to raw HTML. - On the merged-cell fallback, split each onto its own line and pad the block with blank lines so the chunker can split by "\n" instead of force-cutting at the absolute max size. * fix(knowledge): normalize inline HTML tables for all parser engines before chunking Previously NormalizeHTMLTables was only wired into the paddleocr-vl converters, so tables emitted by other engines (notably MinerU) reached the chunker as raw single-line
blocks and were force-cut at the 7500 char hard limit. Add the normalization at the two central pre-chunk boundaries instead: - knowledge_process.go: after all converter/ASR/image markdown rewrites, before the chunker step (covers every parser engine) - knowledge_create.go: before building the splitter config for manual markdown Converters keep their existing calls (idempotent no-op). * fix(chunker): bound Markdown link/image protected patterns to one line The unbounded [^\]]* / [^)]+ character classes in the Markdown link and image protectedPatterns also matched newlines, so a stray '[' left behind by OCR (e.g. a full-width 【公章】 mis-read as half-width) could pair with a distant '](' and swallow whole paragraphs as one protected atomic span. buildUnitsWithProtection then treated the region as a single unit and mergeUnits could not split it, defeating chunking (observed 1 chunk of 1753 runes where 16 chunks were expected). Exclude newlines and bound the link text (<=200) / destination (<=500), matching CommonMark's rule that neither may span a blank line. Regression: protected_spans_test.go — same input now yields multiple chunks with max 125 runes (chunkSize=128) instead of a single 1753-rune chunk; well-formed single-line links/images stay protected. * fix(ocr): keep non-GFM HTML tables chunkable across all ingest paths Conversion fallbacks for
, nested lists, and one-column tables still left single-line HTML, so MinerU output could hit the same oversized-chunk bug. Split those rows, accept one-column GFM, skip fenced examples, and normalize before image resolution so cell images are not escaped. * style: wrap long lines and avoid shadowing max in table-chunk tests Pre-push lll/revive failed on the PR diff versus origin/main. --------- Co-authored-by: neoczr Co-authored-by: wizardchen --- .../html_table_normalize_wiring_test.go | 107 ++++++++++ .../application/service/knowledge_create.go | 5 + .../application/service/knowledge_process.go | 23 ++- internal/application/service/ocr_sanitizer.go | 8 + .../application/service/ocr_sanitizer_test.go | 46 ++++- .../application/service/temporary_document.go | 6 + internal/handler/chunker_debug.go | 2 + .../chunker/protected_spans_test.go | 135 +++++++++++++ internal/infrastructure/chunker/splitter.go | 14 +- .../docparser/html_table_normalizer.go | 128 +++++++++--- .../docparser/html_table_normalizer_test.go | 183 +++++++++++++++++- .../docparser/paddleocr_vl_cloud_converter.go | 2 +- .../docparser/paddleocr_vl_converter.go | 2 +- 13 files changed, 619 insertions(+), 42 deletions(-) create mode 100644 internal/application/service/html_table_normalize_wiring_test.go create mode 100644 internal/infrastructure/chunker/protected_spans_test.go diff --git a/internal/application/service/html_table_normalize_wiring_test.go b/internal/application/service/html_table_normalize_wiring_test.go new file mode 100644 index 000000000..295776c1e --- /dev/null +++ b/internal/application/service/html_table_normalize_wiring_test.go @@ -0,0 +1,107 @@ +package service + +import ( + "os" + "strings" + "testing" + + "github.com/Tencent/WeKnora/internal/infrastructure/docparser" +) + +// Regression guard for the central "normalize inline HTML tables before +// chunking" wiring. The bug this guards: normalization was only wired into the +// paddleocr-vl converters, so tables emitted by other engines (e.g. MinerU) +// reached the chunker as raw single-line
blocks and were force-cut at +// the absolute size limit. +func TestHTMLEmbeddedTableWiring_ProducesChunkableOutput(t *testing.T) { + input := strings.Join([]string{ + "# 检测报告", + "", + "以下是正文段落，用于确认归一化不会破坏普通 markdown。", + "", + // colspan=1: redundant spans must still be convertible to GFM. + `
` + + `` + + `
项目结果
拉伸强度合格
`, + "", + // colspan=6: a real merge -> cannot be GFM, but must stay splittable. + `` + + `
汇总
备注
`, + "", + "收尾段落。", + }, "\n") + + got := docparser.NormalizeHTMLTables(input) + + if strings.Contains(got, `colspan="1"`) { + t.Fatalf("redundant colspan=1 table was not normalized:\n%s", got) + } + if !strings.Contains(got, "|") || !strings.Contains(got, "拉伸强度") { + t.Fatalf("expected colspan=1 table to become GFM, got:\n%s", got) + } + if !strings.Contains(got, " at offset %d is not preceded by a newline:\n%s", abs, got) + } + i = abs + 1 + } +} + +func lastCallBefore(src, call, anchor string) int { + idx := strings.Index(src, anchor) + if idx < 0 { + return -1 + } + return strings.LastIndex(src[:idx], call) +} + +// TestHTMLEmbeddedTableWiring_CallSitesPresent is a lightweight static guard: +// it fails if the central normalization call is removed from a pre-chunk +// (or pre-image-resolution) location. +func TestHTMLEmbeddedTableWiring_CallSitesPresent(t *testing.T) { + call := "docparser.NormalizeHTMLTables(" + + processSrc, err := os.ReadFile("knowledge_process.go") + if err != nil { + t.Fatalf("read knowledge_process.go: %v", err) + } + process := string(processSrc) + if lastCallBefore(process, call, "s.imageResolver.ResolveAndStore") < 0 { + t.Fatalf("knowledge_process.go: %s must run before image resolution", call) + } + if lastCallBefore(process, call, "chunkCfg := buildSplitterConfigFromChunking") < 0 { + t.Fatalf("knowledge_process.go: %s must run before the chunking step", call) + } + + createSrc, err := os.ReadFile("knowledge_create.go") + if err != nil { + t.Fatalf("read knowledge_create.go: %v", err) + } + if lastCallBefore(string(createSrc), call, "chunkCfg := buildSplitterConfigFromChunking") < 0 { + t.Fatalf("knowledge_create.go: %s must run before buildSplitterConfigFromChunking", call) + } + + tempSrc, err := os.ReadFile("temporary_document.go") + if err != nil { + t.Fatalf("read temporary_document.go: %v", err) + } + if lastCallBefore(string(tempSrc), call, "chunker.Split(") < 0 { + t.Fatalf("temporary_document.go: %s must run before chunker.Split", call) + } + + previewSrc, err := os.ReadFile("../../handler/chunker_debug.go") + if err != nil { + t.Fatalf("read chunker_debug.go: %v", err) + } + if !strings.Contains(string(previewSrc), call) { + t.Fatalf("chunker_debug.go is missing the preview %s call", call) + } +} diff --git a/internal/application/service/knowledge_create.go b/internal/application/service/knowledge_create.go index df4f46985..1b19f5a6b 100644 --- a/internal/application/service/knowledge_create.go +++ b/internal/application/service/knowledge_create.go @@ -1276,6 +1276,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context, processOverrides, _ := knowledge.ProcessOverrides() eff := ResolveProcessConfig(kb, processOverrides) + // Normalize inline HTML tables before chunking, for the same reason as the + // file-processing path: parser/OCR output may embed raw blocks that + // the chunker cannot split. Fenced code examples are left untouched. + clean = docparser.NormalizeHTMLTables(clean) + // Manual content is markdown - chunk directly with Go chunker chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig) diff --git a/internal/application/service/knowledge_process.go b/internal/application/service/knowledge_process.go index 8cce86850..648593473 100644 --- a/internal/application/service/knowledge_process.go +++ b/internal/application/service/knowledge_process.go @@ -3689,6 +3689,22 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e convertResult.AudioData = nil } + // Normalize inline HTML tables from any parser engine (MinerU, builtin, + // markitdown, paddleocr-vl, ...) into GFM, or row-newline-separated HTML + // for merged / non-GFM tables, so the chunker can split them. + // Runs before image resolution so inside HTML tables becomes + // markdown the resolver can persist, instead of rewriting data-URI + // tags to `![...](...)` and then escaping them in a later + // HTML→GFM pass. Line endings are normalized first so injected row + // breaks are plain LF. + sanitizeReadResult(convertResult) + if convertResult != nil { + convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent) + if convertResult.MarkdownContent != "" { + convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent) + } + } + // Step 2: Store images and update markdown references var storedImages []docparser.StoredImage @@ -3719,11 +3735,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e logger.Infof(ctx, "Resolved %d total images for knowledge %s", len(storedImages), knowledge.ID) } - // Step 3: Split into chunks using Go chunker. Browser textareas normalize - // pasted content to LF, so normalize uploaded source text before calculating - // chunk boundaries as well. - sanitizeReadResult(convertResult) - convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent) + // Step 3: Split into chunks using Go chunker. Line endings and inline + // HTML tables were normalized before image resolution above. chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig) processOpts := ProcessChunksOptions{ diff --git a/internal/application/service/ocr_sanitizer.go b/internal/application/service/ocr_sanitizer.go index dfa5fb6e6..fba14b033 100644 --- a/internal/application/service/ocr_sanitizer.go +++ b/internal/application/service/ocr_sanitizer.go @@ -5,6 +5,7 @@ import ( "strings" htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2" + "github.com/Tencent/WeKnora/internal/infrastructure/docparser" ) var ( @@ -49,6 +50,13 @@ func sanitizeOCRText(raw string) string { } } + // Convert inline HTML
blocks to GFM tables. This is a safe no-op + // when no
is present, so it runs unconditionally: a markdown body + // with an embedded HTML table does not satisfy looksLikeHTML (it neither + // starts with a tag nor is dominated by tag characters), yet leaving the + // raw markup in place makes the chunker split inside table rows. + text = docparser.NormalizeHTMLTables(text) + if isKnownEmptyReply(text) { return "" } diff --git a/internal/application/service/ocr_sanitizer_test.go b/internal/application/service/ocr_sanitizer_test.go index b6a91c9c0..1281bc28a 100644 --- a/internal/application/service/ocr_sanitizer_test.go +++ b/internal/application/service/ocr_sanitizer_test.go @@ -1,6 +1,10 @@ package service -import "testing" +import ( + "regexp" + "strings" + "testing" +) func TestSanitizeOCRText(t *testing.T) { tests := []struct { @@ -187,6 +191,46 @@ func TestLooksLikeHTML(t *testing.T) { } } +// TestSanitizeOCRText_ConvertsInlineHTMLTable guards against a regression where +// an HTML
embedded in a markdown body survived sanitization untouched: +// the mixed content does not satisfy looksLikeHTML, so the old HTML-to-markdown +// path never ran and the raw
markup reached the chunker. +func TestSanitizeOCRText_ConvertsInlineHTMLTable(t *testing.T) { + para := "本季度公司整体经营情况保持稳定，营收与利润两项核心指标均实现同比增长。" + + "为了便于管理层快速了解经营成果，下表汇总了报告期内的主要财务数据，" + + "供后续经营分析、预算编制以及年度考核等工作参考使用，" + + "请结合实际业务情况综合判断，切勿脱离业务背景单独解读其中任何一项数字。" + tail := "以上数据均来自财务部门审核后的正式报表，统计口径与上一报告期保持一致，" + + "未发生会计政策变更或追溯调整。如有疑问请与财务部门联系确认，" + + "最终解释权归公司财务部门所有。" + input := "# 报告\n\n" + para + "\n\n" + + `
` + + `` + + `
指标数值
营收10亿
利润2.3亿
` + + "\n\n" + tail + "\n\n" + + if looksLikeHTML(input) { + t.Fatalf("test input unexpectedly looks like HTML; the regression test must exercise the mixed-content path") + } + + got := sanitizeOCRText(input) + + if strings.Contains(got, " best { + best = n + } + } + return best +} + +// TestSplitText_OCRStrayBracketDoesNotSwallowDocument is the core regression: +// before the fix the malformed '[' turned the whole document into a single +// protected span (~1700 runes) and SplitText returned one oversized chunk. +func TestSplitText_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) { + doc := buildOCRStrayBracketDoc() + if !strings.Contains(doc, "[公章】") || !strings.Contains(doc, "](来源)") { + t.Fatalf("fixture missing malformed-bracket markers") + } + const chunkSize = 128 + cfg := SplitterConfig{ChunkSize: chunkSize, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", "。"}} + + chunks := SplitText(doc, cfg) + t.Logf("doc=%d runes chunkSize=%d -> chunks=%d max=%d runes", + len([]rune(doc)), chunkSize, len(chunks), maxChunkRunes(chunks)) + if len(chunks) < 2 { + t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks)) + } + if got := maxChunkRunes(chunks); got > 2*chunkSize { + t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize) + } +} + +// TestSplit_OCRStrayBracketDoesNotSwallowDocument exercises the strategy-aware +// entry point with the legacy tier pinned, so the protected-pattern behaviour +// is what is under test rather than the profiler's tier selection. +func TestSplit_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) { + doc := buildOCRStrayBracketDoc() + const chunkSize = 128 + cfg := SplitterConfig{ + ChunkSize: chunkSize, + ChunkOverlap: 0, + Separators: []string{"\n\n", "\n", "。"}, + Strategy: StrategyLegacy, + } + + chunks := Split(doc, cfg) + if len(chunks) < 2 { + t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks)) + } + if got := maxChunkRunes(chunks); got > 2*chunkSize { + t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize) + } +} + +// TestProtectedSpans_NoCrossLineLinkOrImage asserts no protected span may +// contain a newline for the malformed document: link text and destination +// cannot span a blank line (CommonMark). +func TestProtectedSpans_NoCrossLineLinkOrImage(t *testing.T) { + doc := buildOCRStrayBracketDoc() + for _, s := range protectedSpans(doc) { + seg := doc[s.start:s.end] + if strings.Contains(seg, "\n") { + t.Fatalf("protected span spans a newline (%d runes): %q", len([]rune(seg)), seg) + } + } +} + +// TestProtectedSpans_MarkdownLinksAndImagesStillProtected is the positive +// guard: the newline-bounded patterns must keep recognising well-formed +// single-line Markdown links and images. +func TestProtectedSpans_MarkdownLinksAndImagesStillProtected(t *testing.T) { + cases := []string{ + "![alt](http://x/y.png)", + "[文字](http://x.com)", + `[link](url "title")`, + "![](resource://abc)", + } + for _, c := range cases { + spans := protectedSpans(c) + if len(spans) == 0 { + t.Errorf("expected %q to be protected, got no span", c) + continue + } + if spans[0].start != 0 || spans[0].end != len(c) { + t.Errorf("%q: protected span = [%d,%d), want [0,%d)", c, spans[0].start, spans[0].end, len(c)) + } + } +} + +// TestProtectedSpans_LiteralCrossLineExample documents that an isolated +// malformed bracket pair with NO complete '](...)' construct is never treated +// as a protected link/image region. +func TestProtectedSpans_MultilineLinkIntentionallyUnprotected(t *testing.T) { + // CommonMark allows a single soft line break in link text. The protected + // patterns still refuse it so a stray OCR '[' cannot swallow a paragraph. + doc := "[link\ntext](http://example.com)" + if spans := protectedSpans(doc); len(spans) != 0 { + t.Fatalf("expected multiline link to stay unprotected, got %v", spans) + } +} + +func TestProtectedSpans_LiteralCrossLineExample(t *testing.T) { + doc := "[公章】\n\n第一段。\n\n第二段。\n\n第三段（北京)" + if spans := protectedSpans(doc); len(spans) != 0 { + t.Fatalf("expected no protected span, got %v", spans) + } +} diff --git a/internal/infrastructure/chunker/splitter.go b/internal/infrastructure/chunker/splitter.go index ff415eed6..69ff38e43 100644 --- a/internal/infrastructure/chunker/splitter.go +++ b/internal/infrastructure/chunker/splitter.go @@ -114,10 +114,18 @@ func DefaultConfig() SplitterConfig { } // protectedPatterns are regex patterns for content that must not be split. +// +// The Markdown link/image patterns deliberately exclude '\n' and bound the +// link text / destination length: CommonMark forbids them from spanning a +// blank line, and the previous unbounded [^\]]* / [^)]+ let a stray '[' left +// behind by OCR swallow whole paragraphs as one "protected" atomic span, +// defeating chunking entirely. var protectedPatterns = []*regexp.Regexp{ - regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math - regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`), // Markdown images - regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`), // Markdown links + regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math + // Markdown images / links: single line, bounded so a stray OCR '[' + // cannot swallow a paragraph (CommonMark forbids blank-line spans). + regexp.MustCompile(`!\[[^\]\n]{0,200}\]\([^)\n]{1,500}\)`), + regexp.MustCompile(`\[[^\]\n]{1,200}\]\([^)\n]{1,500}\)`), regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"), // Table rows regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"), // Fenced code blocks diff --git a/internal/infrastructure/docparser/html_table_normalizer.go b/internal/infrastructure/docparser/html_table_normalizer.go index ad55b1967..8851c2ba7 100644 --- a/internal/infrastructure/docparser/html_table_normalizer.go +++ b/internal/infrastructure/docparser/html_table_normalizer.go @@ -15,6 +15,10 @@ var ( // block, the form OCR/layout engines such as PaddleOCR-VL emit tables in. htmlTableBlockPattern = regexp.MustCompile(`(?is)]*>.*?`) + // fencedCodeBlockPattern matches CommonMark fenced code blocks so HTML + // examples inside them are left untouched. + fencedCodeBlockPattern = regexp.MustCompile("(?s)(?:```|~~~)[^\\n]*\\n.*?(?:```|~~~)") + // htmlLayoutAttrPattern matches presentational HTML attributes that carry // no semantic value (text-align styles, CSS classes, sizing). Structural // attributes like rowspan/colspan are intentionally excluded. @@ -22,28 +26,45 @@ var ( `(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`, ) - // htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot - // represent; such tables keep their HTML form (attributes stripped) instead. - htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`) + // htmlSpanAttrPattern detects rowspan/colspan values greater than 1, which + // Markdown tables cannot represent; such tables keep their HTML form + // (attributes stripped) instead. Span values of 1 (and the invalid 0) do + // not merge anything and must stay convertible. + htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\s*=\s*["']?(?:[2-9]|\d{2,})`) - // markdownTableSeparatorPattern matches the |---|---| delimiter row that a - // valid GFM table must contain. - markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`) + // htmlTableRowPattern matches the opening tag of each table row, used + // to put every row on its own line so the chunker can split degraded HTML + // tables at "\n" boundaries. + htmlTableRowPattern = regexp.MustCompile(`(?i) blocks embedded in OCR +// NormalizeHTMLTables rewrites inline HTML
blocks embedded in OCR // markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align // styles, which (1) waste tokens on layout markup and (2) are not recognized // by the chunker's table-protection logic, so large tables get split mid-row. // // Each table block is converted to a GFM Markdown table when possible. Tables -// that use rowspan/colspan (which Markdown cannot express) fall back to having -// their presentational attributes stripped so they stay intact as HTML. -func normalizeHTMLTables(md string) string { +// that use rowspan/colspan (which Markdown cannot express), or that the +// converter cannot turn into a valid GFM table (cell
, nested lists, …), +// fall back to presentational-attribute stripping plus one row per line so +// the chunker can still split on "\n". Tables inside fenced code blocks are +// left unchanged. +func NormalizeHTMLTables(md string) string { if !strings.Contains(strings.ToLower(md), "= s[0] && pos < s[1] { + return true } - // Pad with blank lines so the Markdown table is a standalone block that - // the chunker recognizes (and protects) as a table. - return "\n\n" + converted + "\n\n" - }) + } + return false } // stripHTMLLayoutAttrs removes presentational attributes from an HTML fragment @@ -75,3 +133,27 @@ func normalizeHTMLTables(md string) string { func stripHTMLLayoutAttrs(html string) string { return htmlLayoutAttrPattern.ReplaceAllString(html, "") } + +// splitHTMLTableRows puts each table row on its own line. The chunker splits +// on "\n", so a single-line HTML table would otherwise be unsplittable and get +// force-cut at the absolute max size. Whitespace between tags is insignificant +// in HTML, and newlines are only inserted before (never inside a tag). +// A that already starts on its own line is left alone. +func splitHTMLTableRows(block string) string { + locs := htmlTableRowPattern.FindAllStringIndex(block, -1) + if len(locs) == 0 { + return strings.TrimSpace(block) + } + var b strings.Builder + last := 0 + for _, loc := range locs { + b.WriteString(block[last:loc[0]]) + if loc[0] == 0 || block[loc[0]-1] != '\n' { + b.WriteByte('\n') + } + b.WriteString(block[loc[0]:loc[1]]) + last = loc[1] + } + b.WriteString(block[last:]) + return strings.TrimSpace(b.String()) +} diff --git a/internal/infrastructure/docparser/html_table_normalizer_test.go b/internal/infrastructure/docparser/html_table_normalizer_test.go index 8ae31d591..19b036bbc 100644 --- a/internal/infrastructure/docparser/html_table_normalizer_test.go +++ b/internal/infrastructure/docparser/html_table_normalizer_test.go @@ -16,7 +16,7 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) { 结尾。` - got := normalizeHTMLTables(input) + got := NormalizeHTMLTables(input) if strings.Contains(got, "` + + `` + + `` + + `` + + `
指标数值
营收10亿
` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, "1 / rowspan>1) cannot be GFM, so the table stays + // HTML — but each row must start on its own line so the chunker can split it. + input := `` + + `` + + `
合计
A1
B2
` + + got := NormalizeHTMLTables(input) + + if !strings.Contains(got, " to start on a new line, got:\n%q", got) + } + if !strings.Contains(got, "\n\n\n\n") { + t.Fatalf("expected block to be padded at the tail with blank lines, got:\n%q", got) + } + if n := strings.Count(got, "\n at offset %d is not preceded by a newline:\n%s", abs, got) + } + i = abs + 1 + } +} + func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) { // rowspan/colspan cannot be expressed in Markdown, so the table stays HTML - // but its presentational attributes are stripped. - input := `` + + // but its presentational attributes are stripped and rows become splittable. + input := `
合计
` + + `` + `
合计
AB
` - got := normalizeHTMLTables(input) + got := NormalizeHTMLTables(input) if !strings.Contains(got, " inside a cell makes the table plugin give up (it emits a newline). + // The HTML fallback must still put each on its own line. + input := `` + + `
指标
名称		数值
单位
拉伸强度
MPa		合格
A级
` + + got := NormalizeHTMLTables(input) + + if !strings.Contains(got, " table to remain HTML rather than flattened text, got:\n%s", got) + } + assertHTMLRowsSplittable(t, got) + for _, want := range []string{"指标", "名称", "拉伸强度", "合格"} { + if !strings.Contains(got, want) { + t.Fatalf("expected fallback HTML to retain %q, got:\n%s", want, got) + } + } +} + +func TestNormalizeHTMLTables_ListInCellFallsBackToSplittableHTML(t *testing.T) { + input := `` + + `
  • a
  • b
x
cd
` + + got := NormalizeHTMLTables(input) + + if !strings.Contains(got, "目录第一章第二章` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, "profile` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, "![cover](local://images/cover.png)说明` + + got := NormalizeHTMLTables(input) + + if strings.Contains(got, `!\[`) { + t.Fatalf("markdown image syntax escaped:\n%s", got) + } + if !strings.Contains(got, "![cover](local://images/cover.png)") { + t.Fatalf("expected markdown image to survive conversion, got:\n%s", got) + } +} + +func TestNormalizeHTMLTables_CodeFenceTableLeftAlone(t *testing.T) { + input := "示例：\n\n```html\n
ab
\n```\n" + + got := NormalizeHTMLTables(input) + + if got != input { + t.Fatalf("expected fenced HTML table to be unchanged, got:\n%s", got) + } +} + +func TestNormalizeHTMLTables_SpanTableIdempotent(t *testing.T) { + input := `
汇总
备注
` + once := NormalizeHTMLTables(input) + twice := NormalizeHTMLTables(once) + if once != twice { + t.Fatalf("NormalizeHTMLTables is not idempotent for span tables:\n1=%q\n2=%q", once, twice) } } diff --git a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go index 98cfc0133..c05c9561a 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go @@ -78,7 +78,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques return nil, fmt.Errorf("PaddleOCR-VL Cloud fetch results: %w", err) } - mdContent = normalizeHTMLTables(mdContent) + mdContent = NormalizeHTMLTables(mdContent) imageRefs := c.downloadImages(mdContent, imagesURL) mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs) diff --git a/internal/infrastructure/docparser/paddleocr_vl_converter.go b/internal/infrastructure/docparser/paddleocr_vl_converter.go index 313d3b74e..f49e1df58 100644 --- a/internal/infrastructure/docparser/paddleocr_vl_converter.go +++ b/internal/infrastructure/docparser/paddleocr_vl_converter.go @@ -65,7 +65,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (* // wastes tokens and defeats the chunker's table-protection logic. Convert // them to Markdown tables (or strip layout attributes when conversion is // not possible) before downstream processing. - mdContent = normalizeHTMLTables(mdContent) + mdContent = NormalizeHTMLTables(mdContent) imageRefs, mdContent := c.processImages(mdContent, imagesB64) mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)