fix(chunking): 修复内联 HTML 表格与保护模式导致的超长 chunk (#3404)

* fix(ocr): normalize inline HTML tables in VLM/OCR sanitizer

The VLM/OCR path left raw HTML <table> blocks in the body: looksLikeHTML
does not match markdown text with an embedded table, so ocrHTMLToMarkdown
(which also lacked the table plugin) never ran. The chunker then split the
raw markup mid-row, tripping the 7500-char hard limit and invalidating the
heading layer for the whole document.

Reuse docparser.NormalizeHTMLTables (already used by PaddleOCR-VL) instead:
export it and call it unconditionally from sanitizeOCRText, since it is a
safe no-op when no <table> is present.

* fix(ocr): keep merged-cell HTML tables chunkable

- Narrow span detection to values >1 so rowspan=1/colspan=1 tables
  still convert to GFM instead of falling back to raw HTML.
- On the merged-cell fallback, split each <tr> onto its own line and
  pad the block with blank lines so the chunker can split by "\n"
  instead of force-cutting at the absolute max size.

* fix(knowledge): normalize inline HTML tables for all parser engines before chunking

Previously NormalizeHTMLTables was only wired into the paddleocr-vl
converters, so tables emitted by other engines (notably MinerU) reached
the chunker as raw single-line <table> blocks and were force-cut at the
7500 char hard limit.

Add the normalization at the two central pre-chunk boundaries instead:
- knowledge_process.go: after all converter/ASR/image markdown rewrites,
  before the chunker step (covers every parser engine)
- knowledge_create.go: before building the splitter config for manual
  markdown

Converters keep their existing calls (idempotent no-op).

* fix(chunker): bound Markdown link/image protected patterns to one line

The unbounded [^\]]* / [^)]+ character classes in the Markdown link and
image protectedPatterns also matched newlines, so a stray '[' left behind
by OCR (e.g. a full-width 【公章】 mis-read as half-width) could pair with
a distant '](' and swallow whole paragraphs as one protected atomic span.
buildUnitsWithProtection then treated the region as a single unit and
mergeUnits could not split it, defeating chunking (observed 1 chunk of
1753 runes where 16 chunks were expected).

Exclude newlines and bound the link text (<=200) / destination (<=500),
matching CommonMark's rule that neither may span a blank line.

Regression: protected_spans_test.go — same input now yields multiple
chunks with max 125 runes (chunkSize=128) instead of a single 1753-rune
chunk; well-formed single-line links/images stay protected.

* fix(ocr): keep non-GFM HTML tables chunkable across all ingest paths

Conversion fallbacks for <br>, nested lists, and one-column tables still
left single-line HTML, so MinerU output could hit the same oversized-chunk
bug. Split those rows, accept one-column GFM, skip fenced examples, and
normalize before image resolution so cell images are not escaped.

* style: wrap long lines and avoid shadowing max in table-chunk tests

Pre-push lll/revive failed on the PR diff versus origin/main.

---------

Co-authored-by: neoczr <neoczr@users.noreply.github.com>
Co-authored-by: wizardchen <wizardchen@tencent.com>

internal/application/service/html_table_normalize_wiring_test.go

@@ -0,0 +1,107 @@
+package service
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
+)
+
+// Regression guard for the central "normalize inline HTML tables before
+// chunking" wiring. The bug this guards: normalization was only wired into the
+// paddleocr-vl converters, so tables emitted by other engines (e.g. MinerU)
+// reached the chunker as raw single-line <table> blocks and were force-cut at
+// the absolute size limit.
+func TestHTMLEmbeddedTableWiring_ProducesChunkableOutput(t *testing.T) {
+	input := strings.Join([]string{
+		"# 检测报告",
+		"",
+		"以下是正文段落，用于确认归一化不会破坏普通 markdown。",
+		"",
+		// colspan=1: redundant spans must still be convertible to GFM.
+		`<table><tr><td colspan="1" style="text-align:left">项目</td>` +
+			`<td colspan="1">结果</td></tr>` +
+			`<tr><td>拉伸强度</td><td>合格</td></tr></table>`,
+		"",
+		// colspan=6: a real merge -> cannot be GFM, but must stay splittable.
+		`<table><tr><td colspan="6" style="text-align:center">汇总</td></tr>` +
+			`<tr><td colspan="6">备注</td></tr></table>`,
+		"",
+		"收尾段落。",
+	}, "\n")
+
+	got := docparser.NormalizeHTMLTables(input)
+
+	if strings.Contains(got, `colspan="1"`) {
+		t.Fatalf("redundant colspan=1 table was not normalized:\n%s", got)
+	}
+	if !strings.Contains(got, "|") || !strings.Contains(got, "拉伸强度") {
+		t.Fatalf("expected colspan=1 table to become GFM, got:\n%s", got)
+	}
+	if !strings.Contains(got, "<table") {
+		t.Fatalf("expected colspan=6 table to remain HTML, got:\n%s", got)
+	}
+	for i := 0; ; {
+		idx := strings.Index(got[i:], "<tr")
+		if idx < 0 {
+			break
+		}
+		abs := i + idx
+		if abs == 0 || got[abs-1] != '\n' {
+			t.Fatalf("remaining <tr> at offset %d is not preceded by a newline:\n%s", abs, got)
+		}
+		i = abs + 1
+	}
+}
+
+func lastCallBefore(src, call, anchor string) int {
+	idx := strings.Index(src, anchor)
+	if idx < 0 {
+		return -1
+	}
+	return strings.LastIndex(src[:idx], call)
+}
+
+// TestHTMLEmbeddedTableWiring_CallSitesPresent is a lightweight static guard:
+// it fails if the central normalization call is removed from a pre-chunk
+// (or pre-image-resolution) location.
+func TestHTMLEmbeddedTableWiring_CallSitesPresent(t *testing.T) {
+	call := "docparser.NormalizeHTMLTables("
+
+	processSrc, err := os.ReadFile("knowledge_process.go")
+	if err != nil {
+		t.Fatalf("read knowledge_process.go: %v", err)
+	}
+	process := string(processSrc)
+	if lastCallBefore(process, call, "s.imageResolver.ResolveAndStore") < 0 {
+		t.Fatalf("knowledge_process.go: %s must run before image resolution", call)
+	}
+	if lastCallBefore(process, call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {
+		t.Fatalf("knowledge_process.go: %s must run before the chunking step", call)
+	}
+
+	createSrc, err := os.ReadFile("knowledge_create.go")
+	if err != nil {
+		t.Fatalf("read knowledge_create.go: %v", err)
+	}
+	if lastCallBefore(string(createSrc), call, "chunkCfg := buildSplitterConfigFromChunking") < 0 {
+		t.Fatalf("knowledge_create.go: %s must run before buildSplitterConfigFromChunking", call)
+	}
+
+	tempSrc, err := os.ReadFile("temporary_document.go")
+	if err != nil {
+		t.Fatalf("read temporary_document.go: %v", err)
+	}
+	if lastCallBefore(string(tempSrc), call, "chunker.Split(") < 0 {
+		t.Fatalf("temporary_document.go: %s must run before chunker.Split", call)
+	}
+
+	previewSrc, err := os.ReadFile("../../handler/chunker_debug.go")
+	if err != nil {
+		t.Fatalf("read chunker_debug.go: %v", err)
+	}
+	if !strings.Contains(string(previewSrc), call) {
+		t.Fatalf("chunker_debug.go is missing the preview %s call", call)
+	}
+}

internal/application/service/knowledge_create.go

@@ -1276,6 +1276,11 @@ func (s *knowledgeService) triggerManualProcessing(ctx context.Context,
 	processOverrides, _ := knowledge.ProcessOverrides()
 	eff := ResolveProcessConfig(kb, processOverrides)
 
+	// Normalize inline HTML tables before chunking, for the same reason as the
+	// file-processing path: parser/OCR output may embed raw <table> blocks that
+	// the chunker cannot split. Fenced code examples are left untouched.
+	clean = docparser.NormalizeHTMLTables(clean)
+
 	// Manual content is markdown - chunk directly with Go chunker
 	chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
 

internal/application/service/knowledge_process.go

@@ -3689,6 +3689,22 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
 		convertResult.AudioData = nil
 	}
 
+	// Normalize inline HTML tables from any parser engine (MinerU, builtin,
+	// markitdown, paddleocr-vl, ...) into GFM, or row-newline-separated HTML
+	// for merged / non-GFM tables, so the chunker can split them.
+	// Runs before image resolution so <img> inside HTML tables becomes
+	// markdown the resolver can persist, instead of rewriting data-URI
+	// <img> tags to `![...](...)` and then escaping them in a later
+	// HTML→GFM pass. Line endings are normalized first so injected row
+	// breaks are plain LF.
+	sanitizeReadResult(convertResult)
+	if convertResult != nil {
+		convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
+		if convertResult.MarkdownContent != "" {
+			convertResult.MarkdownContent = docparser.NormalizeHTMLTables(convertResult.MarkdownContent)
+		}
+	}
+
 	// Step 2: Store images and update markdown references
 	var storedImages []docparser.StoredImage
 
@@ -3719,11 +3735,8 @@ func (s *knowledgeService) ProcessDocument(ctx context.Context, t *asynq.Task) e
 		logger.Infof(ctx, "Resolved %d total images for knowledge %s", len(storedImages), knowledge.ID)
 	}
 
-	// Step 3: Split into chunks using Go chunker. Browser textareas normalize
-	// pasted content to LF, so normalize uploaded source text before calculating
-	// chunk boundaries as well.
-	sanitizeReadResult(convertResult)
-	convertResult.MarkdownContent = chunker.NormalizeLineEndings(convertResult.MarkdownContent)
+	// Step 3: Split into chunks using Go chunker. Line endings and inline
+	// HTML tables were normalized before image resolution above.
 	chunkCfg := buildSplitterConfigFromChunking(eff.ChunkingConfig)
 
 	processOpts := ProcessChunksOptions{

internal/application/service/ocr_sanitizer.go

@@ -5,6 +5,7 @@ import (
 	"strings"
 
 	htmltomd "github.com/JohannesKaufmann/html-to-markdown/v2"
+	"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
 )
 
 var (
@@ -49,6 +50,13 @@ func sanitizeOCRText(raw string) string {
 		}
 	}
 
+	// Convert inline HTML <table> blocks to GFM tables. This is a safe no-op
+	// when no <table> is present, so it runs unconditionally: a markdown body
+	// with an embedded HTML table does not satisfy looksLikeHTML (it neither
+	// starts with a tag nor is dominated by tag characters), yet leaving the
+	// raw markup in place makes the chunker split inside table rows.
+	text = docparser.NormalizeHTMLTables(text)
+
 	if isKnownEmptyReply(text) {
 		return ""
 	}

internal/application/service/ocr_sanitizer_test.go

@@ -1,6 +1,10 @@
 package service
 
-import "testing"
+import (
+	"regexp"
+	"strings"
+	"testing"
+)
 
 func TestSanitizeOCRText(t *testing.T) {
 	tests := []struct {
@@ -187,6 +191,46 @@ func TestLooksLikeHTML(t *testing.T) {
 	}
 }
 
+// TestSanitizeOCRText_ConvertsInlineHTMLTable guards against a regression where
+// an HTML <table> embedded in a markdown body survived sanitization untouched:
+// the mixed content does not satisfy looksLikeHTML, so the old HTML-to-markdown
+// path never ran and the raw <table> markup reached the chunker.
+func TestSanitizeOCRText_ConvertsInlineHTMLTable(t *testing.T) {
+	para := "本季度公司整体经营情况保持稳定，营收与利润两项核心指标均实现同比增长。" +
+		"为了便于管理层快速了解经营成果，下表汇总了报告期内的主要财务数据，" +
+		"供后续经营分析、预算编制以及年度考核等工作参考使用，" +
+		"请结合实际业务情况综合判断，切勿脱离业务背景单独解读其中任何一项数字。"
+	tail := "以上数据均来自财务部门审核后的正式报表，统计口径与上一报告期保持一致，" +
+		"未发生会计政策变更或追溯调整。如有疑问请与财务部门联系确认，" +
+		"最终解释权归公司财务部门所有。"
+	input := "# 报告\n\n" + para + "\n\n" +
+		`<table><tr><th>指标</th><th>数值</th></tr>` +
+		`<tr><td>营收</td><td>10亿</td></tr>` +
+		`<tr><td>利润</td><td>2.3亿</td></tr></table>` +
+		"\n\n" + tail + "\n\n"
+
+	if looksLikeHTML(input) {
+		t.Fatalf("test input unexpectedly looks like HTML; the regression test must exercise the mixed-content path")
+	}
+
+	got := sanitizeOCRText(input)
+
+	if strings.Contains(got, "<table") {
+		t.Fatalf("expected inline HTML table to be converted, got:\n%s", got)
+	}
+	if !strings.Contains(got, "# 报告") || !strings.Contains(got, "最终解释权归公司财务部门所有。") {
+		t.Fatalf("expected surrounding markdown to be preserved, got:\n%s", got)
+	}
+	for _, want := range []string{"指标", "数值", "营收", "10亿", "利润", "2.3亿"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)
+		}
+	}
+	if !regexp.MustCompile(`\|[-:]{3,}\|`).MatchString(got) {
+		t.Fatalf("expected a GFM table separator row, got:\n%s", got)
+	}
+}
+
 func TestIsKnownEmptyReply(t *testing.T) {
 	tests := []struct {
 		input string

internal/application/service/temporary_document.go

@@ -383,6 +383,8 @@ func (s *temporaryDocumentService) Process(ctx context.Context, task *asynq.Task
 		return nil
 	}
 	content = common.CleanInvalidUTF8(content)
+	content = chunker.NormalizeLineEndings(content)
+	content = docparser.NormalizeHTMLTables(content)
 	lang := chunker.DetectLanguage(content)
 	cfg := chunker.DefaultConfig()
 	cfg.Strategy = chunker.StrategyAuto
@@ -463,6 +465,10 @@ func (s *temporaryDocumentService) parse(ctx context.Context, document *types.Te
 	if err != nil {
 		return "", nil, nil, fmt.Errorf("parse document: %w", err)
 	}
+	if result != nil && result.MarkdownContent != "" {
+		result.MarkdownContent = chunker.NormalizeLineEndings(result.MarkdownContent)
+		result.MarkdownContent = docparser.NormalizeHTMLTables(result.MarkdownContent)
+	}
 	// Capture raw page-image bytes before ResolveAndStore stores/rewrites them,
 	// so the VLM OCR fallback for scanned documents has bytes to work with.
 	maxOCRPages := temporaryDocumentImageOCRMaxPages()

internal/handler/chunker_debug.go

@@ -13,6 +13,7 @@ import (
 	"unicode/utf8"
 
 	"github.com/Tencent/WeKnora/internal/infrastructure/chunker"
+	"github.com/Tencent/WeKnora/internal/infrastructure/docparser"
 	"github.com/Tencent/WeKnora/internal/logger"
 	"github.com/gin-gonic/gin"
 )
@@ -147,6 +148,7 @@ func PreviewChunking(c *gin.Context) {
 		return
 	}
 	text := chunker.NormalizeLineEndings(req.Text)
+	text = docparser.NormalizeHTMLTables(text)
 
 	cfg := chunker.NormalizeSplitterConfig(chunker.SplitterConfig{
 		ChunkSize:    req.ChunkingConfig.ChunkSize,

internal/infrastructure/chunker/protected_spans_test.go

@@ -0,0 +1,135 @@
+package chunker
+
+import (
+	"strconv"
+	"strings"
+	"testing"
+)
+
+// buildOCRStrayBracketDoc models the OCR failure mode behind this regression:
+// a full-width "【公章】" whose opening bracket was mis-read as the half-width
+// '[' (closing bracket lost), followed later by an unrelated half-width
+// "](...)" fragment. CommonMark forbids a link's text/destination from
+// spanning a blank line, but the previous unbounded
+// `\[[^\]]*\]\([^)]+\)` joined the stray '[' to the distant '](' and
+// swallowed the entire document as one protected atomic span.
+func buildOCRStrayBracketDoc() string {
+	var sb strings.Builder
+	sb.WriteString("[公章】\n\n")
+	for i := 0; i < 60; i++ {
+		sb.WriteString("这是第")
+		sb.WriteString(strconv.Itoa(i))
+		sb.WriteString("段用于测试切分的中文内容，包含常见标点符号。\n\n")
+	}
+	sb.WriteString("第三段（北京)\n\n")
+	sb.WriteString("[引用](来源)")
+	return sb.String()
+}
+
+func maxChunkRunes(chunks []Chunk) int {
+	best := 0
+	for _, c := range chunks {
+		if n := len([]rune(c.Content)); n > best {
+			best = n
+		}
+	}
+	return best
+}
+
+// TestSplitText_OCRStrayBracketDoesNotSwallowDocument is the core regression:
+// before the fix the malformed '[' turned the whole document into a single
+// protected span (~1700 runes) and SplitText returned one oversized chunk.
+func TestSplitText_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {
+	doc := buildOCRStrayBracketDoc()
+	if !strings.Contains(doc, "[公章】") || !strings.Contains(doc, "](来源)") {
+		t.Fatalf("fixture missing malformed-bracket markers")
+	}
+	const chunkSize = 128
+	cfg := SplitterConfig{ChunkSize: chunkSize, ChunkOverlap: 0, Separators: []string{"\n\n", "\n", "。"}}
+
+	chunks := SplitText(doc, cfg)
+	t.Logf("doc=%d runes chunkSize=%d -> chunks=%d max=%d runes",
+		len([]rune(doc)), chunkSize, len(chunks), maxChunkRunes(chunks))
+	if len(chunks) < 2 {
+		t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))
+	}
+	if got := maxChunkRunes(chunks); got > 2*chunkSize {
+		t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)
+	}
+}
+
+// TestSplit_OCRStrayBracketDoesNotSwallowDocument exercises the strategy-aware
+// entry point with the legacy tier pinned, so the protected-pattern behaviour
+// is what is under test rather than the profiler's tier selection.
+func TestSplit_OCRStrayBracketDoesNotSwallowDocument(t *testing.T) {
+	doc := buildOCRStrayBracketDoc()
+	const chunkSize = 128
+	cfg := SplitterConfig{
+		ChunkSize:    chunkSize,
+		ChunkOverlap: 0,
+		Separators:   []string{"\n\n", "\n", "。"},
+		Strategy:     StrategyLegacy,
+	}
+
+	chunks := Split(doc, cfg)
+	if len(chunks) < 2 {
+		t.Fatalf("expected multiple chunks, got %d (max=%d runes)", len(chunks), maxChunkRunes(chunks))
+	}
+	if got := maxChunkRunes(chunks); got > 2*chunkSize {
+		t.Fatalf("max chunk %d runes exceeds 2*chunkSize=%d: stray '[' swallowed the document", got, 2*chunkSize)
+	}
+}
+
+// TestProtectedSpans_NoCrossLineLinkOrImage asserts no protected span may
+// contain a newline for the malformed document: link text and destination
+// cannot span a blank line (CommonMark).
+func TestProtectedSpans_NoCrossLineLinkOrImage(t *testing.T) {
+	doc := buildOCRStrayBracketDoc()
+	for _, s := range protectedSpans(doc) {
+		seg := doc[s.start:s.end]
+		if strings.Contains(seg, "\n") {
+			t.Fatalf("protected span spans a newline (%d runes): %q", len([]rune(seg)), seg)
+		}
+	}
+}
+
+// TestProtectedSpans_MarkdownLinksAndImagesStillProtected is the positive
+// guard: the newline-bounded patterns must keep recognising well-formed
+// single-line Markdown links and images.
+func TestProtectedSpans_MarkdownLinksAndImagesStillProtected(t *testing.T) {
+	cases := []string{
+		"![alt](http://x/y.png)",
+		"[文字](http://x.com)",
+		`[link](url "title")`,
+		"![](resource://abc)",
+	}
+	for _, c := range cases {
+		spans := protectedSpans(c)
+		if len(spans) == 0 {
+			t.Errorf("expected %q to be protected, got no span", c)
+			continue
+		}
+		if spans[0].start != 0 || spans[0].end != len(c) {
+			t.Errorf("%q: protected span = [%d,%d), want [0,%d)", c, spans[0].start, spans[0].end, len(c))
+		}
+	}
+}
+
+// TestProtectedSpans_LiteralCrossLineExample documents that an isolated
+// malformed bracket pair with NO complete '](...)' construct is never treated
+// as a protected link/image region.
+func TestProtectedSpans_MultilineLinkIntentionallyUnprotected(t *testing.T) {
+	// CommonMark allows a single soft line break in link text. The protected
+	// patterns still refuse it so a stray OCR '[' cannot swallow a paragraph.
+	doc := "[link\ntext](http://example.com)"
+	if spans := protectedSpans(doc); len(spans) != 0 {
+		t.Fatalf("expected multiline link to stay unprotected, got %v", spans)
+	}
+}
+
+func TestProtectedSpans_LiteralCrossLineExample(t *testing.T) {
+	doc := "[公章】\n\n第一段。\n\n第二段。\n\n第三段（北京)"
+	if spans := protectedSpans(doc); len(spans) != 0 {
+		t.Fatalf("expected no protected span, got %v", spans)
+	}
+}

internal/infrastructure/chunker/splitter.go

@@ -114,10 +114,18 @@ func DefaultConfig() SplitterConfig {
 }
 
 // protectedPatterns are regex patterns for content that must not be split.
+//
+// The Markdown link/image patterns deliberately exclude '\n' and bound the
+// link text / destination length: CommonMark forbids them from spanning a
+// blank line, and the previous unbounded [^\]]* / [^)]+ let a stray '[' left
+// behind by OCR swallow whole paragraphs as one "protected" atomic span,
+// defeating chunking entirely.
 var protectedPatterns = []*regexp.Regexp{
-	regexp.MustCompile(`(?s)\$\$.*?\$\$`),                                                               // LaTeX block math
-	regexp.MustCompile(`!\[[^\]]*\]\([^)]+\)`),                                                          // Markdown images
-	regexp.MustCompile(`\[[^\]]*\]\([^)]+\)`),                                                           // Markdown links
+	regexp.MustCompile(`(?s)\$\$.*?\$\$`), // LaTeX block math
+	// Markdown images / links: single line, bounded so a stray OCR '['
+	// cannot swallow a paragraph (CommonMark forbids blank-line spans).
+	regexp.MustCompile(`!\[[^\]\n]{0,200}\]\([^)\n]{1,500}\)`),
+	regexp.MustCompile(`\[[^\]\n]{1,200}\]\([^)\n]{1,500}\)`),
 	regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+\\s*(?:\\|\\s*:?-{3,}:?\\s*)+\\|[\\r\\n]+"), // Table header+separator
 	regexp.MustCompile("(?m)[ ]*(?:\\|[^|\\n]*)+\\|[\\r\\n]+"),                                          // Table rows
 	regexp.MustCompile("(?s)```(?:\\w+)?[\\r\\n].*?```"),                                                // Fenced code blocks

internal/infrastructure/docparser/html_table_normalizer.go

@@ -15,6 +15,10 @@ var (
 	// block, the form OCR/layout engines such as PaddleOCR-VL emit tables in.
 	htmlTableBlockPattern = regexp.MustCompile(`(?is)<table\b[^>]*>.*?</table>`)
 
+	// fencedCodeBlockPattern matches CommonMark fenced code blocks so HTML
+	// <table> examples inside them are left untouched.
+	fencedCodeBlockPattern = regexp.MustCompile("(?s)(?:```|~~~)[^\\n]*\\n.*?(?:```|~~~)")
+
 	// htmlLayoutAttrPattern matches presentational HTML attributes that carry
 	// no semantic value (text-align styles, CSS classes, sizing). Structural
 	// attributes like rowspan/colspan are intentionally excluded.
@@ -22,28 +26,45 @@ var (
 		`(?is)\s+(?:style|class|align|valign|width|height|bgcolor)\s*=\s*(?:"[^"]*"|'[^']*'|[^\s>]+)`,
 	)
 
-	// htmlSpanAttrPattern detects rowspan/colspan, which Markdown tables cannot
-	// represent; such tables keep their HTML form (attributes stripped) instead.
-	htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\b`)
+	// htmlSpanAttrPattern detects rowspan/colspan values greater than 1, which
+	// Markdown tables cannot represent; such tables keep their HTML form
+	// (attributes stripped) instead. Span values of 1 (and the invalid 0) do
+	// not merge anything and must stay convertible.
+	htmlSpanAttrPattern = regexp.MustCompile(`(?i)\b(?:row|col)span\s*=\s*["']?(?:[2-9]|\d{2,})`)
 
-	// markdownTableSeparatorPattern matches the |---|---| delimiter row that a
-	// valid GFM table must contain.
-	markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)+\|?\s*$`)
+	// htmlTableRowPattern matches the opening <tr> tag of each table row, used
+	// to put every row on its own line so the chunker can split degraded HTML
+	// tables at "\n" boundaries.
+	htmlTableRowPattern = regexp.MustCompile(`(?i)<tr\b`)
+
+	// markdownTableSeparatorPattern matches the |---| delimiter row that a
+	// valid GFM table must contain. The repeating column group is optional so
+	// single-column tables (|---|) are accepted; requiring two or more columns
+	// discarded successful conversions of MinerU figure/TOC tables.
+	markdownTableSeparatorPattern = regexp.MustCompile(`(?m)^\s*\|?\s*:?-+:?\s*(?:\|\s*:?-+:?\s*)*\|?\s*$`)
 )
 
-// normalizeHTMLTables rewrites inline HTML <table> blocks embedded in OCR
+// NormalizeHTMLTables rewrites inline HTML <table> blocks embedded in OCR
 // markdown output. PaddleOCR-VL emits tables as HTML with per-cell text-align
 // styles, which (1) waste tokens on layout markup and (2) are not recognized
 // by the chunker's table-protection logic, so large tables get split mid-row.
 //
 // Each table block is converted to a GFM Markdown table when possible. Tables
-// that use rowspan/colspan (which Markdown cannot express) fall back to having
-// their presentational attributes stripped so they stay intact as HTML.
-func normalizeHTMLTables(md string) string {
+// that use rowspan/colspan (which Markdown cannot express), or that the
+// converter cannot turn into a valid GFM table (cell <br>, nested lists, …),
+// fall back to presentational-attribute stripping plus one row per line so
+// the chunker can still split on "\n". Tables inside fenced code blocks are
+// left unchanged.
+func NormalizeHTMLTables(md string) string {
 	if !strings.Contains(strings.ToLower(md), "<table") {
 		return md
 	}
 
+	locs := htmlTableBlockPattern.FindAllStringIndex(md, -1)
+	if len(locs) == 0 {
+		return md
+	}
+
 	conv := converter.NewConverter(
 		converter.WithPlugins(
 			base.NewBasePlugin(),
@@ -51,23 +72,60 @@ func normalizeHTMLTables(md string) string {
 			table.NewTablePlugin(),
 		),
 	)
+	fences := fencedCodeBlockPattern.FindAllStringIndex(md, -1)
 
-	return htmlTableBlockPattern.ReplaceAllStringFunc(md, func(block string) string {
-		if htmlSpanAttrPattern.MatchString(block) {
-			return stripHTMLLayoutAttrs(block)
+	var b strings.Builder
+	last := 0
+	for _, loc := range locs {
+		b.WriteString(md[last:loc[0]])
+		block := md[loc[0]:loc[1]]
+		if indexInsideSpans(loc[0], fences) {
+			b.WriteString(block)
+			last = loc[1]
+			continue
 		}
-		converted, err := conv.ConvertString(block)
-		if err != nil {
-			return stripHTMLLayoutAttrs(block)
+		// Isolate the rewritten table with blank lines without stacking pads
+		// on repeated NormalizeHTMLTables calls.
+		out := strings.TrimRight(b.String(), "\n")
+		b.Reset()
+		b.WriteString(out)
+		b.WriteString("\n\n")
+		b.WriteString(normalizeOneHTMLTable(conv, block))
+		last = loc[1]
+		for last < len(md) && md[last] == '\n' {
+			last++
 		}
-		converted = strings.TrimSpace(converted)
-		if converted == "" || !markdownTableSeparatorPattern.MatchString(converted) {
-			return stripHTMLLayoutAttrs(block)
+		b.WriteString("\n\n")
+	}
+	b.WriteString(md[last:])
+	return b.String()
+}
+
+func normalizeOneHTMLTable(conv *converter.Converter, block string) string {
+	fallback := func() string {
+		return splitHTMLTableRows(stripHTMLLayoutAttrs(block))
+	}
+	if htmlSpanAttrPattern.MatchString(block) {
+		return fallback()
+	}
+	converted, err := conv.ConvertString(block)
+	if err != nil {
+		return fallback()
+	}
+	converted = unescapeMarkdownImageSyntax(strings.TrimSpace(converted))
+	if converted == "" || !markdownTableSeparatorPattern.MatchString(converted) {
+		return fallback()
+	}
+	return converted
+}
+
+func indexInsideSpans(pos int, spans [][]int) bool {
+	for _, s := range spans {
+		if pos >= s[0] && pos < s[1] {
+			return true
 		}
-		// Pad with blank lines so the Markdown table is a standalone block that
-		// the chunker recognizes (and protects) as a table.
-		return "\n\n" + converted + "\n\n"
-	})
+	}
+	return false
 }
 
 // stripHTMLLayoutAttrs removes presentational attributes from an HTML fragment
@@ -75,3 +133,27 @@ func normalizeHTMLTables(md string) string {
 func stripHTMLLayoutAttrs(html string) string {
 	return htmlLayoutAttrPattern.ReplaceAllString(html, "")
 }
+
+// splitHTMLTableRows puts each table row on its own line. The chunker splits
+// on "\n", so a single-line HTML table would otherwise be unsplittable and get
+// force-cut at the absolute max size. Whitespace between tags is insignificant
+// in HTML, and newlines are only inserted before <tr> (never inside a tag).
+// A <tr> that already starts on its own line is left alone.
+func splitHTMLTableRows(block string) string {
+	locs := htmlTableRowPattern.FindAllStringIndex(block, -1)
+	if len(locs) == 0 {
+		return strings.TrimSpace(block)
+	}
+	var b strings.Builder
+	last := 0
+	for _, loc := range locs {
+		b.WriteString(block[last:loc[0]])
+		if loc[0] == 0 || block[loc[0]-1] != '\n' {
+			b.WriteByte('\n')
+		}
+		b.WriteString(block[loc[0]:loc[1]])
+		last = loc[1]
+	}
+	b.WriteString(block[last:])
+	return strings.TrimSpace(b.String())
+}

internal/infrastructure/docparser/html_table_normalizer_test.go

@@ -16,7 +16,7 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {
 
 结尾。`
 
-	got := normalizeHTMLTables(input)
+	got := NormalizeHTMLTables(input)
 
 	if strings.Contains(got, "<table") {
 		t.Fatalf("expected HTML table to be converted away, got:\n%s", got)
@@ -37,13 +37,92 @@ func TestNormalizeHTMLTables_ConvertsStyledTableToMarkdown(t *testing.T) {
 	}
 }
 
+func TestNormalizeHTMLTables_NoTableUnchanged(t *testing.T) {
+	input := "# 标题\n\n普通段落，没有表格。\n\n| a | b |\n| --- | --- |\n| 1 | 2 |"
+	if got := NormalizeHTMLTables(input); got != input {
+		t.Fatalf("expected content without HTML tables to be unchanged, got:\n%s", got)
+	}
+}
+
+func TestNormalizeHTMLTables_SpanValueOneIsConvertible(t *testing.T) {
+	// rowspan=1 / colspan=1 merge nothing, so the table must still become GFM.
+	// Covers quoted, unquoted and space-padded attribute forms.
+	input := `<table><tr>` +
+		`<td rowspan="1" colspan="1" style="text-align:center;">指标</td>` +
+		`<td rowspan=1 colspan=1>数值</td></tr>` +
+		`<tr><td rowspan="1" colspan='1'>营收</td>` +
+		`<td rowspan = 1 colspan = 1>10亿</td></tr></table>`
+
+	got := NormalizeHTMLTables(input)
+
+	if strings.Contains(got, "<table") {
+		t.Fatalf("expected span=1 table to be converted away, got:\n%s", got)
+	}
+	if !markdownTableSeparatorPattern.MatchString(got) {
+		t.Fatalf("expected a Markdown table separator row, got:\n%s", got)
+	}
+	for _, want := range []string{"指标", "数值", "营收", "10亿"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)
+		}
+	}
+}
+
+func TestNormalizeHTMLTables_RealSpanKeepsHTMLWithRowNewlines(t *testing.T) {
+	// Genuine merges (colspan>1 / rowspan>1) cannot be GFM, so the table stays
+	// HTML — but each row must start on its own line so the chunker can split it.
+	input := `<table><tr><td colspan="6" style="text-align:center;">合计</td></tr>` +
+		`<tr><td rowspan="3">A</td><td>1</td></tr>` +
+		`<tr><td rowspan = 2>B</td><td>2</td></tr></table>`
+
+	got := NormalizeHTMLTables(input)
+
+	if !strings.Contains(got, "<table") {
+		t.Fatalf("expected real span table to remain HTML, got:\n%s", got)
+	}
+	if !strings.Contains(got, `colspan="6"`) || !strings.Contains(got, `rowspan="3"`) {
+		t.Fatalf("expected structural span attrs preserved, got:\n%s", got)
+	}
+	if !strings.Contains(got, "\n<tr") {
+		t.Fatalf("expected each <tr> to start on a new line, got:\n%q", got)
+	}
+	if !strings.Contains(got, "\n\n<table") {
+		t.Fatalf("expected block to be padded at the head with blank lines, got:\n%q", got)
+	}
+	if !strings.HasSuffix(got, "</table>\n\n") {
+		t.Fatalf("expected block to be padded at the tail with blank lines, got:\n%q", got)
+	}
+	if n := strings.Count(got, "\n<tr"); n != 3 {
+		t.Fatalf("expected 3 rows each preceded by a newline, got %d in:\n%q", n, got)
+	}
+}
+
+func assertHTMLRowsSplittable(t *testing.T, got string) {
+	t.Helper()
+	if !strings.Contains(got, "<tr") {
+		t.Fatalf("expected remaining HTML table rows, got:\n%s", got)
+	}
+	for i := 0; ; {
+		idx := strings.Index(got[i:], "<tr")
+		if idx < 0 {
+			return
+		}
+		abs := i + idx
+		if abs == 0 || got[abs-1] != '\n' {
+			t.Fatalf("remaining <tr> at offset %d is not preceded by a newline:\n%s", abs, got)
+		}
+		i = abs + 1
+	}
+}
+
 func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {
 	// rowspan/colspan cannot be expressed in Markdown, so the table stays HTML
-	// but its presentational attributes are stripped.
-	input := `<table><tr><td colspan="2" style="text-align:center;" class="hdr">合计</td></tr>` +
+	// but its presentational attributes are stripped and rows become splittable.
+	input := `<table><tr>` +
+		`<td colspan="2" style="text-align:center;" class="hdr">合计</td></tr>` +
 		`<tr><td style="text-align:left;">A</td><td width="80">B</td></tr></table>`
 
-	got := normalizeHTMLTables(input)
+	got := NormalizeHTMLTables(input)
 
 	if !strings.Contains(got, "<table") {
 		t.Fatalf("expected span table to remain HTML, got:\n%s", got)
@@ -56,11 +135,99 @@ func TestNormalizeHTMLTables_StripsAttrsOnSpanTables(t *testing.T) {
 			t.Fatalf("expected %q to be stripped, got:\n%s", banned, got)
 		}
 	}
+	assertHTMLRowsSplittable(t, got)
 }
 
-func TestNormalizeHTMLTables_NoTableUnchanged(t *testing.T) {
-	input := "# 标题\n\n普通段落，没有表格。\n\n| a | b |\n| --- | --- |\n| 1 | 2 |"
-	if got := normalizeHTMLTables(input); got != input {
-		t.Fatalf("expected content without HTML tables to be unchanged, got:\n%s", got)
+func TestNormalizeHTMLTables_BRCellsStaySplittableHTML(t *testing.T) {
+	// <br> inside a cell makes the table plugin give up (it emits a newline).
+	// The HTML fallback must still put each <tr> on its own line.
+	input := `<table><tr><td>指标<br/>名称</td><td>数值<br/>单位</td></tr>` +
+		`<tr><td>拉伸强度<br/>MPa</td><td>合格<br/>A级</td></tr></table>`
+
+	got := NormalizeHTMLTables(input)
+
+	if !strings.Contains(got, "<table") {
+		t.Fatalf("expected <br> table to remain HTML rather than flattened text, got:\n%s", got)
+	}
+	assertHTMLRowsSplittable(t, got)
+	for _, want := range []string{"指标", "名称", "拉伸强度", "合格"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("expected fallback HTML to retain %q, got:\n%s", want, got)
+		}
+	}
+}
+
+func TestNormalizeHTMLTables_ListInCellFallsBackToSplittableHTML(t *testing.T) {
+	input := `<table><tr><td><ul><li>a</li><li>b</li></ul></td><td>x</td></tr>` +
+		`<tr><td>c</td><td>d</td></tr></table>`
+
+	got := NormalizeHTMLTables(input)
+
+	if !strings.Contains(got, "<table") {
+		t.Fatalf("expected list-in-cell table to remain HTML, got:\n%s", got)
+	}
+	assertHTMLRowsSplittable(t, got)
+}
+
+func TestNormalizeHTMLTables_SingleColumnConvertsToGFM(t *testing.T) {
+	input := `<table><tr><td>目录</td></tr><tr><td>第一章</td></tr><tr><td>第二章</td></tr></table>`
+
+	got := NormalizeHTMLTables(input)
+
+	if strings.Contains(got, "<table") {
+		t.Fatalf("expected single-column table to convert to GFM, got:\n%s", got)
+	}
+	if !markdownTableSeparatorPattern.MatchString(got) {
+		t.Fatalf("expected a Markdown table separator row, got:\n%s", got)
+	}
+	for _, want := range []string{"目录", "第一章", "第二章"} {
+		if !strings.Contains(got, want) {
+			t.Fatalf("expected converted table to retain %q, got:\n%s", want, got)
+		}
+	}
+}
+
+func TestNormalizeHTMLTables_ImgInSingleColumnBecomesGFM(t *testing.T) {
+	input := `<table><tr><td><img src="local://images/profile.png" alt="profile"/></td></tr></table>`
+
+	got := NormalizeHTMLTables(input)
+
+	if strings.Contains(got, "<table") {
+		t.Fatalf("expected single-column image table to convert to GFM, got:\n%s", got)
+	}
+	if !strings.Contains(got, "![profile](local://images/profile.png)") {
+		t.Fatalf("expected markdown image to be preserved, got:\n%s", got)
+	}
+}
+
+func TestNormalizeHTMLTables_MarkdownImageInCellNotEscaped(t *testing.T) {
+	input := `<table><tr><td>![cover](local://images/cover.png)</td><td>说明</td></tr></table>`
+
+	got := NormalizeHTMLTables(input)
+
+	if strings.Contains(got, `!\[`) {
+		t.Fatalf("markdown image syntax escaped:\n%s", got)
+	}
+	if !strings.Contains(got, "![cover](local://images/cover.png)") {
+		t.Fatalf("expected markdown image to survive conversion, got:\n%s", got)
+	}
+}
+
+func TestNormalizeHTMLTables_CodeFenceTableLeftAlone(t *testing.T) {
+	input := "示例：\n\n```html\n<table><tr><td>a</td><td>b</td></tr></table>\n```\n"
+
+	got := NormalizeHTMLTables(input)
+
+	if got != input {
+		t.Fatalf("expected fenced HTML table to be unchanged, got:\n%s", got)
+	}
+}
+
+func TestNormalizeHTMLTables_SpanTableIdempotent(t *testing.T) {
+	input := `<table><tr><td colspan="6">汇总</td></tr><tr><td colspan="6">备注</td></tr></table>`
+	once := NormalizeHTMLTables(input)
+	twice := NormalizeHTMLTables(once)
+	if once != twice {
+		t.Fatalf("NormalizeHTMLTables is not idempotent for span tables:\n1=%q\n2=%q", once, twice)
 	}
 }

internal/infrastructure/docparser/paddleocr_vl_cloud_converter.go

@@ -78,7 +78,7 @@ func (c *PaddleOCRVLCloudReader) Read(ctx context.Context, req *types.ReadReques
 		return nil, fmt.Errorf("PaddleOCR-VL Cloud fetch results: %w", err)
 	}
 
-	mdContent = normalizeHTMLTables(mdContent)
+	mdContent = NormalizeHTMLTables(mdContent)
 
 	imageRefs := c.downloadImages(mdContent, imagesURL)
 	mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)

internal/infrastructure/docparser/paddleocr_vl_converter.go

@@ -65,7 +65,7 @@ func (c *PaddleOCRVLReader) Read(ctx context.Context, req *types.ReadRequest) (*
 	// wastes tokens and defeats the chunker's table-protection logic. Convert
 	// them to Markdown tables (or strip layout attributes when conversion is
 	// not possible) before downstream processing.
-	mdContent = normalizeHTMLTables(mdContent)
+	mdContent = NormalizeHTMLTables(mdContent)
 
 	imageRefs, mdContent := c.processImages(mdContent, imagesB64)
 	mdContent, imageRefs = ensureOriginalImageRef(req, mdContent, imageRefs)