fix(engine): generalize response charset detection with full WHATWG encoding support and remove the uncertain-detection error path

#870

wafenginecore/content_encoding.go

@@ -13,14 +13,142 @@ import (
 	"github.com/klauspost/compress/zstd"
 	"golang.org/x/net/html/charset"
 	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/htmlindex"
 	"golang.org/x/text/encoding/simplifiedchinese"
-	"golang.org/x/text/encoding/unicode"
 	"golang.org/x/text/transform"
 	"io"
 	"net/http"
 	"strings"
+	"unicode/utf8"
 )
 
+// encodingDetectPeekSize 编码自动检测的探测窗口大小（meta声明可能在1024字节之后）
+const encodingDetectPeekSize = 8192
+
+// rawLatin1Charset 内部无损兜底编码名：真正的ISO-8859-1（256字节全双射，解码→处理→回写字节级无损）
+// 不能直接用"iso-8859-1"标签——WHATWG会把它归一到windows-1252，而windows-1252有5个
+// 未定义字节（0x81/0x8D/0x8F/0x90/0x9D）解码会变成U+FFFD，做不到无损往返
+const rawLatin1Charset = "x-raw-latin1"
+
+// resolveCharset 将任意charset名称解析为编码，支持全部WHATWG标准编码（约40种、200+别名）
+// 返回编码实例和规范名；无法识别、auto、或映射到replacement伪编码时返回nil
+func resolveCharset(name string) (encoding.Encoding, string) {
+	name = strings.Trim(strings.TrimSpace(name), `"'`)
+	if name == "" || strings.EqualFold(name, "auto") {
+		return nil, ""
+	}
+	if strings.EqualFold(name, rawLatin1Charset) {
+		return charmap.ISO8859_1, rawLatin1Charset
+	}
+	// 用htmlindex取原始编码而非charset.Lookup：后者返回的编码器自带HTML转义包装，
+	// 不可编码字符会变成&#NNN;（对JSON/JS响应会破坏语法），原始编码器才能配合ReplaceUnsupported
+	enc, lookupErr := htmlindex.Get(name)
+	if lookupErr != nil {
+		return nil, ""
+	}
+	canonical, nameErr := htmlindex.Name(enc)
+	// replacement伪编码（hz-gb-2312/iso-2022-cn等危险标签）解码会把整个body变成U+FFFD，不能使用
+	if nameErr != nil || canonical == "replacement" {
+		return nil, ""
+	}
+	return enc, canonical
+}
+
+// charsetFromContentType 从Content-Type头中提取charset值，未指定时返回空
+func charsetFromContentType(contentType string) string {
+	for _, part := range strings.Split(contentType, ";") {
+		part = strings.TrimSpace(part)
+		if len(part) > 8 && strings.EqualFold(part[:8], "charset=") {
+			return strings.Trim(part[8:], `"'`)
+		}
+	}
+	return ""
+}
+
+// charsetFromHTMLHead 在HTML头部内容中扫描charset声明或DOCTYPE，返回解析出的编码
+// 作为DetermineEncoding（只预扫描前1024字节）的扩窗补充
+func charsetFromHTMLHead(htmlHead string) (encoding.Encoding, string) {
+	if idx := strings.Index(strings.ToLower(htmlHead), "charset="); idx != -1 {
+		substr := strings.TrimLeft(htmlHead[idx+8:], "\"' ")
+		end := len(substr)
+		for i, c := range substr {
+			if !(c == '-' || c == '_' || c == ':' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
+				end = i
+				break
+			}
+		}
+		if enc, canonical := resolveCharset(substr[:end]); enc != nil {
+			return enc, canonical
+		}
+	}
+	// XHTML 1.0 通常默认utf-8
+	if strings.Contains(htmlHead, "xhtml1-transitional.dtd") {
+		return resolveCharset("utf-8")
+	}
+	return nil, ""
+}
+
+// validUTF8Prefix 判断字节序列是否为合法UTF-8（容忍末尾被探测窗口截断的多字节字符）
+func validUTF8Prefix(b []byte) bool {
+	for trim := 0; trim < utf8.UTFMax; trim++ {
+		if utf8.Valid(b) {
+			return true
+		}
+		if len(b) == 0 || b[len(b)-1] < 0x80 {
+			return false
+		}
+		b = b[:len(b)-1]
+	}
+	return false
+}
+
+// looksLikeGBK 无声明时的GBK启发式探测。
+// 西欧latin-1文本中"高位字节+ASCII字母"恰好也是合法GBK双字节（如é=0xE9后跟't'），
+// 为避免误判，要求所有双字节序列的尾字节也是高位字节（GB2312常用汉字尾字节均≥0xA1），
+// 且至少出现2对；宁可放弃判定走无损兜底，也不错误转码
+func looksLikeGBK(b []byte) bool {
+	highPairs := 0
+	for i := 0; i < len(b); {
+		c := b[i]
+		if c < 0x80 {
+			i++
+			continue
+		}
+		if c == 0x80 || c == 0xFF {
+			// 非法GBK首字节
+			return false
+		}
+		if i+1 >= len(b) {
+			// 末尾被探测窗口截断半个双字节字符，容忍
+			break
+		}
+		trail := b[i+1]
+		if trail < 0xA1 || trail == 0xFF {
+			// 尾字节非高位（GBK扩展区）或非法：西欧文本特征，放弃GBK判定
+			return false
+		}
+		highPairs++
+		i += 2
+	}
+	if highPairs < 2 {
+		return false
+	}
+	if gbkDecodesClean(b) {
+		return true
+	}
+	// 末尾可能被探测窗口截断半个双字节字符，剪掉最后一个字节再试
+	return b[len(b)-1] >= 0x80 && gbkDecodesClean(b[:len(b)-1])
+}
+
+func gbkDecodesClean(b []byte) bool {
+	decoded, err := simplifiedchinese.GBK.NewDecoder().Bytes(b)
+	if err != nil {
+		return false
+	}
+	return !bytes.ContainsRune(decoded, utf8.RuneError)
+}
+
 // 返回内容前依据情况进行返回压缩数据
 func (waf *WafEngine) compressContent(res *http.Response, isStaticAssist bool, inputBytes []byte, encodeType string) (respBytes []byte, err error) {
 
@@ -29,38 +157,25 @@ func (waf *WafEngine) compressContent(res *http.Response, isStaticAssist bool, i
 		return inputBytes, errors.New("静态资源或资源类型请求，跳过编码转换")
 	}
 
-	// 首先检查Content-Type头中是否明确指定了字符集
-	contentType := res.Header.Get("Content-Type")
-	var encodedBytes []byte = inputBytes
+	// 优先使用检测阶段返回的编码，否则从Content-Type头提取
 	charsetName := encodeType
-
-	// 如果Content-Type中指定了字符集，需要将UTF-8编码的内容转换回原始编码
-	if charsetName == "" && strings.Contains(contentType, "charset=") {
-
-		parts := strings.Split(contentType, ";")
-		for _, part := range parts {
-			part = strings.TrimSpace(part)
-			if strings.HasPrefix(part, "charset=") {
-				charsetName = strings.TrimPrefix(part, "charset=")
-				charsetName = strings.Trim(charsetName, `"'`)
-				break
-			}
-		}
+	if charsetName == "" {
+		charsetName = charsetFromContentType(res.Header.Get("Content-Type"))
 	}
-	// 根据字符集进行编码转换
+
+	encodedBytes := inputBytes
+	// 内容在处理阶段是UTF-8，回写前需要编码回原始字符集
 	if charsetName != "" && !strings.EqualFold(charsetName, "utf-8") && !strings.EqualFold(charsetName, "utf8") {
-		switch strings.ToLower(charsetName) {
-		case "gbk", "gb2312":
-			// 将UTF-8转换为GBK
-			encoder := simplifiedchinese.GBK.NewEncoder()
-			encodedBytes, err = encoder.Bytes(inputBytes)
-			if err != nil {
-				zlog.Warn("编码转换失败(UTF-8 -> GBK): %v", err)
+		if enc, canonical := resolveCharset(charsetName); enc != nil {
+			// ReplaceUnsupported：目标字符集无法表示的字符替换为占位符而非报错中断
+			converted, encodeErr := encoding.ReplaceUnsupported(enc.NewEncoder()).Bytes(inputBytes)
+			if encodeErr != nil {
+				zlog.Warn("编码转换失败(UTF-8 -> %s): %v", canonical, encodeErr)
 				// 转换失败时使用原始UTF-8内容
-				encodedBytes = inputBytes
+			} else {
+				encodedBytes = converted
 			}
-		// 可以根据需要添加其他编码的支持
-		default:
+		} else {
 			zlog.Debug("不支持的字符集编码转换: %s，保持UTF-8编码", charsetName)
 		}
 	}
@@ -87,7 +202,6 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
 	if err != nil {
 		return bodyBytes, "", fmt.Errorf("读取原始响应体失败: %v", err)
 	}
-	charsetNameFinal := ""
 	// 重新设置响应体，以便后续处理
 	resp.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
 
@@ -96,11 +210,11 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
 		return bodyBytes, "", errors.New("静态资源或资源类型请求，跳过编码转换")
 	}
 
-	// 根据内容编码处理压缩
+	// 根据内容编码处理压缩（从独立reader读取，保持resp.Body完整不被消耗）
 	var bodyReader io.Reader
 	switch resp.Header.Get("Content-Encoding") {
 	case "gzip":
-		gzipReader, gzipErr := gzip.NewReader(resp.Body)
+		gzipReader, gzipErr := gzip.NewReader(bytes.NewReader(bodyBytes))
 		if gzipErr != nil {
 			zlog.Warn("gzip解压失败: %v", gzipErr)
 			// 失败时返回错误
@@ -109,14 +223,13 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
 		bodyReader = gzipReader
 		defer gzipReader.Close()
 	case "deflate":
-		deflateReader := flate.NewReader(resp.Body)
+		deflateReader := flate.NewReader(bytes.NewReader(bodyBytes))
 		bodyReader = deflateReader
 		defer deflateReader.Close()
 	case "br":
-		brotliReader := brotli.NewReader(resp.Body)
-		bodyReader = brotliReader
+		bodyReader = brotli.NewReader(bytes.NewReader(bodyBytes))
 	case "zstd":
-		zstdReader, zstdErr := zstd.NewReader(resp.Body)
+		zstdReader, zstdErr := zstd.NewReader(bytes.NewReader(bodyBytes))
 		if zstdErr != nil {
 			zlog.Warn("zstd解压失败: %v", zstdErr)
 			return bodyBytes, "", fmt.Errorf("zstd解压失败: %v", zstdErr)
@@ -124,116 +237,83 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
 		bodyReader = zstdReader
 		defer zstdReader.Close()
 	default:
-		bodyReader = resp.Body
+		bodyReader = bytes.NewReader(bodyBytes)
 	}
 	// 创建缓冲读取器
-	bufReader := bufio.NewReader(bodyReader)
+	bufReader := bufio.NewReaderSize(bodyReader, encodingDetectPeekSize)
 
-	// 首先检查Content-Type头中是否明确指定了字符集
 	contentType := resp.Header.Get("Content-Type")
 	var currentEncoding encoding.Encoding
+	charsetNameFinal := ""
 
-	if defaultEncoding != "" {
-		switch strings.ToLower(defaultEncoding) {
-		case "utf-8", "utf8":
-			currentEncoding = unicode.UTF8
-			charsetNameFinal = "utf-8"
-			break
-		case "gbk", "gb2312":
-			currentEncoding = simplifiedchinese.GBK
-			charsetNameFinal = "gbk"
-			break
-		default:
-		}
-	} else if strings.Contains(contentType, "charset=") {
-		charsetName := ""
-		parts := strings.Split(contentType, ";")
-		for _, part := range parts {
-			part = strings.TrimSpace(part)
-			if strings.HasPrefix(part, "charset=") {
-				charsetName = strings.TrimPrefix(part, "charset=")
-				charsetName = strings.Trim(charsetName, `"'`)
-				break
-			}
-		}
+	// 优先级1：站点配置的默认编码（auto或空表示走自动检测）
+	if enc, name := resolveCharset(defaultEncoding); enc != nil {
+		currentEncoding = enc
+		charsetNameFinal = name
+	} else if defaultEncoding != "" && !strings.EqualFold(defaultEncoding, "auto") {
+		zlog.Debug("站点默认编码无法识别: %s，转入自动检测", defaultEncoding)
+	}
 
-		// 如果找到了字符集
-		if charsetName != "" {
-			zlog.Debug("从Content-Type中检测到字符集: %s", charsetName)
-			switch strings.ToLower(charsetName) {
-			case "utf-8", "utf8":
-				currentEncoding = unicode.UTF8
-				charsetNameFinal = "utf-8"
-				break
-			case "gbk", "gb2312":
-				currentEncoding = simplifiedchinese.GBK
-				charsetNameFinal = "gbk"
-				break
-			default:
-				currentEncoding = nil // 使用自动检测
+	// 优先级2：Content-Type头中显式指定的字符集
+	if currentEncoding == nil {
+		if headerCharset := charsetFromContentType(contentType); headerCharset != "" {
+			if enc, name := resolveCharset(headerCharset); enc != nil {
+				zlog.Debug("从Content-Type中检测到字符集: %s", name)
+				currentEncoding = enc
+				charsetNameFinal = name
 			}
 		}
 	}
 
-	// 如果没有从Content-Type中获取到编码或获取的编码不支持，则使用自动检测
+	// 优先级3：自动检测（BOM/meta预扫描 → 扩窗meta扫描 → UTF-8校验 → GBK探测 → windows-1252无损兜底）
 	if currentEncoding == nil {
-		// 增加检测字节数到1024，提高准确性
-		peekBytes, peekErr := bufReader.Peek(1024)
+		peekBytes, peekErr := bufReader.Peek(encodingDetectPeekSize)
 		if peekErr != nil && peekErr != io.EOF {
-			return bodyBytes, "", errors.New(fmt.Sprintf("编码检测错误，Peek失败: %v", peekErr))
-		} else {
-			// 先尝试用系统自带的自动检测
-			detectedEncoding, name, certain := charset.DetermineEncoding(peekBytes, contentType)
-
-			if !certain {
-				// 自动检测不确定时，尝试从HTML头部meta标签或DOCTYPE中提取编码
-				htmlHead := string(peekBytes)
-				metaCharset := ""
-				// 1. 检查meta标签
-				metaIdx := strings.Index(strings.ToLower(htmlHead), "charset=")
-				if metaIdx != -1 {
-					// 提取charset=后面的内容
-					substr := htmlHead[metaIdx+8:]
-					// 只取等号后面第一个非字母数字下划线的内容
-					for i, c := range substr {
-						if !(c == '-' || c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
-							metaCharset = substr[:i]
-							break
-						}
-					}
-					if metaCharset == "" {
-						metaCharset = substr
-					}
-					metaCharset = strings.Trim(metaCharset, "\"' />")
-				}
-				// 2. 检查DOCTYPE声明
-				if metaCharset == "" && strings.Contains(htmlHead, "xhtml1-transitional.dtd") {
-					// XHTML 1.0 通常默认utf-8或iso-8859-1
-					metaCharset = "utf-8"
-				}
-				if metaCharset != "" {
-					zlog.Debug("通过HTML头部meta/doctype辅助检测到字符集: %s", metaCharset)
-					switch strings.ToLower(metaCharset) {
-					case "utf-8", "utf8":
-						currentEncoding = unicode.UTF8
-						charsetNameFinal = "utf-8"
-						break
-					case "gbk", "gb2312":
-						currentEncoding = simplifiedchinese.GBK
-						charsetNameFinal = "gbk"
-						break
-					default:
-						zlog.Debug("meta标签检测到未知字符集: %s，保持UTF-8", metaCharset)
-						return bodyBytes, "", errors.New(fmt.Sprintf("编码检测不确定，且未能从HTML头部meta/doctype提取到编码"))
-					}
-				} else {
-					return bodyBytes, "", errors.New(fmt.Sprintf("编码检测不确定，且未能从HTML头部meta/doctype提取到编码"))
-				}
-			} else {
-				zlog.Debug("编码检测确定为: %s", name)
-				currentEncoding = detectedEncoding
+			return bodyBytes, "", fmt.Errorf("编码检测错误，Peek失败: %v", peekErr)
+		}
+		detectedEncoding, name, certain := charset.DetermineEncoding(peekBytes, contentType)
+		// certain来自BOM或头部charset；非windows-1252的不确定结果来自WHATWG meta预扫描，同样可信
+		// windows-1252是DetermineEncoding找不到任何依据时的默认值，需继续用后续手段甄别
+		if name != "replacement" && (certain || name != "windows-1252") {
+			zlog.Debug("编码检测结果: %s (certain=%v)", name, certain)
+			currentEncoding = detectedEncoding
+			charsetNameFinal = name
+		}
+		if currentEncoding == nil {
+			// meta声明可能在1024字节之后，扩大窗口手工扫描
+			if enc, name := charsetFromHTMLHead(string(peekBytes)); enc != nil {
+				zlog.Debug("通过HTML头部meta/doctype辅助检测到字符集: %s", name)
+				currentEncoding = enc
+				charsetNameFinal = name
 			}
 		}
+		if currentEncoding == nil && validUTF8Prefix(peekBytes) {
+			// 无任何声明但内容是合法UTF-8（含纯ASCII），这是线上最常见的情况
+			zlog.Debug("无编码声明，内容通过UTF-8校验，按utf-8处理")
+			charsetNameFinal = "utf-8"
+			currentEncoding, _ = charset.Lookup("utf-8")
+		}
+		if currentEncoding == nil && looksLikeGBK(peekBytes) {
+			zlog.Debug("无编码声明，GBK探测命中，按gbk处理")
+			currentEncoding = simplifiedchinese.GBK
+			charsetNameFinal = "gbk"
+		}
+		if currentEncoding == nil {
+			// ISO-8859-1为256字节全双射：解码→处理→回写字节级无损，ASCII部分的替换仍然有效
+			zlog.Debug("编码无法确定，使用latin-1无损兜底处理")
+			currentEncoding = charmap.ISO8859_1
+			charsetNameFinal = rawLatin1Charset
+		}
+	}
+
+	// UTF-8内容无需转换直接读取，避免个别非法字节被解码器替换为U+FFFD导致内容变动
+	if strings.EqualFold(charsetNameFinal, "utf-8") {
+		resBodyBytes, readErr := io.ReadAll(bufReader)
+		if readErr != nil {
+			zlog.Warn("读取响应体失败: %v", readErr)
+			return bodyBytes, "", fmt.Errorf("读取响应体失败: %v", readErr)
+		}
+		return resBodyBytes, "utf-8", nil
 	}
 
 	// 使用检测到的编码创建转换读取器

wafenginecore/wafengine_test.go

@@ -6,6 +6,13 @@ import (
 	"bytes"
 	"compress/flate"
 	"compress/gzip"
+	"golang.org/x/text/encoding"
+	"golang.org/x/text/encoding/charmap"
+	"golang.org/x/text/encoding/japanese"
+	"golang.org/x/text/encoding/korean"
+	"golang.org/x/text/encoding/simplifiedchinese"
+	"golang.org/x/text/encoding/traditionalchinese"
+	"golang.org/x/text/encoding/unicode"
 	"io"
 	"net/http"
 	"net/http/httptest"
@@ -202,7 +209,7 @@ func TestGetOrgContent(t *testing.T) {
 			contentEncoding: "",
 			content:         "<html><body>没有指定字符集的内容</body></html>",
 			expectedContent: "<html><body>没有指定字符集的内容</body></html>",
-			expectedErr:     true,
+			expectedErr:     false, // UTF-8内容无声明时通过UTF-8校验兜底解出
 		},
 		{
 			// 对于不支持的字符集，我们跳过内容比较，只检查是否有错误
@@ -453,7 +460,7 @@ func TestGetOrgContent_MetaAndDoctypeCharset(t *testing.T) {
 		t.Errorf("DOCTYPE声明检测失败: err=%v, charset=%s, content=%s", err3, charset3, string(content3))
 	}
 
-	// 4. meta标签未知编码
+	// 4. meta标签未知编码：不再报错，内容为合法UTF-8时通过UTF-8校验兜底解出
 	htmlMetaUnknown := `<html><head><meta charset="unknown-charset"></head><body>未知编码</body></html>`
 	resp4 := &http.Response{
 		StatusCode: 200,
@@ -461,9 +468,9 @@ func TestGetOrgContent_MetaAndDoctypeCharset(t *testing.T) {
 		Body:       io.NopCloser(strings.NewReader(htmlMetaUnknown)),
 	}
 	resp4.Header.Set("Content-Type", "text/html")
-	_, _, err4 := waf.getOrgContent(resp4, false, "")
-	if err4 == nil || !strings.Contains(err4.Error(), "编码检测不确定") {
-		t.Errorf("未知meta编码检测失败，期望返回错误，实际: %v", err4)
+	content4, charset4, err4 := waf.getOrgContent(resp4, false, "")
+	if err4 != nil || charset4 != "utf-8" || !strings.Contains(string(content4), "未知编码") {
+		t.Errorf("未知meta编码应兜底为utf-8: err=%v, charset=%s, content=%s", err4, charset4, string(content4))
 	}
 }
 
@@ -917,3 +924,345 @@ func TestShouldAutoJumpHTTPS_Comprehensive(t *testing.T) {
 		}
 	})
 }
+
+// encodeTestBody 用指定编码器将UTF-8原文编码为目标字符集字节，用于构造测试响应体
+func encodeTestBody(t *testing.T, enc encoding.Encoding, text string) []byte {
+	t.Helper()
+	b, err := enc.NewEncoder().Bytes([]byte(text))
+	if err != nil {
+		t.Fatalf("生成测试内容失败: %v", err)
+	}
+	return b
+}
+
+func newEncodingTestResponse(contentType string, body []byte) *http.Response {
+	resp := &http.Response{
+		StatusCode: 200,
+		Header:     make(http.Header),
+		Body:       io.NopCloser(bytes.NewReader(body)),
+	}
+	if contentType != "" {
+		resp.Header.Set("Content-Type", contentType)
+	}
+	return resp
+}
+
+// TestGetOrgContent_MultiCharsets 覆盖WHATWG各编码家族：Content-Type声明与meta声明两条命中路径
+func TestGetOrgContent_MultiCharsets(t *testing.T) {
+	t.Parallel()
+	zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+	waf := &WafEngine{}
+
+	cases := []struct {
+		label     string            // 声明用的charset标签
+		canonical string            // 期望getOrgContent返回的规范名
+		enc       encoding.Encoding // 生成测试字节用的编码器
+		text      string
+		viaMeta   bool // 是否额外测试meta声明路径
+	}{
+		{"gbk", "gbk", simplifiedchinese.GBK, "这是GBK编码的内容", true},
+		{"gb2312", "gbk", simplifiedchinese.GBK, "这是GB2312标签的内容", false},
+		{"gb18030", "gb18030", simplifiedchinese.GB18030, "GB18030扩展字符𠀀测试", true},
+		{"big5", "big5", traditionalchinese.Big5, "這是繁體中文內容", true},
+		{"shift_jis", "shift_jis", japanese.ShiftJIS, "これは日本語です", true},
+		{"euc-jp", "euc-jp", japanese.EUCJP, "日本語テスト", true},
+		{"iso-2022-jp", "iso-2022-jp", japanese.ISO2022JP, "日本語", false},
+		{"euc-kr", "euc-kr", korean.EUCKR, "한국어 테스트", true},
+		{"iso-8859-1", "windows-1252", charmap.Windows1252, "Café résumé", true},
+		{"iso-8859-2", "iso-8859-2", charmap.ISO8859_2, "Příliš žluťoučký", false},
+		{"iso-8859-15", "iso-8859-15", charmap.ISO8859_15, "€ Café", false},
+		{"windows-1252", "windows-1252", charmap.Windows1252, "€ smart “quotes”", true},
+		{"windows-1251", "windows-1251", charmap.Windows1251, "Привет мир", true},
+		{"koi8-r", "koi8-r", charmap.KOI8R, "Русский текст", false},
+		{"iso-8859-7", "iso-8859-7", charmap.ISO8859_7, "Ελληνικά", false},
+		{"windows-1256", "windows-1256", charmap.Windows1256, "مرحبا", false},
+		{"windows-874", "windows-874", charmap.Windows874, "สวัสดี", false},
+		{"ibm866", "ibm866", charmap.CodePage866, "Тест", false},
+		{"macintosh", "macintosh", charmap.Macintosh, "Café™", false},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		// 路径1：Content-Type头声明charset
+		t.Run(tc.label+"_content_type", func(t *testing.T) {
+			body := encodeTestBody(t, tc.enc, "<html><body>"+tc.text+"</body></html>")
+			resp := newEncodingTestResponse("text/html; charset="+tc.label, body)
+			content, cs, err := waf.getOrgContent(resp, false, "")
+			if err != nil || cs != tc.canonical || !strings.Contains(string(content), tc.text) {
+				t.Errorf("Content-Type声明%s检测失败: err=%v, charset=%s, content=%s", tc.label, err, cs, string(content))
+			}
+		})
+		// 路径2：HTML meta标签声明charset
+		if tc.viaMeta {
+			t.Run(tc.label+"_meta", func(t *testing.T) {
+				html := `<html><head><meta charset="` + tc.label + `"></head><body>` + tc.text + `</body></html>`
+				body := encodeTestBody(t, tc.enc, html)
+				resp := newEncodingTestResponse("text/html", body)
+				content, cs, err := waf.getOrgContent(resp, false, "")
+				if err != nil || cs != tc.canonical || !strings.Contains(string(content), tc.text) {
+					t.Errorf("meta声明%s检测失败: err=%v, charset=%s, content=%s", tc.label, err, cs, string(content))
+				}
+			})
+		}
+	}
+
+	// BOM检测：utf-16le / utf-16be / utf-8
+	t.Run("utf-16le_bom", func(t *testing.T) {
+		body := encodeTestBody(t, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), "<html><body>UTF16LE中文内容</body></html>")
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-16le" || !strings.Contains(string(content), "UTF16LE中文内容") {
+			t.Errorf("utf-16le BOM检测失败: err=%v, charset=%s", err, cs)
+		}
+	})
+	t.Run("utf-16be_bom", func(t *testing.T) {
+		body := encodeTestBody(t, unicode.UTF16(unicode.BigEndian, unicode.UseBOM), "<html><body>UTF16BE中文内容</body></html>")
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-16be" || !strings.Contains(string(content), "UTF16BE中文内容") {
+			t.Errorf("utf-16be BOM检测失败: err=%v, charset=%s", err, cs)
+		}
+	})
+	t.Run("utf-8_bom", func(t *testing.T) {
+		body := append([]byte{0xEF, 0xBB, 0xBF}, []byte("<html><body>UTF8BOM中文内容</body></html>")...)
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-8" || !strings.Contains(string(content), "UTF8BOM中文内容") {
+			t.Errorf("utf-8 BOM检测失败: err=%v, charset=%s", err, cs)
+		}
+	})
+
+	// 危险标签：映射到replacement伪编码的标签绝不能采用（否则整个body变U+FFFD），应落入兜底链
+	t.Run("dangerous_label_header", func(t *testing.T) {
+		body := []byte("<html><body>危险编码标签测试</body></html>")
+		resp := newEncodingTestResponse("text/html; charset=hz-gb-2312", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-8" || !strings.Contains(string(content), "危险编码标签测试") {
+			t.Errorf("危险标签hz-gb-2312应兜底为utf-8: err=%v, charset=%s, content=%s", err, cs, string(content))
+		}
+	})
+	t.Run("dangerous_label_meta", func(t *testing.T) {
+		body := []byte(`<html><head><meta charset="utf-7"></head><body>危险meta标签测试</body></html>`)
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-8" || !strings.Contains(string(content), "危险meta标签测试") {
+			t.Errorf("危险标签utf-7应兜底为utf-8: err=%v, charset=%s, content=%s", err, cs, string(content))
+		}
+	})
+}
+
+// TestGetOrgContent_DefaultEncoding 站点"默认编码"配置的优先级与auto特判
+func TestGetOrgContent_DefaultEncoding(t *testing.T) {
+	t.Parallel()
+	zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+	waf := &WafEngine{}
+
+	// 1. defaultEncoding=gbk 优先级最高：即使Content-Type谎报utf-8也按gbk解
+	body1 := encodeTestBody(t, simplifiedchinese.GBK, "<html><body>默认编码优先</body></html>")
+	resp1 := newEncodingTestResponse("text/html; charset=utf-8", body1)
+	content1, cs1, err1 := waf.getOrgContent(resp1, false, "gbk")
+	if err1 != nil || cs1 != "gbk" || !strings.Contains(string(content1), "默认编码优先") {
+		t.Errorf("defaultEncoding=gbk应优先生效: err=%v, charset=%s, content=%s", err1, cs1, string(content1))
+	}
+
+	// 2. defaultEncoding=auto 表示自动检测，顺延到Content-Type
+	body2 := encodeTestBody(t, simplifiedchinese.GBK, "<html><body>auto顺延头部编码</body></html>")
+	resp2 := newEncodingTestResponse("text/html; charset=gbk", body2)
+	content2, cs2, err2 := waf.getOrgContent(resp2, false, "auto")
+	if err2 != nil || cs2 != "gbk" || !strings.Contains(string(content2), "auto顺延头部编码") {
+		t.Errorf("defaultEncoding=auto应顺延Content-Type: err=%v, charset=%s, content=%s", err2, cs2, string(content2))
+	}
+
+	// 3. defaultEncoding=big5（新支持的编码在配置项生效）
+	body3 := encodeTestBody(t, traditionalchinese.Big5, "<html><body>繁體預設編碼</body></html>")
+	resp3 := newEncodingTestResponse("text/html", body3)
+	content3, cs3, err3 := waf.getOrgContent(resp3, false, "big5")
+	if err3 != nil || cs3 != "big5" || !strings.Contains(string(content3), "繁體預設編碼") {
+		t.Errorf("defaultEncoding=big5应生效: err=%v, charset=%s, content=%s", err3, cs3, string(content3))
+	}
+
+	// 4. defaultEncoding为无法识别的值时顺延后续检测链，不报错
+	body4 := []byte("<html><body>无效默认编码顺延</body></html>")
+	resp4 := newEncodingTestResponse("text/html; charset=utf-8", body4)
+	content4, cs4, err4 := waf.getOrgContent(resp4, false, "foobar")
+	if err4 != nil || cs4 != "utf-8" || !strings.Contains(string(content4), "无效默认编码顺延") {
+		t.Errorf("无效defaultEncoding应顺延检测链: err=%v, charset=%s, content=%s", err4, cs4, string(content4))
+	}
+}
+
+// TestGetOrgContent_HeuristicDetection 无声明时的兜底检测链
+func TestGetOrgContent_HeuristicDetection(t *testing.T) {
+	t.Parallel()
+	zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+	waf := &WafEngine{}
+
+	// 1. UTF-8中文内容、无任何charset声明（复现线上"编码检测不确定"WARN）
+	t.Run("utf8_no_declaration", func(t *testing.T) {
+		body := []byte("<html><body>这是没有任何编码声明的UTF-8页面内容</body></html>")
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-8" || !bytes.Equal(content, body) {
+			t.Errorf("无声明UTF-8应通过校验兜底: err=%v, charset=%s", err, cs)
+		}
+	})
+
+	// 2. JSON等非HTML内容、无charset声明
+	t.Run("json_no_charset", func(t *testing.T) {
+		body := []byte(`{"message":"中文数据","code":200}`)
+		resp := newEncodingTestResponse("application/json", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-8" || !bytes.Equal(content, body) {
+			t.Errorf("无声明JSON应按utf-8处理: err=%v, charset=%s", err, cs)
+		}
+	})
+
+	// 3. 纯ASCII内容、无声明
+	t.Run("ascii_no_declaration", func(t *testing.T) {
+		body := []byte("<html><body>plain ascii content only</body></html>")
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-8" || !bytes.Equal(content, body) {
+			t.Errorf("纯ASCII应按utf-8处理: err=%v, charset=%s", err, cs)
+		}
+	})
+
+	// 4. meta声明在1024字节之后（DetermineEncoding预扫描窗口外），靠8KB扩窗扫描命中
+	t.Run("meta_beyond_1024", func(t *testing.T) {
+		html := "<!-- " + strings.Repeat("x", 2048) + ` --><html><head><meta http-equiv="Content-Type" content="text/html; charset=gbk"></head><body>窗口之后的GBK正文</body></html>`
+		body := encodeTestBody(t, simplifiedchinese.GBK, html)
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "gbk" || !strings.Contains(string(content), "窗口之后的GBK正文") {
+			t.Errorf("1024字节后的meta声明应被扩窗扫描命中: err=%v, charset=%s", err, cs)
+		}
+	})
+
+	// 5. 无任何声明的纯GBK内容，靠GBK严格解码探测命中
+	t.Run("gbk_no_declaration", func(t *testing.T) {
+		body := encodeTestBody(t, simplifiedchinese.GBK, "<html><body>这是没有声明的GBK内容，用于启发式探测检查。</body></html>")
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "gbk" || !strings.Contains(string(content), "这是没有声明的GBK内容") {
+			t.Errorf("无声明GBK应被启发式探测命中: err=%v, charset=%s, content=%s", err, cs, string(content))
+		}
+	})
+
+	// 6. 不明单字节数据（非UTF-8也非GBK）→ latin-1无损兜底，不再报错
+	t.Run("unknown_bytes_fallback", func(t *testing.T) {
+		body := []byte("Hello \xe9 w\xf6rld") // latin-1字节，0xE9后跟空格不构成合法UTF-8/GBK
+		resp := newEncodingTestResponse("text/html", body)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != rawLatin1Charset || !strings.Contains(string(content), "Hello") {
+			t.Errorf("不明字节应latin-1兜底: err=%v, charset=%s", err, cs)
+		}
+	})
+
+	// 7. 无声明的西欧latin-1文本：高位字节+ASCII恰好构成合法GBK双字节，不能被误判成GBK，
+	//    必须走latin-1兜底且往返字节无损
+	t.Run("latin1_text_not_gbk", func(t *testing.T) {
+		orig := encodeTestBody(t, charmap.ISO8859_1, "<html><body>Fußgänger café résumé réélection</body></html>")
+		resp := newEncodingTestResponse("text/html", orig)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != rawLatin1Charset {
+			t.Fatalf("无声明西欧文本应latin-1兜底而非误判GBK: err=%v, charset=%s", err, cs)
+		}
+		out, _ := waf.compressContent(resp, false, content, cs)
+		if !bytes.Equal(out, orig) {
+			t.Errorf("西欧文本往返后字节不一致\n原始: % x\n结果: % x", orig, out)
+		}
+	})
+}
+
+// TestContentEncoding_RoundTrip 检测+回写的无损往返：getOrgContent解出UTF-8后经compressContent还原为原始字节
+func TestContentEncoding_RoundTrip(t *testing.T) {
+	t.Parallel()
+	zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+	waf := &WafEngine{}
+
+	cases := []struct {
+		label string
+		enc   encoding.Encoding
+		text  string
+	}{
+		{"gbk", simplifiedchinese.GBK, "简体中文往返测试"},
+		{"gb18030", simplifiedchinese.GB18030, "GB18030往返𠀀测试"},
+		{"big5", traditionalchinese.Big5, "繁體中文往返測試"},
+		{"shift_jis", japanese.ShiftJIS, "日本語ラウンドトリップ"},
+		{"euc-jp", japanese.EUCJP, "日本語往復テスト"},
+		{"iso-2022-jp", japanese.ISO2022JP, "日本語往復"},
+		{"euc-kr", korean.EUCKR, "한국어 왕복 테스트"},
+		{"windows-1251", charmap.Windows1251, "Тест кириллицы"},
+		{"koi8-r", charmap.KOI8R, "Тест КОИ8"},
+		{"windows-1252", charmap.Windows1252, "€ Café “quotes”"},
+		{"windows-874", charmap.Windows874, "ทดสอบภาษาไทย"},
+	}
+
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.label, func(t *testing.T) {
+			orig := encodeTestBody(t, tc.enc, "<html><body>"+tc.text+"</body></html>")
+			resp := newEncodingTestResponse("text/html; charset="+tc.label, orig)
+			content, cs, err := waf.getOrgContent(resp, false, "")
+			if err != nil {
+				t.Fatalf("getOrgContent失败: %v", err)
+			}
+			out, cErr := waf.compressContent(resp, false, content, cs)
+			if cErr != nil {
+				t.Fatalf("compressContent失败: %v", cErr)
+			}
+			if !bytes.Equal(out, orig) {
+				t.Errorf("%s往返后字节不一致\n原始: % x\n结果: % x", tc.label, orig, out)
+			}
+		})
+	}
+
+	// 带BOM的utf-16le无声明往返（BOM在解码时保留为U+FEFF字符，回写后还原）
+	t.Run("utf-16le_bom", func(t *testing.T) {
+		orig := encodeTestBody(t, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), "<html><body>UTF16往返内容</body></html>")
+		resp := newEncodingTestResponse("text/html", orig)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "utf-16le" {
+			t.Fatalf("utf-16le检测失败: err=%v, charset=%s", err, cs)
+		}
+		out, _ := waf.compressContent(resp, false, content, cs)
+		if !bytes.Equal(out, orig) {
+			t.Errorf("utf-16le BOM往返后字节不一致\n原始: % x\n结果: % x", orig, out)
+		}
+	})
+
+	// latin-1兜底路径的256字节全集无损往返（无损兜底的核心保证）
+	t.Run("latin1_fallback_all_bytes", func(t *testing.T) {
+		orig := make([]byte, 256)
+		for i := range orig {
+			orig[i] = byte(i)
+		}
+		resp := newEncodingTestResponse("application/octet-stream", orig)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != rawLatin1Charset {
+			t.Fatalf("全字节内容应latin-1兜底: err=%v, charset=%s", err, cs)
+		}
+		out, _ := waf.compressContent(resp, false, content, cs)
+		if !bytes.Equal(out, orig) {
+			t.Errorf("latin-1兜底往返后字节不一致\n原始: % x\n结果: % x", orig, out)
+		}
+	})
+
+	// 敏感词替换模拟：GBK页面解出UTF-8替换后回写，页面仍可正常按GBK解码且替换生效
+	t.Run("gbk_sensitive_replace", func(t *testing.T) {
+		orig := encodeTestBody(t, simplifiedchinese.GBK, "<html><body>页面包含敏感词汇需要处理</body></html>")
+		resp := newEncodingTestResponse("text/html; charset=gbk", orig)
+		content, cs, err := waf.getOrgContent(resp, false, "")
+		if err != nil || cs != "gbk" {
+			t.Fatalf("GBK检测失败: err=%v, charset=%s", err, cs)
+		}
+		replaced := strings.Replace(string(content), "敏感词汇", "****", 1)
+		out, _ := waf.compressContent(resp, false, []byte(replaced), cs)
+		decoded, dErr := simplifiedchinese.GBK.NewDecoder().Bytes(out)
+		if dErr != nil {
+			t.Fatalf("替换后GBK解码失败: %v", dErr)
+		}
+		if !strings.Contains(string(decoded), "****") || !strings.Contains(string(decoded), "需要处理") || strings.ContainsRune(string(decoded), '�') {
+			t.Errorf("替换后页面异常: %s", string(decoded))
+		}
+	})
+}