diff --git a/wafenginecore/content_encoding.go b/wafenginecore/content_encoding.go
index b60fd93..3b741ef 100644
--- a/wafenginecore/content_encoding.go
+++ b/wafenginecore/content_encoding.go
@@ -13,14 +13,142 @@ import (
"github.com/klauspost/compress/zstd"
"golang.org/x/net/html/charset"
"golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/htmlindex"
"golang.org/x/text/encoding/simplifiedchinese"
- "golang.org/x/text/encoding/unicode"
"golang.org/x/text/transform"
"io"
"net/http"
"strings"
+ "unicode/utf8"
)
+// encodingDetectPeekSize 编码自动检测的探测窗口大小（meta声明可能在1024字节之后）
+const encodingDetectPeekSize = 8192
+
+// rawLatin1Charset 内部无损兜底编码名：真正的ISO-8859-1（256字节全双射，解码→处理→回写字节级无损）
+// 不能直接用"iso-8859-1"标签——WHATWG会把它归一到windows-1252，而windows-1252有5个
+// 未定义字节（0x81/0x8D/0x8F/0x90/0x9D）解码会变成U+FFFD，做不到无损往返
+const rawLatin1Charset = "x-raw-latin1"
+
+// resolveCharset 将任意charset名称解析为编码，支持全部WHATWG标准编码（约40种、200+别名）
+// 返回编码实例和规范名；无法识别、auto、或映射到replacement伪编码时返回nil
+func resolveCharset(name string) (encoding.Encoding, string) {
+ name = strings.Trim(strings.TrimSpace(name), `"'`)
+ if name == "" || strings.EqualFold(name, "auto") {
+ return nil, ""
+ }
+ if strings.EqualFold(name, rawLatin1Charset) {
+ return charmap.ISO8859_1, rawLatin1Charset
+ }
+ // 用htmlindex取原始编码而非charset.Lookup：后者返回的编码器自带HTML转义包装，
+ // 不可编码字符会变成&#NNN;（对JSON/JS响应会破坏语法），原始编码器才能配合ReplaceUnsupported
+ enc, lookupErr := htmlindex.Get(name)
+ if lookupErr != nil {
+ return nil, ""
+ }
+ canonical, nameErr := htmlindex.Name(enc)
+ // replacement伪编码（hz-gb-2312/iso-2022-cn等危险标签）解码会把整个body变成U+FFFD，不能使用
+ if nameErr != nil || canonical == "replacement" {
+ return nil, ""
+ }
+ return enc, canonical
+}
+
+// charsetFromContentType 从Content-Type头中提取charset值，未指定时返回空
+func charsetFromContentType(contentType string) string {
+ for _, part := range strings.Split(contentType, ";") {
+ part = strings.TrimSpace(part)
+ if len(part) > 8 && strings.EqualFold(part[:8], "charset=") {
+ return strings.Trim(part[8:], `"'`)
+ }
+ }
+ return ""
+}
+
+// charsetFromHTMLHead 在HTML头部内容中扫描charset声明或DOCTYPE，返回解析出的编码
+// 作为DetermineEncoding（只预扫描前1024字节）的扩窗补充
+func charsetFromHTMLHead(htmlHead string) (encoding.Encoding, string) {
+ if idx := strings.Index(strings.ToLower(htmlHead), "charset="); idx != -1 {
+ substr := strings.TrimLeft(htmlHead[idx+8:], "\"' ")
+ end := len(substr)
+ for i, c := range substr {
+ if !(c == '-' || c == '_' || c == ':' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
+ end = i
+ break
+ }
+ }
+ if enc, canonical := resolveCharset(substr[:end]); enc != nil {
+ return enc, canonical
+ }
+ }
+ // XHTML 1.0 通常默认utf-8
+ if strings.Contains(htmlHead, "xhtml1-transitional.dtd") {
+ return resolveCharset("utf-8")
+ }
+ return nil, ""
+}
+
+// validUTF8Prefix 判断字节序列是否为合法UTF-8（容忍末尾被探测窗口截断的多字节字符）
+func validUTF8Prefix(b []byte) bool {
+ for trim := 0; trim < utf8.UTFMax; trim++ {
+ if utf8.Valid(b) {
+ return true
+ }
+ if len(b) == 0 || b[len(b)-1] < 0x80 {
+ return false
+ }
+ b = b[:len(b)-1]
+ }
+ return false
+}
+
+// looksLikeGBK 无声明时的GBK启发式探测。
+// 西欧latin-1文本中"高位字节+ASCII字母"恰好也是合法GBK双字节（如é=0xE9后跟't'），
+// 为避免误判，要求所有双字节序列的尾字节也是高位字节（GB2312常用汉字尾字节均≥0xA1），
+// 且至少出现2对；宁可放弃判定走无损兜底，也不错误转码
+func looksLikeGBK(b []byte) bool {
+ highPairs := 0
+ for i := 0; i < len(b); {
+ c := b[i]
+ if c < 0x80 {
+ i++
+ continue
+ }
+ if c == 0x80 || c == 0xFF {
+ // 非法GBK首字节
+ return false
+ }
+ if i+1 >= len(b) {
+ // 末尾被探测窗口截断半个双字节字符，容忍
+ break
+ }
+ trail := b[i+1]
+ if trail < 0xA1 || trail == 0xFF {
+ // 尾字节非高位（GBK扩展区）或非法：西欧文本特征，放弃GBK判定
+ return false
+ }
+ highPairs++
+ i += 2
+ }
+ if highPairs < 2 {
+ return false
+ }
+ if gbkDecodesClean(b) {
+ return true
+ }
+ // 末尾可能被探测窗口截断半个双字节字符，剪掉最后一个字节再试
+ return b[len(b)-1] >= 0x80 && gbkDecodesClean(b[:len(b)-1])
+}
+
+func gbkDecodesClean(b []byte) bool {
+ decoded, err := simplifiedchinese.GBK.NewDecoder().Bytes(b)
+ if err != nil {
+ return false
+ }
+ return !bytes.ContainsRune(decoded, utf8.RuneError)
+}
+
// 返回内容前依据情况进行返回压缩数据
func (waf *WafEngine) compressContent(res *http.Response, isStaticAssist bool, inputBytes []byte, encodeType string) (respBytes []byte, err error) {
@@ -29,38 +157,25 @@ func (waf *WafEngine) compressContent(res *http.Response, isStaticAssist bool, i
return inputBytes, errors.New("静态资源或资源类型请求，跳过编码转换")
}
- // 首先检查Content-Type头中是否明确指定了字符集
- contentType := res.Header.Get("Content-Type")
- var encodedBytes []byte = inputBytes
+ // 优先使用检测阶段返回的编码，否则从Content-Type头提取
charsetName := encodeType
-
- // 如果Content-Type中指定了字符集，需要将UTF-8编码的内容转换回原始编码
- if charsetName == "" && strings.Contains(contentType, "charset=") {
-
- parts := strings.Split(contentType, ";")
- for _, part := range parts {
- part = strings.TrimSpace(part)
- if strings.HasPrefix(part, "charset=") {
- charsetName = strings.TrimPrefix(part, "charset=")
- charsetName = strings.Trim(charsetName, `"'`)
- break
- }
- }
+ if charsetName == "" {
+ charsetName = charsetFromContentType(res.Header.Get("Content-Type"))
}
- // 根据字符集进行编码转换
+
+ encodedBytes := inputBytes
+ // 内容在处理阶段是UTF-8，回写前需要编码回原始字符集
if charsetName != "" && !strings.EqualFold(charsetName, "utf-8") && !strings.EqualFold(charsetName, "utf8") {
- switch strings.ToLower(charsetName) {
- case "gbk", "gb2312":
- // 将UTF-8转换为GBK
- encoder := simplifiedchinese.GBK.NewEncoder()
- encodedBytes, err = encoder.Bytes(inputBytes)
- if err != nil {
- zlog.Warn("编码转换失败(UTF-8 -> GBK): %v", err)
+ if enc, canonical := resolveCharset(charsetName); enc != nil {
+ // ReplaceUnsupported：目标字符集无法表示的字符替换为占位符而非报错中断
+ converted, encodeErr := encoding.ReplaceUnsupported(enc.NewEncoder()).Bytes(inputBytes)
+ if encodeErr != nil {
+ zlog.Warn("编码转换失败(UTF-8 -> %s): %v", canonical, encodeErr)
// 转换失败时使用原始UTF-8内容
- encodedBytes = inputBytes
+ } else {
+ encodedBytes = converted
}
- // 可以根据需要添加其他编码的支持
- default:
+ } else {
zlog.Debug("不支持的字符集编码转换: %s，保持UTF-8编码", charsetName)
}
}
@@ -87,7 +202,6 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
if err != nil {
return bodyBytes, "", fmt.Errorf("读取原始响应体失败: %v", err)
}
- charsetNameFinal := ""
// 重新设置响应体，以便后续处理
resp.Body = io.NopCloser(bytes.NewBuffer(bodyBytes))
@@ -96,11 +210,11 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
return bodyBytes, "", errors.New("静态资源或资源类型请求，跳过编码转换")
}
- // 根据内容编码处理压缩
+ // 根据内容编码处理压缩（从独立reader读取，保持resp.Body完整不被消耗）
var bodyReader io.Reader
switch resp.Header.Get("Content-Encoding") {
case "gzip":
- gzipReader, gzipErr := gzip.NewReader(resp.Body)
+ gzipReader, gzipErr := gzip.NewReader(bytes.NewReader(bodyBytes))
if gzipErr != nil {
zlog.Warn("gzip解压失败: %v", gzipErr)
// 失败时返回错误
@@ -109,14 +223,13 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
bodyReader = gzipReader
defer gzipReader.Close()
case "deflate":
- deflateReader := flate.NewReader(resp.Body)
+ deflateReader := flate.NewReader(bytes.NewReader(bodyBytes))
bodyReader = deflateReader
defer deflateReader.Close()
case "br":
- brotliReader := brotli.NewReader(resp.Body)
- bodyReader = brotliReader
+ bodyReader = brotli.NewReader(bytes.NewReader(bodyBytes))
case "zstd":
- zstdReader, zstdErr := zstd.NewReader(resp.Body)
+ zstdReader, zstdErr := zstd.NewReader(bytes.NewReader(bodyBytes))
if zstdErr != nil {
zlog.Warn("zstd解压失败: %v", zstdErr)
return bodyBytes, "", fmt.Errorf("zstd解压失败: %v", zstdErr)
@@ -124,116 +237,83 @@ func (waf *WafEngine) getOrgContent(resp *http.Response, isStaticAssist bool, de
bodyReader = zstdReader
defer zstdReader.Close()
default:
- bodyReader = resp.Body
+ bodyReader = bytes.NewReader(bodyBytes)
}
// 创建缓冲读取器
- bufReader := bufio.NewReader(bodyReader)
+ bufReader := bufio.NewReaderSize(bodyReader, encodingDetectPeekSize)
- // 首先检查Content-Type头中是否明确指定了字符集
contentType := resp.Header.Get("Content-Type")
var currentEncoding encoding.Encoding
+ charsetNameFinal := ""
- if defaultEncoding != "" {
- switch strings.ToLower(defaultEncoding) {
- case "utf-8", "utf8":
- currentEncoding = unicode.UTF8
- charsetNameFinal = "utf-8"
- break
- case "gbk", "gb2312":
- currentEncoding = simplifiedchinese.GBK
- charsetNameFinal = "gbk"
- break
- default:
- }
- } else if strings.Contains(contentType, "charset=") {
- charsetName := ""
- parts := strings.Split(contentType, ";")
- for _, part := range parts {
- part = strings.TrimSpace(part)
- if strings.HasPrefix(part, "charset=") {
- charsetName = strings.TrimPrefix(part, "charset=")
- charsetName = strings.Trim(charsetName, `"'`)
- break
- }
- }
+ // 优先级1：站点配置的默认编码（auto或空表示走自动检测）
+ if enc, name := resolveCharset(defaultEncoding); enc != nil {
+ currentEncoding = enc
+ charsetNameFinal = name
+ } else if defaultEncoding != "" && !strings.EqualFold(defaultEncoding, "auto") {
+ zlog.Debug("站点默认编码无法识别: %s，转入自动检测", defaultEncoding)
+ }
- // 如果找到了字符集
- if charsetName != "" {
- zlog.Debug("从Content-Type中检测到字符集: %s", charsetName)
- switch strings.ToLower(charsetName) {
- case "utf-8", "utf8":
- currentEncoding = unicode.UTF8
- charsetNameFinal = "utf-8"
- break
- case "gbk", "gb2312":
- currentEncoding = simplifiedchinese.GBK
- charsetNameFinal = "gbk"
- break
- default:
- currentEncoding = nil // 使用自动检测
+ // 优先级2：Content-Type头中显式指定的字符集
+ if currentEncoding == nil {
+ if headerCharset := charsetFromContentType(contentType); headerCharset != "" {
+ if enc, name := resolveCharset(headerCharset); enc != nil {
+ zlog.Debug("从Content-Type中检测到字符集: %s", name)
+ currentEncoding = enc
+ charsetNameFinal = name
}
}
}
- // 如果没有从Content-Type中获取到编码或获取的编码不支持，则使用自动检测
+ // 优先级3：自动检测（BOM/meta预扫描 → 扩窗meta扫描 → UTF-8校验 → GBK探测 → windows-1252无损兜底）
if currentEncoding == nil {
- // 增加检测字节数到1024，提高准确性
- peekBytes, peekErr := bufReader.Peek(1024)
+ peekBytes, peekErr := bufReader.Peek(encodingDetectPeekSize)
if peekErr != nil && peekErr != io.EOF {
- return bodyBytes, "", errors.New(fmt.Sprintf("编码检测错误，Peek失败: %v", peekErr))
- } else {
- // 先尝试用系统自带的自动检测
- detectedEncoding, name, certain := charset.DetermineEncoding(peekBytes, contentType)
-
- if !certain {
- // 自动检测不确定时，尝试从HTML头部meta标签或DOCTYPE中提取编码
- htmlHead := string(peekBytes)
- metaCharset := ""
- // 1. 检查meta标签
- metaIdx := strings.Index(strings.ToLower(htmlHead), "charset=")
- if metaIdx != -1 {
- // 提取charset=后面的内容
- substr := htmlHead[metaIdx+8:]
- // 只取等号后面第一个非字母数字下划线的内容
- for i, c := range substr {
- if !(c == '-' || c == '_' || (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')) {
- metaCharset = substr[:i]
- break
- }
- }
- if metaCharset == "" {
- metaCharset = substr
- }
- metaCharset = strings.Trim(metaCharset, "\"' />")
- }
- // 2. 检查DOCTYPE声明
- if metaCharset == "" && strings.Contains(htmlHead, "xhtml1-transitional.dtd") {
- // XHTML 1.0 通常默认utf-8或iso-8859-1
- metaCharset = "utf-8"
- }
- if metaCharset != "" {
- zlog.Debug("通过HTML头部meta/doctype辅助检测到字符集: %s", metaCharset)
- switch strings.ToLower(metaCharset) {
- case "utf-8", "utf8":
- currentEncoding = unicode.UTF8
- charsetNameFinal = "utf-8"
- break
- case "gbk", "gb2312":
- currentEncoding = simplifiedchinese.GBK
- charsetNameFinal = "gbk"
- break
- default:
- zlog.Debug("meta标签检测到未知字符集: %s，保持UTF-8", metaCharset)
- return bodyBytes, "", errors.New(fmt.Sprintf("编码检测不确定，且未能从HTML头部meta/doctype提取到编码"))
- }
- } else {
- return bodyBytes, "", errors.New(fmt.Sprintf("编码检测不确定，且未能从HTML头部meta/doctype提取到编码"))
- }
- } else {
- zlog.Debug("编码检测确定为: %s", name)
- currentEncoding = detectedEncoding
+ return bodyBytes, "", fmt.Errorf("编码检测错误，Peek失败: %v", peekErr)
+ }
+ detectedEncoding, name, certain := charset.DetermineEncoding(peekBytes, contentType)
+ // certain来自BOM或头部charset；非windows-1252的不确定结果来自WHATWG meta预扫描，同样可信
+ // windows-1252是DetermineEncoding找不到任何依据时的默认值，需继续用后续手段甄别
+ if name != "replacement" && (certain || name != "windows-1252") {
+ zlog.Debug("编码检测结果: %s (certain=%v)", name, certain)
+ currentEncoding = detectedEncoding
+ charsetNameFinal = name
+ }
+ if currentEncoding == nil {
+ // meta声明可能在1024字节之后，扩大窗口手工扫描
+ if enc, name := charsetFromHTMLHead(string(peekBytes)); enc != nil {
+ zlog.Debug("通过HTML头部meta/doctype辅助检测到字符集: %s", name)
+ currentEncoding = enc
+ charsetNameFinal = name
}
}
+ if currentEncoding == nil && validUTF8Prefix(peekBytes) {
+ // 无任何声明但内容是合法UTF-8（含纯ASCII），这是线上最常见的情况
+ zlog.Debug("无编码声明，内容通过UTF-8校验，按utf-8处理")
+ charsetNameFinal = "utf-8"
+ currentEncoding, _ = charset.Lookup("utf-8")
+ }
+ if currentEncoding == nil && looksLikeGBK(peekBytes) {
+ zlog.Debug("无编码声明，GBK探测命中，按gbk处理")
+ currentEncoding = simplifiedchinese.GBK
+ charsetNameFinal = "gbk"
+ }
+ if currentEncoding == nil {
+ // ISO-8859-1为256字节全双射：解码→处理→回写字节级无损，ASCII部分的替换仍然有效
+ zlog.Debug("编码无法确定，使用latin-1无损兜底处理")
+ currentEncoding = charmap.ISO8859_1
+ charsetNameFinal = rawLatin1Charset
+ }
+ }
+
+ // UTF-8内容无需转换直接读取，避免个别非法字节被解码器替换为U+FFFD导致内容变动
+ if strings.EqualFold(charsetNameFinal, "utf-8") {
+ resBodyBytes, readErr := io.ReadAll(bufReader)
+ if readErr != nil {
+ zlog.Warn("读取响应体失败: %v", readErr)
+ return bodyBytes, "", fmt.Errorf("读取响应体失败: %v", readErr)
+ }
+ return resBodyBytes, "utf-8", nil
}
// 使用检测到的编码创建转换读取器
diff --git a/wafenginecore/wafengine_test.go b/wafenginecore/wafengine_test.go
index 4e6b02e..c5972fe 100644
--- a/wafenginecore/wafengine_test.go
+++ b/wafenginecore/wafengine_test.go
@@ -6,6 +6,13 @@ import (
"bytes"
"compress/flate"
"compress/gzip"
+ "golang.org/x/text/encoding"
+ "golang.org/x/text/encoding/charmap"
+ "golang.org/x/text/encoding/japanese"
+ "golang.org/x/text/encoding/korean"
+ "golang.org/x/text/encoding/simplifiedchinese"
+ "golang.org/x/text/encoding/traditionalchinese"
+ "golang.org/x/text/encoding/unicode"
"io"
"net/http"
"net/http/httptest"
@@ -202,7 +209,7 @@ func TestGetOrgContent(t *testing.T) {
contentEncoding: "",
content: "没有指定字符集的内容",
expectedContent: "没有指定字符集的内容",
- expectedErr: true,
+ expectedErr: false, // UTF-8内容无声明时通过UTF-8校验兜底解出
},
{
// 对于不支持的字符集，我们跳过内容比较，只检查是否有错误
@@ -453,7 +460,7 @@ func TestGetOrgContent_MetaAndDoctypeCharset(t *testing.T) {
t.Errorf("DOCTYPE声明检测失败: err=%v, charset=%s, content=%s", err3, charset3, string(content3))
}
- // 4. meta标签未知编码
+ // 4. meta标签未知编码：不再报错，内容为合法UTF-8时通过UTF-8校验兜底解出
htmlMetaUnknown := `未知编码`
resp4 := &http.Response{
StatusCode: 200,
@@ -461,9 +468,9 @@ func TestGetOrgContent_MetaAndDoctypeCharset(t *testing.T) {
Body: io.NopCloser(strings.NewReader(htmlMetaUnknown)),
}
resp4.Header.Set("Content-Type", "text/html")
- _, _, err4 := waf.getOrgContent(resp4, false, "")
- if err4 == nil || !strings.Contains(err4.Error(), "编码检测不确定") {
- t.Errorf("未知meta编码检测失败，期望返回错误，实际: %v", err4)
+ content4, charset4, err4 := waf.getOrgContent(resp4, false, "")
+ if err4 != nil || charset4 != "utf-8" || !strings.Contains(string(content4), "未知编码") {
+ t.Errorf("未知meta编码应兜底为utf-8: err=%v, charset=%s, content=%s", err4, charset4, string(content4))
}
}
@@ -917,3 +924,345 @@ func TestShouldAutoJumpHTTPS_Comprehensive(t *testing.T) {
}
})
}
+
+// encodeTestBody 用指定编码器将UTF-8原文编码为目标字符集字节，用于构造测试响应体
+func encodeTestBody(t *testing.T, enc encoding.Encoding, text string) []byte {
+ t.Helper()
+ b, err := enc.NewEncoder().Bytes([]byte(text))
+ if err != nil {
+ t.Fatalf("生成测试内容失败: %v", err)
+ }
+ return b
+}
+
+func newEncodingTestResponse(contentType string, body []byte) *http.Response {
+ resp := &http.Response{
+ StatusCode: 200,
+ Header: make(http.Header),
+ Body: io.NopCloser(bytes.NewReader(body)),
+ }
+ if contentType != "" {
+ resp.Header.Set("Content-Type", contentType)
+ }
+ return resp
+}
+
+// TestGetOrgContent_MultiCharsets 覆盖WHATWG各编码家族：Content-Type声明与meta声明两条命中路径
+func TestGetOrgContent_MultiCharsets(t *testing.T) {
+ t.Parallel()
+ zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+ waf := &WafEngine{}
+
+ cases := []struct {
+ label string // 声明用的charset标签
+ canonical string // 期望getOrgContent返回的规范名
+ enc encoding.Encoding // 生成测试字节用的编码器
+ text string
+ viaMeta bool // 是否额外测试meta声明路径
+ }{
+ {"gbk", "gbk", simplifiedchinese.GBK, "这是GBK编码的内容", true},
+ {"gb2312", "gbk", simplifiedchinese.GBK, "这是GB2312标签的内容", false},
+ {"gb18030", "gb18030", simplifiedchinese.GB18030, "GB18030扩展字符𠀀测试", true},
+ {"big5", "big5", traditionalchinese.Big5, "這是繁體中文內容", true},
+ {"shift_jis", "shift_jis", japanese.ShiftJIS, "これは日本語です", true},
+ {"euc-jp", "euc-jp", japanese.EUCJP, "日本語テスト", true},
+ {"iso-2022-jp", "iso-2022-jp", japanese.ISO2022JP, "日本語", false},
+ {"euc-kr", "euc-kr", korean.EUCKR, "한국어 테스트", true},
+ {"iso-8859-1", "windows-1252", charmap.Windows1252, "Café résumé", true},
+ {"iso-8859-2", "iso-8859-2", charmap.ISO8859_2, "Příliš žluťoučký", false},
+ {"iso-8859-15", "iso-8859-15", charmap.ISO8859_15, "€ Café", false},
+ {"windows-1252", "windows-1252", charmap.Windows1252, "€ smart “quotes”", true},
+ {"windows-1251", "windows-1251", charmap.Windows1251, "Привет мир", true},
+ {"koi8-r", "koi8-r", charmap.KOI8R, "Русский текст", false},
+ {"iso-8859-7", "iso-8859-7", charmap.ISO8859_7, "Ελληνικά", false},
+ {"windows-1256", "windows-1256", charmap.Windows1256, "مرحبا", false},
+ {"windows-874", "windows-874", charmap.Windows874, "สวัสดี", false},
+ {"ibm866", "ibm866", charmap.CodePage866, "Тест", false},
+ {"macintosh", "macintosh", charmap.Macintosh, "Café™", false},
+ }
+
+ for _, tc := range cases {
+ tc := tc
+ // 路径1：Content-Type头声明charset
+ t.Run(tc.label+"_content_type", func(t *testing.T) {
+ body := encodeTestBody(t, tc.enc, ""+tc.text+"")
+ resp := newEncodingTestResponse("text/html; charset="+tc.label, body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != tc.canonical || !strings.Contains(string(content), tc.text) {
+ t.Errorf("Content-Type声明%s检测失败: err=%v, charset=%s, content=%s", tc.label, err, cs, string(content))
+ }
+ })
+ // 路径2：HTML meta标签声明charset
+ if tc.viaMeta {
+ t.Run(tc.label+"_meta", func(t *testing.T) {
+ html := `` + tc.text + ``
+ body := encodeTestBody(t, tc.enc, html)
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != tc.canonical || !strings.Contains(string(content), tc.text) {
+ t.Errorf("meta声明%s检测失败: err=%v, charset=%s, content=%s", tc.label, err, cs, string(content))
+ }
+ })
+ }
+ }
+
+ // BOM检测：utf-16le / utf-16be / utf-8
+ t.Run("utf-16le_bom", func(t *testing.T) {
+ body := encodeTestBody(t, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), "UTF16LE中文内容")
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-16le" || !strings.Contains(string(content), "UTF16LE中文内容") {
+ t.Errorf("utf-16le BOM检测失败: err=%v, charset=%s", err, cs)
+ }
+ })
+ t.Run("utf-16be_bom", func(t *testing.T) {
+ body := encodeTestBody(t, unicode.UTF16(unicode.BigEndian, unicode.UseBOM), "UTF16BE中文内容")
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-16be" || !strings.Contains(string(content), "UTF16BE中文内容") {
+ t.Errorf("utf-16be BOM检测失败: err=%v, charset=%s", err, cs)
+ }
+ })
+ t.Run("utf-8_bom", func(t *testing.T) {
+ body := append([]byte{0xEF, 0xBB, 0xBF}, []byte("UTF8BOM中文内容")...)
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-8" || !strings.Contains(string(content), "UTF8BOM中文内容") {
+ t.Errorf("utf-8 BOM检测失败: err=%v, charset=%s", err, cs)
+ }
+ })
+
+ // 危险标签：映射到replacement伪编码的标签绝不能采用（否则整个body变U+FFFD），应落入兜底链
+ t.Run("dangerous_label_header", func(t *testing.T) {
+ body := []byte("危险编码标签测试")
+ resp := newEncodingTestResponse("text/html; charset=hz-gb-2312", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-8" || !strings.Contains(string(content), "危险编码标签测试") {
+ t.Errorf("危险标签hz-gb-2312应兜底为utf-8: err=%v, charset=%s, content=%s", err, cs, string(content))
+ }
+ })
+ t.Run("dangerous_label_meta", func(t *testing.T) {
+ body := []byte(`危险meta标签测试`)
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-8" || !strings.Contains(string(content), "危险meta标签测试") {
+ t.Errorf("危险标签utf-7应兜底为utf-8: err=%v, charset=%s, content=%s", err, cs, string(content))
+ }
+ })
+}
+
+// TestGetOrgContent_DefaultEncoding 站点"默认编码"配置的优先级与auto特判
+func TestGetOrgContent_DefaultEncoding(t *testing.T) {
+ t.Parallel()
+ zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+ waf := &WafEngine{}
+
+ // 1. defaultEncoding=gbk 优先级最高：即使Content-Type谎报utf-8也按gbk解
+ body1 := encodeTestBody(t, simplifiedchinese.GBK, "默认编码优先")
+ resp1 := newEncodingTestResponse("text/html; charset=utf-8", body1)
+ content1, cs1, err1 := waf.getOrgContent(resp1, false, "gbk")
+ if err1 != nil || cs1 != "gbk" || !strings.Contains(string(content1), "默认编码优先") {
+ t.Errorf("defaultEncoding=gbk应优先生效: err=%v, charset=%s, content=%s", err1, cs1, string(content1))
+ }
+
+ // 2. defaultEncoding=auto 表示自动检测，顺延到Content-Type
+ body2 := encodeTestBody(t, simplifiedchinese.GBK, "auto顺延头部编码")
+ resp2 := newEncodingTestResponse("text/html; charset=gbk", body2)
+ content2, cs2, err2 := waf.getOrgContent(resp2, false, "auto")
+ if err2 != nil || cs2 != "gbk" || !strings.Contains(string(content2), "auto顺延头部编码") {
+ t.Errorf("defaultEncoding=auto应顺延Content-Type: err=%v, charset=%s, content=%s", err2, cs2, string(content2))
+ }
+
+ // 3. defaultEncoding=big5（新支持的编码在配置项生效）
+ body3 := encodeTestBody(t, traditionalchinese.Big5, "繁體預設編碼")
+ resp3 := newEncodingTestResponse("text/html", body3)
+ content3, cs3, err3 := waf.getOrgContent(resp3, false, "big5")
+ if err3 != nil || cs3 != "big5" || !strings.Contains(string(content3), "繁體預設編碼") {
+ t.Errorf("defaultEncoding=big5应生效: err=%v, charset=%s, content=%s", err3, cs3, string(content3))
+ }
+
+ // 4. defaultEncoding为无法识别的值时顺延后续检测链，不报错
+ body4 := []byte("无效默认编码顺延")
+ resp4 := newEncodingTestResponse("text/html; charset=utf-8", body4)
+ content4, cs4, err4 := waf.getOrgContent(resp4, false, "foobar")
+ if err4 != nil || cs4 != "utf-8" || !strings.Contains(string(content4), "无效默认编码顺延") {
+ t.Errorf("无效defaultEncoding应顺延检测链: err=%v, charset=%s, content=%s", err4, cs4, string(content4))
+ }
+}
+
+// TestGetOrgContent_HeuristicDetection 无声明时的兜底检测链
+func TestGetOrgContent_HeuristicDetection(t *testing.T) {
+ t.Parallel()
+ zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+ waf := &WafEngine{}
+
+ // 1. UTF-8中文内容、无任何charset声明（复现线上"编码检测不确定"WARN）
+ t.Run("utf8_no_declaration", func(t *testing.T) {
+ body := []byte("这是没有任何编码声明的UTF-8页面内容")
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-8" || !bytes.Equal(content, body) {
+ t.Errorf("无声明UTF-8应通过校验兜底: err=%v, charset=%s", err, cs)
+ }
+ })
+
+ // 2. JSON等非HTML内容、无charset声明
+ t.Run("json_no_charset", func(t *testing.T) {
+ body := []byte(`{"message":"中文数据","code":200}`)
+ resp := newEncodingTestResponse("application/json", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-8" || !bytes.Equal(content, body) {
+ t.Errorf("无声明JSON应按utf-8处理: err=%v, charset=%s", err, cs)
+ }
+ })
+
+ // 3. 纯ASCII内容、无声明
+ t.Run("ascii_no_declaration", func(t *testing.T) {
+ body := []byte("plain ascii content only")
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-8" || !bytes.Equal(content, body) {
+ t.Errorf("纯ASCII应按utf-8处理: err=%v, charset=%s", err, cs)
+ }
+ })
+
+ // 4. meta声明在1024字节之后（DetermineEncoding预扫描窗口外），靠8KB扩窗扫描命中
+ t.Run("meta_beyond_1024", func(t *testing.T) {
+ html := "窗口之后的GBK正文`
+ body := encodeTestBody(t, simplifiedchinese.GBK, html)
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "gbk" || !strings.Contains(string(content), "窗口之后的GBK正文") {
+ t.Errorf("1024字节后的meta声明应被扩窗扫描命中: err=%v, charset=%s", err, cs)
+ }
+ })
+
+ // 5. 无任何声明的纯GBK内容，靠GBK严格解码探测命中
+ t.Run("gbk_no_declaration", func(t *testing.T) {
+ body := encodeTestBody(t, simplifiedchinese.GBK, "这是没有声明的GBK内容，用于启发式探测检查。")
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "gbk" || !strings.Contains(string(content), "这是没有声明的GBK内容") {
+ t.Errorf("无声明GBK应被启发式探测命中: err=%v, charset=%s, content=%s", err, cs, string(content))
+ }
+ })
+
+ // 6. 不明单字节数据（非UTF-8也非GBK）→ latin-1无损兜底，不再报错
+ t.Run("unknown_bytes_fallback", func(t *testing.T) {
+ body := []byte("Hello \xe9 w\xf6rld") // latin-1字节，0xE9后跟空格不构成合法UTF-8/GBK
+ resp := newEncodingTestResponse("text/html", body)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != rawLatin1Charset || !strings.Contains(string(content), "Hello") {
+ t.Errorf("不明字节应latin-1兜底: err=%v, charset=%s", err, cs)
+ }
+ })
+
+ // 7. 无声明的西欧latin-1文本：高位字节+ASCII恰好构成合法GBK双字节，不能被误判成GBK，
+ // 必须走latin-1兜底且往返字节无损
+ t.Run("latin1_text_not_gbk", func(t *testing.T) {
+ orig := encodeTestBody(t, charmap.ISO8859_1, "Fußgänger café résumé réélection")
+ resp := newEncodingTestResponse("text/html", orig)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != rawLatin1Charset {
+ t.Fatalf("无声明西欧文本应latin-1兜底而非误判GBK: err=%v, charset=%s", err, cs)
+ }
+ out, _ := waf.compressContent(resp, false, content, cs)
+ if !bytes.Equal(out, orig) {
+ t.Errorf("西欧文本往返后字节不一致\n原始: % x\n结果: % x", orig, out)
+ }
+ })
+}
+
+// TestContentEncoding_RoundTrip 检测+回写的无损往返：getOrgContent解出UTF-8后经compressContent还原为原始字节
+func TestContentEncoding_RoundTrip(t *testing.T) {
+ t.Parallel()
+ zlog.InitZLog(global.GWAF_LOG_DEBUG_ENABLE, "json")
+ waf := &WafEngine{}
+
+ cases := []struct {
+ label string
+ enc encoding.Encoding
+ text string
+ }{
+ {"gbk", simplifiedchinese.GBK, "简体中文往返测试"},
+ {"gb18030", simplifiedchinese.GB18030, "GB18030往返𠀀测试"},
+ {"big5", traditionalchinese.Big5, "繁體中文往返測試"},
+ {"shift_jis", japanese.ShiftJIS, "日本語ラウンドトリップ"},
+ {"euc-jp", japanese.EUCJP, "日本語往復テスト"},
+ {"iso-2022-jp", japanese.ISO2022JP, "日本語往復"},
+ {"euc-kr", korean.EUCKR, "한국어 왕복 테스트"},
+ {"windows-1251", charmap.Windows1251, "Тест кириллицы"},
+ {"koi8-r", charmap.KOI8R, "Тест КОИ8"},
+ {"windows-1252", charmap.Windows1252, "€ Café “quotes”"},
+ {"windows-874", charmap.Windows874, "ทดสอบภาษาไทย"},
+ }
+
+ for _, tc := range cases {
+ tc := tc
+ t.Run(tc.label, func(t *testing.T) {
+ orig := encodeTestBody(t, tc.enc, ""+tc.text+"")
+ resp := newEncodingTestResponse("text/html; charset="+tc.label, orig)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil {
+ t.Fatalf("getOrgContent失败: %v", err)
+ }
+ out, cErr := waf.compressContent(resp, false, content, cs)
+ if cErr != nil {
+ t.Fatalf("compressContent失败: %v", cErr)
+ }
+ if !bytes.Equal(out, orig) {
+ t.Errorf("%s往返后字节不一致\n原始: % x\n结果: % x", tc.label, orig, out)
+ }
+ })
+ }
+
+ // 带BOM的utf-16le无声明往返（BOM在解码时保留为U+FEFF字符，回写后还原）
+ t.Run("utf-16le_bom", func(t *testing.T) {
+ orig := encodeTestBody(t, unicode.UTF16(unicode.LittleEndian, unicode.UseBOM), "UTF16往返内容")
+ resp := newEncodingTestResponse("text/html", orig)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "utf-16le" {
+ t.Fatalf("utf-16le检测失败: err=%v, charset=%s", err, cs)
+ }
+ out, _ := waf.compressContent(resp, false, content, cs)
+ if !bytes.Equal(out, orig) {
+ t.Errorf("utf-16le BOM往返后字节不一致\n原始: % x\n结果: % x", orig, out)
+ }
+ })
+
+ // latin-1兜底路径的256字节全集无损往返（无损兜底的核心保证）
+ t.Run("latin1_fallback_all_bytes", func(t *testing.T) {
+ orig := make([]byte, 256)
+ for i := range orig {
+ orig[i] = byte(i)
+ }
+ resp := newEncodingTestResponse("application/octet-stream", orig)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != rawLatin1Charset {
+ t.Fatalf("全字节内容应latin-1兜底: err=%v, charset=%s", err, cs)
+ }
+ out, _ := waf.compressContent(resp, false, content, cs)
+ if !bytes.Equal(out, orig) {
+ t.Errorf("latin-1兜底往返后字节不一致\n原始: % x\n结果: % x", orig, out)
+ }
+ })
+
+ // 敏感词替换模拟：GBK页面解出UTF-8替换后回写，页面仍可正常按GBK解码且替换生效
+ t.Run("gbk_sensitive_replace", func(t *testing.T) {
+ orig := encodeTestBody(t, simplifiedchinese.GBK, "页面包含敏感词汇需要处理")
+ resp := newEncodingTestResponse("text/html; charset=gbk", orig)
+ content, cs, err := waf.getOrgContent(resp, false, "")
+ if err != nil || cs != "gbk" {
+ t.Fatalf("GBK检测失败: err=%v, charset=%s", err, cs)
+ }
+ replaced := strings.Replace(string(content), "敏感词汇", "****", 1)
+ out, _ := waf.compressContent(resp, false, []byte(replaced), cs)
+ decoded, dErr := simplifiedchinese.GBK.NewDecoder().Bytes(out)
+ if dErr != nil {
+ t.Fatalf("替换后GBK解码失败: %v", dErr)
+ }
+ if !strings.Contains(string(decoded), "****") || !strings.Contains(string(decoded), "需要处理") || strings.ContainsRune(string(decoded), '�') {
+ t.Errorf("替换后页面异常: %s", string(decoded))
+ }
+ })
+}