From 0e1c34e436a7c065fc86f509ac467a4e466362ac Mon Sep 17 00:00:00 2001 From: samwaf Date: Wed, 5 Feb 2025 17:00:15 +0800 Subject: [PATCH] fix:spider timeout --- global/global.go | 3 +- wafbot/determine_search.go | 183 +++++++++++++++++++++++++++---------- wafbot/dns_search.go | 7 +- wafbot/dns_search_test.go | 2 +- wafenginecore/checkbot.go | 10 +- waftask/task_config.go | 5 + 6 files changed, 150 insertions(+), 60 deletions(-) diff --git a/global/global.go b/global/global.go index 7fd9ba6..c1ecbd6 100644 --- a/global/global.go +++ b/global/global.go @@ -35,7 +35,8 @@ var ( GWAF_RUNTIME_QPS uint64 = 0 //当前qps GWAF_RUNTIME_LOG_PROCESS uint64 = 0 //log 处理速度 - GWAF_RUNTIME_DNS_SERVER string = "119.29.29.29" //反向查询DNS的IP + GWAF_RUNTIME_DNS_SERVER string = "119.29.29.29" //反向查询DNS的IP + GWAF_RUNTIME_DNS_TIMEOUT int64 = 500 // DNS 查询超时时间 单位毫秒 GWAF_RUNTIME_RECORD_LOG_TYPE string = "all" // 记录日志形式： 全部(all),非正常(abnormal) GWAF_RUNTIME_IS_UPDATETING bool = false //是否正在升级中 diff --git a/wafbot/determine_search.go b/wafbot/determine_search.go index 505cfc2..5921223 100644 --- a/wafbot/determine_search.go +++ b/wafbot/determine_search.go @@ -2,6 +2,8 @@ package wafbot import ( "SamWaf/utils" + "errors" + "net" "strings" ) @@ -42,6 +44,11 @@ func DetermineNormalSearch(userAgent, ip string) BotResult { func baiduSpider(ip string) BotResult { //先查询本地库 //然后远端查询 + fakeSpiderResult := BotResult{ + IsBot: true, + IsNormalBot: false, + BotName: "伪装百度爬虫", + } lookup, err := ReverseDNSLookup(ip) if err == nil { if len(lookup) > 0 { @@ -52,25 +59,36 @@ func baiduSpider(ip string) BotResult { BotName: "百度爬虫", } } else { + return fakeSpiderResult + } + } else { + return fakeSpiderResult + } + } else { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsTimeout { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是百度爬虫", + IsNormalBot: true, + BotName: "查询超时", + } + } else if dnsErr.IsNotFound { + return fakeSpiderResult + } else { + return BotResult{ + IsBot: true, + IsNormalBot: true, + BotName: "查询失败", } } } else { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是百度爬虫", + IsNormalBot: true, + BotName: "查询失败", } } - } else { - return BotResult{ - IsBot: true, - IsNormalBot: true, - BotName: "查询超时", - } } } @@ -81,6 +99,11 @@ func baiduSpider(ip string) BotResult { func googleSpider(ip string) BotResult { //先查询本地库 //然后远端查询 + fakeSpiderResult := BotResult{ + IsBot: true, + IsNormalBot: false, + BotName: "伪装Google爬虫", + } lookup, err := ReverseDNSLookup(ip) if err == nil { if len(lookup) > 0 { @@ -103,25 +126,36 @@ func googleSpider(ip string) BotResult { BotName: "Google爬虫(用户触发)", } } else { + return fakeSpiderResult + } + } else { + return fakeSpiderResult + } + } else { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsTimeout { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是Google爬虫", + IsNormalBot: true, + BotName: "查询超时", + } + } else if dnsErr.IsNotFound { + return fakeSpiderResult + } else { + return BotResult{ + IsBot: true, + IsNormalBot: true, + BotName: "查询失败", } } } else { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是Google爬虫", + IsNormalBot: true, + BotName: "查询失败", } } - } else { - return BotResult{ - IsBot: true, - IsNormalBot: true, - BotName: "查询超时", - } } } @@ -132,6 +166,11 @@ bing的蜘蛛 func bingSpider(ip string) BotResult { //先查询本地库 //然后远端查询 + fakeSpiderResult := BotResult{ + IsBot: true, + IsNormalBot: false, + BotName: "伪装Bing爬虫", + } lookup, err := ReverseDNSLookup(ip) if err == nil { if len(lookup) > 0 { @@ -142,25 +181,36 @@ func bingSpider(ip string) BotResult { BotName: "Bing爬虫", } } else { + return fakeSpiderResult + } + } else { + return fakeSpiderResult + } + } else { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsTimeout { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是Bing爬虫", + IsNormalBot: true, + BotName: "查询超时", + } + } else if dnsErr.IsNotFound { + return fakeSpiderResult + } else { + return BotResult{ + IsBot: true, + IsNormalBot: true, + BotName: "查询失败", } } } else { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是Bing爬虫", + IsNormalBot: true, + BotName: "查询失败", } } - } else { - return BotResult{ - IsBot: true, - IsNormalBot: true, - BotName: "可能不是Bing爬虫", - } } } @@ -171,6 +221,11 @@ sogou蜘蛛 func sogouSpider(ip string) BotResult { //先查询本地库 //然后远端查询 + fakeSpiderResult := BotResult{ + IsBot: true, + IsNormalBot: false, + BotName: "伪装搜狗爬虫", + } lookup, err := ReverseDNSLookup(ip) if err == nil { if len(lookup) > 0 { @@ -181,25 +236,36 @@ func sogouSpider(ip string) BotResult { BotName: "搜狗爬虫", } } else { + return fakeSpiderResult + } + } else { + return fakeSpiderResult + } + } else { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsTimeout { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是搜狗爬虫", + IsNormalBot: true, + BotName: "查询超时", + } + } else if dnsErr.IsNotFound { + return fakeSpiderResult + } else { + return BotResult{ + IsBot: true, + IsNormalBot: true, + BotName: "查询失败", } } } else { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是搜狗爬虫", + IsNormalBot: true, + BotName: "查询失败", } } - } else { - return BotResult{ - IsBot: true, - IsNormalBot: true, - BotName: "查询超时", - } } } @@ -265,6 +331,12 @@ UC 搜索 func yisouSpider(ip string) BotResult { //先查询本地库 //然后远端查询 + + fakeSpiderResult := BotResult{ + IsBot: true, + IsNormalBot: false, + BotName: "伪装神马搜索爬虫", + } lookup, err := ReverseDNSLookup(ip) if err == nil { if len(lookup) > 0 { @@ -275,25 +347,36 @@ func yisouSpider(ip string) BotResult { BotName: "神马搜索爬虫", } } else { + return fakeSpiderResult + } + } else { + return fakeSpiderResult + } + } else { + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + if dnsErr.IsTimeout { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是神马搜索爬虫", + IsNormalBot: true, + BotName: "查询超时", + } + } else if dnsErr.IsNotFound { + return fakeSpiderResult + } else { + return BotResult{ + IsBot: true, + IsNormalBot: true, + BotName: "查询失败", } } } else { return BotResult{ IsBot: true, - IsNormalBot: false, - BotName: "可能不是神马搜索爬虫", + IsNormalBot: true, + BotName: "查询失败", } } - } else { - return BotResult{ - IsBot: true, - IsNormalBot: true, - BotName: "查询超时", - } } } diff --git a/wafbot/dns_search.go b/wafbot/dns_search.go index 56dd43e..a78879d 100644 --- a/wafbot/dns_search.go +++ b/wafbot/dns_search.go @@ -3,7 +3,6 @@ package wafbot import ( "SamWaf/global" "context" - "fmt" "net" "time" ) @@ -18,7 +17,8 @@ func ReverseDNSLookup(ipAddress string) ([]string, error) { return net.Dial("udp", global.GWAF_RUNTIME_DNS_SERVER+":53") }, }} - ctxWithTimeout, cancel := context.WithTimeout(ctx, 500*time.Millisecond) + //TODO 请注意此处得时间 + ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(global.GWAF_RUNTIME_DNS_TIMEOUT)*time.Millisecond) defer cancel() names, err := d.Resolver.LookupAddr(ctxWithTimeout, ipAddress) @@ -26,8 +26,7 @@ func ReverseDNSLookup(ipAddress string) ([]string, error) { //zlog.Debug("搜索引擎查询耗时", elapsed.String()) if err != nil { - return nil, fmt.Errorf("逆向 DNS 查询失败: %s", err) + return nil, err } - return names, nil } diff --git a/wafbot/dns_search_test.go b/wafbot/dns_search_test.go index 9cd6201..fbe6e5b 100644 --- a/wafbot/dns_search_test.go +++ b/wafbot/dns_search_test.go @@ -17,7 +17,7 @@ func TestReverseGoogleDNSLookup(t *testing.T) { } func TestReverseDNSLookup(t *testing.T) { - lookup, err := ReverseDNSLookup("3.3.77.3") + lookup, err := ReverseDNSLookup("106.8.139.201") if err == nil { for _, s := range lookup { fmt.Println(s) diff --git a/wafenginecore/checkbot.go b/wafenginecore/checkbot.go index a6dc627..ccab003 100644 --- a/wafenginecore/checkbot.go +++ b/wafenginecore/checkbot.go @@ -51,12 +51,14 @@ func (waf *WafEngine) CheckBot(r *http.Request, weblogbean *innerbean.WebLog, fo } result.Title = botResult.BotName result.Content = "请正确访问" + + if !isBotCacheExist { + //如果是bot 加入cache里面 + global.GCACHE_WAFCACHE.SetWithTTl(enums.CACHE_DNS_BOT_IP+weblogbean.SRC_IP, botResult, time.Duration(global.GCONFIG_RECORD_DNS_BOT_EXPIRE_HOURS)*time.Hour) + } return result } - if !isBotCacheExist { - //如果是bot 加入cache里面 - global.GCACHE_WAFCACHE.SetWithTTl(enums.CACHE_DNS_BOT_IP+weblogbean.SRC_IP, botResult, time.Duration(global.GCONFIG_RECORD_DNS_BOT_EXPIRE_HOURS)*time.Hour) - } + } else { //如果不是bot 加入到正常cache里面 global.GCACHE_WAFCACHE.SetWithTTl(enums.CACHE_DNS_NORMAL_IP+weblogbean.SRC_IP, weblogbean.SRC_IP, time.Duration(global.GCONFIG_RECORD_DNS_NORMAL_EXPIRE_HOURS)*time.Hour) diff --git a/waftask/task_config.go b/waftask/task_config.go index c06c872..6003651 100644 --- a/waftask/task_config.go +++ b/waftask/task_config.go @@ -63,6 +63,9 @@ func setConfigIntValue(name string, value int64, change int) { case "enable_debug": global.GCONFIG_RECORD_DEBUG_ENABLE = value break + case "dns_timeout": + global.GWAF_RUNTIME_DNS_TIMEOUT = value + break default: zlog.Warn("Unknown config item:", name) } @@ -153,6 +156,8 @@ func TaskLoadSetting(initLoad bool) { updateConfigIntItem(initLoad, "system", "auto_load_ssl_file", global.GCONFIG_RECORD_AUTO_LOAD_SSL, "是否每天凌晨3点自动加载ssl证书", "int", "") updateConfigStringItem(initLoad, "system", "dns_server", global.GWAF_RUNTIME_DNS_SERVER, "DNS服务器", "options", "119.29.29.29|腾讯DNS,8.8.8.8|谷歌DNS") + updateConfigIntItem(initLoad, "system", "dns_timeout", global.GWAF_RUNTIME_DNS_TIMEOUT, "DNS 查询超时时间 单位毫秒", "int", "") + updateConfigStringItem(initLoad, "system", "record_log_type", global.GWAF_RUNTIME_RECORD_LOG_TYPE, "日志记录类型", "options", "all|全部,abnormal|非正常") updateConfigStringItem(initLoad, "system", "gwaf_center_enable", global.GWAF_CENTER_ENABLE, "中心开关", "bool", "false|关闭,true|开启") updateConfigStringItem(initLoad, "system", "gwaf_center_url", global.GWAF_CENTER_URL, "中心URL", "string", "")