opensource/SamWaf
3
0
Fork 0
mirror of https://gitee.com/samwaf/SamWaf.git synced 2026-09-24 22:39:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix:spider timeout

Browse Source
This commit is contained in:
samwaf
2025-02-05 17:00:15 +08:00
parent 92c9395ba5
commit 0e1c34e436
6 changed files with 150 additions and 60 deletions
Download Patch File Download Diff File Expand all files Collapse all files
global/global.go
+2 -1
View File
@@ -35,7 +35,8 @@ var (
GWAF_RUNTIME_QPS uint64 = 0 //当前qps
GWAF_RUNTIME_LOG_PROCESS uint64 = 0 //log 处理速度
GWAF_RUNTIME_DNS_SERVER string = "119.29.29.29" //反向查询DNS的IP
GWAF_RUNTIME_DNS_SERVER string = "119.29.29.29" //反向查询DNS的IP
GWAF_RUNTIME_DNS_TIMEOUT int64 = 500 // DNS 查询超时时间 单位毫秒
GWAF_RUNTIME_RECORD_LOG_TYPE string = "all" // 记录日志形式： 全部(all),非正常(abnormal)
GWAF_RUNTIME_IS_UPDATETING bool = false //是否正在升级中
wafbot/determine_search.go
+133 -50
View File
@@ -2,6 +2,8 @@ package wafbot
import (
"SamWaf/utils"
"errors"
"net"
"strings"
)
@@ -42,6 +44,11 @@ func DetermineNormalSearch(userAgent, ip string) BotResult {
func baiduSpider(ip string) BotResult {
//先查询本地库
//然后远端查询
fakeSpiderResult := BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "伪装百度爬虫",
}
lookup, err := ReverseDNSLookup(ip)
if err == nil {
if len(lookup) > 0 {
@@ -52,25 +59,36 @@ func baiduSpider(ip string) BotResult {
BotName: "百度爬虫",
}
} else {
return fakeSpiderResult
}
} else {
return fakeSpiderResult
}
} else {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
if dnsErr.IsTimeout {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是百度爬虫",
IsNormalBot: true,
BotName: "查询超时",
}
} else if dnsErr.IsNotFound {
return fakeSpiderResult
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是百度爬虫",
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询超时",
}
}
}
@@ -81,6 +99,11 @@ func baiduSpider(ip string) BotResult {
func googleSpider(ip string) BotResult {
//先查询本地库
//然后远端查询
fakeSpiderResult := BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "伪装Google爬虫",
}
lookup, err := ReverseDNSLookup(ip)
if err == nil {
if len(lookup) > 0 {
@@ -103,25 +126,36 @@ func googleSpider(ip string) BotResult {
BotName: "Google爬虫(用户触发)",
}
} else {
return fakeSpiderResult
}
} else {
return fakeSpiderResult
}
} else {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
if dnsErr.IsTimeout {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是Google爬虫",
IsNormalBot: true,
BotName: "查询超时",
}
} else if dnsErr.IsNotFound {
return fakeSpiderResult
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是Google爬虫",
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询超时",
}
}
}
@@ -132,6 +166,11 @@ bing的蜘蛛
func bingSpider(ip string) BotResult {
//先查询本地库
//然后远端查询
fakeSpiderResult := BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "伪装Bing爬虫",
}
lookup, err := ReverseDNSLookup(ip)
if err == nil {
if len(lookup) > 0 {
@@ -142,25 +181,36 @@ func bingSpider(ip string) BotResult {
BotName: "Bing爬虫",
}
} else {
return fakeSpiderResult
}
} else {
return fakeSpiderResult
}
} else {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
if dnsErr.IsTimeout {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是Bing爬虫",
IsNormalBot: true,
BotName: "查询超时",
}
} else if dnsErr.IsNotFound {
return fakeSpiderResult
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是Bing爬虫",
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "可能不是Bing爬虫",
}
}
}
@@ -171,6 +221,11 @@ sogou蜘蛛
func sogouSpider(ip string) BotResult {
//先查询本地库
//然后远端查询
fakeSpiderResult := BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "伪装搜狗爬虫",
}
lookup, err := ReverseDNSLookup(ip)
if err == nil {
if len(lookup) > 0 {
@@ -181,25 +236,36 @@ func sogouSpider(ip string) BotResult {
BotName: "搜狗爬虫",
}
} else {
return fakeSpiderResult
}
} else {
return fakeSpiderResult
}
} else {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
if dnsErr.IsTimeout {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是搜狗爬虫",
IsNormalBot: true,
BotName: "查询超时",
}
} else if dnsErr.IsNotFound {
return fakeSpiderResult
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是搜狗爬虫",
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询超时",
}
}
}
@@ -265,6 +331,12 @@ UC 搜索
func yisouSpider(ip string) BotResult {
//先查询本地库
//然后远端查询
fakeSpiderResult := BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "伪装神马搜索爬虫",
}
lookup, err := ReverseDNSLookup(ip)
if err == nil {
if len(lookup) > 0 {
@@ -275,25 +347,36 @@ func yisouSpider(ip string) BotResult {
BotName: "神马搜索爬虫",
}
} else {
return fakeSpiderResult
}
} else {
return fakeSpiderResult
}
} else {
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
if dnsErr.IsTimeout {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是神马搜索爬虫",
IsNormalBot: true,
BotName: "查询超时",
}
} else if dnsErr.IsNotFound {
return fakeSpiderResult
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: false,
BotName: "可能不是神马搜索爬虫",
IsNormalBot: true,
BotName: "查询失败",
}
}
} else {
return BotResult{
IsBot: true,
IsNormalBot: true,
BotName: "查询超时",
}
}
}
wafbot/dns_search.go
+3 -4
View File
@@ -3,7 +3,6 @@ package wafbot
import (
"SamWaf/global"
"context"
"fmt"
"net"
"time"
)
@@ -18,7 +17,8 @@ func ReverseDNSLookup(ipAddress string) ([]string, error) {
return net.Dial("udp", global.GWAF_RUNTIME_DNS_SERVER+":53")
},
}}
ctxWithTimeout, cancel := context.WithTimeout(ctx, 500*time.Millisecond)
//TODO 请注意此处得时间
ctxWithTimeout, cancel := context.WithTimeout(ctx, time.Duration(global.GWAF_RUNTIME_DNS_TIMEOUT)*time.Millisecond)
defer cancel()
names, err := d.Resolver.LookupAddr(ctxWithTimeout, ipAddress)
@@ -26,8 +26,7 @@ func ReverseDNSLookup(ipAddress string) ([]string, error) {
//zlog.Debug("搜索引擎查询耗时", elapsed.String())
if err != nil {
return nil, fmt.Errorf("逆向 DNS 查询失败: %s", err)
return nil, err
}
return names, nil
}
wafbot/dns_search_test.go
+1 -1
View File
@@ -17,7 +17,7 @@ func TestReverseGoogleDNSLookup(t *testing.T) {
}
func TestReverseDNSLookup(t *testing.T) {
lookup, err := ReverseDNSLookup("3.3.77.3")
lookup, err := ReverseDNSLookup("106.8.139.201")
if err == nil {
for _, s := range lookup {
fmt.Println(s)
wafenginecore/checkbot.go
+6 -4
View File
@@ -51,12 +51,14 @@ func (waf *WafEngine) CheckBot(r *http.Request, weblogbean *innerbean.WebLog, fo
}
result.Title = botResult.BotName
result.Content = "请正确访问"
if !isBotCacheExist {
//如果是bot 加入cache里面
global.GCACHE_WAFCACHE.SetWithTTl(enums.CACHE_DNS_BOT_IP+weblogbean.SRC_IP, botResult, time.Duration(global.GCONFIG_RECORD_DNS_BOT_EXPIRE_HOURS)*time.Hour)
}
return result
}
if !isBotCacheExist {
//如果是bot 加入cache里面
global.GCACHE_WAFCACHE.SetWithTTl(enums.CACHE_DNS_BOT_IP+weblogbean.SRC_IP, botResult, time.Duration(global.GCONFIG_RECORD_DNS_BOT_EXPIRE_HOURS)*time.Hour)
}
} else {
//如果不是bot 加入到正常cache里面
global.GCACHE_WAFCACHE.SetWithTTl(enums.CACHE_DNS_NORMAL_IP+weblogbean.SRC_IP, weblogbean.SRC_IP, time.Duration(global.GCONFIG_RECORD_DNS_NORMAL_EXPIRE_HOURS)*time.Hour)
waftask/task_config.go
+5
View File
@@ -63,6 +63,9 @@ func setConfigIntValue(name string, value int64, change int) {
case "enable_debug":
global.GCONFIG_RECORD_DEBUG_ENABLE = value
break
case "dns_timeout":
global.GWAF_RUNTIME_DNS_TIMEOUT = value
break
default:
zlog.Warn("Unknown config item:", name)
}
@@ -153,6 +156,8 @@ func TaskLoadSetting(initLoad bool) {
updateConfigIntItem(initLoad, "system", "auto_load_ssl_file", global.GCONFIG_RECORD_AUTO_LOAD_SSL, "是否每天凌晨3点自动加载ssl证书", "int", "")
updateConfigStringItem(initLoad, "system", "dns_server", global.GWAF_RUNTIME_DNS_SERVER, "DNS服务器", "options", "119.29.29.29|腾讯DNS,8.8.8.8|谷歌DNS")
updateConfigIntItem(initLoad, "system", "dns_timeout", global.GWAF_RUNTIME_DNS_TIMEOUT, "DNS 查询超时时间 单位毫秒", "int", "")
updateConfigStringItem(initLoad, "system", "record_log_type", global.GWAF_RUNTIME_RECORD_LOG_TYPE, "日志记录类型", "options", "all|全部,abnormal|非正常")
updateConfigStringItem(initLoad, "system", "gwaf_center_enable", global.GWAF_CENTER_ENABLE, "中心开关", "bool", "false|关闭,true|开启")
updateConfigStringItem(initLoad, "system", "gwaf_center_url", global.GWAF_CENTER_URL, "中心URL", "string", "")