diff --git a/ThirdLicense b/ThirdLicense index c4b916b..f2145eb 100644 --- a/ThirdLicense +++ b/ThirdLicense @@ -95,3 +95,6 @@ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLI corazawaf This project includes components from [corazawaf], which is licensed under the Apache License 2.0. + +GeoLite2-Country.mmdb +This project includes ipv6 data from [maxmind] diff --git a/docs/compile.md b/docs/compile.md index b107f5b..5ff41f0 100644 --- a/docs/compile.md +++ b/docs/compile.md @@ -88,7 +88,8 @@ docker run --rm -v %cd%:/workspace samwaflocalcompile 前端: 使用TDesign Vue Starter 后端: gorm,excelize(360EntSecGroup-Skylar),godlp(bytedance),gin,gocron, grule-rule-engine,ip2region,sqlitedriver,viper,libinjection-go,corazawaf - +数据: +ipv6(GeoLite2-Country.mmdb) by maxmind ## TODO List - 完善对应功能test \ No newline at end of file diff --git a/exedata/GeoLite2-Country.mmdb b/exedata/GeoLite2-Country.mmdb new file mode 100644 index 0000000..9c751dc Binary files /dev/null and b/exedata/GeoLite2-Country.mmdb differ diff --git a/global/global.go b/global/global.go index bd60c10..96ce36a 100644 --- a/global/global.go +++ b/global/global.go @@ -10,6 +10,8 @@ import ( "SamWaf/wafowasp" "SamWaf/wafsnowflake" "github.com/bytedance/godlp/dlpheader" + "github.com/lionsoul2014/ip2region/binding/golang/xdb" + "github.com/oschwald/geoip2-golang" "gorm.io/gorm" "strconv" "time" @@ -78,7 +80,12 @@ var ( /*****CACHE相关*********/ GCACHE_WAFCACHE *cache.WafCache //cache GCACHE_WECHAT_ACCESS string = "" //微信访问密钥 - GCACHE_IP_CBUFF []byte // IP相关缓存 + + /*********IP相关**************/ + GCACHE_IP_CBUFF []byte // IP相关缓存 + GCACHE_IP_V6_COUNTRY_CBUFF []byte // IPv6国家相关缓存 + GCACHE_IPV4_SEARCHER *xdb.Searcher //IPV4得查询器 + GCACHE_IPV6_SEARCHER *geoip2.Reader // IPV6得查询器 GDATA_DELETE_INTERVAL int64 = 180 // 删除180天前的数据 diff --git a/go.mod b/go.mod index 245a607..d665548 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/hyperjumptech/grule-rule-engine v1.11.0 github.com/kardianos/service v1.2.2 github.com/lionsoul2014/ip2region/binding/golang v0.0.0-20220907060842-b2ba5d58e48d + github.com/oschwald/geoip2-golang v1.11.0 github.com/pengge/go-wxsqlite3 v0.0.0-20231127082057-d869bc67f783 github.com/pengge/sqlitedriver v0.0.0-20231127095117-b0f000e40c2c github.com/satori/go.uuid v1.2.0 @@ -65,6 +66,7 @@ require ( github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 // indirect + github.com/oschwald/maxminddb-golang v1.13.0 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.5 // indirect github.com/petar-dambovaliev/aho-corasick v0.0.0-20240411101913-e07a1f0e8eb4 // indirect diff --git a/go.sum b/go.sum index 837bc22..8adbaa7 100644 --- a/go.sum +++ b/go.sum @@ -249,6 +249,10 @@ github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9G github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= +github.com/oschwald/geoip2-golang v1.11.0 h1:hNENhCn1Uyzhf9PTmquXENiWS6AlxAEnBII6r8krA3w= +github.com/oschwald/geoip2-golang v1.11.0/go.mod h1:P9zG+54KPEFOliZ29i7SeYZ/GM6tfEL+rgSn03hYuUo= +github.com/oschwald/maxminddb-golang v1.13.0 h1:R8xBorY71s84yO06NgTmQvqvTvlS/bnYZrrWX1MElnU= +github.com/oschwald/maxminddb-golang v1.13.0/go.mod h1:BU0z8BfFVhi1LQaonTwwGQlsHUEu9pWNdMfmq4ztm0o= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= diff --git a/main.go b/main.go index 8ca4b88..36b6ceb 100644 --- a/main.go +++ b/main.go @@ -55,6 +55,9 @@ import ( //go:embed exedata/ip2region.xdb var Ip2regionBytes []byte // 当前目录,解析为[]byte类型 +//go:embed exedata/GeoLite2-Country.mmdb +var Ipv6CountryBytes []byte // IPv6国家解析 + //go:embed exedata/ldpconfig.yml var ldpConfig string //隐私防护ldp @@ -133,6 +136,21 @@ func (m *wafSystenService) run() { zlog.Info("IP database ip2region.xdb loaded into cache, size: ", len(global.GCACHE_IP_CBUFF), ip2RegionFilePath) } + //检测是否存在IPV6得数据包 + ipv6RegionFilePath := filepath.Join(utils.GetCurrentDir(), "data", "GeoLite2-Country.mmdb") + // 检查文件是否存在 + if _, err := os.Stat(ipv6RegionFilePath); os.IsNotExist(err) { + global.GCACHE_IP_V6_COUNTRY_CBUFF = Ipv6CountryBytes + } else { + // 读取文件内容 + fileBytes, err := ioutil.ReadFile(ipv6RegionFilePath) + if err != nil { + log.Fatalf("Failed to read IPv6 database file GeoLite2-Country.mmdb: %v", err) + } + global.GCACHE_IP_V6_COUNTRY_CBUFF = fileBytes + // 检查是否成功读取 + zlog.Info("IPv6 database file GeoLite2-Country.mmdb loaded into cache, size: ", len(global.GCACHE_IP_V6_COUNTRY_CBUFF), ipv6RegionFilePath) + } global.GWAF_DLP_CONFIG = ldpConfig global.GWAF_REG_PUBLIC_KEY = publicKey @@ -619,6 +637,9 @@ func (m *wafSystenService) stopSamWaf() { webmanager.CloseLocalServer() zlog.Debug("Shutdown SamWaf WebManager finished") + zlog.Debug("Shutdown SamWaf IPDatabase...") + utils.CloseIPDatabase() + zlog.Debug("Shutdown SamWaf IPDatabase finished") } // 优雅升级 diff --git a/utils/common.go b/utils/common.go index 3cdb11f..9440a62 100644 --- a/utils/common.go +++ b/utils/common.go @@ -6,6 +6,7 @@ import ( "SamWaf/model" "fmt" "github.com/lionsoul2014/ip2region/binding/golang/xdb" + "github.com/oschwald/geoip2-golang" "io/ioutil" "net" "net/http" @@ -115,20 +116,47 @@ func GetPublicIP() string { } func GetCountry(ip string) []string { - // 2、用全局的 cBuff 创建完全基于内存的查询对象。 - searcher, err := xdb.NewWithBuffer(global.GCACHE_IP_CBUFF) - if err != nil { - fmt.Printf("failed to create searcher with content: %s\n", err) + if IsValidIPv6(ip) { + a := "ipv6|ipv6|ipv6|ipv6|ipv6" + if global.GCACHE_IPV6_SEARCHER == nil { + db, err := geoip2.FromBytes(global.GCACHE_IP_V6_COUNTRY_CBUFF) + if err != nil { + zlog.Error("Failed to open GeoLite2-Country.mmdb:", err) + return strings.Split(a, "|") + } + global.GCACHE_IPV6_SEARCHER = db + } + ipv6 := net.ParseIP(ip) + record, err := global.GCACHE_IPV6_SEARCHER.Country(ipv6) + if err != nil { + zlog.Error("Failed to Search GeoLite2-Country.mmdb:", err) + return strings.Split(a, "|") + } + if record.Country.Names == nil { + a = "内网" + "||||" + } else { + a = record.Country.Names["zh-CN"] + "||||" + } + return strings.Split(a, "|") } - defer searcher.Close() + //IPV4得查询逻辑 + if global.GCACHE_IPV4_SEARCHER == nil { + // 2、用全局的 cBuff 创建完全基于内存的查询对象。 + searcher, err := xdb.NewWithBuffer(global.GCACHE_IP_CBUFF) + if err != nil { + fmt.Printf("failed to create searcher with content: %s\n", err) + + } + global.GCACHE_IPV4_SEARCHER = searcher + } // do the search var tStart = time.Now() // 备注:并发使用,每个 goroutine 需要创建一个独立的 searcher 对象。 - region, err := searcher.SearchByStr(ip) + region, err := global.GCACHE_IPV4_SEARCHER.SearchByStr(ip) if err != nil { fmt.Printf("failed to SearchIP(%s): %s\n", ip, err) return []string{"无", "无"} @@ -145,6 +173,19 @@ func GetCountry(ip string) []string { return regions } +// CloseIPDatabase 关闭IP数据库 +func CloseIPDatabase() { + if global.GCACHE_IPV4_SEARCHER != nil { + global.GCACHE_IPV4_SEARCHER.Close() + } + if global.GCACHE_IPV6_SEARCHER != nil { + err := global.GCACHE_IPV6_SEARCHER.Close() + if err != nil { + return + } + } +} + // PortCheck 检查端口是否可用,可用-true 不可用-false func PortCheck(port int) bool { conn, err := net.DialTimeout("tcp", fmt.Sprintf(":%d", port), time.Second) @@ -283,6 +324,13 @@ func IsValidIPv4(ip string) bool { return parsedIP != nil && parsedIP.To4() != nil } +// 验证IPv6地址是否有效 +func IsValidIPv6(ip string) bool { + parsedIP := net.ParseIP(ip) + // 如果解析出的 IP 类型是 IPv6 且不是 IPv4 映射地址(IPv4-mapped IPv6 addresses) + return parsedIP != nil && strings.Contains(ip, ":") && parsedIP.To4() == nil +} + // 获取纯域名 func GetPureDomain(host string) string { diff --git a/utils/common_test.go b/utils/common_test.go new file mode 100644 index 0000000..225c6aa --- /dev/null +++ b/utils/common_test.go @@ -0,0 +1,33 @@ +package utils + +import ( + "fmt" + "github.com/oschwald/geoip2-golang" + "log" + "net" + "testing" +) + +func TestIPv6(t *testing.T) { + db, err := geoip2.Open("../data/ipv6/GeoLite2-Country.mmdb") + if err != nil { + log.Fatal(err) + } + defer db.Close() + // If you are using strings that may be invalid, check that ip is not nil + ip := net.ParseIP("2409:8087:3c02:21:0:1:0:100a") + record, err := db.City(ip) + if err != nil { + log.Fatal(err) + } + fmt.Printf("国家: %v\n", record.Country.Names["zh-CN"]) + fmt.Sprintf("%v", record) + fmt.Printf("Portuguese (BR) city name: %v\n", record.City.Names["pt-BR"]) + if len(record.Subdivisions) > 0 { + fmt.Printf("English subdivision name: %v\n", record.Subdivisions[0].Names["en"]) + } + fmt.Printf("Russian country name: %v\n", record.Country.Names["ru"]) + fmt.Printf("ISO country code: %v\n", record.Country.IsoCode) + fmt.Printf("Time zone: %v\n", record.Location.TimeZone) + fmt.Printf("Coordinates: %v, %v\n", record.Location.Latitude, record.Location.Longitude) +} diff --git a/utils/rulehelper_test.go b/utils/rulehelper_test.go index 8b44a5b..d473e4a 100644 --- a/utils/rulehelper_test.go +++ b/utils/rulehelper_test.go @@ -23,8 +23,6 @@ rule R1ca0bf1c409e4c1b823c995afe7733b0 "禁止一些robotahrefs" salience 10 { } ` var ruleconfigs []model.Rules rule := model.Rules{ - Id: 0, - TenantId: "", HostCode: "", RuleCode: "", RuleName: "", @@ -32,7 +30,6 @@ rule R1ca0bf1c409e4c1b823c995afe7733b0 "禁止一些robotahrefs" salience 10 { RuleContentJSON: "", RuleVersionName: "", RuleVersion: 0, - UserCode: "", IsPublicRule: 0, IsManualRule: 0, RuleStatus: 0, diff --git a/wafenginecore/wafengine.go b/wafenginecore/wafengine.go index d5b0ece..28871ee 100644 --- a/wafenginecore/wafengine.go +++ b/wafenginecore/wafengine.go @@ -419,6 +419,9 @@ func (waf *WafEngine) getClientIP(r *http.Request, headers ...string) (error, st if utils.IsValidIPv4(trimmedIP) { return nil, trimmedIP, "0" } + if utils.IsValidIPv6(ip) { + return nil, ip, "0" + } return errors.New("invalid IPv4 address from header: " + header + " value:" + ip), "", "" } } @@ -428,11 +431,25 @@ func (waf *WafEngine) getClientIP(r *http.Request, headers ...string) (error, st if err != nil { return err, "", "" } - if !utils.IsValidIPv4(ip) { - return errors.New("invalid IPv4 address from RemoteAddr" + " value:" + ip), "", "" + + // 验证 IPv4 地址 + if utils.IsValidIPv4(ip) { + return nil, ip, port } - return nil, ip, port + // 如果是 IPv6 地址,进一步检查是否是有效的 IPv6 地址 + if strings.Contains(ip, ":") { + // 如果是 IPv6 地址,进一步检查是否是有效的 IPv6 地址 + if strings.Contains(ip, ":") { + if utils.IsValidIPv6(ip) { + return nil, ip, port + } else { + return errors.New("invalid IPv6 address from RemoteAddr: " + ip), "", "" + } + } + } + + return errors.New("invalid IP address (not IPv4 or IPv6): " + ip), "", "" } // EchoErrorInfo ruleName 对内记录 blockInfo 对外展示