diff --git a/api/waf_stat.go b/api/waf_stat.go index d72c0fb..4018c47 100644 --- a/api/waf_stat.go +++ b/api/waf_stat.go @@ -1,6 +1,7 @@ package api import ( + "SamWaf/common/zlog" "SamWaf/model/common/response" "SamWaf/model/request" @@ -50,7 +51,13 @@ func (w *WafStatApi) StatRumtimeSysinfoApi(c *gin.Context) { // @Router /wafstatsumday [get] func (w *WafStatApi) StatHomeSumDayApi(c *gin.Context) { - wafStat, _ := wafStatService.StatHomeSumDayApi() + wafStat, err := wafStatService.StatHomeSumDayApi() + if err != nil { + // 详细错误只落日志,响应体不回传 SQL/表结构信息 + zlog.Error("获取今日访问统计失败", err) + response.FailWithMessage("获取统计数据失败", c) + return + } response.OkWithDetailed(wafStat, "获取成功", c) } @@ -68,7 +75,12 @@ func (w *WafStatApi) StatHomeSumDayRangeApi(c *gin.Context) { var req request.WafStatsDayRangeReq err := c.ShouldBind(&req) if err == nil { - wafStat, _ := wafStatService.StatHomeSumDayRangeApi(req) + wafStat, err := wafStatService.StatHomeSumDayRangeApi(req) + if err != nil { + zlog.Error("按日期范围统计访问量失败", err) + response.FailWithMessage("获取统计数据失败", c) + return + } response.OkWithDetailed(wafStat, "获取成功", c) } else { @@ -90,7 +102,12 @@ func (w *WafStatApi) StatHomeSumDayTopIPRangeApi(c *gin.Context) { var req request.WafStatsDayRangeReq err := c.ShouldBind(&req) if err == nil { - wafStat, _ := wafStatService.StatHomeSumDayTopIPRangeApi(req) + wafStat, err := wafStatService.StatHomeSumDayTopIPRangeApi(req) + if err != nil { + zlog.Error("统计攻击IP排行失败", err) + response.FailWithMessage("获取统计数据失败", c) + return + } response.OkWithDetailed(wafStat, "获取成功", c) } else { @@ -112,7 +129,12 @@ func (w *WafStatApi) StatSiteOverviewApi(c *gin.Context) { var req request.WafStatsSiteOverviewReq err := c.ShouldBind(&req) if err == nil { - data, _ := wafStatService.StatSiteOverviewApi(req) + data, err := wafStatService.StatSiteOverviewApi(req) + if err != nil { + zlog.Error("站点综合概览查询失败", err) + response.FailWithMessage("获取统计数据失败", c) + return + } response.OkWithDetailed(data, "获取成功", c) } else { response.FailWithMessage("解析失败", c) @@ -133,7 +155,12 @@ func (w *WafStatApi) StatSiteDetailApi(c *gin.Context) { var req request.WafStatsSiteDetailReq err := c.ShouldBind(&req) if err == nil { - data, _ := wafStatService.StatSiteDetailApi(req) + data, err := wafStatService.StatSiteDetailApi(req) + if err != nil { + zlog.Error("站点详情趋势查询失败", err) + response.FailWithMessage("获取统计数据失败", c) + return + } response.OkWithDetailed(data, "获取成功", c) } else { response.FailWithMessage("解析失败", c) diff --git a/service/waf_service/cross_engine_stats_test.go b/service/waf_service/cross_engine_stats_test.go index 08851f8..00b3cfe 100644 --- a/service/waf_service/cross_engine_stats_test.go +++ b/service/waf_service/cross_engine_stats_test.go @@ -1,13 +1,18 @@ //go:build crossdb // stats 库三库回归:验证统计聚合(sum/group by 等方言 SQL)+ 结构化 SQL 查询 -// 在 SQLite/MySQL/PostgreSQL 上都能执行且结果合理。由 TestCrossEngine 每引擎调一次。 +// 在 SQLite/MySQL/PostgreSQL 上都能执行且结果正确。由 TestCrossEngine 每引擎调一次。 +// +// 注意:这里的断言必须校验**返回值**而不只是 err。历史上 StatSiteOverviewApi 吞掉了 +// 所有 Scan 错误恒返回 nil,导致 ONLY_FULL_GROUP_BY 违规(SELECT 非聚合列 host + +// GROUP BY host_code)在 MySQL/PG 上静默返回空列表却测试全绿。 package waf_service import ( "SamWaf/common/uuid" "SamWaf/model" req "SamWaf/model/request" + response2 "SamWaf/model/response" "strconv" "testing" "time" @@ -15,37 +20,162 @@ import ( "gorm.io/gorm" ) -func runStatsCases(t *testing.T, statsdb *gorm.DB) { - today := time.Now().Format("20060102") - dayInt, _ := strconv.Atoi(today) +// findSite 从站点概览列表里按 host_code 取出一项 +func findSite(t *testing.T, list []response2.WafSiteStatDetail, hostCode string) response2.WafSiteStatDetail { + t.Helper() + for _, d := range list { + if d.HostCode == hostCode { + return d + } + } + t.Fatalf("站点概览里找不到 host_code=%s(实际 %d 项)", hostCode, len(list)) + return response2.WafSiteStatDetail{} +} - // 种子:今日 StatsDay(阻止 5、放行 10) - must(t, statsdb.Create(&model.StatsDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: "h1", Host: "a.com", Day: dayInt, Type: "阻止", Count: 5}).Error) - must(t, statsdb.Create(&model.StatsDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: "h1", Host: "a.com", Day: dayInt, Type: "放行", Count: 10}).Error) - // 种子:今日站点统计 - must(t, statsdb.Create(&model.StatsSiteDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: "h1", Host: "a.com", Day: dayInt, TotalCount: 15, AttackCount: 5, NormalCount: 10}).Error) +func eq64(t *testing.T, what string, got, want int64) { + t.Helper() + if got != want { + t.Fatalf("%s 期望 %d,实际 %d", what, want, got) + } +} + +func runStatsCases(t *testing.T, statsdb *gorm.DB) { + now := time.Now() + today := now.Format("20060102") + yesterday := now.AddDate(0, 0, -1).Format("20060102") + todayInt, _ := strconv.Atoi(today) + yesterdayInt, _ := strconv.Atoi(yesterday) + curHourTs := (now.Unix() / 3600) * 3600 + + newStatsDay := func(hostCode, host string, day int, typ string, cnt int) *model.StatsDay { + return &model.StatsDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: hostCode, Host: host, Day: day, Type: typ, Count: cnt} + } + newIPDay := func(hostCode, host string, day int, ip, typ string, cnt int) *model.StatsIPDay { + return &model.StatsIPDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: hostCode, Host: host, Day: day, IP: ip, Type: typ, Count: cnt} + } + + // —— 种子:StatsDay(今日 阻止5/放行10,昨日 阻止3)—— + must(t, statsdb.Create(newStatsDay("h1", "a.com", todayInt, "阻止", 5)).Error) + must(t, statsdb.Create(newStatsDay("h1", "a.com", todayInt, "放行", 10)).Error) + must(t, statsdb.Create(newStatsDay("h1", "a.com", yesterdayInt, "阻止", 3)).Error) + + // —— 种子:StatsSiteDay,h1 跨两天 + h2 一天 —— + // h1 两天合计:total 20 / attack 6 / normal 14 / in 1536 / out 3072 / timeSpent 200 + must(t, statsdb.Create(&model.StatsSiteDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: "h1", Host: "a.com", Day: todayInt, + TotalCount: 15, AttackCount: 5, NormalCount: 10, TrafficIn: 1024, TrafficOut: 2048, TotalTimeSpent: 150}).Error) + must(t, statsdb.Create(&model.StatsSiteDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: "h1", Host: "a.com", Day: yesterdayInt, + TotalCount: 5, AttackCount: 1, NormalCount: 4, TrafficIn: 512, TrafficOut: 1024, TotalTimeSpent: 50}).Error) + must(t, statsdb.Create(&model.StatsSiteDay{BaseOrm: newBase(uuid.GenUUID()), HostCode: "h2", Host: "b.com", Day: todayInt, + TotalCount: 3, AttackCount: 0, NormalCount: 3, TrafficIn: 128, TrafficOut: 256, TotalTimeSpent: 30}).Error) + + // —— 种子:StatsSiteHour(当前整点,供 24h 趋势用)—— + must(t, statsdb.Create(&model.StatsSiteHour{BaseOrm: newBase(uuid.GenUUID()), HostCode: "h1", Host: "a.com", HourTime: curHourTs, + TotalCount: 8, AttackCount: 2, NormalCount: 6, TrafficIn: 64, TrafficOut: 128, TotalTimeSpent: 80}).Error) + + // —— 种子:StatsIPDay(此前套件从未种过,聚合一直跑在空表上)—— + // h1 今日去重 IP = {1.1.1.1, 2.2.2.2} = 2;h1 两天去重 = 再加 3.3.3.3 = 3;h2 今日 = 1 + must(t, statsdb.Create(newIPDay("h1", "a.com", todayInt, "1.1.1.1", "阻止", 3)).Error) + must(t, statsdb.Create(newIPDay("h1", "a.com", todayInt, "2.2.2.2", "阻止", 2)).Error) + must(t, statsdb.Create(newIPDay("h1", "a.com", todayInt, "1.1.1.1", "放行", 7)).Error) + must(t, statsdb.Create(newIPDay("h1", "a.com", yesterdayInt, "3.3.3.3", "阻止", 1)).Error) + must(t, statsdb.Create(newIPDay("h2", "b.com", todayInt, "4.4.4.4", "放行", 3)).Error) t.Run("StatHomeSumDay", func(t *testing.T) { res, err := WafStatServiceApp.StatHomeSumDayApi() fatalIf(t, err) - if res.AttackCountOfToday != 5 { - t.Fatalf("今日攻击数聚合期望 5,实际 %d", res.AttackCountOfToday) - } + eq64(t, "今日攻击数", res.AttackCountOfToday, 5) + eq64(t, "今日访问数", res.VisitCountOfToday, 15) + eq64(t, "昨日攻击数", res.AttackCountOfYesterday, 3) + // 去重 IP:今日放行 {1.1.1.1, 4.4.4.4}=2,今日阻止 {1.1.1.1, 2.2.2.2}=2 + eq64(t, "今日放行去重IP数", res.NormalIpCountOfToday, 2) + eq64(t, "今日拦截去重IP数", res.IllegalIpCountOfToday, 2) }) t.Run("StatHomeSumDayRange", func(t *testing.T) { - _, err := WafStatServiceApp.StatHomeSumDayRangeApi(req.WafStatsDayRangeReq{StartDay: today, EndDay: today}) - fatalIf(t, err) // 只验证聚合 SQL 在三方言可执行,不崩即可 + res, err := WafStatServiceApp.StatHomeSumDayRangeApi(req.WafStatsDayRangeReq{StartDay: yesterday, EndDay: today}) + fatalIf(t, err) + eq64(t, "区间今日攻击数", res.AttackCountOfRange[todayInt], 5) + eq64(t, "区间昨日攻击数", res.AttackCountOfRange[yesterdayInt], 3) + eq64(t, "区间今日放行数", res.NormalCountOfRange[todayInt], 10) }) t.Run("StatHomeSumDayTopIPRange", func(t *testing.T) { - _, err := WafStatServiceApp.StatHomeSumDayTopIPRangeApi(req.WafStatsDayRangeReq{StartDay: today, EndDay: today}) + res, err := WafStatServiceApp.StatHomeSumDayTopIPRangeApi(req.WafStatsDayRangeReq{StartDay: yesterday, EndDay: today}) fatalIf(t, err) + if len(res.AttackIPOfRange) != 3 { + t.Fatalf("攻击 TopIP 期望 3 条,实际 %d", len(res.AttackIPOfRange)) + } + // 按 sum(count) 倒序,1.1.1.1(3) 居首 + if res.AttackIPOfRange[0].IP != "1.1.1.1" || res.AttackIPOfRange[0].Count != 3 { + t.Fatalf("攻击 TopIP 首位期望 1.1.1.1/3,实际 %s/%d", res.AttackIPOfRange[0].IP, res.AttackIPOfRange[0].Count) + } + if len(res.NormalIPOfRange) != 2 { + t.Fatalf("放行 TopIP 期望 2 条,实际 %d", len(res.NormalIPOfRange)) + } }) + // 本次 #896 回归点:SELECT 非聚合列 host + GROUP BY host_code, + // 在 MySQL(ONLY_FULL_GROUP_BY)/PostgreSQL 上会直接报错,必须用聚合函数取值。 t.Run("StatSiteOverview", func(t *testing.T) { - _, err := WafStatServiceApp.StatSiteOverviewApi(req.WafStatsSiteOverviewReq{StartDay: today, EndDay: today}) + res, err := WafStatServiceApp.StatSiteOverviewApi(req.WafStatsSiteOverviewReq{StartDay: yesterday, EndDay: today}) fatalIf(t, err) + if len(res.SiteList) != 2 { + t.Fatalf("站点概览期望 2 个站点,实际 %d", len(res.SiteList)) + } + // 按 PV 倒序,h1(20) 在 h2(3) 之前 + if res.SiteList[0].HostCode != "h1" { + t.Fatalf("站点概览应按 PV 倒序,首位期望 h1,实际 %s", res.SiteList[0].HostCode) + } + h1 := findSite(t, res.SiteList, "h1") + if h1.Host != "a.com" { + t.Fatalf("h1 域名期望 a.com,实际 %q", h1.Host) + } + eq64(t, "h1 两天 PV", h1.TotalCount, 20) + eq64(t, "h1 两天拦截", h1.AttackCount, 6) + eq64(t, "h1 两天放行", h1.NormalCount, 14) + eq64(t, "h1 两天去重IP(UV)", h1.UvCount, 3) + if h1.AvgTimeMs != 10 { // 200ms / 20 次 + t.Fatalf("h1 平均耗时期望 10ms,实际 %v", h1.AvgTimeMs) + } + eq64(t, "总 PV", res.TotalPv, 23) + eq64(t, "总拦截", res.TotalAttack, 6) + eq64(t, "总 UV", res.TotalUv, 4) + }) + + t.Run("GetTodaySiteStatsByHostCodes", func(t *testing.T) { + m := WafStatServiceApp.GetTodaySiteStatsByHostCodes([]string{"h1", "h2"}) + if len(m) != 2 { + t.Fatalf("今日站点统计期望 2 项,实际 %d", len(m)) + } + eq64(t, "h1 今日 PV", m["h1"].TodayPvCount, 15) + eq64(t, "h1 今日拦截", m["h1"].TodayAttackCount, 5) + eq64(t, "h1 今日 UV", m["h1"].TodayUvCount, 2) + eq64(t, "h1 今日入流量", m["h1"].TodayTrafficIn, 1024) + eq64(t, "h2 今日 PV", m["h2"].TodayPvCount, 3) + }) + + t.Run("StatSiteDetail_7d", func(t *testing.T) { + res, err := WafStatServiceApp.StatSiteDetailApi(req.WafStatsSiteDetailReq{HostCode: "h1", TimeRange: "7d"}) + fatalIf(t, err) + if len(res.DayTrend) != 2 { + t.Fatalf("7d 趋势期望 2 天,实际 %d", len(res.DayTrend)) + } + if res.DayTrend[0].Day != yesterdayInt || res.DayTrend[1].Day != todayInt { + t.Fatalf("7d 趋势应按天升序,实际 %d,%d", res.DayTrend[0].Day, res.DayTrend[1].Day) + } + eq64(t, "7d 昨日 UV", res.DayTrend[0].UvCount, 1) + eq64(t, "7d 今日 UV", res.DayTrend[1].UvCount, 2) + eq64(t, "7d 总请求", res.TotalCountSum, 20) + }) + + t.Run("StatSiteDetail_24h", func(t *testing.T) { + res, err := WafStatServiceApp.StatSiteDetailApi(req.WafStatsSiteDetailReq{HostCode: "h1", TimeRange: "24h"}) + fatalIf(t, err) + if len(res.HourTrend) != 24 { + t.Fatalf("24h 趋势期望 24 个点,实际 %d", len(res.HourTrend)) + } + eq64(t, "24h 总请求", res.TotalCountSum, 8) + eq64(t, "24h 总耗时", res.TotalTimeSpentSum, 80) }) // 结构化 SQL 查询(动态选库 + 列白名单),对 stats 库 stats_days 计数 diff --git a/service/waf_service/waf_stat.go b/service/waf_service/waf_stat.go index df0a078..d08d55a 100644 --- a/service/waf_service/waf_stat.go +++ b/service/waf_service/waf_stat.go @@ -1,6 +1,7 @@ package waf_service import ( + "SamWaf/common/zlog" "SamWaf/global" "SamWaf/model" "SamWaf/model/request" @@ -23,29 +24,44 @@ func (receiver *WafStatService) StatHomeSumDayApi() (response2.WafStat, error) { currentDay, _ := strconv.Atoi(time.Now().Format("20060102")) yesterdayDay, _ := strconv.Atoi(time.Now().AddDate(0, 0, -1).Format("20060102")) + // 空表时 sum() 返回 NULL,直接 Scan 进 int64 会报 "converting NULL to int64", + // 用 coalesce 兜底成 0(sqlite/mysql/pg 通用),这样错误才能真实反映方言问题。 var AttackCountOfToday int64 - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? and type = ? ", - currentDay, "阻止").Select("sum(count) as vcnt").Row().Scan(&AttackCountOfToday) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? and type = ? ", + currentDay, "阻止").Select("coalesce(sum(count),0) as vcnt").Row().Scan(&AttackCountOfToday); err != nil { + return response2.WafStat{}, err + } var VisitCountOfToday int64 - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? ", - currentDay).Select("sum(count) as vcnt").Row().Scan(&VisitCountOfToday) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? ", + currentDay).Select("coalesce(sum(count),0) as vcnt").Row().Scan(&VisitCountOfToday); err != nil { + return response2.WafStat{}, err + } var AttackCountOfYesterday int64 - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? and type = ? ", - yesterdayDay, "阻止").Select("sum(count) as vcnt").Row().Scan(&AttackCountOfYesterday) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? and type = ? ", + yesterdayDay, "阻止").Select("coalesce(sum(count),0) as vcnt").Row().Scan(&AttackCountOfYesterday); err != nil { + return response2.WafStat{}, err + } var VisitCountOfYesterday int64 - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? ", - yesterdayDay).Select("sum(count) as vcnt").Row().Scan(&VisitCountOfYesterday) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day = ? ", + yesterdayDay).Select("coalesce(sum(count),0) as vcnt").Row().Scan(&VisitCountOfYesterday); err != nil { + return response2.WafStat{}, err + } + // Group("ip") + Count:GORM 在存在 GROUP BY 时用 RowsAffected 覆盖结果,语义即去重 IP 数 var NormalIpCountOfToday int64 - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).Where("day = ? and type = ? ", - currentDay, "放行").Group("ip").Count(&NormalIpCountOfToday) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).Where("day = ? and type = ? ", + currentDay, "放行").Group("ip").Count(&NormalIpCountOfToday).Error; err != nil { + return response2.WafStat{}, err + } var IllegalIpCountOfToday int64 - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).Where("day = ? and type = ? ", - currentDay, "阻止").Group("ip").Count(&IllegalIpCountOfToday) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}).Where("day = ? and type = ? ", + currentDay, "阻止").Group("ip").Count(&IllegalIpCountOfToday).Error; err != nil { + return response2.WafStat{}, err + } return response2.WafStat{ AttackCountOfToday: AttackCountOfToday, VisitCountOfToday: VisitCountOfToday, @@ -77,11 +93,15 @@ func (receiver *WafStatService) StatHomeSumDayRangeApi(req request.WafStatsDayRa } var AttackCountOfRange []model.StatsDayCount - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day between ? and ? and type = ? ", - req.StartDay, req.EndDay, "阻止").Select("day,sum(count) as count").Group("day").Scan(&AttackCountOfRange) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day between ? and ? and type = ? ", + req.StartDay, req.EndDay, "阻止").Select("day,sum(count) as count").Group("day").Scan(&AttackCountOfRange).Error; err != nil { + return response2.WafStatRange{}, err + } var NormalCountOfRange []model.StatsDayCount - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day between ? and ? and type = ? ", - req.StartDay, req.EndDay, "放行").Select("day,sum(count) as count").Group("day").Scan(&NormalCountOfRange) + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsDay{}).Where("day between ? and ? and type = ? ", + req.StartDay, req.EndDay, "放行").Select("day,sum(count) as count").Group("day").Scan(&NormalCountOfRange).Error; err != nil { + return response2.WafStatRange{}, err + } for i := 0; i < len(AttackCountOfRange); i++ { bean := AttackCountOfRange[i] @@ -105,11 +125,13 @@ func (receiver *WafStatService) StatHomeSumDayRangeApi(req request.WafStatsDayRa } func (receiver *WafStatService) StatHomeSumDayTopIPRangeApi(req request.WafStatsDayRangeReq) (response2.WafIPStats, error) { var AttackCountOfRange []model.StatsIPCount - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). Where("day between ? and ? and type = ? ", req.StartDay, req.EndDay, "阻止"). Select("ip,sum(count) as count").Group("ip").Order("sum(count) desc"). Limit(10). - Scan(&AttackCountOfRange) + Scan(&AttackCountOfRange).Error; err != nil { + return response2.WafIPStats{}, err + } var AttackCountOfRangeMore []model.StatsIPCountMore ipTagDB := global.GetIPTagDB() // 使用封装方法获取数据库连接 @@ -130,12 +152,14 @@ func (receiver *WafStatService) StatHomeSumDayTopIPRangeApi(req request.WafStats } var NormalCountOfRange []model.StatsIPCount - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). Where("day between ? and ? and type = ? ", req.StartDay, req.EndDay, "放行").Select("ip,sum(count) as count"). Group("ip").Order("sum(count) desc"). Limit(10). - Scan(&NormalCountOfRange) + Scan(&NormalCountOfRange).Error; err != nil { + return response2.WafIPStats{}, err + } var NormalCountOfRangeMore []model.StatsIPCountMore for i := range NormalCountOfRange { @@ -193,11 +217,15 @@ func (receiver *WafStatService) GetTodaySiteStatsByHostCodes(hostCodes []string) TrafficOut int64 } var siteRows []siteRow - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). + // 本方法无 error 返回值(列表页附带查询,查不到就不显示),但错误必须落日志, + // 否则方言不兼容这类问题只会表现为"数字全是 0"而无从排查 + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). Where("day = ? and host_code in ?", currentDay, hostCodes). Select("host_code, sum(total_count) as total_count, sum(attack_count) as attack_count, sum(traffic_in) as traffic_in, sum(traffic_out) as traffic_out"). Group("host_code"). - Scan(&siteRows) + Scan(&siteRows).Error; err != nil { + zlog.Error("查询今日站点统计失败", err) + } for _, row := range siteRows { statsMap[row.HostCode] = response2.HostTodayStat{ @@ -213,11 +241,13 @@ func (receiver *WafStatService) GetTodaySiteStatsByHostCodes(hostCodes []string) UvCount int64 } var uvRows []uvRow - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). Where("day = ? and host_code in ?", currentDay, hostCodes). Select("host_code, count(distinct ip) as uv_count"). Group("host_code"). - Scan(&uvRows) + Scan(&uvRows).Error; err != nil { + zlog.Error("查询今日站点UV失败", err) + } for _, row := range uvRows { stat := statsMap[row.HostCode] @@ -242,11 +272,17 @@ func (receiver *WafStatService) StatSiteOverviewApi(req request.WafStatsSiteOver TotalTimeSpent int64 } var rows []siteRow - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). + // host 是 host_code 的域名快照,语义上一对一,但 MySQL(ONLY_FULL_GROUP_BY)/PostgreSQL + // 不允许 SELECT 非聚合列,必须用 max() 取值;若改成 group by host_code,host, + // 站点在区间内改过域名时会被拆成多行,前端出现重复站点。 + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). Where("day between ? and ?", req.StartDay, req.EndDay). - Select("host_code, host, sum(total_count) as total_count, sum(attack_count) as attack_count, sum(normal_count) as normal_count, sum(traffic_in) as traffic_in, sum(traffic_out) as traffic_out, sum(total_time_spent) as total_time_spent"). + Select("host_code, max(host) as host, sum(total_count) as total_count, sum(attack_count) as attack_count, sum(normal_count) as normal_count, sum(traffic_in) as traffic_in, sum(traffic_out) as traffic_out, sum(total_time_spent) as total_time_spent"). Group("host_code"). - Scan(&rows) + Order("sum(total_count) desc"). + Scan(&rows).Error; err != nil { + return response2.WafSiteOverview{}, err + } // 2) 根据 host_code 回填站点备注 type hostRemarkRow struct { @@ -263,10 +299,12 @@ func (receiver *WafStatService) StatSiteOverviewApi(req request.WafStatsSiteOver hostRemarkMap := make(map[string]string) if len(hostCodes) > 0 { var hostRows []hostRemarkRow - global.GWAF_LOCAL_DB.Model(&model.Hosts{}). + if err := global.GWAF_LOCAL_DB.Model(&model.Hosts{}). Where("code in ?", hostCodes). Select("code, remarks"). - Scan(&hostRows) + Scan(&hostRows).Error; err != nil { + return response2.WafSiteOverview{}, err + } for _, r := range hostRows { hostRemarkMap[r.Code] = r.Remarks } @@ -278,11 +316,13 @@ func (receiver *WafStatService) StatSiteOverviewApi(req request.WafStatsSiteOver UvCount int64 } var uvRows []uvRow - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). Where("day between ? and ?", req.StartDay, req.EndDay). Select("host_code, count(distinct ip) as uv_count"). Group("host_code"). - Scan(&uvRows) + Scan(&uvRows).Error; err != nil { + return response2.WafSiteOverview{}, err + } uvMap := make(map[string]int64) for _, r := range uvRows { uvMap[r.HostCode] = r.UvCount @@ -337,9 +377,11 @@ func (receiver *WafStatService) StatSiteDetailApi(req request.WafStatsSiteDetail // 最近1小时,按小时查(最多2个点:上一整点+当前整点) startTs := (now.Add(-1*time.Hour).Unix() / 3600) * 3600 var pts []model.StatsSiteHour - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteHour{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteHour{}). Where("host_code = ? and hour_time >= ?", req.HostCode, startTs). - Order("hour_time asc").Scan(&pts) + Order("hour_time asc").Scan(&pts).Error; err != nil { + return response2.WafSiteDetail{}, err + } ptMap := make(map[int64]model.StatsSiteHour) for _, p := range pts { @@ -365,9 +407,11 @@ func (receiver *WafStatService) StatSiteDetailApi(req request.WafStatsSiteDetail // 最近24小时,按小时查(最多24个点:过去23小时+当前整点) startTs := (now.Add(-23*time.Hour).Unix() / 3600) * 3600 var pts []model.StatsSiteHour - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteHour{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteHour{}). Where("host_code = ? and hour_time >= ?", req.HostCode, startTs). - Order("hour_time asc").Scan(&pts) + Order("hour_time asc").Scan(&pts).Error; err != nil { + return response2.WafSiteDetail{}, err + } ptMap := make(map[int64]model.StatsSiteHour) for _, p := range pts { @@ -400,20 +444,24 @@ func (receiver *WafStatService) StatSiteDetailApi(req request.WafStatsSiteDetail TotalTimeSpent int64 } var dayRows []dayRow - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). Where("host_code = ? and day between ? and ?", req.HostCode, startDay, endDay). Select("day, sum(total_count) as total_count, sum(attack_count) as attack_count, sum(normal_count) as normal_count, sum(total_time_spent) as total_time_spent"). - Group("day").Order("day asc").Scan(&dayRows) + Group("day").Order("day asc").Scan(&dayRows).Error; err != nil { + return response2.WafSiteDetail{}, err + } // UV type uvDay struct { Day int UvCount int64 } var uvDays []uvDay - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). Where("host_code = ? and day between ? and ?", req.HostCode, startDay, endDay). Select("day, count(distinct ip) as uv_count"). - Group("day").Order("day asc").Scan(&uvDays) + Group("day").Order("day asc").Scan(&uvDays).Error; err != nil { + return response2.WafSiteDetail{}, err + } uvDayMap := make(map[int]int64) for _, u := range uvDays { uvDayMap[u.Day] = u.UvCount @@ -438,19 +486,23 @@ func (receiver *WafStatService) StatSiteDetailApi(req request.WafStatsSiteDetail TotalTimeSpent int64 } var dayRows []dayRow - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteDay{}). Where("host_code = ? and day between ? and ?", req.HostCode, startDay, endDay). Select("day, sum(total_count) as total_count, sum(attack_count) as attack_count, sum(normal_count) as normal_count, sum(total_time_spent) as total_time_spent"). - Group("day").Order("day asc").Scan(&dayRows) + Group("day").Order("day asc").Scan(&dayRows).Error; err != nil { + return response2.WafSiteDetail{}, err + } type uvDay struct { Day int UvCount int64 } var uvDays []uvDay - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsIPDay{}). Where("host_code = ? and day between ? and ?", req.HostCode, startDay, endDay). Select("day, count(distinct ip) as uv_count"). - Group("day").Order("day asc").Scan(&uvDays) + Group("day").Order("day asc").Scan(&uvDays).Error; err != nil { + return response2.WafSiteDetail{}, err + } uvDayMap := make(map[int]int64) for _, u := range uvDays { uvDayMap[u.Day] = u.UvCount @@ -468,9 +520,11 @@ func (receiver *WafStatService) StatSiteDetailApi(req request.WafStatsSiteDetail // 默认 24h startTs := (now.Add(-23*time.Hour).Unix() / 3600) * 3600 var pts []model.StatsSiteHour - global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteHour{}). + if err := global.GWAF_LOCAL_STATS_DB.Model(&model.StatsSiteHour{}). Where("host_code = ? and hour_time >= ?", req.HostCode, startTs). - Order("hour_time asc").Scan(&pts) + Order("hour_time asc").Scan(&pts).Error; err != nil { + return response2.WafSiteDetail{}, err + } ptMap := make(map[int64]model.StatsSiteHour) for _, p := range pts {