diff --git a/.gitignore b/.gitignore index 81ccd7b..d288bda 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ /release/ /vue/dist/ /data/local_log.db -/data/local_stats.db \ No newline at end of file +/data/local_stats.db +/data/local.db \ No newline at end of file diff --git a/build-releases-win-upx.bat b/build-releases-win-upx.bat index f400793..15fc774 100644 --- a/build-releases-win-upx.bat +++ b/build-releases-win-upx.bat @@ -2,4 +2,4 @@ SET CGO_ENABLED=1 SET GOOS=windows SET GOARCH=amd64 SET GIN_MODE=release -go build -ldflags="-X SamWaf/global.GWAF_RELEASE=true -X SamWaf/global.GWAF_RELEASE_VERSION_NAME=20231109 -X SamWaf/global.GWAF_RELEASE_VERSION=v1.0.121 -s -w" -o %cd%/release/SamWaf64.exe main.go && %cd%/upx/win64/upx -9 %cd%/release/SamWaf64.exe \ No newline at end of file +go build -ldflags="-X SamWaf/global.GWAF_RELEASE=true -X SamWaf/global.GWAF_RELEASE_VERSION_NAME=20231109 -X SamWaf/global.GWAF_RELEASE_VERSION=v1.0.124 -s -w" -o %cd%/release/SamWaf64.exe main.go && %cd%/upx/win64/upx -9 %cd%/release/SamWaf64.exe \ No newline at end of file diff --git a/build_docker_release_linux.bat b/build_docker_release_linux.bat index 89104d9..20c9d7f 100644 --- a/build_docker_release_linux.bat +++ b/build_docker_release_linux.bat @@ -1 +1 @@ -docker run --rm -v "$PWD":/media/sf_SamWaf -w /media/sf_SamWaf -e CGO_ENABLED=1 -e GOPROXY=https://goproxy.cn,direct golang:1.19 go build -v -ldflags="-X SamWaf/global.GWAF_RELEASE=true -X SamWaf/global.GWAF_RELEASE_VERSION_NAME=20231109 -X SamWaf/global.GWAF_RELEASE_VERSION=v1.0.121 -s -w -extldflags "-static"" -o /media/sf_SamWaf/release/SamWafLinux64 main.go \ No newline at end of file +docker run --rm -v "$PWD":/media/sf_SamWaf -w /media/sf_SamWaf -e CGO_ENABLED=1 -e GOPROXY=https://goproxy.cn,direct golang:1.19 go build -v -ldflags="-X SamWaf/global.GWAF_RELEASE=true -X SamWaf/global.GWAF_RELEASE_VERSION_NAME=20231109 -X SamWaf/global.GWAF_RELEASE_VERSION=v1.0.124 -s -w -extldflags "-static"" -o /media/sf_SamWaf/release/SamWafLinux64 main.go \ No newline at end of file diff --git a/global/global.go b/global/global.go index 1ad1863..d465414 100644 --- a/global/global.go +++ b/global/global.go @@ -29,19 +29,20 @@ var ( GWAF_GLOBAL_HOST_NAME string = "全局网站:0" //全局网站 - GWAF_LOCAL_DB *gorm.DB //通用本地数据库，尊重用户隐私 - GWAF_LOCAL_LOG_DB *gorm.DB //通用本地数据库存日志数据，尊重用户隐私 - GWAF_LOCAL_STATS_DB *gorm.DB //通用本地数据库存放统计数据，尊重用户隐私 - GWAF_REMOTE_DB *gorm.DB //仅当用户使用云数据库 - GWAF_LOCAL_SERVER_PORT int = 26666 // 本地local端口 - GWAF_USER_CODE string // 当前识别号 - GWAF_CUSTOM_SERVER_NAME string // 当前服务器自定义名称 - GWAF_TENANT_ID string // 当前租户ID - GWAF_RELEASE string = "false" // 当前是否为发行版 - GWAF_RELEASE_VERSION_NAME string = "1.0" // 发行版的版本号名称 - GWAF_RELEASE_VERSION string = "v1.0.0" // 发行版的版本号 - GWAF_LAST_UPDATE_TIME time.Time // 上次时间 - GWAF_DLP dlpheader.EngineAPI // 脱敏引擎 + GWAF_LOCAL_DB *gorm.DB //通用本地数据库，尊重用户隐私 + GWAF_LOCAL_LOG_DB *gorm.DB //通用本地数据库存日志数据，尊重用户隐私 + GWAF_LOCAL_STATS_DB *gorm.DB //通用本地数据库存放统计数据，尊重用户隐私 + GWAF_REMOTE_DB *gorm.DB //仅当用户使用云数据库 + GWAF_LOCAL_SERVER_PORT int = 26666 // 本地local端口 + GWAF_USER_CODE string // 当前识别号 + GWAF_CUSTOM_SERVER_NAME string // 当前服务器自定义名称 + GWAF_TENANT_ID string // 当前租户ID + GWAF_RELEASE string = "false" // 当前是否为发行版 + GWAF_RELEASE_VERSION_NAME string = "1.0" // 发行版的版本号名称 + GWAF_RELEASE_VERSION string = "v1.0.0" // 发行版的版本号 + GWAF_LAST_UPDATE_TIME time.Time // 上次时间 + + GWAF_DLP dlpheader.EngineAPI // 脱敏引擎 /**链聚合**/ GWAF_CHAN_HOST = make(chan model.Hosts, 10) //主机链 @@ -75,6 +76,9 @@ var ( GUPDATE_VERSION_URL string = "https://update.samwaf.com/" // GWAF_SNOWFLAKE_GEN *wafsnowflake.Snowflake //雪花算法 + + //任务开关信息 + GWAF_SWITCH_TASK_COUNTER bool ) func GetCurrentVersionInt() int { diff --git a/main.go b/main.go index 12c36bf..0927d1e 100644 --- a/main.go +++ b/main.go @@ -68,8 +68,6 @@ func (m *wafSystenService) run() { zlog.Info(rversion) - global.GWAF_LAST_UPDATE_TIME = time.Now() - syscall.Setenv("ZONEINFO", utils.GetCurrentDir()+"//data//zoneinfo") //守护程序开始 @@ -134,10 +132,14 @@ func (m *wafSystenService) run() { timezone, _ := time.LoadLocation("Asia/Shanghai") s := gocron.NewScheduler(timezone) + global.GWAF_LAST_UPDATE_TIME = time.Now() // 每10秒执行一次 s.Every(10).Seconds().Do(func() { - zlog.Debug("i am alive") - go waftask.TaskCounter() + if global.GWAF_SWITCH_TASK_COUNTER == false { + go waftask.TaskCounter() + } else { + zlog.Debug("统计还没完成，调度任务PASS") + } }) // 获取延迟信息 s.Every(1).Minutes().Do(func() { diff --git a/waftask/localtaskcounter.go b/waftask/localtaskcounter.go index 7125750..a21beb1 100644 --- a/waftask/localtaskcounter.go +++ b/waftask/localtaskcounter.go @@ -57,7 +57,7 @@ type CountCityResult struct { */ func TaskCounter() { - + global.GWAF_SWITCH_TASK_COUNTER = true /*dateTime, err := time.Parse("2006-01-02", "2023-01-01") if err != nil { fmt.Println("解析日期出错:", err) @@ -65,13 +65,20 @@ func TaskCounter() { } currenyDayBak := dateTime*/ + /** + 1.首次是当前日期，查询当前时间以后的所有数据，备份当前日期 + 2.查询使用备份日期，倒退10秒，查询这个时候所有的数据 + 3. + + */ + currenyDayBak := time.Now() - currentDayBak3Second := currenyDayBak.Add(-3 * time.Second) - currenyDayMillisecondsBak := currentDayBak3Second.UnixNano() / 1e6 //倒查3秒 + currenyDayMillisecondsBak := (global.GWAF_LAST_UPDATE_TIME.Add(-10 * time.Second).UnixNano()) / 1e6 //倒退10秒 + //一、 主机聚合统计 { var resultHosts []CountHostResult - global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host FROM \"web_logs\" where task_flag = ? and unix_add_time <= ? GROUP BY host_code, user_code,action,tenant_id,day,host", + global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host FROM \"web_logs\" where task_flag = ? and unix_add_time > ? GROUP BY host_code, user_code,action,tenant_id,day,host", 1, currenyDayMillisecondsBak).Scan(&resultHosts) /**** 1.如果不存在则创建 @@ -114,7 +121,7 @@ func TaskCounter() { //二、 IP聚合统计 { var resultIP []CountIPResult - global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,src_ip as ip FROM \"web_logs\" where task_flag = ? and unix_add_time <= ? GROUP BY host_code, user_code,action,tenant_id,day,host,ip", + global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,src_ip as ip FROM \"web_logs\" where task_flag = ? and unix_add_time > ? GROUP BY host_code, user_code,action,tenant_id,day,host,ip", 1, currenyDayMillisecondsBak).Scan(&resultIP) /**** 1.如果不存在则创建 @@ -160,7 +167,7 @@ func TaskCounter() { //三、 城市信息聚合统计 { var resultCitys []CountCityResult - global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,country,province,city FROM \"web_logs\" where task_flag = ? and unix_add_time <= ? GROUP BY host_code, user_code,action,tenant_id,day,host,country,province,city", + global.GWAF_LOCAL_LOG_DB.Raw("SELECT host_code, user_code,tenant_id ,action,count(req_uuid) as count,day,host,country,province,city FROM \"web_logs\" where task_flag = ? and unix_add_time > ? GROUP BY host_code, user_code,action,tenant_id,day,host,country,province,city", 1, currenyDayMillisecondsBak).Scan(&resultCitys) /**** 1.如果不存在则创建 @@ -205,6 +212,7 @@ func TaskCounter() { } } global.GWAF_LAST_UPDATE_TIME = currenyDayBak + global.GWAF_SWITCH_TASK_COUNTER = false } func TaskWechatAccessToken() {