From 2707b5350e00a42c2e1a6b9fbfdfac88638f871e Mon Sep 17 00:00:00 2001 From: samwaf Date: Tue, 27 Oct 2026 14:42:10 +0800 Subject: [PATCH] fix: support integer widths in website Excel imports --- api/waf_common.go | 258 ++--------- api/waf_common_test.go | 437 ++++++++++++++++-- service/waf_service/waf_common_import.go | 215 +++++++++ service/waf_service/waf_common_import_test.go | 185 ++++++++ wafupgradenotice/upgrade_notes.yaml | 16 + 5 files changed, 860 insertions(+), 251 deletions(-) create mode 100644 service/waf_service/waf_common_import.go create mode 100644 service/waf_service/waf_common_import_test.go diff --git a/api/waf_common.go b/api/waf_common.go index 9904c7e..2bf09d1 100644 --- a/api/waf_common.go +++ b/api/waf_common.go @@ -1,14 +1,10 @@ package api import ( - "SamWaf/common/uuid" - "SamWaf/customtype" - "SamWaf/global" "SamWaf/model" - "SamWaf/model/baseorm" "SamWaf/model/common/response" "SamWaf/model/request" - "errors" + "SamWaf/service/waf_service" "fmt" "log" "net/http" @@ -23,11 +19,6 @@ import ( type WafCommonApi struct { } -type ReturnImportData struct { - SuccessInt int - FailInt int - Msg string -} // 导出excel func (w *WafCommonApi) ExportExcelApi(c *gin.Context) { @@ -67,7 +58,19 @@ func (w *WafCommonApi) ExportExcelApi(c *gin.Context) { if field.Name == "BaseOrm" { f.SetCellValue(sheetName, fmt.Sprintf("%s%d", w.GetColumnName(j), rowNum), "") } else { - colValue := rowValue.Field(j).Interface() + value := rowValue.Field(j) + colValue := value.Interface() + // Excel 数值仅保留 15 位有效数字，大整数使用文本单元格。 + switch value.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + if n := value.Int(); n > 999999999999999 || n < -999999999999999 { + colValue = strconv.FormatInt(n, 10) + } + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + if n := value.Uint(); n > 999999999999999 { + colValue = strconv.FormatUint(n, 10) + } + } f.SetCellValue(sheetName, fmt.Sprintf("%s%d", w.GetColumnName(j), rowNum), colValue) } } @@ -93,45 +96,39 @@ func (w *WafCommonApi) ExportExcelApi(c *gin.Context) { } } func (w *WafCommonApi) ImportExcelApi(c *gin.Context) { - file, err := c.FormFile("file") // "excelFile" 对应前端上传的文件字段名 + importTable, ok := c.GetPostForm("import_table") + if !ok || importTable != "hosts" { + c.JSON(http.StatusBadRequest, gin.H{"error": "import_table 仅支持 hosts"}) + return + } + importCodeStrategy, ok := c.GetPostForm("import_code_strategy") + if !ok || (importCodeStrategy != "0" && importCodeStrategy != "1") { + c.JSON(http.StatusBadRequest, gin.H{"error": "import_code_strategy 仅支持 0 或 1"}) + return + } + file, err := c.FormFile("file") if err != nil { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) + c.JSON(http.StatusBadRequest, gin.H{"error": "读取上传文件失败"}) return } - // 生成文件名 - fileName := time.Now().Format("20060102150405") + "import.xlsx" - - if err := c.SaveUploadedFile(file, fileName); // 保存文件到指定路径 - err != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) - return - } - importTable, succResult := c.GetPostForm("import_table") - if !succResult { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - } - importCodeStrategy, succResult := c.GetPostForm("import_code_strategy") - if !succResult { - c.JSON(http.StatusBadRequest, gin.H{"error": err.Error()}) - } - // 创建 Excel 文件 - f, err := excelize.OpenFile(fileName) + reader, err := file.Open() if err != nil { - fmt.Println(err) + c.JSON(http.StatusBadRequest, gin.H{"error": "打开上传文件失败"}) return } - - // Get all the rows in the Sheet1. - rows, err := f.GetRows("Sheet1") + defer reader.Close() + f, err := excelize.OpenReader(reader) if err != nil { - fmt.Println(err) + c.JSON(http.StatusBadRequest, gin.H{"error": "解析 Excel 文件失败"}) return } - ret := saveDataToDatabase(importTable, rows, importCodeStrategy) - // 删除临时文件 - if err := os.Remove(fileName); err != nil { - log.Println("无法删除临时文件：", err) + defer f.Close() + rows, err := f.GetRows("Sheet1", excelize.Options{RawCellValue: true}) + if err != nil { + c.JSON(http.StatusBadRequest, gin.H{"error": "读取 Sheet1 失败"}) + return } + ret := waf_service.ImportHostsExcel(rows, importCodeStrategy) response.OkWithDetailed(ret, "上传完成", c) } @@ -160,187 +157,6 @@ func (w *WafCommonApi) GetColumnName(colIdx int) string { return colName } -// 通用数据插入函数 -func saveDataToDatabase(tableName string, rows [][]string, importCodeStrategy string) ReturnImportData { - successInt := 0 - failInt := 0 - msg := "" - - processImportData(&model.Hosts{}, tableName, rows, &successInt, &failInt, &msg, importCodeStrategy) - - return ReturnImportData{ - SuccessInt: successInt, - FailInt: failInt, - Msg: msg, - } -} - -// processImportData 是通用的导入数据函数 -func processImportData(dataType interface{}, tableName string, rows [][]string, successInt, failInt *int, msg *string, importCodeStrategy string) { - var header []string - needJumpFristCol := false - rowNumber := 0 - var dataMap map[string]string - - // 获取结构体类型和字段信息 - dataValue := reflect.ValueOf(dataType).Elem() - dataTypeFields := dataValue.Type() - - // 获取表头 - for _, row := range rows { - for _, colCell := range row { - if colCell == " - " { - needJumpFristCol = true - continue - } - header = append(header, colCell) - } - break - } - - isBadData := false - // 处理数据 获取数据并插入数据库 - for _, row := range rows { - isBadData = false - if rowNumber == 0 && needJumpFristCol { - rowNumber++ - continue - } - - // 建立一个 map 来存储数据 - dataMap = make(map[string]string) - colNumber := 0 - for _, colCell := range row { - if colNumber == 0 && needJumpFristCol { - colNumber++ - continue - } - headerNumber := colNumber - if needJumpFristCol { - headerNumber = headerNumber - 1 - } - dataMap[header[headerNumber]] = colCell - colNumber++ - } - - // 动态创建结构体实例，并映射数据 - newInstance := reflect.New(dataValue.Type()).Elem() - - for fieldIdx := 0; fieldIdx < dataTypeFields.NumField(); fieldIdx++ { - field := dataTypeFields.Field(fieldIdx) - fieldName := field.Name - jsonTag := field.Tag.Get("json") - - // 如果 dataMap 中有匹配的字段 - if val, exists := dataMap[jsonTag]; exists { - //排除一些特定数据 - if tableName == "hosts" && fieldName == "Host" && (val == "全局网站" || val == "") { - isBadData = true - break - } - //检查数据是否已经存在 - if tableName == "hosts" && fieldName == "Host" { - if importCodeStrategy == "1" { - errMsg, err := checkHostCodeData(dataMap["code"]) - if err != nil { - *msg += fmt.Sprintf("行 %d, 检测数据合法性时候 出错: %v |", rowNumber, errMsg) - *failInt++ - isBadData = true - break - } - } - errMsg, err := checkHostPortData(dataMap["host"], dataMap["port"]) - if err != nil { - *msg += fmt.Sprintf("行 %d, 检测数据合法性时候 出错: %v |", rowNumber, errMsg) - *failInt++ - isBadData = true - break - } - } - // 将字段值设置到结构体中 - fieldVal := newInstance.Field(fieldIdx) - // 转换并设置字段的值 - switch fieldVal.Kind() { - case reflect.String: - if tableName == "hosts" { - if importCodeStrategy == "0" && fieldName == "Code" { - fieldVal.SetString(uuid.GenUUID()) - } else { - fieldVal.SetString(val) - } - } else { - fieldVal.SetString(val) - } - case reflect.Int: - intVal, err := strconv.Atoi(val) - if err != nil { - *msg += fmt.Sprintf("行 %d, 字段 %s 转换为 int 错误: %v |", rowNumber, fieldName, err) - *failInt++ - isBadData = true - break - } - fieldVal.SetInt(int64(intVal)) - default: - *msg += fmt.Sprintf("不支持的字段类型: %s |", fieldVal.Kind()) - *failInt++ - isBadData = true - break - } - } else if fieldName == "BaseOrm" { - fieldVal := newInstance.Field(fieldIdx) - // 给 BaseOrm 赋值 - baseOrm := baseorm.BaseOrm{ - Id: uuid.GenUUID(), // 新生成的 ID - USER_CODE: global.GWAF_USER_CODE, - Tenant_ID: global.GWAF_TENANT_ID, - CREATE_TIME: customtype.JsonTime(time.Now()), - UPDATE_TIME: customtype.JsonTime(time.Now()), - } - // 将 BaseOrm 设置到结构体字段 - fieldVal.Set(reflect.ValueOf(baseOrm)) - } else { - *msg += fmt.Sprintf("行 %d, 缺少字段 %s 数据 |", rowNumber, fieldName) - *failInt++ - isBadData = true - break - } - } - if isBadData == false { - if err := global.GWAF_LOCAL_DB.Create(newInstance.Interface()); err != nil { - errGorm := err.Error - if errGorm != nil { - *msg += fmt.Sprintf("行 %d 插入失败: %v |", rowNumber, err.Error) - *failInt++ - } else { - *successInt++ - } - } - } - - rowNumber++ - } -} - -// checkHostData 检查host信息是否合法 -func checkHostCodeData(code string) (string, error) { - // 唯一性校验：检查 `Code` 是否已存在 - if wafHostService.GetDetailByCodeApi(code).Code != "" { - errorMsg := "Code 数据已存在不进行插入" - // 数据已存在，不插入 - return errorMsg, errors.New(errorMsg) - } - return "数据正常", nil -} -func checkHostPortData(host string, port string) (string, error) { - //唯一性校验：检查 `Host` 和 `Port` 的组合是否已存在 - if err := wafHostService.CheckIsExist(host, port); err == nil { - errorMsg := "Host+Port 数据已存在不进行插入" - // 数据已存在，不插入 - return errorMsg, errors.New(errorMsg) - } - return "数据正常", nil -} - // HeartbeatApi 心跳数据 func (w *WafCommonApi) HeartbeatApi(c *gin.Context) { response.Ok(c) diff --git a/api/waf_common_test.go b/api/waf_common_test.go index 5ef8fe4..7bb4d97 100644 --- a/api/waf_common_test.go +++ b/api/waf_common_test.go @@ -2,49 +2,426 @@ package api import ( "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "SamWaf/service/waf_service" + "bytes" + "encoding/json" "fmt" - "github.com/gin-gonic/gin" + "mime/multipart" "net/http" "net/http/httptest" - "net/url" + "reflect" + "strconv" + "strings" "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/xuri/excelize/v2" ) -// 测试用例 ExportExcelApi -func TestExportExcelApi(t *testing.T) { - // 创建一个基于 Gin 的引擎 - r := gin.Default() - - global.GWAF_RELEASE_VERSION = "111" - r.GET("/api/v1/export", new(WafCommonApi).ExportExcelApi) - // 创建一个模拟的 HTTP 请求 WafCommonReq 对象 - // 创建一个模拟的 HTTP 请求 - queryParams := url.Values{} - queryParams.Set("table_name", "hosts") - - req, err := http.NewRequest(http.MethodGet, "/api/v1/export?"+queryParams.Encode(), nil) - if err != nil { - t.Fatalf("无法创建请求：%v", err) +// 每次从当前 Hosts 生成样本；新增类型必须明确提供非零测试值。 +func excelHostFixture(t *testing.T) model.Hosts { + t.Helper() + host := model.Hosts{} + v := reflect.ValueOf(&host).Elem() + for i := 0; i < v.NumField(); i++ { + field := v.Type().Field(i) + if field.Name == "BaseOrm" { + continue + } + if tag := field.Tag.Get("json"); tag == "" || tag == "-" || strings.Contains(tag, ",") { + t.Fatalf("Hosts.%s needs an explicit Excel field contract: %q", field.Name, tag) + } + switch value := v.Field(i); value.Kind() { + case reflect.String: + value.SetString("测试-" + field.Name) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32: + value.SetInt(1) + case reflect.Int64: + value.SetInt(9223372036854775807) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + value.SetUint(1) + case reflect.Bool: + value.SetBool(true) + case reflect.Float32, reflect.Float64: + value.SetFloat(1.25) + default: + t.Fatalf("add Excel round-trip coverage for Hosts.%s (%s)", field.Name, field.Type) + } } + host.BaseOrm = baseorm.BaseOrm{Id: "export-source", USER_CODE: global.GWAF_USER_CODE, Tenant_ID: global.GWAF_TENANT_ID} + host.Host, host.Port, host.Code = "import.example", 9112, "export-source-code" + host.GLOBAL_HOST = 0 + return host +} - // 创建一个响应记录器 +func excelHostRows(host model.Hosts) [][]string { + v := reflect.ValueOf(host) + header, row := make([]string, v.NumField()), make([]string, v.NumField()) + for i := range header { + if v.Type().Field(i).Name == "BaseOrm" { + header[i] = " - " + continue + } + header[i] = v.Type().Field(i).Tag.Get("json") + row[i] = fmt.Sprint(v.Field(i).Interface()) + } + return [][]string{header, row} +} + +func setExcelHostCell(t *testing.T, rows [][]string, field, value string) { + t.Helper() + for i, header := range rows[0] { + if header == field { + rows[1][i] = value + return + } + } + t.Fatalf("missing fixture column %s", field) +} + +func assertExcelImportedHost(t *testing.T, want model.Hosts, strategy string) { + t.Helper() + var hosts []model.Hosts + if err := global.GWAF_LOCAL_DB.Find(&hosts).Error; err != nil { + t.Fatal(err) + } + if len(hosts) != 1 { + t.Fatalf("want exactly one imported host, got %d", len(hosts)) + } + got := hosts[0] + if got.Id == "" || got.Id == want.Id || got.USER_CODE != global.GWAF_USER_CODE || got.Tenant_ID != global.GWAF_TENANT_ID || time.Time(got.CREATE_TIME).IsZero() || time.Time(got.UPDATE_TIME).IsZero() { + t.Fatalf("imported BaseOrm metadata was not generated: %+v", got.BaseOrm) + } + gv, wv := reflect.ValueOf(got), reflect.ValueOf(want) + for i := 0; i < wv.NumField(); i++ { + name := wv.Type().Field(i).Name + if name == "BaseOrm" { + continue + } + if name == "Code" && strategy == "0" { + if got.Code == "" || got.Code == want.Code { + t.Fatal("strategy 0 must generate a new host code") + } + continue + } + if !reflect.DeepEqual(gv.Field(i).Interface(), wv.Field(i).Interface()) { + t.Errorf("Hosts.%s changed during import: want %v, got %v", name, wv.Field(i), gv.Field(i)) + } + } +} + +func exportExcelHost(t *testing.T, host model.Hosts) []byte { + t.Helper() + if err := global.GWAF_LOCAL_DB.Create(&host).Error; err != nil { + t.Fatal(err) + } + r := gin.New() + r.GET("/export", new(WafCommonApi).ExportExcelApi) rec := httptest.NewRecorder() - - // 将模拟的请求发送到测试的 API 路由 - r.ServeHTTP(rec, req) - - // 检查响应状态码是否为 200 + r.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/export?table_name=hosts", nil)) if rec.Code != http.StatusOK { - t.Errorf("期望的状态码：%d，实际状态码：%d", http.StatusOK, rec.Code) + t.Fatalf("export HTTP %d: %s", rec.Code, rec.Body.String()) + } + f, err := excelize.OpenReader(bytes.NewReader(rec.Body.Bytes())) + if err != nil { + t.Fatalf("response is not an Excel workbook: %v", err) + } + defer f.Close() + rows, err := f.GetRows("Sheet1") + if err != nil { + t.Fatal(err) + } + if want := excelHostRows(host); !reflect.DeepEqual(rows, want) { + t.Fatalf("exported columns or cell values differ from current Hosts fields\nwant: %v\ngot: %v", want, rows) + } + for i, field := range rows[0] { + if field == "emergency_until" && host.EmergencyUntil > 999999999999999 { + cell, err := excelize.CoordinatesToCellName(i+1, 2) + if err != nil { + t.Fatal(err) + } + kind, err := f.GetCellType("Sheet1", cell) + if err != nil || (kind != excelize.CellTypeSharedString && kind != excelize.CellTypeInlineString) { + t.Fatalf("large integer must be exported as text: %v %v", kind, err) + } + } + } + return append([]byte(nil), rec.Body.Bytes()...) +} + +func importExcelRequest(t *testing.T, workbook []byte, table, strategy string, includeFile bool) *httptest.ResponseRecorder { + t.Helper() + var body bytes.Buffer + writer := multipart.NewWriter(&body) + if includeFile { + part, err := writer.CreateFormFile("file", "hosts.xlsx") + if err != nil { + t.Fatal(err) + } + if _, err := part.Write(workbook); err != nil { + t.Fatal(err) + } + } + for key, value := range map[string]string{"import_table": table, "import_code_strategy": strategy} { + if value != "" { + if err := writer.WriteField(key, value); err != nil { + t.Fatal(err) + } + } + } + if err := writer.Close(); err != nil { + t.Fatal(err) + } + r := gin.New() + r.POST("/import", func(c *gin.Context) { + c.Set("is_openapi", true) + new(WafCommonApi).ImportExcelApi(c) + }) + req := httptest.NewRequest(http.MethodPost, "/import", &body) + req.Header.Set("Content-Type", writer.FormDataContentType()) + rec := httptest.NewRecorder() + r.ServeHTTP(rec, req) + return rec +} + +func TestExportExcelApi(t *testing.T) { + setupNotifyTestEnv(t, &model.Hosts{}) + t.Chdir(t.TempDir()) + exportExcelHost(t, excelHostFixture(t)) +} + +func TestHostsExcelRoundTripAllFields(t *testing.T) { + for _, strategy := range []string{"0", "1"} { + t.Run("strategy_"+strategy, func(t *testing.T) { + setupNotifyTestEnv(t, &model.Hosts{}) + t.Chdir(t.TempDir()) + host := excelHostFixture(t) + workbook := exportExcelHost(t, host) + if err := global.GWAF_LOCAL_DB.Where("id = ?", host.Id).Delete(&model.Hosts{}).Error; err != nil { + t.Fatal(err) + } + rec := importExcelRequest(t, workbook, "hosts", strategy, true) + var response struct { + Code int + Data waf_service.ReturnImportData + } + if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatalf("decode import result: %v: %s", err, rec.Body.String()) + } + if rec.Code != http.StatusOK || response.Code != 0 || response.Data.SuccessInt != 1 || response.Data.FailInt != 0 || response.Data.Msg != "" { + t.Fatalf("round-trip failed: HTTP %d %s", rec.Code, rec.Body.String()) + } + assertExcelImportedHost(t, host, strategy) + }) + } +} + +func TestHostsExcelImportRawNumericCell(t *testing.T) { + setupNotifyTestEnv(t, &model.Hosts{}) + t.Chdir(t.TempDir()) + host := excelHostFixture(t) + workbook := exportExcelHost(t, host) + f, err := excelize.OpenReader(bytes.NewReader(workbook)) + if err != nil { + t.Fatal(err) + } + defer f.Close() + rows, err := f.GetRows("Sheet1") + if err != nil { + t.Fatal(err) + } + for i, field := range rows[0] { + if field == "emergency_until" { + cell, err := excelize.CoordinatesToCellName(i+1, 2) + if err != nil { + t.Fatal(err) + } + if err := f.SetCellValue("Sheet1", cell, host.EmergencyUntil); err != nil { + t.Fatal(err) + } + } + } + buffer, err := f.WriteToBuffer() + if err != nil { + t.Fatal(err) + } + if err := global.GWAF_LOCAL_DB.Where("id = ?", host.Id).Delete(&model.Hosts{}).Error; err != nil { + t.Fatal(err) + } + rec := importExcelRequest(t, buffer.Bytes(), "hosts", "1", true) + var response struct { + Code int + Data waf_service.ReturnImportData + } + if err := json.Unmarshal(rec.Body.Bytes(), &response); err != nil { + t.Fatal(err) + } + if rec.Code != http.StatusOK || response.Code != 0 || response.Data.SuccessInt != 1 || response.Data.FailInt != 0 || response.Data.Msg != "" { + t.Fatalf("raw numeric cell import failed: HTTP %d %s", rec.Code, rec.Body.String()) + } + assertExcelImportedHost(t, host, "1") +} + +func TestHostsExcelImportRows(t *testing.T) { + cases := []struct { + name string + edit func(*testing.T, [][]string) [][]string + fail bool + message string + }{ + {"int64_zero", func(t *testing.T, r [][]string) [][]string { setExcelHostCell(t, r, "emergency_until", "0"); return r }, false, ""}, + {"int64_max", func(t *testing.T, r [][]string) [][]string { + setExcelHostCell(t, r, "emergency_until", "9223372036854775807") + return r + }, false, ""}, + {"int64_invalid", func(t *testing.T, r [][]string) [][]string { + setExcelHostCell(t, r, "emergency_until", "bad") + return r + }, true, "EmergencyUntil"}, + {"int64_overflow", func(t *testing.T, r [][]string) [][]string { + setExcelHostCell(t, r, "emergency_until", "9223372036854775808") + return r + }, true, "EmergencyUntil"}, + {"two_invalid_cells_one_failure", func(t *testing.T, r [][]string) [][]string { + setExcelHostCell(t, r, "ssl", "bad") + setExcelHostCell(t, r, "remote_port", "bad") + return r + }, true, "Ssl"}, + {"without_base_column", func(t *testing.T, r [][]string) [][]string { return [][]string{r[0][1:], r[1][1:]} }, false, ""}, + {"missing_model_column", func(t *testing.T, r [][]string) [][]string { + for i, field := range r[0] { + if field == "emergency_until" { + return [][]string{append(r[0][:i], r[0][i+1:]...), append(r[1][:i], r[1][i+1:]...)} + } + } + t.Fatal("emergency_until column missing") + return nil + }, true, "EmergencyUntil"}, + {"extra_nonempty_cell", func(t *testing.T, r [][]string) [][]string { r[1] = append(r[1], "extra"); return r }, true, ""}, + {"duplicate_header", func(t *testing.T, r [][]string) [][]string { + r[0] = append(r[0], "emergency_until") + r[1] = append(r[1], "0") + return r + }, true, "EmergencyUntil"}, + {"reordered_columns", func(t *testing.T, r [][]string) [][]string { + for i, j := 0, len(r[0])-1; i < j; i, j = i+1, j-1 { + r[0][i], r[0][j] = r[0][j], r[0][i] + r[1][i], r[1][j] = r[1][j], r[1][i] + } + return r + }, false, ""}, + {"trailing_empty_string", func(t *testing.T, r [][]string) [][]string { + for i, field := range r[0] { + if field == "remarks" { + r[0][i], r[0][len(r[0])-1] = r[0][len(r[0])-1], r[0][i] + r[1][i] = r[1][len(r[1])-1] + r[1] = r[1][:len(r[1])-1] + return r + } + } + t.Fatal("remarks column missing") + return nil + }, false, ""}, + {"empty_workbook", func(t *testing.T, r [][]string) [][]string { return nil }, true, ""}, + {"blank_row", func(t *testing.T, r [][]string) [][]string { return [][]string{r[0], {}, r[1]} }, false, ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + setupNotifyTestEnv(t, &model.Hosts{}) + host := excelHostFixture(t) + rows := tc.edit(t, excelHostRows(host)) + result := waf_service.ImportHostsExcel(rows, "1") + var count int64 + if err := global.GWAF_LOCAL_DB.Model(&model.Hosts{}).Count(&count).Error; err != nil { + t.Fatal(err) + } + if tc.fail { + if result.SuccessInt != 0 || result.FailInt != 1 || count != 0 || !strings.Contains(result.Msg, tc.message) { + t.Fatalf("invalid import must fail once without insertion: %+v, count=%d", result, count) + } + return + } + if result.SuccessInt != 1 || result.FailInt != 0 || result.Msg != "" || count != 1 { + t.Fatalf("valid import failed: %+v count=%d", result, count) + } + if tc.name == "int64_zero" { + host.EmergencyUntil = 0 + } + if tc.name == "int64_max" { + host.EmergencyUntil = 9223372036854775807 + } + if tc.name == "trailing_empty_string" { + host.REMARKS = "" + } + assertExcelImportedHost(t, host, "1") + }) + } +} + +func TestHostsExcelImportDuplicatesAndBatch(t *testing.T) { + for _, duplicate := range []string{"code", "host_port"} { + t.Run(duplicate, func(t *testing.T) { + setupNotifyTestEnv(t, &model.Hosts{}) + host := excelHostFixture(t) + existing := host + if duplicate == "code" { + existing.Host = "other.example" + } else { + existing.Code = "other-code" + } + if err := global.GWAF_LOCAL_DB.Create(&existing).Error; err != nil { + t.Fatal(err) + } + result := waf_service.ImportHostsExcel(excelHostRows(host), "1") + if result.FailInt != 1 || result.SuccessInt != 0 { + t.Fatalf("duplicate accepted: %+v", result) + } + }) + } + t.Run("invalid_row_then_valid", func(t *testing.T) { + setupNotifyTestEnv(t, &model.Hosts{}) + host := excelHostFixture(t) + rows := excelHostRows(host) + valid := append([]string(nil), rows[1]...) + setExcelHostCell(t, rows, "emergency_until", "invalid") + rows = append(rows, valid) + result := waf_service.ImportHostsExcel(rows, "1") + if result.SuccessInt != 1 || result.FailInt != 1 || !strings.Contains(result.Msg, "2") { + t.Fatalf("batch accounting incorrect: %+v", result) + } + assertExcelImportedHost(t, host, "1") + }) +} + +func TestHostsExcelImportBadRequests(t *testing.T) { + for _, tc := range []struct { + name, table, strategy string + file bool + }{ + {"no_file", "hosts", "0", false}, + {"no_table", "", "0", true}, + {"unknown_table", "unknown", "0", true}, + {"no_strategy", "hosts", "", true}, + {"unknown_strategy", "hosts", "2", true}, + {"invalid_workbook", "hosts", "0", true}, + } { + t.Run(tc.name, func(t *testing.T) { + rec := importExcelRequest(t, []byte("invalid workbook"), tc.table, tc.strategy, tc.file) + if rec.Code != http.StatusBadRequest || rec.Body.Len() == 0 { + t.Fatalf("want explained HTTP400, got %d: %s", rec.Code, rec.Body.String()) + } + }) } } func TestGetColumnName(t *testing.T) { - colIdx := 500 - colName := "" - for colIdx >= 0 { - colName = fmt.Sprintf("%c", 'A'+colIdx%26) + colName - colIdx = colIdx/26 - 1 + api := new(WafCommonApi) + for index, want := range map[int]string{0: "A", 25: "Z", 26: "AA", 51: "AZ", 52: "BA", 500: "SG"} { + if got := api.GetColumnName(index); got != want { + t.Errorf("column %s: want %s, got %s", strconv.Itoa(index), want, got) + } } - fmt.Println(colName) } diff --git a/service/waf_service/waf_common_import.go b/service/waf_service/waf_common_import.go new file mode 100644 index 0000000..d1eaff3 --- /dev/null +++ b/service/waf_service/waf_common_import.go @@ -0,0 +1,215 @@ +package waf_service + +import ( + "SamWaf/common/uuid" + "SamWaf/customtype" + "SamWaf/global" + "SamWaf/model" + "SamWaf/model/baseorm" + "errors" + "fmt" + "math" + "reflect" + "strconv" + "strings" + "time" +) + +type ReturnImportData struct { + SuccessInt int + FailInt int + Msg string +} + +// ImportHostsExcel 导入包含完整 Hosts 表头的网站数据。 +func ImportHostsExcel(rows [][]string, importCodeStrategy string) ReturnImportData { + var result ReturnImportData + if importCodeStrategy != "0" && importCodeStrategy != "1" { + result.FailInt = 1 + result.Msg = "导入策略无效，仅支持 0 或 1" + return result + } + processImportData(rows, importCodeStrategy, &result) + return result +} + +func processImportData(rows [][]string, strategy string, result *ReturnImportData) { + if len(rows) == 0 { + result.FailInt = 1 + result.Msg = "行 1, 缺少表头 |" + return + } + typ := reflect.TypeOf(model.Hosts{}) + columns := make(map[string]int) + for column, name := range rows[0] { + if name == " - " || name == "" { + continue + } + if _, exists := columns[name]; exists { + // 只输出模型字段信息，不回显表头文本。 + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if field.Name != "BaseOrm" && field.Tag.Get("json") == name { + result.FailInt = 1 + result.Msg = fmt.Sprintf("行 1, 字段 %s (%s) 表头重复 |", field.Name, field.Type) + return + } + } + result.FailInt = 1 + result.Msg = fmt.Sprintf("行 1, 第 %d 列表头重复 |", column+1) + return + } + columns[name] = column + } + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if field.Name == "BaseOrm" { + continue + } + if _, exists := columns[field.Tag.Get("json")]; !exists { + result.FailInt = 1 + result.Msg = fmt.Sprintf("行 1, 缺少字段 %s (%s) 表头 |", field.Name, field.Type) + return + } + } + + for index, row := range rows[1:] { + rowNumber := index + 2 + if importRowEmpty(row) { + continue + } + var rowErr error + for column := len(rows[0]); column < len(row); column++ { + if strings.TrimSpace(row[column]) != "" { + rowErr = fmt.Errorf("第 %d 列超出表头范围", column+1) + break + } + } + if rowErr != nil { + result.addRowFailure(rowNumber, rowErr) + continue + } + hostName := importCell(row, columns["host"]) + if strings.TrimSpace(hostName) == "" || hostName == "全局网站" { + continue + } + + var host model.Hosts + value := reflect.ValueOf(&host).Elem() + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + if field.Name == "BaseOrm" { + continue + } + if err := setImportScalar(value.Field(i), importCell(row, columns[field.Tag.Get("json")])); err != nil { + rowErr = fmt.Errorf("字段 %s (%s) %s", field.Name, field.Type, err) + break + } + } + if rowErr != nil { + result.addRowFailure(rowNumber, rowErr) + continue + } + if strategy == "0" { + host.Code = uuid.GenUUID() + } else if err := checkHostCodeData(host.Code); err != nil { + result.addRowFailure(rowNumber, err) + continue + } + if err := checkHostPortData(host.Host, host.Port); err != nil { + result.addRowFailure(rowNumber, err) + continue + } + now := customtype.JsonTime(time.Now()) + host.BaseOrm = baseorm.BaseOrm{ + Id: uuid.GenUUID(), USER_CODE: global.GWAF_USER_CODE, Tenant_ID: global.GWAF_TENANT_ID, + CREATE_TIME: now, UPDATE_TIME: now, + } + if err := global.GWAF_LOCAL_DB.Create(&host).Error; err != nil { + result.addRowFailure(rowNumber, errors.New("插入失败")) + continue + } + result.SuccessInt++ + } +} + +func (r *ReturnImportData) addRowFailure(rowNumber int, err error) { + r.FailInt++ + r.Msg += fmt.Sprintf("行 %d, %s |", rowNumber, err) +} + +func importCell(row []string, column int) string { + if column >= len(row) { + return "" + } + return row[column] +} + +func importRowEmpty(row []string) bool { + for _, cell := range row { + if strings.TrimSpace(cell) != "" { + return false + } + } + return true +} + +// setImportScalar 转换成功后才写入字段；错误不包含单元格内容。 +func setImportScalar(field reflect.Value, text string) error { + if !field.IsValid() || !field.CanSet() { + return errors.New("字段不可写") + } + switch field.Kind() { + case reflect.String: + field.SetString(text) + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + n, err := strconv.ParseInt(text, 10, field.Type().Bits()) + if err != nil { + return errors.New("整数无效或超出范围") + } + field.SetInt(n) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + n, err := strconv.ParseUint(text, 10, field.Type().Bits()) + if err != nil { + return errors.New("无符号整数无效或超出范围") + } + field.SetUint(n) + case reflect.Bool: + b, err := strconv.ParseBool(text) + if err != nil { + return errors.New("布尔值无效") + } + field.SetBool(b) + case reflect.Float32, reflect.Float64: + n, err := strconv.ParseFloat(text, field.Type().Bits()) + if err != nil || math.IsNaN(n) || math.IsInf(n, 0) { + return errors.New("浮点数无效或超出范围") + } + field.SetFloat(n) + default: + return errors.New("不支持的字段类型") + } + return nil +} + +func checkHostCodeData(code string) error { + var count int64 + if global.GWAF_LOCAL_DB == nil || global.GWAF_LOCAL_DB.Model(&model.Hosts{}).Where("code = ?", code).Count(&count).Error != nil { + return errors.New("字段 Code (string) 重复检查失败") + } + if count > 0 { + return errors.New("字段 Code (string) 数据已存在不进行插入") + } + return nil +} + +func checkHostPortData(host string, port int) error { + var count int64 + if global.GWAF_LOCAL_DB == nil || global.GWAF_LOCAL_DB.Model(&model.Hosts{}).Where("host = ? AND port = ?", host, port).Count(&count).Error != nil { + return errors.New("字段 Host (string) + Port (int) 重复检查失败") + } + if count > 0 { + return errors.New("字段 Host (string) + Port (int) 数据已存在不进行插入") + } + return nil +} diff --git a/service/waf_service/waf_common_import_test.go b/service/waf_service/waf_common_import_test.go new file mode 100644 index 0000000..2569a2a --- /dev/null +++ b/service/waf_service/waf_common_import_test.go @@ -0,0 +1,185 @@ +package waf_service + +import ( + "math" + "math/big" + "reflect" + "strconv" + "strings" + "testing" +) + +func TestSetImportScalarSignedWidths(t *testing.T) { + for _, sample := range []interface{}{int(0), int8(0), int16(0), int32(0), int64(0)} { + typ := reflect.TypeOf(sample) + t.Run(typ.String(), func(t *testing.T) { + limit := new(big.Int).Lsh(big.NewInt(1), uint(typ.Bits()-1)) + min := new(big.Int).Neg(new(big.Int).Set(limit)) + max := new(big.Int).Sub(new(big.Int).Set(limit), big.NewInt(1)) + for _, valid := range []string{min.String(), max.String(), "0", "-1", "+1"} { + field := reflect.New(typ).Elem() + if err := setImportScalar(field, valid); err != nil { + t.Fatalf("valid boundary rejected: %v", err) + } + want, _ := strconv.ParseInt(valid, 10, typ.Bits()) + if field.Int() != want { + t.Fatalf("got %d, want %d", field.Int(), want) + } + } + for _, invalid := range []string{ + new(big.Int).Sub(min, big.NewInt(1)).String(), + new(big.Int).Add(max, big.NewInt(1)).String(), + "", "invalid-secret", "1.5", "1e3", "0x10", " 1", "1 ", + } { + assertImportScalarRejected(t, typ, invalid) + } + }) + } +} + +func TestSetImportScalarUnsignedWidths(t *testing.T) { + for _, sample := range []interface{}{uint(0), uint8(0), uint16(0), uint32(0), uint64(0), uintptr(0)} { + typ := reflect.TypeOf(sample) + t.Run(typ.String(), func(t *testing.T) { + limit := new(big.Int).Lsh(big.NewInt(1), uint(typ.Bits())) + max := new(big.Int).Sub(new(big.Int).Set(limit), big.NewInt(1)) + for _, valid := range []string{"0", "1", max.String()} { + field := reflect.New(typ).Elem() + if err := setImportScalar(field, valid); err != nil { + t.Fatalf("valid boundary rejected: %v", err) + } + want, _ := strconv.ParseUint(valid, 10, typ.Bits()) + if field.Uint() != want { + t.Fatalf("got %d, want %d", field.Uint(), want) + } + } + for _, invalid := range []string{limit.String(), "-1", "", "invalid-secret", "1.5", "0x10", " 1"} { + assertImportScalarRejected(t, typ, invalid) + } + }) + } +} + +func TestSetImportScalarStringAndBool(t *testing.T) { + var text string + for _, valid := range []string{"", " original text ", "中文"} { + if err := setImportScalar(reflect.ValueOf(&text).Elem(), valid); err != nil || text != valid { + t.Fatal("string value was not preserved") + } + } + for _, valid := range []string{"true", "false", "1", "0", "TRUE", "FALSE"} { + var value bool + if err := setImportScalar(reflect.ValueOf(&value).Elem(), valid); err != nil { + t.Fatalf("valid bool rejected: %v", err) + } + want, _ := strconv.ParseBool(valid) + if value != want { + t.Fatalf("got %v, want %v", value, want) + } + } + for _, invalid := range []string{"", "2", "invalid-secret", " true"} { + assertImportScalarRejected(t, reflect.TypeOf(false), invalid) + } +} + +func TestSetImportScalarFloatWidths(t *testing.T) { + for _, sample := range []interface{}{float32(0), float64(0)} { + typ := reflect.TypeOf(sample) + t.Run(typ.String(), func(t *testing.T) { + max, smallest := math.MaxFloat64, math.SmallestNonzeroFloat64 + overflow := "1e309" + if typ.Bits() == 32 { + max, smallest, overflow = math.MaxFloat32, math.SmallestNonzeroFloat32, "1e39" + } + for _, valid := range []string{ + "0", "-1.25", "1.5e2", + strconv.FormatFloat(max, 'g', -1, typ.Bits()), + strconv.FormatFloat(-max, 'g', -1, typ.Bits()), + strconv.FormatFloat(smallest, 'g', -1, typ.Bits()), + } { + field := reflect.New(typ).Elem() + if err := setImportScalar(field, valid); err != nil { + t.Fatalf("valid float rejected: %v", err) + } + want, _ := strconv.ParseFloat(valid, typ.Bits()) + if field.Float() != want { + t.Fatalf("got %v, want %v", field.Float(), want) + } + } + for _, invalid := range []string{"", "invalid-secret", overflow, "-" + overflow, "NaN", "Inf", "+Inf", "-Inf", " 1"} { + assertImportScalarRejected(t, typ, invalid) + } + }) + } +} + +func TestSetImportScalarUnsupportedAndUnwritable(t *testing.T) { + for _, sample := range []interface{}{ + struct{}{}, []string{}, [1]int{}, map[string]string{}, (*int)(nil), complex64(0), complex128(0), + make(chan int), (func())(nil), + } { + typ := reflect.TypeOf(sample) + t.Run(typ.String(), func(t *testing.T) { + assertImportScalarRejected(t, typ, "invalid-secret") + }) + } + var value interface{} + if err := setImportScalar(reflect.ValueOf(&value).Elem(), "invalid-secret"); err == nil { + t.Fatal("interface field accepted") + } + for _, field := range []reflect.Value{reflect.Value{}, reflect.ValueOf(1)} { + if err := setImportScalar(field, "invalid-secret"); err == nil { + t.Fatal("unwritable field accepted") + } + } +} + +func TestSetImportScalarNamedTypes(t *testing.T) { + type signed int64 + type unsigned uint16 + type text string + type flag bool + type number float32 + for _, tc := range []struct { + initial interface{} + input string + want interface{} + }{ + {signed(0), "9223372036854775807", signed(math.MaxInt64)}, + {unsigned(0), "65535", unsigned(math.MaxUint16)}, + {text(""), "unchanged", text("unchanged")}, + {flag(false), "true", flag(true)}, + {number(0), "1.25", number(1.25)}, + } { + field := reflect.New(reflect.TypeOf(tc.initial)).Elem() + if err := setImportScalar(field, tc.input); err != nil || !reflect.DeepEqual(field.Interface(), tc.want) { + t.Fatalf("named scalar conversion failed: %v", err) + } + } +} + +func assertImportScalarRejected(t *testing.T, typ reflect.Type, input string) { + t.Helper() + field := reflect.New(typ).Elem() + switch field.Kind() { + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + field.SetInt(7) + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr: + field.SetUint(7) + case reflect.Bool: + field.SetBool(true) + case reflect.Float32, reflect.Float64: + field.SetFloat(7) + } + before := field.Interface() + err := setImportScalar(field, input) + if err == nil { + t.Fatal("invalid value or unsupported type accepted") + } + if input != "" && strings.Contains(err.Error(), input) { + t.Fatal("error contains the input value") + } + if !reflect.DeepEqual(field.Interface(), before) { + t.Fatal("failed conversion changed the field") + } +} diff --git a/wafupgradenotice/upgrade_notes.yaml b/wafupgradenotice/upgrade_notes.yaml index 9a06a23..d645add 100644 --- a/wafupgradenotice/upgrade_notes.yaml +++ b/wafupgradenotice/upgrade_notes.yaml @@ -12,6 +12,22 @@ # 5. fresh_install: true 的条目只在"全新安装"时生成，不参与版本区间计算 notes: + - id: v1_3_25_hosts_excel_integer_import + version: v1.3.25 + kind: check + level: low + page: /waf-host/wafhost + apply: + type: navigate + zh: + title: 网站 Excel 导入支持各位宽整数 + detail: 网站 Excel 导入支持包括紧急模式到期时间在内的整数字段，转换错误会标明行号与字段，每行最多计一次失败。导入仍要求完整字段表头；旧文件缺少新增列时，请使用当前版本重新导出后填写。 + revert: 无需调整配置；导入前请确认文件字段完整，并核对导入结果中的成功数和失败原因。 + en: + title: Website Excel import supports all integer widths + detail: Website Excel import supports integer fields including the emergency-mode expiry time. Conversion errors identify the row and field, with at most one failure per row. A complete column header is still required; if an older file lacks new columns, export a fresh file from the current version before filling it in. + revert: No configuration changes are needed. Check that the file has all columns before import, then review the success count and failure reasons. + - id: v1_3_25_website_access_diagnostic version: v1.3.25 kind: check