fix: support db checksum algorithm of SHA256 (#19681)

Co-authored-by: Qiu Jian <qiujian@yunionyun.com>

go.mod

@@ -95,7 +95,7 @@ require (
 	yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
 	yunion.io/x/pkg v1.10.1-0.20240303050651-73685b15a96e
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
-	yunion.io/x/sqlchemy v1.1.3-0.20240305180726-ccd03e6aa837
+	yunion.io/x/sqlchemy v1.1.3-0.20240309051602-9ab0cc051bcc
 	yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c
 )
 

go.sum

@@ -1223,7 +1223,7 @@ yunion.io/x/pkg v1.10.1-0.20240303050651-73685b15a96e h1:NuYu+z0dOT93aoNjg3W3PkL
 yunion.io/x/pkg v1.10.1-0.20240303050651-73685b15a96e/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
-yunion.io/x/sqlchemy v1.1.3-0.20240305180726-ccd03e6aa837 h1:2KhiZJdbKVK6ksl542z5BzDsD+bgpHEtYHCEuA8Y48A=
-yunion.io/x/sqlchemy v1.1.3-0.20240305180726-ccd03e6aa837/go.mod h1:5W8ghvJ4TNt/r2yDjjD3i4QsZgIiJX45dhRQBGWPHsQ=
+yunion.io/x/sqlchemy v1.1.3-0.20240309051602-9ab0cc051bcc h1:aoz/hVPigGxG/fGCpDmC3Wi57/yDbbGQO7T5SGj6l4c=
+yunion.io/x/sqlchemy v1.1.3-0.20240309051602-9ab0cc051bcc/go.mod h1:5W8ghvJ4TNt/r2yDjjD3i4QsZgIiJX45dhRQBGWPHsQ=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ=
 yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=

pkg/apis/identity/consts.go

@@ -165,6 +165,7 @@ var (
 			"db_checksum_skip_init",
 			"db_checksum_tables",
 			"enable_db_checksum_tables",
+			"db_checksum_hash_algorithm",
 			"auto_sync_table",
 			"exit_after_db_init",
 			"global_virtual_resource_namespace",

pkg/cloudcommon/consts/db.go

@@ -14,6 +14,8 @@
 
 package consts
 
+import "yunion.io/x/log"
+
 var (
 	QueryOffsetOptimization = false
 
@@ -22,6 +24,8 @@ var (
 	defaultDBDialect string
 
 	defaultDBConnectionString string
+
+	defaultDBChecksumHashAlgorithm string
 )
 
 func SetDefaultDB(dialect, connStr string) {
@@ -36,3 +40,15 @@ func DefaultDBDialect() string {
 func DefaultDBConnStr() string {
 	return defaultDBConnectionString
 }
+
+func SetDefaultDBChecksumHashAlgorithm(alg string) {
+	log.Infof("Set default DB checksum hash algorithm: %s", alg)
+	defaultDBChecksumHashAlgorithm = alg
+}
+
+func DefaultDBChecksumHashAlgorithm() string {
+	if len(defaultDBChecksumHashAlgorithm) > 0 {
+		return defaultDBChecksumHashAlgorithm
+	}
+	return "sha256"
+}

pkg/cloudcommon/database.go

@@ -143,6 +143,10 @@ func InitDB(options *common_options.DBOptions) {
 	}
 	// lm := lockman.NewNoopLockManager()
 
+	if options.EnableDBChecksumTables && len(options.DBChecksumHashAlgorithm) > 0 {
+		consts.SetDefaultDBChecksumHashAlgorithm(options.DBChecksumHashAlgorithm)
+	}
+
 	initDBNotifier()
 	startInitInformer(options)
 }

pkg/cloudcommon/db/checksum.go

@@ -16,9 +16,11 @@ package db
 
 import (
 	"crypto/md5"
+	"crypto/sha256"
 	"fmt"
 	"reflect"
 	"sort"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
@@ -27,6 +29,7 @@ import (
 	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
+	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/util/splitable"
 )
 
@@ -126,13 +129,19 @@ func CheckRecordChecksumConsistent(model IModel) error {
 }
 
 func calculateRecordChecksumByValues(vals []interface{}) string {
-	ss := ""
+	ss := strings.Builder{}
 	for _, val := range vals {
-		ss += fmt.Sprintf("\n%v", val)
+		ss.WriteString(fmt.Sprintf("\n%v", val))
 	}
-	hStr := md5.Sum([]byte(ss))
-	sum := fmt.Sprintf("%x", hStr)
-	log.Debugf("calculate values string: %s checksum: %s", ss, sum)
+	var sum string
+	algStr := consts.DefaultDBChecksumHashAlgorithm()
+	switch algStr {
+	case "md5":
+		sum = fmt.Sprintf("%x", md5.Sum([]byte(ss.String())))
+	default:
+		sum = fmt.Sprintf("%x", sha256.Sum256([]byte(ss.String())))
+	}
+	// log.Debugf("calculate values string: %s alg: %s, checksum: %s", ss, algStr, sum)
 	return sum
 }
 

pkg/cloudcommon/options/options.go

@@ -164,6 +164,8 @@ type DBOptions struct {
 	EnableDBChecksumTables bool `help:"Enable DB tables with record checksum for consistency"`
 	DBChecksumSkipInit     bool `help:"Skip DB tables with record checksum calculation when init" default:"false"`
 
+	DBChecksumHashAlgorithm string `help:"hash algorithm for db checksum hash" choices:"md5|sha256" default:"sha256"`
+
 	AutoSyncTable   bool `help:"Automatically synchronize table changes if differences are detected"`
 	ExitAfterDBInit bool `help:"Exit program after db initialization" default:"false"`
 

pkg/util/splitable/metadata.go

@@ -42,7 +42,7 @@ func (spec *SSplitTableSpec) getTableLastMeta() (*STableMetadata, error) {
 func (spec *SSplitTableSpec) GetTableMetaByTime(recordTime time.Time) (*STableMetadata, error) {
 	q := spec.metaSpec.Query().Desc("id").IsFalse("deleted")
 	if !recordTime.IsZero() {
-		q = q.LE("start_date", recordTime)
+		q = q.LE("start_date", recordTime.Add(time.Second))
 	}
 	meta := new(STableMetadata)
 	err := q.First(meta)

vendor/modules.txt

@@ -1624,7 +1624,7 @@ yunion.io/x/pkg/utils
 # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 ## explicit; go 1.12
 yunion.io/x/s3cli
-# yunion.io/x/sqlchemy v1.1.3-0.20240305180726-ccd03e6aa837
+# yunion.io/x/sqlchemy v1.1.3-0.20240309051602-9ab0cc051bcc
 ## explicit; go 1.17
 yunion.io/x/sqlchemy
 yunion.io/x/sqlchemy/backends

vendor/yunion.io/x/sqlchemy/backends/dameng/column.go

@@ -505,12 +505,12 @@ func (c *STimeTypeColumn) ConvertFromString(str string) interface{} {
 // NewTimeTypeColumn return an instance of STimeTypeColumn
 func NewTimeTypeColumn(name string, typeStr string, tagmap map[string]string, isPointer bool) STimeTypeColumn {
 	tagmap, v, ok := utils.TagPop(tagmap, sqlchemy.TAG_PRECISION)
-	prec := 6
+	prec := 0
 	if ok {
 		prec, _ = strconv.Atoi(v)
-		if prec > 6 || prec <= 0 {
+		if prec > 6 || prec < 0 {
 			log.Warningf("datetime field %s precision %d change to 6", name, prec)
-			prec = 6
+			prec = 0
 		}
 	}
 	dc := STimeTypeColumn{