From cb9bcd95866690c504570d9e9feb10621dba7d51 Mon Sep 17 00:00:00 2001 From: ioito Date: Wed, 19 Jan 2022 19:08:27 +0800 Subject: [PATCH] fix(common): metadata update key --- go.mod | 4 +- go.sum | 8 ++-- pkg/cloudcommon/db/metadata.go | 3 +- pkg/multicloud/aliyun/resource_tags.go | 48 +++++++++++++------ vendor/modules.txt | 4 +- vendor/yunion.io/x/jsonutils/access.go | 2 +- .../x/sqlchemy/backends/clickhouse/sync.go | 2 +- .../x/sqlchemy/backends/mysql/sync.go | 2 +- .../x/sqlchemy/backends/sqlite/sync.go | 2 +- vendor/yunion.io/x/sqlchemy/backends_mock.go | 4 ++ vendor/yunion.io/x/sqlchemy/sync.go | 1 + vendor/yunion.io/x/sqlchemy/update.go | 7 +++ 12 files changed, 59 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index fede148576..49215f1e3d 100644 --- a/go.mod +++ b/go.mod @@ -152,11 +152,11 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 - yunion.io/x/jsonutils v0.0.0-20211213041607-f6733e38783e + yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8 yunion.io/x/log v0.0.0-20201210064738-43181789dc74 yunion.io/x/ovsdb v0.0.0-20200526071744-27bf0940cbc7 yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e - yunion.io/x/sqlchemy v0.0.0-20211226083340-1ba177bc58be + yunion.io/x/sqlchemy v0.0.0-20220119131341-fadec84c0638 yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce ) diff --git a/go.sum b/go.sum index cdf31a88e3..c442992e4b 100644 --- a/go.sum +++ b/go.sum @@ -942,8 +942,8 @@ sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII= yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= -yunion.io/x/jsonutils v0.0.0-20211213041607-f6733e38783e h1:zOVxZ0HrDOHLTm5YxkDKUm1x5jMVKbQ/iBSAyn5l1+Q= -yunion.io/x/jsonutils v0.0.0-20211213041607-f6733e38783e/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE= +yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8 h1:yFJVMnUoolC8mlUxa+LMTObYmqJ2H7iyxGIaXxT1fOc= +yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE= yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE= yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= yunion.io/x/log v0.0.0-20201210064738-43181789dc74 h1:7D+sQ/XaUTUEm+NCrKXOhXBKlzEd0RyS2qZ4vBGjx2o= @@ -957,7 +957,7 @@ yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97 h1:lY+5NlWwO2aIkfNnVOGKHmZvMt yunion.io/x/pkg v0.0.0-20211116020154-6a76ba2f7e97/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo= yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v0.0.0-20211226083340-1ba177bc58be h1:LCmMlLgRLr6EmnMQtzbsys6QGY2wfFk+1IOgb2tG/7E= -yunion.io/x/sqlchemy v0.0.0-20211226083340-1ba177bc58be/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= +yunion.io/x/sqlchemy v0.0.0-20220119131341-fadec84c0638 h1:Tc7bMp/mahI1Zua5dxBNrXSKl4hCUSA7CPBcqBnK8Uw= +yunion.io/x/sqlchemy v0.0.0-20220119131341-fadec84c0638/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce h1:kU8xE7O5uZ1GSJVMZHoJ+jrNL7csUQHYGyAPW9QfNpE= yunion.io/x/structarg v0.0.0-20200720093445-9f850fa222ce/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= diff --git a/pkg/cloudcommon/db/metadata.go b/pkg/cloudcommon/db/metadata.go index 1535a210c6..630106a643 100644 --- a/pkg/cloudcommon/db/metadata.go +++ b/pkg/cloudcommon/db/metadata.go @@ -576,6 +576,7 @@ func (manager *SMetadataManager) rawSetValues(ctx context.Context, objType strin rV, rD := record.Value, record.Deleted _, err = Update(&record, func() error { record.Value = newRecord.Value + record.Key = key record.Deleted = newRecord.Deleted return nil }) @@ -675,7 +676,7 @@ func (manager *SMetadataManager) rawGetAll(objType, objId string, keys []string, ret := make(map[string]string) for _, rec := range records { if len(rec.Value) > 0 || strings.HasPrefix(rec.Key, USER_TAG_PREFIX) || strings.HasPrefix(rec.Key, CLOUD_TAG_PREFIX) { - ret[strings.ToLower(rec.Key)] = rec.Value + ret[rec.Key] = rec.Value } } return ret, nil diff --git a/pkg/multicloud/aliyun/resource_tags.go b/pkg/multicloud/aliyun/resource_tags.go index 3ce3de4c26..441dcea305 100644 --- a/pkg/multicloud/aliyun/resource_tags.go +++ b/pkg/multicloud/aliyun/resource_tags.go @@ -20,6 +20,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/cloudprovider" ) @@ -97,28 +98,45 @@ func (self *SRegion) UntagResource(serviceType string, resourceType string, resI } func (self *SRegion) SetResourceTags(serviceType string, resourceType string, resId string, tags map[string]string, replace bool) error { - err := self.TagResource(serviceType, resourceType, resId, tags) - if err != nil { - return errors.Wrapf(err, "TagResource") - } - if !replace || len(tags) == 0 { - return nil - } _, _tags, err := self.ListSysAndUserTags(serviceType, resourceType, resId) if err != nil { return errors.Wrapf(err, "ListTags") } - tagMaps := map[string]string{} - for k, v := range tags { - tagMaps[strings.ToLower(k)] = v + keys, upperKeys := []string{}, []string{} + for k := range tags { + keys = append(keys, k) + upperKeys = append(upperKeys, strings.ToUpper(k)) } - keys := []string{} - for k := range _tags { - if _, ok := tagMaps[strings.ToLower(k)]; !ok { - keys = append(keys, k) + if replace { + if len(tags) > 0 { + removeKeys := []string{} + for k := range _tags { + if !utils.IsInStringArray(k, keys) { + removeKeys = append(removeKeys, k) + } + } + if len(removeKeys) > 0 { + err := self.UntagResource(serviceType, resourceType, resId, removeKeys) + if err != nil { + return errors.Wrapf(err, "UntagResource") + } + } + } + } else { + removeKeys := []string{} + for k := range _tags { + if !utils.IsInStringArray(k, keys) && utils.IsInStringArray(strings.ToUpper(k), upperKeys) { + removeKeys = append(removeKeys, k) + } + } + if len(removeKeys) > 0 { + err := self.UntagResource(serviceType, resourceType, resId, removeKeys) + if err != nil { + return errors.Wrapf(err, "UntagResource") + } } } - return self.UntagResource(serviceType, resourceType, resId, keys) + return self.TagResource(serviceType, resourceType, resId, tags) } func (self *SRegion) TagResource(serviceType string, resourceType string, resourceId string, tags map[string]string) error { diff --git a/vendor/modules.txt b/vendor/modules.txt index 07471dc46c..7621e0df6e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1186,7 +1186,7 @@ sigs.k8s.io/yaml yunion.io/x/executor/apis yunion.io/x/executor/client yunion.io/x/executor/server -# yunion.io/x/jsonutils v0.0.0-20211213041607-f6733e38783e +# yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8 yunion.io/x/jsonutils # yunion.io/x/log v0.0.0-20201210064738-43181789dc74 yunion.io/x/log @@ -1229,7 +1229,7 @@ yunion.io/x/pkg/util/workqueue yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e yunion.io/x/s3cli -# yunion.io/x/sqlchemy v0.0.0-20211226083340-1ba177bc58be +# yunion.io/x/sqlchemy v0.0.0-20220119131341-fadec84c0638 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends yunion.io/x/sqlchemy/backends/clickhouse diff --git a/vendor/yunion.io/x/jsonutils/access.go b/vendor/yunion.io/x/jsonutils/access.go index b08e18ec16..7d687d3e13 100644 --- a/vendor/yunion.io/x/jsonutils/access.go +++ b/vendor/yunion.io/x/jsonutils/access.go @@ -112,7 +112,7 @@ func (this *JSONDict) Add(o JSONObject, keys ...string) error { obj.Set(keys[i], o) } else { o, ok := obj.data.Get(keys[i]) - if !ok { + if !ok || o == JSONNull { obj.Set(keys[i], NewDict()) o, ok = obj.data.Get(keys[i]) } diff --git a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go index c8fb7e2565..430eaf4cb1 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/clickhouse/sync.go @@ -77,7 +77,7 @@ func (clickhouse *SClickhouseBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpe /* IGNORE DROP STATEMENT */ for _, col := range changes.RemoveColumns { sql := fmt.Sprintf("DROP COLUMN `%s`", col.Name()) - log.Infof("ALTER TABLE %s %s;", ts.Name(), sql) + log.Debugf("skip ALTER TABLE %s %s;", ts.Name(), sql) // alters = append(alters, sql) // ignore drop statement // if the column is auto_increment integer column, diff --git a/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go b/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go index bf2d50c8a6..84f7dfa1a6 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/mysql/sync.go @@ -61,7 +61,7 @@ func (mysql *SMySQLBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, changes /* IGNORE DROP STATEMENT */ for _, col := range changes.RemoveColumns { sql := fmt.Sprintf("DROP COLUMN `%s`", col.Name()) - log.Infof("ALTER TABLE %s %s;", ts.Name(), sql) + log.Debugf("skip ALTER TABLE %s %s;", ts.Name(), sql) // alters = append(alters, sql) // ignore drop statement // if the column is auto_increment integer column, diff --git a/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go b/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go index b7f3cc0904..54438ecf7e 100644 --- a/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go +++ b/vendor/yunion.io/x/sqlchemy/backends/sqlite/sync.go @@ -61,7 +61,7 @@ func (sqlite *SSqliteBackend) CommitTableChangeSQL(ts sqlchemy.ITableSpec, chang /* IGNORE DROP STATEMENT */ for _, col := range changes.RemoveColumns { sql := fmt.Sprintf("DROP COLUMN `%s`", col.Name()) - log.Infof("ALTER TABLE %s %s;", ts.Name(), sql) + log.Debugf("skip ALTER TABLE %s %s;", ts.Name(), sql) // alters = append(alters, sql) // ignore drop statement // if the column is auto_increment integer column, diff --git a/vendor/yunion.io/x/sqlchemy/backends_mock.go b/vendor/yunion.io/x/sqlchemy/backends_mock.go index a35ca82a1c..6474a4dbba 100644 --- a/vendor/yunion.io/x/sqlchemy/backends_mock.go +++ b/vendor/yunion.io/x/sqlchemy/backends_mock.go @@ -51,6 +51,10 @@ func (c *sMockColumn) IsNumeric() bool { return false } +func (c *sMockColumn) IsText() bool { + return c.sqlType == "string" +} + func (c *sMockColumn) IsZero(val interface{}) bool { return reflect.ValueOf(val).IsZero() } diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 650892eb42..fcc26b5ab8 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -202,6 +202,7 @@ func (ts *STableSpec) Sync() error { sqls := ts.SyncSQL() if sqls != nil { for _, sql := range sqls { + log.Infof(sql) _, err := ts.Database().Exec(sql) if err != nil { log.Errorf("exec sql error %s: %s", sql, err) diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index 356dad3199..5ad3254232 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -128,6 +128,13 @@ func (us *SUpdateSession) saveUpdateSql(dt interface{}) (*sUpdateSQLResult, erro nf, _ := fields.GetInterface(k) if c.IsPrimary() { if !gotypes.IsNil(of) && !c.IsZero(of) { + if c.IsText() { + ov, _ := of.(string) + nv, _ := nf.(string) + if ov != nv && strings.EqualFold(ov, nv) { + setters[k] = SUpdateDiff{old: of, new: nf, col: c} + } + } primaries[k] = c.ConvertFromValue(of) } else if c.IsText() { primaries[k] = ""