opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix(region): aws redis tags (#16937)

Browse Source
This commit is contained in:
屈轩
2023-05-05 14:42:11 +08:00
committed by GitHub
parent 1419c6cee9
commit 446eeda5e8
16 changed files with 249 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+3 -3
View File
@@ -84,14 +84,14 @@ require (
k8s.io/client-go v0.19.3
k8s.io/cluster-bootstrap v0.19.3
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230428051538-2fa935f30d76
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230504125407-bc1e1d01d41c
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656
yunion.io/x/jsonutils v1.0.1-0.20230428104347-7c2fdff8e8e7
yunion.io/x/log v1.0.1-0.20230411060016-feb3f46ab361
yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900
yunion.io/x/pkg v1.0.1-0.20230427104242-202edd119017
yunion.io/x/pkg v1.0.1-0.20230504073602-0a74096f836a
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
yunion.io/x/sqlchemy v1.1.2-0.20230422192549-7a9668176f46
yunion.io/x/sqlchemy v1.1.2-0.20230504021010-f3e76263d633
yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6
)
go.sum
+6 -6
View File
@@ -1185,8 +1185,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230428051538-2fa935f30d76 h1:+3TVCaGlZsheKC6klo6gONQ4EYGjzGsaJ1JDJmBwdy0=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230428051538-2fa935f30d76/go.mod h1:crMeQeaNaZefTXfXbQkoj5SStggqkSNVABHtYBFjM3Y=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230504125407-bc1e1d01d41c h1:I2/soITFPHV7aAZsaKNmoixcRyAKZcByAqSnm54V6c8=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230504125407-bc1e1d01d41c/go.mod h1:crMeQeaNaZefTXfXbQkoj5SStggqkSNVABHtYBFjM3Y=
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656 h1:0zlZD5uhZoIHgLVAWCz2aHaYk2ZrNsACCYD7R6EIBII=
yunion.io/x/executor v0.0.0-20211018100936-39a2cd966656/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
@@ -1204,12 +1204,12 @@ yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZV
yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
yunion.io/x/pkg v1.0.1-0.20220819080140-55b265a8658e/go.mod h1:nYOMflWG1wQC6301rGVglQva9bYbJxHThEJUxRAgMFc=
yunion.io/x/pkg v1.0.1-0.20230427104242-202edd119017 h1:UbdLKvnWMSWnRt24LKoaBr2vGI2QfgDG9GbugCKjpAQ=
yunion.io/x/pkg v1.0.1-0.20230427104242-202edd119017/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
yunion.io/x/pkg v1.0.1-0.20230504073602-0a74096f836a h1:8fv6DAHrq+YtvdDIB7QqvmhbqYtBpqmydg1GS7dim9Q=
yunion.io/x/pkg v1.0.1-0.20230504073602-0a74096f836a/go.mod h1:ksCJVQ+DwKrJ5QBEoU8pzrDFfDaZVAFH/iJ6yQCYxJk=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
yunion.io/x/sqlchemy v1.0.1/go.mod h1:FTdwPdGhMgh4E+UFXc9klI1Ok34fMuybTT+jLhOaIjI=
yunion.io/x/sqlchemy v1.1.2-0.20230422192549-7a9668176f46 h1:M/Xv+Qu6/dSMyuvhgbs52/adr+O5k5NXtpF7czYVCJU=
yunion.io/x/sqlchemy v1.1.2-0.20230422192549-7a9668176f46/go.mod h1:xXVfJLIxxTRTnPEaVIIoRZXqWyO8+takIOYhc+zO5KE=
yunion.io/x/sqlchemy v1.1.2-0.20230504021010-f3e76263d633 h1:llBmGecLpkEw47PwH2mCyiusNfnWZBdtX/ymhWPdbn4=
yunion.io/x/sqlchemy v1.1.2-0.20230504021010-f3e76263d633/go.mod h1:uuPVZEyEq3sWd5vf9VjGSy6lZzof22X87OEHw9sddJQ=
yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6 h1:WuWXhY3DvhdRTzWCJ/kwt3Ss6KIq7+KqJwb+esvNGwU=
yunion.io/x/structarg v0.0.0-20220312084958-9c6c79c7d1c6/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng=
pkg/compute/models/elasticcache_instances.go
+2 -2
View File
@@ -145,10 +145,10 @@ type SElasticcache struct {
// 维护开始时间，格式为HH:mmZ
// example: 02:00Z
MaintainStartTime string `width:"8" charset:"ascii" nullable:"true" list:"user" create:"optional" json:"maintain_start_time"`
MaintainStartTime string `width:"16" charset:"ascii" nullable:"true" list:"user" create:"optional" json:"maintain_start_time"`
// 维护结束时间
MaintainEndTime string `width:"8" charset:"ascii" nullable:"true" list:"user" create:"optional" json:"maintain_end_time"`
MaintainEndTime string `width:"16" charset:"ascii" nullable:"true" list:"user" create:"optional" json:"maintain_end_time"`
// 访问密码？ on （开启密码）|off （免密码访问）
AuthMode string `width:"8" charset:"ascii" nullable:"false" list:"user" create:"optional" json:"auth_mode"`
vendor/modules.txt
Vendored
+3 -3
View File
@@ -1457,7 +1457,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.2.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230428051538-2fa935f30d76
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20230504125407-bc1e1d01d41c
## explicit; go 1.18
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
@@ -1548,7 +1548,7 @@ yunion.io/x/log/hooks
yunion.io/x/ovsdb/cli_util
yunion.io/x/ovsdb/schema/ovn_nb
yunion.io/x/ovsdb/types
# yunion.io/x/pkg v1.0.1-0.20230427104242-202edd119017
# yunion.io/x/pkg v1.0.1-0.20230504073602-0a74096f836a
## explicit; go 1.18
yunion.io/x/pkg/appctx
yunion.io/x/pkg/errors
@@ -1603,7 +1603,7 @@ yunion.io/x/pkg/utils
# yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
## explicit; go 1.12
yunion.io/x/s3cli
# yunion.io/x/sqlchemy v1.1.2-0.20230422192549-7a9668176f46
# yunion.io/x/sqlchemy v1.1.2-0.20230504021010-f3e76263d633
## explicit; go 1.17
yunion.io/x/sqlchemy
yunion.io/x/sqlchemy/backends
vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/elasticache_instance.go
Generated Vendored
+65 -2
View File
@@ -15,6 +15,7 @@
package aws
import (
"fmt"
"strconv"
"strings"
"time"
@@ -432,6 +433,68 @@ func (self *SElasticache) GetPrivateIpAddr() string {
return ""
}
func (self *SElasticache) GetTags() (map[string]string, error) {
params := map[string]string{
"ResourceName": *self.replicaGroup.ARN,
}
tags := AwsTags{}
err := self.region.redisRequest("ListTagsForResource", params, &tags)
if err != nil {
return nil, errors.Wrapf(err, "ListTagsForResource")
}
return tags.GetTags()
}
func (self *SElasticache) SetTags(tags map[string]string, replace bool) error {
oldTags, err := self.GetTags()
if err != nil {
return errors.Wrapf(err, "GetTags")
}
added, removed := map[string]string{}, map[string]string{}
for k, v := range tags {
oldValue, ok := oldTags[k]
if !ok {
added[k] = v
} else if oldValue != v {
removed[k] = oldValue
added[k] = v
}
}
if replace {
for k, v := range oldTags {
newValue, ok := tags[k]
if !ok {
removed[k] = v
} else if v != newValue {
added[k] = newValue
removed[k] = v
}
}
}
if len(removed) > 0 {
params := map[string]string{
"ResourceName": *self.replicaGroup.ARN,
}
i := 1
for k := range tags {
params[fmt.Sprintf("TagKeys.member.%d", i)] = k
}
return self.region.redisRequest("RemoveTagsFromResource", params, nil)
}
if len(added) > 0 {
params := map[string]string{
"ResourceName": *self.replicaGroup.ARN,
}
i := 1
for k, v := range tags {
params[fmt.Sprintf("Tags.member.%d.Key", i)] = k
params[fmt.Sprintf("Tags.member.%d.Value", i)] = v
}
return self.region.redisRequest("AddTagsToResource", params, nil)
}
return nil
}
func (self *SElasticache) GetPrivateConnectPort() int {
for _, nodeGroup := range self.replicaGroup.NodeGroups {
if nodeGroup != nil && nodeGroup.PrimaryEndpoint != nil && nodeGroup.PrimaryEndpoint.Port != nil {
@@ -464,7 +527,7 @@ func (self *SElasticache) GetMaintainStartTime() string {
}
splited := strings.Split(window, "-")
return splited[0]
return strings.Trim(splited[0], "")
}
func (self *SElasticache) GetMaintainEndTime() string {
@@ -479,7 +542,7 @@ func (self *SElasticache) GetMaintainEndTime() string {
splited := strings.Split(window, "-")
if len(splited) == 2 {
return splited[1]
return strings.Trim(splited[1], "")
}
return ""
}
vendor/yunion.io/x/cloudmux/pkg/multicloud/aws/region.go
Generated Vendored
+5
View File
@@ -138,6 +138,7 @@ const (
ROUTE53_SERVICE_NAME = "route53"
ELASTICACHE_SERVICE_NAME = "elasticache"
ELASTICACHE_SERVICE_ID = "ElastiCache"
ELB_SERVICE_NAME = "elasticloadbalancing"
ELB_SERVICE_ID = "Elastic Load Balancing v2"
@@ -257,6 +258,10 @@ func (self *SRegion) rdsRequest(apiName string, params map[string]string, retval
return self.client.request(self.RegionId, RDS_SERVICE_NAME, RDS_SERVICE_ID, "2014-10-31", apiName, params, retval, true)
}
func (self *SRegion) redisRequest(apiName string, params map[string]string, retval interface{}) error {
return self.client.request(self.RegionId, ELASTICACHE_SERVICE_NAME, ELASTICACHE_SERVICE_ID, "2015-02-02", apiName, params, retval, true)
}
func (self *SRegion) ec2Request(apiName string, params map[string]string, retval interface{}) error {
return self.client.request(self.RegionId, EC2_SERVICE_NAME, EC2_SERVICE_ID, "2016-11-15", apiName, params, retval, true)
}
vendor/yunion.io/x/pkg/utils/utils.go
Generated Vendored
+14 -6
View File
@@ -116,15 +116,14 @@ func DecodeMeta(str string) string {
}
func IsInStringArray(val string, array []string) bool {
for _, ele := range array {
if ele == val {
return true
}
}
return false
return IsInArray(val, array)
}
func InStringArray(val string, array []string) (ok bool, i int) {
return InArray2(val, array)
}
func InArray2[T int | uint | int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64 | float32 | float64 | string](val T, array []T) (ok bool, i int) {
for i = range array {
if ok = array[i] == val; ok {
return
@@ -133,6 +132,15 @@ func InStringArray(val string, array []string) (ok bool, i int) {
return
}
func IsInArray[T int | uint | int8 | uint8 | int16 | uint16 | int32 | uint32 | int64 | uint64 | float32 | float64 | string](val T, array []T) bool {
for _, ele := range array {
if ele == val {
return true
}
}
return false
}
func InArray(v interface{}, in interface{}) (ok bool, i int) {
val := reflect.Indirect(reflect.ValueOf(in))
switch val.Kind() {
vendor/yunion.io/x/sqlchemy/backends/clickhouse/clickhouse.go
Generated Vendored
+3
View File
@@ -258,6 +258,9 @@ func (click *SClickhouseBackend) GetColumnSpecByFieldType(table *sqlchemy.STable
}
col := NewFloatColumn(fieldname, "Float64", tagmap, isPointer)
return &col
case reflect.Map, reflect.Slice:
col := NewCompoundColumn(fieldname, tagmap, isPointer)
return &col
}
if fieldType.Implements(gotypes.ISerializableType) {
col := NewCompoundColumn(fieldname, tagmap, isPointer)
vendor/yunion.io/x/sqlchemy/backends/mysql/mysql.go
Generated Vendored
+3
View File
@@ -213,6 +213,9 @@ func (mysql *SMySQLBackend) GetColumnSpecByFieldType(table *sqlchemy.STableSpec,
}
col := NewFloatColumn(fieldname, colType, tagmap, isPointer)
return &col
case reflect.Map, reflect.Slice:
col := NewCompoundColumn(fieldname, getTextSqlType(tagmap), tagmap, isPointer)
return &col
}
if fieldType.Implements(gotypes.ISerializableType) {
col := NewCompoundColumn(fieldname, getTextSqlType(tagmap), tagmap, isPointer)
vendor/yunion.io/x/sqlchemy/backends/sqlite/sqlite.go
Generated Vendored
+3
View File
@@ -170,6 +170,9 @@ func (sqlite *SSqliteBackend) GetColumnSpecByFieldType(table *sqlchemy.STableSpe
case reflect.Float32, reflect.Float64:
col := NewFloatColumn(fieldname, tagmap, isPointer)
return &col
case reflect.Map, reflect.Slice:
col := NewCompoundColumn(fieldname, tagmap, isPointer)
return &col
}
if fieldType.Implements(gotypes.ISerializableType) {
col := NewCompoundColumn(fieldname, tagmap, isPointer)
vendor/yunion.io/x/sqlchemy/column.go
Generated Vendored
-2
View File
@@ -19,7 +19,6 @@ import (
"strconv"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/gotypes"
"yunion.io/x/pkg/utils"
)
@@ -385,7 +384,6 @@ type SBaseCompoundColumn struct{}
func (c *SBaseCompoundColumn) ConvertFromString(str string) interface{} {
json, err := jsonutils.ParseString(str)
if err != nil {
log.Errorf("ParseString fail %s", err)
json = jsonutils.JSONNull
}
return json.String()
vendor/yunion.io/x/sqlchemy/functions.go
Generated Vendored
+1 -1
View File
@@ -73,7 +73,7 @@ func (ff *SFunctionFieldBase) Name() string {
// Label implementation of SFunctionFieldBase for IQueryField
func (ff *SFunctionFieldBase) Label(label string) IQueryField {
if len(label) > 0 && label != ff.alias {
if len(label) > 0 {
ff.alias = label
}
return ff
vendor/yunion.io/x/sqlchemy/insert_batch.go
Generated Vendored
+137
View File
@@ -0,0 +1,137 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package sqlchemy
import (
"bytes"
"fmt"
"reflect"
"strings"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/gotypes"
"yunion.io/x/pkg/util/reflectutils"
"yunion.io/x/pkg/util/timeutils"
)
const (
sqlLineLimit = 100
)
func (t *STableSpec) InsertBatch(dataList []interface{}) error {
var sql string
var fieldCount int
{
buffer := new(bytes.Buffer)
buffer.WriteString("INSERT INTO `")
buffer.WriteString(t.Name())
buffer.WriteString("` (")
headers := make([]string, 0)
format := make([]string, 0)
for _, col := range t.Columns() {
if col.IsAutoIncrement() {
continue
}
name := col.Name()
headers = append(headers, fmt.Sprintf("`%s`", name))
if col.IsCreatedAt() || col.IsUpdatedAt() {
if t.Database().backend.SupportMixedInsertVariables() {
format = append(format, t.Database().backend.CurrentUTCTimeStampString())
} else {
format = append(format, "?")
fieldCount++
}
continue
}
format = append(format, "?")
fieldCount++
}
buffer.WriteString(strings.Join(headers, ","))
buffer.WriteString(") VALUES ")
buffer.WriteString("(")
buffer.WriteString(strings.Join(format, ","))
buffer.WriteString(")")
sql = buffer.String()
if DEBUG_SQLCHEMY {
log.Debugf("batchInsert SQL: %s", buffer.String())
}
}
batchParams := make([][]interface{}, 0)
now := timeutils.UtcNow()
errs := make([]error, 0)
for i := range dataList {
v := dataList[i]
var params []interface{}
modelValue := reflect.Indirect(reflect.ValueOf(v))
beforeInsert(modelValue)
dataFields := reflectutils.FetchStructFieldValueSet(modelValue)
for _, col := range t.Columns() {
if col.IsAutoIncrement() {
continue
}
if col.IsCreatedAt() || col.IsUpdatedAt() {
if !t.Database().backend.SupportMixedInsertVariables() {
params = append(params, now)
}
continue
}
ov, find := dataFields.GetInterface(col.Name())
if !find || gotypes.IsNil(ov) || col.IsZero(ov) {
// empty column
if col.IsSupportDefault() && (len(col.Default()) > 0 || col.IsString()) {
params = append(params, col.ConvertFromString(col.Default()))
} else {
params = append(params, nil)
}
} else {
params = append(params, col.ConvertFromValue(ov))
}
}
if len(params) != fieldCount {
log.Errorf("expect %d got %d(%#v)", fieldCount, len(params), params)
}
batchParams = append(batchParams, params)
if len(batchParams) >= sqlLineLimit || (i+1) == len(dataList) {
results, err := t.Database().TxBatchExec(sql, batchParams)
if err != nil {
return errors.Wrap(err, "TxBatchExec")
}
for _, result := range results {
if result.Error != nil {
errs = append(errs, result.Error)
}
}
if len(errs) != 0 {
return errors.NewAggregate(errs)
}
batchParams = make([][]interface{}, 0)
}
}
return nil
}
vendor/yunion.io/x/sqlchemy/reflect.go
Generated Vendored
+2 -3
View File
@@ -22,7 +22,6 @@ import (
"time"
"yunion.io/x/jsonutils"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
"yunion.io/x/pkg/gotypes"
"yunion.io/x/pkg/tristate"
@@ -72,9 +71,9 @@ func GetStringValue(dat interface{}) string {
serializable, ok := value.Interface().(gotypes.ISerializable)
if ok {
return serializable.String()
} else {
return jsonutils.Marshal(value.Interface()).String()
}
log.Errorf("cannot convert %v to string", value)
return ""
}
func setValueBySQLString(value reflect.Value, val string) error {
vendor/yunion.io/x/sqlchemy/table.go
Generated Vendored
+1 -1
View File
@@ -353,7 +353,7 @@ func (c *STableField) Reference() string {
// Label implementation of STableField for IQueryField
func (c *STableField) Label(label string) IQueryField {
if len(label) > 0 && label != c.spec.Name() {
if len(label) > 0 {
c.alias = label
}
return c
vendor/yunion.io/x/sqlchemy/union.go
Generated Vendored
+1 -1
View File
@@ -53,7 +53,7 @@ func (sqf *SUnionQueryField) Reference() string {
// Label implementation of SUnionQueryField for IQueryField
func (sqf *SUnionQueryField) Label(label string) IQueryField {
if len(label) > 0 && label != sqf.name {
if len(label) > 0 {
sqf.alias = label
}
return sqf