From 68e28bfe491674d00898a315deaad373b869b6c2 Mon Sep 17 00:00:00 2001 From: Qu Xuan Date: Thu, 4 Mar 2021 15:38:38 +0800 Subject: [PATCH] fix: optimzed tag sync --- pkg/cloudcommon/db/standalone_anon.go | 20 +- pkg/compute/guestdrivers/managedvirtual.go | 29 +- pkg/compute/regiondrivers/managedvirtual.go | 38 ++- pkg/multicloud/aliyun/aliyun.go | 6 + pkg/multicloud/aliyun/dbinstance.go | 29 +- .../aliyun/elasticcache_instance.go | 30 +- pkg/multicloud/aliyun/instance.go | 86 ++---- pkg/multicloud/aliyun/loadbalancer.go | 27 +- pkg/multicloud/aliyun/region.go | 17 +- pkg/multicloud/aliyun/resource_tags.go | 287 +++++++----------- pkg/multicloud/aliyun/shell/resource_tag.go | 58 ++-- 11 files changed, 233 insertions(+), 394 deletions(-) diff --git a/pkg/cloudcommon/db/standalone_anon.go b/pkg/cloudcommon/db/standalone_anon.go index a67d922090..c63ae03492 100644 --- a/pkg/cloudcommon/db/standalone_anon.go +++ b/pkg/cloudcommon/db/standalone_anon.go @@ -16,7 +16,6 @@ package db import ( "context" - "reflect" "strings" "yunion.io/x/jsonutils" @@ -291,22 +290,11 @@ func (model *SStandaloneAnonResourceBase) SetCloudMetadataAll(ctx context.Contex if err != nil { return errors.Wrap(err, "SetAll") } - userTags, err := model.GetAllUserMetadata() - if err != nil { - return errors.Wrap(err, "model.GetAllUserMetadata()") + userTags := map[string]interface{}{} + for k, v := range dictstore { + userTags[strings.Replace(k, CLOUD_TAG_PREFIX, USER_TAG_PREFIX, 1)] = v } - cloudTags, err := model.GetAllCloudMetadata() - if err != nil { - return errors.Wrap(err, "model.GetAllCloudMetadata()") - } - if !reflect.DeepEqual(userTags, cloudTags) { - cloudTags2 := make(map[string]interface{}) - for k, v := range cloudTags { - cloudTags2[USER_TAG_PREFIX+k] = v - } - return model.SetUserMetadataAll(ctx, cloudTags2, userCred) - } - return nil + return Metadata.SetAll(ctx, model, userTags, userCred, USER_TAG_PREFIX) } func (model *SStandaloneAnonResourceBase) SetSysCloudMetadataAll(ctx context.Context, dictstore map[string]interface{}, userCred mcclient.TokenCredential) error { diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index 9ccc1e67bd..18f15b3d68 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -1235,21 +1235,20 @@ func (self *SManagedVirtualizedGuestDriver) RequestRemoteUpdate(ctx context.Cont } tags, err := guest.GetAllUserMetadata() if err != nil { - log.Errorf("GetAllUserMetadata fail %s", err) - } else { - tagsUpdateInfo := cloudprovider.TagsUpdateInfo{OldTags: oldTags, NewTags: tags} - err := iVM.SetTags(tags, replaceTags) - if err != nil { - logclient.AddSimpleActionLog(guest, logclient.ACT_UPDATE_TAGS, err, userCred, false) - return errors.Wrap(err, "iVM.SetMetadata") - } - logclient.AddSimpleActionLog(guest, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, true) - // sync back cloud metadata - iVM.Refresh() - err = models.SyncVirtualResourceMetadata(ctx, userCred, guest, iVM) - if err != nil { - return errors.Wrap(err, "syncVirtualResourceMetadata") - } + return errors.Wrapf(err, "GetAllUserMetadata") + } + tagsUpdateInfo := cloudprovider.TagsUpdateInfo{OldTags: oldTags, NewTags: tags} + err = iVM.SetTags(tags, replaceTags) + if err != nil { + logclient.AddSimpleActionLog(guest, logclient.ACT_UPDATE_TAGS, err, userCred, false) + return errors.Wrap(err, "iVM.SetTags") + } + logclient.AddSimpleActionLog(guest, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, true) + // sync back cloud metadata + iVM.Refresh() + err = models.SyncVirtualResourceMetadata(ctx, userCred, guest, iVM) + if err != nil { + return errors.Wrap(err, "syncVirtualResourceMetadata") } err = iVM.UpdateVM(ctx, guest.Name) diff --git a/pkg/compute/regiondrivers/managedvirtual.go b/pkg/compute/regiondrivers/managedvirtual.go index e46be7b9d2..7b3b02972b 100644 --- a/pkg/compute/regiondrivers/managedvirtual.go +++ b/pkg/compute/regiondrivers/managedvirtual.go @@ -335,17 +335,16 @@ func (self *SManagedVirtualizationRegionDriver) RequestRemoteUpdateLoadbalancer( return nil, errors.Wrap(err, "iLoadbalancer.GetTags()") } tags, err := lb.GetAllUserMetadata() - tagsUpdateInfo := cloudprovider.TagsUpdateInfo{OldTags: oldTags, NewTags: tags} if err != nil { - log.Errorf("GetAllUserMetadata fail %s", err) - } else { - err := iLoadbalancer.SetTags(tags, replaceTags) - if err != nil { - logclient.AddActionLogWithStartable(task, lb, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, false) - return nil, errors.Wrap(err, "iLoadbalancer.SetMetadata") - } - logclient.AddActionLogWithStartable(task, lb, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, true) + return nil, errors.Wrapf(err, "lb.GetAllUserMetadata") } + tagsUpdateInfo := cloudprovider.TagsUpdateInfo{OldTags: oldTags, NewTags: tags} + err = iLoadbalancer.SetTags(tags, replaceTags) + if err != nil { + logclient.AddActionLogWithStartable(task, lb, logclient.ACT_UPDATE_TAGS, err, userCred, false) + return nil, errors.Wrap(err, "iLoadbalancer.SetMetadata") + } + logclient.AddActionLogWithStartable(task, lb, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, true) return nil, nil }) return nil @@ -2632,8 +2631,8 @@ func (self *SManagedVirtualizationRegionDriver) RequestRemoteUpdateDBInstance(ct tagsUpdateInfo := cloudprovider.TagsUpdateInfo{OldTags: oldTags, NewTags: tags} err = iRds.SetTags(tags, replaceTags) if err != nil { - logclient.AddActionLogWithStartable(task, instance, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, false) - return nil, errors.Wrap(err, "iRds.SetMetadata") + logclient.AddActionLogWithStartable(task, instance, logclient.ACT_UPDATE_TAGS, err, userCred, false) + return nil, errors.Wrap(err, "iRds.SetTags") } logclient.AddActionLogWithStartable(task, instance, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, true) return nil, nil @@ -3025,17 +3024,16 @@ func (self *SManagedVirtualizationRegionDriver) RequestRemoteUpdateElasticcache( return nil, errors.Wrap(err, "iElasticcache.GetTags()") } tags, err := elasticcache.GetAllUserMetadata() - tagsUpdateInfo := cloudprovider.TagsUpdateInfo{OldTags: oldTags, NewTags: tags} if err != nil { - log.Errorf("GetAllUserMetadata fail %s", err) - } else { - err := iElasticcache.SetTags(tags, replaceTags) - if err != nil { - logclient.AddActionLogWithStartable(task, elasticcache, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, false) - return nil, errors.Wrap(err, "iElasticcache.SetMetadata") - } - logclient.AddActionLogWithStartable(task, elasticcache, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, true) + return nil, errors.Wrapf(err, "GetAllUserMetadata") } + tagsUpdateInfo := cloudprovider.TagsUpdateInfo{OldTags: oldTags, NewTags: tags} + err = iElasticcache.SetTags(tags, replaceTags) + if err != nil { + logclient.AddActionLogWithStartable(task, elasticcache, logclient.ACT_UPDATE_TAGS, err, userCred, false) + return nil, errors.Wrap(err, "iElasticcache.SetTags") + } + logclient.AddActionLogWithStartable(task, elasticcache, logclient.ACT_UPDATE_TAGS, tagsUpdateInfo, userCred, true) return nil, nil }) return nil diff --git a/pkg/multicloud/aliyun/aliyun.go b/pkg/multicloud/aliyun/aliyun.go index f7400d15ca..3dda2ac958 100644 --- a/pkg/multicloud/aliyun/aliyun.go +++ b/pkg/multicloud/aliyun/aliyun.go @@ -63,6 +63,12 @@ const ( ALIYUN_CBN_API_VERSION = "2017-09-12" ALIYUN_CDN_API_VERSION = "2018-05-10" ALIYUN_IMS_API_VERSION = "2019-08-15" + + ALIYUN_SERVICE_ECS = "ecs" + ALIYUN_SERVICE_VPC = "vpc" + ALIYUN_SERVICE_RDS = "rds" + ALIYUN_SERVICE_SLB = "slb" + ALIYUN_SERVICE_KVS = "kvs" ) var ( diff --git a/pkg/multicloud/aliyun/dbinstance.go b/pkg/multicloud/aliyun/dbinstance.go index c7fe466454..047442f331 100644 --- a/pkg/multicloud/aliyun/dbinstance.go +++ b/pkg/multicloud/aliyun/dbinstance.go @@ -708,7 +708,7 @@ func (region *SRegion) CreateIDBInstance(desc *cloudprovider.SManagedDBInstanceC if err != nil { return nil, errors.Wrap(err, `resp.GetString("DBInstanceId")`) } - region.SetResourceTags("rds", "INSTANCE", []string{instanceId}, desc.Tags, false) + region.SetResourceTags(ALIYUN_SERVICE_RDS, "INSTANCE", instanceId, desc.Tags, false) return region.GetIDBInstanceById(instanceId) } @@ -847,31 +847,18 @@ func (region *SRegion) RenewDBInstance(instanceId string, bc billing.SBillingCyc return err } -func (rds *SDBInstance) GetMetadata() *jsonutils.JSONDict { - data := jsonutils.NewDict() - tags, err := rds.region.ListResourceTags("rds", "INSTANCE", []string{rds.GetId()}) - if err != nil { - log.Errorf(`[err:%s]rds.region.FetchResourceTags("slb", "instance", []string{rds.GetId()})`, err.Error()) - return nil - } - if _, ok := tags[rds.GetId()]; !ok { - return nil - } - data.Update(jsonutils.Marshal(tags[rds.GetId()])) - return data -} - func (rds *SDBInstance) GetTags() (map[string]string, error) { - tags, err := rds.region.ListResourceTags("rds", "INSTANCE", []string{rds.GetId()}) + tags, err := rds.region.ListTags(ALIYUN_SERVICE_RDS, "INSTANCE", rds.GetId()) if err != nil { - return nil, errors.Wrap(err, `rds.region.ListResourceTags`) + return nil, errors.Wrap(err, `rds.region.ListTags`) } - if _, ok := tags[rds.GetId()]; !ok { - return nil, cloudprovider.ErrNotFound + ret := map[string]string{} + for _, tag := range tags { + ret[tag.TagKey] = tag.TagValue } - return *tags[rds.GetId()], nil + return ret, nil } func (rds *SDBInstance) SetTags(tags map[string]string, replace bool) error { - return rds.region.SetResourceTags("rds", "INSTANCE", []string{rds.GetId()}, tags, replace) + return rds.region.SetResourceTags(ALIYUN_SERVICE_RDS, "INSTANCE", rds.GetId(), tags, replace) } diff --git a/pkg/multicloud/aliyun/elasticcache_instance.go b/pkg/multicloud/aliyun/elasticcache_instance.go index b8b162b59b..b79b84d406 100644 --- a/pkg/multicloud/aliyun/elasticcache_instance.go +++ b/pkg/multicloud/aliyun/elasticcache_instance.go @@ -653,7 +653,7 @@ func (self *SRegion) CreateIElasticcaches(ec *cloudprovider.SCloudElasticCacheIn if err != nil { return nil, errors.Wrap(err, "region.CreateIElasticcaches") } - self.SetResourceTags("kvs", "INSTANCE", []string{ret.InstanceID}, ec.Tags, false) + self.SetResourceTags(ALIYUN_SERVICE_KVS, "INSTANCE", ret.InstanceID, ec.Tags, true) ret.region = self return ret, nil @@ -925,34 +925,20 @@ func (self *SElasticcache) GetICloudElasticcacheBackup(backupId string) (cloudpr return nil, cloudprovider.ErrNotFound } -func (instance *SElasticcache) GetMetadata() *jsonutils.JSONDict { - data := jsonutils.NewDict() - tags, err := instance.region.ListResourceTags("kvs", "INSTANCE", []string{instance.GetId()}) - if err != nil { - log.Errorf(`[err:%s]instance.region.FetchResourceTags("kvs", "instance", []string{instance.GetId()})`, err.Error()) - return nil - } - if _, ok := tags[instance.GetId()]; !ok { - return nil - } - data.Update(jsonutils.Marshal(tags[instance.GetId()])) - return data -} - func (instance *SElasticcache) GetTags() (map[string]string, error) { - tags, err := instance.region.ListResourceTags("kvs", "INSTANCE", []string{instance.GetId()}) + tags, err := instance.region.ListTags(ALIYUN_SERVICE_KVS, "INSTANCE", instance.GetId()) if err != nil { - return nil, errors.Wrap(err, "instance.region.ListResourceTags") + return nil, errors.Wrap(err, "instance.region.ListTags") } - if _, ok := tags[instance.GetId()]; !ok { - log.Debugf("SElasticcache.GetTags %s, tags %#v", instance.GetId(), tags) - return nil, cloudprovider.ErrNotFound + ret := map[string]string{} + for _, tag := range tags { + ret[tag.TagKey] = tag.TagValue } - return *tags[instance.GetId()], nil + return ret, nil } func (instance *SElasticcache) SetTags(tags map[string]string, replace bool) error { - return instance.region.SetResourceTags("kvs", "INSTANCE", []string{instance.GetId()}, tags, replace) + return instance.region.SetResourceTags(ALIYUN_SERVICE_KVS, "INSTANCE", instance.GetId(), tags, replace) } func (self *SElasticcache) UpdateSecurityGroups(secgroupIds []string) error { diff --git a/pkg/multicloud/aliyun/instance.go b/pkg/multicloud/aliyun/instance.go index d30ec42c7f..6e45200be2 100644 --- a/pkg/multicloud/aliyun/instance.go +++ b/pkg/multicloud/aliyun/instance.go @@ -129,6 +129,9 @@ type SInstance struct { VlanId string VpcAttributes SVpcAttributes ZoneId string + + sys map[string]string + user map[string]string } // {"AutoReleaseTime":"","ClusterId":"","Cpu":1,"CreationTime":"2018-05-23T07:58Z","DedicatedHostAttribute":{"DedicatedHostId":"","DedicatedHostName":""},"Description":"","DeviceAvailable":true,"EipAddress":{"AllocationId":"","InternetChargeType":"","IpAddress":""},"ExpiredTime":"2018-05-30T16:00Z","GPUAmount":0,"GPUSpec":"","HostName":"iZ2ze57isp1ali72tzkjowZ","ImageId":"centos_7_04_64_20G_alibase_201701015.vhd","InnerIpAddress":{"IpAddress":[]},"InstanceChargeType":"PrePaid","InstanceId":"i-2ze57isp1ali72tzkjow","InstanceName":"gaoxianqi-test-7days","InstanceNetworkType":"vpc","InstanceType":"ecs.t5-lc2m1.nano","InstanceTypeFamily":"ecs.t5","InternetChargeType":"PayByBandwidth","InternetMaxBandwidthIn":-1,"InternetMaxBandwidthOut":0,"IoOptimized":true,"Memory":512,"NetworkInterfaces":{"NetworkInterface":[{"MacAddress":"00:16:3e:10:f0:c9","NetworkInterfaceId":"eni-2zecqsagtpztl6x5hu2r","PrimaryIpAddress":"192.168.220.214"}]},"OSName":"CentOS 7.4 64位","OSType":"linux","OperationLocks":{"LockReason":[]},"PublicIpAddress":{"IpAddress":[]},"Recyclable":false,"RegionId":"cn-beijing","ResourceGroupId":"","SaleCycle":"Week","SecurityGroupIds":{"SecurityGroupId":["sg-2zecqsagtpztl6x9zynl"]},"SerialNumber":"df05d9b4-df3d-4400-88d1-5f843f0dd088","SpotPriceLimit":0.000000,"SpotStrategy":"NoSpot","StartTime":"2018-05-23T07:58Z","Status":"Running","StoppedMode":"Not-applicable","VlanId":"","VpcAttributes":{"NatIpAddress":"","PrivateIpAddress":{"IpAddress":["192.168.220.214"]},"VSwitchId":"vsw-2ze9cqwza4upoyujq1thd","VpcId":"vpc-2zer4jy8ix3i8f0coc5uw"},"ZoneId":"cn-beijing-f"} @@ -166,56 +169,9 @@ func (self *SRegion) GetInstances(zoneId string, ids []string, offset int, limit return instances, int(total), nil } -func (self *SRegion) fetchTags(resourceType string, resourceId string) (*jsonutils.JSONDict, error) { - // 资源类型。取值范围： - // disk - // instance - // image - // securitygroup - // snapshot - var page int64 = 1 - var pageSize int64 = 50 - params := make(map[string]string) - params["RegionId"] = self.RegionId - params["ResourceType"] = resourceType - params["ResourceId"] = resourceId - params["PageSize"] = fmt.Sprintf("%d", pageSize) - params["PageNumber"] = fmt.Sprintf("%d", page) - ret, err := self.ecsRequest("DescribeTags", params) - if err != nil { - return nil, err - } - - tags := jsonutils.NewDict() - result, _ := ret.GetArray("Tags", "Tag") - for _, item := range result { - k, _ := item.GetString("TagKey") - v, _ := item.Get("TagValue") - if len(k) > 0 { - tags.Set(k, v) - } - } - - total, _ := ret.Int("TotalCount") - for ; total > page*pageSize; page++ { - params["PageSize"] = fmt.Sprintf("%d", pageSize) - params["PageNumber"] = fmt.Sprintf("%d", page) - ret, err := self.ecsRequest("DescribeTags", params) - if err != nil { - return nil, err - } - - result, _ := ret.GetArray("Tags", "Tag") - for _, item := range result { - k, _ := item.GetString("TagKey") - v, _ := item.Get("TagValue") - if len(k) > 0 { - tags.Set(k, v) - } - } - } - - return tags, nil +type SAliyunTag struct { + TagKey string + TagValue string } func (self *SInstance) GetSecurityGroupIds() ([]string, error) { @@ -233,11 +189,11 @@ func (self *SInstance) GetMetadata() *jsonutils.JSONDict { priceKey := fmt.Sprintf("%s::%s::%s::%s::%s", self.RegionId, self.InstanceType, self.InstanceNetworkType, self.OSType, optimized) data.Add(jsonutils.NewString(priceKey), "price_key") - tags, err := self.host.zone.region.fetchTags("instance", self.InstanceId) + _, tags, err := self.fetchTags() if err != nil { log.Errorln(err) } - data.Update(tags) + data.Update(jsonutils.Marshal(tags)) data.Add(jsonutils.NewString(self.host.zone.GetGlobalId()), "zone_ext_id") if len(self.ImageId) > 0 { @@ -270,20 +226,31 @@ func (self *SInstance) GetSysTags() map[string]string { } } } + sys, _, _ := self.fetchTags() + for k, v := range sys { + data[k] = v + } return data } func (self *SInstance) GetTags() (map[string]string, error) { - tags, err := self.host.zone.region.fetchTags("instance", self.InstanceId) + _, tags, err := self.fetchTags() if err != nil { - return nil, errors.Wrap(err, "self.host.zone.region.fetchTags") + return nil, errors.Wrap(err, "self.fetchTags") } - data := map[string]string{} - err = tags.Unmarshal(&data) + return tags, nil +} + +func (self *SInstance) fetchTags() (map[string]string, map[string]string, error) { + if self.sys != nil || self.user != nil { + return self.sys, self.user, nil + } + var err error + self.sys, self.user, err = self.host.zone.region.ListSysAndUserTags(ALIYUN_SERVICE_ECS, "instance", self.InstanceId) if err != nil { - return nil, errors.Wrap(err, "tags.Unmarshal") + return nil, nil, errors.Wrapf(err, "ListSysAndUserTags") } - return data, nil + return self.sys, self.user, nil } func (self *SInstance) GetIHost() cloudprovider.ICloudHost { @@ -442,6 +409,7 @@ func (self *SInstance) GetStatus() string { } func (self *SInstance) Refresh() error { + self.sys, self.user = nil, nil new, err := self.host.zone.region.GetInstance(self.InstanceId) if err != nil { return err @@ -1143,7 +1111,7 @@ func (self *SInstance) SetAutoRenew(autoRenew bool) error { } func (self *SInstance) SetTags(tags map[string]string, replace bool) error { - return self.host.zone.region.SetResourceTags("ecs", "instance", []string{self.InstanceId}, tags, replace) + return self.host.zone.region.SetResourceTags(ALIYUN_SERVICE_ECS, "instance", self.InstanceId, tags, replace) } func (self *SRegion) SaveImage(instanceId string, opts *cloudprovider.SaveImageOptions) (*SImage, error) { diff --git a/pkg/multicloud/aliyun/loadbalancer.go b/pkg/multicloud/aliyun/loadbalancer.go index 43c702cdb9..2686add9c5 100644 --- a/pkg/multicloud/aliyun/loadbalancer.go +++ b/pkg/multicloud/aliyun/loadbalancer.go @@ -103,29 +103,16 @@ func (lb *SLoadbalancer) GetStatus() string { return api.LB_STATUS_DISABLED } -func (lb *SLoadbalancer) GetMetadata() *jsonutils.JSONDict { - data := jsonutils.NewDict() - tags, err := lb.region.ListResourceTags("slb", "instance", []string{lb.GetId()}) - if err != nil { - log.Errorf(`[err:%s]lb.region.FetchResourceTags("slb", "instance", []string{lb.GetId()})`, err.Error()) - return nil - } - if _, ok := tags[lb.GetId()]; !ok { - return nil - } - data.Update(jsonutils.Marshal(tags[lb.GetId()])) - return data -} - func (lb *SLoadbalancer) GetTags() (map[string]string, error) { - tags, err := lb.region.ListResourceTags("slb", "instance", []string{lb.GetId()}) + tags, err := lb.region.ListTags(ALIYUN_SERVICE_SLB, "instance", lb.GetId()) if err != nil { - return nil, errors.Wrap(err, "lb.region.ListResourceTags") + return nil, errors.Wrap(err, "lb.region.ListTags") } - if _, ok := tags[lb.GetId()]; !ok { - return nil, cloudprovider.ErrNotFound + ret := map[string]string{} + for _, tag := range tags { + ret[tag.TagKey] = tag.TagValue } - return *tags[lb.GetId()], nil + return ret, nil } func (lb *SLoadbalancer) GetAddress() string { @@ -424,7 +411,7 @@ func (lb *SLoadbalancer) GetProjectId() string { } func (lb *SLoadbalancer) SetTags(tags map[string]string, replace bool) error { - return lb.region.SetResourceTags("slb", "instance", []string{lb.LoadBalancerId}, tags, replace) + return lb.region.SetResourceTags(ALIYUN_SERVICE_SLB, "instance", lb.LoadBalancerId, tags, replace) } // mapping aliyun finance zoneId to aliyun finance ecs zoneId diff --git a/pkg/multicloud/aliyun/region.go b/pkg/multicloud/aliyun/region.go index 06d007c6db..c214a77aa3 100644 --- a/pkg/multicloud/aliyun/region.go +++ b/pkg/multicloud/aliyun/region.go @@ -166,21 +166,6 @@ func (self *SRegion) kvsRequest(action string, params map[string]string) (jsonut return jsonRequest(client, "r-kvstore.aliyuncs.com", ALIYUN_API_VERSION_KVS, action, params, self.client.debug) } -func (self *SRegion) tagRequest(serviceType string, action string, params map[string]string) (jsonutils.JSONObject, error) { - switch serviceType { - case "ecs": - return self.ecsRequest(action, params) - case "rds": - return self.rdsRequest(action, params) - case "slb": - return self.lbRequest(action, params) - case "kvs": - return self.kvsRequest(action, params) - default: - return nil, errors.Wrapf(errors.ErrNotSupported, "not support %service tag", serviceType) - } -} - type LBRegion struct { RegionEndpoint string RegionId string @@ -969,7 +954,7 @@ func (region *SRegion) CreateILoadBalancer(loadbalancer *cloudprovider.SLoadbala if err != nil { return nil, err } - region.SetResourceTags("slb", "instance", []string{loadBalancerID}, loadbalancer.Tags, false) + region.SetResourceTags(ALIYUN_SERVICE_SLB, "instance", loadBalancerID, loadbalancer.Tags, false) iLoadbalancer, err := region.GetLoadbalancerDetail(loadBalancerID) if err != nil { return nil, err diff --git a/pkg/multicloud/aliyun/resource_tags.go b/pkg/multicloud/aliyun/resource_tags.go index 8d41998243..568167070f 100644 --- a/pkg/multicloud/aliyun/resource_tags.go +++ b/pkg/multicloud/aliyun/resource_tags.go @@ -16,6 +16,7 @@ package aliyun import ( "fmt" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" @@ -23,222 +24,138 @@ import ( "yunion.io/x/onecloud/pkg/cloudprovider" ) -type STagResource struct { - ResourceType string `json:"ResourceType"` - TagValue string `json:"TagValue"` - ResourceID string `json:"ResourceId"` - TagKey string `json:"TagKey"` +func (self *SRegion) tagRequest(serviceType, action string, params map[string]string) (jsonutils.JSONObject, error) { + switch serviceType { + case ALIYUN_SERVICE_ECS: + return self.ecsRequest(action, params) + case ALIYUN_SERVICE_VPC: + return self.vpcRequest(action, params) + case ALIYUN_SERVICE_RDS: + return self.rdsRequest(action, params) + case ALIYUN_SERVICE_SLB: + return self.lbRequest(action, params) + case ALIYUN_SERVICE_KVS: + return self.kvsRequest(action, params) + default: + return nil, fmt.Errorf("invalid service type") + } } -func (self *SRegion) rawListTagResources(serviceType string, resourceType string, resIds []string, token string) ([]STagResource, string, error) { - if len(resIds) > 50 { - return nil, "", errors.Wrap(cloudprovider.ErrNotSupported, "resource count exceed 50 for one request") - } +// 资源类型。取值范围： +// disk, instance, image, securitygroup, snapshot +func (self *SRegion) ListTags(serviceType string, resourceType string, resourceId string) ([]SAliyunTag, error) { + tags := []SAliyunTag{} params := make(map[string]string) + params["RegionId"] = self.RegionId params["ResourceType"] = resourceType - for i := range resIds { - params[fmt.Sprintf("ResourceId.%d", i+1)] = resIds[i] - } - if len(token) != 0 { - params["NextToken"] = token - } - ret, err := self.tagRequest(serviceType, "ListTagResources", params) - if err != nil { - return nil, "", errors.Wrapf(err, `self.tagRequest(%s,"ListTagResources", %s)`, serviceType, jsonutils.Marshal(params).String()) - } - tagResources := []STagResource{} - err = ret.Unmarshal(&tagResources, "TagResources", "TagResource") - if err != nil { - return nil, "", errors.Wrapf(err, "(%s).Unmarshal(&tagResources)", ret.String()) - } - nextToken, _ := ret.GetString("NextToken") - return tagResources, nextToken, nil -} - -func splitStringSlice(resIds []string, stride int) [][]string { - result := [][]string{} - i := 0 - for i < len(resIds)/stride { - result = append(result, resIds[i*stride:i*stride+stride]) - i++ - } - remainder := len(resIds) % stride - if remainder != 0 { - result = append(result, resIds[i*stride:i*stride+remainder]) - } - return result -} - -func splitTags(tags map[string]string, stride int) []map[string]string { - tagsGroups := []map[string]string{} - tagsGroup := map[string]string{} - for k, v := range tags { - tagsGroup[k] = v - if len(tagsGroup) == stride { - tagsGroups = append(tagsGroups, tagsGroup) - tagsGroup = map[string]string{} + params["ResourceId.1"] = resourceId + params["PageSize"] = "50" + var page int = 1 + for { + params["PageNumber"] = fmt.Sprintf("%d", page) + resp, err := self.tagRequest(serviceType, "DescribeTags", params) + if err != nil { + return nil, errors.Wrapf(err, "%s DescribeTags %s", serviceType, params) } - } - if len(tagsGroup) > 0 { - tagsGroups = append(tagsGroups, tagsGroup) - } - return tagsGroups -} - -func (self *SRegion) ListResourceTags(serviceType string, resourceType string, resIds []string) (map[string]*map[string]string, error) { - tags := make(map[string]*map[string]string) - tagReources := []STagResource{} - nextToken := "" - resIdsGroups := splitStringSlice(resIds, 50) - for i := range resIdsGroups { - for { - _tagResource, nextToken, err := self.rawListTagResources(serviceType, resourceType, resIdsGroups[i], nextToken) - if err != nil { - return nil, errors.Wrapf(err, "self.rawListTagResources(%s,%s,%s)", resourceType, resIds, nextToken) - } - tagReources = append(tagReources, _tagResource...) - if len(_tagResource) == 0 || len(nextToken) == 0 { - break - } - + part := []SAliyunTag{} + err = resp.Unmarshal(&part, "Tags", "Tag") + if err != nil { + return nil, errors.Wrapf(err, "resp.Unmarshal") } - } - for _, r := range tagReources { - if tagMapPtr, ok := tags[r.ResourceID]; !ok { - tagMap := map[string]string{ - r.TagKey: r.TagValue, - } - tags[r.ResourceID] = &tagMap - } else { - tagMap := *tagMapPtr - tagMap[r.TagKey] = r.TagValue + tags = append(tags, part...) + total, _ := resp.Int("TotalCount") + if len(tags) >= int(total) { + break } + page++ } return tags, nil } -func (self *SRegion) rawTagResources(serviceType string, resourceType string, resIds []string, tags map[string]string) error { - if len(resIds) > 50 { - return errors.Wrap(cloudprovider.ErrNotSupported, "resource count exceed 50 for one request") +func (self *SRegion) UntagResource(serviceType string, resourceType string, resId string, keys []string) error { + if len(resId) == 0 || len(keys) == 0 { + return nil } + + params := map[string]string{ + "RegionId": self.RegionId, + "ResourceId.1": resId, + "ResourceType": resourceType, + } + for i, key := range keys { + params[fmt.Sprintf("TagKey.%d", i+1)] = key + } + + _, err := self.tagRequest(serviceType, "UntagResources", params) + return errors.Wrapf(err, "UntagResources %s", params) +} + +func (self *SRegion) SetResourceTags(serviceType string, resourceType string, resId string, tags map[string]string, replace bool) error { + err := self.TagResource(serviceType, resourceType, resId, tags) + if err != nil { + return errors.Wrapf(err, "TagResource") + } + if !replace { + return nil + } + _, _tags, err := self.ListSysAndUserTags(serviceType, resourceType, resId) + if err != nil { + return errors.Wrapf(err, "ListTags") + } + tagMaps := map[string]string{} + for k, v := range tags { + tagMaps[strings.ToLower(k)] = v + } + keys := []string{} + for k, _ := range _tags { + if _, ok := tagMaps[strings.ToLower(k)]; !ok { + keys = append(keys, k) + } + } + return self.UntagResource(serviceType, resourceType, resId, keys) +} + +func (self *SRegion) TagResource(serviceType string, resourceType string, resourceId string, tags map[string]string) error { if len(tags) > 20 { return errors.Wrap(cloudprovider.ErrNotSupported, "tags count exceed 20 for one request") } params := make(map[string]string) params["ResourceType"] = resourceType - for i := range resIds { - params[fmt.Sprintf("ResourceId.%d", i+1)] = resIds[i] - } + params["ResourceId.1"] = resourceId i := 0 for k, v := range tags { + if strings.HasPrefix(k, "aliyun") || + strings.HasPrefix(k, "acs:") || + strings.HasPrefix(k, "http://") || + strings.HasPrefix(k, "https://") || + strings.HasPrefix(v, "http://") || + strings.HasPrefix(v, "https://") || + strings.HasPrefix(v, "acs:") { + continue + } params[fmt.Sprintf("Tag.%d.Key", i+1)] = k params[fmt.Sprintf("Tag.%d.Value", i+1)] = v i++ } _, err := self.tagRequest(serviceType, "TagResources", params) if err != nil { - return errors.Wrapf(err, `self.tagRequest(%s,"TagResources", %s)`, serviceType, jsonutils.Marshal(params).String()) + return errors.Wrapf(err, "TagResources %s %s", resourceId, params) } return nil } -func (self *SRegion) TagResources(serviceType string, resourceType string, resIds []string, tags map[string]string) error { - if len(resIds) == 0 || len(tags) == 0 { - return nil - } - resIdsGroups := splitStringSlice(resIds, 50) - tagsGroups := splitTags(tags, 20) - for i := range resIdsGroups { - for j := range tagsGroups { - err := self.rawTagResources(serviceType, resourceType, resIdsGroups[i], tagsGroups[j]) - if err != nil { - return errors.Wrapf(err, "self.rawTagResources(resourceType, resIdsGroups[i], tagsGroups[i])") - } - } - } - return nil -} - -func (self *SRegion) rawUntagResources(serviceType string, resourceType string, resIds []string, tags []string) error { - if len(resIds) > 50 { - return errors.Wrap(cloudprovider.ErrNotSupported, "resource count exceed 50 for one request") - } - if len(tags) > 20 { - return errors.Wrap(cloudprovider.ErrNotSupported, "tags count exceed 20 for one request") - } - params := make(map[string]string) - params["ResourceType"] = resourceType - for i := range resIds { - params[fmt.Sprintf("ResourceId.%d", i+1)] = resIds[i] - } - for i := range tags { - params[fmt.Sprintf("TagKey.%d", i+1)] = tags[i] - } - _, err := self.tagRequest(serviceType, "UntagResources", params) +func (self *SRegion) ListSysAndUserTags(serviceType string, resourceType string, resourceId string) (map[string]string, map[string]string, error) { + tags, err := self.ListTags(serviceType, resourceType, resourceId) if err != nil { - return errors.Wrapf(err, `self.tagRequest(%s,"UntagResources", %s)`, serviceType, jsonutils.Marshal(params).String()) + return nil, nil, errors.Wrapf(err, "ListTags(%s, %s)", resourceType, resourceId) } - return nil -} - -func (self *SRegion) UntagResources(serviceType string, resourceType string, resIds []string, tags []string) error { - if len(resIds) == 0 || len(tags) == 0 { - return nil - } - resIdsGroups := splitStringSlice(resIds, 50) - tagsGroups := splitStringSlice(tags, 20) - for i := range resIdsGroups { - for j := range tagsGroups { - err := self.rawUntagResources(serviceType, resourceType, resIdsGroups[i], tagsGroups[j]) - if err != nil { - return errors.Wrapf(err, "self.rawTagResources(resourceType, resIdsGroups[i], tagsGroups[i])") - } + sys, user := map[string]string{}, map[string]string{} + for _, tag := range tags { + if strings.HasPrefix(tag.TagKey, "aliyun") || strings.HasPrefix(tag.TagKey, "acs:") { + sys[tag.TagKey] = tag.TagValue + continue } + user[tag.TagKey] = tag.TagValue } - return nil -} - -func (self *SRegion) SetResourceTags(serviceType string, resourceType string, resIds []string, tags map[string]string, replace bool) error { - oldTags, err := self.ListResourceTags(serviceType, resourceType, resIds) - if err != nil { - return errors.Wrapf(err, "self.ListResourceTags(%s,%s)", resourceType, resIds) - } - for i := range resIds { - _, ok := oldTags[resIds[i]] - if !ok { - err := self.TagResources(serviceType, resourceType, []string{resIds[i]}, tags) - if err != nil { - return errors.Wrap(err, "self.TagResources(resourceType, []string{resIds[i]}, tags)") - } - } else { - oldResourceTags := *oldTags[resIds[i]] - addTags := map[string]string{} - for k, v := range tags { - if _, ok := oldResourceTags[k]; !ok { - addTags[k] = v - } else { - if oldResourceTags[k] != v { - addTags[k] = v - } - } - } - delTags := []string{} - if replace { - for k := range oldResourceTags { - if _, ok := tags[k]; !ok { - delTags = append(delTags, k) - } - } - } - err := self.UntagResources(serviceType, resourceType, []string{resIds[i]}, delTags) - if err != nil { - return errors.Wrap(err, "self.UntagResources(resourceType, []string{resIds[i]}, delTags)") - } - err = self.TagResources(serviceType, resourceType, []string{resIds[i]}, addTags) - if err != nil { - return errors.Wrap(err, "self.TagResources(resourceType, []string{resIds[i]}, addTags)") - } - } - } - return nil + return sys, user, nil } diff --git a/pkg/multicloud/aliyun/shell/resource_tag.go b/pkg/multicloud/aliyun/shell/resource_tag.go index a816a97909..ad206a0a6d 100644 --- a/pkg/multicloud/aliyun/shell/resource_tag.go +++ b/pkg/multicloud/aliyun/shell/resource_tag.go @@ -15,45 +15,63 @@ package shell import ( - "fmt" "strings" + "yunion.io/x/pkg/errors" + "yunion.io/x/onecloud/pkg/multicloud/aliyun" "yunion.io/x/onecloud/pkg/util/shellutils" ) func init() { type TagGetOptions struct { - SERVICE string `help:"service, eg. ecs"` - RESTYPE string `help:"resource type, eg. instance"` - ID []string `help:"resource Id, eg. ins-123xxx"` + SERVICE string `help:"service, eg. ecs" choices:"ecs|kvs|rds|vpc|slb"` + RESTYPE string `help:"resource type, eg. instance"` + ID string `help:"resource Id, eg. ins-123xxx"` } - shellutils.R(&TagGetOptions{}, "tag-show", "show tag of a specific resource", func(cli *aliyun.SRegion, args *TagGetOptions) error { - tags, err := cli.ListResourceTags(args.SERVICE, args.RESTYPE, args.ID) + shellutils.R(&TagGetOptions{}, "tag-list", "List tag of a specific resource", func(cli *aliyun.SRegion, args *TagGetOptions) error { + tags, err := cli.ListTags(args.SERVICE, args.RESTYPE, args.ID) if err != nil { return err } - for id, tag := range tags { - fmt.Println(id, *tag) - } + printObject(tags) return nil }) + type TagOptions struct { + TagGetOptions + KEY string + VALUE string + } + shellutils.R(&TagOptions{}, "tag-resource", "set tags of a specific resource", func(cli *aliyun.SRegion, args *TagOptions) error { + return cli.TagResource(args.SERVICE, args.RESTYPE, args.ID, map[string]string{args.KEY: args.VALUE}) + }) + + type UnTagOptions struct { + TagGetOptions + KEY []string + } + + shellutils.R(&UnTagOptions{}, "untag-resource", "un tags of a specific resource", func(cli *aliyun.SRegion, args *UnTagOptions) error { + return cli.UntagResource(args.SERVICE, args.RESTYPE, args.ID, args.KEY) + }) + type TagSetOptions struct { TagGetOptions - Tag []string `help:"tag to set, key:value"` - Replace bool `help:"replace all tags"` + VALUES []string + Replace bool } + shellutils.R(&TagSetOptions{}, "tag-set", "set tags of a specific resource", func(cli *aliyun.SRegion, args *TagSetOptions) error { - tags := make(map[string]string) - for _, t := range args.Tag { - parts := strings.Split(t, ":") - tags[parts[0]] = parts[1] + tags := map[string]string{} + for _, value := range args.VALUES { + v := strings.Split(value, ":") + if len(v) != 2 { + return errors.Errorf("invalid tag %s", value) + } + tags[v[0]] = v[1] } - err := cli.SetResourceTags(args.SERVICE, args.RESTYPE, args.ID, tags, args.Replace) - if err != nil { - return err - } - return nil + return cli.SetResourceTags(args.SERVICE, args.RESTYPE, args.ID, tags, args.Replace) }) + }