diff --git a/pkg/apis/compute/cloudaccount.go b/pkg/apis/compute/cloudaccount.go index 2cbd9948ac..ceb5ef0d43 100644 --- a/pkg/apis/compute/cloudaccount.go +++ b/pkg/apis/compute/cloudaccount.go @@ -330,6 +330,9 @@ type CloudaccountDetail struct { ProxySetting proxyapi.SProxySetting `json:"proxy_setting"` ProjectMappingResourceInfo + + // 上次同步耗时 + LastSyncCost string } func (self CloudaccountDetail) GetMetricTags() map[string]string { @@ -529,6 +532,8 @@ type SyncRangeInput struct { Force bool `json:"force"` FullSync bool `json:"full_sync"` DeepSync bool `json:"deep_sync"` + // 极速模式(差量更新) + Xor bool `json:"xor"` Region []string `json:"region"` Zone []string `json:"zone"` diff --git a/pkg/apis/compute/cloudprovider.go b/pkg/apis/compute/cloudprovider.go index efa652d265..a2bd07a966 100644 --- a/pkg/apis/compute/cloudprovider.go +++ b/pkg/apis/compute/cloudprovider.go @@ -141,6 +141,9 @@ type CloudproviderDetails struct { ReadOnly bool `json:"read_only"` ProjectMappingResourceInfo + + // 上次同步耗时 + LastSyncCost string } // 云订阅输入参数 diff --git a/pkg/apis/compute/cloudproviderregion.go b/pkg/apis/compute/cloudproviderregion.go index dc93c8c7e6..f6c65b576c 100644 --- a/pkg/apis/compute/cloudproviderregion.go +++ b/pkg/apis/compute/cloudproviderregion.go @@ -38,6 +38,9 @@ type CloudproviderregionDetails struct { // 支持服务列表 Capabilities []string `json:"capabilities"` + + // 上次同步耗时 + LastSyncCost string `json:"last_sync_cost"` } type CloudproviderregionListInput struct { diff --git a/pkg/compute/guestdrivers/managedvirtual.go b/pkg/compute/guestdrivers/managedvirtual.go index b75ffd463d..0431349441 100644 --- a/pkg/compute/guestdrivers/managedvirtual.go +++ b/pkg/compute/guestdrivers/managedvirtual.go @@ -185,7 +185,7 @@ func (self *SManagedVirtualizedGuestDriver) RequestSaveImage(ctx context.Context if err != nil { return nil, errors.Wrapf(err, "iRegion.GetIStoragecacheById(%s)", caches[i].ExternalId) } - result := caches[i].SyncCloudImages(ctx, userCred, iStoragecache, region) + result := caches[i].SyncCloudImages(ctx, userCred, iStoragecache, region, true) log.Infof("sync cloud image for storagecache %s result: %s", caches[i].Name, result.Result()) } } diff --git a/pkg/compute/models/access_group_caches.go b/pkg/compute/models/access_group_caches.go index b525be565c..d02bbdb423 100644 --- a/pkg/compute/models/access_group_caches.go +++ b/pkg/compute/models/access_group_caches.go @@ -254,9 +254,15 @@ func (self *SAccessGroupCache) RealDelete(ctx context.Context, userCred mcclient return self.SStatusStandaloneResourceBase.Delete(ctx, userCred) } -func (self *SCloudregion) SyncAccessGroups(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, iAccessGroups []cloudprovider.ICloudAccessGroup) compare.SyncResult { - lockman.LockRawObject(ctx, self.Id, "access_groups") - defer lockman.ReleaseRawObject(ctx, self.Id, "access_groups") +func (self *SCloudregion) SyncAccessGroups( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + iAccessGroups []cloudprovider.ICloudAccessGroup, + xor bool, +) compare.SyncResult { + lockman.LockRawObject(ctx, self.Id, AccessGroupManager.Keyword()) + defer lockman.ReleaseRawObject(ctx, self.Id, AccessGroupManager.Keyword()) result := compare.SyncResult{} @@ -284,13 +290,15 @@ func (self *SCloudregion) SyncAccessGroups(ctx context.Context, userCred mcclien } result.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithAccessGroup(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithAccessGroup(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { err := provider.newFromCloudAccessGroup(ctx, userCred, self, added[i]) diff --git a/pkg/compute/models/app.go b/pkg/compute/models/app.go index 6a0551564d..6d8748105b 100644 --- a/pkg/compute/models/app.go +++ b/pkg/compute/models/app.go @@ -206,7 +206,13 @@ func (self *SCloudregion) GetApps(managerId string) ([]SApp, error) { return ret, nil } -func (self *SCloudregion) SyncApps(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudApp) compare.SyncResult { +func (self *SCloudregion) SyncApps( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudApp, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, AppManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) defer lockman.ReleaseRawObject(ctx, AppManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) result := compare.SyncResult{} @@ -237,14 +243,16 @@ func (self *SCloudregion) SyncApps(ctx context.Context, userCred mcclient.TokenC result.Delete() } - // sync with cloud app - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudApp(ctx, userCred, provider, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + // sync with cloud app + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudApp(ctx, userCred, provider, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } // new one diff --git a/pkg/compute/models/buckets.go b/pkg/compute/models/buckets.go index b5e2a7af67..e34edee49a 100644 --- a/pkg/compute/models/buckets.go +++ b/pkg/compute/models/buckets.go @@ -118,9 +118,9 @@ func (manager *SBucketManager) fetchBuckets(provider *SCloudprovider, region *SC return buckets, nil } -func (manager *SBucketManager) syncBuckets(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, buckets []cloudprovider.ICloudBucket) compare.SyncResult { - lockman.LockRawObject(ctx, "buckets", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "buckets", fmt.Sprintf("%s-%s", provider.Id, region.Id)) +func (manager *SBucketManager) syncBuckets(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, buckets []cloudprovider.ICloudBucket, xor bool) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) syncResult := compare.SyncResult{} @@ -149,12 +149,14 @@ func (manager *SBucketManager) syncBuckets(ctx context.Context, userCred mcclien syncResult.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudBucket(ctx, userCred, commonext[i], provider, false) - if err != nil { - syncResult.UpdateError(err) - } else { - syncResult.Update() + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithCloudBucket(ctx, userCred, commonext[i], provider, false) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } } } for i := 0; i < len(added); i += 1 { diff --git a/pkg/compute/models/cdn_domains.go b/pkg/compute/models/cdn_domains.go index d7695c632f..5c1973ec30 100644 --- a/pkg/compute/models/cdn_domains.go +++ b/pkg/compute/models/cdn_domains.go @@ -129,7 +129,7 @@ func (self *SCloudprovider) GetCDNDomains() ([]SCDNDomain, error) { return domains, nil } -func (self *SCloudprovider) SyncCDNDomains(ctx context.Context, userCred mcclient.TokenCredential, exts []cloudprovider.ICloudCDNDomain) compare.SyncResult { +func (self *SCloudprovider) SyncCDNDomains(ctx context.Context, userCred mcclient.TokenCredential, exts []cloudprovider.ICloudCDNDomain, xor bool) compare.SyncResult { lockman.LockRawObject(ctx, CDNDomainManager.Keyword(), self.Id) defer lockman.ReleaseRawObject(ctx, CDNDomainManager.Keyword(), self.Id) @@ -160,13 +160,15 @@ func (self *SCloudprovider) SyncCDNDomains(ctx context.Context, userCred mcclien } result.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudCDNDomain(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudCDNDomain(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { _, err := self.newFromCloudCDNDomain(ctx, userCred, added[i]) diff --git a/pkg/compute/models/cloudaccounts.go b/pkg/compute/models/cloudaccounts.go index 1f5cbbadf6..ff62360598 100644 --- a/pkg/compute/models/cloudaccounts.go +++ b/pkg/compute/models/cloudaccounts.go @@ -1283,6 +1283,9 @@ func (manager *SCloudaccountManager) FetchCustomizeColumns( EnabledStatusInfrasResourceBaseDetails: stdRows[i], ProjectMappingResourceInfo: pmRows[i], } + if !account.LastSyncEndAt.IsZero() && !account.LastSync.IsZero() { + detail.LastSyncCost = account.LastSyncEndAt.Sub(account.LastSync).Round(time.Second).String() + } if proxySetting, ok := proxySettings[account.ProxySettingId]; ok { detail.ProxySetting.Id = proxySetting.Id detail.ProxySetting.Name = proxySetting.Name @@ -2752,7 +2755,7 @@ func (self *SCloudaccount) GetDnsZoneCaches() ([]SDnsZoneCache, error) { return caches, nil } -func (self *SCloudaccount) SyncDnsZones(ctx context.Context, userCred mcclient.TokenCredential, dnsZones []cloudprovider.ICloudDnsZone) ([]SDnsZone, []cloudprovider.ICloudDnsZone, compare.SyncResult) { +func (self *SCloudaccount) SyncDnsZones(ctx context.Context, userCred mcclient.TokenCredential, dnsZones []cloudprovider.ICloudDnsZone, xor bool) ([]SDnsZone, []cloudprovider.ICloudDnsZone, compare.SyncResult) { lockman.LockRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-dnszone", self.Id)) defer lockman.ReleaseRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-dnszone", self.Id)) @@ -2790,10 +2793,12 @@ func (self *SCloudaccount) SyncDnsZones(ctx context.Context, userCred mcclient.T } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudDnsZone(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(errors.Wrapf(err, "SyncWithCloudDnsZone")) - continue + if !xor { + err = commondb[i].SyncWithCloudDnsZone(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(errors.Wrapf(err, "SyncWithCloudDnsZone")) + continue + } } zone, err := commondb[i].GetDnsZone() if err != nil { diff --git a/pkg/compute/models/cloudimages.go b/pkg/compute/models/cloudimages.go index 016387589f..b0d785c077 100644 --- a/pkg/compute/models/cloudimages.go +++ b/pkg/compute/models/cloudimages.go @@ -99,7 +99,7 @@ func SyncPublicCloudImages(ctx context.Context, userCred mcclient.TokenCredentia continue } - err = regions[i].SyncCloudImages(ctx, userCred, !isStart) + err = regions[i].SyncCloudImages(ctx, userCred, !isStart, false) if err != nil { log.Errorf("SyncCloudImages for region %s(%s) error: %v", regions[i].Name, regions[i].Id, err) continue diff --git a/pkg/compute/models/cloudprovider_quotas.go b/pkg/compute/models/cloudprovider_quotas.go index 8e0617aecb..37089fb80a 100644 --- a/pkg/compute/models/cloudprovider_quotas.go +++ b/pkg/compute/models/cloudprovider_quotas.go @@ -196,7 +196,7 @@ func (manager *SCloudproviderQuotaManager) GetQuotas(provider *SCloudprovider, r return quotas, nil } -func (manager *SCloudproviderQuotaManager) SyncQuotas(ctx context.Context, userCred mcclient.TokenCredential, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, region *SCloudregion, quotaRange string, iQuotas []cloudprovider.ICloudQuota) compare.SyncResult { +func (manager *SCloudproviderQuotaManager) SyncQuotas(ctx context.Context, userCred mcclient.TokenCredential, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, region *SCloudregion, quotaRange string, iQuotas []cloudprovider.ICloudQuota, xor bool) compare.SyncResult { key := provider.Id if region != nil { key = fmt.Sprintf("%s-%s", key, region.Id) @@ -230,13 +230,15 @@ func (manager *SCloudproviderQuotaManager) SyncQuotas(ctx context.Context, userC } } - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudQuota(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudQuota(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/cloudproviderregions.go b/pkg/compute/models/cloudproviderregions.go index 4eb4fbceea..cda51115b2 100644 --- a/pkg/compute/models/cloudproviderregions.go +++ b/pkg/compute/models/cloudproviderregions.go @@ -128,7 +128,11 @@ func (manager *SCloudproviderregionManager) FetchCustomizeColumns( rows[i].JointResourceBaseDetails = jointRows[i] rows[i].CloudregionResourceInfo = regionRows[i] rows[i].Capabilities, _ = objs[i].(*SCloudproviderregion).getCapabilities() - managerIds[i] = objs[i].(*SCloudproviderregion).CloudproviderId + cpr := objs[i].(*SCloudproviderregion) + managerIds[i] = cpr.CloudproviderId + if !cpr.LastSync.IsZero() && !cpr.LastSyncEndAt.IsZero() { + rows[i].LastSyncCost = cpr.LastSyncEndAt.Sub(cpr.LastSync).Round(time.Second).String() + } } managers := make(map[string]SCloudprovider) @@ -436,7 +440,7 @@ func (self *SCloudproviderregion) DoSync(ctx context.Context, userCred mcclient. syncRange.DeepSync = true } } - log.Debugf("need to do deep sync? ... %v", syncRange.DeepSync) + log.Debugf("need to do deep sync? ... %v, xor? ... %v", syncRange.DeepSync, syncRange.Xor) if localRegion.isManaged() { remoteRegion, err := driver.GetIRegionById(localRegion.ExternalId) diff --git a/pkg/compute/models/cloudproviders.go b/pkg/compute/models/cloudproviders.go index d99cb14754..58ea627b11 100644 --- a/pkg/compute/models/cloudproviders.go +++ b/pkg/compute/models/cloudproviders.go @@ -1068,6 +1068,9 @@ func (manager *SCloudproviderManager) FetchCustomizeColumns( SyncStatus2: provider.getSyncStatus2(), ProjectMappingResourceInfo: pmRows[i], } + if !provider.LastSync.IsZero() && !provider.LastSyncEndAt.IsZero() { + rows[i].LastSyncCost = provider.LastSyncEndAt.Sub(provider.LastSync).Round(time.Second).String() + } capabilities, _ := CloudproviderCapabilityManager.getCapabilities(provider.Id) if len(capabilities) > 0 { rows[i].Capabilities = capabilities @@ -1986,7 +1989,7 @@ func (self *SCloudprovider) GetInterVpcNetworks() ([]SInterVpcNetwork, error) { } -func (self *SCloudprovider) SyncInterVpcNetwork(ctx context.Context, userCred mcclient.TokenCredential, interVpcNetworks []cloudprovider.ICloudInterVpcNetwork) ([]SInterVpcNetwork, []cloudprovider.ICloudInterVpcNetwork, compare.SyncResult) { +func (self *SCloudprovider) SyncInterVpcNetwork(ctx context.Context, userCred mcclient.TokenCredential, interVpcNetworks []cloudprovider.ICloudInterVpcNetwork, xor bool) ([]SInterVpcNetwork, []cloudprovider.ICloudInterVpcNetwork, compare.SyncResult) { lockman.LockRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-interVpcNetwork", self.Id)) defer lockman.ReleaseRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-interVpcNetwork", self.Id)) @@ -2022,10 +2025,12 @@ func (self *SCloudprovider) SyncInterVpcNetwork(ctx context.Context, userCred mc } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudInterVpcNetwork(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(errors.Wrapf(err, "SyncWithCloudInterVpcNetwork")) - continue + if !xor { + err = commondb[i].SyncWithCloudInterVpcNetwork(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(errors.Wrapf(err, "SyncWithCloudInterVpcNetwork")) + continue + } } localNetworks = append(localNetworks, commondb[i]) remoteNetworks = append(remoteNetworks, commonext[i]) diff --git a/pkg/compute/models/cloudregions.go b/pkg/compute/models/cloudregions.go index 1ebc33f808..deb7de3c57 100644 --- a/pkg/compute/models/cloudregions.go +++ b/pkg/compute/models/cloudregions.go @@ -1002,9 +1002,9 @@ func (self *SCloudregion) GetSystemImageCount() (int, error) { return q.CountWithError() } -func (self *SCloudregion) SyncCloudImages(ctx context.Context, userCred mcclient.TokenCredential, refresh bool) error { - lockman.LockRawObject(ctx, "cloudimages", self.Id) - defer lockman.ReleaseRawObject(ctx, "cloudimages", self.Id) +func (self *SCloudregion) SyncCloudImages(ctx context.Context, userCred mcclient.TokenCredential, refresh, xor bool) error { + lockman.LockRawObject(ctx, CloudimageManager.Keyword(), self.Id) + defer lockman.ReleaseRawObject(ctx, CloudimageManager.Keyword(), self.Id) systemImageCount, err := self.GetSystemImageCount() if err != nil { @@ -1047,13 +1047,15 @@ func (self *SCloudregion) SyncCloudImages(ctx context.Context, userCred mcclient result.Delete() } - for i := 0; i < len(commonext); i++ { - err := commondb[i].syncWithImage(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(errors.Wrapf(err, "updateCachedImage")) - continue + if !xor { + for i := 0; i < len(commonext); i++ { + err := commondb[i].syncWithImage(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(errors.Wrapf(err, "updateCachedImage")) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index 299a153db3..2fb6d332a7 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -72,8 +72,8 @@ type sStoragecacheSyncPair struct { isNew bool } -func (pair *sStoragecacheSyncPair) syncCloudImages(ctx context.Context, userCred mcclient.TokenCredential) compare.SyncResult { - return pair.local.SyncCloudImages(ctx, userCred, pair.remote, pair.region) +func (pair *sStoragecacheSyncPair) syncCloudImages(ctx context.Context, userCred mcclient.TokenCredential, xor bool) compare.SyncResult { + return pair.local.SyncCloudImages(ctx, userCred, pair.remote, pair.region, xor) } func isInCache(pairs []sStoragecacheSyncPair, localCacheId string) bool { @@ -86,7 +86,7 @@ func isInCache(pairs []sStoragecacheSyncPair, localCacheId string) bool { return false } -func syncRegionQuotas(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, driver cloudprovider.ICloudProvider, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncRegionQuotas(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, driver cloudprovider.ICloudProvider, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, xor bool) error { quotas, err := func() ([]cloudprovider.ICloudQuota, error) { defer syncResults.AddRequestCost(CloudproviderQuotaManager)() return remoteRegion.GetICloudQuotas() @@ -98,7 +98,7 @@ func syncRegionQuotas(ctx context.Context, userCred mcclient.TokenCredential, sy } result := func() compare.SyncResult { defer syncResults.AddSqlCost(CloudproviderQuotaManager)() - return CloudproviderQuotaManager.SyncQuotas(ctx, userCred, provider.GetOwnerId(), provider, localRegion, api.CLOUD_PROVIDER_QUOTA_RANGE_CLOUDREGION, quotas) + return CloudproviderQuotaManager.SyncQuotas(ctx, userCred, provider.GetOwnerId(), provider, localRegion, api.CLOUD_PROVIDER_QUOTA_RANGE_CLOUDREGION, quotas, xor) }() syncResults.Add(CloudproviderQuotaManager, result) msg := result.Result() @@ -110,7 +110,7 @@ func syncRegionQuotas(ctx context.Context, userCred mcclient.TokenCredential, sy return nil } -func syncRegionZones(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) ([]SZone, []cloudprovider.ICloudZone, error) { +func syncRegionZones(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, xor bool) ([]SZone, []cloudprovider.ICloudZone, error) { zones, err := func() ([]cloudprovider.ICloudZone, error) { defer syncResults.AddRequestCost(ZoneManager)() return remoteRegion.GetIZones() @@ -122,7 +122,7 @@ func syncRegionZones(ctx context.Context, userCred mcclient.TokenCredential, syn } localZones, remoteZones, result := func() ([]SZone, []cloudprovider.ICloudZone, compare.SyncResult) { defer syncResults.AddSqlCost(ZoneManager)() - return ZoneManager.SyncZones(ctx, userCred, localRegion, zones) + return ZoneManager.SyncZones(ctx, userCred, localRegion, zones, xor) }() syncResults.Add(ZoneManager, result) msg := result.Result() @@ -132,11 +132,10 @@ func syncRegionZones(ctx context.Context, userCred mcclient.TokenCredential, syn return nil, nil, fmt.Errorf(msg) } db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) return localZones, remoteZones, nil } -func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, localRegion *SCloudregion) { +func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, localRegion *SCloudregion, xor bool) { if localRegion == nil { log.Debugf("local region is nil, skipp...") return @@ -156,7 +155,7 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca if cnt == 0 { // 提前同步instance type.如果同步失败可能导致vm 内存显示为0 - if ret := SyncServerSkusByRegion(ctx, userCred, localRegion, nil); ret.IsError() { + if ret := SyncServerSkusByRegion(ctx, userCred, localRegion, nil, xor); ret.IsError() { msg := fmt.Sprintf("Get Skus for region %s failed %s", localRegion.GetName(), ret.Result()) log.Errorln(msg) // 暂时不终止同步 @@ -173,12 +172,20 @@ func syncRegionSkus(ctx context.Context, userCred mcclient.TokenCredential, loca } if cnt == 0 { - SyncElasticCacheSkusByRegion(ctx, userCred, localRegion) + SyncElasticCacheSkusByRegion(ctx, userCred, localRegion, xor) } } } -func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionEips( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { eips, err := func() ([]cloudprovider.ICloudEIP, error) { defer syncResults.AddRequestCost(ElasticipManager)() return remoteRegion.GetIEips() @@ -191,7 +198,7 @@ func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, sync result := func() compare.SyncResult { defer syncResults.AddSqlCost(ElasticipManager)() - return ElasticipManager.SyncEips(ctx, userCred, provider, localRegion, eips, provider.GetOwnerId()) + return ElasticipManager.SyncEips(ctx, userCred, provider, localRegion, eips, provider.GetOwnerId(), syncRange.Xor) }() syncResults.Add(ElasticipManager, result) @@ -201,10 +208,9 @@ func syncRegionEips(ctx context.Context, userCred mcclient.TokenCredential, sync if result.IsError() { return } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) } -func syncRegionBuckets(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) { +func syncRegionBuckets(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, xor bool) { buckets, err := func() ([]cloudprovider.ICloudBucket, error) { defer syncResults.AddRequestCost(BucketManager)() return remoteRegion.GetIBuckets() @@ -217,7 +223,7 @@ func syncRegionBuckets(ctx context.Context, userCred mcclient.TokenCredential, s result := func() compare.SyncResult { defer syncResults.AddSqlCost(BucketManager)() - return BucketManager.syncBuckets(ctx, userCred, provider, localRegion, buckets) + return BucketManager.syncBuckets(ctx, userCred, provider, localRegion, buckets, xor) }() syncResults.Add(BucketManager, result) @@ -229,10 +235,17 @@ func syncRegionBuckets(ctx context.Context, userCred mcclient.TokenCredential, s return } db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) } -func syncRegionVPCs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionVPCs( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { vpcs, err := func() ([]cloudprovider.ICloudVpc, error) { defer syncResults.AddRequestCost(VpcManager)() return remoteRegion.GetIVpcs() @@ -245,7 +258,7 @@ func syncRegionVPCs(ctx context.Context, userCred mcclient.TokenCredential, sync localVpcs, remoteVpcs, result := func() ([]SVpc, []cloudprovider.ICloudVpc, compare.SyncResult) { defer syncResults.AddSqlCost(VpcManager)() - return VpcManager.SyncVPCs(ctx, userCred, provider, localRegion, vpcs) + return VpcManager.SyncVPCs(ctx, userCred, provider, localRegion, vpcs, syncRange.Xor) }() syncResults.Add(VpcManager, result) @@ -257,7 +270,6 @@ func syncRegionVPCs(ctx context.Context, userCred mcclient.TokenCredential, sync return } db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) globalVpcIds := []string{} for j := 0; j < len(localVpcs); j += 1 { func() { @@ -301,13 +313,21 @@ func syncRegionAccessGroups(ctx context.Context, userCred mcclient.TokenCredenti result := func() compare.SyncResult { defer syncResults.AddSqlCost(AccessGroupManager)() - return localRegion.SyncAccessGroups(ctx, userCred, provider, accessGroups) + return localRegion.SyncAccessGroups(ctx, userCred, provider, accessGroups, syncRange.Xor) }() syncResults.Add(AccessGroupCacheManager, result) log.Infof("Sync Access Group Caches for region %s result: %s", localRegion.Name, result.Result()) } -func syncRegionFileSystems(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionFileSystems( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { filesystems, err := func() ([]cloudprovider.ICloudFileSystem, error) { defer syncResults.AddRequestCost(FileSystemManager)() return remoteRegion.GetICloudFileSystems() @@ -322,7 +342,7 @@ func syncRegionFileSystems(ctx context.Context, userCred mcclient.TokenCredentia localFSs, removeFSs, result := func() ([]SFileSystem, []cloudprovider.ICloudFileSystem, compare.SyncResult) { defer syncResults.AddSqlCost(FileSystemManager)() - return localRegion.SyncFileSystems(ctx, userCred, provider, filesystems) + return localRegion.SyncFileSystems(ctx, userCred, provider, filesystems, syncRange.Xor) }() syncResults.Add(FileSystemManager, result) log.Infof("Sync FileSystem for region %s result: %s", localRegion.Name, result.Result()) @@ -337,12 +357,18 @@ func syncRegionFileSystems(ctx context.Context, userCred mcclient.TokenCredentia return } - syncFileSystemMountTargets(ctx, userCred, &localFSs[j], removeFSs[j]) + syncFileSystemMountTargets(ctx, userCred, &localFSs[j], removeFSs[j], syncRange.Xor) }() } } -func syncFileSystemMountTargets(ctx context.Context, userCred mcclient.TokenCredential, localFs *SFileSystem, remoteFs cloudprovider.ICloudFileSystem) { +func syncFileSystemMountTargets( + ctx context.Context, + userCred mcclient.TokenCredential, + localFs *SFileSystem, + remoteFs cloudprovider.ICloudFileSystem, + xor bool, +) { mountTargets, err := remoteFs.GetMountTargets() if err != nil { if errors.Cause(err) == cloudprovider.ErrNotImplemented || errors.Cause(err) == cloudprovider.ErrNotSupported { @@ -351,11 +377,19 @@ func syncFileSystemMountTargets(ctx context.Context, userCred mcclient.TokenCred log.Errorf("GetMountTargets for %s error: %v", localFs.Name, err) return } - result := localFs.SyncMountTargets(ctx, userCred, mountTargets) + result := localFs.SyncMountTargets(ctx, userCred, mountTargets, xor) log.Infof("SyncMountTargets for FileSystem %s result: %s", localFs.Name, result.Result()) } -func syncVpcPeerConnections(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localVpc *SVpc, remoteVpc cloudprovider.ICloudVpc, syncRange *SSyncRange) { +func syncVpcPeerConnections( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localVpc *SVpc, + remoteVpc cloudprovider.ICloudVpc, + syncRange *SSyncRange, +) { peerConnections, err := func() ([]cloudprovider.ICloudVpcPeeringConnection, error) { defer syncResults.AddRequestCost(VpcPeeringConnectionManager)() return remoteVpc.GetICloudVpcPeeringConnections() @@ -370,7 +404,7 @@ func syncVpcPeerConnections(ctx context.Context, userCred mcclient.TokenCredenti result := func() compare.SyncResult { defer syncResults.AddSqlCost(VpcPeeringConnectionManager)() - return localVpc.SyncVpcPeeringConnections(ctx, userCred, peerConnections) + return localVpc.SyncVpcPeeringConnections(ctx, userCred, peerConnections, syncRange.Xor) }() syncResults.Add(VpcPeeringConnectionManager, result) @@ -410,7 +444,7 @@ func syncVpcSecGroup(ctx context.Context, userCred mcclient.TokenCredential, syn _, _, result := func() ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { defer syncResults.AddSqlCost(SecurityGroupManager)() - return SecurityGroupCacheManager.SyncSecurityGroupCaches(ctx, userCred, provider, secgroups, localVpc) + return SecurityGroupCacheManager.SyncSecurityGroupCaches(ctx, userCred, provider, secgroups, localVpc, syncRange.Xor) }() syncResults.Add(SecurityGroupCacheManager, result) @@ -434,7 +468,7 @@ func syncVpcRouteTables(ctx context.Context, userCred mcclient.TokenCredential, } localRouteTables, remoteRouteTables, result := func() ([]SRouteTable, []cloudprovider.ICloudRouteTable, compare.SyncResult) { defer syncResults.AddSqlCost(RouteTableManager)() - return RouteTableManager.SyncRouteTables(ctx, userCred, localVpc, routeTables, provider) + return RouteTableManager.SyncRouteTables(ctx, userCred, localVpc, routeTables, provider, syncRange.Xor) }() syncResults.Add(RouteTableManager, result) @@ -453,8 +487,8 @@ func syncVpcRouteTables(ctx context.Context, userCred mcclient.TokenCredential, if localRouteTables[i].Deleted { return } - localRouteTables[i].SyncRouteTableRouteSets(ctx, userCred, remoteRouteTables[i], provider) - localRouteTables[i].SyncRouteTableAssociations(ctx, userCred, remoteRouteTables[i], provider) + localRouteTables[i].SyncRouteTableRouteSets(ctx, userCred, remoteRouteTables[i], provider, syncRange.Xor) + localRouteTables[i].SyncRouteTableAssociations(ctx, userCred, remoteRouteTables[i], provider, syncRange.Xor) }() } } @@ -471,7 +505,7 @@ func syncIPv6Gateways(ctx context.Context, userCred mcclient.TokenCredential, sy } result := func() compare.SyncResult { defer syncResults.AddSqlCost(IPv6GatewayManager)() - return localVpc.SyncIPv6Gateways(ctx, userCred, exts, provider) + return localVpc.SyncIPv6Gateways(ctx, userCred, exts, provider, syncRange.Xor) }() syncResults.Add(IPv6GatewayManager, result) @@ -499,7 +533,7 @@ func syncVpcNatgateways(ctx context.Context, userCred mcclient.TokenCredential, } localNatGateways, remoteNatGateways, result := func() ([]SNatGateway, []cloudprovider.ICloudNatGateway, compare.SyncResult) { defer syncResults.AddSqlCost(NatGatewayManager)() - return NatGatewayManager.SyncNatGateways(ctx, userCred, provider.GetOwnerId(), provider, localVpc, natGateways) + return NatGatewayManager.SyncNatGateways(ctx, userCred, provider.GetOwnerId(), provider, localVpc, natGateways, syncRange.Xor) }() syncResults.Add(NatGatewayManager, result) @@ -521,8 +555,8 @@ func syncVpcNatgateways(ctx context.Context, userCred mcclient.TokenCredential, } syncNatGatewayEips(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i]) - syncNatDTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i]) - syncNatSTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i]) + syncNatDTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i], syncRange.Xor) + syncNatSTable(ctx, userCred, provider, &localNatGateways[i], remoteNatGateways[i], syncRange.Xor) }() } } @@ -543,14 +577,21 @@ func syncNatGatewayEips(ctx context.Context, userCred mcclient.TokenCredential, db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) } -func syncNatDTable(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, localNatGateway *SNatGateway, remoteNatGateway cloudprovider.ICloudNatGateway) { +func syncNatDTable( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + localNatGateway *SNatGateway, + remoteNatGateway cloudprovider.ICloudNatGateway, + xor bool, +) { dtable, err := remoteNatGateway.GetINatDTable() if err != nil { msg := fmt.Sprintf("GetINatDTable for NatGateway %s failed %s", remoteNatGateway.GetName(), err) log.Errorf(msg) return } - result := NatDEntryManager.SyncNatDTable(ctx, userCred, provider, localNatGateway, dtable) + result := NatDEntryManager.SyncNatDTable(ctx, userCred, provider, localNatGateway, dtable, xor) msg := result.Result() log.Infof("SyncNatDTable for NatGateway %s result: %s", localNatGateway.Name, msg) if result.IsError() { @@ -559,14 +600,21 @@ func syncNatDTable(ctx context.Context, userCred mcclient.TokenCredential, provi db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) } -func syncNatSTable(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, localNatGateway *SNatGateway, remoteNatGateway cloudprovider.ICloudNatGateway) { +func syncNatSTable( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + localNatGateway *SNatGateway, + remoteNatGateway cloudprovider.ICloudNatGateway, + xor bool, +) { stable, err := remoteNatGateway.GetINatSTable() if err != nil { msg := fmt.Sprintf("GetINatSTable for NatGateway %s failed %s", remoteNatGateway.GetName(), err) log.Errorf(msg) return } - result := NatSEntryManager.SyncNatSTable(ctx, userCred, provider, localNatGateway, stable) + result := NatSEntryManager.SyncNatSTable(ctx, userCred, provider, localNatGateway, stable, xor) msg := result.Result() log.Infof("SyncNatSTable for NatGateway %s result: %s", localNatGateway.Name, msg) if result.IsError() { @@ -595,7 +643,7 @@ func syncVpcWires(ctx context.Context, userCred mcclient.TokenCredential, syncRe syncResults.AddSqlCost(WireManager)() } }() - return WireManager.SyncWires(ctx, userCred, localVpc, wires, provider) + return WireManager.SyncWires(ctx, userCred, localVpc, wires, provider, syncRange.Xor) }() if syncResults != nil { @@ -608,8 +656,7 @@ func syncVpcWires(ctx context.Context, userCred mcclient.TokenCredential, syncRe if result.IsError() { return } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) + for i := 0; i < len(localWires); i += 1 { func() { lockman.LockObject(ctx, &localWires[i]) @@ -643,7 +690,7 @@ func syncWireNetworks(ctx context.Context, userCred mcclient.TokenCredential, sy syncResults.AddSqlCost(NetworkManager)() } }() - return NetworkManager.SyncNetworks(ctx, userCred, localWire, nets, provider) + return NetworkManager.SyncNetworks(ctx, userCred, localWire, nets, provider, syncRange.Xor) }() if syncResults != nil { @@ -656,11 +703,19 @@ func syncWireNetworks(ctx context.Context, userCred mcclient.TokenCredential, sy if result.IsError() { return } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) } -func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider, localZone *SZone, remoteZone cloudprovider.ICloudZone, syncRange *SSyncRange, storageCachePairs []sStoragecacheSyncPair) []sStoragecacheSyncPair { +func syncZoneStorages( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + driver cloudprovider.ICloudProvider, + localZone *SZone, + remoteZone cloudprovider.ICloudZone, + syncRange *SSyncRange, + storageCachePairs []sStoragecacheSyncPair, +) []sStoragecacheSyncPair { storages, err := func() ([]cloudprovider.ICloudStorage, error) { defer syncResults.AddRequestCost(StorageManager)() return remoteZone.GetIStorages() @@ -672,7 +727,7 @@ func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, sy } localStorages, remoteStorages, result := func() ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { defer syncResults.AddSqlCost(StorageManager)() - return StorageManager.SyncStorages(ctx, userCred, provider, localZone, storages) + return StorageManager.SyncStorages(ctx, userCred, provider, localZone, storages, syncRange.Xor) }() syncResults.Add(StorageManager, result) @@ -683,8 +738,6 @@ func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, sy if result.IsError() { return nil } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) newCacheIds := make([]sStoragecacheSyncPair, 0) for i := 0; i < len(localStorages); i += 1 { @@ -697,7 +750,7 @@ func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, sy } if !isInCache(storageCachePairs, localStorages[i].StoragecacheId) && !isInCache(newCacheIds, localStorages[i].StoragecacheId) { - cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i]) + cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i], syncRange.Xor) if cachePair.remote != nil && cachePair.local != nil { newCacheIds = append(newCacheIds, cachePair) } @@ -710,14 +763,14 @@ func syncZoneStorages(ctx context.Context, userCred mcclient.TokenCredential, sy return newCacheIds } -func syncStorageCaches(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, localStorage *SStorage, remoteStorage cloudprovider.ICloudStorage) (cachePair sStoragecacheSyncPair) { +func syncStorageCaches(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, localStorage *SStorage, remoteStorage cloudprovider.ICloudStorage, xor bool) (cachePair sStoragecacheSyncPair) { log.Debugf("syncStorageCaches for storage %s", localStorage.GetId()) remoteCache := remoteStorage.GetIStoragecache() if remoteCache == nil { log.Errorf("remote storageCache is nil") return } - localCache, isNew, err := StoragecacheManager.SyncWithCloudStoragecache(ctx, userCred, remoteCache, provider) + localCache, isNew, err := StoragecacheManager.SyncWithCloudStoragecache(ctx, userCred, remoteCache, provider, xor) if err != nil { msg := fmt.Sprintf("SyncWithCloudStoragecache for storage %s failed %s", remoteStorage.GetName(), err) log.Errorf(msg) @@ -747,7 +800,7 @@ func syncStorageDisks(ctx context.Context, userCred mcclient.TokenCredential, sy } _, _, result := func() ([]SDisk, []cloudprovider.ICloudDisk, compare.SyncResult) { defer syncResults.AddSqlCost(DiskManager)() - return DiskManager.SyncDisks(ctx, userCred, driver, localStorage, disks, provider.GetOwnerId()) + return DiskManager.SyncDisks(ctx, userCred, driver, localStorage, disks, provider.GetOwnerId(), syncRange.Xor) }() syncResults.Add(DiskManager, result) @@ -758,11 +811,19 @@ func syncStorageDisks(ctx context.Context, userCred mcclient.TokenCredential, sy if result.IsError() { return } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) } -func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider, localZone *SZone, remoteZone cloudprovider.ICloudZone, syncRange *SSyncRange, storageCachePairs []sStoragecacheSyncPair) []sStoragecacheSyncPair { +func syncZoneHosts( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + driver cloudprovider.ICloudProvider, + localZone *SZone, + remoteZone cloudprovider.ICloudZone, + syncRange *SSyncRange, + storageCachePairs []sStoragecacheSyncPair, +) []sStoragecacheSyncPair { hosts, err := func() ([]cloudprovider.ICloudHost, error) { defer syncResults.AddRequestCost(HostManager)() return remoteZone.GetIHosts() @@ -774,7 +835,7 @@ func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncR } localHosts, remoteHosts, result := func() ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { defer syncResults.AddSqlCost(HostManager)() - return HostManager.SyncHosts(ctx, userCred, provider, localZone, hosts) + return HostManager.SyncHosts(ctx, userCred, provider, localZone, hosts, syncRange.Xor) }() syncResults.Add(HostManager, result) @@ -787,7 +848,6 @@ func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncR } var newCachePairs []sStoragecacheSyncPair db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) for i := 0; i < len(localHosts); i += 1 { if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) { continue @@ -800,8 +860,7 @@ func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncR return } - syncMetadata(ctx, userCred, &localHosts[i], remoteHosts[i]) - newCachePairs = syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs) + newCachePairs = syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs, syncRange.Xor) syncHostWires(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i]) syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange) }() @@ -809,7 +868,7 @@ func syncZoneHosts(ctx context.Context, userCred mcclient.TokenCredential, syncR return newCachePairs } -func syncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localHost *SHost, remoteHost cloudprovider.ICloudHost, storageCachePairs []sStoragecacheSyncPair) []sStoragecacheSyncPair { +func syncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localHost *SHost, remoteHost cloudprovider.ICloudHost, storageCachePairs []sStoragecacheSyncPair, xor bool) []sStoragecacheSyncPair { storages, err := func() ([]cloudprovider.ICloudStorage, error) { defer syncResults.AddRequestCost(HoststorageManager)() return remoteHost.GetIStorages() @@ -821,7 +880,7 @@ func syncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, sy } localStorages, remoteStorages, result := func() ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { defer syncResults.AddSqlCost(HoststorageManager)() - return localHost.SyncHostStorages(ctx, userCred, storages, provider) + return localHost.SyncHostStorages(ctx, userCred, storages, provider, xor) }() syncResults.Add(HoststorageManager, result) @@ -832,14 +891,12 @@ func syncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, sy if result.IsError() { return nil } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) newCacheIds := make([]sStoragecacheSyncPair, 0) for i := 0; i < len(localStorages); i += 1 { syncMetadata(ctx, userCred, &localStorages[i], remoteStorages[i]) if !isInCache(storageCachePairs, localStorages[i].StoragecacheId) && !isInCache(newCacheIds, localStorages[i].StoragecacheId) { - cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i]) + cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i], xor) if cachePair.remote != nil && cachePair.local != nil { newCacheIds = append(newCacheIds, cachePair) } @@ -882,7 +939,6 @@ func syncHostWires(ctx context.Context, userCred mcclient.TokenCredential, syncR return } db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) } func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider, localHost *SHost, remoteHost cloudprovider.ICloudHost, syncRange *SSyncRange) { @@ -898,7 +954,7 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes syncVMPairs, result := func() ([]SGuestSyncResult, compare.SyncResult) { defer syncResults.AddSqlCost(GuestManager)() - return localHost.SyncHostVMs(ctx, userCred, driver, vms, provider.GetOwnerId()) + return localHost.SyncHostVMs(ctx, userCred, driver, vms, provider.GetOwnerId(), syncRange.Xor) }() syncResults.Add(GuestManager, result) @@ -906,11 +962,7 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes msg := result.Result() notes := fmt.Sprintf("SyncHostVMs for host %s result: %s", localHost.Name, msg) log.Infof(notes) - /*if result.IsError() { - return - }*/ - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) if result.IsError() { logclient.AddSimpleActionLog(provider, logclient.ACT_CLOUD_SYNC, notes, userCred, false) } @@ -932,7 +984,15 @@ func syncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, syncRes } -func syncVMPeripherals(ctx context.Context, userCred mcclient.TokenCredential, local *SGuest, remote cloudprovider.ICloudVM, host *SHost, provider *SCloudprovider, driver cloudprovider.ICloudProvider) { +func syncVMPeripherals( + ctx context.Context, + userCred mcclient.TokenCredential, + local *SGuest, + remote cloudprovider.ICloudVM, + host *SHost, + provider *SCloudprovider, + driver cloudprovider.ICloudProvider, +) { err := syncVMNics(ctx, userCred, provider, host, local, remote) if err != nil { log.Errorf("syncVMNics error %s", err) @@ -955,11 +1015,16 @@ func syncVMPeripherals(ctx context.Context, userCred mcclient.TokenCredential, l } } -func syncVMNics(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, host *SHost, localVM *SGuest, remoteVM cloudprovider.ICloudVM) error { +func syncVMNics( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + host *SHost, + localVM *SGuest, + remoteVM cloudprovider.ICloudVM, +) error { nics, err := remoteVM.GetINics() if err != nil { - // msg := fmt.Sprintf("GetINics for VM %s failed %s", remoteVM.GetName(), err) - // log.Errorf(msg) return errors.Wrap(err, "remoteVM.GetINics") } result := localVM.SyncVMNics(ctx, userCred, host, nics, nil) @@ -969,16 +1034,20 @@ func syncVMNics(ctx context.Context, userCred mcclient.TokenCredential, provider if result.IsError() { return result.AllError() } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) return nil } -func syncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, driver cloudprovider.ICloudProvider, host *SHost, localVM *SGuest, remoteVM cloudprovider.ICloudVM) error { +func syncVMDisks( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + driver cloudprovider.ICloudProvider, + host *SHost, + localVM *SGuest, + remoteVM cloudprovider.ICloudVM, +) error { disks, err := remoteVM.GetIDisks() if err != nil { - // msg := fmt.Sprintf("GetIDisks for VM %s failed %s", remoteVM.GetName(), err) - // log.Errorf(msg) return errors.Wrap(err, "remoteVM.GetIDisks") } result := localVM.SyncVMDisks(ctx, userCred, driver, host, disks, provider.GetOwnerId()) @@ -988,16 +1057,12 @@ func syncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, provide if result.IsError() { return result.AllError() } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.Params), notes, task.UserCred, true) return nil } func syncVMEip(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, localVM *SGuest, remoteVM cloudprovider.ICloudVM) error { eip, err := remoteVM.GetIEIP() if err != nil { - // msg := fmt.Sprintf("GetIEIP for VM %s failed %s", remoteVM.GetName(), err) - // log.Errorf(msg) return errors.Wrap(err, "remoteVM.GetIEIP") } result := localVM.SyncVMEip(ctx, userCred, provider, eip, provider.GetOwnerId()) @@ -1006,7 +1071,6 @@ func syncVMEip(ctx context.Context, userCred mcclient.TokenCredential, provider if result.IsError() { return result.AllError() } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) return nil } @@ -1018,7 +1082,14 @@ func syncVMSecgroups(ctx context.Context, userCred mcclient.TokenCredential, pro return localVM.SyncVMSecgroups(ctx, userCred, secgroupIds) } -func syncSkusFromPrivateCloud(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, region *SCloudregion, remoteRegion cloudprovider.ICloudRegion) { +func syncSkusFromPrivateCloud( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + region *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) { skus, err := remoteRegion.GetISkus() if err != nil { msg := fmt.Sprintf("GetISkus for region %s(%s) failed %v", region.Name, region.Id, err) @@ -1026,7 +1097,7 @@ func syncSkusFromPrivateCloud(ctx context.Context, userCred mcclient.TokenCreden return } - result := ServerSkuManager.SyncPrivateCloudSkus(ctx, userCred, region, skus) + result := ServerSkuManager.SyncPrivateCloudSkus(ctx, userCred, region, skus, xor) syncResults.Add(ServerSkuManager, result) @@ -1041,7 +1112,15 @@ func syncSkusFromPrivateCloud(ctx context.Context, userCred mcclient.TokenCreden } } -func syncRegionDBInstances(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionDBInstances( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { instances, err := func() ([]cloudprovider.ICloudDBInstance, error) { defer syncResults.AddRequestCost(DBInstanceManager)() return remoteRegion.GetIDBInstances() @@ -1053,7 +1132,7 @@ func syncRegionDBInstances(ctx context.Context, userCred mcclient.TokenCredentia } localInstances, remoteInstances, result := func() ([]SDBInstance, []cloudprovider.ICloudDBInstance, compare.SyncResult) { defer syncResults.AddSqlCost(DBInstanceManager)() - return DBInstanceManager.SyncDBInstances(ctx, userCred, provider.GetOwnerId(), provider, localRegion, instances) + return DBInstanceManager.SyncDBInstances(ctx, userCred, provider.GetOwnerId(), provider, localRegion, instances, syncRange.Xor) }() syncResults.Add(DBInstanceManager, result) @@ -1232,7 +1311,7 @@ func syncDBInstanceBackups(ctx context.Context, userCred mcclient.TokenCredentia } provider := localInstance.GetCloudprovider() - result := DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, provider, localInstance, region, backups) + result := DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, provider, localInstance, region, backups, false) syncResults.Add(DBInstanceBackupManager, result) msg := result.Result() @@ -1260,7 +1339,15 @@ func syncDBInstanceParameters(ctx context.Context, userCred mcclient.TokenCreden return nil } -func syncRegionDBInstanceBackups(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) error { +func syncRegionDBInstanceBackups( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) error { backups, err := func() ([]cloudprovider.ICloudDBInstanceBackup, error) { defer syncResults.AddRequestCost(DBInstanceBackupManager)() return remoteRegion.GetIDBInstanceBackups() @@ -1271,7 +1358,7 @@ func syncRegionDBInstanceBackups(ctx context.Context, userCred mcclient.TokenCre result := func() compare.SyncResult { defer syncResults.AddSqlCost(DBInstanceBackupManager)() - return DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, provider, nil, localRegion, backups) + return DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, provider, nil, localRegion, backups, syncRange.Xor) }() syncResults.Add(DBInstanceBackupManager, result) @@ -1352,7 +1439,15 @@ func syncDBInstanceAccountPrivileges(ctx context.Context, userCred mcclient.Toke return nil } -func syncWafIPSets(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncWafIPSets( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { ipSets, err := func() ([]cloudprovider.ICloudWafIPSet, error) { defer syncResults.AddRequestCost(WafIPSetManager)() return remoteRegion.GetICloudWafIPSets() @@ -1365,7 +1460,7 @@ func syncWafIPSets(ctx context.Context, userCred mcclient.TokenCredential, syncR result := func() compare.SyncResult { defer syncResults.AddSqlCost(WafIPSetManager)() - return localRegion.SyncWafIPSets(ctx, userCred, provider, ipSets) + return localRegion.SyncWafIPSets(ctx, userCred, provider, ipSets, xor) }() syncResults.Add(WafIPSetManager, result) @@ -1376,7 +1471,15 @@ func syncWafIPSets(ctx context.Context, userCred mcclient.TokenCredential, syncR return nil } -func syncWafRegexSets(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncWafRegexSets( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { rSets, err := func() ([]cloudprovider.ICloudWafRegexSet, error) { defer syncResults.AddRequestCost(WafRegexSetManager)() return remoteRegion.GetICloudWafRegexSets() @@ -1388,7 +1491,7 @@ func syncWafRegexSets(ctx context.Context, userCred mcclient.TokenCredential, sy } result := func() compare.SyncResult { defer syncResults.AddSqlCost(WafRegexSetManager)() - return localRegion.SyncWafRegexSets(ctx, userCred, provider, rSets) + return localRegion.SyncWafRegexSets(ctx, userCred, provider, rSets, xor) }() syncResults.Add(WafRegexSetManager, result) log.Infof("SyncWafRegexSets for region %s result: %s", localRegion.Name, result.Result()) @@ -1398,7 +1501,15 @@ func syncWafRegexSets(ctx context.Context, userCred mcclient.TokenCredential, sy return nil } -func syncMongoDBs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncMongoDBs( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { dbs, err := func() ([]cloudprovider.ICloudMongoDB, error) { defer syncResults.AddRequestCost(MongoDBManager)() return remoteRegion.GetICloudMongoDBs() @@ -1411,7 +1522,7 @@ func syncMongoDBs(ctx context.Context, userCred mcclient.TokenCredential, syncRe _, _, result := func() ([]SMongoDB, []cloudprovider.ICloudMongoDB, compare.SyncResult) { defer syncResults.AddSqlCost(MongoDBManager)() - return localRegion.SyncMongoDBs(ctx, userCred, provider, dbs) + return localRegion.SyncMongoDBs(ctx, userCred, provider, dbs, xor) }() syncResults.Add(MongoDBManager, result) msg := result.Result() @@ -1423,7 +1534,15 @@ func syncMongoDBs(ctx context.Context, userCred mcclient.TokenCredential, syncRe return nil } -func syncElasticSearchs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncElasticSearchs( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { iEss, err := func() ([]cloudprovider.ICloudElasticSearch, error) { defer syncResults.AddRequestCost(ElasticSearchManager)() return remoteRegion.GetIElasticSearchs() @@ -1436,7 +1555,7 @@ func syncElasticSearchs(ctx context.Context, userCred mcclient.TokenCredential, result := func() compare.SyncResult { defer syncResults.AddSqlCost(ElasticSearchManager)() - return localRegion.SyncElasticSearchs(ctx, userCred, provider, iEss) + return localRegion.SyncElasticSearchs(ctx, userCred, provider, iEss, xor) }() syncResults.Add(ElasticSearchManager, result) msg := result.Result() @@ -1447,7 +1566,15 @@ func syncElasticSearchs(ctx context.Context, userCred mcclient.TokenCredential, return nil } -func syncKafkas(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncKafkas( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { iKafkas, err := func() ([]cloudprovider.ICloudKafka, error) { defer syncResults.AddRequestCost(KafkaManager)() return remoteRegion.GetICloudKafkas() @@ -1460,7 +1587,7 @@ func syncKafkas(ctx context.Context, userCred mcclient.TokenCredential, syncResu result := func() compare.SyncResult { defer syncResults.AddSqlCost(KafkaManager)() - return localRegion.SyncKafkas(ctx, userCred, provider, iKafkas) + return localRegion.SyncKafkas(ctx, userCred, provider, iKafkas, xor) }() syncResults.Add(KafkaManager, result) msg := result.Result() @@ -1471,7 +1598,15 @@ func syncKafkas(ctx context.Context, userCred mcclient.TokenCredential, syncResu return nil } -func syncApps(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncApps( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { iApps, err := func() ([]cloudprovider.ICloudApp, error) { defer syncResults.AddRequestCost(AppManager)() return remoteRegion.GetICloudApps() @@ -1483,7 +1618,7 @@ func syncApps(ctx context.Context, userCred mcclient.TokenCredential, syncResult } result := func() compare.SyncResult { defer syncResults.AddSqlCost(AppManager)() - return localRegion.SyncApps(ctx, userCred, provider, iApps) + return localRegion.SyncApps(ctx, userCred, provider, iApps, xor) }() syncResults.Add(AppManager, result) msg := result.Result() @@ -1494,7 +1629,15 @@ func syncApps(ctx context.Context, userCred mcclient.TokenCredential, syncResult return nil } -func syncKubeClusters(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncKubeClusters( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { iClusters, err := func() ([]cloudprovider.ICloudKubeCluster, error) { defer syncResults.AddRequestCost(KubeClusterManager)() return remoteRegion.GetICloudKubeClusters() @@ -1506,7 +1649,7 @@ func syncKubeClusters(ctx context.Context, userCred mcclient.TokenCredential, sy } localClusters, remoteClusters, result := func() ([]SKubeCluster, []cloudprovider.ICloudKubeCluster, compare.SyncResult) { defer syncResults.AddSqlCost(KubeClusterManager)() - return localRegion.SyncKubeClusters(ctx, userCred, provider, iClusters) + return localRegion.SyncKubeClusters(ctx, userCred, provider, iClusters, xor) }() syncResults.Add(KubeClusterManager, result) msg := result.Result() @@ -1591,7 +1734,15 @@ func syncKubeClusterNodes(ctx context.Context, userCred mcclient.TokenCredential return nil } -func syncWafInstances(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncWafInstances( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { wafIns, err := func() ([]cloudprovider.ICloudWafInstance, error) { defer syncResults.AddRequestCost(WafInstanceManager)() return remoteRegion.GetICloudWafInstances() @@ -1604,7 +1755,7 @@ func syncWafInstances(ctx context.Context, userCred mcclient.TokenCredential, sy localWafs, remoteWafs, result := func() ([]SWafInstance, []cloudprovider.ICloudWafInstance, compare.SyncResult) { defer syncResults.AddSqlCost(WafInstanceManager)() - return localRegion.SyncWafInstances(ctx, userCred, provider, wafIns) + return localRegion.SyncWafInstances(ctx, userCred, provider, wafIns, xor) }() syncResults.Add(WafInstanceManager, result) msg := result.Result() @@ -1656,7 +1807,15 @@ func syncWafRules(ctx context.Context, userCred mcclient.TokenCredential, syncRe return nil } -func syncRegionSnapshots(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionSnapshots( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { snapshots, err := func() ([]cloudprovider.ICloudSnapshot, error) { defer syncResults.AddRequestCost(SnapshotManager)() return remoteRegion.GetISnapshots() @@ -1669,7 +1828,7 @@ func syncRegionSnapshots(ctx context.Context, userCred mcclient.TokenCredential, result := func() compare.SyncResult { defer syncResults.AddSqlCost(SnapshotManager)() - return SnapshotManager.SyncSnapshots(ctx, userCred, provider, localRegion, snapshots, provider.GetOwnerId()) + return SnapshotManager.SyncSnapshots(ctx, userCred, provider, localRegion, snapshots, provider.GetOwnerId(), syncRange.Xor) }() syncResults.Add(SnapshotManager, result) @@ -1679,10 +1838,17 @@ func syncRegionSnapshots(ctx context.Context, userCred mcclient.TokenCredential, if result.IsError() { return } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) } -func syncRegionSnapshotPolicies(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionSnapshotPolicies( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { snapshotPolicies, err := func() ([]cloudprovider.ICloudSnapshotPolicy, error) { defer syncResults.AddRequestCost(SnapshotPolicyManager)() return remoteRegion.GetISnapshotPolicies() @@ -1694,7 +1860,7 @@ func syncRegionSnapshotPolicies(ctx context.Context, userCred mcclient.TokenCred result := func() compare.SyncResult { defer syncResults.AddSqlCost(SnapshotPolicyManager)() - return SnapshotPolicyManager.SyncSnapshotPolicies(ctx, userCred, provider, localRegion, snapshotPolicies, provider.GetOwnerId()) + return SnapshotPolicyManager.SyncSnapshotPolicies(ctx, userCred, provider, localRegion, snapshotPolicies, provider.GetOwnerId(), syncRange.Xor) }() syncResults.Add(SnapshotPolicyManager, result) msg := result.Result() @@ -1704,7 +1870,15 @@ func syncRegionSnapshotPolicies(ctx context.Context, userCred mcclient.TokenCred } } -func syncRegionNetworkInterfaces(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionNetworkInterfaces( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { networkInterfaces, err := func() ([]cloudprovider.ICloudNetworkInterface, error) { defer syncResults.AddRequestCost(NetworkInterfaceManager)() return remoteRegion.GetINetworkInterfaces() @@ -1717,7 +1891,7 @@ func syncRegionNetworkInterfaces(ctx context.Context, userCred mcclient.TokenCre localInterfaces, remoteInterfaces, result := func() ([]SNetworkInterface, []cloudprovider.ICloudNetworkInterface, compare.SyncResult) { defer syncResults.AddSqlCost(NetworkInterfaceManager)() - return NetworkInterfaceManager.SyncNetworkInterfaces(ctx, userCred, provider, localRegion, networkInterfaces) + return NetworkInterfaceManager.SyncNetworkInterfaces(ctx, userCred, provider, localRegion, networkInterfaces, syncRange.Xor) }() syncResults.Add(NetworkInterfaceManager, result) @@ -1779,18 +1953,18 @@ func syncPublicCloudProviderInfo( storageCachePairs := make([]sStoragecacheSyncPair, 0) if cloudprovider.IsSupportQuota(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_QUOTA) { - syncRegionQuotas(ctx, userCred, syncResults, driver, provider, localRegion, remoteRegion) + syncRegionQuotas(ctx, userCred, syncResults, driver, provider, localRegion, remoteRegion, syncRange.Xor) } - localZones, remoteZones, _ := syncRegionZones(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + localZones, remoteZones, _ := syncRegionZones(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) if !driver.GetFactory().NeedSyncSkuFromCloud() { - syncRegionSkus(ctx, userCred, localRegion) - SyncRegionDBInstanceSkus(ctx, userCred, localRegion.Id, true) - SyncRegionNatSkus(ctx, userCred, localRegion.Id, true) - SyncRegionNasSkus(ctx, userCred, localRegion.Id, true) + syncRegionSkus(ctx, userCred, localRegion, syncRange.Xor) + SyncRegionDBInstanceSkus(ctx, userCred, localRegion.Id, true, syncRange.Xor) + SyncRegionNatSkus(ctx, userCred, localRegion.Id, true, syncRange.Xor) + SyncRegionNasSkus(ctx, userCred, localRegion.Id, true, syncRange.Xor) } else { - syncSkusFromPrivateCloud(ctx, userCred, syncResults, localRegion, remoteRegion) + syncSkusFromPrivateCloud(ctx, userCred, syncResults, localRegion, remoteRegion, syncRange.Xor) if cloudprovider.IsSupportRds(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_RDS) { syncDBInstanceSkus(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) } @@ -1805,7 +1979,7 @@ func syncPublicCloudProviderInfo( // no need to lock public cloud region as cloud region for public cloud is readonly if cloudprovider.IsSupportObjectstore(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE) { - syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportCompute(driver) { @@ -1868,33 +2042,42 @@ func syncPublicCloudProviderInfo( } if cloudprovider.IsSupportWaf(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_WAF) { - syncWafIPSets(ctx, userCred, syncResults, provider, localRegion, remoteRegion) - syncWafRegexSets(ctx, userCred, syncResults, provider, localRegion, remoteRegion) - syncWafInstances(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncWafIPSets(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) + syncWafRegexSets(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) + syncWafInstances(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportMongoDB(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_MONGO_DB) { - syncMongoDBs(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncMongoDBs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportElasticSearch(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_ES) { - syncElasticSearchs(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncElasticSearchs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportKafka(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_KAFKA) { - syncKafkas(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncKafkas(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportApp(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_APP) { - syncApps(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncApps(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportContainer(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_CONTAINER) { - syncKubeClusters(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncKubeClusters(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportTablestore(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_TABLESTORE) { - syncTablestore(ctx, userCred, syncResults, provider, localRegion, remoteRegion) + syncTablestore(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) + } + + if cloudprovider.IsSupportModelartsPool(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_MODELARTES) { + syncModelartsPoolSkus(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) + syncModelartsPools(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) + } + + if cloudprovider.IsSupportMiscResources(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_MISC) { + syncMiscResources(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange.Xor) } if cloudprovider.IsSupportCompute(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { @@ -1904,7 +2087,7 @@ func syncPublicCloudProviderInfo( if storageCachePairs[i].isNew || syncRange.DeepSync || !driver.GetFactory().IsPublicCloud() { result := func() compare.SyncResult { defer syncResults.AddRequestCost(CachedimageManager)() - return storageCachePairs[i].syncCloudImages(ctx, userCred) + return storageCachePairs[i].syncCloudImages(ctx, userCred, syncRange.Xor) }() syncResults.Add(CachedimageManager, result) @@ -1915,15 +2098,6 @@ func syncPublicCloudProviderInfo( } } - if cloudprovider.IsSupportModelartsPool(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_MODELARTES) { - syncModelartsPoolSkus(ctx, userCred, syncResults, provider, localRegion, remoteRegion) - syncModelartsPools(ctx, userCred, syncResults, provider, localRegion, remoteRegion) - } - - if cloudprovider.IsSupportMiscResources(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_MISC) { - syncMiscResources(ctx, userCred, syncResults, provider, localRegion, remoteRegion) - } - return nil } @@ -1964,7 +2138,7 @@ func syncOnPremiseCloudProviderStorage(ctx context.Context, userCred mcclient.To log.Errorf(msg) return nil } - localStorages, remoteStorages, result := StorageManager.SyncStorages(ctx, userCred, provider, zone, istorages) + localStorages, remoteStorages, result := StorageManager.SyncStorages(ctx, userCred, provider, zone, istorages, syncRange.Xor) syncResults.Add(StorageManager, result) msg := result.Result() @@ -1985,7 +2159,7 @@ func syncOnPremiseCloudProviderStorage(ctx context.Context, userCred mcclient.To } if !isInCache(storageCachePairs, localStorages[i].StoragecacheId) { - cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i]) + cachePair := syncStorageCaches(ctx, userCred, provider, &localStorages[i], remoteStorages[i], syncRange.Xor) if cachePair.remote != nil && cachePair.local != nil { storageCachePairs = append(storageCachePairs, cachePair) } @@ -2018,7 +2192,7 @@ func syncOnPremiseCloudProviderInfo( localRegion := CloudregionManager.FetchDefaultRegion() if cloudprovider.IsSupportObjectstore(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_OBJECTSTORE) { - syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, iregion) + syncRegionBuckets(ctx, userCred, syncResults, provider, localRegion, iregion, syncRange.Xor) } var storageCachePairs []sStoragecacheSyncPair @@ -2036,7 +2210,7 @@ func syncOnPremiseCloudProviderInfo( localHosts, remoteHosts, result := func() ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { defer syncResults.AddSqlCost(HostManager)() - return HostManager.SyncHosts(ctx, userCred, provider, nil, ihosts) + return HostManager.SyncHosts(ctx, userCred, provider, nil, ihosts, syncRange.Xor) }() syncResults.Add(HostManager, result) @@ -2049,7 +2223,7 @@ func syncOnPremiseCloudProviderInfo( if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) { continue } - newCachePairs := syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs) + newCachePairs := syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs, syncRange.Xor) if len(newCachePairs) > 0 { storageCachePairs = append(storageCachePairs, newCachePairs...) } @@ -2066,7 +2240,7 @@ func syncOnPremiseCloudProviderInfo( // if storageCachePairs[i].isNew || syncRange.DeepSync { result := func() compare.SyncResult { defer syncResults.AddRequestCost(CachedimageManager)() - return storageCachePairs[i].syncCloudImages(ctx, userCred) + return storageCachePairs[i].syncCloudImages(ctx, userCred, syncRange.Xor) }() syncResults.Add(CachedimageManager, result) @@ -2115,8 +2289,6 @@ func syncHostNics(ctx context.Context, userCred mcclient.TokenCredential, provid if result.IsError() { return } - // db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - // logclient.AddActionLog(provider, getAction(task.GetParams()), notes, task.GetUserCred(), true) } func (manager *SCloudproviderregionManager) fetchRecordsByQuery(q *sqlchemy.SQuery) []SCloudproviderregion { @@ -2218,14 +2390,14 @@ func SyncCloudaccountResources(ctx context.Context, userCred mcclient.TokenCrede } if cloudprovider.IsSupportProject(provider) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_PROJECT) { - err = syncProjects(ctx, userCred, SSyncResultSet{}, account, provider) + err = syncProjects(ctx, userCred, SSyncResultSet{}, account, provider, syncRange.Xor) if err != nil { log.Errorf("Sync project for account %s error: %v", account.Name, err) } } if cloudprovider.IsSupportDnsZone(provider) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_DNSZONE) { - err = syncDns(ctx, userCred, SSyncResultSet{}, account, provider) + err = syncDns(ctx, userCred, SSyncResultSet{}, account, provider, syncRange.Xor) if err != nil { log.Errorf("Sync dns zone for account %s error: %v", account.Name, err) } @@ -2234,7 +2406,7 @@ func SyncCloudaccountResources(ctx context.Context, userCred mcclient.TokenCrede return nil } -func syncProjects(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, account *SCloudaccount, provider cloudprovider.ICloudProvider) error { +func syncProjects(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, account *SCloudaccount, provider cloudprovider.ICloudProvider, xor bool) error { lockman.LockRawObject(ctx, ExternalProjectManager.Keyword(), account.Id) defer lockman.ReleaseRawObject(ctx, ExternalProjectManager.Keyword(), account.Id) @@ -2248,7 +2420,7 @@ func syncProjects(ctx context.Context, userCred mcclient.TokenCredential, syncRe result := func() compare.SyncResult { defer syncResults.AddSqlCost(ExternalProjectManager)() - return ExternalProjectManager.SyncProjects(ctx, userCred, account, projects) + return ExternalProjectManager.SyncProjects(ctx, userCred, account, projects, xor) }() syncResults.Add(ExternalProjectManager, result) @@ -2261,7 +2433,7 @@ func syncProjects(ctx context.Context, userCred mcclient.TokenCredential, syncRe return nil } -func syncDns(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, account *SCloudaccount, provider cloudprovider.ICloudProvider) error { +func syncDns(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, account *SCloudaccount, provider cloudprovider.ICloudProvider, xor bool) error { lockman.LockRawObject(ctx, DnsZoneCacheManager.Keyword(), account.Id) defer lockman.ReleaseRawObject(ctx, DnsZoneCacheManager.Keyword(), account.Id) @@ -2269,7 +2441,7 @@ func syncDns(ctx context.Context, userCred mcclient.TokenCredential, syncResults if err != nil { return errors.Wrapf(err, "GetICloudDnsZones") } - localZones, remoteZones, result := account.SyncDnsZones(ctx, userCred, dnsZones) + localZones, remoteZones, result := account.SyncDnsZones(ctx, userCred, dnsZones, xor) log.Infof("Sync dns zones for cloudaccount %s result: %s", account.Name, result.Result()) for i := 0; i < len(localZones); i++ { func() { @@ -2280,7 +2452,7 @@ func syncDns(ctx context.Context, userCred mcclient.TokenCredential, syncResults return } - result := localZones[i].SyncDnsRecordSets(ctx, userCred, account.Provider, remoteZones[i]) + result := localZones[i].SyncDnsRecordSets(ctx, userCred, account.Provider, remoteZones[i], xor) log.Infof("Sync dns records for dns zone %s result: %s", localZones[i].GetName(), result.Result()) }() } @@ -2294,21 +2466,21 @@ func SyncCloudproviderResources(ctx context.Context, userCred mcclient.TokenCred } if cloudprovider.IsSupportCDN(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_CDN) { - err = syncCdnDomains(ctx, userCred, SSyncResultSet{}, provider, driver) + err = syncCdnDomains(ctx, userCred, SSyncResultSet{}, provider, driver, syncRange.Xor) if err != nil { log.Errorf("syncCdnDomains error: %v", err) } } if cloudprovider.IsSupportInterVpcNetwork(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_INTERVPCNETWORK) { - err = syncInterVpcNetworks(ctx, userCred, SSyncResultSet{}, provider, driver) + err = syncInterVpcNetworks(ctx, userCred, SSyncResultSet{}, provider, driver, syncRange.Xor) if err != nil { log.Errorf("syncInterVpcNetworks error: %v", err) } } if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NETWORK) { - err = syncGlobalVpcs(ctx, userCred, SSyncResultSet{}, provider, driver) + err = syncGlobalVpcs(ctx, userCred, SSyncResultSet{}, provider, driver, syncRange.Xor) if err != nil { log.Errorf("syncGlobalVpcs error: %v", err) } @@ -2317,23 +2489,23 @@ func SyncCloudproviderResources(ctx context.Context, userCred mcclient.TokenCred return nil } -func syncCdnDomains(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider) error { +func syncCdnDomains(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider, xor bool) error { domains, err := driver.GetICloudCDNDomains() if err != nil { return err } - result := provider.SyncCDNDomains(ctx, userCred, domains) + result := provider.SyncCDNDomains(ctx, userCred, domains, xor) log.Infof("Sync CDN for provider %s result: %s", provider.Name, result.Result()) return nil } -func syncInterVpcNetworks(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider) error { +func syncInterVpcNetworks(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider, xor bool) error { networks, err := driver.GetICloudInterVpcNetworks() if err != nil { return errors.Wrapf(err, "GetICloudInterVpcNetworks") } - localNetwork, remoteNetwork, result := provider.SyncInterVpcNetwork(ctx, userCred, networks) + localNetwork, remoteNetwork, result := provider.SyncInterVpcNetwork(ctx, userCred, networks, xor) log.Infof("Sync inter vpc network for cloudprovider %s result: %s", provider.GetName(), result.Result()) for i := range localNetwork { lockman.LockObject(ctx, &localNetwork[i]) @@ -2342,12 +2514,12 @@ func syncInterVpcNetworks(ctx context.Context, userCred mcclient.TokenCredential if localNetwork[i].Deleted { continue } - localNetwork[i].SyncInterVpcNetworkRouteSets(ctx, userCred, remoteNetwork[i]) + localNetwork[i].SyncInterVpcNetworkRouteSets(ctx, userCred, remoteNetwork[i], xor) } return nil } -func syncGlobalVpcs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider) error { +func syncGlobalVpcs(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, driver cloudprovider.ICloudProvider, xor bool) error { gvpcs, err := driver.GetICloudGlobalVpcs() if err != nil { if errors.Cause(err) == cloudprovider.ErrNotImplemented || errors.Cause(err) == cloudprovider.ErrNotSupported { @@ -2356,12 +2528,20 @@ func syncGlobalVpcs(ctx context.Context, userCred mcclient.TokenCredential, sync return err } - result := provider.SyncGlobalVpcs(ctx, userCred, gvpcs) + result := provider.SyncGlobalVpcs(ctx, userCred, gvpcs, xor) log.Infof("Sync global vpcs for cloudprovider %s result: %s", provider.GetName(), result.Result()) return nil } -func syncTablestore(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncTablestore( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { iTablestores, err := func() ([]cloudprovider.ICloudTablestore, error) { defer syncResults.AddRequestCost(TablestoreManager)() return remoteRegion.GetICloudTablestores() @@ -2373,7 +2553,7 @@ func syncTablestore(ctx context.Context, userCred mcclient.TokenCredential, sync } result := func() compare.SyncResult { defer syncResults.AddSqlCost(TablestoreManager)() - return localRegion.SyncTablestores(ctx, userCred, iTablestores, provider) + return localRegion.SyncTablestores(ctx, userCred, iTablestores, provider, xor) }() syncResults.Add(TablestoreManager, result) msg := result.Result() @@ -2384,38 +2564,62 @@ func syncTablestore(ctx context.Context, userCred mcclient.TokenCredential, sync return nil } -func syncModelartsPools(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncModelartsPools( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { ipools, err := remoteRegion.GetIModelartsPools() if err != nil { msg := fmt.Sprintf("GetIModelartsPools for provider %s failed %s", err, ipools) log.Errorf(msg) return err } - result := localRegion.SyncModelartsPools(ctx, userCred, provider, ipools) + result := localRegion.SyncModelartsPools(ctx, userCred, provider, ipools, xor) log.Infof("SyncModelartsPools for region %s result: %s", provider.GetName(), result.Result()) return nil } -func syncModelartsPoolSkus(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncModelartsPoolSkus( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { ipools, err := remoteRegion.GetIModelartsPoolSku() if err != nil { msg := fmt.Sprintf("GetIModelartsPoolSku for provider %s failed %s", err, ipools) log.Errorf(msg) return err } - result := localRegion.SyncModelartsPoolSkus(ctx, userCred, provider, ipools) + result := localRegion.SyncModelartsPoolSkus(ctx, userCred, provider, ipools, xor) log.Infof("SyncModelartsPoolSkus for region %s result: %s", provider.GetName(), result.Result()) return nil } -func syncMiscResources(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion) error { +func syncMiscResources( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + xor bool, +) error { exts, err := remoteRegion.GetIMiscResources() if err != nil { msg := fmt.Sprintf("GetIMiscResources for provider %s failed %v", provider.Name, err) log.Errorf(msg) return err } - result := localRegion.SyncMiscResources(ctx, userCred, provider, exts) + result := localRegion.SyncMiscResources(ctx, userCred, provider, exts, xor) log.Infof("SyncMiscResources for provider %s result: %s", provider.GetName(), result.Result()) return nil } diff --git a/pkg/compute/models/cloudsyncelasticcache.go b/pkg/compute/models/cloudsyncelasticcache.go index 0861da9f59..65539a984c 100644 --- a/pkg/compute/models/cloudsyncelasticcache.go +++ b/pkg/compute/models/cloudsyncelasticcache.go @@ -29,7 +29,15 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" ) -func syncElasticcaches(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncElasticcaches( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { extCacheDBs, err := func() ([]cloudprovider.ICloudElasticcache, error) { defer syncResults.AddRequestCost(ElasticcacheManager)() return remoteRegion.GetIElasticcaches() @@ -42,7 +50,7 @@ func syncElasticcaches(ctx context.Context, userCred mcclient.TokenCredential, s localInstances, remoteInstances, result := func() ([]SElasticcache, []cloudprovider.ICloudElasticcache, compare.SyncResult) { defer syncResults.AddSqlCost(ElasticcacheManager)() - return ElasticcacheManager.SyncElasticcaches(ctx, userCred, provider.GetOwnerId(), provider, localRegion, extCacheDBs) + return ElasticcacheManager.SyncElasticcaches(ctx, userCred, provider.GetOwnerId(), provider, localRegion, extCacheDBs, syncRange.Xor) }() syncResults.Add(ElasticcacheManager, result) diff --git a/pkg/compute/models/cloudsyncelb.go b/pkg/compute/models/cloudsyncelb.go index 71326fd707..df081e80a7 100644 --- a/pkg/compute/models/cloudsyncelb.go +++ b/pkg/compute/models/cloudsyncelb.go @@ -28,7 +28,15 @@ import ( "yunion.io/x/onecloud/pkg/mcclient" ) -func syncRegionLoadbalancerCertificates(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionLoadbalancerCertificates( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { certificates, err := func() ([]cloudprovider.ICloudLoadbalancerCertificate, error) { defer syncResults.AddRequestCost(LoadbalancerCertificateManager)() return remoteRegion.GetILoadBalancerCertificates() @@ -40,7 +48,7 @@ func syncRegionLoadbalancerCertificates(ctx context.Context, userCred mcclient.T } result := func() compare.SyncResult { defer syncResults.AddSqlCost(LoadbalancerCertificateManager)() - return provider.SyncLoadbalancerCertificates(ctx, userCred, localRegion, certificates) + return provider.SyncLoadbalancerCertificates(ctx, userCred, localRegion, certificates, syncRange.Xor) }() syncResults.Add(CachedLoadbalancerCertificateManager, result) @@ -52,7 +60,15 @@ func syncRegionLoadbalancerCertificates(ctx context.Context, userCred mcclient.T } } -func syncRegionLoadbalancerAcls(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionLoadbalancerAcls( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { acls, err := func() ([]cloudprovider.ICloudLoadbalancerAcl, error) { defer syncResults.AddRequestCost(LoadbalancerAclManager)() return remoteRegion.GetILoadBalancerAcls() @@ -76,7 +92,15 @@ func syncRegionLoadbalancerAcls(ctx context.Context, userCred mcclient.TokenCred } } -func syncRegionLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localRegion *SCloudregion, remoteRegion cloudprovider.ICloudRegion, syncRange *SSyncRange) { +func syncRegionLoadbalancers( + ctx context.Context, + userCred mcclient.TokenCredential, + syncResults SSyncResultSet, + provider *SCloudprovider, + localRegion *SCloudregion, + remoteRegion cloudprovider.ICloudRegion, + syncRange *SSyncRange, +) { lbs, err := func() ([]cloudprovider.ICloudLoadbalancer, error) { defer syncResults.AddRequestCost(LoadbalancerManager)() return remoteRegion.GetILoadBalancers() @@ -89,7 +113,7 @@ func syncRegionLoadbalancers(ctx context.Context, userCred mcclient.TokenCredent func() { defer syncResults.AddSqlCost(LoadbalancerManager)() - localLbs, remoteLbs, result := LoadbalancerManager.SyncLoadbalancers(ctx, userCred, provider, localRegion, lbs) + localLbs, remoteLbs, result := LoadbalancerManager.SyncLoadbalancers(ctx, userCred, provider, localRegion, lbs, syncRange.Xor) syncResults.Add(LoadbalancerManager, result) diff --git a/pkg/compute/models/dbinstance_backups.go b/pkg/compute/models/dbinstance_backups.go index bfb5b4d8c9..cf98526486 100644 --- a/pkg/compute/models/dbinstance_backups.go +++ b/pkg/compute/models/dbinstance_backups.go @@ -386,9 +386,17 @@ func (backup *SDBInstanceBackup) GetIDBInstanceBackup(ctx context.Context) (clou return iRegion.GetIDBInstanceBackupById(backup.ExternalId) } -func (manager *SDBInstanceBackupManager) SyncDBInstanceBackups(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, instance *SDBInstance, region *SCloudregion, cloudBackups []cloudprovider.ICloudDBInstanceBackup) compare.SyncResult { - lockman.LockRawObject(ctx, "dbinstance-backups", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "dbinstance-backups", fmt.Sprintf("%s-%s", provider.Id, region.Id)) +func (manager *SDBInstanceBackupManager) SyncDBInstanceBackups( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + instance *SDBInstance, + region *SCloudregion, + cloudBackups []cloudprovider.ICloudDBInstanceBackup, + xor bool, +) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) result := compare.SyncResult{} dbBackups, err := region.GetDBInstanceBackups(provider, instance) @@ -415,12 +423,14 @@ func (manager *SDBInstanceBackupManager) SyncDBInstanceBackups(ctx context.Conte } } - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudDBInstanceBackup(ctx, userCred, commonext[i], provider) - if err != nil { - result.UpdateError(err) - } else { - result.Update() + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudDBInstanceBackup(ctx, userCred, commonext[i], provider) + if err != nil { + result.UpdateError(err) + } else { + result.Update() + } } } diff --git a/pkg/compute/models/dbinstance_skus.go b/pkg/compute/models/dbinstance_skus.go index cacc0b0d08..b0e235140d 100644 --- a/pkg/compute/models/dbinstance_skus.go +++ b/pkg/compute/models/dbinstance_skus.go @@ -479,9 +479,15 @@ func (manager *SDBInstanceSkuManager) GetDBInstanceSkus(provider, cloudregionId, return skus, nil } -func (manager *SDBInstanceSkuManager) SyncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, meta *SSkuResourcesMeta) compare.SyncResult { - lockman.LockRawObject(ctx, "dbinstance-skus", region.Id) - defer lockman.ReleaseRawObject(ctx, "dbinstance-skus", region.Id) +func (manager *SDBInstanceSkuManager) SyncDBInstanceSkus( + ctx context.Context, + userCred mcclient.TokenCredential, + region *SCloudregion, + meta *SSkuResourcesMeta, + xor bool, +) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), region.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), region.Id) syncResult := compare.SyncResult{} @@ -516,12 +522,14 @@ func (manager *SDBInstanceSkuManager) SyncDBInstanceSkus(ctx context.Context, us syncResult.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - } else { - syncResult.Update() + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } } } for i := 0; i < len(added); i += 1 { @@ -559,7 +567,7 @@ func (manager *SDBInstanceSkuManager) newFromCloudSku(ctx context.Context, userC return manager.TableSpec().Insert(ctx, sku) } -func SyncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart bool) { +func SyncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart, xor bool) { if isStart { q := DBInstanceSkuManager.Query() if len(regionId) > 0 { @@ -599,7 +607,7 @@ func SyncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCreden log.Infof("region %s(%s) not support dbinstance, skip sync", region.Name, region.Id) continue } - result := DBInstanceSkuManager.SyncDBInstanceSkus(ctx, userCred, ®ion, meta) + result := DBInstanceSkuManager.SyncDBInstanceSkus(ctx, userCred, ®ion, meta, xor) msg := result.Result() notes := fmt.Sprintf("sync rds sku for region %s result: %s", region.Name, msg) log.Infof(notes) @@ -608,7 +616,7 @@ func SyncRegionDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCreden } func SyncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { - SyncRegionDBInstanceSkus(ctx, userCred, "", isStart) + SyncRegionDBInstanceSkus(ctx, userCred, "", isStart, false) } func (manager *SDBInstanceSkuManager) ListItemExportKeys(ctx context.Context, @@ -674,7 +682,12 @@ func (self *SCloudregion) GetDBInstanceSkus() ([]SDBInstanceSku, error) { return skus, nil } -func (self *SCloudregion) SyncDBInstanceSkus(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudDBInstanceSku) compare.SyncResult { +func (self *SCloudregion) SyncDBInstanceSkus( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudDBInstanceSku, +) compare.SyncResult { lockman.LockRawObject(ctx, DBInstanceSkuManager.Keyword(), self.Id) defer lockman.ReleaseRawObject(ctx, DBInstanceSkuManager.Keyword(), self.Id) diff --git a/pkg/compute/models/dbinstances.go b/pkg/compute/models/dbinstances.go index 127fde11dd..7dcb5cf3ac 100644 --- a/pkg/compute/models/dbinstances.go +++ b/pkg/compute/models/dbinstances.go @@ -1419,7 +1419,15 @@ func (manager *SDBInstanceManager) SyncDBInstanceMasterId(ctx context.Context, u } } -func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred mcclient.TokenCredential, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, region *SCloudregion, cloudDBInstances []cloudprovider.ICloudDBInstance) ([]SDBInstance, []cloudprovider.ICloudDBInstance, compare.SyncResult) { +func (manager *SDBInstanceManager) SyncDBInstances( + ctx context.Context, + userCred mcclient.TokenCredential, + syncOwnerId mcclient.IIdentityProvider, + provider *SCloudprovider, + region *SCloudregion, + cloudDBInstances []cloudprovider.ICloudDBInstance, + xor bool, +) ([]SDBInstance, []cloudprovider.ICloudDBInstance, compare.SyncResult) { lockman.LockRawObject(ctx, "dbinstances", fmt.Sprintf("%s-%s", provider.Id, region.Id)) defer lockman.ReleaseRawObject(ctx, "dbinstances", fmt.Sprintf("%s-%s", provider.Id, region.Id)) @@ -1458,15 +1466,17 @@ func (manager *SDBInstanceManager) SyncDBInstances(ctx context.Context, userCred } } - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudDBInstance(ctx, userCred, provider, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudDBInstance(ctx, userCred, provider, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + localDBInstances = append(localDBInstances, commondb[i]) + remoteDBInstances = append(remoteDBInstances, commonext[i]) + syncResult.Update() } - localDBInstances = append(localDBInstances, commondb[i]) - remoteDBInstances = append(remoteDBInstances, commonext[i]) - syncResult.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/disks.go b/pkg/compute/models/disks.go index e7ba065896..113ed3d435 100644 --- a/pkg/compute/models/disks.go +++ b/pkg/compute/models/disks.go @@ -1451,11 +1451,9 @@ func (manager *SDiskManager) findOrCreateDisk(ctx context.Context, userCred mccl return diskObj.(*SDisk), nil } -func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, storage *SStorage, disks []cloudprovider.ICloudDisk, syncOwnerId mcclient.IIdentityProvider) ([]SDisk, []cloudprovider.ICloudDisk, compare.SyncResult) { - // syncOwnerId := projectId - - lockman.LockRawObject(ctx, "disks", storage.Id) - defer lockman.ReleaseRawObject(ctx, "disks", storage.Id) +func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, storage *SStorage, disks []cloudprovider.ICloudDisk, syncOwnerId mcclient.IIdentityProvider, xor bool) ([]SDisk, []cloudprovider.ICloudDisk, compare.SyncResult) { + lockman.LockRawObject(ctx, manager.Keyword(), storage.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), storage.Id) localDisks := make([]SDisk, 0) remoteDisks := make([]cloudprovider.ICloudDisk, 0) @@ -1499,10 +1497,12 @@ func (manager *SDiskManager) SyncDisks(ctx context.Context, userCred mcclient.To syncResult.Delete() continue } - err = commondb[i].syncWithCloudDisk(ctx, userCred, provider, commonext[i], -1, syncOwnerId, storage.ManagerId) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + err = commondb[i].syncWithCloudDisk(ctx, userCred, provider, commonext[i], -1, syncOwnerId, storage.ManagerId) + if err != nil { + syncResult.UpdateError(err) + continue + } } localDisks = append(localDisks, commondb[i]) remoteDisks = append(remoteDisks, commonext[i]) diff --git a/pkg/compute/models/dns_zones.go b/pkg/compute/models/dns_zones.go index 9d71f884fb..1e5a81a048 100644 --- a/pkg/compute/models/dns_zones.go +++ b/pkg/compute/models/dns_zones.go @@ -604,7 +604,7 @@ func (self *SDnsZone) GetDnsRecordSets() ([]SDnsRecordSet, error) { return records, nil } -func (self *SDnsZone) SyncDnsRecordSets(ctx context.Context, userCred mcclient.TokenCredential, provider string, ext cloudprovider.ICloudDnsZone) compare.SyncResult { +func (self *SDnsZone) SyncDnsRecordSets(ctx context.Context, userCred mcclient.TokenCredential, provider string, ext cloudprovider.ICloudDnsZone, xor bool) compare.SyncResult { lockman.LockRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) defer lockman.ReleaseRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) @@ -666,25 +666,27 @@ func (self *SDnsZone) SyncDnsRecordSets(ctx context.Context, userCred mcclient.T result.Delete() } - for i := range update { - _record, err := DnsRecordSetManager.FetchById(update[i].Id) - if err != nil { - result.UpdateError(errors.Wrapf(err, "DnsRecordSetManager.FetchById(%s)", del[i].Id)) - continue - } - record := _record.(*SDnsRecordSet) - caches, err := self.GetDnsZoneCaches() - if err != nil { - result.UpdateError(errors.Wrapf(err, "GetDnsZoneCaches")) - continue - } - if self.ZoneType == string(cloudprovider.PrivateZone) || len(caches) < 2 { - err = record.syncWithCloudDnsRecord(ctx, userCred, provider, update[i]) + if !xor { + for i := range update { + _record, err := DnsRecordSetManager.FetchById(update[i].Id) if err != nil { - result.UpdateError(errors.Wrapf(err, "syncWithCloudDnsRecord")) + result.UpdateError(errors.Wrapf(err, "DnsRecordSetManager.FetchById(%s)", del[i].Id)) continue } - result.Update() + record := _record.(*SDnsRecordSet) + caches, err := self.GetDnsZoneCaches() + if err != nil { + result.UpdateError(errors.Wrapf(err, "GetDnsZoneCaches")) + continue + } + if self.ZoneType == string(cloudprovider.PrivateZone) || len(caches) < 2 { + err = record.syncWithCloudDnsRecord(ctx, userCred, provider, update[i]) + if err != nil { + result.UpdateError(errors.Wrapf(err, "syncWithCloudDnsRecord")) + continue + } + result.Update() + } } } diff --git a/pkg/compute/models/elastic_search.go b/pkg/compute/models/elastic_search.go index 7b8145f2f3..4171e79b75 100644 --- a/pkg/compute/models/elastic_search.go +++ b/pkg/compute/models/elastic_search.go @@ -259,7 +259,13 @@ func (self *SCloudregion) GetElasticSearchs(managerId string) ([]SElasticSearch, return ret, nil } -func (self *SCloudregion) SyncElasticSearchs(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudElasticSearch) compare.SyncResult { +func (self *SCloudregion) SyncElasticSearchs( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudElasticSearch, + xor bool, +) compare.SyncResult { // 加锁防止重入 lockman.LockRawObject(ctx, ElasticSearchManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) defer lockman.ReleaseRawObject(ctx, ElasticSearchManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) @@ -293,14 +299,16 @@ func (self *SCloudregion) SyncElasticSearchs(ctx context.Context, userCred mccli result.Delete() } - // 和云上资源属性进行同步 - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudElasticSearch(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + // 和云上资源属性进行同步 + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudElasticSearch(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } // 创建本地没有的云上资源 diff --git a/pkg/compute/models/elasticcache_instances.go b/pkg/compute/models/elasticcache_instances.go index 960ea8e84d..5e307a2167 100644 --- a/pkg/compute/models/elasticcache_instances.go +++ b/pkg/compute/models/elasticcache_instances.go @@ -498,9 +498,17 @@ func (manager *SElasticcacheManager) QueryDistinctExtraField(q *sqlchemy.SQuery, return q, httperrors.ErrNotFound } -func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, userCred mcclient.TokenCredential, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, region *SCloudregion, cloudElasticcaches []cloudprovider.ICloudElasticcache) ([]SElasticcache, []cloudprovider.ICloudElasticcache, compare.SyncResult) { - lockman.LockRawObject(ctx, "elastic-cache", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "elastic-cache", fmt.Sprintf("%s-%s", provider.Id, region.Id)) +func (manager *SElasticcacheManager) SyncElasticcaches( + ctx context.Context, + userCred mcclient.TokenCredential, + syncOwnerId mcclient.IIdentityProvider, + provider *SCloudprovider, + region *SCloudregion, + cloudElasticcaches []cloudprovider.ICloudElasticcache, + xor bool, +) ([]SElasticcache, []cloudprovider.ICloudElasticcache, compare.SyncResult) { + lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) localElasticcaches := []SElasticcache{} remoteElasticcaches := []cloudprovider.ICloudElasticcache{} @@ -537,15 +545,17 @@ func (manager *SElasticcacheManager) SyncElasticcaches(ctx context.Context, user } } - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudElasticcache(ctx, userCred, provider, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudElasticcache(ctx, userCred, provider, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + localElasticcaches = append(localElasticcaches, commondb[i]) + remoteElasticcaches = append(remoteElasticcaches, commonext[i]) + syncResult.Update() } - localElasticcaches = append(localElasticcaches, commondb[i]) - remoteElasticcaches = append(remoteElasticcaches, commonext[i]) - syncResult.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/elasticcache_skus.go b/pkg/compute/models/elasticcache_skus.go index 57676c5f91..a00c94a774 100644 --- a/pkg/compute/models/elasticcache_skus.go +++ b/pkg/compute/models/elasticcache_skus.go @@ -413,9 +413,9 @@ func (manager *SElasticcacheSkuManager) FetchSkusByRegion(regionID string) ([]SE return skus, nil } -func (manager *SElasticcacheSkuManager) SyncElasticcacheSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult { - lockman.LockRawObject(ctx, "elastic-cache-skus", region.Id) - defer lockman.ReleaseRawObject(ctx, "elastic-cache-skus", region.Id) +func (manager *SElasticcacheSkuManager) SyncElasticcacheSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta, xor bool) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), region.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), region.Id) syncResult := compare.SyncResult{} @@ -450,12 +450,14 @@ func (manager *SElasticcacheSkuManager) SyncElasticcacheSkus(ctx context.Context syncResult.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - } else { - syncResult.Update() + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } } } for i := 0; i < len(added); i += 1 { diff --git a/pkg/compute/models/elasticips.go b/pkg/compute/models/elasticips.go index e7154e7dcc..bf4cda482c 100644 --- a/pkg/compute/models/elasticips.go +++ b/pkg/compute/models/elasticips.go @@ -375,7 +375,15 @@ func (self *SElasticip) GetShortDesc(ctx context.Context) *jsonutils.JSONDict { return desc } -func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, eips []cloudprovider.ICloudEIP, syncOwnerId mcclient.IIdentityProvider) compare.SyncResult { +func (manager *SElasticipManager) SyncEips( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + region *SCloudregion, + eips []cloudprovider.ICloudEIP, + syncOwnerId mcclient.IIdentityProvider, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, manager.KeywordPlural(), region.Id) defer lockman.ReleaseRawObject(ctx, manager.KeywordPlural(), region.Id) @@ -413,11 +421,13 @@ func (manager *SElasticipManager) SyncEips(ctx context.Context, userCred mcclien syncResult.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudEip(ctx, userCred, provider, commonext[i], syncOwnerId) - if err != nil { - syncResult.UpdateError(err) - } else { + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudEip(ctx, userCred, provider, commonext[i], syncOwnerId) + if err != nil { + syncResult.UpdateError(err) + continue + } syncResult.Update() } } diff --git a/pkg/compute/models/external_projects.go b/pkg/compute/models/external_projects.go index 7cd8d7ed81..706e66343d 100644 --- a/pkg/compute/models/external_projects.go +++ b/pkg/compute/models/external_projects.go @@ -179,9 +179,9 @@ func (manager *SExternalProjectManager) GetProject(externalId string, providerId return project, q.First(project) } -func (manager *SExternalProjectManager) SyncProjects(ctx context.Context, userCred mcclient.TokenCredential, account *SCloudaccount, projects []cloudprovider.ICloudProject) compare.SyncResult { - lockman.LockRawObject(ctx, "external-projects", account.Id) - defer lockman.ReleaseRawObject(ctx, "external-projects", account.Id) +func (manager *SExternalProjectManager) SyncProjects(ctx context.Context, userCred mcclient.TokenCredential, account *SCloudaccount, projects []cloudprovider.ICloudProject, xor bool) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), account.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), account.Id) syncResult := compare.SyncResult{} @@ -210,12 +210,14 @@ func (manager *SExternalProjectManager) SyncProjects(ctx context.Context, userCr syncResult.Delete() } } - for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudProject(ctx, userCred, account, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - } else { - syncResult.Update() + if !xor { + for i := 0; i < len(commondb); i++ { + err = commondb[i].SyncWithCloudProject(ctx, userCred, account, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } } } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/filesystem.go b/pkg/compute/models/filesystem.go index e53798a20f..1a591819cc 100644 --- a/pkg/compute/models/filesystem.go +++ b/pkg/compute/models/filesystem.go @@ -313,9 +313,15 @@ func (self *SCloudregion) GetFileSystems() ([]SFileSystem, error) { return ret, nil } -func (self *SCloudregion) SyncFileSystems(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, filesystems []cloudprovider.ICloudFileSystem) ([]SFileSystem, []cloudprovider.ICloudFileSystem, compare.SyncResult) { - lockman.LockRawObject(ctx, self.Id, "filesystems") - defer lockman.ReleaseRawObject(ctx, self.Id, "filesystems") +func (self *SCloudregion) SyncFileSystems( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + filesystems []cloudprovider.ICloudFileSystem, + xor bool, +) ([]SFileSystem, []cloudprovider.ICloudFileSystem, compare.SyncResult) { + lockman.LockRawObject(ctx, self.Id, FileSystemManager.Keyword()) + defer lockman.ReleaseRawObject(ctx, self.Id, FileSystemManager.Keyword()) result := compare.SyncResult{} @@ -347,10 +353,12 @@ func (self *SCloudregion) SyncFileSystems(ctx context.Context, userCred mcclient result.Delete() } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudFileSystem(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + err = commondb[i].SyncWithCloudFileSystem(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } } localFSs = append(localFSs, commondb[i]) remoteFSs = append(remoteFSs, commonext[i]) @@ -440,7 +448,7 @@ func (self *SFileSystem) ValidateDeleteCondition(ctx context.Context, info jsonu } func (self *SFileSystem) SyncAllWithCloudFileSystem(ctx context.Context, userCred mcclient.TokenCredential, fs cloudprovider.ICloudFileSystem) error { - syncFileSystemMountTargets(ctx, userCred, self, fs) + syncFileSystemMountTargets(ctx, userCred, self, fs, false) return self.SyncWithCloudFileSystem(ctx, userCred, fs) } diff --git a/pkg/compute/models/globalvpcs.go b/pkg/compute/models/globalvpcs.go index 0668655944..d2cca21e7d 100644 --- a/pkg/compute/models/globalvpcs.go +++ b/pkg/compute/models/globalvpcs.go @@ -294,7 +294,7 @@ func (self *SCloudprovider) GetGlobalVpcs() ([]SGlobalVpc, error) { return vpcs, nil } -func (self *SCloudprovider) SyncGlobalVpcs(ctx context.Context, userCred mcclient.TokenCredential, exts []cloudprovider.ICloudGlobalVpc) compare.SyncResult { +func (self *SCloudprovider) SyncGlobalVpcs(ctx context.Context, userCred mcclient.TokenCredential, exts []cloudprovider.ICloudGlobalVpc, xor bool) compare.SyncResult { lockman.LockRawObject(ctx, GlobalVpcManager.Keyword(), self.Id) defer lockman.ReleaseRawObject(ctx, GlobalVpcManager.Keyword(), self.Id) @@ -326,13 +326,15 @@ func (self *SCloudprovider) SyncGlobalVpcs(ctx context.Context, userCred mcclien result.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudGlobalVpc(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudGlobalVpc(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index 8644a7bb6d..07c6387e9b 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -3389,7 +3389,13 @@ func getCloudNicNetwork(ctx context.Context, vnic cloudprovider.ICloudNic, host return localNet, nil } -func (self *SGuest) SyncVMNics(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, vnics []cloudprovider.ICloudNic, ipList []string) compare.SyncResult { +func (self *SGuest) SyncVMNics( + ctx context.Context, + userCred mcclient.TokenCredential, + host *SHost, + vnics []cloudprovider.ICloudNic, + ipList []string, +) compare.SyncResult { result := compare.SyncResult{} nics, err := self.GetNetworks("") @@ -3599,7 +3605,14 @@ func (self *SGuest) attach2Disk(ctx context.Context, disk *SDisk, userCred mccli return err } -func (self *SGuest) SyncVMDisks(ctx context.Context, userCred mcclient.TokenCredential, provider cloudprovider.ICloudProvider, host *SHost, vdisks []cloudprovider.ICloudDisk, syncOwnerId mcclient.IIdentityProvider) compare.SyncResult { +func (self *SGuest) SyncVMDisks( + ctx context.Context, + userCred mcclient.TokenCredential, + provider cloudprovider.ICloudProvider, + host *SHost, + vdisks []cloudprovider.ICloudDisk, + syncOwnerId mcclient.IIdentityProvider, +) compare.SyncResult { lockman.LockRawObject(ctx, self.Id, DiskManager.Keyword()) defer lockman.ReleaseRawObject(ctx, self.Id, DiskManager.Keyword()) diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 443121baf5..dc4228d831 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -1805,13 +1805,13 @@ func (manager *SHostManager) getHostsByZoneProvider(zone *SZone, provider *SClou return hosts, nil } -func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { +func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, hosts []cloudprovider.ICloudHost, xor bool) ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { key := provider.Id if zone != nil { key = fmt.Sprintf("%s-%s", zone.Id, provider.Id) } - lockman.LockRawObject(ctx, "hosts", key) - defer lockman.ReleaseRawObject(ctx, "hosts", key) + lockman.LockRawObject(ctx, manager.Keyword(), key) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), key) localHosts := make([]SHost, 0) remoteHosts := make([]cloudprovider.ICloudHost, 0) @@ -1846,14 +1846,15 @@ func (manager *SHostManager) SyncHosts(ctx context.Context, userCred mcclient.To } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudHost(ctx, userCred, commonext[i], provider) - if err != nil { - syncResult.UpdateError(err) - } else { - localHosts = append(localHosts, commondb[i]) - remoteHosts = append(remoteHosts, commonext[i]) - syncResult.Update() + if !xor { + err = commondb[i].syncWithCloudHost(ctx, userCred, commonext[i], provider) + if err != nil { + syncResult.UpdateError(err) + } } + localHosts = append(localHosts, commondb[i]) + remoteHosts = append(remoteHosts, commonext[i]) + syncResult.Update() } for i := 0; i < len(added); i += 1 { new, err := manager.NewFromCloudHost(ctx, userCred, added[i], provider, zone) @@ -1944,6 +1945,7 @@ func (self *SHost) syncWithCloudHost(ctx context.Context, userCred mcclient.Toke SyncCloudDomain(userCred, self, provider.GetOwnerId()) self.SyncShareState(ctx, userCred, provider.getAccountShareInfo()) } + syncMetadata(ctx, userCred, self, extHost) if err := self.syncSchedtags(ctx, userCred, extHost); err != nil { log.Errorf("syncSchedtags fail: %v", err) @@ -2214,7 +2216,7 @@ func (manager *SHostManager) NewFromCloudHost(ctx context.Context, userCred mccl return &host, nil } -func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, storages []cloudprovider.ICloudStorage, provider *SCloudprovider) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { +func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.TokenCredential, storages []cloudprovider.ICloudStorage, provider *SCloudprovider, xor bool) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { lockman.LockRawObject(ctx, "storages", self.Id) defer lockman.ReleaseRawObject(ctx, "storages", self.Id) @@ -2263,15 +2265,16 @@ func (self *SHost) SyncHostStorages(ctx context.Context, userCred mcclient.Token } for i := 0; i < len(commondb); i += 1 { - log.Infof("host %s is still connected with %s, to update ...", self.Id, commondb[i].Id) - err := self.syncWithCloudHostStorage(ctx, userCred, &commondb[i], commonext[i], provider) - if err != nil { - syncResult.UpdateError(err) - } else { - localStorages = append(localStorages, commondb[i]) - remoteStorages = append(remoteStorages, commonext[i]) - syncResult.Update() + if !xor { + log.Infof("host %s is still connected with %s, to update ...", self.Id, commondb[i].Id) + err := self.syncWithCloudHostStorage(ctx, userCred, &commondb[i], commonext[i], provider) + if err != nil { + syncResult.UpdateError(err) + } } + localStorages = append(localStorages, commondb[i]) + remoteStorages = append(remoteStorages, commonext[i]) + syncResult.Update() } for i := 0; i < len(added); i += 1 { @@ -2494,9 +2497,9 @@ func IsNeedSkipSync(ext cloudprovider.ICloudResource) (bool, string) { return false, "" } -func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, iprovider cloudprovider.ICloudProvider, vms []cloudprovider.ICloudVM, syncOwnerId mcclient.IIdentityProvider) ([]SGuestSyncResult, compare.SyncResult) { - lockman.LockRawObject(ctx, "guests", self.Id) - defer lockman.ReleaseRawObject(ctx, "guests", self.Id) +func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCredential, iprovider cloudprovider.ICloudProvider, vms []cloudprovider.ICloudVM, syncOwnerId mcclient.IIdentityProvider, xor bool) ([]SGuestSyncResult, compare.SyncResult) { + lockman.LockRawObject(ctx, GuestManager.Keyword(), self.Id) + defer lockman.ReleaseRawObject(ctx, GuestManager.Keyword(), self.Id) syncVMPairs := make([]SGuestSyncResult, 0) syncResult := compare.SyncResult{} @@ -2535,30 +2538,32 @@ func (self *SHost) SyncHostVMs(ctx context.Context, userCred mcclient.TokenCrede } } - for i := 0; i < len(commondb); i += 1 { - skip, key := IsNeedSkipSync(commonext[i]) - if skip { - log.Infof("delete server %s(%s) with system tag key: %s", commonext[i].GetName(), commonext[i].GetGlobalId(), key) - err := commondb[i].purge(ctx, userCred) - if err != nil { - syncResult.DeleteError(err) + if !xor { + for i := 0; i < len(commondb); i += 1 { + skip, key := IsNeedSkipSync(commonext[i]) + if skip { + log.Infof("delete server %s(%s) with system tag key: %s", commonext[i].GetName(), commonext[i].GetGlobalId(), key) + err := commondb[i].purge(ctx, userCred) + if err != nil { + syncResult.DeleteError(err) + continue + } + syncResult.Delete() continue } - syncResult.Delete() - continue + err := commondb[i].syncWithCloudVM(ctx, userCred, iprovider, self, commonext[i], syncOwnerId, true) + if err != nil { + syncResult.UpdateError(err) + continue + } + syncVMPair := SGuestSyncResult{ + Local: &commondb[i], + Remote: commonext[i], + IsNew: false, + } + syncVMPairs = append(syncVMPairs, syncVMPair) + syncResult.Update() } - err := commondb[i].syncWithCloudVM(ctx, userCred, iprovider, self, commonext[i], syncOwnerId, true) - if err != nil { - syncResult.UpdateError(err) - continue - } - syncVMPair := SGuestSyncResult{ - Local: &commondb[i], - Remote: commonext[i], - IsNew: false, - } - syncVMPairs = append(syncVMPairs, syncVMPair) - syncResult.Update() } for i := 0; i < len(added); i += 1 { diff --git a/pkg/compute/models/inter_vpc_network.go b/pkg/compute/models/inter_vpc_network.go index a434eb0246..9139bb34de 100644 --- a/pkg/compute/models/inter_vpc_network.go +++ b/pkg/compute/models/inter_vpc_network.go @@ -492,7 +492,7 @@ func (self *SInterVpcNetwork) GetInterVpcNetworkRouteSets() ([]SInterVpcNetworkR return routes, nil } -func (self *SInterVpcNetwork) SyncInterVpcNetworkRouteSets(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudInterVpcNetwork) compare.SyncResult { +func (self *SInterVpcNetwork) SyncInterVpcNetworkRouteSets(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudInterVpcNetwork, xor bool) compare.SyncResult { lockman.LockRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) defer lockman.ReleaseRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) @@ -528,13 +528,15 @@ func (self *SInterVpcNetwork) SyncInterVpcNetworkRouteSets(ctx context.Context, } } - for i := 0; i < len(commondb); i++ { - err := commondb[i].syncWithCloudRouteSet(ctx, userCred, self, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].syncWithCloudRouteSet(ctx, userCred, self, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + syncResult.Update() } - syncResult.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/ipv6_gateways.go b/pkg/compute/models/ipv6_gateways.go index b617a34c51..91a242d18e 100644 --- a/pkg/compute/models/ipv6_gateways.go +++ b/pkg/compute/models/ipv6_gateways.go @@ -82,7 +82,13 @@ func (self *SVpc) GetIPv6Gateways() ([]SIPv6Gateway, error) { return ret, err } -func (self *SVpc) SyncIPv6Gateways(ctx context.Context, userCred mcclient.TokenCredential, exts []cloudprovider.ICloudIPv6Gateway, provider *SCloudprovider) compare.SyncResult { +func (self *SVpc) SyncIPv6Gateways( + ctx context.Context, + userCred mcclient.TokenCredential, + exts []cloudprovider.ICloudIPv6Gateway, + provider *SCloudprovider, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, IPv6GatewayManager.Keyword(), self.Id) defer lockman.ReleaseRawObject(ctx, IPv6GatewayManager.Keyword(), self.Id) @@ -113,13 +119,15 @@ func (self *SVpc) SyncIPv6Gateways(ctx context.Context, userCred mcclient.TokenC result.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudIPv6Gateway(ctx, userCred, commonext[i], provider) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudIPv6Gateway(ctx, userCred, commonext[i], provider) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { _, err := self.newFromCloudIPv6Gateway(ctx, userCred, added[i], provider) diff --git a/pkg/compute/models/kafka.go b/pkg/compute/models/kafka.go index 38b81c07c2..b5f27a7818 100644 --- a/pkg/compute/models/kafka.go +++ b/pkg/compute/models/kafka.go @@ -258,7 +258,13 @@ func (self *SCloudregion) GetKafkas(managerId string) ([]SKafka, error) { return ret, nil } -func (self *SCloudregion) SyncKafkas(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudKafka) compare.SyncResult { +func (self *SCloudregion) SyncKafkas( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudKafka, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, KafkaManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) defer lockman.ReleaseRawObject(ctx, KafkaManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) @@ -291,14 +297,16 @@ func (self *SCloudregion) SyncKafkas(ctx context.Context, userCred mcclient.Toke result.Delete() } - // 和云上资源属性进行同步 - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudKafka(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + // 和云上资源属性进行同步 + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudKafka(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } // 创建本地没有的云上资源 diff --git a/pkg/compute/models/kube_clusters.go b/pkg/compute/models/kube_clusters.go index f0a2a1ef66..c7b4151e0c 100644 --- a/pkg/compute/models/kube_clusters.go +++ b/pkg/compute/models/kube_clusters.go @@ -132,7 +132,13 @@ func (self *SCloudregion) GetKubeClusters(managerId string) ([]SKubeCluster, err return clusters, nil } -func (self *SCloudregion) SyncKubeClusters(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, clusters []cloudprovider.ICloudKubeCluster) ([]SKubeCluster, []cloudprovider.ICloudKubeCluster, compare.SyncResult) { +func (self *SCloudregion) SyncKubeClusters( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + clusters []cloudprovider.ICloudKubeCluster, + xor bool, +) ([]SKubeCluster, []cloudprovider.ICloudKubeCluster, compare.SyncResult) { lockman.LockRawObject(ctx, KubeClusterManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) defer lockman.ReleaseRawObject(ctx, KubeClusterManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) @@ -165,15 +171,17 @@ func (self *SCloudregion) SyncKubeClusters(ctx context.Context, userCred mcclien result.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudKubeCluster(ctx, userCred, commonext[i], provider) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudKubeCluster(ctx, userCred, commonext[i], provider) + if err != nil { + result.UpdateError(err) + continue + } + localClusters = append(localClusters, commondb[i]) + remoteClusters = append(remoteClusters, commonext[i]) + result.Update() } - localClusters = append(localClusters, commondb[i]) - remoteClusters = append(remoteClusters, commonext[i]) - result.Update() } for i := 0; i < len(added); i += 1 { newKubeCluster, err := self.newFromCloudKubeCluster(ctx, userCred, added[i], provider) diff --git a/pkg/compute/models/loadbalancercachedacls.go b/pkg/compute/models/loadbalancercachedacls.go index 67da0bbe14..c7ecad828b 100644 --- a/pkg/compute/models/loadbalancercachedacls.go +++ b/pkg/compute/models/loadbalancercachedacls.go @@ -227,6 +227,7 @@ func (acl *SCachedLoadbalancerAcl) SyncWithCloudLoadbalancerAcl(ctx context.Cont if err != nil { return errors.Wrap(err, "cacheLoadbalancerAcl.sync.Update") } + syncMetadata(ctx, userCred, acl, extAcl) db.OpsLog.LogSyncUpdate(acl, diff, userCred) return nil } @@ -390,9 +391,16 @@ func (man *SCachedLoadbalancerAclManager) getLoadbalancerAclByRegion(provider *S return nil, cloudprovider.ErrNotFound } -func (man *SCachedLoadbalancerAclManager) SyncLoadbalancerAcls(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, acls []cloudprovider.ICloudLoadbalancerAcl, syncRange *SSyncRange) compare.SyncResult { - lockman.LockRawObject(ctx, "acls", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "acls", fmt.Sprintf("%s-%s", provider.Id, region.Id)) +func (man *SCachedLoadbalancerAclManager) SyncLoadbalancerAcls( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + region *SCloudregion, + acls []cloudprovider.ICloudLoadbalancerAcl, + syncRange *SSyncRange, +) compare.SyncResult { + lockman.LockRawObject(ctx, man.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, man.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) syncResult := compare.SyncResult{} @@ -421,21 +429,21 @@ func (man *SCachedLoadbalancerAclManager) SyncLoadbalancerAcls(ctx context.Conte syncResult.Delete() } } - for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerAcl(ctx, userCred, commonext[i], provider.GetOwnerId()) - if err != nil { - syncResult.UpdateError(err) - } else { - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - syncResult.Update() + if !syncRange.Xor { + for i := 0; i < len(commondb); i++ { + err = commondb[i].SyncWithCloudLoadbalancerAcl(ctx, userCred, commonext[i], provider.GetOwnerId()) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } } } for i := 0; i < len(added); i++ { - local, err := man.newFromCloudLoadbalancerAcl(ctx, userCred, provider, added[i], region, provider.GetOwnerId()) + _, err := man.newFromCloudLoadbalancerAcl(ctx, userCred, provider, added[i], region, provider.GetOwnerId()) if err != nil { syncResult.AddError(err) } else { - syncMetadata(ctx, userCred, local, added[i]) syncResult.Add() } } @@ -496,6 +504,7 @@ func (man *SCachedLoadbalancerAclManager) newFromCloudLoadbalancerAcl(ctx contex if err != nil { return nil, errors.Wrap(err, "Insert") } + syncMetadata(ctx, userCred, &acl, extAcl) db.OpsLog.LogEvent(&acl, db.ACT_CREATE, acl.GetShortDesc(ctx), userCred) return &acl, nil diff --git a/pkg/compute/models/loadbalancercachedcertificates.go b/pkg/compute/models/loadbalancercachedcertificates.go index c5fe496ac7..7d549ffe96 100644 --- a/pkg/compute/models/loadbalancercachedcertificates.go +++ b/pkg/compute/models/loadbalancercachedcertificates.go @@ -404,7 +404,13 @@ func (self *SCloudprovider) getLoadbalancerCertificatesByRegion(region *SCloudre return ret, nil } -func (self *SCloudprovider) SyncLoadbalancerCertificates(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, certificates []cloudprovider.ICloudLoadbalancerCertificate) compare.SyncResult { +func (self *SCloudprovider) SyncLoadbalancerCertificates( + ctx context.Context, + userCred mcclient.TokenCredential, + region *SCloudregion, + certificates []cloudprovider.ICloudLoadbalancerCertificate, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, CachedLoadbalancerCertificateManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, region.Id)) defer lockman.ReleaseRawObject(ctx, CachedLoadbalancerCertificateManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, region.Id)) @@ -435,13 +441,15 @@ func (self *SCloudprovider) SyncLoadbalancerCertificates(ctx context.Context, us } syncResult.Delete() } - for i := 0; i < len(commondb); i++ { - err = commondb[i].SyncWithCloudLoadbalancerCertificate(ctx, userCred, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err = commondb[i].SyncWithCloudLoadbalancerCertificate(ctx, userCred, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + syncResult.Update() } - syncResult.Update() } for i := 0; i < len(added); i++ { err := self.newFromCloudLoadbalancerCertificate(ctx, userCred, added[i], region) diff --git a/pkg/compute/models/loadbalancers.go b/pkg/compute/models/loadbalancers.go index 22bc43b57a..506663c3ce 100644 --- a/pkg/compute/models/loadbalancers.go +++ b/pkg/compute/models/loadbalancers.go @@ -965,7 +965,14 @@ func (man *SLoadbalancerManager) getLocalLoadbalancers(ctx context.Context, user return ret, nil } -func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, lbs []cloudprovider.ICloudLoadbalancer) ([]SLoadbalancer, []cloudprovider.ICloudLoadbalancer, compare.SyncResult) { +func (man *SLoadbalancerManager) SyncLoadbalancers( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + region *SCloudregion, + lbs []cloudprovider.ICloudLoadbalancer, + xor bool, +) ([]SLoadbalancer, []cloudprovider.ICloudLoadbalancer, compare.SyncResult) { lockman.LockRawObject(ctx, man.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) defer lockman.ReleaseRawObject(ctx, man.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) @@ -1005,15 +1012,17 @@ func (man *SLoadbalancerManager) SyncLoadbalancers(ctx context.Context, userCred syncResult.Delete() } } - for i := 0; i < len(commondb); i++ { - err = commondb[i].syncWithCloudLoadbalancer(ctx, userCred, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err = commondb[i].syncWithCloudLoadbalancer(ctx, userCred, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + localLbs = append(localLbs, commondb[i]) + remoteLbs = append(remoteLbs, commonext[i]) + syncResult.Update() } - localLbs = append(localLbs, commondb[i]) - remoteLbs = append(remoteLbs, commonext[i]) - syncResult.Update() } for i := 0; i < len(added); i++ { lb, err := region.newFromCloudLoadbalancer(ctx, userCred, provider, added[i]) diff --git a/pkg/compute/models/miscresources.go b/pkg/compute/models/miscresources.go index a0aa52c348..72ac185425 100644 --- a/pkg/compute/models/miscresources.go +++ b/pkg/compute/models/miscresources.go @@ -108,7 +108,13 @@ func (self *SCloudregion) GetMiscResources() ([]SMiscResource, error) { return misc, nil } -func (self *SCloudregion) SyncMiscResources(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudMiscResource) compare.SyncResult { +func (self *SCloudregion) SyncMiscResources( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudMiscResource, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, CloudproviderManager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) defer lockman.ReleaseRawObject(ctx, CloudproviderManager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) @@ -139,13 +145,15 @@ func (self *SCloudregion) SyncMiscResources(ctx context.Context, userCred mcclie } result.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudMiscResource(ctx, userCred, commonext[i], provider) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudMiscResource(ctx, userCred, commonext[i], provider) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { _, err := self.newFromCloudMiscResource(ctx, userCred, added[i], provider) diff --git a/pkg/compute/models/modelarts_pool_sku.go b/pkg/compute/models/modelarts_pool_sku.go index d72966482a..bcc28d9478 100644 --- a/pkg/compute/models/modelarts_pool_sku.go +++ b/pkg/compute/models/modelarts_pool_sku.go @@ -197,10 +197,16 @@ func (self *SCloudregion) GetModelartsPoolSkus() ([]SModelartsPoolSku, error) { return ret, nil } -func (self *SCloudregion) SyncModelartsPoolSkus(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudModelartsPoolSku) compare.SyncResult { +func (self *SCloudregion) SyncModelartsPoolSkus( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudModelartsPoolSku, + xor bool, +) compare.SyncResult { // 加锁防止重入 - lockman.LockRawObject(ctx, self.Provider, "modelarts-pool-sku") - defer lockman.ReleaseRawObject(ctx, self.Provider, "modelarts-pool-sku") + lockman.LockRawObject(ctx, self.Provider, ModelartsPoolSkuManager.Keyword()) + defer lockman.ReleaseRawObject(ctx, self.Provider, ModelartsPoolSkuManager.Keyword()) result := compare.SyncResult{} dbPoolSku, err := self.GetModelartsPoolSkus() if err != nil { @@ -228,14 +234,16 @@ func (self *SCloudregion) SyncModelartsPoolSkus(ctx context.Context, userCred mc result.Delete() } - // 和云上资源属性进行同步 - for i := 0; i < len(commondb); i++ { - err := commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + // 和云上资源属性进行同步 + for i := 0; i < len(commondb); i++ { + err := commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } // 创建本地没有的云上资源 diff --git a/pkg/compute/models/modelarts_pools.go b/pkg/compute/models/modelarts_pools.go index a35fd0d367..15bbf63208 100644 --- a/pkg/compute/models/modelarts_pools.go +++ b/pkg/compute/models/modelarts_pools.go @@ -235,7 +235,13 @@ func (self *SCloudregion) GetPools(managerId string) ([]SModelartsPool, error) { return ret, nil } -func (self *SCloudregion) SyncModelartsPools(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudModelartsPool) compare.SyncResult { +func (self *SCloudregion) SyncModelartsPools( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudModelartsPool, + xor bool, +) compare.SyncResult { // 加锁防止重入 lockman.LockRawObject(ctx, ModelartsPoolManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) defer lockman.ReleaseRawObject(ctx, ModelartsPoolManager.KeywordPlural(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) @@ -267,14 +273,16 @@ func (self *SCloudregion) SyncModelartsPools(ctx context.Context, userCred mccli result.Delete() } - // 和云上资源属性进行同步 - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudModelartsPool(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + // 和云上资源属性进行同步 + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudModelartsPool(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } // 创建本地没有的云上资源 diff --git a/pkg/compute/models/mongodb.go b/pkg/compute/models/mongodb.go index b963b8a69a..166adf8814 100644 --- a/pkg/compute/models/mongodb.go +++ b/pkg/compute/models/mongodb.go @@ -405,9 +405,15 @@ func (self *SCloudregion) GetMongoDBs(managerId string) ([]SMongoDB, error) { return dbs, nil } -func (self *SCloudregion) SyncMongoDBs(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, cloudMongoDBs []cloudprovider.ICloudMongoDB) ([]SMongoDB, []cloudprovider.ICloudMongoDB, compare.SyncResult) { - lockman.LockRawObject(ctx, "mongodbs", fmt.Sprintf("%s-%s", provider.Id, self.Id)) - defer lockman.ReleaseRawObject(ctx, "mongodbs", fmt.Sprintf("%s-%s", provider.Id, self.Id)) +func (self *SCloudregion) SyncMongoDBs( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + cloudMongoDBs []cloudprovider.ICloudMongoDB, + xor bool, +) ([]SMongoDB, []cloudprovider.ICloudMongoDB, compare.SyncResult) { + lockman.LockRawObject(ctx, MongoDBManager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) + defer lockman.ReleaseRawObject(ctx, MongoDBManager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, self.Id)) localMongoDBs := []SMongoDB{} remoteMongoDBs := []cloudprovider.ICloudMongoDB{} @@ -438,15 +444,17 @@ func (self *SCloudregion) SyncMongoDBs(ctx context.Context, userCred mcclient.To result.Delete() } - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudMongoDB(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudMongoDB(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + localMongoDBs = append(localMongoDBs, commondb[i]) + remoteMongoDBs = append(remoteMongoDBs, commonext[i]) + result.Update() } - localMongoDBs = append(localMongoDBs, commondb[i]) - remoteMongoDBs = append(remoteMongoDBs, commonext[i]) - result.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/mount_targets.go b/pkg/compute/models/mount_targets.go index d8e339d371..8328bde877 100644 --- a/pkg/compute/models/mount_targets.go +++ b/pkg/compute/models/mount_targets.go @@ -386,7 +386,12 @@ func (self *SFileSystem) GetMountTargets() ([]SMountTarget, error) { return mounts, nil } -func (self *SFileSystem) SyncMountTargets(ctx context.Context, userCred mcclient.TokenCredential, extMounts []cloudprovider.ICloudMountTarget) compare.SyncResult { +func (self *SFileSystem) SyncMountTargets( + ctx context.Context, + userCred mcclient.TokenCredential, + extMounts []cloudprovider.ICloudMountTarget, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, self.Id, MountTargetManager.KeywordPlural()) lockman.ReleaseRawObject(ctx, self.Id, MountTargetManager.KeywordPlural()) @@ -416,13 +421,15 @@ func (self *SFileSystem) SyncMountTargets(ctx context.Context, userCred mcclient } result.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithMountTarget(ctx, userCred, self.ManagerId, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithMountTarget(ctx, userCred, self.ManagerId, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { err := self.newFromCloudMountTarget(ctx, userCred, added[i]) diff --git a/pkg/compute/models/nas_skus.go b/pkg/compute/models/nas_skus.go index 2d47b44fad..06ab8e21d2 100644 --- a/pkg/compute/models/nas_skus.go +++ b/pkg/compute/models/nas_skus.go @@ -198,7 +198,7 @@ func (self SNasSku) GetGlobalId() string { return self.ExternalId } -func (self *SCloudregion) SyncNasSkus(ctx context.Context, userCred mcclient.TokenCredential, meta *SSkuResourcesMeta) compare.SyncResult { +func (self *SCloudregion) SyncNasSkus(ctx context.Context, userCred mcclient.TokenCredential, meta *SSkuResourcesMeta, xor bool) compare.SyncResult { lockman.LockRawObject(ctx, self.Id, "nas-sku") defer lockman.ReleaseRawObject(ctx, self.Id, "nas-sku") @@ -235,13 +235,15 @@ func (self *SCloudregion) SyncNasSkus(ctx context.Context, userCred mcclient.Tok } syncResult.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + syncResult.Update() } - syncResult.Update() } for i := 0; i < len(added); i += 1 { err = self.newFromCloudNasSku(ctx, userCred, added[i]) @@ -274,13 +276,13 @@ func (self *SCloudregion) newFromCloudNasSku(ctx context.Context, userCred mccli } func SyncNasSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { - err := SyncRegionNasSkus(ctx, userCred, "", isStart) + err := SyncRegionNasSkus(ctx, userCred, "", isStart, false) if err != nil { log.Errorf("SyncRegionNasSkus error: %v", err) } } -func SyncRegionNasSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart bool) error { +func SyncRegionNasSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart, xor bool) error { if isStart { q := NasSkuManager.Query() if len(regionId) > 0 { @@ -317,7 +319,7 @@ func SyncRegionNasSkus(ctx context.Context, userCred mcclient.TokenCredential, r log.Infof("region %s(%s) not support nas, skip sync", regions[i].Name, regions[i].Id) continue } - result := regions[i].SyncNasSkus(ctx, userCred, meta) + result := regions[i].SyncNasSkus(ctx, userCred, meta, xor) msg := result.Result() notes := fmt.Sprintf("SyncNasSkus for region %s result: %s", regions[i].Name, msg) log.Infof(notes) diff --git a/pkg/compute/models/nat_skus.go b/pkg/compute/models/nat_skus.go index d69fe80809..0911a9387d 100644 --- a/pkg/compute/models/nat_skus.go +++ b/pkg/compute/models/nat_skus.go @@ -201,9 +201,9 @@ func (self SNatSku) GetGlobalId() string { return self.ExternalId } -func (self *SCloudregion) SyncNatSkus(ctx context.Context, userCred mcclient.TokenCredential, meta *SSkuResourcesMeta) compare.SyncResult { - lockman.LockRawObject(ctx, self.Id, "nat-sku") - defer lockman.ReleaseRawObject(ctx, self.Id, "nat-sku") +func (self *SCloudregion) SyncNatSkus(ctx context.Context, userCred mcclient.TokenCredential, meta *SSkuResourcesMeta, xor bool) compare.SyncResult { + lockman.LockRawObject(ctx, self.Id, NatSkuManager.Keyword()) + defer lockman.ReleaseRawObject(ctx, self.Id, NatSkuManager.Keyword()) syncResult := compare.SyncResult{} @@ -238,13 +238,15 @@ func (self *SCloudregion) SyncNatSkus(ctx context.Context, userCred mcclient.Tok } syncResult.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + syncResult.Update() } - syncResult.Update() } for i := 0; i < len(added); i += 1 { err = self.newFromCloudNatSku(ctx, userCred, added[i]) @@ -277,13 +279,13 @@ func (self *SCloudregion) newFromCloudNatSku(ctx context.Context, userCred mccli } func SyncNatSkus(ctx context.Context, userCred mcclient.TokenCredential, isStart bool) { - err := SyncRegionNatSkus(ctx, userCred, "", isStart) + err := SyncRegionNatSkus(ctx, userCred, "", isStart, false) if err != nil { log.Errorf("SyncRegionNatSkus error: %v", err) } } -func SyncRegionNatSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart bool) error { +func SyncRegionNatSkus(ctx context.Context, userCred mcclient.TokenCredential, regionId string, isStart, xor bool) error { if isStart { q := NatSkuManager.Query() if len(regionId) > 0 { @@ -320,7 +322,7 @@ func SyncRegionNatSkus(ctx context.Context, userCred mcclient.TokenCredential, r log.Infof("region %s(%s) not support nat, skip sync", regions[i].Name, regions[i].Id) continue } - result := regions[i].SyncNatSkus(ctx, userCred, meta) + result := regions[i].SyncNatSkus(ctx, userCred, meta, xor) msg := result.Result() notes := fmt.Sprintf("SyncNatSkus for region %s result: %s", regions[i].Name, msg) log.Infof(notes) diff --git a/pkg/compute/models/natdtable.go b/pkg/compute/models/natdtable.go index a7eb1b3a18..2f285d552a 100644 --- a/pkg/compute/models/natdtable.go +++ b/pkg/compute/models/natdtable.go @@ -176,11 +176,18 @@ func (man *SNatDEntryManager) ValidateCreateData(ctx context.Context, userCred m return input, nil } -func (manager *SNatDEntryManager) SyncNatDTable(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, nat *SNatGateway, extDTable []cloudprovider.ICloudNatDEntry) compare.SyncResult { +func (manager *SNatDEntryManager) SyncNatDTable( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + nat *SNatGateway, + extDTable []cloudprovider.ICloudNatDEntry, + xor bool, +) compare.SyncResult { syncOwnerId := provider.GetOwnerId() - lockman.LockRawObject(ctx, "dtable", nat.Id) - defer lockman.ReleaseRawObject(ctx, "dtable", nat.Id) + lockman.LockRawObject(ctx, manager.Keyword(), nat.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), nat.Id) result := compare.SyncResult{} dbNatDTables, err := nat.GetDTable() @@ -207,23 +214,23 @@ func (manager *SNatDEntryManager) SyncNatDTable(ctx context.Context, userCred mc } } - for i := 0; i < len(commondb); i += 1 { - err := commondb[i].SyncWithCloudNatDTable(ctx, userCred, commonext[i], syncOwnerId) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err := commondb[i].SyncWithCloudNatDTable(ctx, userCred, commonext[i], syncOwnerId) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - result.Update() } for i := 0; i < len(added); i += 1 { - routeTableNew, err := manager.newFromCloudNatDTable(ctx, userCred, syncOwnerId, nat, added[i]) + _, err := manager.newFromCloudNatDTable(ctx, userCred, syncOwnerId, nat, added[i]) if err != nil { result.AddError(err) continue } - syncMetadata(ctx, userCred, routeTableNew, added[i]) result.Add() } return result @@ -264,6 +271,7 @@ func (self *SNatDEntry) SyncWithCloudNatDTable(ctx context.Context, userCred mcc SyncCloudDomain(userCred, self, syncOwnerId) + syncMetadata(ctx, userCred, self, extEntry) db.OpsLog.LogSyncUpdate(self, diff, userCred) return nil } @@ -299,6 +307,7 @@ func (manager *SNatDEntryManager) newFromCloudNatDTable(ctx context.Context, use } SyncCloudDomain(userCred, &table, ownerId) + syncMetadata(ctx, userCred, &table, extEntry) db.OpsLog.LogEvent(&table, db.ACT_CREATE, table.GetShortDesc(ctx), userCred) diff --git a/pkg/compute/models/natgateways.go b/pkg/compute/models/natgateways.go index d2121ef79a..ccf39572d8 100644 --- a/pkg/compute/models/natgateways.go +++ b/pkg/compute/models/natgateways.go @@ -392,9 +392,17 @@ func (manager *SNatGatewayManager) FetchCustomizeColumns( return rows } -func (manager *SNatGatewayManager) SyncNatGateways(ctx context.Context, userCred mcclient.TokenCredential, syncOwnerId mcclient.IIdentityProvider, provider *SCloudprovider, vpc *SVpc, cloudNatGateways []cloudprovider.ICloudNatGateway) ([]SNatGateway, []cloudprovider.ICloudNatGateway, compare.SyncResult) { - lockman.LockRawObject(ctx, "natgateways", vpc.Id) - defer lockman.ReleaseRawObject(ctx, "natgateways", vpc.Id) +func (manager *SNatGatewayManager) SyncNatGateways( + ctx context.Context, + userCred mcclient.TokenCredential, + syncOwnerId mcclient.IIdentityProvider, + provider *SCloudprovider, + vpc *SVpc, + cloudNatGateways []cloudprovider.ICloudNatGateway, + xor bool, +) ([]SNatGateway, []cloudprovider.ICloudNatGateway, compare.SyncResult) { + lockman.LockRawObject(ctx, manager.Keyword(), vpc.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), vpc.Id) localNatGateways := make([]SNatGateway, 0) remoteNatGateways := make([]cloudprovider.ICloudNatGateway, 0) @@ -425,10 +433,12 @@ func (manager *SNatGatewayManager) SyncNatGateways(ctx context.Context, userCred } for i := 0; i < len(commondb); i += 1 { - err := commondb[i].SyncWithCloudNatGateway(ctx, userCred, provider, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + err := commondb[i].SyncWithCloudNatGateway(ctx, userCred, provider, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } } localNatGateways = append(localNatGateways, commondb[i]) remoteNatGateways = append(remoteNatGateways, commonext[i]) diff --git a/pkg/compute/models/natstable.go b/pkg/compute/models/natstable.go index 9fa5c8e0f0..6c9968b878 100644 --- a/pkg/compute/models/natstable.go +++ b/pkg/compute/models/natstable.go @@ -230,11 +230,18 @@ func (man *SNatSEntryManager) ValidateCreateData(ctx context.Context, userCred m return input, nil } -func (manager *SNatSEntryManager) SyncNatSTable(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, nat *SNatGateway, extTable []cloudprovider.ICloudNatSEntry) compare.SyncResult { +func (manager *SNatSEntryManager) SyncNatSTable( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + nat *SNatGateway, + extTable []cloudprovider.ICloudNatSEntry, + xor bool, +) compare.SyncResult { syncOwnerId := provider.GetOwnerId() - lockman.LockRawObject(ctx, "stable", nat.Id) - defer lockman.ReleaseRawObject(ctx, "stable", nat.Id) + lockman.LockRawObject(ctx, manager.Keyword(), nat.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), nat.Id) result := compare.SyncResult{} dbNatSTables, err := nat.GetSTable() @@ -261,23 +268,23 @@ func (manager *SNatSEntryManager) SyncNatSTable(ctx context.Context, userCred mc } } - for i := 0; i < len(commondb); i += 1 { - err := commondb[i].SyncWithCloudNatSTable(ctx, userCred, commonext[i], syncOwnerId, provider.Id) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err := commondb[i].SyncWithCloudNatSTable(ctx, userCred, commonext[i], syncOwnerId, provider.Id) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - result.Update() } for i := 0; i < len(added); i += 1 { - routeTableNew, err := manager.newFromCloudNatSTable(ctx, userCred, syncOwnerId, nat, added[i], provider.Id) + _, err := manager.newFromCloudNatSTable(ctx, userCred, syncOwnerId, nat, added[i], provider.Id) if err != nil { result.AddError(err) continue } - syncMetadata(ctx, userCred, routeTableNew, added[i]) result.Add() } return result @@ -319,6 +326,7 @@ func (self *SNatSEntry) SyncWithCloudNatSTable(ctx context.Context, userCred mcc } SyncCloudDomain(userCred, self, syncOwnerId) + syncMetadata(ctx, userCred, self, extEntry) db.OpsLog.LogSyncUpdate(self, diff, userCred) return nil @@ -366,6 +374,7 @@ func (manager *SNatSEntryManager) newFromCloudNatSTable(ctx context.Context, use } SyncCloudDomain(userCred, &table, ownerId) + syncMetadata(ctx, userCred, &table, extEntry) db.OpsLog.LogEvent(&table, db.ACT_CREATE, table.GetShortDesc(ctx), userCred) diff --git a/pkg/compute/models/networkinterfaces.go b/pkg/compute/models/networkinterfaces.go index 45e928eb72..057a411056 100644 --- a/pkg/compute/models/networkinterfaces.go +++ b/pkg/compute/models/networkinterfaces.go @@ -214,9 +214,16 @@ func (manager *SNetworkInterfaceManager) getNetworkInterfacesByProviderId(provid return nics, nil } -func (manager *SNetworkInterfaceManager) SyncNetworkInterfaces(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, exts []cloudprovider.ICloudNetworkInterface) ([]SNetworkInterface, []cloudprovider.ICloudNetworkInterface, compare.SyncResult) { - lockman.LockRawObject(ctx, "network-interfaces", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "network-interfaces", fmt.Sprintf("%s-%s", provider.Id, region.Id)) +func (manager *SNetworkInterfaceManager) SyncNetworkInterfaces( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + region *SCloudregion, + exts []cloudprovider.ICloudNetworkInterface, + xor bool, +) ([]SNetworkInterface, []cloudprovider.ICloudNetworkInterface, compare.SyncResult) { + lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) localResources := make([]SNetworkInterface, 0) remoteResources := make([]cloudprovider.ICloudNetworkInterface, 0) @@ -246,16 +253,17 @@ func (manager *SNetworkInterfaceManager) SyncNetworkInterfaces(ctx context.Conte } } - for i := 0; i < len(commondb); i += 1 { - err := commondb[i].SyncWithCloudNetworkInterface(ctx, userCred, provider, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err := commondb[i].SyncWithCloudNetworkInterface(ctx, userCred, provider, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + localResources = append(localResources, commondb[i]) + remoteResources = append(remoteResources, commonext[i]) + syncResult.Update() } - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - localResources = append(localResources, commondb[i]) - remoteResources = append(remoteResources, commonext[i]) - syncResult.Update() } for i := 0; i < len(added); i += 1 { @@ -311,6 +319,7 @@ func (self *SNetworkInterface) SyncWithCloudNetworkInterface(ctx context.Context } SyncCloudDomain(userCred, self, provider.GetOwnerId()) + syncMetadata(ctx, userCred, self, ext) db.OpsLog.LogSyncUpdate(self, diff, userCred) return nil } diff --git a/pkg/compute/models/networks.go b/pkg/compute/models/networks.go index 45478d0e59..1f16903286 100644 --- a/pkg/compute/models/networks.go +++ b/pkg/compute/models/networks.go @@ -549,11 +549,18 @@ func (manager *SNetworkManager) getNetworksByWire(wire *SWire) ([]SNetwork, erro return nets, nil */ } -func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mcclient.TokenCredential, wire *SWire, nets []cloudprovider.ICloudNetwork, provider *SCloudprovider) ([]SNetwork, []cloudprovider.ICloudNetwork, compare.SyncResult) { +func (manager *SNetworkManager) SyncNetworks( + ctx context.Context, + userCred mcclient.TokenCredential, + wire *SWire, + nets []cloudprovider.ICloudNetwork, + provider *SCloudprovider, + xor bool, +) ([]SNetwork, []cloudprovider.ICloudNetwork, compare.SyncResult) { syncOwnerId := provider.GetOwnerId() - lockman.LockRawObject(ctx, "networks", wire.Id) - defer lockman.ReleaseRawObject(ctx, "networks", wire.Id) + lockman.LockRawObject(ctx, manager.Keyword(), wire.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), wire.Id) localNets := make([]SNetwork, 0) remoteNets := make([]cloudprovider.ICloudNetwork, 0) @@ -591,11 +598,13 @@ func (manager *SNetworkManager) SyncNetworks(ctx context.Context, userCred mccli syncResult.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudNetwork(ctx, userCred, commonext[i], syncOwnerId, provider) - if err != nil { - syncResult.UpdateError(err) - } else { + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudNetwork(ctx, userCred, commonext[i], syncOwnerId, provider) + if err != nil { + syncResult.UpdateError(err) + continue + } localNets = append(localNets, commondb[i]) remoteNets = append(remoteNets, commonext[i]) syncResult.Update() diff --git a/pkg/compute/models/routetables.go b/pkg/compute/models/routetables.go index 08050a0796..b668066b15 100644 --- a/pkg/compute/models/routetables.go +++ b/pkg/compute/models/routetables.go @@ -356,9 +356,16 @@ func (manager *SRouteTableManager) FetchCustomizeColumns( return rows } -func (man *SRouteTableManager) SyncRouteTables(ctx context.Context, userCred mcclient.TokenCredential, vpc *SVpc, cloudRouteTables []cloudprovider.ICloudRouteTable, provider *SCloudprovider) ([]SRouteTable, []cloudprovider.ICloudRouteTable, compare.SyncResult) { - lockman.LockRawObject(ctx, "route-tables", vpc.Id) - defer lockman.ReleaseRawObject(ctx, "route-tables", vpc.Id) +func (man *SRouteTableManager) SyncRouteTables( + ctx context.Context, + userCred mcclient.TokenCredential, + vpc *SVpc, + cloudRouteTables []cloudprovider.ICloudRouteTable, + provider *SCloudprovider, + xor bool, +) ([]SRouteTable, []cloudprovider.ICloudRouteTable, compare.SyncResult) { + lockman.LockRawObject(ctx, man.Keyword(), vpc.Id) + defer lockman.ReleaseRawObject(ctx, man.Keyword(), vpc.Id) localRouteTables := make([]SRouteTable, 0) remoteRouteTables := make([]cloudprovider.ICloudRouteTable, 0) @@ -387,16 +394,17 @@ func (man *SRouteTableManager) SyncRouteTables(ctx context.Context, userCred mcc } } - for i := 0; i < len(commondb); i += 1 { - err := commondb[i].SyncWithCloudRouteTable(ctx, userCred, vpc, commonext[i], provider) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err := commondb[i].SyncWithCloudRouteTable(ctx, userCred, vpc, commonext[i], provider) + if err != nil { + syncResult.UpdateError(err) + continue + } + localRouteTables = append(localRouteTables, commondb[i]) + remoteRouteTables = append(remoteRouteTables, commonext[i]) + syncResult.Update() } - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - localRouteTables = append(localRouteTables, commondb[i]) - remoteRouteTables = append(remoteRouteTables, commonext[i]) - syncResult.Update() } for i := 0; i < len(added); i += 1 { @@ -522,12 +530,18 @@ func (self *SRouteTable) SyncWithCloudRouteTable(ctx context.Context, userCred m SyncCloudDomain(userCred, self, provider.GetOwnerId()) self.SyncShareState(ctx, userCred, provider.getAccountShareInfo()) } - + syncMetadata(ctx, userCred, self, cloudRouteTable) db.OpsLog.LogSyncUpdate(self, diff, userCred) return nil } -func (self *SRouteTable) SyncRouteTableRouteSets(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudRouteTable, provider *SCloudprovider) compare.SyncResult { +func (self *SRouteTable) SyncRouteTableRouteSets( + ctx context.Context, + userCred mcclient.TokenCredential, + ext cloudprovider.ICloudRouteTable, + provider *SCloudprovider, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) defer lockman.ReleaseRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) @@ -569,13 +583,15 @@ func (self *SRouteTable) SyncRouteTableRouteSets(ctx context.Context, userCred m } } - for i := 0; i < len(commondb); i++ { - err := commondb[i].syncWithCloudRouteSet(ctx, userCred, provider, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].syncWithCloudRouteSet(ctx, userCred, provider, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + syncResult.Update() } - syncResult.Update() } for i := 0; i < len(added); i++ { @@ -600,7 +616,13 @@ func (self *SRouteTable) GetRouteTableRouteSets() ([]SRouteTableRouteSet, error) return routes, nil } -func (self *SRouteTable) SyncRouteTableAssociations(ctx context.Context, userCred mcclient.TokenCredential, ext cloudprovider.ICloudRouteTable, provider *SCloudprovider) compare.SyncResult { +func (self *SRouteTable) SyncRouteTableAssociations( + ctx context.Context, + userCred mcclient.TokenCredential, + ext cloudprovider.ICloudRouteTable, + provider *SCloudprovider, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) defer lockman.ReleaseRawObject(ctx, self.Keyword(), fmt.Sprintf("%s-records", self.Id)) @@ -638,13 +660,15 @@ func (self *SRouteTable) SyncRouteTableAssociations(ctx context.Context, userCre } } - for i := 0; i < len(commondb); i++ { - err := commondb[i].syncWithCloudAssociation(ctx, userCred, provider, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].syncWithCloudAssociation(ctx, userCred, provider, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + continue + } + syncResult.Update() } - syncResult.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/secgroupcache.go b/pkg/compute/models/secgroupcache.go index 56501e1ee9..927c556f9a 100644 --- a/pkg/compute/models/secgroupcache.go +++ b/pkg/compute/models/secgroupcache.go @@ -418,9 +418,16 @@ func (self *SSecurityGroupCache) syncWithCloudSecurityGroup(ctx context.Context, return rules, nil } -func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, secgroups []cloudprovider.ICloudSecurityGroup, vpc *SVpc) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { - lockman.LockRawObject(ctx, "secgroups", vpc.Id) - defer lockman.ReleaseRawObject(ctx, "secgroups", vpc.Id) +func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + secgroups []cloudprovider.ICloudSecurityGroup, + vpc *SVpc, + xor bool, +) ([]SSecurityGroup, []cloudprovider.ICloudSecurityGroup, compare.SyncResult) { + lockman.LockRawObject(ctx, manager.Keyword(), vpc.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), vpc.Id) localSecgroups := []SSecurityGroup{} remoteSecgroups := []cloudprovider.ICloudSecurityGroup{} @@ -483,14 +490,16 @@ func (manager *SSecurityGroupCacheManager) SyncSecurityGroupCaches(ctx context.C rules := []SSecurityGroupRule{} - for i := 0; i < len(commondb); i++ { - _rules, err := commondb[i].syncWithCloudSecurityGroup(ctx, userCred, provider, commonext[i]) - if err != nil { - syncResult.UpdateError(errors.Wrapf(err, "syncWithCloudSecurityGroup")) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + _rules, err := commondb[i].syncWithCloudSecurityGroup(ctx, userCred, provider, commonext[i]) + if err != nil { + syncResult.UpdateError(errors.Wrapf(err, "syncWithCloudSecurityGroup")) + continue + } + rules = append(rules, _rules...) + syncResult.Update() } - rules = append(rules, _rules...) - syncResult.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/skus.go b/pkg/compute/models/skus.go index c7ad47535f..ebec1d156b 100644 --- a/pkg/compute/models/skus.go +++ b/pkg/compute/models/skus.go @@ -997,9 +997,15 @@ func (manager *SServerSkuManager) PendingDeleteInvalidSku() error { return nil } -func (manager *SServerSkuManager) SyncPrivateCloudSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, skus []cloudprovider.ICloudSku) compare.SyncResult { - lockman.LockRawObject(ctx, "serverskus", region.Id) - defer lockman.ReleaseRawObject(ctx, "serverskus", region.Id) +func (manager *SServerSkuManager) SyncPrivateCloudSkus( + ctx context.Context, + userCred mcclient.TokenCredential, + region *SCloudregion, + skus []cloudprovider.ICloudSku, + xor bool, +) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), region.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), region.Id) result := compare.SyncResult{} @@ -1029,12 +1035,14 @@ func (manager *SServerSkuManager) SyncPrivateCloudSkus(ctx context.Context, user result.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithPrivateCloudSku(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithPrivateCloudSku(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { @@ -1176,9 +1184,9 @@ func (manager *SServerSkuManager) FetchSkusByRegion(regionID string) ([]SServerS return skus, nil } -func (manager *SServerSkuManager) SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult { - lockman.LockRawObject(ctx, "serverskus", region.Id) - defer lockman.ReleaseRawObject(ctx, "serverskus", region.Id) +func (manager *SServerSkuManager) SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta, xor bool) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), region.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), region.Id) syncResult := compare.SyncResult{} @@ -1213,12 +1221,14 @@ func (manager *SServerSkuManager) SyncServerSkus(ctx context.Context, userCred m syncResult.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) - if err != nil { - syncResult.UpdateError(err) - } else { - syncResult.Update() + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithCloudSku(ctx, userCred, commonext[i]) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } } } for i := 0; i < len(added); i += 1 { diff --git a/pkg/compute/models/skus_tools.go b/pkg/compute/models/skus_tools.go index d8523781e0..845605de7b 100644 --- a/pkg/compute/models/skus_tools.go +++ b/pkg/compute/models/skus_tools.go @@ -478,7 +478,7 @@ func SyncElasticCacheSkus(ctx context.Context, userCred mcclient.TokenCredential region := &cloudregions[i] if region.GetDriver().IsSupportedElasticcache() { - result := ElasticcacheSkuManager.SyncElasticcacheSkus(ctx, userCred, region, meta) + result := ElasticcacheSkuManager.SyncElasticcacheSkus(ctx, userCred, region, meta, false) notes := fmt.Sprintf("SyncElasticCacheSkusByRegion %s result: %s", region.Name, result.Result()) log.Infof(notes) } else { @@ -489,7 +489,7 @@ func SyncElasticCacheSkus(ctx context.Context, userCred mcclient.TokenCredential } // 同步Region elasticcache sku列表. -func SyncElasticCacheSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion) error { +func SyncElasticCacheSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, xor bool) error { if !region.GetDriver().IsSupportedElasticcache() { notes := fmt.Sprintf("SyncElasticCacheSkusByRegion %s not support elasticcache", region.Name) log.Infof(notes) @@ -501,7 +501,7 @@ func SyncElasticCacheSkusByRegion(ctx context.Context, userCred mcclient.TokenCr return errors.Wrap(err, "SyncElasticCacheSkusByRegion.FetchSkuResourcesMeta") } - result := ElasticcacheSkuManager.SyncElasticcacheSkus(ctx, userCred, region, meta) + result := ElasticcacheSkuManager.SyncElasticcacheSkus(ctx, userCred, region, meta, xor) notes := fmt.Sprintf("SyncElasticCacheSkusByRegion %s result: %s", region.Name, result.Result()) log.Infof(notes) return nil @@ -552,7 +552,7 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt continue } - result := ServerSkuManager.SyncServerSkus(ctx, userCred, region, meta) + result := ServerSkuManager.SyncServerSkus(ctx, userCred, region, meta, false) notes := fmt.Sprintf("SyncServerSkusByRegion %s result: %s", region.Name, result.Result()) log.Infof(notes) } @@ -563,7 +563,7 @@ func SyncServerSkus(ctx context.Context, userCred mcclient.TokenCredential, isSt } // 同步指定region sku列表 -func SyncServerSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta) compare.SyncResult { +func SyncServerSkusByRegion(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, extSkuMeta *SSkuResourcesMeta, xor bool) compare.SyncResult { result := compare.SyncResult{} var err error if extSkuMeta == nil { @@ -574,7 +574,7 @@ func SyncServerSkusByRegion(ctx context.Context, userCred mcclient.TokenCredenti } } - result = ServerSkuManager.SyncServerSkus(ctx, userCred, region, extSkuMeta) + result = ServerSkuManager.SyncServerSkus(ctx, userCred, region, extSkuMeta, xor) notes := fmt.Sprintf("SyncServerSkusByRegion %s result: %s", region.Name, result.Result()) log.Infof(notes) diff --git a/pkg/compute/models/snapshotpolicy.go b/pkg/compute/models/snapshotpolicy.go index 124a711f79..2ccb3b04e8 100644 --- a/pkg/compute/models/snapshotpolicy.go +++ b/pkg/compute/models/snapshotpolicy.go @@ -379,13 +379,18 @@ func (sp *SSnapshotPolicy) GetCloudproviderId() string { return "" } -// ==================================================== sync =========================================================== -func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context, userCred mcclient.TokenCredential, - provider *SCloudprovider, region *SCloudregion, cloudSPs []cloudprovider.ICloudSnapshotPolicy, - syncOwnerId mcclient.IIdentityProvider) compare.SyncResult { +func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + region *SCloudregion, + cloudSPs []cloudprovider.ICloudSnapshotPolicy, + syncOwnerId mcclient.IIdentityProvider, + xor bool, +) compare.SyncResult { - lockman.LockRawObject(ctx, "snapshotpolicies", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "snapshotpolicies", fmt.Sprintf("%s-%s", provider.Id, region.Id)) + lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) syncResult := compare.SyncResult{} // Fetch allsnapshotpolicy caches @@ -459,19 +464,21 @@ func (manager *SSnapshotPolicyManager) SyncSnapshotPolicies(ctx context.Context, syncResult = manager.allNewFromCloudSnapshotPolicy(ctx, userCred, added, region, syncOwnerId, provider, syncResult) - for i := range commondb { - _, err = db.Update(commondb[i], func() error { - commondb[i].Status = api.SNAPSHOT_POLICY_CACHE_STATUS_READY - if len(commonext[i].GetName()) == 0 { - commondb[i].Name = commonext[i].GetId() - } else { - commondb[i].Name = commonext[i].GetName() + if !xor { + for i := range commondb { + _, err = db.Update(commondb[i], func() error { + commondb[i].Status = api.SNAPSHOT_POLICY_CACHE_STATUS_READY + if len(commonext[i].GetName()) == 0 { + commondb[i].Name = commonext[i].GetId() + } else { + commondb[i].Name = commonext[i].GetName() + } + return nil + }) + if err != nil { + syncResult.UpdateError(err) + continue } - return nil - }) - if err != nil { - syncResult.UpdateError(err) - } else { syncResult.Update() } } diff --git a/pkg/compute/models/snapshots.go b/pkg/compute/models/snapshots.go index af107bdd7b..d51a17bd3a 100644 --- a/pkg/compute/models/snapshots.go +++ b/pkg/compute/models/snapshots.go @@ -985,9 +985,17 @@ func (manager *SSnapshotManager) getProviderSnapshotsByRegion(region *SCloudregi return snapshots, nil } -func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, snapshots []cloudprovider.ICloudSnapshot, syncOwnerId mcclient.IIdentityProvider) compare.SyncResult { - lockman.LockRawObject(ctx, "snapshots", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "snapshots", fmt.Sprintf("%s-%s", provider.Id, region.Id)) +func (manager *SSnapshotManager) SyncSnapshots( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + region *SCloudregion, + snapshots []cloudprovider.ICloudSnapshot, + syncOwnerId mcclient.IIdentityProvider, + xor bool, +) compare.SyncResult { + lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) syncResult := compare.SyncResult{} dbSnapshots, err := manager.getProviderSnapshotsByRegion(region, provider) @@ -1009,15 +1017,17 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc err = removed[i].syncRemoveCloudSnapshot(ctx, userCred) if err != nil { syncResult.DeleteError(err) - } else { - syncResult.Delete() + continue } + syncResult.Delete() } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudSnapshot(ctx, userCred, commonext[i], syncOwnerId, region) - if err != nil { - syncResult.UpdateError(err) - } else { + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudSnapshot(ctx, userCred, commonext[i], syncOwnerId, region) + if err != nil { + syncResult.UpdateError(err) + continue + } syncResult.Update() } } @@ -1025,9 +1035,9 @@ func (manager *SSnapshotManager) SyncSnapshots(ctx context.Context, userCred mcc _, err := manager.newFromCloudSnapshot(ctx, userCred, added[i], region, syncOwnerId, provider) if err != nil { syncResult.AddError(err) - } else { - syncResult.Add() + continue } + syncResult.Add() } return syncResult } diff --git a/pkg/compute/models/storagecaches.go b/pkg/compute/models/storagecaches.go index 0b9c6760ee..66ef23b277 100644 --- a/pkg/compute/models/storagecaches.go +++ b/pkg/compute/models/storagecaches.go @@ -197,7 +197,7 @@ func (self *SStoragecache) getHostId() (string, error) { return ret, nil } -func (manager *SStoragecacheManager) SyncWithCloudStoragecache(ctx context.Context, userCred mcclient.TokenCredential, cloudCache cloudprovider.ICloudStoragecache, provider *SCloudprovider) (*SStoragecache, bool, error) { +func (manager *SStoragecacheManager) SyncWithCloudStoragecache(ctx context.Context, userCred mcclient.TokenCredential, cloudCache cloudprovider.ICloudStoragecache, provider *SCloudprovider, xor bool) (*SStoragecache, bool, error) { lockman.LockClass(ctx, manager, db.GetLockClassKey(manager, userCred)) defer lockman.ReleaseClass(ctx, manager, db.GetLockClassKey(manager, userCred)) @@ -217,7 +217,9 @@ func (manager *SStoragecacheManager) SyncWithCloudStoragecache(ctx context.Conte } } else { localCache := localCacheObj.(*SStoragecache) - localCache.syncWithCloudStoragecache(ctx, userCred, cloudCache, provider) + if !xor { + localCache.syncWithCloudStoragecache(ctx, userCred, cloudCache, provider) + } return localCache, false, nil } } @@ -600,12 +602,13 @@ func (self *SStoragecache) SyncCloudImages( userCred mcclient.TokenCredential, iStoragecache cloudprovider.ICloudStoragecache, region *SCloudregion, + xor bool, ) compare.SyncResult { lockman.LockObject(ctx, self) defer lockman.ReleaseObject(ctx, self) - lockman.LockRawObject(ctx, "cachedimages", self.Id) - defer lockman.ReleaseRawObject(ctx, "cachedimages", self.Id) + lockman.LockRawObject(ctx, CachedimageManager.Keyword(), self.Id) + defer lockman.ReleaseRawObject(ctx, CachedimageManager.Keyword(), self.Id) result := compare.SyncResult{} @@ -616,7 +619,7 @@ func (self *SStoragecache) SyncCloudImages( } if driver.IsPublicCloud() { err = func() error { - err := region.SyncCloudImages(ctx, userCred, false) + err := region.SyncCloudImages(ctx, userCred, false, xor) if err != nil { return errors.Wrapf(err, "SyncCloudImages") } @@ -642,7 +645,7 @@ func (self *SStoragecache) SyncCloudImages( result.Error(errors.Wrapf(err, "GetICustomizedCloudImages")) return result } - result = self.syncCloudImages(ctx, userCred, localCachedImages, remoteImages) + result = self.syncCloudImages(ctx, userCred, localCachedImages, remoteImages, xor) } else { log.Debugln("localCachedImages started") localCachedImages, err := self.getCachedImages() @@ -656,7 +659,7 @@ func (self *SStoragecache) SyncCloudImages( result.Error(errors.Wrapf(err, "GetICloudImages")) return result } - result = self.syncCloudImages(ctx, userCred, localCachedImages, remoteImages) + result = self.syncCloudImages(ctx, userCred, localCachedImages, remoteImages, xor) } return result @@ -667,6 +670,7 @@ func (cache *SStoragecache) syncCloudImages( userCred mcclient.TokenCredential, localCachedImages []SStoragecachedimage, remoteImages []cloudprovider.ICloudImage, + xor bool, ) compare.SyncResult { syncResult := compare.SyncResult{} @@ -695,12 +699,14 @@ func (cache *SStoragecache) syncCloudImages( syncResult.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudImage(ctx, userCred, syncOwnerId, commonext[i], cache.ManagerId) - if err != nil { - syncResult.UpdateError(err) - } else { - syncResult.Update() + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].syncWithCloudImage(ctx, userCred, syncOwnerId, commonext[i], cache.ManagerId) + if err != nil { + syncResult.UpdateError(err) + } else { + syncResult.Update() + } } } for i := 0; i < len(added); i += 1 { @@ -876,8 +882,8 @@ func (self *SStoragecache) getSystemImageCount() (int, error) { } func (self *SStoragecache) CheckCloudimages(ctx context.Context, userCred mcclient.TokenCredential, regionName, regionId string) error { - lockman.LockRawObject(ctx, "cachedimages", regionId) - defer lockman.ReleaseRawObject(ctx, "cachedimages", regionId) + lockman.LockRawObject(ctx, CachedimageManager.Keyword(), regionId) + defer lockman.ReleaseRawObject(ctx, CachedimageManager.Keyword(), regionId) result := compare.SyncResult{} diff --git a/pkg/compute/models/storages.go b/pkg/compute/models/storages.go index 2ce1acc638..2e3824fd0f 100644 --- a/pkg/compute/models/storages.go +++ b/pkg/compute/models/storages.go @@ -693,7 +693,7 @@ func (manager *SStorageManager) scanLegacyStorages() error { return nil } -func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, storages []cloudprovider.ICloudStorage) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { +func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, zone *SZone, storages []cloudprovider.ICloudStorage, xor bool) ([]SStorage, []cloudprovider.ICloudStorage, compare.SyncResult) { var resId string if zone != nil { resId = fmt.Sprintf("%s-%s", provider.Id, zone.Id) @@ -743,15 +743,18 @@ func (manager *SStorageManager) SyncStorages(ctx context.Context, userCred mccli } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudStorage(ctx, userCred, commonext[i], provider) - if err != nil { - syncResult.UpdateError(err) - } else { + if !xor { + err = commondb[i].syncWithCloudStorage(ctx, userCred, commonext[i], provider) + if err != nil { + syncResult.UpdateError(err) + continue + } syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - localStorages = append(localStorages, commondb[i]) - remoteStorages = append(remoteStorages, commonext[i]) - syncResult.Update() } + + localStorages = append(localStorages, commondb[i]) + remoteStorages = append(remoteStorages, commonext[i]) + syncResult.Update() } for i := 0; i < len(added); i += 1 { new, err := manager.newFromCloudStorage(ctx, userCred, added[i], provider, zone) diff --git a/pkg/compute/models/tablestores.go b/pkg/compute/models/tablestores.go index 475d5d6fa2..3f48fa2e34 100644 --- a/pkg/compute/models/tablestores.go +++ b/pkg/compute/models/tablestores.go @@ -81,7 +81,13 @@ func (self *SCloudregion) GetTablestores() ([]STablestore, error) { return ret, err } -func (self *SCloudregion) SyncTablestores(ctx context.Context, userCred mcclient.TokenCredential, exts []cloudprovider.ICloudTablestore, provider *SCloudprovider) compare.SyncResult { +func (self *SCloudregion) SyncTablestores( + ctx context.Context, + userCred mcclient.TokenCredential, + exts []cloudprovider.ICloudTablestore, + provider *SCloudprovider, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, TablestoreManager.Keyword(), self.Id) defer lockman.ReleaseRawObject(ctx, TablestoreManager.Keyword(), self.Id) @@ -112,13 +118,15 @@ func (self *SCloudregion) SyncTablestores(ctx context.Context, userCred mcclient result.Delete() } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudTablestore(ctx, userCred, commonext[i], provider) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudTablestore(ctx, userCred, commonext[i], provider) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i += 1 { _, err := self.newFromCloudTablestore(ctx, userCred, added[i], provider) diff --git a/pkg/compute/models/vpc_peering_connections.go b/pkg/compute/models/vpc_peering_connections.go index 9a9c164e77..1f2502d3e3 100644 --- a/pkg/compute/models/vpc_peering_connections.go +++ b/pkg/compute/models/vpc_peering_connections.go @@ -381,6 +381,8 @@ func (self *SVpcPeeringConnection) SyncWithCloudPeerConnection(ctx context.Conte SyncCloudDomain(userCred, self, provider.GetOwnerId()) self.SyncShareState(ctx, userCred, provider.getAccountShareInfo()) } + + syncMetadata(ctx, userCred, self, ext) return nil } diff --git a/pkg/compute/models/vpcs.go b/pkg/compute/models/vpcs.go index 219981e00c..c5e79f915d 100644 --- a/pkg/compute/models/vpcs.go +++ b/pkg/compute/models/vpcs.go @@ -430,9 +430,16 @@ func (self *SVpc) setDefault(def bool) error { return err } -func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, region *SCloudregion, vpcs []cloudprovider.ICloudVpc) ([]SVpc, []cloudprovider.ICloudVpc, compare.SyncResult) { - lockman.LockRawObject(ctx, "vpcs", fmt.Sprintf("%s-%s", provider.Id, region.Id)) - defer lockman.ReleaseRawObject(ctx, "vpcs", fmt.Sprintf("%s-%s", provider.Id, region.Id)) +func (manager *SVpcManager) SyncVPCs( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + region *SCloudregion, + vpcs []cloudprovider.ICloudVpc, + xor bool, +) ([]SVpc, []cloudprovider.ICloudVpc, compare.SyncResult) { + lockman.LockRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), fmt.Sprintf("%s-%s", provider.Id, region.Id)) localVPCs := make([]SVpc, 0) remoteVPCs := make([]cloudprovider.ICloudVpc, 0) @@ -471,10 +478,12 @@ func (manager *SVpcManager) SyncVPCs(ctx context.Context, userCred mcclient.Toke } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudVpc(ctx, userCred, commonext[i], provider) - if err != nil { - syncResult.UpdateError(err) - continue + if !xor { + err = commondb[i].SyncWithCloudVpc(ctx, userCred, commonext[i], provider) + if err != nil { + syncResult.UpdateError(err) + continue + } } localVPCs = append(localVPCs, commondb[i]) remoteVPCs = append(remoteVPCs, commonext[i]) @@ -648,7 +657,7 @@ func (self *SVpc) SyncRouteTables(ctx context.Context, userCred mcclient.TokenCr if err != nil { return errors.Wrapf(err, "GetIRouteTables for vpc %s failed", ivpc.GetId()) } - _, _, result := RouteTableManager.SyncRouteTables(ctx, userCred, self, routeTables, self.GetCloudprovider()) + _, _, result := RouteTableManager.SyncRouteTables(ctx, userCred, self, routeTables, self.GetCloudprovider(), false) if result.IsError() { return errors.Wrapf(result.AllError(), "RouteTableManager.SyncRouteTables(%s,%s)", jsonutils.Marshal(self).String(), jsonutils.Marshal(routeTables).String()) } @@ -1634,7 +1643,12 @@ func (self *SVpc) BackSycVpcPeeringConnectionsVpc(exts []cloudprovider.ICloudVpc } -func (self *SVpc) SyncVpcPeeringConnections(ctx context.Context, userCred mcclient.TokenCredential, exts []cloudprovider.ICloudVpcPeeringConnection) compare.SyncResult { +func (self *SVpc) SyncVpcPeeringConnections( + ctx context.Context, + userCred mcclient.TokenCredential, + exts []cloudprovider.ICloudVpcPeeringConnection, + xor bool, +) compare.SyncResult { result := compare.SyncResult{} dbPeers, err := self.GetVpcPeeringConnections() @@ -1667,14 +1681,15 @@ func (self *SVpc) SyncVpcPeeringConnections(ctx context.Context, userCred mcclie } } - for i := 0; i < len(commondb); i += 1 { - err = commondb[i].SyncWithCloudPeerConnection(ctx, userCred, commonext[i], provider) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i += 1 { + err = commondb[i].SyncWithCloudPeerConnection(ctx, userCred, commonext[i], provider) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - result.Update() } for i := 0; i < len(added); i += 1 { diff --git a/pkg/compute/models/waf_instances.go b/pkg/compute/models/waf_instances.go index cb4c2efedf..e028d89ed3 100644 --- a/pkg/compute/models/waf_instances.go +++ b/pkg/compute/models/waf_instances.go @@ -293,7 +293,13 @@ func (self *SCloudregion) GetWafInstances(managerId string) ([]SWafInstance, err return wafs, err } -func (self *SCloudregion) SyncWafInstances(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudWafInstance) ([]SWafInstance, []cloudprovider.ICloudWafInstance, compare.SyncResult) { +func (self *SCloudregion) SyncWafInstances( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudWafInstance, + xor bool, +) ([]SWafInstance, []cloudprovider.ICloudWafInstance, compare.SyncResult) { lockman.LockRawObject(ctx, WafInstanceManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) defer lockman.ReleaseRawObject(ctx, WafInstanceManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) @@ -326,15 +332,17 @@ func (self *SCloudregion) SyncWafInstances(ctx context.Context, userCred mcclien result.Delete() } - for i := 0; i < len(commondb); i++ { - err := commondb[i].SyncWithCloudWafInstance(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].SyncWithCloudWafInstance(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + localWafs = append(localWafs, commondb[i]) + remoteWafs = append(remoteWafs, commonext[i]) + result.Update() } - localWafs = append(localWafs, commondb[i]) - remoteWafs = append(remoteWafs, commonext[i]) - result.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/waf_ipset_caches.go b/pkg/compute/models/waf_ipset_caches.go index c3dab29e91..2ac90191b1 100644 --- a/pkg/compute/models/waf_ipset_caches.go +++ b/pkg/compute/models/waf_ipset_caches.go @@ -305,7 +305,13 @@ func (self *SCloudregion) newFromCloudWafIPSet(ctx context.Context, userCred mcc return WafIPSetCacheManager.TableSpec().Insert(ctx, cache) } -func (self *SCloudregion) SyncWafIPSets(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudWafIPSet) compare.SyncResult { +func (self *SCloudregion) SyncWafIPSets( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudWafIPSet, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, WafIPSetCacheManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) defer lockman.ReleaseRawObject(ctx, WafIPSetCacheManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) @@ -336,13 +342,15 @@ func (self *SCloudregion) SyncWafIPSets(ctx context.Context, userCred mcclient.T result.Delete() } - for i := 0; i < len(commondb); i++ { - err := commondb[i].syncWithCloudIPSet(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].syncWithCloudIPSet(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/waf_regexset_caches.go b/pkg/compute/models/waf_regexset_caches.go index 45451d6089..f3248996c2 100644 --- a/pkg/compute/models/waf_regexset_caches.go +++ b/pkg/compute/models/waf_regexset_caches.go @@ -305,7 +305,13 @@ func (self *SCloudregion) newFromCloudWafRegexSet(ctx context.Context, userCred return WafRegexSetCacheManager.TableSpec().Insert(ctx, cache) } -func (self *SCloudregion) SyncWafRegexSets(ctx context.Context, userCred mcclient.TokenCredential, provider *SCloudprovider, exts []cloudprovider.ICloudWafRegexSet) compare.SyncResult { +func (self *SCloudregion) SyncWafRegexSets( + ctx context.Context, + userCred mcclient.TokenCredential, + provider *SCloudprovider, + exts []cloudprovider.ICloudWafRegexSet, + xor bool, +) compare.SyncResult { lockman.LockRawObject(ctx, WafRegexSetCacheManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) defer lockman.ReleaseRawObject(ctx, WafRegexSetCacheManager.Keyword(), fmt.Sprintf("%s-%s", self.Id, provider.Id)) @@ -336,13 +342,15 @@ func (self *SCloudregion) SyncWafRegexSets(ctx context.Context, userCred mcclien result.Delete() } - for i := 0; i < len(commondb); i++ { - err := commondb[i].syncWithCloudRegexSet(ctx, userCred, commonext[i]) - if err != nil { - result.UpdateError(err) - continue + if !xor { + for i := 0; i < len(commondb); i++ { + err := commondb[i].syncWithCloudRegexSet(ctx, userCred, commonext[i]) + if err != nil { + result.UpdateError(err) + continue + } + result.Update() } - result.Update() } for i := 0; i < len(added); i++ { diff --git a/pkg/compute/models/wires.go b/pkg/compute/models/wires.go index 908bc93016..11217150b2 100644 --- a/pkg/compute/models/wires.go +++ b/pkg/compute/models/wires.go @@ -286,9 +286,16 @@ func (manager *SWireManager) getWiresByVpcAndZone(vpc *SVpc, zone *SZone) ([]SWi return wires, nil } -func (manager *SWireManager) SyncWires(ctx context.Context, userCred mcclient.TokenCredential, vpc *SVpc, wires []cloudprovider.ICloudWire, provider *SCloudprovider) ([]SWire, []cloudprovider.ICloudWire, compare.SyncResult) { - lockman.LockRawObject(ctx, "wires", vpc.Id) - defer lockman.ReleaseRawObject(ctx, "wires", vpc.Id) +func (manager *SWireManager) SyncWires( + ctx context.Context, + userCred mcclient.TokenCredential, + vpc *SVpc, + wires []cloudprovider.ICloudWire, + provider *SCloudprovider, + xor bool, +) ([]SWire, []cloudprovider.ICloudWire, compare.SyncResult) { + lockman.LockRawObject(ctx, manager.Keyword(), vpc.Id) + defer lockman.ReleaseRawObject(ctx, manager.Keyword(), vpc.Id) localWires := make([]SWire, 0) remoteWires := make([]cloudprovider.ICloudWire, 0) @@ -327,26 +334,25 @@ func (manager *SWireManager) SyncWires(ctx context.Context, userCred mcclient.To } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudWire(ctx, userCred, commonext[i], vpc, provider) - if err != nil { - syncResult.UpdateError(err) - } else { - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - localWires = append(localWires, commondb[i]) - remoteWires = append(remoteWires, commonext[i]) + if !xor { + err = commondb[i].syncWithCloudWire(ctx, userCred, commonext[i], vpc, provider) + if err != nil { + syncResult.UpdateError(err) + } syncResult.Update() } + localWires = append(localWires, commondb[i]) + remoteWires = append(remoteWires, commonext[i]) } for i := 0; i < len(added); i += 1 { - new, err := manager.newFromCloudWire(ctx, userCred, added[i], vpc, provider) + wire, err := manager.newFromCloudWire(ctx, userCred, added[i], vpc, provider) if err != nil { syncResult.AddError(err) - } else { - syncMetadata(ctx, userCred, new, added[i]) - localWires = append(localWires, *new) - remoteWires = append(remoteWires, added[i]) - syncResult.Add() + continue } + localWires = append(localWires, *wire) + remoteWires = append(remoteWires, added[i]) + syncResult.Add() } return localWires, remoteWires, syncResult @@ -409,6 +415,7 @@ func (self *SWire) syncWithCloudWire(ctx context.Context, userCred mcclient.Toke } else if self.IsEmulated { self.SaveSharedInfo(apis.TOwnerSource(vpc.PublicSrc), ctx, userCred, vpc.GetSharedInfo()) } + syncMetadata(ctx, userCred, self, extWire) db.OpsLog.LogSyncUpdate(self, diff, userCred) return err @@ -477,6 +484,7 @@ func (manager *SWireManager) newFromCloudWire(ctx context.Context, userCred mccl wire.SyncShareState(ctx, userCred, provider.getAccountShareInfo()) } + syncMetadata(ctx, userCred, &wire, extWire) db.OpsLog.LogEvent(&wire, db.ACT_CREATE, wire.GetShortDesc(ctx), userCred) return &wire, nil } diff --git a/pkg/compute/models/zones.go b/pkg/compute/models/zones.go index 56afa5eb54..0751119c78 100644 --- a/pkg/compute/models/zones.go +++ b/pkg/compute/models/zones.go @@ -280,7 +280,13 @@ func (zone *SZone) GetI18N(ctx context.Context) *jsonutils.JSONDict { return zone.GetModelI18N(ctx, zone) } -func (manager *SZoneManager) SyncZones(ctx context.Context, userCred mcclient.TokenCredential, region *SCloudregion, zones []cloudprovider.ICloudZone) ([]SZone, []cloudprovider.ICloudZone, compare.SyncResult) { +func (manager *SZoneManager) SyncZones( + ctx context.Context, + userCred mcclient.TokenCredential, + region *SCloudregion, + zones []cloudprovider.ICloudZone, + xor bool, +) ([]SZone, []cloudprovider.ICloudZone, compare.SyncResult) { lockman.LockRawObject(ctx, "zones", region.Id) defer lockman.ReleaseRawObject(ctx, "zones", region.Id) @@ -314,26 +320,25 @@ func (manager *SZoneManager) SyncZones(ctx context.Context, userCred mcclient.To } } for i := 0; i < len(commondb); i += 1 { - err = commondb[i].syncWithCloudZone(ctx, userCred, commonext[i], region) - if err != nil { - syncResult.UpdateError(err) - } else { - syncMetadata(ctx, userCred, &commondb[i], commonext[i]) - localZones = append(localZones, commondb[i]) - remoteZones = append(remoteZones, commonext[i]) - syncResult.Update() + if !xor { + err = commondb[i].syncWithCloudZone(ctx, userCred, commonext[i], region) + if err != nil { + syncResult.UpdateError(err) + } } + localZones = append(localZones, commondb[i]) + remoteZones = append(remoteZones, commonext[i]) + syncResult.Update() } for i := 0; i < len(added); i += 1 { - new, err := manager.newFromCloudZone(ctx, userCred, added[i], region) + zone, err := manager.newFromCloudZone(ctx, userCred, added[i], region) if err != nil { syncResult.AddError(err) - } else { - syncMetadata(ctx, userCred, new, added[i]) - localZones = append(localZones, *new) - remoteZones = append(remoteZones, added[i]) - syncResult.Add() + continue } + localZones = append(localZones, *zone) + remoteZones = append(remoteZones, added[i]) + syncResult.Add() } return localZones, remoteZones, syncResult @@ -370,6 +375,7 @@ func (self *SZone) syncWithCloudZone(ctx context.Context, userCred mcclient.Toke log.Errorf("syncWithCloudZone error %s", err) return err } + syncMetadata(ctx, userCred, self, extZone) db.OpsLog.LogSyncUpdate(self, diff, userCred) return nil } @@ -405,6 +411,7 @@ func (manager *SZoneManager) newFromCloudZone(ctx context.Context, userCred mccl if err != nil { return nil, errors.Wrap(err, "SyncI18ns") } + syncMetadata(ctx, userCred, &zone, extZone) db.OpsLog.LogEvent(&zone, db.ACT_CREATE, zone.GetShortDesc(ctx), userCred) return &zone, nil diff --git a/pkg/compute/regiondrivers/aws.go b/pkg/compute/regiondrivers/aws.go index 7cfdac3c17..cdbd830b71 100644 --- a/pkg/compute/regiondrivers/aws.go +++ b/pkg/compute/regiondrivers/aws.go @@ -363,7 +363,7 @@ func (self *SAwsRegionDriver) RequestDeleteVpc(ctx context.Context, userCred mcc } } - _, _, result := models.SecurityGroupCacheManager.SyncSecurityGroupCaches(ctx, userCred, provider, []cloudprovider.ICloudSecurityGroup{}, vpc) + _, _, result := models.SecurityGroupCacheManager.SyncSecurityGroupCaches(ctx, userCred, provider, []cloudprovider.ICloudSecurityGroup{}, vpc, true) if result.IsError() { return nil, fmt.Errorf("SyncSecurityGroupCaches %s", result.Result()) } diff --git a/pkg/compute/regiondrivers/google.go b/pkg/compute/regiondrivers/google.go index 08ea3f8a05..8b985b9fd8 100644 --- a/pkg/compute/regiondrivers/google.go +++ b/pkg/compute/regiondrivers/google.go @@ -231,7 +231,7 @@ func (self *SGoogleRegionDriver) RequestCreateDBInstanceBackup(ctx context.Conte region, _ := backup.GetRegion() - result := models.DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, backup.GetCloudprovider(), instance, region, backups) + result := models.DBInstanceBackupManager.SyncDBInstanceBackups(ctx, userCred, backup.GetCloudprovider(), instance, region, backups, false) log.Infof("SyncDBInstanceBackups for dbinstance %s(%s) result: %s", instance.Name, instance.Id, result.Result()) instance.SetStatus(userCred, api.DBINSTANCE_RUNNING, "") return nil, nil diff --git a/pkg/compute/tasks/cloud_account_sync_skus_task.go b/pkg/compute/tasks/cloud_account_sync_skus_task.go index 0925fca623..0ed0c8d90c 100644 --- a/pkg/compute/tasks/cloud_account_sync_skus_task.go +++ b/pkg/compute/tasks/cloud_account_sync_skus_task.go @@ -92,7 +92,7 @@ func (self *CloudAccountSyncSkusTask) OnInit(ctx context.Context, obj db.IStanda return } - type SyncFunc func(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, extSkuMeta *models.SSkuResourcesMeta) compare.SyncResult + type SyncFunc func(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, extSkuMeta *models.SSkuResourcesMeta, xor bool) compare.SyncResult var syncFunc SyncFunc for _, region := range regions { switch res { @@ -103,15 +103,15 @@ func (self *CloudAccountSyncSkusTask) OnInit(ctx context.Context, obj db.IStanda case models.DBInstanceSkuManager.Keyword(): syncFunc = models.DBInstanceSkuManager.SyncDBInstanceSkus case models.NatSkuManager.Keyword(): - result := region.SyncNatSkus(ctx, self.GetUserCred(), meta) + result := region.SyncNatSkus(ctx, self.GetUserCred(), meta, false) log.Infof("Sync %s %s skus for region %s result: %s", region.Provider, res, region.Name, result.Result()) case models.NasSkuManager.Keyword(): - result := region.SyncNasSkus(ctx, self.GetUserCred(), meta) + result := region.SyncNasSkus(ctx, self.GetUserCred(), meta, false) log.Infof("Sync %s %s skus for region %s result: %s", region.Provider, res, region.Name, result.Result()) } if syncFunc != nil { - result := syncFunc(ctx, self.GetUserCred(), ®ion, meta) + result := syncFunc(ctx, self.GetUserCred(), ®ion, meta, false) log.Infof("Sync %s %s skus for region %s result: %s", region.Provider, res, region.Name, result.Result()) } } diff --git a/pkg/compute/tasks/cloud_region_sync_skus_task.go b/pkg/compute/tasks/cloud_region_sync_skus_task.go index 0625e491b0..aa270505eb 100644 --- a/pkg/compute/tasks/cloud_region_sync_skus_task.go +++ b/pkg/compute/tasks/cloud_region_sync_skus_task.go @@ -51,7 +51,7 @@ func (self *CloudRegionSyncSkusTask) OnInit(ctx context.Context, obj db.IStandal return } - type SyncFunc func(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, extSkuMeta *models.SSkuResourcesMeta) compare.SyncResult + type SyncFunc func(ctx context.Context, userCred mcclient.TokenCredential, region *models.SCloudregion, extSkuMeta *models.SSkuResourcesMeta, xor bool) compare.SyncResult var syncFunc SyncFunc switch res { case models.ServerSkuManager.Keyword(): @@ -61,12 +61,12 @@ func (self *CloudRegionSyncSkusTask) OnInit(ctx context.Context, obj db.IStandal case models.DBInstanceSkuManager.Keyword(): syncFunc = models.DBInstanceSkuManager.SyncDBInstanceSkus case models.NatSkuManager.Keyword(): - result := region.SyncNatSkus(ctx, self.GetUserCred(), meta) + result := region.SyncNatSkus(ctx, self.GetUserCred(), meta, false) log.Infof("Sync %s %s skus for region %s result: %s", region.Provider, res, region.Name, result.Result()) } if syncFunc != nil { - result := syncFunc(ctx, self.GetUserCred(), region, meta) + result := syncFunc(ctx, self.GetUserCred(), region, meta, false) log.Infof("Sync %s %s skus for region %s result: %s", region.Provider, res, region.Name, result.Result()) if result.IsError() { self.taskFailed(ctx, region, result.Result()) diff --git a/pkg/compute/tasks/cloudregion_sync_images_task.go b/pkg/compute/tasks/cloudregion_sync_images_task.go index 4a20bae58d..1ee7358f49 100644 --- a/pkg/compute/tasks/cloudregion_sync_images_task.go +++ b/pkg/compute/tasks/cloudregion_sync_images_task.go @@ -44,7 +44,7 @@ func (self *CloudregionSyncImagesTask) taskFailed(ctx context.Context, region *m func (self *CloudregionSyncImagesTask) OnInit(ctx context.Context, obj db.IStandaloneModel, body jsonutils.JSONObject) { region := obj.(*models.SCloudregion) - err := region.SyncCloudImages(ctx, self.GetUserCred(), true) + err := region.SyncCloudImages(ctx, self.GetUserCred(), true, false) if err != nil { self.taskFailed(ctx, region, errors.Wrapf(err, "SyncCloudImages")) return diff --git a/pkg/compute/tasks/dnszone_syncstatus_task.go b/pkg/compute/tasks/dnszone_syncstatus_task.go index e7f430ddbf..6a6ad59a73 100644 --- a/pkg/compute/tasks/dnszone_syncstatus_task.go +++ b/pkg/compute/tasks/dnszone_syncstatus_task.go @@ -64,7 +64,7 @@ func (self *DnsZoneSyncstatusTask) OnInit(ctx context.Context, obj db.IStandalon if len(caches) == 1 { account, _ := caches[i].GetCloudaccount() if account != nil { - dnsZone.SyncDnsRecordSets(ctx, self.GetUserCred(), account.Provider, iZone) + dnsZone.SyncDnsRecordSets(ctx, self.GetUserCred(), account.Provider, iZone, false) } } } diff --git a/pkg/compute/tasks/inter_vpc_network_syncstatus_task.go b/pkg/compute/tasks/inter_vpc_network_syncstatus_task.go index 394528be75..ecdd05f089 100644 --- a/pkg/compute/tasks/inter_vpc_network_syncstatus_task.go +++ b/pkg/compute/tasks/inter_vpc_network_syncstatus_task.go @@ -57,7 +57,7 @@ func (self *InterVpcNetworkSyncstatusTask) OnInit(ctx context.Context, obj db.IS return } - result := snetwork.SyncInterVpcNetworkRouteSets(ctx, self.UserCred, inetwork) + result := snetwork.SyncInterVpcNetworkRouteSets(ctx, self.UserCred, inetwork, false) log.Infof("sync routes for %s result: %s", snetwork.GetName(), result.Result()) logclient.AddActionLogWithStartable(self, snetwork, logclient.ACT_SYNC_STATUS, nil, self.UserCred, true) diff --git a/pkg/compute/tasks/route_table_syncstatus_task.go b/pkg/compute/tasks/route_table_syncstatus_task.go index 1ae6fcf5f8..52a5492ffd 100644 --- a/pkg/compute/tasks/route_table_syncstatus_task.go +++ b/pkg/compute/tasks/route_table_syncstatus_task.go @@ -61,14 +61,14 @@ func (self *RouteTableSyncStatusTask) OnInit(ctx context.Context, obj db.IStanda self.taskFailed(ctx, routeTable, errors.Wrapf(err, "SyncWithCloudRouteTable")) return } - result := routeTable.SyncRouteTableRouteSets(ctx, self.GetUserCred(), iRouteTable, nil) + result := routeTable.SyncRouteTableRouteSets(ctx, self.GetUserCred(), iRouteTable, nil, false) if result.IsError() { self.taskFailed(ctx, routeTable, errors.Wrapf(result.AllError(), "SyncRouteTableRouteSets")) return } log.Infof("sync route table for %s result: %s", routeTable.GetName(), result.Result()) - result = routeTable.SyncRouteTableAssociations(ctx, self.GetUserCred(), iRouteTable, nil) + result = routeTable.SyncRouteTableAssociations(ctx, self.GetUserCred(), iRouteTable, nil, false) if result.IsError() { self.taskFailed(ctx, routeTable, errors.Wrapf(result.AllError(), "routeTable.SyncRouteTableAssociations()")) return