From b081dc43de35aa1d1643b1749d814f2caed35dc4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=B1=88=E8=BD=A9?= Date: Wed, 26 Feb 2025 15:18:23 +0800 Subject: [PATCH] fix(region): fast sync image (#22185) --- go.mod | 2 +- go.sum | 4 +- pkg/apis/compute/cloudaccount.go | 2 +- pkg/apis/monitor/zz_generated.model.go | 8 +- pkg/compute/models/cloudsync.go | 148 ++++++++---------- pkg/compute/models/guest_actions.go | 4 + vendor/modules.txt | 2 +- .../x/cloudmux/pkg/cloudprovider/consts.go | 1 + 8 files changed, 82 insertions(+), 89 deletions(-) diff --git a/go.mod b/go.mod index e3a43e173c..1ee7cb2067 100644 --- a/go.mod +++ b/go.mod @@ -87,7 +87,7 @@ require ( k8s.io/client-go v0.19.3 k8s.io/cluster-bootstrap v0.19.3 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250214014137-798f62b12a5f + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250225101418-162e21a39499 yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 yunion.io/x/jsonutils v1.0.1-0.20240930100528-1671a2d0d22f yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 diff --git a/go.sum b/go.sum index 59a2df7a96..b93395c353 100644 --- a/go.sum +++ b/go.sum @@ -1274,8 +1274,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250214014137-798f62b12a5f h1:UMh7Guhl+M7Rdj+vKEkIyqM2j0rg620YPv/74Bkh/Sc= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250214014137-798f62b12a5f/go.mod h1:KQ/jWx7bZlmjCE711KEWuvHW/dzpdr/UTlBjjutkj0Y= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250225101418-162e21a39499 h1:+Hzzk52JkHCyLlPWASkn+t7TWve2zRbnLj5VEEjF4JM= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250225101418-162e21a39499/go.mod h1:KQ/jWx7bZlmjCE711KEWuvHW/dzpdr/UTlBjjutkj0Y= yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256 h1:kLKQ6zbgPDQflRwoHFAjxNChcbhXIFgsUVLkJwiXu/8= yunion.io/x/executor v0.0.0-20241205080005-48f5b1212256/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= diff --git a/pkg/apis/compute/cloudaccount.go b/pkg/apis/compute/cloudaccount.go index e42f10c69d..8c9ef425e9 100644 --- a/pkg/apis/compute/cloudaccount.go +++ b/pkg/apis/compute/cloudaccount.go @@ -565,7 +565,7 @@ type SyncRangeInput struct { // 按资源类型同步，可输入多个 // enmu: project, compute, network, eip, loadbalancer, objectstore, rds, cache, event, cloudid, dnszone, public_ip, intervpcnetwork, saml_auth, quota, nat, nas, waf, mongodb, es, kafka, app, cdn, container, ipv6_gateway, tablestore, modelarts, vpcpeer, misc - Resources []string `json:"resources" choices:"project|compute|network|eip|loadbalancer|objectstore|rds|cache|event|cloudid|dnszone|public_ip|intervpcnetwork|saml_auth|quota|nat|nas|waf|mongodb|es|kafka|app|cdn|container|ipv6_gateway|tablestore|modelarts|vpcpeer|misc"` + Resources []string `json:"resources" choices:"project|compute|network|eip|loadbalancer|objectstore|rds|cache|event|cloudid|dnszone|public_ip|intervpcnetwork|saml_auth|quota|nat|nas|waf|mongodb|es|kafka|app|cdn|container|ipv6_gateway|tablestore|modelarts|vpcpeer|misc|image"` } type iRes interface { diff --git a/pkg/apis/monitor/zz_generated.model.go b/pkg/apis/monitor/zz_generated.model.go index da471eab08..86b0a4d801 100644 --- a/pkg/apis/monitor/zz_generated.model.go +++ b/pkg/apis/monitor/zz_generated.model.go @@ -29,11 +29,11 @@ type SAlert struct { apis.SStatusStandaloneResourceBase SMonitorScopedResource // Frequency is evaluate period - Frequency int64 `json:"frequency"` + Frequency int64 `json:"frequency"` Settings *AlertSetting `json:"settings"` - Level string `json:"level"` - Message string `json:"message"` - UsedBy string `json:"used_by"` + Level string `json:"level"` + Message string `json:"message"` + UsedBy string `json:"used_by"` // Silenced bool ExecutionError string `json:"execution_error"` // If an alert rule has a configured `For` and the query violates the configured threshold diff --git a/pkg/compute/models/cloudsync.go b/pkg/compute/models/cloudsync.go index 11ae695fcc..7aee96b381 100644 --- a/pkg/compute/models/cloudsync.go +++ b/pkg/compute/models/cloudsync.go @@ -866,8 +866,10 @@ func syncZoneStorages( newCacheIds = append(newCacheIds, cachePair) } } - if !remoteStorages[i].DisableSync() { - syncStorageDisks(ctx, userCred, syncResults, provider, driver, &localStorages[i], remoteStorages[i], syncRange) + if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { + if !remoteStorages[i].DisableSync() { + syncStorageDisks(ctx, userCred, syncResults, provider, driver, &localStorages[i], remoteStorages[i], syncRange) + } } }() } @@ -974,9 +976,11 @@ func syncZoneHosts( } newCachePairs = syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs, syncRange.Xor) - syncHostNics(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i]) - // syncHostWires(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i]) - syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange) + + if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { + syncHostNics(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i]) + syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange) + } }() } return newCachePairs @@ -2162,12 +2166,13 @@ func syncPublicCloudProviderInfo( if cloudprovider.IsSupportCompute(driver) { if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NETWORK) || syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NAT) || + syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_IMAGE) || syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_EIP) { // 需要先同步vpc，避免私有云eip找不到network if !(driver.GetFactory().IsPublicCloud() && !syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NETWORK)) && syncRange.IsNotSkipSyncResource(VpcManager) { syncRegionVPCs(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) } - if syncRange.IsNotSkipSyncResource(ElasticipManager) { + if syncRange.IsNotSkipSyncResource(ElasticipManager) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_EIP) { syncRegionEips(ctx, userCred, syncResults, provider, localRegion, remoteRegion, syncRange) } @@ -2177,7 +2182,7 @@ func syncPublicCloudProviderInfo( } - if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { + if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) || syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_IMAGE) { for j := 0; j < len(localZones); j += 1 { @@ -2311,7 +2316,7 @@ func syncPublicCloudProviderInfo( } } - if cloudprovider.IsSupportCompute(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { + if cloudprovider.IsSupportCompute(driver) && (syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) || syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_IMAGE)) { log.Debugf("storageCachePairs count %d", len(storageCachePairs)) for i := range storageCachePairs { // always sync private cloud cached images @@ -2397,8 +2402,10 @@ func syncOnPremiseCloudProviderStorage(ctx context.Context, userCred mcclient.To storageCachePairs = append(storageCachePairs, cachePair) } } - if !remoteStorages[i].DisableSync() { - syncStorageDisks(ctx, userCred, syncResults, provider, driver, &localStorages[i], remoteStorages[i], syncRange) + if cloudprovider.IsSupportCompute(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { + if !remoteStorages[i].DisableSync() { + syncStorageDisks(ctx, userCred, syncResults, provider, driver, &localStorages[i], remoteStorages[i], syncRange) + } } }() } @@ -2431,63 +2438,72 @@ func syncOnPremiseCloudProviderInfo( } var storageCachePairs []sStoragecacheSyncPair - if cloudprovider.IsSupportCompute(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { - remoteVpcs, err := iregion.GetIVpcs() - if err != nil { - msg := fmt.Sprintf("GetIVpcs for provider %s failed %s", provider.GetName(), err) - log.Errorf(msg) - return err - } + if cloudprovider.IsSupportCompute(driver) && + (syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) || + syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_IMAGE) || + syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NETWORK)) { + zone, err := getZoneForOnPremiseCloudRegion(ctx, userCred, iregion) if err != nil { msg := fmt.Sprintf("Can't get zone for Premise cloud region %s error: %v", iregion.GetName(), err) log.Errorf(msg) return errors.Wrap(err, "getZoneForOnPremiseCloudRegion") } - { - // sync wires - localVpc := VpcManager.FetchDefaultVpc() - syncVpcWires(ctx, userCred, syncResults, provider, localVpc, remoteVpcs[0], zone, syncRange) + + if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) || syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_NETWORK) { + remoteVpcs, err := iregion.GetIVpcs() + if err != nil { + msg := fmt.Sprintf("GetIVpcs for provider %s failed %s", provider.GetName(), err) + log.Errorf(msg) + return err + } + { + // sync wires + localVpc := VpcManager.FetchDefaultVpc() + syncVpcWires(ctx, userCred, syncResults, provider, localVpc, remoteVpcs[0], zone, syncRange) + } } storageCachePairs = syncOnPremiseCloudProviderStorage(ctx, userCred, syncResults, provider, iregion, driver, zone, syncRange) - ihosts, err := func() ([]cloudprovider.ICloudHost, error) { - defer syncResults.AddRequestCost(HostManager)() - return iregion.GetIHosts() - }() - if err != nil { - msg := fmt.Sprintf("GetIHosts for provider %s failed %s", provider.GetName(), err) - log.Errorf(msg) - return err - } - localHosts, remoteHosts, result := func() ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { - defer syncResults.AddSqlCost(HostManager)() - return HostManager.SyncHosts(ctx, userCred, provider, zone, nil, ihosts, syncRange.Xor) - }() - - syncResults.Add(HostManager, result) - - msg := result.Result() - notes := fmt.Sprintf("SyncHosts for provider %s result: %s", provider.Name, msg) - log.Infof(notes) - provider.SyncError(result, notes, userCred) - - for i := 0; i < len(localHosts); i += 1 { - if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) { - continue + if syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { + ihosts, err := func() ([]cloudprovider.ICloudHost, error) { + defer syncResults.AddRequestCost(HostManager)() + return iregion.GetIHosts() + }() + if err != nil { + msg := fmt.Sprintf("GetIHosts for provider %s failed %s", provider.GetName(), err) + log.Errorf(msg) + return err } - newCachePairs := syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs, syncRange.Xor) - if len(newCachePairs) > 0 { - storageCachePairs = append(storageCachePairs, newCachePairs...) + + localHosts, remoteHosts, result := func() ([]SHost, []cloudprovider.ICloudHost, compare.SyncResult) { + defer syncResults.AddSqlCost(HostManager)() + return HostManager.SyncHosts(ctx, userCred, provider, zone, nil, ihosts, syncRange.Xor) + }() + + syncResults.Add(HostManager, result) + + msg := result.Result() + notes := fmt.Sprintf("SyncHosts for provider %s result: %s", provider.Name, msg) + log.Infof(notes) + provider.SyncError(result, notes, userCred) + + for i := 0; i < len(localHosts); i += 1 { + if len(syncRange.Host) > 0 && !utils.IsInStringArray(localHosts[i].Id, syncRange.Host) { + continue + } + newCachePairs := syncHostStorages(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i], storageCachePairs, syncRange.Xor) + if len(newCachePairs) > 0 { + storageCachePairs = append(storageCachePairs, newCachePairs...) + } + syncHostNics(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i]) + syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange) } - syncHostNics(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i]) - // syncOnPremiseHostWires(ctx, userCred, syncResults, provider, &localHosts[i], remoteHosts[i]) - syncHostVMs(ctx, userCred, syncResults, provider, driver, &localHosts[i], remoteHosts[i], syncRange) } } - if cloudprovider.IsSupportCompute(driver) && syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) { + if cloudprovider.IsSupportCompute(driver) && (syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_COMPUTE) || syncRange.NeedSyncResource(cloudprovider.CLOUD_CAPABILITY_IMAGE)) { log.Debugf("storageCachePairs count %d", len(storageCachePairs)) for i := range storageCachePairs { // alway sync on-premise cached images @@ -2509,34 +2525,6 @@ func syncOnPremiseCloudProviderInfo( return nil } -/*func syncOnPremiseHostWires(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localHost *SHost, remoteHost cloudprovider.ICloudHost) { - log.Infof("start to sync OnPremeseHostWires") - if provider.Provider != api.CLOUD_PROVIDER_VMWARE { - return - } - func() { - defer func() { - if syncResults != nil { - syncResults.AddSqlCost(NetInterfaceManager)() - } - }() - result := localHost.SyncEsxiHostWires(ctx, userCred, remoteHost) - if syncResults != nil { - syncResults.Add(NetInterfaceManager, result) - } - - msg := result.Result() - notes := fmt.Sprintf("SyncEsxiHostWires for host %s result: %s", localHost.Name, msg) - if result.IsError() { - log.Errorf(notes) - return - } else { - log.Infof(notes) - } - db.OpsLog.LogEvent(provider, db.ACT_SYNC_HOST_COMPLETE, msg, userCred) - }() -}*/ - func syncHostNics(ctx context.Context, userCred mcclient.TokenCredential, syncResults SSyncResultSet, provider *SCloudprovider, localHost *SHost, remoteHost cloudprovider.ICloudHost) { defer func() { if syncResults != nil { diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index a10cec67e9..89e5b0c2f7 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -29,6 +29,7 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/billing" "yunion.io/x/pkg/util/httputils" @@ -1665,6 +1666,9 @@ func (self *SGuest) PerformRebuildRoot( } diskCat := self.CategorizeDisks() + if gotypes.IsNil(diskCat.Root) { + return nil, httperrors.NewInputParameterError("no root disk is found") + } if img.MinDiskMB == 0 || img.Status != imageapi.IMAGE_STATUS_ACTIVE { return nil, httperrors.NewInputParameterError("invlid image") } diff --git a/vendor/modules.txt b/vendor/modules.txt index eb6dcdb08d..d006833630 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1576,7 +1576,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.2.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250214014137-798f62b12a5f +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20250225101418-162e21a39499 ## explicit; go 1.21 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing diff --git a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go index 30a5f513da..f7a17d23a2 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go +++ b/vendor/yunion.io/x/cloudmux/pkg/cloudprovider/consts.go @@ -59,6 +59,7 @@ const ( CLOUD_CAPABILITY_VPC_PEER = "vpcpeer" // 对等连接 CLOUD_CAPABILITY_MISC = "misc" CLOUD_CAPABILITY_CERT = "sslcertificates" // 证书 + CLOUD_CAPABILITY_IMAGE = "image" // 镜像 ) const (