fix(region): support vmware resource pool (#23828)

This commit is contained in:
屈轩
2025-11-25 19:07:27 +08:00
committed by GitHub
parent 19c0680c4b
commit 2e58bba807
14 changed files with 211 additions and 78 deletions
+1 -1
View File
@@ -88,7 +88,7 @@ require (
k8s.io/client-go v0.19.3
k8s.io/cluster-bootstrap v0.19.3
moul.io/http2curl/v2 v2.3.0
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251124060319-e1c2af05b72d
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251125082616-286af01528f8
yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0
yunion.io/x/jsonutils v1.0.1-0.20250507052344-1abcf4f443b1
yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91
+2 -2
View File
@@ -1279,8 +1279,8 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK
sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q=
sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251124060319-e1c2af05b72d h1:aQpxoj99ETA2SOHb7BMhNdGmNzMwiG3EIH0wYb63Q+0=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251124060319-e1c2af05b72d/go.mod h1:rw1H4VRSx5lxvcNaTwzFOHPCxUIUR9tS2TekttVq+qc=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251125082616-286af01528f8 h1:/+oHV9UbwwGeRHtNAosPyv+4A6l6xhOoBAP3AyLHpuw=
yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251125082616-286af01528f8/go.mod h1:rw1H4VRSx5lxvcNaTwzFOHPCxUIUR9tS2TekttVq+qc=
yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0 h1:msG4SiDSVU7CrXH06WuHlNEZXIooTcmNbfrIGHuIHBU=
yunion.io/x/executor v0.0.0-20250518005516-5402e9e0bed0/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
+3
View File
@@ -451,6 +451,9 @@ func (self *SESXiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gues
if len(projectId) > 0 {
config.Add(jsonutils.NewString(projectId), "desc", "project_id")
}
if len(guest.ResourcePool) > 0 {
config.Add(jsonutils.NewString(guest.ResourcePool), "desc", "resource_pool")
}
}
config.Add(jsonutils.Marshal(accessInfo), "datastore")
+25
View File
@@ -192,6 +192,9 @@ type SGuest struct {
HealthStatus string `width:"36" charset:"ascii" nullable:"true" default:"ok" list:"user"`
// Used for guest rescue
RescueMode bool `nullable:"false" default:"false" list:"user" create:"optional"`
// 资源池,仅vmware指定调度标签时内部使用
ResourcePool string `width:"64" charset:"utf8" nullable:"true" create:"optional"`
}
func (manager *SGuestManager) GetPropertyStatistics(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*apis.StatusStatistic, error) {
@@ -2479,6 +2482,28 @@ func (guest *SGuest) PostCreate(ctx context.Context, userCred mcclient.TokenCred
guest.setUserData(ctx, userCred, userData)
}
if guest.Hypervisor == api.HYPERVISOR_ESXI {
schedtags := []api.SchedtagConfig{}
data.Unmarshal(&schedtags, "schedtags")
for _, tag := range schedtags {
if tag.ResourceType != HostManager.KeywordPlural() {
continue
}
meta := db.SMetadata{}
db.Metadata.Query().
Equals("obj_type", SchedtagManager.Keyword()).
Equals("obj_id", tag.Id).
Equals("key", cloudprovider.METADATA_POOL_ID).First(&meta)
if len(meta.Value) > 0 {
db.Update(guest, func() error {
guest.ResourcePool = meta.Value
return nil
})
}
}
}
if guest.GetDriver().GetMaxSecurityGroupCount() > 0 {
secgroups, _ := jsonutils.GetStringArray(data, "secgroups")
for _, secgroupId := range secgroups {
+53 -40
View File
@@ -2333,9 +2333,9 @@ func (hh *SHost) syncWithCloudHost(ctx context.Context, userCred mcclient.TokenC
syncMetadata(ctx, userCred, hh, extHost, account.ReadOnly)
}
if err := hh.syncSchedtags(ctx, userCred, extHost); err != nil {
log.Errorf("syncSchedtags fail: %v", err)
return err
err = hh.syncSchedtags(ctx, userCred, extHost)
if err != nil && errors.Cause(err) != cloudprovider.ErrNotFound && errors.Cause(err) != errors.ErrNotImplemented {
log.Errorf("syncSchedtags for %s fail: %v", hh.Name, err)
}
if len(diff) > 0 {
@@ -2375,8 +2375,11 @@ var (
METADATA_EXT_SCHEDTAG_KEY = "ext:schedtag"
)
func (s *SHost) getAllSchedtagsWithExtSchedtagKey(ctx context.Context, userCred mcclient.TokenCredential) (map[string]*SSchedtag, error) {
q := SchedtagManager.Query().Equals("resource_type", HostManager.KeywordPlural())
func (h *SHost) getAllSchedtagsWithExtSchedtagKey() (map[string]*SSchedtag, error) {
metaSQ := db.Metadata.Query("obj_id").Equals("obj_type", SchedtagManager.KeywordPlural()).Equals("key", METADATA_EXT_SCHEDTAG_KEY).SubQuery()
hostSQ := HostManager.Query("id").Equals("manager_id", h.ManagerId).SubQuery()
hSQ := HostschedtagManager.Query("schedtag_id").In("host_id", hostSQ).SubQuery()
q := SchedtagManager.Query().Equals("resource_type", HostManager.KeywordPlural()).In("id", metaSQ).In("id", hSQ)
sts := make([]SSchedtag, 0, 5)
err := db.FetchModelObjects(SchedtagManager, q, &sts)
if err != nil {
@@ -2384,79 +2387,93 @@ func (s *SHost) getAllSchedtagsWithExtSchedtagKey(ctx context.Context, userCred
}
stMap := make(map[string]*SSchedtag)
for i := range sts {
extTagName := sts[i].GetMetadata(ctx, METADATA_EXT_SCHEDTAG_KEY, userCred)
if len(extTagName) == 0 {
continue
}
stMap[extTagName] = &sts[i]
stMap[sts[i].Id] = &sts[i]
}
return stMap, nil
}
func (s *SHost) syncSchedtags(ctx context.Context, userCred mcclient.TokenCredential, extHost cloudprovider.ICloudHost) error {
stq := SchedtagManager.Query()
subq := HostschedtagManager.Query("schedtag_id").Equals("host_id", s.Id).SubQuery()
stq = stq.Join(subq, sqlchemy.Equals(stq.Field("id"), subq.Field("schedtag_id")))
func (h *SHost) GetSchedtags() ([]SSchedtag, error) {
sq := HostschedtagManager.Query("schedtag_id").Equals("host_id", h.Id).SubQuery()
q := SchedtagManager.Query().In("id", sq)
schedtags := make([]SSchedtag, 0)
err := db.FetchModelObjects(SchedtagManager, stq, &schedtags)
err := db.FetchModelObjects(SchedtagManager, q, &schedtags)
if err != nil {
return errors.Wrap(err, "db.FetchModelObjects")
return nil, errors.Wrap(err, "db.FetchModelObjects")
}
extSchedtagStrs, err := extHost.GetSchedtags()
return schedtags, nil
}
func (h *SHost) syncSchedtags(ctx context.Context, userCred mcclient.TokenCredential, extHost cloudprovider.ICloudHost) error {
schedtags, err := h.GetSchedtags()
if err != nil {
return errors.Wrap(err, "GetSchedtags")
}
extSchedTags, err := extHost.GetSchedtags()
if err != nil {
return errors.Wrap(err, "extHost.GetSchedtags")
}
extStStrSet := sets.NewString(extSchedtagStrs...)
extTagMap := map[string]*cloudprovider.Schedtag{}
extTagIdSet := sets.NewString()
for i := range extSchedTags {
extSchedtag := &extSchedTags[i]
extTagIdSet.Insert(extSchedtag.Id)
extTagMap[extSchedtag.Id] = &extSchedTags[i]
}
removed := make([]*SSchedtag, 0)
removedIds := make([]string, 0)
for i := range schedtags {
stag := &schedtags[i]
extTagName := stag.GetMetadata(ctx, METADATA_EXT_SCHEDTAG_KEY, userCred)
if len(extTagName) == 0 {
extTagId := stag.GetMetadata(ctx, METADATA_EXT_SCHEDTAG_KEY, userCred)
if len(extTagId) == 0 {
continue
}
if !extStStrSet.Has(extTagName) {
if !extTagIdSet.Has(extTagId) {
removed = append(removed, stag)
removedIds = append(removedIds, stag.GetId())
} else {
extStStrSet.Delete(extTagName)
extTagIdSet.Delete(extTagId)
}
}
added := extStStrSet.UnsortedList()
added := extTagIdSet.UnsortedList()
var stagMap map[string]*SSchedtag
if len(added) > 0 {
stagMap, err = s.getAllSchedtagsWithExtSchedtagKey(ctx, userCred)
stagMap, err = h.getAllSchedtagsWithExtSchedtagKey()
if err != nil {
return errors.Wrap(err, "getAllSchedtagsWithExtSchedtagKey")
}
}
for _, stStr := range added {
st, ok := stagMap[stStr]
for _, extSchedId := range added {
st, ok := stagMap[extSchedId]
if !ok {
st = &SSchedtag{
ResourceType: HostManager.KeywordPlural(),
}
st.DomainId = s.DomainId
st.Name = stStr
st.DomainId = h.DomainId
st.Name = extTagMap[extSchedId].Name
st.Description = "Sync from cloud"
st.SetModelManager(SchedtagManager, st)
err := SchedtagManager.TableSpec().Insert(ctx, st)
if err != nil {
return errors.Wrapf(err, "unable to create schedtag %q", stStr)
return errors.Wrapf(err, "unable to create schedtag %s", st.Name)
}
st.SetMetadata(ctx, METADATA_EXT_SCHEDTAG_KEY, stStr, userCred)
meta := make(map[string]interface{})
meta[METADATA_EXT_SCHEDTAG_KEY] = extSchedId
for k, v := range extTagMap[extSchedId].Meta {
meta[k] = v
}
st.SetAllMetadata(ctx, meta, userCred)
}
// attach
hostschedtag := &SHostschedtag{
HostId: s.GetId(),
HostId: h.GetId(),
}
hostschedtag.SetModelManager(HostschedtagManager, hostschedtag)
hostschedtag.SchedtagId = st.GetId()
err = HostschedtagManager.TableSpec().Insert(ctx, hostschedtag)
if err != nil {
return errors.Wrapf(err, "unable to create hostschedtag for tag %q host %q", stStr, s.GetId())
return errors.Wrapf(err, "unable to create hostschedtag for tag %s host %s", st.Name, h.GetId())
}
}
@@ -2464,7 +2481,7 @@ func (s *SHost) syncSchedtags(ctx context.Context, userCred mcclient.TokenCreden
return nil
}
q := HostschedtagManager.Query().Equals("host_id", s.GetId()).In("schedtag_id", removedIds)
q := HostschedtagManager.Query().Equals("host_id", h.GetId()).In("schedtag_id", removedIds)
hostschedtags := make([]SHostschedtag, 0, len(removedIds))
err = db.FetchModelObjects(HostschedtagManager, q, &hostschedtags)
if err != nil {
@@ -2588,9 +2605,9 @@ func (manager *SHostManager) NewFromCloudHost(ctx context.Context, userCred mccl
SyncCloudDomain(userCred, &host, provider.GetOwnerId())
if err := host.syncSchedtags(ctx, userCred, extHost); err != nil {
log.Errorf("newFromCloudHost fail in syncSchedtags %v", err)
return nil, err
err = host.syncSchedtags(ctx, userCred, extHost)
if err != nil && errors.Cause(err) != cloudprovider.ErrNotFound && errors.Cause(err) != errors.ErrNotImplemented {
log.Errorf("syncSchedtags %s fail %v", host.Name, err)
}
if provider != nil {
@@ -3464,10 +3481,6 @@ func (hh *SHost) GetBaremetalServer() *SGuest {
return &guest
}
func (hh *SHost) GetSchedtags() []SSchedtag {
return GetSchedtags(HostschedtagManager, hh.Id)
}
type SHostGuestResourceUsage struct {
GuestCount int
GuestVcpuCount int
-8
View File
@@ -173,14 +173,6 @@ func (host *SHost) GetVersion() string {
return host.Version
}
func (host *SHost) GetSchedtags() ([]string, error) {
ret := []string{}
for _, tag := range host.Schedtags {
ret = append(ret, tag.Name)
}
return ret, nil
}
type SHostNic struct {
host *SHost
+1 -1
View File
@@ -1605,7 +1605,7 @@ sigs.k8s.io/structured-merge-diff/v4/value
# sigs.k8s.io/yaml v1.2.0
## explicit; go 1.12
sigs.k8s.io/yaml
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251124060319-e1c2af05b72d
# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20251125082616-286af01528f8
## explicit; go 1.21
yunion.io/x/cloudmux/pkg/apis
yunion.io/x/cloudmux/pkg/apis/billing
+1 -1
View File
@@ -318,7 +318,7 @@ type ICloudHost interface {
CreateVM(desc *SManagedVMCreateConfig) (ICloudVM, error)
GetIHostNics() ([]ICloudHostNetInterface, error)
GetSchedtags() ([]string, error)
GetSchedtags() ([]Schedtag, error)
GetOvnVersion() string // just for cloudpods host
}
+25
View File
@@ -0,0 +1,25 @@
// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cloudprovider
const (
METADATA_POOL_ID = "__pool_id"
)
type Schedtag struct {
Name string
Id string
Meta map[string]string
}
-4
View File
@@ -230,10 +230,6 @@ func (self *SNode) getIWires() ([]cloudprovider.ICloudWire, error) {
return ret, nil
}
func (self *SNode) GetSchedtags() ([]string, error) {
return []string{}, nil
}
func (self *SNode) GetIsMaintenance() bool {
return self.Status == "maintain"
}
+31 -2
View File
@@ -15,7 +15,13 @@
package esxi
import (
"strings"
"github.com/vmware/govmomi/object"
"github.com/vmware/govmomi/vim25/mo"
"yunion.io/x/cloudmux/pkg/cloudprovider"
"yunion.io/x/log"
"yunion.io/x/pkg/errors"
)
type SCluster struct {
@@ -26,6 +32,29 @@ func NewCluster(manager *SESXiClient, cluster *mo.ClusterComputeResource, dc *SD
return &SCluster{SManagedObject: newManagedObject(manager, cluster, dc)}
}
func (cluster *SCluster) getoCluster() *mo.ClusterComputeResource {
return cluster.object.(*mo.ClusterComputeResource)
func (cluster *SCluster) listResourcePools() ([]mo.ResourcePool, error) {
var pools []mo.ResourcePool
err := cluster.manager.scanMObjects(cluster.object.Entity().Self, RESOURCEPOOL_PROPS, &pools)
if err != nil {
return nil, errors.Wrap(err, "scanMObjects")
}
return pools, nil
}
func (cluster *SCluster) SyncResourcePool(name string) (*object.ResourcePool, error) {
pools, err := cluster.listResourcePools()
if err != nil {
return nil, errors.Wrap(err, "listResourcePools")
}
for i := range pools {
pool := NewResourcePool(cluster.manager, &pools[i], cluster.datacenter)
if strings.EqualFold(pool.GetId(), name) || strings.EqualFold(strings.Join(pool.GetPath(), "/"), name) ||
strings.EqualFold(strings.Join(pool.GetPath(), "|"), name) ||
strings.EqualFold(pool.GetName(), name) {
log.Infof("SyncResourcePool: %s found", strings.Join(pool.GetPath(), "|"))
return object.NewResourcePool(cluster.manager.client.Client, pools[i].Reference()), nil
}
}
log.Infof("SyncResourcePool: %s not found", name)
return nil, errors.Wrap(cloudprovider.ErrNotFound, "SyncResourcePool")
}
+58 -17
View File
@@ -145,6 +145,61 @@ func (host *SHost) GetName() string {
return formatName(host.SManagedObject.GetName())
}
func (host *SHost) getCluster() (*mo.ClusterComputeResource, error) {
moHost := host.getHostSystem()
if moHost.Parent.Type != "ClusterComputeResource" {
return nil, errors.Wrapf(cloudprovider.ErrNotFound, "host %s parent is not the cluster resource", host.GetName())
}
cluster := &mo.ClusterComputeResource{}
err := host.manager.reference2Object(*moHost.Parent, []string{"name", "resourcePool"}, cluster)
if err != nil {
return nil, errors.Wrap(err, "SESXiClient.reference2Object")
}
return cluster, nil
}
func (host *SHost) GetCluster() (*SCluster, error) {
cluster, err := host.getCluster()
if err != nil {
return nil, errors.Wrap(err, "getCluster")
}
return NewCluster(host.manager, cluster, host.datacenter), nil
}
func (host *SHost) GetSchedtags() ([]cloudprovider.Schedtag, error) {
dc, err := host.GetDatacenter()
if err != nil {
return nil, err
}
cluster, err := host.GetCluster()
if err != nil {
return nil, err
}
ret := []cloudprovider.Schedtag{}
ret = append(ret, cloudprovider.Schedtag{
Name: fmt.Sprintf("cluster:|%s|%s|%s", host.datacenter.manager.cpcfg.Name, dc.GetName(), cluster.GetName()),
Id: fmt.Sprintf("%s|%s|%s", host.datacenter.manager.cpcfg.Id, dc.GetId(), cluster.GetId()),
})
pools, err := cluster.listResourcePools()
if err != nil {
return nil, err
}
for i := range pools {
pool := NewResourcePool(host.manager, &pools[i], host.datacenter)
if pool.IsDefault() {
continue
}
ret = append(ret, cloudprovider.Schedtag{
Name: fmt.Sprintf("pool:|%s|%s|%s|%s", host.datacenter.manager.cpcfg.Name, dc.GetName(), cluster.GetName(), strings.Join(pool.GetPath(), "|")),
Id: fmt.Sprintf("%s|%s|%s|%s", host.datacenter.manager.cpcfg.Id, dc.GetId(), cluster.GetId(), pool.GetId()),
Meta: map[string]string{
cloudprovider.METADATA_POOL_ID: pool.GetId(),
},
})
}
return ret, nil
}
func (host *SHost) getHostSystem() *mo.HostSystem {
return host.object.(*mo.HostSystem)
}
@@ -1769,25 +1824,11 @@ func (host *SHost) SyncResourcePool(name string) (*object.ResourcePool, error) {
if len(name) == 0 {
return host.GetResourcePool()
}
dc, err := host.GetDatacenter()
cluster, err := host.GetCluster()
if err != nil {
return nil, errors.Wrap(err, "GetDatacenter")
return host.GetResourcePool()
}
pools, err := dc.listResourcePools()
if err != nil {
return nil, errors.Wrap(err, "ListResourcePools")
}
for i := range pools {
pool := NewResourcePool(host.manager, &pools[i], host.datacenter)
if strings.EqualFold(strings.Join(pool.GetPath(), "/"), name) ||
strings.EqualFold(strings.Join(pool.GetPath(), "|"), name) ||
strings.EqualFold(pool.GetName(), name) {
log.Debugf("SyncResourcePool: %s found", strings.Join(pool.GetPath(), "|"))
return object.NewResourcePool(host.manager.client.Client, pools[i].Reference()), nil
}
}
log.Errorf("SyncResourcePool: %s not found", name)
return host.GetResourcePool()
return cluster.SyncResourcePool(name)
}
func (host *SHost) GetSiblingHosts() ([]*SHost, error) {
+6
View File
@@ -15,6 +15,8 @@
package esxi
import (
"strings"
"github.com/vmware/govmomi/vim25/mo"
api "yunion.io/x/cloudmux/pkg/apis/compute"
@@ -53,6 +55,10 @@ func (pool *SResourcePool) getParentEntity(obj *mo.ManagedEntity) *mo.ManagedEnt
return nil
}
func (pool *SResourcePool) IsDefault() bool {
return strings.EqualFold(pool.GetName(), "Resources")
}
func (pool *SResourcePool) fetchPath() []string {
path := []string{pool.GetName()}
obj := pool.object.Entity()
+5 -2
View File
@@ -14,7 +14,10 @@
package multicloud
import "yunion.io/x/cloudmux/pkg/apis"
import (
"yunion.io/x/cloudmux/pkg/apis"
"yunion.io/x/cloudmux/pkg/cloudprovider"
)
type SHostBase struct {
SResourceBase
@@ -33,7 +36,7 @@ func (host *SHostBase) GetReservedMemoryMb() int {
return 0
}
func (host *SHostBase) GetSchedtags() ([]string, error) {
func (host *SHostBase) GetSchedtags() ([]cloudprovider.Schedtag, error) {
return nil, nil
}