Merge pull request #7848 from rainzm/automated-cherry-pick-of-#7845-upstream-release-3.2

Automated cherry pick of #7845: Feat & Fix & Refactor for ESXi

pkg/compute/guestdrivers/esxi.go

@@ -368,3 +368,42 @@ func (self *SESXiGuestDriver) RequestAssociateEip(ctx context.Context, userCred
 func (self *SESXiGuestDriver) IsSupportCdrom(guest *models.SGuest) (bool, error) {
 	return false, nil
 }
+
+func (self *SESXiGuestDriver) RequestSyncstatusOnHost(ctx context.Context, guest *models.SGuest, host *models.SHost, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
+	ihost, err := host.GetIHost()
+	if err != nil {
+		return nil, err
+	}
+	ivm, err := ihost.GetIVMById(guest.GetExternalId())
+	if err != nil && errors.Cause(err) != errors.ErrNotFound {
+		return nil, err
+	}
+	// VM may be migrated by Vcenter, try to find VM from whole datacenter.
+	if err != nil {
+		ehost := ihost.(*esxi.SHost)
+		dc, err := ehost.GetDatacenter()
+		if err != nil {
+			return nil, errors.Wrapf(err, "ehost.GetDatacenter")
+		}
+		vm, err := dc.FetchVMById(guest.GetExternalId())
+		if err != nil {
+			log.Errorf("fail to find ivm by id %q in dc %q: %v", guest.GetExternalId(), dc.GetName(), err)
+			return nil, err
+		}
+		ihost = vm.GetIHost()
+		host = models.HostManager.FetchHostByExtId(ihost.GetGlobalId())
+		if host == nil {
+			return nil, errors.Wrapf(errors.ErrNotFound, "find ivm %q in ihost %q which is not existed here", guest.GetExternalId(), ihost.GetGlobalId())
+		}
+		ivm = vm
+	}
+	err = guest.SyncAllWithCloudVM(ctx, userCred, host, ivm)
+	if err != nil {
+		return nil, err
+	}
+
+	status := GetCloudVMStatus(ivm)
+	body := jsonutils.NewDict()
+	body.Add(jsonutils.NewString(status), "status")
+	return body, nil
+}

pkg/compute/models/hosts.go

@@ -5240,3 +5240,15 @@ func (manager *SHostManager) ListItemExportKeys(ctx context.Context,
 	}
 	return q, nil
 }
+
+func (manager *SHostManager) FetchHostByExtId(extid string) *SHost {
+	host := SHost{}
+	host.SetModelManager(manager, &host)
+	err := manager.Query().Equals("external_id", extid).First(&host)
+	if err != nil {
+		log.Errorf("fetchHostByExtId fail %s", err)
+		return nil
+	} else {
+		return &host
+	}
+}

pkg/hostman/storageman/imagecachemanager_agent.go

@@ -225,7 +225,7 @@ func (c *SAgentImageCacheManager) perfetchTemplateVMImageCache(ctx context.Conte
 	if err != nil {
 		return nil, errors.Wrap(err, "host.GetDatacenter")
 	}
-	_, err = dc.GetTemplateVMById(data.ImageExternalId)
+	_, err = dc.FetchTemplateVMById(data.ImageExternalId)
 	if err != nil {
 		return nil, err
 	}

pkg/multicloud/esxi/datacenter.go

@@ -18,6 +18,8 @@ import (
 	"strings"
 
 	"github.com/vmware/govmomi/object"
+	"github.com/vmware/govmomi/property"
+	"github.com/vmware/govmomi/view"
 	"github.com/vmware/govmomi/vim25/mo"
 	"github.com/vmware/govmomi/vim25/types"
 
@@ -237,32 +239,85 @@ func (dc *SDatacenter) getDcObj() *object.Datacenter {
 	return object.NewDatacenter(dc.manager.client.Client, dc.object.Reference())
 }
 
-// fetchVms will identify if VM is a template and return two different arrays; the latter contains all template vms.
-func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference, all bool) ([]cloudprovider.ICloudVM, []*SVirtualMachine, error) {
-	var vms []mo.VirtualMachine
+func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference, all bool) ([]*SVirtualMachine, error) {
+	var movms []mo.VirtualMachine
 	if vmRefs != nil {
-		err := dc.manager.references2Objects(vmRefs, VIRTUAL_MACHINE_PROPS, &vms)
+		err := dc.manager.references2Objects(vmRefs, VIRTUAL_MACHINE_PROPS, &movms)
 		if err != nil {
-			return nil, nil, errors.Wrap(err, "dc.manager.references2Objects")
+			return nil, errors.Wrap(err, "dc.manager.references2Objects")
 		}
 	}
 
 	// avoid applying new memory and copying
-	retVms := make([]cloudprovider.ICloudVM, 0, len(vms)/2)
-	templateVMs := make([]*SVirtualMachine, 0, 2)
-	for i := 0; i < len(vms); i += 1 {
-		if all || !strings.HasPrefix(vms[i].Entity().Name, api.ESXI_IMAGE_CACHE_TMP_PREFIX) {
-			vmObj := NewVirtualMachine(dc.manager, &vms[i], dc)
-			if vms[i].Config != nil && vms[i].Config.Template {
-				templateVMs = append(templateVMs, vmObj)
-				continue
-			}
-			if vmObj != nil {
-				retVms = append(retVms, vmObj)
-			}
+	vms := make([]*SVirtualMachine, 0, len(movms))
+	for i := range movms {
+		if all || !strings.HasPrefix(movms[i].Entity().Name, api.ESXI_IMAGE_CACHE_TMP_PREFIX) {
+			vms = append(vms, NewVirtualMachine(dc.manager, &movms[i], dc))
 		}
 	}
-	return retVms, templateVMs, nil
+	return vms, nil
+}
+
+func (dc *SDatacenter) FetchVMs() ([]*SVirtualMachine, error) {
+	return dc.fetchVMs(property.Filter{})
+}
+
+func (dc *SDatacenter) FetchNoTemplateVMs() ([]*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = false
+	return dc.fetchVMs(filter)
+}
+
+func (dc *SDatacenter) fetchVMs(filter property.Filter) ([]*SVirtualMachine, error) {
+	odc := dc.getObjectDatacenter()
+	root := odc.Reference()
+	m := view.NewManager(dc.manager.client.Client)
+	v, err := m.CreateContainerView(dc.manager.context, root, []string{"VirtualMachine"}, true)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		_ = v.Destroy(dc.manager.context)
+	}()
+	objs, err := v.Find(dc.manager.context, []string{"VirtualMachine"}, filter)
+	if err != nil {
+		return nil, err
+	}
+	vms, err := dc.fetchVms(objs, false)
+	return vms, err
+}
+
+func (dc *SDatacenter) FetchTemplateVMs() ([]*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = true
+	return dc.fetchVMs(filter)
+}
+
+func (dc *SDatacenter) FetchTemplateVMById(id string) (*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["config.template"] = true
+	filter["summary.config.uuid"] = id
+	vms, err := dc.fetchVMs(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
+}
+
+func (dc *SDatacenter) FetchVMById(id string) (*SVirtualMachine, error) {
+	filter := property.Filter{}
+	filter["summary.config.uuid"] = id
+	vms, err := dc.fetchVMs(filter)
+	if err != nil {
+		return nil, err
+	}
+	if len(vms) == 0 {
+		return nil, errors.ErrNotFound
+	}
+	return vms[0], nil
 }
 
 func (dc *SDatacenter) fetchDatastores(datastoreRefs []types.ManagedObjectReference) ([]cloudprovider.ICloudStorage, error) {
@@ -381,24 +436,3 @@ func (dc *SDatacenter) GetTemplateVMs() ([]*SVirtualMachine, error) {
 	}
 	return templateVms, nil
 }
-
-func (dc *SDatacenter) GetTemplateVMById(id string) (*SVirtualMachine, error) {
-	id = dc.manager.getPrivateId(id)
-	hosts, err := dc.GetIHosts()
-	if err != nil {
-		return nil, errors.Wrap(err, "SDatacenter.GetIHosts")
-	}
-	for _, ihost := range hosts {
-		host := ihost.(*SHost)
-		tvms, err := host.GetTemplateVMs()
-		if err != nil {
-			return nil, errors.Wrap(err, "host.GetTemplateVMs")
-		}
-		for i := range tvms {
-			if tvms[i].GetGlobalId() == id {
-				return tvms[i], nil
-			}
-		}
-	}
-	return nil, cloudprovider.ErrNotFound
-}

pkg/multicloud/esxi/host.go

@@ -179,6 +179,7 @@ func (self *SHost) fetchVMs(all bool) error {
 	}
 
 	MAX_TRIES := 3
+	var vms []*SVirtualMachine
 	for tried := 0; tried < MAX_TRIES; tried += 1 {
 		hostVms := self.getHostSystem().Vm
 		if len(hostVms) == 0 {
@@ -186,15 +187,20 @@ func (self *SHost) fetchVMs(all bool) error {
 			return nil
 		}
 
-		vms, templatevms, err := dc.fetchVms(hostVms, all)
+		vms, err = dc.fetchVms(hostVms, all)
 		if err != nil {
 			log.Errorf("dc.fetchVms fail %s", err)
 			time.Sleep(time.Second)
 			self.Refresh()
 			continue
 		}
-		self.vms = vms
-		self.tempalteVMs = templatevms
+	}
+	for _, vm := range vms {
+		if vm.IsTemplate() {
+			self.tempalteVMs = append(self.tempalteVMs, vm)
+		} else {
+			self.vms = append(self.vms, vm)
+		}
 	}
 	return nil
 }
@@ -684,9 +690,13 @@ func (self *SHost) CreateVM2(ctx context.Context, ds *SDatastore, params SCreate
 	if err != nil {
 		return nil, errors.Wrap(err, "SEsxiClient.FindHostByIp")
 	}
-	temvm, err := imgHost.GetTemplateVMById(imageInfo.ImageExternalId)
+	dc, err := imgHost.GetDatacenter()
 	if err != nil {
-		return nil, errors.Wrap(err, "SHost.GetTemplateVMById")
+		return nil, errors.Wrap(err, "host.GetDatacenter")
+	}
+	temvm, err := dc.FetchTemplateVMById(imageInfo.ImageExternalId)
+	if err != nil {
+		return nil, errors.Wrapf(err, "datacenter.TemplateVMById for image %q and datacenter %q", imageInfo.ImageExternalId, dc.GetId())
 	}
 	return self.CloneVM(ctx, temvm, ds, params)
 }

pkg/multicloud/esxi/shell/virtualmachine.go

@@ -27,28 +27,53 @@ import (
 
 func init() {
 	type VirtualMachineListOptions struct {
-		HOSTIP   string `help:"Host IP"`
-		Template bool   `help:"Whether it is tempalte virtual machine"`
+		Datacenter string `help:"Datacenter"`
+		HostIP     string `help:"HostIP"`
+		Template   bool   `help:"Whether it is tempalte virtual machine, default:false"`
 	}
 	shellutils.R(&VirtualMachineListOptions{}, "vm-list", "List vms of a host", func(cli *esxi.SESXiClient, args *VirtualMachineListOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		if args.Template {
-			vms, err := host.GetTemplateVMs()
+		switch {
+		case len(args.HostIP) > 0:
+			host, err := cli.FindHostByIp(args.HostIP)
+			if err != nil {
+				return err
+			}
+			if args.Template {
+				vms, err := host.GetTemplateVMs()
+				if err != nil {
+					return err
+				}
+				printList(vms, []string{})
+				return nil
+			}
+			vms, err := host.GetIVMs2()
 			if err != nil {
 				return err
 			}
 			printList(vms, []string{})
 			return nil
+		case len(args.Datacenter) > 0:
+			dc, err := cli.FindDatacenterByMoId(args.Datacenter)
+			if err != nil {
+				return errors.Wrap(err, "FindDatacenterByMoId")
+			}
+			var vms []*esxi.SVirtualMachine
+			if args.Template {
+				vms, err = dc.FetchTemplateVMs()
+				if err != nil {
+					return errors.Wrap(err, "FetchTemplateVMs")
+				}
+			} else {
+				vms, err = dc.FetchNoTemplateVMs()
+				if err != nil {
+					return errors.Wrap(err, "FetchNoTemplateVMs")
+				}
+			}
+			printList(vms, []string{})
+			return nil
+		default:
+			return fmt.Errorf("Both Datacenter and HostIP cannot be empty")
 		}
-		vms, err := host.GetIVMs2()
-		if err != nil {
-			return err
-		}
-		printList(vms, []string{})
-		return nil
 	})
 
 	type VirtualMachineCloneOptions struct {
@@ -91,24 +116,45 @@ func init() {
 	})
 
 	type VirtualMachineShowOptions struct {
-		HOSTIP   string `help:"Host IP"`
-		VMID     string `help:"VM ID"`
-		Template bool
+		Datacenter string `help:"Datacenter"`
+		HostIP     string `help:"Host IP"`
+		VMID       string `help:"VM ID"`
+	}
+	getVM := func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) (*esxi.SVirtualMachine, error) {
+		var vm *esxi.SVirtualMachine
+		switch {
+		case len(args.HostIP) > 0:
+			host, err := cli.FindHostByIp(args.HostIP)
+			if err != nil {
+				return nil, errors.Wrap(err, "FindHostByIp")
+			}
+			ivm, err := host.GetIVMById(args.VMID)
+			if err != nil && errors.Cause(err) != errors.ErrNotFound {
+				return nil, err
+			}
+			if err != nil {
+				vm, err = host.GetTemplateVMById(args.VMID)
+				if err != nil {
+					return nil, errors.Wrap(err, "GetTemplateVMById")
+				}
+			}
+			vm = ivm.(*esxi.SVirtualMachine)
+		case len(args.Datacenter) > 0:
+			dc, err := cli.FindDatacenterByMoId(args.Datacenter)
+			if err != nil {
+				return nil, errors.Wrap(err, "FindDatacenterByMoId")
+			}
+			vm, err = dc.FetchVMById(args.VMID)
+			if err != nil {
+				return nil, errors.Wrap(err, "FetchVMById")
+			}
+		default:
+			return nil, fmt.Errorf("Both Datacenter and HostIP cannot be empty")
+		}
+		return vm, nil
 	}
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-show", "Show vm details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		if args.Template {
-			vm, err := host.GetTemplateVMById(args.VMID)
-			if err != nil {
-				return err
-			}
-			printObject(vm)
-			return nil
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -117,11 +163,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-nics", "Show vm nics details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -134,11 +176,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-disks", "Show vm disks details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -151,17 +189,12 @@ func init() {
 	})
 
 	type VirtualMachineDiskResizeOptions struct {
-		HOSTIP  string `help:"host ip"`
-		VMID    string `help:"virtual machine UUID"`
-		DISKIDX int    `help:"disk index"`
-		SIZEGB  int64  `help:"new size of disk"`
+		VirtualMachineShowOptions
+		DISKIDX int   `help:"disk index"`
+		SIZEGB  int64 `help:"new size of disk"`
 	}
 	shellutils.R(&VirtualMachineDiskResizeOptions{}, "vm-disk-resize", "Resize a vm disk", func(cli *esxi.SESXiClient, args *VirtualMachineDiskResizeOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, &args.VirtualMachineShowOptions)
 		if err != nil {
 			return err
 		}
@@ -178,11 +211,7 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-vnc", "Show vm VNC details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
-		if err != nil {
-			return err
-		}
-		vm, err := host.GetIVMById(args.VMID)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
@@ -195,15 +224,11 @@ func init() {
 	})
 
 	shellutils.R(&VirtualMachineShowOptions{}, "vm-file-status", "Show vm files details", func(cli *esxi.SESXiClient, args *VirtualMachineShowOptions) error {
-		host, err := cli.FindHostByIp(args.HOSTIP)
+		vm, err := getVM(cli, args)
 		if err != nil {
 			return err
 		}
-		vm, err := host.GetIVMById(args.VMID)
-		if err != nil {
-			return err
-		}
-		err = vm.(*esxi.SVirtualMachine).CheckFileInfo(context.Background())
+		err = vm.CheckFileInfo(context.Background())
 		if err != nil {
 			return err
 		}

pkg/multicloud/esxi/storage.go

@@ -259,7 +259,14 @@ func (self *SDatastore) getVMs() ([]cloudprovider.ICloudVM, error) {
 	if len(vms) == 0 {
 		return nil, nil
 	}
-	ret, _, err := dc.fetchVms(vms, false)
+	svms, err := dc.fetchVms(vms, false)
+	if err != nil {
+		return nil, err
+	}
+	ret := make([]cloudprovider.ICloudVM, len(svms))
+	for i := range svms {
+		ret[i] = svms[i]
+	}
 	return ret, err
 }
 

pkg/multicloud/esxi/virtualmachine.go

@@ -72,10 +72,7 @@ func (d byDiskType) Less(i, j int) bool {
 
 func NewVirtualMachine(manager *SESXiClient, vm *mo.VirtualMachine, dc *SDatacenter) *SVirtualMachine {
 	svm := &SVirtualMachine{SManagedObject: newManagedObject(manager, vm, dc)}
-	err := svm.fetchHardwareInfo()
-	if err != nil {
-		return nil
-	}
+	svm.fetchHardwareInfo()
 	return svm
 }
 
@@ -708,7 +705,7 @@ func (self *SVirtualMachine) UpdateUserData(userData string) error {
 	return nil
 }
 
-func (self *SVirtualMachine) fetchHardwareInfo() error {
+func (self *SVirtualMachine) fetchHardwareInfo() {
 	self.vnics = make([]SVirtualNIC, 0)
 	self.vdisks = make([]SVirtualDisk, 0)
 	self.cdroms = make([]SVirtualCdrom, 0)
@@ -722,7 +719,8 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 	}
 
 	if moVM == nil || moVM.Config == nil || moVM.Config.Hardware.Device == nil {
-		return errors.Error("invalid vm config")
+		log.Errorf("invalid vm config, moVM: %v", moVM)
+		return
 	}
 
 	for i := 0; i < len(moVM.Config.Hardware.Device); i += 1 {
@@ -750,7 +748,6 @@ func (self *SVirtualMachine) fetchHardwareInfo() error {
 	sort.Slice(self.vdisks, func(i, j int) bool {
 		return self.vdisks[i].GetIndex() < self.vdisks[j].GetIndex()
 	})
-	return nil
 }
 
 func (self *SVirtualMachine) getVdev(key int32) SVirtualDevice {
@@ -1218,3 +1215,8 @@ func (self *SVirtualMachine) FindMinDiffKey(limit int32) int32 {
 	}
 	return limit
 }
+
+func (self *SVirtualMachine) IsTemplate() bool {
+	movm := self.getVirtualMachine()
+	return movm.Config != nil && movm.Config.Template
+}