diff --git a/pkg/multicloud/esxi/datacenter.go b/pkg/multicloud/esxi/datacenter.go index f089b17f01..30de8900c7 100644 --- a/pkg/multicloud/esxi/datacenter.go +++ b/pkg/multicloud/esxi/datacenter.go @@ -25,6 +25,7 @@ import ( "github.com/vmware/govmomi/vim25/types" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/util/sets" api "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/cloudprovider" @@ -297,6 +298,33 @@ func (dc *SDatacenter) fetchFakeTemplateVMs(movms []mo.VirtualMachine, regex str return dc.fetchVms(objs, false) } +func (dc *SDatacenter) fetchVmsFromCache(vmRefs []types.ManagedObjectReference) ([]*SVirtualMachine, error) { + vmRefSet := sets.NewString() + for i := range vmRefs { + vmRefSet.Insert(vmRefs[i].String()) + } + ihosts, err := dc.GetIHosts() + if err != nil { + return nil, err + } + ret := make([]*SVirtualMachine, 0, len(vmRefs)) + for i := range ihosts { + ivms, err := ihosts[i].GetIVMs() + if err != nil { + return nil, err + } + for i := range ivms { + vm := ivms[i].(*SVirtualMachine) + s := vm.getVirtualMachine().Self.String() + if vmRefSet.Has(s) { + ret = append(ret, vm) + vmRefSet.Delete(s) + } + } + } + return ret, nil +} + func (dc *SDatacenter) fetchVms(vmRefs []types.ManagedObjectReference, all bool) ([]*SVirtualMachine, error) { var movms []mo.VirtualMachine if vmRefs != nil { diff --git a/pkg/multicloud/esxi/host.go b/pkg/multicloud/esxi/host.go index f2cdcb80ce..c9faa1ac18 100644 --- a/pkg/multicloud/esxi/host.go +++ b/pkg/multicloud/esxi/host.go @@ -19,7 +19,6 @@ import ( "fmt" "regexp" "strings" - "time" "github.com/vmware/govmomi/object" "github.com/vmware/govmomi/vim25/mo" @@ -36,7 +35,20 @@ import ( "yunion.io/x/onecloud/pkg/multicloud" ) -var HOST_SYSTEM_PROPS = []string{"name", "parent", "summary", "config", "hardware", "vm", "datastore", "network"} +var ( + hostConfigProps = []string{"config.network", "config.storageDevice"} + hostSummaryProps = []string{"summary.runtime", "summary.hardware", "summary.config.product", "summary.managementServerIp"} + hostHardWareProps = []string{"hardware.systemInfo"} +) + +var HOST_SYSTEM_PROPS []string + +func init() { + HOST_SYSTEM_PROPS = []string{"name", "parent", "vm", "datastore", "network"} + HOST_SYSTEM_PROPS = append(HOST_SYSTEM_PROPS, hostConfigProps...) + HOST_SYSTEM_PROPS = append(HOST_SYSTEM_PROPS, hostSummaryProps...) + HOST_SYSTEM_PROPS = append(HOST_SYSTEM_PROPS, hostHardWareProps...) +} type SHostStorageAdapterInfo struct { Device string @@ -200,22 +212,16 @@ func (self *SHost) fetchVMs(all bool) error { return err } - MAX_TRIES := 3 var vms []*SVirtualMachine - for tried := 0; tried < MAX_TRIES; tried += 1 { - hostVms := self.getHostSystem().Vm - if len(hostVms) == 0 { - // log.Errorf("host VMs are nil!!!!!") - return nil - } + hostVms := self.getHostSystem().Vm + if len(hostVms) == 0 { + // log.Errorf("host VMs are nil!!!!!") + return nil + } - vms, err = dc.fetchVms(hostVms, all) - if err != nil { - log.Errorf("dc.fetchVms fail %s", err) - time.Sleep(time.Second) - self.Refresh() - continue - } + vms, err = dc.fetchVms(hostVms, all) + if err != nil { + return err } for _, vm := range vms { if vm.IsTemplate() { @@ -285,20 +291,7 @@ func (self *SHost) GetIWires() ([]cloudprovider.ICloudWire, error) { } func (self *SHost) GetIStorages() ([]cloudprovider.ICloudStorage, error) { - moHost := self.getHostSystem() - dc, err := self.GetDatacenter() - if err != nil { - return nil, err - } - istorages := make([]cloudprovider.ICloudStorage, len(moHost.Datastore)) - for i := 0; i < len(moHost.Datastore); i += 1 { - storage, err := dc.GetIStorageByMoId(moRefId(moHost.Datastore[i])) - if err != nil { - return nil, err - } - istorages[i] = storage - } - return istorages, nil + return self.GetDataStores() } func (self *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, error) { @@ -315,6 +308,9 @@ func (self *SHost) GetIStorageById(id string) (cloudprovider.ICloudStorage, erro } func (self *SHost) GetEnabled() bool { + if self.getHostSystem().Summary.Runtime.InMaintenanceMode { + return false + } return true } @@ -324,6 +320,9 @@ func (self *SHost) GetHostStatus() string { HostSystemConnectionStateNotResponding = HostSystemConnectionState("notResponding") HostSystemConnectionStateDisconnected = HostSystemConnectionState("disconnected") */ + if self.getHostSystem().Summary.Runtime.InMaintenanceMode { + return api.HOST_OFFLINE + } switch self.getHostSystem().Summary.Runtime.ConnectionState { case types.HostSystemConnectionStateConnected: return api.HOST_ONLINE @@ -489,14 +488,21 @@ type SSysInfo struct { func (self *SHost) GetSysInfo() jsonutils.JSONObject { sysinfo := SSysInfo{} - sysinfo.Manufacture = self.getHostSystem().Summary.Hardware.Vendor - sysinfo.Model = self.getHostSystem().Summary.Hardware.Model - sysinfo.SerialNumber = self.getHostSystem().Hardware.SystemInfo.SerialNumber + host := self.getHostSystem() + sysinfo.Manufacture = host.Summary.Hardware.Vendor + sysinfo.Model = host.Summary.Hardware.Model + if host.Hardware != nil { + sysinfo.SerialNumber = host.Hardware.SystemInfo.SerialNumber + } return jsonutils.Marshal(&sysinfo) } func (self *SHost) GetSN() string { - return self.getHostSystem().Hardware.SystemInfo.SerialNumber + host := self.getHostSystem() + if host.Hardware != nil { + return host.Hardware.SystemInfo.SerialNumber + } + return "" } func (self *SHost) GetCpuCount() int { @@ -1393,28 +1399,19 @@ func (host *SHost) fetchDatastores() error { return nil } - dc, err := host.GetDatacenter() + dss := host.getHostSystem().Datastore + var datastores []mo.Datastore + err := host.manager.references2Objects(dss, DATASTORE_PROPS, &datastores) if err != nil { return err } - - MAX_TRIES := 3 - for tried := 0; tried < MAX_TRIES; tried += 1 { - hostDss := host.getHostSystem().Datastore - if len(hostDss) == 0 { - // log.Errorf("host VMs are nil!!!!!") - return nil + host.datastores = make([]cloudprovider.ICloudStorage, 0) + for i := 0; i < len(datastores); i += 1 { + ds := NewDatastore(host.manager, &datastores[i], host.datacenter) + dsId := ds.GetGlobalId() + if len(dsId) > 0 { + host.datastores = append(host.datastores, ds) } - - dss, err := dc.fetchDatastores(hostDss) - if err != nil { - log.Errorf("dc.fetchVms fail %s", err) - time.Sleep(time.Second) - host.Refresh() - continue - } - host.datastores = dss - break } return nil } diff --git a/pkg/multicloud/esxi/manager.go b/pkg/multicloud/esxi/manager.go index cd0ca69b2e..b7b3e9317b 100644 --- a/pkg/multicloud/esxi/manager.go +++ b/pkg/multicloud/esxi/manager.go @@ -342,10 +342,10 @@ func (cli *SESXiClient) newDatacenterFromMo(mo *mo.Datacenter) (*SDatacenter, er if err != nil { return nil, errors.Wrap(err, "dc.scanHosts") } - err = dc.scanDatastores() - if err != nil { - return nil, errors.Wrap(err, "dc.scanDatastores") - } + // err = dc.scanDatastores() + // if err != nil { + // return nil, errors.Wrap(err, "dc.scanDatastores") + // } return dc, nil } diff --git a/pkg/multicloud/esxi/network.go b/pkg/multicloud/esxi/network.go index 409f1180e8..e7879683c7 100644 --- a/pkg/multicloud/esxi/network.go +++ b/pkg/multicloud/esxi/network.go @@ -43,8 +43,8 @@ const ( VLAN_MODE_TRUNK = "trunk" ) -var NETWORK_PROPS = []string{"name", "parent", "summary", "host", "vm"} -var DVPORTGROUP_PROPS = []string{"name", "parent", "summary", "host", "vm", "config", "key"} +var NETWORK_PROPS = []string{"name", "parent", "host"} +var DVPORTGROUP_PROPS = []string{"name", "parent", "host", "config", "key"} type SNetwork struct { SManagedObject diff --git a/pkg/multicloud/esxi/storage.go b/pkg/multicloud/esxi/storage.go index 2731526698..3232ee9e0d 100644 --- a/pkg/multicloud/esxi/storage.go +++ b/pkg/multicloud/esxi/storage.go @@ -61,6 +61,7 @@ type SDatastore struct { // vms []cloudprovider.ICloudVM ihosts []cloudprovider.ICloudHost + idisks []cloudprovider.ICloudDisk storageCache *SDatastoreImageCache } @@ -330,7 +331,7 @@ func (self *SDatastore) getVMs() ([]cloudprovider.ICloudVM, error) { if len(vms) == 0 { return nil, nil } - svms, err := dc.fetchVms(vms, false) + svms, err := dc.fetchVmsFromCache(vms) if err != nil { return nil, err } @@ -357,20 +358,32 @@ func (self *SDatastore) GetIDiskById(idStr string) (cloudprovider.ICloudDisk, er return nil, cloudprovider.ErrNotFound } -func (self *SDatastore) GetIDisks() ([]cloudprovider.ICloudDisk, error) { +func (self *SDatastore) fetchDisks() error { vms, err := self.getVMs() if err != nil { - return nil, err + return err } allDisks := make([]cloudprovider.ICloudDisk, 0) for i := 0; i < len(vms); i += 1 { disks, err := vms[i].GetIDisks() if err != nil { - return nil, err + return err } allDisks = append(allDisks, disks...) } - return allDisks, nil + self.idisks = allDisks + return nil +} + +func (self *SDatastore) GetIDisks() ([]cloudprovider.ICloudDisk, error) { + if self.idisks != nil { + return self.idisks, nil + } + err := self.fetchDisks() + if err != nil { + return nil, err + } + return self.idisks, nil } func (self *SDatastore) isLocalVMFS() bool { diff --git a/pkg/multicloud/esxi/storagecache.go b/pkg/multicloud/esxi/storagecache.go index cb8da0612d..3d0de21a98 100644 --- a/pkg/multicloud/esxi/storagecache.go +++ b/pkg/multicloud/esxi/storagecache.go @@ -153,10 +153,47 @@ func (self *SDatastoreImageCache) GetIImageInImagecache() ([]cloudprovider.IClou return ret, nil } +var ErrTimeConsuming = errors.Error("time consuming") + +func (self *SDatastoreImageCache) getTemplateVMsFromCache() ([]*SVirtualMachine, error) { + ihosts, err := self.datastore.datacenter.GetIHosts() + if err != nil { + return nil, err + } + dsRef := self.datastore.getDatastore().Self + ret := make([]*SVirtualMachine, 0) + for i := range ihosts { + tvms := ihosts[i].(*SHost).tempalteVMs + if tvms == nil { + return nil, ErrTimeConsuming + } + for _, vm := range tvms { + dss := vm.getVirtualMachine().Datastore + for _, ds := range dss { + if ds == dsRef { + ret = append(ret, vm) + break + } + } + } + } + return ret, nil +} + func (self *SDatastoreImageCache) GetIImageInTemplateVMs() ([]cloudprovider.ICloudImage, error) { ret := make([]cloudprovider.ICloudImage, 0, 2) log.Infof("start to GetIImages") + datastore := self.datastore + if datastore.datacenter.ihosts != nil { + vms, err := self.getTemplateVMsFromCache() + if err == nil { + for i := range vms { + ret = append(ret, NewVMTemplate(vms[i], self)) + } + return ret, nil + } + } realTemplates, err := self.getTempalteVMs() if err != nil { return nil, errors.Wrap(err, "getTemplateVMs") diff --git a/pkg/multicloud/esxi/virtualmachine.go b/pkg/multicloud/esxi/virtualmachine.go index de1eec41bd..39abc37068 100644 --- a/pkg/multicloud/esxi/virtualmachine.go +++ b/pkg/multicloud/esxi/virtualmachine.go @@ -49,7 +49,20 @@ import ( "yunion.io/x/onecloud/pkg/util/version" ) -var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "runtime", "summary", "config", "guest", "resourcePool", "layoutEx", "snapshot"} +var ( + vmSummaryProps = []string{"summary.runtime.powerState", "summary.config.uuid", "summary.config.memorySizeMB", "summary.config.numCpu"} + // vmConfigProps = []string{"config.template", "config.alternateGuestName", "config.hardware", "config.guestId", "config.guestFullName", "config.firmware", "config.version", "config.createDate"} + vmGuestProps = []string{"guest.net", "guest.guestState", "guest.toolsStatus", "guest.toolsRunningStatus", "guest.toolsVersion"} + vmLayoutExProps = []string{"layoutEx.file"} +) + +var VIRTUAL_MACHINE_PROPS = []string{"name", "parent", "resourcePool", "snapshot", "config"} + +func init() { + VIRTUAL_MACHINE_PROPS = append(VIRTUAL_MACHINE_PROPS, vmSummaryProps...) + // VIRTUAL_MACHINE_PROPS = append(VIRTUAL_MACHINE_PROPS, vmConfigProps...) + VIRTUAL_MACHINE_PROPS = append(VIRTUAL_MACHINE_PROPS, vmGuestProps...) +} type SVirtualMachine struct { multicloud.SInstanceBase @@ -128,10 +141,10 @@ func (self *SVirtualMachine) GetGlobalId() string { } func (self *SVirtualMachine) GetStatus() string { - err := self.CheckFileInfo(context.Background()) - if err != nil { - return api.VM_UNKNOWN - } + // err := self.CheckFileInfo(context.Background()) + // if err != nil { + // return api.VM_UNKNOWN + // } vm := object.NewVirtualMachine(self.manager.client.Client, self.getVirtualMachine().Self) state, err := vm.PowerState(self.manager.context) if err != nil { @@ -735,10 +748,10 @@ func (self *SVirtualMachine) fetchHardwareInfo() error { moVM := self.getVirtualMachine() - MAX_TRIES := 3 - for tried := 0; tried < MAX_TRIES && (moVM == nil || moVM.Config == nil || moVM.Config.Hardware.Device == nil); tried += 1 { - time.Sleep(time.Second) - } + // MAX_TRIES := 3 + // for tried := 0; tried < MAX_TRIES && (moVM == nil || moVM.Config == nil || moVM.Config.Hardware.Device == nil); tried += 1 { + // time.Sleep(time.Second) + // } if moVM == nil || moVM.Config == nil || moVM.Config.Hardware.Device == nil { return fmt.Errorf("invalid vm") @@ -903,6 +916,20 @@ func (self *SVirtualMachine) devNumWithCtrlKey(ctrlKey int32) int { return n } +func (self *SVirtualMachine) getLayoutEx() *types.VirtualMachineFileLayoutEx { + vm := self.getVirtualMachine() + if vm.LayoutEx != nil { + return vm.LayoutEx + } + var nvm mo.VirtualMachine + err := self.manager.reference2Object(vm.Self, vmLayoutExProps, &nvm) + if err != nil { + log.Errorf("unable to fetch LayoutEx.File from vc: %v", err) + } + vm.LayoutEx = nvm.LayoutEx + return vm.LayoutEx +} + func (self *SVirtualMachine) CreateDisk(ctx context.Context, sizeMb int, uuid string, driver string) error { if driver == "pvscsi" { driver = "scsi" @@ -979,11 +1006,11 @@ func (self *SVirtualMachine) createDriverAndDisk(ctx context.Context, sizeMb int } func (self *SVirtualMachine) copyRootDisk(ctx context.Context, imagePath string) (string, error) { - movm := self.getVirtualMachine() - if movm.LayoutEx == nil || len(movm.LayoutEx.File) == 0 { + layoutEx := self.getLayoutEx() + if layoutEx == nil || len(layoutEx.File) == 0 { return "", fmt.Errorf("invalid LayoutEx") } - file := movm.LayoutEx.File[0].Name + file := layoutEx.File[0].Name // find stroage host := self.GetIHost() storages, err := host.GetIStorages() @@ -1097,9 +1124,9 @@ func (self *SVirtualMachine) getResourcePool() (*SResourcePool, error) { } func (self *SVirtualMachine) CheckFileInfo(ctx context.Context) error { - vm := self.getVirtualMachine() - if vm.LayoutEx != nil && len(vm.LayoutEx.File) > 0 { - file := vm.LayoutEx.File[0] + layoutEx := self.getLayoutEx() + if layoutEx != nil && len(layoutEx.File) > 0 { + file := layoutEx.File[0] host := self.GetIHost() storages, err := host.GetIStorages() if err != nil {