diff --git a/build/docker/Dockerfile.baremetal-agent b/build/docker/Dockerfile.baremetal-agent index 8bb3c02e37..2385b4a78d 100644 --- a/build/docker/Dockerfile.baremetal-agent +++ b/build/docker/Dockerfile.baremetal-agent @@ -1,4 +1,4 @@ -FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.0.1 +FROM registry.cn-beijing.aliyuncs.com/yunionio/baremetal-base:v0.2 MAINTAINER "Zexi Li " diff --git a/build/docker/Dockerfile.baremetal-base b/build/docker/Dockerfile.baremetal-base index 7f96bb029e..3b1054b5d0 100644 --- a/build/docker/Dockerfile.baremetal-base +++ b/build/docker/Dockerfile.baremetal-base @@ -5,13 +5,16 @@ MAINTAINER "Yaoqi Wan " RUN mkdir -p /opt/yunion/bin RUN apk update && \ - apk add --no-cache ipmitool ethtool tzdata ca-certificates && \ + apk add --no-cache ipmitool ethtool tzdata ca-certificates cdrkit coreutils && \ rm -rf /var/cache/apk/* RUN cp /usr/share/zoneinfo/Asia/Shanghai /etc/localtime RUN mkdir -p /opt/cloud/yunion/baremetal +# udpate latest pci.ids +RUN wget -O /opt/cloud/yunion/baremetal/pci.ids http://pci-ids.ucw.cz/v2.2/pci.ids + # download baremetal-pxerom-*.x86_64.rpm and unpack rpm in current directory COPY ./opt/cloud/yunion/baremetal/bootia32.efi /opt/cloud/yunion/baremetal COPY ./opt/cloud/yunion/baremetal/bootx64.efi /opt/cloud/yunion/baremetal @@ -27,13 +30,3 @@ COPY ./opt/cloud/yunion/baremetal/libutil.c32 /opt/cloud/yunion/baremetal COPY ./opt/cloud/yunion/baremetal/lpxelinux.0 /opt/cloud/yunion/baremetal COPY ./opt/cloud/yunion/baremetal/menu.c32 /opt/cloud/yunion/baremetal COPY ./opt/cloud/yunion/baremetal/pxelinux.0 /opt/cloud/yunion/baremetal - -# add mkisofs with bundles libraries -ADD ./mkisofs /bin/mkisofs -ADD ./.mkisofs.bin /bin/.mkisofs.bin -ADD ./bundles/mkisofs /bin/bundles/mkisofs -# use GNU timeout command replace busybox timeout command -RUN rm /usr/bin/timeout -ADD ./timeout /bin/timeout -ADD ./.timeout.bin /bin/.timeout.bin -ADD ./bundles/timeout /bin/bundles/timeout \ No newline at end of file diff --git a/pkg/apis/compute/host.go b/pkg/apis/compute/host.go index b308aaef0e..e9ad70cf02 100644 --- a/pkg/apis/compute/host.go +++ b/pkg/apis/compute/host.go @@ -23,13 +23,23 @@ import ( type HostSpec struct { apis.Meta - Cpu int `json:"cpu"` - Mem int `json:"mem"` - NicCount int `json:"nic_count"` - Manufacture string `json:"manufacture"` - Model string `json:"model"` - Disk DiskDriverSpec `json:"disk"` - Driver string `json:"driver"` + Cpu int `json:"cpu"` + Mem int `json:"mem"` + NicCount int `json:"nic_count"` + Manufacture string `json:"manufacture"` + Model string `json:"model"` + Disk DiskDriverSpec `json:"disk"` + Driver string `json:"driver"` + IsolatedDevices []IsolatedDeviceSpec `json:"isolated_devices"` +} + +type IsolatedDeviceSpec struct { + apis.Meta + + DevType string `json:"dev_type"` + Model string `json:"model"` + PciId string `json:"pci_id"` + Vendor string `json:"vendor"` } type DiskDriverSpec map[string]DiskAdapterSpec diff --git a/pkg/apis/compute/isolated_device.go b/pkg/apis/compute/isolated_device.go index ef47c7dcee..3672207dc2 100644 --- a/pkg/apis/compute/isolated_device.go +++ b/pkg/apis/compute/isolated_device.go @@ -53,6 +53,9 @@ type IsolatedDeviceListInput struct { // 设备VENDOE编号 VendorDeviceId []string `json:"vendor_device_id"` + + // 展示物理机的上的设备 + ShowBaremetalIsolatedDevices bool `json:"show_baremetal_isolated_devices"` } type IsolatedDeviceCreateInput struct { diff --git a/pkg/baremetal/manager.go b/pkg/baremetal/manager.go index 4ea03440e2..4eb58f451d 100644 --- a/pkg/baremetal/manager.go +++ b/pkg/baremetal/manager.go @@ -536,6 +536,10 @@ func newBaremetalInstance(man *SBaremetalManager, desc jsonutils.JSONObject) (*S return bm, nil } +func (b *SBaremetalInstance) GetDHCPServerIP() (net.IP, error) { + return b.manager.Agent.GetDHCPServerIP() +} + func (b *SBaremetalInstance) GetClientSession() *mcclient.ClientSession { return b.manager.GetClientSession() } diff --git a/pkg/baremetal/tasks/baseprepare.go b/pkg/baremetal/tasks/baseprepare.go index 0a9a180abf..0005b024aa 100644 --- a/pkg/baremetal/tasks/baseprepare.go +++ b/pkg/baremetal/tasks/baseprepare.go @@ -16,6 +16,7 @@ package tasks import ( "fmt" + "net" "strings" "time" @@ -33,6 +34,7 @@ import ( "yunion.io/x/onecloud/pkg/baremetal/utils/ipmitool" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/compute/baremetal" + "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/modules" "yunion.io/x/onecloud/pkg/util/logclient" @@ -55,14 +57,15 @@ func newBaremetalPrepareTask(baremetal IBaremetal, userCred mcclient.TokenCreden } type baremetalPrepareInfo struct { - sysInfo *types.SSystemInfo - cpuInfo *types.SCPUInfo - dmiCpuInfo *types.SDMICPUInfo - memInfo *types.SDMIMemInfo - nicsInfo []*types.SNicDevInfo - diskInfo []*baremetal.BaremetalStorage - storageDriver string - ipmiInfo *types.SIPMIInfo + sysInfo *types.SSystemInfo + cpuInfo *types.SCPUInfo + dmiCpuInfo *types.SDMICPUInfo + memInfo *types.SDMIMemInfo + nicsInfo []*types.SNicDevInfo + diskInfo []*baremetal.BaremetalStorage + storageDriver string + ipmiInfo *types.SIPMIInfo + isolatedDevicesInfo []*isolated_device.PCIDevice } func (task *sBaremetalPrepareTask) GetStartTime() time.Time { @@ -95,6 +98,14 @@ func (task *sBaremetalPrepareTask) prepareBaremetalInfo(cli *ssh.Client) (*barem if err != nil { return nil, err } + dhcpServerIp, err := task.baremetal.GetDHCPServerIP() + if err != nil { + log.Errorf("failed get dhcp server %s", err) + } + isolatedDevicesInfo, err := getIsolatedDevicesInfo(cli, dhcpServerIp) + if err != nil { + return nil, err + } raidDiskInfo, nonRaidDiskInfo, pcieDiskInfo, err := detect_storages.DetectStorageInfo(cli, true) if err != nil { @@ -140,6 +151,7 @@ func (task *sBaremetalPrepareTask) prepareBaremetalInfo(cli *ssh.Client) (*barem diskInfo, storageDriver, ipmiInfo, + isolatedDevicesInfo, }, nil } @@ -393,6 +405,12 @@ func (task *sBaremetalPrepareTask) updateBmInfo(cli *ssh.Client, i *baremetalPre log.Errorf("sendStorageInfo error: %v", err) return errors.Wrap(err, "task.sendStorageInfo") } + if len(i.isolatedDevicesInfo) > 0 { + err = task.sendIsolatedDevicesInfo(task.getClientSession(), i.isolatedDevicesInfo) + if err != nil { + return errors.Wrap(err, "send isolated devices info") + } + } // XXX do not change nic order anymore // for i := range nicsInfo { // if nicsInfo[i].Mac.String() == adminNic.GetMac().String() { @@ -654,6 +672,69 @@ func getNicsInfo(cli *ssh.Client) ([]*types.SNicDevInfo, error) { return sysutils.ParseNicInfo(ret), nil } +func getIsolatedDevicesInfo(cli *ssh.Client, ip net.IP) ([]*isolated_device.PCIDevice, error) { + // fetch pci.ids from baremetal agent + var updatedPciids bool + if ip != nil { + out, err := cli.Run(fmt.Sprintf("tftp -g -r pci.ids %s -l /pci.ids", ip)) + if err != nil { + log.Errorf("tftp failed download pciids %s %s", err, out) + } else { + updatedPciids = true + } + } + + var bootVgaPath = []string{} + lines, err := cli.Run("ls /sys/bus/pci/devices/*/boot_vga") + if err != nil { + log.Errorf("failed find boot vga %s", err) + } + for i := 0; i < len(lines); i++ { + bootVgaPath = append(bootVgaPath, strings.TrimSpace(lines[i])) + } + + cmd := "lspci -nnmm | egrep '3D|VGA'" + if updatedPciids { + cmd = "lspci -i /pci.ids -nnmm | egrep '3D|VGA'" + } + + lines, err = cli.Run(cmd) + if err != nil { + return nil, errors.Wrapf(err, "run %s", cmd) + } + devs := []*isolated_device.PCIDevice{} + for _, line := range lines { + if len(line) > 0 { + dev := isolated_device.NewPCIDevice2(line) + if len(dev.Addr) > 0 && !isBootVga(cli, dev, bootVgaPath) { + devs = append(devs, dev) + } + } + } + return devs, nil +} + +func isBootVga(cli *ssh.Client, dev *isolated_device.PCIDevice, bootVgaPath []string) bool { + for i := 0; i < len(bootVgaPath); i++ { + if strings.Contains(bootVgaPath[i], dev.Addr) { + out, err := cli.RawRun(fmt.Sprintf("cat %s", bootVgaPath[i])) + if err != nil { + log.Errorf("cat boot_vga %s failed %s", bootVgaPath[i], err) + return false + } else if len(out) > 0 { + if strings.HasPrefix(out[0], "1") { + log.Infof("device %s is boot vga", bootVgaPath[i]) + return true + } else { + return false + } + } + break + } + } + return false +} + func isIPMIEnable(cli *ssh.Client) (bool, error) { ret, err := cli.Run("/usr/sbin/dmidecode -t 38") if err != nil { @@ -690,6 +771,18 @@ func (task *sBaremetalPrepareTask) sendStorageInfo(size int64) error { return err } +func (task *sBaremetalPrepareTask) sendIsolatedDevicesInfo( + session *mcclient.ClientSession, devs []*isolated_device.PCIDevice, +) error { + for i := 0; i < len(devs); i++ { + dev := isolated_device.NewGPUHPCDevice(devs[i]) + if err := dev.SyncDeviceInfo(session, task.baremetal.GetId()); err != nil { + return errors.Wrap(err, "sync device info") + } + } + return nil +} + func (task *sBaremetalPrepareTask) doNicWireProbe(cli *ssh.Client, nic *types.SNicDevInfo) error { maxTries := 6 for tried := 0; tried < maxTries; tried++ { diff --git a/pkg/baremetal/tasks/interface.go b/pkg/baremetal/tasks/interface.go index 0fd2e595cc..193ca6bb77 100644 --- a/pkg/baremetal/tasks/interface.go +++ b/pkg/baremetal/tasks/interface.go @@ -67,6 +67,7 @@ type IBaremetal interface { InitializeServer(session *mcclient.ClientSession, name string) error SaveSSHConfig(remoteAddr string, key string) error ServerLoadDesc() error + GetDHCPServerIP() (net.IP, error) } type IBmManager interface { diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 08fd0714d4..dca93d1a8c 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -1144,6 +1144,17 @@ func (self *SHost) GetSpec(statusCheck bool) *jsonutils.JSONDict { } specInfo.Manufacture = manufacture specInfo.Model = model + devices := IsolatedDeviceManager.FindByHost(self.Id) + if len(devices) > 0 { + specInfo.IsolatedDevices = make([]api.IsolatedDeviceSpec, len(devices)) + for i := 0; i < len(devices); i++ { + specInfo.IsolatedDevices[i].DevType = devices[i].DevType + specInfo.IsolatedDevices[i].Model = devices[i].Model + specInfo.IsolatedDevices[i].PciId = devices[i].VendorDeviceId + specInfo.IsolatedDevices[i].Vendor = devices[i].getVendor() + } + } + return specInfo.JSON(specInfo) } diff --git a/pkg/compute/models/isolated_devices.go b/pkg/compute/models/isolated_devices.go index 83cf44cd93..02ee4a0a52 100644 --- a/pkg/compute/models/isolated_devices.go +++ b/pkg/compute/models/isolated_devices.go @@ -224,6 +224,11 @@ func (manager *SIsolatedDeviceManager) ListItemFilter( q = q.In("vendor_device_id", query.VendorDeviceId) } + if !query.ShowBaremetalIsolatedDevices { + sq := HostManager.Query("id").Equals("host_type", api.HOST_TYPE_HYPERVISOR).SubQuery() + q = q.In("host_id", sq) + } + return q, nil } @@ -601,7 +606,7 @@ func (self *SIsolatedDevice) GetSpec(statusCheck bool) *jsonutils.JSONDict { return nil } host := self.getHost() - if host.Status != api.BAREMETAL_RUNNING || !host.GetEnabled() { + if host.Status != api.BAREMETAL_RUNNING || !host.GetEnabled() || host.HostType != api.HOST_TYPE_HYPERVISOR { return nil } } diff --git a/pkg/compute/tasks/baremetal_delete_task.go b/pkg/compute/tasks/baremetal_delete_task.go index f3d6c87dc2..db6002169c 100644 --- a/pkg/compute/tasks/baremetal_delete_task.go +++ b/pkg/compute/tasks/baremetal_delete_task.go @@ -53,12 +53,11 @@ func (self *BaremetalDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneM return } log.Errorln(err.Error()) - self.OnFailure(ctx, baremetal, nil) + self.OnFailure(ctx, baremetal, jsonutils.NewString(err.Error())) } } func (self *BaremetalDeleteTask) OnDeleteBaremetalComplete(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { - baremetal.RealDelete(ctx, self.UserCred) self.SetStageComplete(ctx, nil) } diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 7f2cedc721..ccd195e5f8 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -1413,7 +1413,7 @@ func (h *SHostInfo) onGetIsolatedDeviceSucc(objs []jsonutils.JSONObject) { func (h *SHostInfo) uploadIsolatedDevices() { for _, dev := range h.IsolatedDeviceMan.Devices { - if err := dev.SyncDeviceInfo(h); err != nil { + if err := dev.SyncDeviceInfo(h.GetSession(), h.HostId); err != nil { h.onFail(fmt.Sprintf("Sync device %s: %v", dev.String(), err)) } } diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 92a04eb5f7..e7b7595b73 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -87,7 +87,7 @@ type IDevice interface { GetIOMMUGroupDeviceCmd() string GetVGACmd() string GetCPUCmd() string - SyncDeviceInfo(IHost) error + SyncDeviceInfo(session *mcclient.ClientSession, hostId string) error } type IsolatedDeviceManager struct { @@ -115,7 +115,7 @@ func (man *IsolatedDeviceManager) fillPCIDevices() error { return nil } for idx, gpu := range gpus { - man.Devices = append(man.Devices, newGPUHPCDevice(gpu)) + man.Devices = append(man.Devices, NewGPUHPCDevice(gpu)) log.Infof("Add GPU device: %d => %#v", idx, gpu) } return nil @@ -225,18 +225,18 @@ func (dev *sBaseDevice) SetDetectedOnHost(probe bool) { dev.detectedOnHost = probe } -func (dev *sBaseDevice) SyncDeviceInfo(host IHost) error { +func (dev *sBaseDevice) SyncDeviceInfo(session *mcclient.ClientSession, hostId string) error { if len(dev.hostId) == 0 { - dev.hostId = host.GetHostId() + dev.hostId = hostId } data := dev.GetApiResourceData() if len(dev.GetCloudId()) != 0 { log.Infof("Update %s isolated_device: %s", dev.GetCloudId(), data.String()) - _, err := modules.IsolatedDevices.Update(host.GetSession(), dev.GetCloudId(), data) + _, err := modules.IsolatedDevices.Update(session, dev.GetCloudId(), data) return err } log.Infof("Create new isolated_device: %s", data.String()) - _, err := modules.IsolatedDevices.Create(host.GetSession(), data) + _, err := modules.IsolatedDevices.Create(session, data) return err } @@ -407,7 +407,7 @@ type sGPUHPCDevice struct { *sGPUBaseDevice } -func newGPUHPCDevice(dev *PCIDevice) *sGPUHPCDevice { +func NewGPUHPCDevice(dev *PCIDevice) *sGPUHPCDevice { gpuDev := &sGPUHPCDevice{ sGPUBaseDevice: newGPUBaseDevice(dev), } diff --git a/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go similarity index 83% rename from pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go rename to pkg/scheduler/algorithm/predicates/isolated_device_predicate.go index 97ead51bab..4a581a793f 100644 --- a/pkg/scheduler/algorithm/predicates/guest/isolated_device_predicate.go +++ b/pkg/scheduler/algorithm/predicates/isolated_device_predicate.go @@ -12,20 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package guest +package predicates import ( "fmt" - "yunion.io/x/onecloud/pkg/scheduler/algorithm/predicates" - "yunion.io/x/onecloud/pkg/scheduler/cache/candidate" "yunion.io/x/onecloud/pkg/scheduler/core" ) // IsolatedDevicePredicate check mode, and number of scheduled // device configurations and current resources. type IsolatedDevicePredicate struct { - predicates.BasePredicate + BasePredicate } func (f *IsolatedDevicePredicate) Name() string { @@ -45,14 +43,9 @@ func (f *IsolatedDevicePredicate) PreExecute(u *core.Unit, cs []core.Candidater) } func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool, []core.PredicateFailureReason, error) { - h := predicates.NewPredicateHelper(f, u, c) + h := NewPredicateHelper(f, u, c) reqIsoDevs := u.SchedData().IsolatedDevices - // TODO: use interface function - hc, ok := c.(*candidate.HostDesc) - if !ok { - return false, nil, fmt.Errorf("Candidater is not *candidate.HostDesc") - } - + getter := c.Getter() minCapacity := int64(0xFFFFFFFF) // check by specify device id @@ -60,7 +53,7 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool if len(dev.Id) == 0 { continue } - if fDev := hc.GetIsolatedDevice(dev.Id); fDev != nil { + if fDev := getter.GetIsolatedDevice(dev.Id); fDev != nil { if len(fDev.GuestID) != 0 { h.Exclude(fmt.Sprintf("IsolatedDevice %q already used by guest %q", dev.Id, fDev.GuestID)) return h.GetResult() @@ -73,8 +66,8 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool } reqCount := len(reqIsoDevs) - freeCount := len(hc.UnusedIsolatedDevices()) - hc.GetPendingUsage().IsolatedDevice - totalCount := len(hc.IsolatedDevices) + freeCount := len(getter.UnusedIsolatedDevices()) - getter.GetPendingUsage().IsolatedDevice + totalCount := len(getter.GetIsolatedDevices()) // check host isolated device count if freeCount < reqCount { @@ -93,7 +86,7 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool } } for devType, reqCount := range devTypeRequest { - freeCount := len(hc.UnusedIsolatedDevicesByType(devType)) + freeCount := len(getter.UnusedIsolatedDevicesByType(devType)) if freeCount < reqCount { h.Exclude(fmt.Sprintf("IsolatedDevice type %q not enough, request: %d, hostFree: %d", devType, reqCount, freeCount)) return h.GetResult() @@ -112,7 +105,7 @@ func (f *IsolatedDevicePredicate) Execute(u *core.Unit, c core.Candidater) (bool } } for vendorModel, reqCount := range devVendorModelRequest { - freeCount := len(hc.UnusedIsolatedDevicesByVendorModel(vendorModel)) + freeCount := len(getter.UnusedIsolatedDevicesByVendorModel(vendorModel)) if freeCount < reqCount { h.Exclude(fmt.Sprintf("IsolatedDevice vendor:model %q not enough, request: %d, hostFree: %d", vendorModel, reqCount, freeCount)) return h.GetResult() diff --git a/pkg/scheduler/algorithmprovider/baremetal.go b/pkg/scheduler/algorithmprovider/baremetal.go index f4088c5695..d729250265 100644 --- a/pkg/scheduler/algorithmprovider/baremetal.go +++ b/pkg/scheduler/algorithmprovider/baremetal.go @@ -40,5 +40,6 @@ func baremetalPredicates() sets.String { factory.RegisterFitPredicate("i-NetschedtagFilter", &predicates.NetworkSchedtagPredicate{}), factory.RegisterFitPredicate("k-NetBondingFilter", &predicatebm.NetBondingPredicate{}), factory.RegisterFitPredicate("l-CdromFilter", &predicatebm.CdromBootPredicate{}), + factory.RegisterFitPredicate("m-IsolatedDevicesFilter", &predicates.IsolatedDevicePredicate{}), ) } diff --git a/pkg/scheduler/algorithmprovider/defaults.go b/pkg/scheduler/algorithmprovider/defaults.go index c0111d00a8..57919b4403 100644 --- a/pkg/scheduler/algorithmprovider/defaults.go +++ b/pkg/scheduler/algorithmprovider/defaults.go @@ -40,7 +40,7 @@ func defaultPredicates() sets.String { factory.RegisterFitPredicate("h-GuestMemoryFilter", &predicateguest.MemoryPredicate{}), factory.RegisterFitPredicate("i-GuestStorageFilter", &predicateguest.StoragePredicate{}), factory.RegisterFitPredicate("j-GuestNetworkFilter", &predicates.NetworkPredicate{}), - factory.RegisterFitPredicate("k-GuestIsolatedDeviceFilter", &predicateguest.IsolatedDevicePredicate{}), + factory.RegisterFitPredicate("k-GuestIsolatedDeviceFilter", &predicates.IsolatedDevicePredicate{}), factory.RegisterFitPredicate("l-GuestResourceTypeFilter", &predicates.ResourceTypePredicate{}), factory.RegisterFitPredicate("m-GuestDiskschedtagFilter", &predicates.DiskSchedtagPredicate{}), factory.RegisterFitPredicate("n-ServerSkuFilter", &predicates.InstanceTypePredicate{}), diff --git a/pkg/scheduler/cache/candidate/baremetals.go b/pkg/scheduler/cache/candidate/baremetals.go index 974ed25c79..0789aae0a1 100644 --- a/pkg/scheduler/cache/candidate/baremetals.go +++ b/pkg/scheduler/cache/candidate/baremetals.go @@ -137,7 +137,9 @@ func (bb *BaremetalBuilder) init(ids []string) error { errMessageChannel := make(chan error, 2) defer close(errMessageChannel) - setFuncs := []func(){} + setFuncs := []func(){ + func() { bb.setIsolatedDevs(ids, errMessageChannel) }, + } for _, f := range setFuncs { wg.Wrap(f) @@ -198,7 +200,7 @@ func (bb *BaremetalBuilder) build() ([]interface{}, error) { } func (bb *BaremetalBuilder) buildOne(hostObj *computemodels.SHost) (interface{}, error) { - baseDesc, err := newBaseHostDesc(hostObj) + baseDesc, err := newBaseHostDesc(bb.baseBuilder, hostObj) if err != nil { return nil, err } diff --git a/pkg/scheduler/cache/candidate/base.go b/pkg/scheduler/cache/candidate/base.go index d5fa408279..bad028d718 100644 --- a/pkg/scheduler/cache/candidate/base.go +++ b/pkg/scheduler/cache/candidate/base.go @@ -16,6 +16,7 @@ package candidate import ( "fmt" + "strings" "yunion.io/x/jsonutils" "yunion.io/x/log" @@ -28,6 +29,7 @@ import ( "yunion.io/x/onecloud/pkg/cloudcommon/types" computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/api" + "yunion.io/x/onecloud/pkg/scheduler/core" schedmodels "yunion.io/x/onecloud/pkg/scheduler/models" ) @@ -43,6 +45,8 @@ type BaseHostDesc struct { NetInterfaces map[string][]computemodels.SNetInterface `json:"net_interfaces"` Storages []*api.CandidateStorage `json:"storages"` + IsolatedDevices []*core.IsolatedDeviceDesc `json:"isolated_devices"` + Tenants map[string]int64 `json:"tenants"` HostSchedtags []computemodels.SSchedtag `json:"schedtags"` @@ -217,6 +221,38 @@ func (b baseHostGetter) GetQuotaKeys(s *api.SchedInfo) computemodels.SComputeRes return b.h.getQuotaKeys(s) } +func (b baseHostGetter) GetPendingUsage() *schedmodels.SPendingUsage { + return b.h.GetPendingUsage() +} + +func (b baseHostGetter) UnusedIsolatedDevices() []*core.IsolatedDeviceDesc { + return b.h.UnusedIsolatedDevices() +} + +func (b baseHostGetter) UnusedIsolatedDevicesByType(devType string) []*core.IsolatedDeviceDesc { + return b.h.UnusedIsolatedDevicesByType(devType) +} + +func (b baseHostGetter) UnusedIsolatedDevicesByVendorModel(vendorModel string) []*core.IsolatedDeviceDesc { + return b.h.UnusedIsolatedDevicesByModel(vendorModel) +} + +func (b baseHostGetter) UnusedIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc { + return b.h.UnusedIsolatedDevicesByModel(model) +} + +func (b baseHostGetter) GetIsolatedDevice(devID string) *core.IsolatedDeviceDesc { + return b.h.GetIsolatedDevice(devID) +} + +func (b baseHostGetter) UnusedGpuDevices() []*core.IsolatedDeviceDesc { + return b.h.UnusedGpuDevices() +} + +func (b baseHostGetter) GetIsolatedDevices() []*core.IsolatedDeviceDesc { + return b.h.GetIsolatedDevices() +} + func reviseResourceType(resType string) string { if resType == "" { return computeapi.HostResourceTypeDefault @@ -224,7 +260,7 @@ func reviseResourceType(resType string) string { return resType } -func newBaseHostDesc(host *computemodels.SHost) (*BaseHostDesc, error) { +func newBaseHostDesc(b *baseBuilder, host *computemodels.SHost) (*BaseHostDesc, error) { host.ResourceType = reviseResourceType(host.ResourceType) desc := &BaseHostDesc{ SHost: host, @@ -269,6 +305,10 @@ func newBaseHostDesc(host *computemodels.SHost) (*BaseHostDesc, error) { return nil, fmt.Errorf("Fill ipmi info error: %v", err) } + if err := desc.fillIsolatedDevices(b, host); err != nil { + return nil, fmt.Errorf("Fill isolated devices error: %v", err) + } + desc.fillSharedDomains() desc.PendingUsage = desc.GetPendingUsage().ToMap() @@ -316,6 +356,94 @@ func (b BaseHostDesc) GetResourceType() string { return b.ResourceType } +func (h *BaseHostDesc) UnusedIsolatedDevices() []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + for _, dev := range h.IsolatedDevices { + if len(dev.GuestID) == 0 { + ret = append(ret, dev) + } + } + return ret +} + +func (h *BaseHostDesc) UnusedIsolatedDevicesByType(devType string) []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if dev.DevType == devType { + ret = append(ret, dev) + } + } + return ret +} + +func (h *BaseHostDesc) UnusedIsolatedDevicesByVendorModel(vendorModel string) []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + vm := core.NewVendorModelByStr(vendorModel) + for _, dev := range h.UnusedIsolatedDevices() { + if dev.GetVendorModel().IsMatch(vm) { + ret = append(ret, dev) + } + } + return ret +} + +func (h *BaseHostDesc) UnusedIsolatedDevicesByModel(model string) []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if strings.Contains(dev.Model, model) { + ret = append(ret, dev) + } + } + return ret +} + +func (h *BaseHostDesc) GetIsolatedDevice(devID string) *core.IsolatedDeviceDesc { + for _, dev := range h.IsolatedDevices { + if dev.ID == devID { + return dev + } + } + return nil +} + +func (h *BaseHostDesc) GetIsolatedDevices() []*core.IsolatedDeviceDesc { + return h.IsolatedDevices +} + +func (h *BaseHostDesc) UnusedGpuDevices() []*core.IsolatedDeviceDesc { + ret := make([]*core.IsolatedDeviceDesc, 0) + for _, dev := range h.UnusedIsolatedDevices() { + if strings.HasPrefix(dev.DevType, "GPU") { + ret = append(ret, dev) + } + } + return ret +} + +func (h *BaseHostDesc) fillIsolatedDevices(b *baseBuilder, host *computemodels.SHost) error { + allDevs := b.getIsolatedDevices(host.Id) + if len(allDevs) == 0 { + return nil + } + + devs := make([]*core.IsolatedDeviceDesc, len(allDevs)) + for index, devModel := range allDevs { + dev := &core.IsolatedDeviceDesc{ + ID: devModel.Id, + GuestID: devModel.GuestId, + HostID: devModel.HostId, + DevType: devModel.DevType, + Model: devModel.Model, + Addr: devModel.Addr, + VendorDeviceID: devModel.VendorDeviceId, + } + devs[index] = dev + } + h.IsolatedDevices = devs + + return nil +} + func (b *BaseHostDesc) fillCloudProvider(host *computemodels.SHost) error { b.Cloudprovider = host.GetCloudprovider() if b.Cloudprovider != nil { @@ -546,6 +674,8 @@ func HostResidentTenantCount(id string) (map[string]int64, error) { type DescBuilder struct { actor BuildActor + + isolatedDevicesDict map[string][]interface{} } func NewDescBuilder(act BuildActor) *DescBuilder { diff --git a/pkg/scheduler/cache/candidate/builder.go b/pkg/scheduler/cache/candidate/builder.go index 86c98a3da3..445f8df446 100644 --- a/pkg/scheduler/cache/candidate/builder.go +++ b/pkg/scheduler/cache/candidate/builder.go @@ -17,6 +17,7 @@ package candidate import ( "time" + "yunion.io/x/pkg/utils" "yunion.io/x/sqlchemy" computeapi "yunion.io/x/onecloud/pkg/apis/compute" @@ -26,6 +27,8 @@ import ( type baseBuilder struct { resourceType string + + isolatedDevicesDict map[string][]interface{} } func newBaseBuilder(resourceType string) *baseBuilder { @@ -38,6 +41,35 @@ func (b *baseBuilder) Type() string { return b.resourceType } +func (b *baseBuilder) getIsolatedDevices(hostID string) (devs []computemodels.SIsolatedDevice) { + devObjs, ok := b.isolatedDevicesDict[hostID] + devs = make([]computemodels.SIsolatedDevice, 0) + if !ok { + return + } + for _, obj := range devObjs { + dev := obj.(computemodels.SIsolatedDevice) + devs = append(devs, dev) + } + return +} + +func (b *baseBuilder) setIsolatedDevs(ids []string, errMessageChannel chan error) { + devs := computemodels.IsolatedDeviceManager.FindByHosts(ids) + dict, err := utils.GroupBy(devs, func(obj interface{}) (string, error) { + dev, ok := obj.(computemodels.SIsolatedDevice) + if !ok { + return "", utils.ConvertError(obj, "computemodels.SIsolatedDevice") + } + return dev.HostId, nil + }) + if err != nil { + errMessageChannel <- err + return + } + b.isolatedDevicesDict = dict +} + func FetchModelIds(q *sqlchemy.SQuery) ([]string, error) { rs, err := q.Rows() if err != nil { diff --git a/pkg/scheduler/cache/candidate/hosts.go b/pkg/scheduler/cache/candidate/hosts.go index 721dc8227e..b71aac170a 100644 --- a/pkg/scheduler/cache/candidate/hosts.go +++ b/pkg/scheduler/cache/candidate/hosts.go @@ -133,11 +133,10 @@ type HostDesc struct { RunningGuestCount int64 `json:"running_guest_count"` //Groups *GroupCounts `json:"groups"` - Metadata map[string]string `json:"metadata"` - IsolatedDevices []*IsolatedDeviceDesc `json:"isolated_devices"` - IsMaintenance bool `json:"is_maintenance"` - GuestReservedResource *ReservedResource `json:"guest_reserved_resource"` - GuestReservedResourceUsed *ReservedResource `json:"guest_reserved_used"` + Metadata map[string]string `json:"metadata"` + IsMaintenance bool `json:"is_maintenance"` + GuestReservedResource *ReservedResource `json:"guest_reserved_resource"` + GuestReservedResourceUsed *ReservedResource `json:"guest_reserved_used"` } type ReservedResource struct { @@ -243,7 +242,7 @@ type HostBuilder struct { //guestMetadatasDict map[string][]interface{} //diskStats []models.StorageCapacity - isolatedDevicesDict map[string][]interface{} + // isolatedDevicesDict map[string][]interface{} cpuIOLoads map[string]map[string]float64 @@ -415,66 +414,6 @@ func (h *HostDesc) IndexKey() string { return h.Id } -func (h *HostDesc) UnusedIsolatedDevices() []*IsolatedDeviceDesc { - ret := make([]*IsolatedDeviceDesc, 0) - for _, dev := range h.IsolatedDevices { - if len(dev.GuestID) == 0 { - ret = append(ret, dev) - } - } - return ret -} - -func (h *HostDesc) UnusedIsolatedDevicesByType(devType string) []*IsolatedDeviceDesc { - ret := make([]*IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { - if dev.DevType == devType { - ret = append(ret, dev) - } - } - return ret -} - -func (h *HostDesc) UnusedIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc { - ret := make([]*IsolatedDeviceDesc, 0) - vm := NewVendorModelByStr(vendorModel) - for _, dev := range h.UnusedIsolatedDevices() { - if dev.GetVendorModel().IsMatch(vm) { - ret = append(ret, dev) - } - } - return ret -} - -func (h *HostDesc) UnusedIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc { - ret := make([]*IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { - if strings.Contains(dev.Model, model) { - ret = append(ret, dev) - } - } - return ret -} - -func (h *HostDesc) GetIsolatedDevice(devID string) *IsolatedDeviceDesc { - for _, dev := range h.IsolatedDevices { - if dev.ID == devID { - return dev - } - } - return nil -} - -func (h *HostDesc) UnusedGpuDevices() []*IsolatedDeviceDesc { - ret := make([]*IsolatedDeviceDesc, 0) - for _, dev := range h.UnusedIsolatedDevices() { - if strings.HasPrefix(dev.DevType, "GPU") { - ret = append(ret, dev) - } - } - return ret -} - type WaitGroupWrapper struct { gosync.WaitGroup } @@ -767,22 +706,6 @@ func (b *HostBuilder) setGuests(ids []string, errMessageChannel chan error) { //return //} -func (b *HostBuilder) setIsolatedDevs(ids []string, errMessageChannel chan error) { - devs := computemodels.IsolatedDeviceManager.FindByHosts(ids) - dict, err := utils.GroupBy(devs, func(obj interface{}) (string, error) { - dev, ok := obj.(computemodels.SIsolatedDevice) - if !ok { - return "", utils.ConvertError(obj, "computemodels.SIsolatedDevice") - } - return dev.HostId, nil - }) - if err != nil { - errMessageChannel <- err - return - } - b.isolatedDevicesDict = dict -} - /*func (b *HostBuilder) setDiskStats(errMessageChannel chan error) { storageIDs := make([]string, len(b.storages)) func() { @@ -865,7 +788,7 @@ func (b *HostBuilder) build() ([]interface{}, error) { } func (b *HostBuilder) buildOne(host *computemodels.SHost) (interface{}, error) { - baseDesc, err := newBaseHostDesc(host) + baseDesc, err := newBaseHostDesc(b.baseBuilder, host) if err != nil { return nil, err } @@ -889,7 +812,6 @@ func (b *HostBuilder) buildOne(host *computemodels.SHost) (interface{}, error) { b.fillGuestsResourceInfo, //b.fillResidentGroups, b.fillMetadata, - b.fillIsolatedDevices, b.fillCPUIOLoads, } @@ -1079,79 +1001,6 @@ func (b *HostBuilder) fillMetadata(desc *HostDesc, host *computemodels.SHost) er return nil } -type IsolatedDeviceDesc struct { - ID string - GuestID string - HostID string - DevType string - Model string - Addr string - VendorDeviceID string -} - -func (i *IsolatedDeviceDesc) VendorID() string { - return strings.Split(i.VendorDeviceID, ":")[0] -} - -type VendorModel struct { - Vendor string - Model string -} - -func NewVendorModelByStr(desc string) *VendorModel { - vm := new(VendorModel) - // desc format is ':' - parts := strings.Split(desc, ":") - if len(parts) == 1 { - vm.Model = parts[0] - } else if len(parts) == 2 { - vm.Vendor = parts[0] - vm.Model = parts[1] - } - return vm -} - -func (vm *VendorModel) IsMatch(target *VendorModel) bool { - if vm.Model == "" || target.Model == "" { - return false - } - vendorMatch := false - modelMatch := false - if target.Vendor != "" { - if vm.Vendor == target.Vendor { - vendorMatch = true - } else if computeapi.ID_VENDOR_MAP[vm.Vendor] == target.Vendor { - vendorMatch = true - } - } else { - vendorMatch = true - } - if vm.Model == target.Model { - modelMatch = true - } - return vendorMatch && modelMatch -} - -func (i *IsolatedDeviceDesc) GetVendorModel() *VendorModel { - return &VendorModel{ - Vendor: i.VendorID(), - Model: i.Model, - } -} - -func (b *HostBuilder) getIsolatedDevices(hostID string) (devs []computemodels.SIsolatedDevice) { - devObjs, ok := b.isolatedDevicesDict[hostID] - devs = make([]computemodels.SIsolatedDevice, 0) - if !ok { - return - } - for _, obj := range devObjs { - dev := obj.(computemodels.SIsolatedDevice) - devs = append(devs, dev) - } - return -} - func (b *HostBuilder) getUsedIsolatedDevices(hostID string) (devs []computemodels.SIsolatedDevice) { devs = make([]computemodels.SIsolatedDevice, 0) for _, dev := range b.getIsolatedDevices(hostID) { @@ -1193,31 +1042,6 @@ func (b *HostBuilder) getUnusedIsolatedDevices(hostID string) (devs []computemod return } -func (b *HostBuilder) fillIsolatedDevices(desc *HostDesc, host *computemodels.SHost) error { - - allDevs := b.getIsolatedDevices(host.Id) - if len(allDevs) == 0 { - return nil - } - - devs := make([]*IsolatedDeviceDesc, len(allDevs)) - for index, devModel := range allDevs { - dev := &IsolatedDeviceDesc{ - ID: devModel.Id, - GuestID: devModel.GuestId, - HostID: devModel.HostId, - DevType: devModel.DevType, - Model: devModel.Model, - Addr: devModel.Addr, - VendorDeviceID: devModel.VendorDeviceId, - } - devs[index] = dev - } - desc.IsolatedDevices = devs - - return nil -} - func (b *HostBuilder) fillCPUIOLoads(desc *HostDesc, host *computemodels.SHost) error { desc.CPULoad = b.loadByName(host.Id, "cpu_load") desc.IOLoad = b.loadByName(host.Id, "io_load") diff --git a/pkg/scheduler/core/types.go b/pkg/scheduler/core/types.go index 96c24809d1..71e71d76f1 100644 --- a/pkg/scheduler/core/types.go +++ b/pkg/scheduler/core/types.go @@ -15,14 +15,18 @@ package core import ( + "strings" + "yunion.io/x/jsonutils" + computeapi "yunion.io/x/onecloud/pkg/apis/compute" schedapi "yunion.io/x/onecloud/pkg/apis/scheduler" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/compute/baremetal" computemodels "yunion.io/x/onecloud/pkg/compute/models" "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core/score" + schedmodels "yunion.io/x/onecloud/pkg/scheduler/models" ) const ( @@ -97,6 +101,17 @@ type CandidatePropertyGetter interface { GetIpmiInfo() types.SIPMIInfo GetQuotaKeys(s *api.SchedInfo) computemodels.SComputeResourceKeys + + GetPendingUsage() *schedmodels.SPendingUsage + + // isloatedDevices + UnusedIsolatedDevices() []*IsolatedDeviceDesc + UnusedIsolatedDevicesByType(devType string) []*IsolatedDeviceDesc + UnusedIsolatedDevicesByVendorModel(vendorModel string) []*IsolatedDeviceDesc + UnusedIsolatedDevicesByModel(model string) []*IsolatedDeviceDesc + GetIsolatedDevice(devID string) *IsolatedDeviceDesc + UnusedGpuDevices() []*IsolatedDeviceDesc + GetIsolatedDevices() []*IsolatedDeviceDesc } // Candidater replace host Candidate resource info @@ -207,3 +222,63 @@ func NewAllocatedResource() *AllocatedResource { Nets: make([]*schedapi.CandidateNet, 0), } } + +type IsolatedDeviceDesc struct { + ID string + GuestID string + HostID string + DevType string + Model string + Addr string + VendorDeviceID string +} + +func (i *IsolatedDeviceDesc) VendorID() string { + return strings.Split(i.VendorDeviceID, ":")[0] +} + +func (i *IsolatedDeviceDesc) GetVendorModel() *VendorModel { + return &VendorModel{ + Vendor: i.VendorID(), + Model: i.Model, + } +} + +type VendorModel struct { + Vendor string + Model string +} + +func NewVendorModelByStr(desc string) *VendorModel { + vm := new(VendorModel) + // desc format is ':' + parts := strings.Split(desc, ":") + if len(parts) == 1 { + vm.Model = parts[0] + } else if len(parts) == 2 { + vm.Vendor = parts[0] + vm.Model = parts[1] + } + return vm +} + +func (vm *VendorModel) IsMatch(target *VendorModel) bool { + if vm.Model == "" || target.Model == "" { + return false + } + vendorMatch := false + modelMatch := false + if target.Vendor != "" { + if vm.Vendor == target.Vendor { + vendorMatch = true + } else if computeapi.ID_VENDOR_MAP[vm.Vendor] == target.Vendor { + vendorMatch = true + } + } else { + vendorMatch = true + } + if vm.Model == target.Model { + modelMatch = true + } + return vendorMatch && modelMatch +}