From 53570fd78a6e6ebd78442e495c51241596170355 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Wed, 10 Apr 2019 11:45:10 +0800 Subject: [PATCH] fix: 1. baremetal always boot from pxe 2. baremetal prepare retry (#307) multiple local ipmi address --- pkg/baremetal/manager.go | 23 +++- pkg/baremetal/pxe/dhcp.go | 12 +- pkg/baremetal/tasks/base.go | 20 +-- pkg/baremetal/tasks/basedeploy.go | 2 +- pkg/baremetal/tasks/baseprepare.go | 122 +++++++++++------- pkg/baremetal/tasks/interface.go | 2 +- pkg/baremetal/tasks/start.go | 4 +- pkg/baremetal/tasks/unmaintenance.go | 2 +- pkg/baremetal/utils/disktool/disktool.go | 3 +- .../isolated_device/isolated_device.go | 4 +- pkg/util/tftp/tftp.go | 2 +- 11 files changed, 117 insertions(+), 79 deletions(-) diff --git a/pkg/baremetal/manager.go b/pkg/baremetal/manager.go index 3ac44245c8..0003968810 100644 --- a/pkg/baremetal/manager.go +++ b/pkg/baremetal/manager.go @@ -691,15 +691,24 @@ func (b *SBaremetalInstance) GetNotifyUrl() string { } func (b *SBaremetalInstance) GetTFTPResponse() string { - return fmt.Sprintf( - `default start + resp := `default start serial 1 115200 + label start menu label ^Start menu default - kernel kernel - append initrd=initramfs token=%s url=%s`, - auth.GetTokenString(), b.GetNotifyUrl()) +` + + if b.NeedPXEBoot() { + resp += " kernel kernel\n" + resp += fmt.Sprintf(" append initrd=initramfs token=%s url=%s", + auth.GetTokenString(), b.GetNotifyUrl()) + } else { + resp += " COM32 chain.c32\n" + resp += " APPEND hd0 0\n" + } + log.Infof("%s", resp) + return resp } func (b *SBaremetalInstance) GetTaskQueue() *tasks.TaskQueue { @@ -934,6 +943,7 @@ func (b *SBaremetalInstance) DoPXEBoot() error { return fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId()) } +/* func (b *SBaremetalInstance) DoDiskBoot() error { log.Infof("Do DISK Boot ........., wait") b.ClearSSHConfig() @@ -943,6 +953,7 @@ func (b *SBaremetalInstance) DoDiskBoot() error { } return fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId()) } +*/ func (b *SBaremetalInstance) GetPowerStatus() (string, error) { ipmiCli := b.GetIPMITool() @@ -1019,7 +1030,7 @@ func (b *SBaremetalInstance) StartBaremetalResetBMCTask(userCred mcclient.TokenC } func (b *SBaremetalInstance) DelayedServerReset(_ jsonutils.JSONObject) (jsonutils.JSONObject, error) { - err := b.DoDiskBoot() + err := b.DoPXEBoot() return nil, err } diff --git a/pkg/baremetal/pxe/dhcp.go b/pkg/baremetal/pxe/dhcp.go index 2c515ab2f5..95190d5a3b 100644 --- a/pkg/baremetal/pxe/dhcp.go +++ b/pkg/baremetal/pxe/dhcp.go @@ -167,12 +167,14 @@ func (req *dhcpRequest) fetchConfig(session *mcclient.ClientSession) (*dhcp.Resp if err != nil { return nil, err } - if req.baremetalInstance.NeedPXEBoot() { - return req.baremetalInstance.GetPXEDHCPConfig(req.ClientArch) - } + // always response PXE request + // let bootloader decide boot local or remote + // if req.baremetalInstance.NeedPXEBoot() { + return req.baremetalInstance.GetPXEDHCPConfig(req.ClientArch) + // } // ignore - log.Warningf("No need to pxeboot, ignore the request ...(mac:%s guid:%s)", req.ClientMac, req.ClientGuid) - return nil, nil + // log.Warningf("No need to pxeboot, ignore the request ...(mac:%s guid:%s)", req.ClientMac, req.ClientGuid) + // return nil, nil } else { // handle normal DHCP request bmInstance := req.baremetalManager.GetBaremetalByMac(req.ClientMac) diff --git a/pkg/baremetal/tasks/base.go b/pkg/baremetal/tasks/base.go index 4962cda31d..e209098d93 100644 --- a/pkg/baremetal/tasks/base.go +++ b/pkg/baremetal/tasks/base.go @@ -283,25 +283,15 @@ func (self *SBaremetalTaskBase) EnsurePowerShutdown(soft bool) error { return nil } -func (self *SBaremetalTaskBase) EnsurePowerUp(bootdev string) error { - log.Infof("EnsurePowerUp: bootdev=%s", bootdev) - var bootFunc func() error = nil - switch bootdev { - case "pxe": - bootFunc = self.Baremetal.DoPXEBoot - case "disk": - bootFunc = self.Baremetal.DoDiskBoot - } - if bootFunc == nil { - return fmt.Errorf("No boot func %s found", bootdev) - } +func (self *SBaremetalTaskBase) EnsurePowerUp() error { + log.Infof("EnsurePowerUp: bootdev=pxe") status, err := self.Baremetal.GetPowerStatus() if err != nil { return err } for status == "" || status == types.POWER_STATUS_OFF { if status == types.POWER_STATUS_OFF { - err = bootFunc() + err = self.Baremetal.DoPXEBoot() if err != nil { return err } @@ -376,7 +366,7 @@ func (self *SBaremetalPXEBootTaskBase) InitPXEBootTask(pxeBootTask IPXEBootTask, if err := self.EnsurePowerShutdown(false); err != nil { return self, fmt.Errorf("EnsurePowerShutdown: %v", err) } - if err := self.EnsurePowerUp("pxe"); err != nil { + if err := self.EnsurePowerUp(); err != nil { return self, fmt.Errorf("EnsurePowerUp to pxe: %v", err) } // this stage will be called by baremetalInstance when pxe start notify @@ -406,7 +396,7 @@ func (self *SBaremetalPXEBootTaskBase) WaitForShutdown(ctx context.Context, args } func (self *SBaremetalPXEBootTaskBase) OnStopComplete(ctx context.Context, args interface{}) error { - err := self.EnsurePowerUp("pxe") + err := self.EnsurePowerUp() if err != nil { return err } diff --git a/pkg/baremetal/tasks/basedeploy.go b/pkg/baremetal/tasks/basedeploy.go index d7215c44dd..d0957c7488 100644 --- a/pkg/baremetal/tasks/basedeploy.go +++ b/pkg/baremetal/tasks/basedeploy.go @@ -80,7 +80,7 @@ func (self *SBaremetalServerBaseDeployTask) OnPXEBoot(ctx context.Context, term return err } if onFinishAction == "restart" { - err = self.EnsurePowerUp("disk") + err = self.EnsurePowerUp() if err != nil { return err } diff --git a/pkg/baremetal/tasks/baseprepare.go b/pkg/baremetal/tasks/baseprepare.go index fbd34c294a..369990d0e2 100644 --- a/pkg/baremetal/tasks/baseprepare.go +++ b/pkg/baremetal/tasks/baseprepare.go @@ -122,6 +122,7 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error { log.Errorf("Lan channel %d MAC address is empty", lanChannel) continue } + ipmiNic := &types.SNicDevInfo{ Mac: conf.Mac, Up: false, @@ -132,10 +133,12 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error { rootId := ipmitool.GetRootId(ipmiSysInfo) err = ipmitool.SetLanUserPasswd(sshIPMI, lanChannel, rootId, ipmiUser, ipmiPasswd) if err != nil { + // ignore the error log.Errorf("Lan channel %d set user password error: %v", lanChannel, err) } err = ipmitool.EnableLanAccess(sshIPMI, lanChannel) if err != nil { + // ignore the error log.Errorf("Lan channel %d enable lan access error: %v", lanChannel, err) } @@ -148,55 +151,20 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error { } if len(tryAddrs) > 0 && !o.Options.ForceDhcpProbeIpmi { for _, tryAddr := range tryAddrs { - log.Infof("IP addr found in IPMI config, try use %s as IPMI address", tryAddr) - ipConf, err := task.getIPMIIPConfig(tryAddr) - if err != nil { - log.Errorf("Failed to get IPMI ipconfig for %s", tryAddr) - continue - } - err = ipmitool.SetLanStatic(sshIPMI, lanChannel, ipConf.IPAddr, ipConf.Netmask, ipConf.Gateway) - if err != nil { - log.Errorf("Failed to set IPMI static net config %#v for %s", *ipConf, tryAddr) - continue - } - time.Sleep(1 * time.Second) - conf, err = ipmitool.GetLanConfig(sshIPMI, lanChannel) - if err != nil { - log.Errorf("Failed to get lan config after set static network: %v", err) - continue - } - log.Infof("Get lan config %#v", *conf) - if conf.IPAddr == "" || conf.IPAddr != tryAddr { - log.Errorf("Failed to set ipmi lan channel %d static ipaddr", lanChannel) - continue - } - rmcpIPMI := ipmitool.NewLanPlusIPMI(tryAddr, ipmiUser, ipmiPasswd) - conf2, err := ipmitool.GetLanConfig(rmcpIPMI, lanChannel) - if err != nil { - log.Errorf("Failed to get lan channel %d config use RMCP mode: %v", lanChannel, err) - continue - } - if len(conf2.Mac) != 0 && - conf2.Mac.String() == conf.Mac.String() && - conf2.IPAddr != "" && conf2.IPAddr == tryAddr { - // make sure the ipaddr is a IPMI address - // enable the netif - if err := task.sendNicInfo(ipmiNic, -1, types.NIC_TYPE_IPMI, false, tryAddr); err != nil { - log.Errorf("Fail to set existing BMC IP address to %s", tryAddr) - } else { - ipmiInfo.IpAddr = tryAddr - ipmiLanChannel = lanChannel - break - } - } else { - log.Errorf("Use RMCP mode get invalid lan config: %#v", conf2) - } - if ipmiLanChannel >= 0 { - // found and set config on lanChannel + tryResult := task.tryLocalIpmiAddr(sshIPMI, ipmiNic, lanChannel, + ipmiUser, ipmiPasswd, tryAddr) + if tryResult { + ipmiInfo.IpAddr = tryAddr + ipmiLanChannel = lanChannel break } } + if ipmiLanChannel >= 0 { + // found and set config on lanChannel + break + } } + if len(tryAddrs) > 0 { task.baremetal.SetExistingIPMIIPAddr(tryAddrs[0]) } @@ -325,6 +293,70 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error { return nil } +func (task *sBaremetalPrepareTask) tryLocalIpmiAddr(sshIPMI *ipmitool.SSHIPMI, ipmiNic *types.SNicDevInfo, lanChannel int, ipmiUser, ipmiPasswd, tryAddr string) bool { + log.Infof("IP addr found in IPMI config, try use %s as IPMI address", tryAddr) + ipConf, err := task.getIPMIIPConfig(tryAddr) + if err != nil { + log.Errorf("Failed to get IPMI ipconfig for %s", tryAddr) + return false + } + err = ipmitool.SetLanStatic(sshIPMI, lanChannel, ipConf.IPAddr, ipConf.Netmask, ipConf.Gateway) + if err != nil { + log.Errorf("Failed to set IPMI static net config %#v for %s", *ipConf, tryAddr) + return false + } + + var conf *types.SIPMILanConfig + + tried := 0 + maxTries := 5 + time.Sleep(2 * time.Second) + for tried = 0; tried < maxTries; tried += 1 { + conf, err = ipmitool.GetLanConfig(sshIPMI, lanChannel) + if err != nil { + log.Errorf("Failed to get lan config after set static network: %v", err) + continue + } + log.Infof("Get lan config %#v", *conf) + if conf.IPAddr == "" || conf.IPAddr != tryAddr { + log.Errorf("Failed to set ipmi lan channel %d static ipaddr", lanChannel) + continue + } + break + } + if tried >= maxTries { + log.Errorf("Failed to get lan config after %s tries", tried) + return false + } + rmcpIPMI := ipmitool.NewLanPlusIPMI(tryAddr, ipmiUser, ipmiPasswd) + for tried = 0; tried < maxTries; tried += 1 { + conf2, err := ipmitool.GetLanConfig(rmcpIPMI, lanChannel) + if err != nil { + log.Errorf("Failed to get lan channel %d config use RMCP mode: %v", lanChannel, err) + continue + } + if len(conf2.Mac) != 0 && + conf2.Mac.String() == conf.Mac.String() && + conf2.IPAddr != "" && conf2.IPAddr == tryAddr { + break + } else { + log.Errorf("fail to rmcp get IPMI ip config %v", conf2) + time.Sleep(5 * time.Second) + } + } + if tried < maxTries { + // make sure the ipaddr is a IPMI address + // enable the netif + err := task.sendNicInfo(ipmiNic, -1, types.NIC_TYPE_IPMI, false, tryAddr) + if err != nil { + log.Errorf("Fail to set existing BMC IP address to %s", tryAddr) + } else { + return true + } + } + return false +} + func (task *sBaremetalPrepareTask) getIPMIUserPasswd(sysInfo *types.SIPMISystemInfo) (string, string, string) { var ( ipmiUser string diff --git a/pkg/baremetal/tasks/interface.go b/pkg/baremetal/tasks/interface.go index b1cde7abde..548c7f222f 100644 --- a/pkg/baremetal/tasks/interface.go +++ b/pkg/baremetal/tasks/interface.go @@ -48,7 +48,7 @@ type IBaremetal interface { GetPowerStatus() (string, error) DoPowerShutdown(soft bool) error DoPXEBoot() error - DoDiskBoot() error + // DoDiskBoot() error RemoveServer() } diff --git a/pkg/baremetal/tasks/start.go b/pkg/baremetal/tasks/start.go index db14590f4f..aaade7dad7 100644 --- a/pkg/baremetal/tasks/start.go +++ b/pkg/baremetal/tasks/start.go @@ -37,8 +37,8 @@ func NewBaremetalServerStartTask( self := &SBaremetalServerStartTask{ SBaremetalTaskBase: baseTask, } - if err := self.Baremetal.DoDiskBoot(); err != nil { - return nil, fmt.Errorf("DoDiskBoot: %v", err) + if err := self.Baremetal.DoPXEBoot(); err != nil { + return nil, fmt.Errorf("DoPXEBoot: %v", err) } self.SetStage(self.WaitForStart) ExecuteTask(self, nil) diff --git a/pkg/baremetal/tasks/unmaintenance.go b/pkg/baremetal/tasks/unmaintenance.go index c925491fc4..1631c3295f 100644 --- a/pkg/baremetal/tasks/unmaintenance.go +++ b/pkg/baremetal/tasks/unmaintenance.go @@ -42,7 +42,7 @@ func NewBaremetalUnmaintenanceTask( if err != nil { return task, fmt.Errorf("EnsurePowerShutdown hard: %v", err) } - err = task.EnsurePowerUp("disk") + err = task.EnsurePowerUp() if err != nil { return task, fmt.Errorf("EnsurePowerUp disk: %v", err) } diff --git a/pkg/baremetal/utils/disktool/disktool.go b/pkg/baremetal/utils/disktool/disktool.go index 82079eee54..08b806f31f 100644 --- a/pkg/baremetal/utils/disktool/disktool.go +++ b/pkg/baremetal/utils/disktool/disktool.go @@ -104,7 +104,8 @@ func (p *Partition) Format(fs string, uuid string) error { cmd = []string{"/usr/sbin/mkfs.ext3"} cmdUUID = []string{"/usr/sbin/tune2fs", "-U", uuid} case "ext4": - cmd = []string{"/usr/sbin/mkfs.ext4", "-O", "^64bit", "-E", "lazy_itable_init=1"} + // for baremetal, force 64bit support large disks + cmd = []string{"/usr/sbin/mkfs.ext4", "-O", "64bit", "-E", "lazy_itable_init=1"} cmdUUID = []string{"/usr/sbin/tune2fs", "-U", uuid} case "ext4dev": cmd = []string{"/usr/sbin/mkfs.ext4dev", "-E", "lazy_itable_init=1"} diff --git a/pkg/hostman/isolated_device/isolated_device.go b/pkg/hostman/isolated_device/isolated_device.go index 291eb85d69..1cf330d52c 100644 --- a/pkg/hostman/isolated_device/isolated_device.go +++ b/pkg/hostman/isolated_device/isolated_device.go @@ -113,7 +113,9 @@ func (man *IsolatedDeviceManager) fillPCIDevices() error { // only support gpu by now gpus, err := getPassthroughGPUS() if err != nil { - return fmt.Errorf("getPassthroughGPUS: %v", err) + // ignore getPassthroughGPUS error on old machines without VGA devices + log.Errorf("getPassthroughGPUS: %v", err) + return nil } for idx, gpu := range gpus { man.Devices = append(man.Devices, newGPUHPCDevice(gpu)) diff --git a/pkg/util/tftp/tftp.go b/pkg/util/tftp/tftp.go index 61bef301c2..625604a9f9 100644 --- a/pkg/util/tftp/tftp.go +++ b/pkg/util/tftp/tftp.go @@ -117,7 +117,7 @@ func (s *Server) Serve(l net.PacketConn) error { if err := l.SetDeadline(time.Time{}); err != nil { return err } - buf := make([]byte, 512) + buf := make([]byte, 2048) for { n, addr, err := l.ReadFrom(buf) if err != nil {