From 6c5bba7dd2b377898f38831d7f9c343ffb3264b1 Mon Sep 17 00:00:00 2001 From: Zexi Li Date: Tue, 25 May 2021 13:41:22 +0800 Subject: [PATCH] feat(baremetal): support management of none BMC host --- pkg/baremetal/cronjobs.go | 2 +- pkg/baremetal/handler/handlers.go | 10 +- pkg/baremetal/manager.go | 137 +++++++++++++++++- pkg/baremetal/tasks/base.go | 65 ++++++--- pkg/baremetal/tasks/basedeploy.go | 45 ++++-- pkg/baremetal/tasks/baseprepare.go | 2 +- pkg/baremetal/tasks/create.go | 5 +- pkg/baremetal/tasks/interface.go | 5 + pkg/baremetal/tasks/rebuild.go | 5 +- pkg/baremetal/tasks/reprepare.go | 5 + pkg/baremetal/tasks/start.go | 24 +-- pkg/baremetal/tasks/stop.go | 13 +- pkg/baremetal/tasks/unmaintenance.go | 29 ++-- pkg/baremetal/types/types.go | 1 + pkg/cloudcommon/types/types.go | 6 + pkg/compute/guestdrivers/baremetals.go | 2 +- pkg/compute/models/hosts.go | 8 + .../tasks/baremetal_maintenance_task.go | 3 +- .../baremetal_server_sync_status_task.go | 5 +- .../tasks/baremetal_sync_status_task.go | 8 + .../tasks/baremetal_unmaintenance_task.go | 5 +- pkg/util/ssh/ssh.go | 33 ++++- 22 files changed, 352 insertions(+), 66 deletions(-) diff --git a/pkg/baremetal/cronjobs.go b/pkg/baremetal/cronjobs.go index eab92d7675..8e5342febf 100644 --- a/pkg/baremetal/cronjobs.go +++ b/pkg/baremetal/cronjobs.go @@ -96,7 +96,7 @@ func (job *SStatusProbeJob) Do(ctx context.Context, now time.Time) error { if bStatus == api.BAREMETAL_READY || bStatus == api.BAREMETAL_RUNNING || bStatus == api.BAREMETAL_UNKNOWN { ps, err := job.baremetal.GetPowerStatus() if err != nil { - return errors.Wrap(err, "GetPowerStatus") + return errors.Wrap(err, "StatusProbeJob get power status") } job.lastTime = now pps := PowerStatusToBaremetalStatus(ps) diff --git a/pkg/baremetal/handler/handlers.go b/pkg/baremetal/handler/handlers.go index f4d1795796..42eed723ec 100644 --- a/pkg/baremetal/handler/handlers.go +++ b/pkg/baremetal/handler/handlers.go @@ -212,9 +212,13 @@ func handleServerStart(ctx *Context, bm *baremetal.SBaremetalInstance, _ baremet } func handleServerStop(ctx *Context, bm *baremetal.SBaremetalInstance, _ baremetaltypes.IBaremetalServer) { - if err := bm.StartServerStopTask(ctx.UserCred(), ctx.TaskId(), ctx.Data()); err != nil { - ctx.ResponseError(httperrors.NewGeneralError(err)) - return + if bm.HasBMC() { + if err := bm.StartServerStopTask(ctx.UserCred(), ctx.TaskId(), ctx.Data()); err != nil { + ctx.ResponseError(httperrors.NewGeneralError(err)) + return + } + } else { + bm.StartBaremetalMaintenanceTask(ctx.UserCred(), ctx.TaskId(), ctx.Data()) } ctx.ResponseOk() } diff --git a/pkg/baremetal/manager.go b/pkg/baremetal/manager.go index 4d9c24e024..77f2b9eec4 100644 --- a/pkg/baremetal/manager.go +++ b/pkg/baremetal/manager.go @@ -927,13 +927,18 @@ func (b *SBaremetalInstance) NeedPXEBoot() bool { taskNeedPXEBoot = true } ret := false - if taskNeedPXEBoot || (task == nil && len(serverId) == 0 && b.GetHostType() == "baremetal") { + if taskNeedPXEBoot || (task == nil && len(serverId) == 0 && b.GetHostType() == "baremetal") || (task == nil && b.IsMaintenance()) { ret = true } log.Infof("Check task %s, server %s NeedPXEBoot: %v", taskName, serverId, ret) return ret } +func (b *SBaremetalInstance) IsMaintenance() bool { + isMt, _ := b.desc.Bool("is_maintenance") + return isMt +} + func (b *SBaremetalInstance) GetHostType() string { hostType, _ := b.desc.GetString("host_type") return hostType @@ -1347,6 +1352,112 @@ func (b *SBaremetalInstance) SetExistingIPMIIPAddr(ipAddr string) { b.desc.Set("ipmi_info", info) } +func (b *SBaremetalInstance) HasBMC() bool { + conf := b.GetIPMIConfig() + if conf == nil { + return false + } + return true +} + +func (b *SBaremetalInstance) GetHostSSHClient() (*ssh.Client, error) { + conf, err := b.GetSSHConfig() + if err != nil { + return nil, errors.Wrap(err, "Get host ssh config") + } + if conf == nil { + return nil, errors.Errorf("Host ssh config is empty") + } + sshCli, err := ssh.NewClient(conf.RemoteIP, 22, "root", conf.Password, "") + if err != nil { + return nil, errors.Wrap(err, "New ssh client") + } + return sshCli, nil +} + +func (b *SBaremetalInstance) GetServerSSHClient() (*ssh.Client, error) { + s := b.GetServer() + if s == nil { + return nil, errors.Error("No server") + } + + privateKey, err := modules.Sshkeypairs.FetchPrivateKey(context.TODO(), auth.AdminCredential()) + if err != nil { + return nil, errors.Wrapf(err, "Get server %s login info", s.GetId()) + } + nics := s.GetNics() + var errs []error + for _, nic := range nics { + if nic.LinkUp && nic.Ip != "" { + for _, user := range []string{"cloudroot", "root"} { + sshCli, err := ssh.NewClient(nic.Ip, 22, user, "", privateKey) + if err != nil { + err = errors.Wrapf(err, "New server %s ssh client %s@%s", s.GetName(), user, nic.Ip) + errs = append(errs, err) + } else { + return sshCli, nil + } + } + } + } + + return nil, errors.NewAggregate(errs) +} + +func (b *SBaremetalInstance) SSHReachable() (bool, error) { + var errs []error + if _, err := b.GetHostSSHClient(); err != nil { + errs = append(errs, err) + } else { + // host ssh reachable + return true, nil + } + if _, err := b.GetServerSSHClient(); err != nil { + errs = append(errs, err) + } else { + // server ssh reachable + return true, nil + } + return false, errors.NewAggregate(errs) +} + +func (b *SBaremetalInstance) sshRun(hostCmd string, serverCmd string) ([]string, error) { + hostCli, err := b.GetHostSSHClient() + if err != nil { + log.Warningf("Get host ssh client error: %v", err) + } else { + if hostCli != nil { + return hostCli.RawRun(hostCmd) + } + } + + serverCli, err := b.GetServerSSHClient() + if err != nil { + return nil, errors.Wrapf(err, "Get baremetal %s server ssh client", b.GetName()) + } + return serverCli.RunWithTTY(serverCmd) +} + +func (b *SBaremetalInstance) SSHReboot() error { + if _, err := b.sshRun("/sbin/reboot", "sudo shutdown -r now && exit"); err != nil { + if !ssh.IsExitMissingError(err) { + return errors.Wrap(err, "Try reboot") + } + } + b.ClearSSHConfig() + return nil +} + +func (b *SBaremetalInstance) SSHShutdown() error { + if _, err := b.sshRun("/sbin/poweroff", "sudo shutdown -h now && exit"); err != nil { + if ssh.IsExitMissingError(err) { + return nil + } + return errors.Wrap(err, "Try poweroff") + } + return nil +} + func (b *SBaremetalInstance) GetIPMITool() *ipmitool.LanPlusIPMI { conf := b.GetIPMIConfig() if conf == nil { @@ -1429,9 +1540,31 @@ func (b *SBaremetalInstance) DoDiskBoot() error { */ func (b *SBaremetalInstance) GetPowerStatus() (string, error) { + status, err := b.getPowerStatus() + if err != nil { + if errors.Cause(err) != types.ErrIPMIToolNull { + return "", errors.Wrap(err, "GetPowerStatus") + } else if b.HasBMC() { + return "", errors.Wrap(err, "GetPowerStatus from ipmi") + } + } + return status, nil +} + +func (b *SBaremetalInstance) getPowerStatus() (string, error) { ipmiCli := b.GetIPMITool() if ipmiCli == nil { - return "", fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId()) + if _, err := b.GetHostSSHClient(); err == nil { + return types.POWER_STATUS_ON, nil + } else { + log.Warningf("Use host %s ssh client get powerstatus: %v", b.GetName(), err) + } + if _, err := b.GetServerSSHClient(); err == nil { + return types.POWER_STATUS_ON, nil + } else { + log.Warningf("Use server %s ssh client get powerstatus: %v", b.GetServerName(), err) + } + return "", errors.Wrapf(types.ErrIPMIToolNull, "Baremetal %s", b.GetId()) } return ipmitool.GetChassisPowerStatus(ipmiCli) } diff --git a/pkg/baremetal/tasks/base.go b/pkg/baremetal/tasks/base.go index b14e8a5256..9a8983ea57 100644 --- a/pkg/baremetal/tasks/base.go +++ b/pkg/baremetal/tasks/base.go @@ -342,6 +342,29 @@ func (self *SBaremetalTaskBase) EnsurePowerUp() error { return nil } +func (self *SBaremetalTaskBase) EnsureSSHReboot() error { + if err := self.Baremetal.SSHReboot(); err != nil { + return errors.Wrap(err, "Ensure ssh reboot") + } + + var ( + err error + canReach bool + ) + maxTries := 20 + startTime := time.Now() + for count := 0; count < maxTries; count++ { + times := (count % 10) + 1 + log.Infof("Try %s ssh connection after reboot %d times, %s passed", self.Baremetal.GetName(), times, time.Now().Sub(startTime)) + canReach, err = self.Baremetal.SSHReachable() + if canReach { + return nil + } + time.Sleep(10 * time.Second * time.Duration(times)) + } + return errors.Wrapf(err, "Test %s ssh connection after reboot", self.Baremetal.GetName()) +} + func (self *SBaremetalTaskBase) NeedPXEBoot() bool { return false } @@ -400,27 +423,35 @@ func (self *SBaremetalPXEBootTaskBase) InitPXEBootTask(ctx context.Context, args self.PxeBoot = false } - // Do soft reboot - if self.data != nil && jsonutils.QueryBoolean(self.data, "soft_boot", false) { - self.startTime = time.Now() - if err := self.Baremetal.DoPowerShutdown(true); err != nil { - // ignore error - log.Errorf("DoPowerShutdown error: %v", err) + if !self.Baremetal.HasBMC() { + // Try remote ssh reboot + if err := self.Baremetal.SSHReboot(); err != nil { + return errors.Wrap(err, "Try ssh reboot") } - //self.CallNextStage(self, self.WaitForShutdown, nil) - self.SetStage(self.WaitForShutdown) + } else { + // Do soft reboot + if self.data != nil && jsonutils.QueryBoolean(self.data, "soft_boot", false) { + self.startTime = time.Now() + if err := self.Baremetal.DoPowerShutdown(true); err != nil { + // ignore error + log.Errorf("DoPowerShutdown error: %v", err) + } + //self.CallNextStage(self, self.WaitForShutdown, nil) + self.SetStage(self.WaitForShutdown) - return nil + return nil + } + + // shutdown and power up to PXE mode + if err := self.EnsurePowerShutdown(false); err != nil { + return errors.Wrap(err, "EnsurePowerShutdown") + } + + if err := self.EnsurePowerUp(); err != nil { + return errors.Wrap(err, "EnsurePowerUp to pxe") + } } - // shutdown and power up to PXE mode - if err := self.EnsurePowerShutdown(false); err != nil { - return errors.Wrap(err, "EnsurePowerShutdown") - } - - if err := self.EnsurePowerUp(); err != nil { - return errors.Wrap(err, "EnsurePowerUp to pxe") - } // this stage will be called by baremetalInstance when pxe start notify self.SetSSHStage(self.IPXEBootTask().OnPXEBoot) return nil diff --git a/pkg/baremetal/tasks/basedeploy.go b/pkg/baremetal/tasks/basedeploy.go index 0860de364c..725c7edcd4 100644 --- a/pkg/baremetal/tasks/basedeploy.go +++ b/pkg/baremetal/tasks/basedeploy.go @@ -22,6 +22,7 @@ import ( "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/ssh" ) @@ -35,6 +36,8 @@ type IServerBaseDeployTask interface { type SBaremetalServerBaseDeployTask struct { SBaremetalPXEBootTaskBase + + needPXEBoot bool } func newBaremetalServerBaseDeployTask( @@ -45,6 +48,7 @@ func newBaremetalServerBaseDeployTask( ) SBaremetalServerBaseDeployTask { task := SBaremetalServerBaseDeployTask{ SBaremetalPXEBootTaskBase: newBaremetalPXEBootTaskBase(userCred, baremetal, taskId, data), + needPXEBoot: true, } // any inheritance must call: // task.SetStage(task.InitPXEBootTask) @@ -59,6 +63,10 @@ func (self *SBaremetalServerBaseDeployTask) GetName() string { return "BaremetalServerBaseDeployTask" } +func (self *SBaremetalServerBaseDeployTask) NeedPXEBoot() bool { + return self.needPXEBoot +} + func (self *SBaremetalServerBaseDeployTask) GetFinishAction() string { if self.data != nil { action, _ := self.data.GetString("on_finish") @@ -88,23 +96,42 @@ func (self *SBaremetalServerBaseDeployTask) OnPXEBoot(ctx context.Context, term if err != nil { return errors.Wrap(err, "Sync disk") } + if err := self.IServerBaseDeployTask().PostDeploys(term); err != nil { return errors.Wrap(err, "post deploy") } + onFinishAction := self.GetFinishAction() if utils.IsInStringArray(onFinishAction, []string{"restart", "shutdown"}) { - err = self.EnsurePowerShutdown(false) - if err != nil { - return errors.Wrap(err, "Ensure power off") - } - if onFinishAction == "restart" { - err = self.EnsurePowerUp() - if err != nil { - return errors.Wrap(err, "Ensure power up") + if self.Baremetal.HasBMC() { + if err := self.EnsurePowerShutdown(false); err != nil { + return errors.Wrap(err, "Ensure power off") + } + if onFinishAction == "restart" { + if err := self.EnsurePowerUp(); err != nil { + return errors.Wrap(err, "Ensure power up") + } + } + self.Baremetal.AutoSyncAllStatus() + } else { + if onFinishAction == "shutdown" { + log.Infof("None BMC baremetal can't shutdown when deploying") + /* + * if err := self.Baremetal.SSHShutdown(); err != nil { + * return errors.Wrap(err, "Try ssh shutdown") + * } + */ + } else { + // do restart + // hack: ssh reboot to disk + self.needPXEBoot = false + if err := self.EnsureSSHReboot(); err != nil { + return errors.Wrap(err, "Try ssh reboot") + } } } + self.Baremetal.SyncAllStatus(types.POWER_STATUS_ON) } - self.Baremetal.AutoSyncAllStatus() SetTaskComplete(self, result) return nil } diff --git a/pkg/baremetal/tasks/baseprepare.go b/pkg/baremetal/tasks/baseprepare.go index 7dcedd9f20..b5e8f69dee 100644 --- a/pkg/baremetal/tasks/baseprepare.go +++ b/pkg/baremetal/tasks/baseprepare.go @@ -313,7 +313,7 @@ func (task *sBaremetalPrepareTask) DoPrepare(cli *ssh.Client) error { // set ipmi nic address and user password if err = task.configIPMISetting(cli, infos); err != nil { logclient.AddActionLogWithStartable(task, task.baremetal, logclient.ACT_PREPARE, err, task.userCred, false) - return err + return errors.Wrap(err, "Config IPMI setting") } if err = task.updateBmInfo(cli, infos); err != nil { diff --git a/pkg/baremetal/tasks/create.go b/pkg/baremetal/tasks/create.go index e0c56186cf..c9402a56e6 100644 --- a/pkg/baremetal/tasks/create.go +++ b/pkg/baremetal/tasks/create.go @@ -90,7 +90,10 @@ func doPoweroff(term *ssh.Client) error { } func (self *SBaremetalServerCreateTask) PostDeploys(term *ssh.Client) error { - return doPoweroff(term) + if self.Baremetal.HasBMC() { + return doPoweroff(term) + } + return nil } func (self *SBaremetalServerCreateTask) onError(term *ssh.Client, err error) error { diff --git a/pkg/baremetal/tasks/interface.go b/pkg/baremetal/tasks/interface.go index 193ca6bb77..67a98ebd1e 100644 --- a/pkg/baremetal/tasks/interface.go +++ b/pkg/baremetal/tasks/interface.go @@ -68,6 +68,11 @@ type IBaremetal interface { SaveSSHConfig(remoteAddr string, key string) error ServerLoadDesc() error GetDHCPServerIP() (net.IP, error) + + HasBMC() bool + SSHReachable() (bool, error) + SSHReboot() error + SSHShutdown() error } type IBmManager interface { diff --git a/pkg/baremetal/tasks/rebuild.go b/pkg/baremetal/tasks/rebuild.go index e8cb93f312..411b8118d7 100644 --- a/pkg/baremetal/tasks/rebuild.go +++ b/pkg/baremetal/tasks/rebuild.go @@ -65,5 +65,8 @@ func (self *SBaremetalServerRebuildTask) DoDeploys(term *ssh.Client) (jsonutils. } func (self *SBaremetalServerRebuildTask) PostDeploys(term *ssh.Client) error { - return doPoweroff(term) + if self.Baremetal.HasBMC() { + return doPoweroff(term) + } + return nil } diff --git a/pkg/baremetal/tasks/reprepare.go b/pkg/baremetal/tasks/reprepare.go index b3d065afbf..16643ebddc 100644 --- a/pkg/baremetal/tasks/reprepare.go +++ b/pkg/baremetal/tasks/reprepare.go @@ -48,3 +48,8 @@ func (self *SBaremetalReprepareTask) DoDeploys(term *ssh.Client) (jsonutils.JSON err := task.DoPrepare(term) return nil, err } + +func (self *SBaremetalReprepareTask) PostDeploys(term *ssh.Client) error { + self.Baremetal.AutoSyncStatus() + return nil +} diff --git a/pkg/baremetal/tasks/start.go b/pkg/baremetal/tasks/start.go index e8014f5544..0452eea493 100644 --- a/pkg/baremetal/tasks/start.go +++ b/pkg/baremetal/tasks/start.go @@ -44,16 +44,22 @@ func NewBaremetalServerStartTask( } func (self *SBaremetalServerStartTask) DoBoot(ctx context.Context, args interface{}) error { - conf := self.Baremetal.GetRawIPMIConfig() - if !conf.CdromBoot { - err := self.Baremetal.DoPXEBoot() - if err != nil { - return errors.Wrap(err, "DoPXEBoot") + if self.Baremetal.HasBMC() { + conf := self.Baremetal.GetRawIPMIConfig() + if !conf.CdromBoot { + err := self.Baremetal.DoPXEBoot() + if err != nil { + return errors.Wrap(err, "DoPXEBoot") + } + } else { + err := self.Baremetal.DoRedfishPowerOn() + if err != nil { + return errors.Wrap(err, "DoRedfishPowerOn") + } } } else { - err := self.Baremetal.DoRedfishPowerOn() - if err != nil { - return errors.Wrap(err, "DoRedfishPowerOn") + if err := self.Baremetal.SSHReboot(); err != nil { + return errors.Wrap(err, "Try reboot") } } self.SetStage(self.WaitForStart) @@ -68,7 +74,7 @@ func (self *SBaremetalServerStartTask) GetName() string { func (self *SBaremetalServerStartTask) WaitForStart(ctx context.Context, args interface{}) error { status, err := self.Baremetal.GetPowerStatus() if err != nil { - return errors.Wrap(err, "GetPowerStatus") + return errors.Wrap(err, "Wait for start") } log.Infof("%s WaitForStart status=%s", self.GetName(), status) if status == types.POWER_STATUS_ON { diff --git a/pkg/baremetal/tasks/stop.go b/pkg/baremetal/tasks/stop.go index 5d1fd3bb97..d1a3298e44 100644 --- a/pkg/baremetal/tasks/stop.go +++ b/pkg/baremetal/tasks/stop.go @@ -46,9 +46,16 @@ func NewBaremetalServerStopTask( } func (task *SBaremetalServerStopTask) DoStop(ctx context.Context, args interface{}) error { - task.SetStage(task.WaitForStop) - if err := task.Baremetal.DoPowerShutdown(true); err != nil { - log.Errorf("Do power shutdown error: %s", err) + if task.Baremetal.HasBMC() { + task.SetStage(task.WaitForStop) + if err := task.Baremetal.DoPowerShutdown(true); err != nil { + log.Errorf("Do power shutdown error: %s", err) + } + } else { + if err := task.Baremetal.SSHShutdown(); err != nil { + return errors.Wrap(err, "Try ssh shutdown") + } + task.SetStage(task.OnStopComplete) } task.startTime = time.Now() ExecuteTask(task, nil) diff --git a/pkg/baremetal/tasks/unmaintenance.go b/pkg/baremetal/tasks/unmaintenance.go index ec51e2d848..cec48f9cbf 100644 --- a/pkg/baremetal/tasks/unmaintenance.go +++ b/pkg/baremetal/tasks/unmaintenance.go @@ -16,9 +16,9 @@ package tasks import ( "context" - "fmt" "yunion.io/x/jsonutils" + "yunion.io/x/pkg/errors" baremetalstatus "yunion.io/x/onecloud/pkg/baremetal/status" "yunion.io/x/onecloud/pkg/cloudcommon/types" @@ -45,14 +45,23 @@ func NewBaremetalUnmaintenanceTask( func (task *SBaremetalUnmaintenanceTask) DoUnmaintenance(ctx context.Context, args interface{}) error { var err error - if jsonutils.QueryBoolean(task.data, "guest_running", false) { - err = task.EnsurePowerShutdown(false) - if err != nil { - return fmt.Errorf("EnsurePowerShutdown hard: %v", err) - } - err = task.EnsurePowerUp() - if err != nil { - return fmt.Errorf("EnsurePowerUp disk: %v", err) + hasBMC := task.Baremetal.HasBMC() + if jsonutils.QueryBoolean(task.data, "guest_running", false) || !hasBMC { + if hasBMC { + err = task.EnsurePowerShutdown(false) + if err != nil { + return errors.Errorf("EnsurePowerShutdown hard: %v", err) + } + err = task.EnsurePowerUp() + if err != nil { + return errors.Errorf("EnsurePowerUp disk: %v", err) + } + } else { + if task.Baremetal.GetServer() != nil { + if err := task.EnsureSSHReboot(); err != nil { + return errors.Wrap(err, "Do unmaintenance for none BMC server") + } + } } task.Baremetal.SyncStatus(baremetalstatus.RUNNING, "") SetTaskComplete(task, nil) @@ -61,7 +70,7 @@ func (task *SBaremetalUnmaintenanceTask) DoUnmaintenance(ctx context.Context, ar task.SetStage(task.WaitForStop) err = task.EnsurePowerShutdown(true) if err != nil { - return fmt.Errorf("EnsurePowerShutdown soft: %v", err) + return errors.Errorf("EnsurePowerShutdown soft: %v", err) } ExecuteTask(task, nil) return nil diff --git a/pkg/baremetal/types/types.go b/pkg/baremetal/types/types.go index 6d3d23be1e..7e4cfad792 100644 --- a/pkg/baremetal/types/types.go +++ b/pkg/baremetal/types/types.go @@ -36,6 +36,7 @@ type IBaremetalServer interface { SyncPartitionSize(term *ssh.Client, parts []*disktool.Partition) ([]jsonutils.JSONObject, error) DoDeploy(term *ssh.Client, data jsonutils.JSONObject, isInit bool) (jsonutils.JSONObject, error) SaveDesc(desc jsonutils.JSONObject) error + GetNics() []types.SServerNic GetNicByMac(mac net.HardwareAddr) *types.SNic GetRootTemplateId() string diff --git a/pkg/cloudcommon/types/types.go b/pkg/cloudcommon/types/types.go index 65d1366f3d..e71ccf70de 100644 --- a/pkg/cloudcommon/types/types.go +++ b/pkg/cloudcommon/types/types.go @@ -17,6 +17,12 @@ package types import ( "net" "strings" + + "yunion.io/x/pkg/errors" +) + +const ( + ErrIPMIToolNull = errors.Error("IPMI tool is null") ) type SSHConfig struct { diff --git a/pkg/compute/guestdrivers/baremetals.go b/pkg/compute/guestdrivers/baremetals.go index be4f75ec24..9acb197b39 100644 --- a/pkg/compute/guestdrivers/baremetals.go +++ b/pkg/compute/guestdrivers/baremetals.go @@ -345,7 +345,7 @@ func (self *SBaremetalGuestDriver) RequestStopGuestForDelete(ctx context.Context return guest.StartGuestStopTask(ctx, task.GetUserCred(), true, false, task.GetTaskId()) } if host != nil && !host.GetEnabled() && !purge { - return fmt.Errorf("fail to contact baremetal") + return errors.Errorf("fail to contact baremetal") } task.ScheduleRun(nil) return nil diff --git a/pkg/compute/models/hosts.go b/pkg/compute/models/hosts.go index 63d567bdd5..cf59158a53 100644 --- a/pkg/compute/models/hosts.go +++ b/pkg/compute/models/hosts.go @@ -3935,6 +3935,14 @@ func (self *SHost) AllowPerformPrepare(ctx context.Context, return db.IsAdminAllowPerform(userCred, self, "prepare") } +func (self *SHost) HasBMC() bool { + ipmiInfo, _ := self.GetIpmiInfo() + if ipmiInfo.Username != "" && ipmiInfo.Password != "" { + return true + } + return false +} + func (self *SHost) isRedfishCapable() bool { ipmiInfo, _ := self.GetIpmiInfo() if ipmiInfo.Verified && ipmiInfo.RedfishApi { diff --git a/pkg/compute/tasks/baremetal_maintenance_task.go b/pkg/compute/tasks/baremetal_maintenance_task.go index a23ec9a387..fff757deff 100644 --- a/pkg/compute/tasks/baremetal_maintenance_task.go +++ b/pkg/compute/tasks/baremetal_maintenance_task.go @@ -69,12 +69,13 @@ func (self *BaremetalMaintenanceTask) OnEnterMaintenantModeSucc(ctx context.Cont metadatas["__maint_guest_running"] = guestRunning } baremetal.SetAllMetadata(ctx, metadatas, self.UserCred) + baremetal.StartSyncConfig(ctx, self.GetUserCred(), "") self.SetStageComplete(ctx, nil) } func (self *BaremetalMaintenanceTask) OnEnterMaintenantModeSuccFailed(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { self.SetStageFailed(ctx, body) - baremetal.StartSyncstatus(ctx, self.UserCred, "") + baremetal.StartSyncstatus(ctx, self.GetUserCred(), "") guest := baremetal.GetBaremetalServer() if guest != nil { guest.StartSyncstatus(ctx, self.UserCred, "") diff --git a/pkg/compute/tasks/baremetal_server_sync_status_task.go b/pkg/compute/tasks/baremetal_server_sync_status_task.go index 1c00ecc5b3..1f93cc1d46 100644 --- a/pkg/compute/tasks/baremetal_server_sync_status_task.go +++ b/pkg/compute/tasks/baremetal_server_sync_status_task.go @@ -57,6 +57,7 @@ func (self *BaremetalServerSyncStatusTask) OnInit(ctx context.Context, obj db.IS func (self *BaremetalServerSyncStatusTask) OnGuestStatusTaskComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) { var status string var hostStatus string + host := guest.GetHost() if data.Contains("status") { statusStr, _ := data.GetString("status") switch statusStr { @@ -69,6 +70,9 @@ func (self *BaremetalServerSyncStatusTask) OnGuestStatusTaskComplete(ctx context case "admin": status = api.VM_ADMIN hostStatus = api.HOST_STATUS_RUNNING + if !host.IsMaintenance && !host.HasBMC() { + status = api.VM_READY + } default: status = api.VM_INIT hostStatus = api.HOST_STATUS_UNKNOWN @@ -78,7 +82,6 @@ func (self *BaremetalServerSyncStatusTask) OnGuestStatusTaskComplete(ctx context hostStatus = api.HOST_STATUS_UNKNOWN } guest.SetStatus(self.UserCred, status, "BaremetalServerSyncStatusTask") - host := guest.GetHost() host.SetStatus(self.UserCred, hostStatus, "BaremetalServerSyncStatusTask") self.SetStageComplete(ctx, nil) diff --git a/pkg/compute/tasks/baremetal_sync_status_task.go b/pkg/compute/tasks/baremetal_sync_status_task.go index ad57b28266..8ee65dc983 100644 --- a/pkg/compute/tasks/baremetal_sync_status_task.go +++ b/pkg/compute/tasks/baremetal_sync_status_task.go @@ -56,6 +56,10 @@ func (self *BaremetalSyncStatusTask) OnSyncstatusComplete(ctx context.Context, b self.SetStageComplete(ctx, nil) } +func (self *BaremetalSyncStatusTask) OnSyncstatusCompleteFailed(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { + self.SetStageFailed(ctx, body) +} + type BaremetalSyncAllGuestsStatusTask struct { SBaremetalBaseTask } @@ -112,6 +116,10 @@ func (self *BaremetalSyncAllGuestsStatusTask) OnGuestSyncStatusComplete(ctx cont self.SetStageComplete(ctx, nil) } +func (self *BaremetalSyncAllGuestsStatusTask) OnGuestSyncStatusCompleteFailed(ctx context.Context, baremetal *models.SHost, body jsonutils.JSONObject) { + self.SetStageFailed(ctx, body) +} + func init() { taskman.RegisterTask(BaremetalSyncStatusTask{}) taskman.RegisterTask(BaremetalSyncAllGuestsStatusTask{}) diff --git a/pkg/compute/tasks/baremetal_unmaintenance_task.go b/pkg/compute/tasks/baremetal_unmaintenance_task.go index 051541c9e6..b8359ad5c9 100644 --- a/pkg/compute/tasks/baremetal_unmaintenance_task.go +++ b/pkg/compute/tasks/baremetal_unmaintenance_task.go @@ -63,10 +63,11 @@ func (self *BaremetalUnmaintenanceTask) OnUnmaintenantComplete(ctx context.Conte "__maint_password": "None", "__maint_ip": "None", } - baremetal.SetAllMetadata(ctx, metadatas, self.UserCred) + baremetal.SetAllMetadata(ctx, metadatas, self.GetUserCred()) self.SetStageComplete(ctx, nil) guest := baremetal.GetBaremetalServer() if guest != nil { - guest.StartSyncstatus(ctx, self.UserCred, "") + guest.StartSyncstatus(ctx, self.GetUserCred(), "") } + baremetal.StartSyncConfig(ctx, self.GetUserCred(), "") } diff --git a/pkg/util/ssh/ssh.go b/pkg/util/ssh/ssh.go index a5ccd01848..176ea660bc 100644 --- a/pkg/util/ssh/ssh.go +++ b/pkg/util/ssh/ssh.go @@ -144,18 +144,23 @@ func (s *Client) GetConfig() ClientConfig { } func (s *Client) RawRun(cmds ...string) ([]string, error) { - return s.run(false, cmds, nil) + return s.run(false, cmds, nil, false) } func (s *Client) Run(cmds ...string) ([]string, error) { - return s.run(true, cmds, nil) + return s.run(true, cmds, nil, false) } func (s *Client) RunWithInput(input io.Reader, cmds ...string) ([]string, error) { - return s.run(true, cmds, input) + return s.run(true, cmds, input, false) } -func (s *Client) run(parseOutput bool, cmds []string, input io.Reader) ([]string, error) { +// RunWithTTY request Pty before run command. +func (s *Client) RunWithTTY(cmds ...string) ([]string, error) { + return s.run(false, cmds, nil, true) +} + +func (s *Client) run(parseOutput bool, cmds []string, input io.Reader, withPty bool) ([]string, error) { ret := []string{} for _, cmd := range cmds { session, err := s.client.NewSession() @@ -163,6 +168,18 @@ func (s *Client) run(parseOutput bool, cmds []string, input io.Reader) ([]string return nil, err } defer session.Close() + + if withPty { + modes := ssh.TerminalModes{ + ssh.ECHO: 1, // enable echoing + ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud + ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud + } + if err := session.RequestPty("xterm", 24, 80, modes); err != nil { + return nil, errors.Wrap(err, "Setup TTY") + } + } + log.Debugf("Run command: %s", cmd) var stdOut bytes.Buffer var stdErr bytes.Buffer @@ -255,3 +272,11 @@ func (s *Client) RunTerminal() error { } return nil } + +func IsExitMissingError(err error) bool { + errStr := new(ssh.ExitMissingError).Error() + if strings.Contains(err.Error(), errStr) { + return true + } + return false +}