diff --git a/pkg/hostman/diskutils/qemu_kvm/driver.go b/pkg/hostman/diskutils/qemu_kvm/driver.go index 97ca75aca2..434a7d7184 100644 --- a/pkg/hostman/diskutils/qemu_kvm/driver.go +++ b/pkg/hostman/diskutils/qemu_kvm/driver.go @@ -41,7 +41,7 @@ import ( "yunion.io/x/onecloud/pkg/util/sysutils" ) -var BASE_SSH_PORT = 22222 +var BASE_SSH_PORT = 28000 var ( QEMU_KVM_PATH = "/usr/libexec/qemu-kvm" @@ -154,7 +154,7 @@ func (m *QemuDeployManager) unsetPort(port int) { func (m *QemuDeployManager) GetSshFreePort() int { port := m.GetFreePortByBase(BASE_SSH_PORT + m.lastUsedSshPort) m.lastUsedSshPort = port - BASE_SSH_PORT - if m.lastUsedSshPort > 10000 { + if m.lastUsedSshPort >= 2000 { m.lastUsedSshPort = 0 } return port @@ -631,7 +631,7 @@ func (d *QemuBaseDriver) startCmds( cmd += __("-append vsyscall=emulate") cmd += fwOpts cmd += __("-device virtio-serial-pci") - cmd += __("-netdev user,id=hostnet0,hostfwd=tcp::%d-:22", sshPort) + cmd += __("-netdev user,id=hostnet0,hostfwd=tcp:127.0.0.1:%d-:22", sshPort) cmd += __("-device virtio-net-pci,netdev=hostnet0") cmd += __("-device virtio-scsi-pci,id=scsi") diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index 36b24decb1..04d965760f 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -1537,7 +1537,9 @@ func (m *SGuestManager) RequestVerifyDirtyServer(s *SKVMGuestInstance) { log.Errorf("Dirty server request start error: %s", err) } else if jsonutils.QueryBoolean(ret, "guest_unknown_need_clean", false) { m.Delete(s.Id) - s.CleanGuest(context.Background(), true) + if err := s.cleanDirtyGuest(context.Background()); err != nil { + log.Errorf("failed clean dirty server %s: %s", s.Id, err) + } } } diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 1cadc6d6c9..9705751756 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -503,6 +503,10 @@ func (s *SKVMGuestInstance) HomeDir() string { return path.Join(s.manager.ServersPath, s.Id) } +func (s *SKVMGuestInstance) RecycleDir() string { + return path.Join(s.manager.ServersPath, "recycle") +} + func (s *SKVMGuestInstance) PrepareDir() error { output, err := procutils.NewCommand("mkdir", "-p", s.HomeDir()).Output() if err != nil { @@ -2109,12 +2113,20 @@ func (s *SKVMGuestInstance) CleanGuest(ctx context.Context, params interface{}) return nil, nil } +func (s *SKVMGuestInstance) cleanDirtyGuest(ctx context.Context) error { + for s.IsRunning() { + s.ForceStop() + time.Sleep(time.Second * 1) + } + return s.Delete(ctx, false, true) +} + func (s *SKVMGuestInstance) StartDelete(ctx context.Context, migrated bool) error { for s.IsRunning() { s.ForceStop() time.Sleep(time.Second * 1) } - return s.Delete(ctx, migrated) + return s.Delete(ctx, migrated, false) } func (s *SKVMGuestInstance) ForceStop() bool { @@ -2234,13 +2246,27 @@ func (s *SKVMGuestInstance) delFlatFiles(ctx context.Context) error { return nil } -func (s *SKVMGuestInstance) Delete(ctx context.Context, migrated bool) error { +func (s *SKVMGuestInstance) Delete(ctx context.Context, migrated, recycle bool) error { if err := s.delTmpDisks(ctx, migrated); err != nil { return errors.Wrap(err, "delTmpDisks") } - output, err := procutils.NewCommand("rm", "-rf", s.HomeDir()).Output() - if err != nil { - return errors.Wrapf(err, "rm %s failed: %s", s.HomeDir(), output) + + if recycle { + if !fileutils2.Exists(s.RecycleDir()) { + output, err := procutils.NewCommand("mkdir", "-p", s.RecycleDir()).Output() + if err != nil { + return errors.Wrapf(err, "mkdir %s failed: %s", s.RecycleDir(), output) + } + } + output, err := procutils.NewCommand("mv", "-f", s.HomeDir(), s.RecycleDir()).Output() + if err != nil { + return errors.Wrapf(err, "mv %s to %s failed: %s", s.HomeDir(), s.RecycleDir(), output) + } + } else { + output, err := procutils.NewCommand("rm", "-rf", s.HomeDir()).Output() + if err != nil { + return errors.Wrapf(err, "rm %s failed: %s", s.HomeDir(), output) + } } return nil } diff --git a/pkg/hostman/isolated_device/gpu.go b/pkg/hostman/isolated_device/gpu.go index bd931e365e..0826a7e521 100644 --- a/pkg/hostman/isolated_device/gpu.go +++ b/pkg/hostman/isolated_device/gpu.go @@ -110,7 +110,16 @@ func getPassthroughGPUs(filteredAddrs []string, enableWhitelist bool, whitelistM warns = append(warns, errors.Wrapf(err, "get dev %s iommu group devices", dev.Addr)) continue } - if err := dev.forceBindVFIOPCIDriver(o.HostOptions.UseBootVga); err != nil { + + if isBootVga, err := dev.IsBootVGA(); err != nil { + warns = append(warns, errors.Wrapf(err, "check dev %s is boot vga devices", dev.Addr)) + continue + } else if isBootVga && !o.HostOptions.UseBootVga { + log.Infof("skip boot vga device %s", dev.Addr) + continue + } + + if err := dev.forceBindVFIOPCIDriver(); err != nil { warns = append(warns, errors.Wrapf(err, "force bind vfio-pci driver %s", dev.Addr)) continue } @@ -201,7 +210,7 @@ func NewPCIDevice(addr string, executors ...IExecutor) (*PCIDevice, error) { if err := dev.checkSameIOMMUGroupDevice(); err != nil { return nil, err } - if err := dev.forceBindVFIOPCIDriver(o.HostOptions.UseBootVga); err != nil { + if err := dev.forceBindVFIOPCIDriver(); err != nil { return nil, fmt.Errorf("Force bind vfio-pci driver: %v", err) } return dev, nil @@ -382,27 +391,18 @@ func (d *PCIDevice) IsBootVGA() (bool, error) { return false, nil } -func (d *PCIDevice) forceBindVFIOPCIDriver(useBootVGA bool) error { +func (d *PCIDevice) forceBindVFIOPCIDriver() error { if !utils.IsInArray(d.ClassCode, GpuClassCodes) { return nil } - isBootVGA, err := d.IsBootVGA() - if err != nil { - return err - } - if !useBootVGA && isBootVGA { - log.Infof("%#v is boot vga card, skip it", d) - return nil - } - if d.IsVFIOPCIDriverUsed() { - log.Infof("%s already use vfio-pci driver", d) - return nil - } - devs := []*PCIDevice{} devs = append(devs, d.RestIOMMUGroupDevs...) devs = append(devs, d) for _, dev := range devs { + if dev.IsVFIOPCIDriverUsed() { + log.Infof("%s already use vfio-pci driver", d.Addr) + continue + } if err := dev.bindAddrVFIOPCI(); err != nil { return fmt.Errorf("bind %s vfio-pci driver: %v", dev, err) } @@ -437,11 +437,20 @@ func (d *PCIDevice) unbindDriver() error { func (d *PCIDevice) bindDriver() error { vendorDevId := fmt.Sprintf("%s %s", d.VendorId, d.DeviceId) - return fileutils2.FilePutContents( + err := fileutils2.FilePutContents( "/sys/bus/pci/drivers/vfio-pci/new_id", fmt.Sprintf("%s\n", vendorDevId), false, ) + if err != nil { + log.Errorf("failed write %s to %s, try bind addr", vendorDevId, "/sys/bus/pci/drivers/vfio-pci/new_id") + } else { + return nil + } + if err := fileutils2.FilePutContents("/sys/bus/pci/drivers/vfio-pci/bind", fmt.Sprintf("0000:%s", d.Addr), false); err != nil { + return fmt.Errorf("bind driver %s : %v", d.Addr, err) + } + return nil } func (d *PCIDevice) fillPCIEInfo(executor IExecutor) error {