From d6721519ce4571eb334bcce6fd46a297262ec18e Mon Sep 17 00:00:00 2001 From: wanyaoqi <18528551+wanyaoqi@users.noreply.github.com> Date: Thu, 8 Dec 2022 14:42:46 +0800 Subject: [PATCH] fix(host): don't set ovs interface external id on migrating (#15497) Signed-off-by: wanyaoqi Signed-off-by: wanyaoqi --- pkg/hostman/guestman/guesttasks.go | 15 +++++++++- pkg/hostman/guestman/qemu-kvm.go | 24 +++++++++++++-- pkg/hostman/guestman/qemu-kvmhelper.go | 12 ++++---- pkg/hostman/hostinfo/hostbridge/hostbridge.go | 18 ++++++----- .../hostinfo/hostbridge/linux_bridge.go | 16 ++++++---- pkg/hostman/hostinfo/hostbridge/ovs.go | 30 ++++++++++++------- 6 files changed, 81 insertions(+), 34 deletions(-) diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 1e4056d94b..1544dbf83d 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -1335,7 +1335,6 @@ func (s *SGuestResumeTask) onConfirmRunning(status string) { return } if err := s.onGuestPrelaunch(); err != nil { - s.ForceStop() s.taskFailed(err.Error()) return } @@ -1379,6 +1378,20 @@ func (s *SGuestResumeTask) onGetBlockInfo(blocks []monitor.QemuBlock) { } func (s *SGuestResumeTask) resumeGuest() { + if s.resumed { + s.taskFailed("resume guest twice") + return + } + + if s.Desc.IsVolatileHost { + if err := s.prepareNicsForVolatileGuestResume(); err != nil { + s.taskFailed(err.Error()) + return + } + s.Desc.IsVolatileHost = false + s.SaveLiveDesc(s.Desc) + } + s.startTime = time.Now() s.Monitor.SimpleCommand("cont", s.onResumeSucc) } diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 7e46d2817c..58ad546970 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -479,10 +479,12 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa } // get live migrate listen port - if jsonutils.QueryBoolean(data, "need_migrate", false) || s.Desc.IsSlave { + if s.LiveMigrateDestPort == nil && + (jsonutils.QueryBoolean(data, "need_migrate", false) || s.Desc.IsSlave) { migratePort := s.manager.GetLiveMigrateFreePort() defer s.manager.unsetPort(migratePort) - data.Set("live_migrate_port", jsonutils.NewInt(int64(migratePort))) + migratePortInt64 := int64(migratePort) + s.LiveMigrateDestPort = &migratePortInt64 } err = s.saveScripts(data) @@ -1099,6 +1101,10 @@ func (s *SKVMGuestInstance) IsSlave() bool { return s.Desc.IsSlave } +func (s *SKVMGuestInstance) IsMigratingDestGuest() bool { + return s.LiveMigrateDestPort != nil +} + func (s *SKVMGuestInstance) DiskCount() int { return len(s.Desc.Disks) } @@ -2485,6 +2491,20 @@ func (s *SKVMGuestInstance) PrepareDisksMigrate(liveMigrage bool) (*jsonutils.JS return disksBackFile, nil } +func (s *SKVMGuestInstance) prepareNicsForVolatileGuestResume() error { + for _, nic := range s.Desc.Nics { + bridge := nic.Bridge + dev := s.manager.GetHost().GetBridgeDev(bridge) + if dev == nil { + return fmt.Errorf("Can't find bridge %s", bridge) + } + if err := dev.OnVolatileGuestResume(nic); err != nil { + return errors.Wrap(err, "dev.OnVolatileGuestResume") + } + } + return nil +} + func (s *SKVMGuestInstance) onlineResizeDisk(ctx context.Context, diskId string, sizeMB int64) { task := NewGuestOnlineResizeDiskTask(ctx, s, diskId, sizeMB) task.Start() diff --git a/pkg/hostman/guestman/qemu-kvmhelper.go b/pkg/hostman/guestman/qemu-kvmhelper.go index f52e1e1379..806a652803 100644 --- a/pkg/hostman/guestman/qemu-kvmhelper.go +++ b/pkg/hostman/guestman/qemu-kvmhelper.go @@ -269,11 +269,11 @@ func (s *SKVMGuestInstance) generateNicScripts(nic *desc.SGuestNetwork) error { if dev == nil { return fmt.Errorf("Can't find bridge %s", bridge) } - isSlave := s.IsSlave() - if err := dev.GenerateIfupScripts(s.getNicUpScriptPath(nic), nic, isSlave); err != nil { + isVolatileHost := s.IsSlave() || s.IsMigratingDestGuest() + if err := dev.GenerateIfupScripts(s.getNicUpScriptPath(nic), nic, isVolatileHost); err != nil { return errors.Wrap(err, "GenerateIfupScripts") } - if err := dev.GenerateIfdownScripts(s.getNicDownScriptPath(nic), nic, isSlave); err != nil { + if err := dev.GenerateIfdownScripts(s.getNicDownScriptPath(nic), nic, isVolatileHost); err != nil { return errors.Wrap(err, "GenerateIfdownScripts") } return nil @@ -487,17 +487,15 @@ function nic_mtu() { input.IsKVMSupport = s.IsKvmSupport() input.ExtraOptions = append(input.ExtraOptions, s.extraOptions()) - liveMigratePort, _ := data.Int("live_migrate_port") if jsonutils.QueryBoolean(data, "need_migrate", false) { input.NeedMigrate = true - input.LiveMigratePort = uint(liveMigratePort) - s.LiveMigrateDestPort = &liveMigratePort + input.LiveMigratePort = uint(*s.LiveMigrateDestPort) if jsonutils.QueryBoolean(data, "live_migrate_use_tls", false) { s.LiveMigrateUseTls = true input.LiveMigrateUseTLS = true } } else if s.Desc.IsSlave { - input.LiveMigratePort = uint(liveMigratePort) + input.LiveMigratePort = uint(*s.LiveMigrateDestPort) } qemuOpts, err := qemu.GenerateStartOptions(input) diff --git a/pkg/hostman/hostinfo/hostbridge/hostbridge.go b/pkg/hostman/hostinfo/hostbridge/hostbridge.go index 068f9ad93b..1a9e82e672 100644 --- a/pkg/hostman/hostinfo/hostbridge/hostbridge.go +++ b/pkg/hostman/hostinfo/hostbridge/hostbridge.go @@ -53,12 +53,14 @@ type IBridgeDriver interface { PersistentConfig() error DisableDHCPClient() (bool, error) - GenerateIfupScripts(scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error - GenerateIfdownScripts(scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error + GenerateIfupScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error + GenerateIfdownScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error RegisterHostlocalServer(mac, ip string) error - getUpScripts(nic *desc.SGuestNetwork, isSlave bool) (string, error) - getDownScripts(nic *desc.SGuestNetwork, isSlave bool) (string, error) + getUpScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) + getDownScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) + + OnVolatileGuestResume(nic *desc.SGuestNetwork) error Bridge() string } @@ -358,16 +360,16 @@ func (d *SBaseBridgeDriver) saveFileExecutable(scriptPath, script string) error return os.Chmod(scriptPath, syscall.S_IRUSR|syscall.S_IWUSR|syscall.S_IXUSR) } -func (d *SBaseBridgeDriver) generateIfdownScripts(driver IBridgeDriver, scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error { - script, err := driver.getDownScripts(nic, isSlave) +func (d *SBaseBridgeDriver) generateIfdownScripts(driver IBridgeDriver, scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { + script, err := driver.getDownScripts(nic, isVolatileHost) if err != nil { return errors.Wrap(err, "getDownScripts") } return d.saveFileExecutable(scriptPath, script) } -func (d *SBaseBridgeDriver) generateIfupScripts(driver IBridgeDriver, scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error { - script, err := driver.getUpScripts(nic, isSlave) +func (d *SBaseBridgeDriver) generateIfupScripts(driver IBridgeDriver, scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { + script, err := driver.getUpScripts(nic, isVolatileHost) if err != nil { log.Errorln(err) return err diff --git a/pkg/hostman/hostinfo/hostbridge/linux_bridge.go b/pkg/hostman/hostinfo/hostbridge/linux_bridge.go index ca23f16e32..cfce88b8cf 100644 --- a/pkg/hostman/hostinfo/hostbridge/linux_bridge.go +++ b/pkg/hostman/hostinfo/hostbridge/linux_bridge.go @@ -83,15 +83,19 @@ func (l *SLinuxBridgeDriver) Interfaces() ([]string, error) { return infs, nil } -func (l *SLinuxBridgeDriver) GenerateIfdownScripts(scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error { - return l.generateIfdownScripts(l, scriptPath, nic, isSlave) +func (l *SLinuxBridgeDriver) GenerateIfdownScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { + return l.generateIfdownScripts(l, scriptPath, nic, isVolatileHost) } -func (l *SLinuxBridgeDriver) GenerateIfupScripts(scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error { - return l.generateIfupScripts(l, scriptPath, nic, isSlave) +func (l *SLinuxBridgeDriver) GenerateIfupScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { + return l.generateIfupScripts(l, scriptPath, nic, isVolatileHost) } -func (l *SLinuxBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isSlave bool) (string, error) { +func (l *SLinuxBridgeDriver) OnVolatileGuestResume(nic *desc.SGuestNetwork) error { + return nil +} + +func (l *SLinuxBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) { s := "#!/bin/bash\n\n" s += fmt.Sprintf("switch='%s'\n", l.bridge) if options.HostOptions.TunnelPaddingBytes > 0 { @@ -103,7 +107,7 @@ func (l *SLinuxBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isSlave bool) return s, nil } -func (l *SLinuxBridgeDriver) getDownScripts(nic *desc.SGuestNetwork, isSlave bool) (string, error) { +func (l *SLinuxBridgeDriver) getDownScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) { s := "#!/bin/sh\n\n" s += fmt.Sprintf("switch='%s'\n", l.bridge) s += "brctl show ${switch} | grep $1\n" diff --git a/pkg/hostman/hostinfo/hostbridge/ovs.go b/pkg/hostman/hostinfo/hostbridge/ovs.go index 4876ed871f..b8c8554890 100644 --- a/pkg/hostman/hostinfo/hostbridge/ovs.go +++ b/pkg/hostman/hostinfo/hostbridge/ovs.go @@ -118,15 +118,27 @@ func (d *SOVSBridgeDriver) PersistentConfig() error { return nil } -func (o *SOVSBridgeDriver) GenerateIfdownScripts(scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error { - return o.generateIfdownScripts(o, scriptPath, nic, isSlave) +func (o *SOVSBridgeDriver) GenerateIfdownScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { + return o.generateIfdownScripts(o, scriptPath, nic, isVolatileHost) } -func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic *desc.SGuestNetwork, isSlave bool) error { - return o.generateIfupScripts(o, scriptPath, nic, isSlave) +func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { + return o.generateIfupScripts(o, scriptPath, nic, isVolatileHost) } -func (o *SOVSBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isSlave bool) (string, error) { +func (o *SOVSBridgeDriver) OnVolatileGuestResume(nic *desc.SGuestNetwork) error { + cmd := []string{"ovs-vsctl", "set", "Interface", nic.Ifname, + fmt.Sprintf("external_ids:iface-id=iface-%s-%s", nic.NetId, nic.Ifname), + } + output, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output() + if err != nil { + log.Errorf("failed exec %v: %s %s", cmd, err, output) + return errors.Wrapf(err, "set interface external_ids failed: %s", output) + } + return nil +} + +func (o *SOVSBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) { var ( bridge = o.bridge.String() ifname = nic.Ifname @@ -173,10 +185,8 @@ func (o *SOVSBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isSlave bool) ( s += " TAG=\"tag=$VLAN_ID\"\n" s += "fi\n" s += "ovs-vsctl add-port $SWITCH $IF $TAG\n" - if vpcProvider == compute.VPC_PROVIDER_OVN { - if !isSlave { - s += "ovs-vsctl set Interface $IF external_ids:iface-id=iface-$NET_ID-$IF\n" - } + if vpcProvider == compute.VPC_PROVIDER_OVN && !isVolatileHost { + s += "ovs-vsctl set Interface $IF external_ids:iface-id=iface-$NET_ID-$IF\n" } s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" @@ -196,7 +206,7 @@ func (o *SOVSBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isSlave bool) ( return s, nil } -func (o *SOVSBridgeDriver) getDownScripts(nic *desc.SGuestNetwork, isSlave bool) (string, error) { +func (o *SOVSBridgeDriver) getDownScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) { var ( bridge = o.bridge.String() ifname = nic.Ifname