Merge pull request #12969 from rainzm/network/restart

feat(region): restart network after change ipaddr for guest

pkg/apis/compute/guest_const.go

@@ -146,6 +146,9 @@ const (
 	VM_UPDATE_TAGS        = "update_tags"
 	VM_UPDATE_TAGS_FAILED = "update_tags_fail"
 
+	VM_RESTART_NETWORK        = "restart_network"
+	VM_RESTART_NETWORK_FAILED = "restart_network_failed"
+
 	SHUTDOWN_STOP      = "stop"
 	SHUTDOWN_TERMINATE = "terminate"
 

pkg/compute/models/guest_actions.go

@@ -720,8 +720,29 @@ func (self *SGuest) PerformAttachdisk(ctx context.Context, userCred mcclient.Tok
 	return nil, self.GetDriver().StartGuestAttachDiskTask(ctx, userCred, self, taskData, "")
 }
 
-func (self *SGuest) StartSyncTask(ctx context.Context, userCred mcclient.TokenCredential, firewallOnly bool,
-	parentTaskId string) error {
+func (self *SGuest) StartRestartNetworkTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, ip string) error {
+	data := jsonutils.NewDict()
+	data.Set("ip", jsonutils.NewString(ip))
+	if task, err := taskman.TaskManager.NewTask(ctx, "GuestRestartNetworkTask", self, userCred, data, parentTaskId, "", nil); err != nil {
+		log.Errorln(err)
+		return err
+	} else {
+		task.ScheduleRun(nil)
+	}
+	return nil
+}
+
+func (self *SGuest) startSyncTask(ctx context.Context, userCred mcclient.TokenCredential, firewallOnly bool, parentTaskId string, data *jsonutils.JSONDict) error {
+	if firewallOnly {
+		data.Add(jsonutils.JSONTrue, "fw_only")
+	} else if err := self.SetStatus(userCred, api.VM_SYNC_CONFIG, ""); err != nil {
+		log.Errorln(err)
+		return err
+	}
+	return self.doSyncTask(ctx, data, userCred, parentTaskId)
+}
+
+func (self *SGuest) StartSyncTask(ctx context.Context, userCred mcclient.TokenCredential, firewallOnly bool, parentTaskId string) error {
 
 	data := jsonutils.NewDict()
 	if firewallOnly {
@@ -2095,8 +2116,14 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T
 	}
 	logclient.AddActionLogWithContext(ctx, self, logclient.ACT_VM_CHANGE_NIC, notes, userCred, true)
 
-	err = self.StartSyncTask(ctx, userCred, false, "")
-	return nil, err
+	restartNetwork, _ := data.Bool("restart_network")
+
+	taskData := jsonutils.NewDict()
+	if self.Hypervisor == api.HYPERVISOR_KVM && restartNetwork {
+		taskData.Set("restart_network", jsonutils.JSONTrue)
+		taskData.Set("prev_ip", jsonutils.NewString(gn.IpAddr))
+	}
+	return nil, self.startSyncTask(ctx, userCred, true, "", taskData)
 }
 
 func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerDetachnetworkInput) (jsonutils.JSONObject, error) {

pkg/compute/tasks/guest_restart_network_task.go

@@ -0,0 +1,209 @@
+package tasks
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/tristate"
+
+	ansible_api "yunion.io/x/onecloud/pkg/apis/ansible"
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
+	"yunion.io/x/onecloud/pkg/compute/models"
+	"yunion.io/x/onecloud/pkg/devtool/utils"
+	"yunion.io/x/onecloud/pkg/mcclient/auth"
+	ansible_modules "yunion.io/x/onecloud/pkg/mcclient/modules/ansible"
+	"yunion.io/x/onecloud/pkg/util/ansiblev2"
+	"yunion.io/x/onecloud/pkg/util/logclient"
+)
+
+type GuestRestartNetworkTask struct {
+	SGuestBaseTask
+}
+
+func init() {
+	taskman.RegisterTask(GuestRestartNetworkTask{})
+}
+
+func (self *GuestRestartNetworkTask) taskFailed(ctx context.Context, guest *models.SGuest, clean func() error, err error) {
+	guest.SetStatus(self.GetUserCred(), api.VM_RESTART_NETWORK_FAILED, err.Error())
+	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_RESTART_NETWORK, jsonutils.NewString(err.Error()), self.UserCred, false)
+	if clean != nil {
+		err := clean()
+		if err != nil {
+			log.Errorf("unable to clean: %s", err.Error())
+		}
+	}
+	self.SetStageFailed(ctx, nil)
+}
+
+func (self *GuestRestartNetworkTask) OnCloseIpMacSrcCheckComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	guest.SetStatus(self.GetUserCred(), api.VM_RESTART_NETWORK, "restart network")
+
+	vpc, err := guest.GetVpc()
+	if err != nil {
+		self.taskFailed(ctx, guest, nil, err)
+		return
+	}
+	ip, _ := self.Params.GetString("ip")
+	session := auth.GetAdminSession(ctx, "", "")
+	sshable, clean, err := utils.CheckSshableForYunionCloud(session, utils.SServerInfo{
+		Id:         guest.GetId(),
+		Ip:         ip,
+		Hypervisor: guest.Hypervisor,
+		VpcId:      vpc.GetId(),
+	})
+	log.Infof("start to CheckSshableForYunionCloud")
+	if err != nil {
+		self.taskFailed(ctx, guest, clean, err)
+		return
+	}
+	log.Infof("ssable: %s", jsonutils.Marshal(sshable))
+	if !sshable.Ok {
+		self.taskFailed(ctx, guest, clean, fmt.Errorf("guest %s is not sshable", guest.GetId()))
+		return
+	}
+
+	playbook := `- hosts: all
+  become: true
+
+  tasks:
+    - name: "restart network"
+      service:
+        name: network
+        state: restarted
+      async: 20
+      poll: 0`
+	params := jsonutils.NewDict()
+	params.Set("playbook", jsonutils.NewString(playbook))
+
+	vars := map[string]interface{}{
+		"ansible_port": fmt.Sprintf("%d", sshable.Port),
+		"ansible_user": sshable.User,
+	}
+	host := ansiblev2.NewHost()
+	host.Vars = vars
+	inv := ansiblev2.NewInventory()
+	inv.SetHost(sshable.Host, host)
+
+	params.Set("inventory", jsonutils.NewString(inv.String()))
+	params.Set("generate_name", jsonutils.NewString(fmt.Sprintf("%s-restart-network", guest.Name)))
+
+	apb, err := ansible_modules.AnsiblePlaybooksV2.Create(session, params)
+	if err != nil {
+		self.taskFailed(ctx, guest, clean, err)
+		return
+	}
+	id, _ := apb.GetString("id")
+	defer func() {
+		_, err := ansible_modules.AnsiblePlaybooksV2.Delete(session, id, nil)
+		if err != nil {
+			log.Errorf("unable to delete ansibleplaybook %s: %v", id, err)
+		}
+	}()
+	times, waitTimes := 0, time.Second
+Loop:
+	for times < 10 {
+		time.Sleep(waitTimes)
+		times++
+		waitTimes += time.Second * time.Duration(times)
+		apd, err := ansible_modules.AnsiblePlaybooksV2.GetSpecific(session, id, "status", nil)
+		if err != nil {
+			continue
+		}
+		status, _ := apd.GetString("status")
+		switch status {
+		case ansible_api.AnsiblePlaybookStatusInit, ansible_api.AnsiblePlaybookStatusRunning:
+			continue
+		case ansible_api.AnsiblePlaybookStatusFailed, ansible_api.AnsiblePlaybookStatusCanceled, ansible_api.AnsiblePlaybookStatusUnknown:
+			apd, err := ansible_modules.AnsiblePlaybooksV2.GetSpecific(session, id, "output", nil)
+			if err != nil {
+				self.taskFailed(ctx, guest, nil, errors.Wrapf(err, "ansibleplaybook %s exec failed and can't get its output", id))
+				return
+			}
+			output, _ := apd.GetString("output")
+			self.taskFailed(ctx, guest, clean, fmt.Errorf("exec ansibleplaybook failed, its output:\n %s", output))
+			return
+		case ansible_api.AnsiblePlaybookStatusSucceeded:
+			break Loop
+		}
+	}
+
+	guest.SetStatus(self.GetUserCred(), api.VM_RUNNING, "")
+	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_RESTART_NETWORK, "", self.UserCred, true)
+	if clean != nil {
+		err := clean()
+		if err != nil {
+			log.Errorf("unable to clean: %s", err.Error())
+		}
+	}
+
+	if !self.Params.Contains("src_ip_check") {
+		self.SetStageComplete(ctx, nil)
+		return
+	}
+
+	srcIpCheck, _ := self.Params.Bool("src_ip_check")
+	srcMacCheck, _ := self.Params.Bool("src_mac_check")
+	_, err = db.Update(guest, func() error {
+		guest.SrcIpCheck = tristate.NewFromBool(srcIpCheck)
+		guest.SrcMacCheck = tristate.NewFromBool(srcMacCheck)
+		return nil
+	})
+	if err != nil {
+		self.taskFailed(ctx, guest, nil, err)
+		return
+	}
+	self.SetStage("OnResumeIpMacSrcCheckComplete", nil)
+	err = guest.StartSyncTask(ctx, self.GetUserCred(), false, self.Id)
+	if err != nil {
+		self.taskFailed(ctx, guest, nil, err)
+	}
+}
+
+func (self *GuestRestartNetworkTask) OnResumeIpMacSrcCheckComplete(ctx context.Context, obj db.IScopedResourceManager, data jsonutils.JSONObject) {
+	self.SetStageComplete(ctx, nil)
+}
+
+func (self *GuestRestartNetworkTask) OnResumeIpMacSrcCheckCompleteFailed(ctx context.Context, obj db.IScopedResourceManager, data jsonutils.JSONObject) {
+	self.SetStageFailed(ctx, data)
+}
+
+func (self *GuestRestartNetworkTask) OnCloseIpMacSrcCheckCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	guest.SetStatus(self.GetUserCred(), api.VM_RESTART_NETWORK_FAILED, data.String())
+	logclient.AddActionLogWithStartable(self, guest, logclient.ACT_RESTART_NETWORK, data, self.UserCred, false)
+	self.SetStageFailed(ctx, nil)
+}
+
+func (self *GuestRestartNetworkTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
+	guest := obj.(*models.SGuest)
+	if guest.SrcIpCheck.IsTrue() || guest.SrcMacCheck.IsTrue() {
+		data := jsonutils.NewDict()
+		data.Set("src_ip_check", jsonutils.NewBool(guest.SrcIpCheck.Bool()))
+		data.Set("src_mac_check", jsonutils.NewBool(guest.SrcMacCheck.Bool()))
+		_, err := db.Update(guest, func() error {
+			guest.SrcIpCheck = tristate.False
+			guest.SrcMacCheck = tristate.False
+			return nil
+		})
+		if err != nil {
+			self.taskFailed(ctx, guest, nil, err)
+			return
+		}
+		self.SetStage("OnCloseIpMacSrcCheckComplete", data)
+		err = guest.StartSyncTask(ctx, self.GetUserCred(), false, self.Id)
+		if err != nil {
+			self.taskFailed(ctx, guest, nil, err)
+			return
+		}
+	} else {
+		self.OnCloseIpMacSrcCheckComplete(ctx, obj, data)
+	}
+}

pkg/compute/tasks/guest_sync_task.go

@@ -54,6 +54,17 @@ func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStand
 	guest := obj.(*models.SGuest)
 	if fwOnly, _ := self.GetParams().Bool("fw_only"); fwOnly {
 		db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred)
+		if restart, _ := self.Params.Bool("restart_network"); !restart {
+			self.SetStageComplete(ctx, nil)
+			return
+		}
+		prevIp, err := self.Params.GetString("prev_ip")
+		if err != nil {
+			log.Errorf("unable to get prev_ip when restart_network is true when sync guest")
+			self.SetStageComplete(ctx, nil)
+			return
+		}
+		guest.StartRestartNetworkTask(ctx, self.UserCred, "", prevIp)
 		self.SetStageComplete(ctx, guest.GetShortDesc(ctx))
 	} else if data.Contains("task") {
 		// XXX this is only applied to KVM, which will call task_complete twice

pkg/devtool/utils/ssh.go

@@ -85,14 +85,39 @@ func checkSshableForOtherCloud(session *mcclient.ClientSession, serverId string)
 	return sshable, nil
 }
 
-func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *comapi.ServerDetails) (sshable SSHable, clean bool, err error) {
-	if serverDetail.IPs == "" {
-		err = fmt.Errorf("empty ips for server %s", serverDetail.Id)
+type SServerInfo struct {
+	Id         string
+	Ip         string
+	VpcId      string
+	Hypervisor string
+}
+
+func CheckSshableForYunionCloud(session *mcclient.ClientSession, serverInfo SServerInfo) (sshable SSHable, cleanFunc func() error, err error) {
+	sshable, clean, err := checkSshableForYunionCloud(session, serverInfo)
+	if err != nil {
 		return
 	}
-	ips := strings.Split(serverDetail.IPs, ",")
-	ip := strings.TrimSpace(ips[0])
-	if serverDetail.Hypervisor == comapi.HYPERVISOR_BAREMETAL || serverDetail.VpcId == "" || serverDetail.VpcId == comapi.DEFAULT_VPC_ID {
+	if clean {
+		cleanFunc = func() error {
+			proxyAddr := sshable.Host
+			proxyPort := sshable.Port
+			params := jsonutils.NewDict()
+			params.Set("proto", jsonutils.NewString("tcp"))
+			params.Set("proxy_addr", jsonutils.NewString(proxyAddr))
+			params.Set("proxy_port", jsonutils.NewInt(int64(proxyPort)))
+			_, err := modules.Servers.PerformAction(session, serverInfo.Id, "close-forward", params)
+			if err != nil {
+				return errors.Wrapf(err, "unable to close forward(addr %q, port %d, proto %q) for server %s", proxyAddr, proxyPort, "tcp", serverInfo.Id)
+			}
+			return nil
+		}
+	}
+	return
+}
+
+func checkSshableForYunionCloud(session *mcclient.ClientSession, serverInfo SServerInfo) (sshable SSHable, clean bool, err error) {
+	ip := serverInfo.Ip
+	if serverInfo.Hypervisor == comapi.HYPERVISOR_BAREMETAL || serverInfo.VpcId == "" || serverInfo.VpcId == comapi.DEFAULT_VPC_ID {
 		sshable = SSHable{
 			Ok:   true,
 			User: "cloudroot",
@@ -105,9 +130,9 @@ func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *c
 	lfParams.Set("proto", jsonutils.NewString("tcp"))
 	lfParams.Set("port", jsonutils.NewInt(22))
 	lfParams.Set("addr", jsonutils.NewString(ip))
-	data, err := modules.Servers.PerformAction(session, serverDetail.Id, "list-forward", lfParams)
+	data, err := modules.Servers.PerformAction(session, serverInfo.Id, "list-forward", lfParams)
 	if err != nil {
-		err = errors.Wrapf(err, "unable to List Forward for server %s", serverDetail.Id)
+		err = errors.Wrapf(err, "unable to List Forward for server %s", serverInfo.Id)
 		return
 	}
 	var openForward bool
@@ -125,9 +150,9 @@ func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *c
 
 	var forward jsonutils.JSONObject
 	if openForward {
-		forward, err = modules.Servers.PerformAction(session, serverDetail.Id, "open-forward", lfParams)
+		forward, err = modules.Servers.PerformAction(session, serverInfo.Id, "open-forward", lfParams)
 		if err != nil {
-			err = errors.Wrapf(err, "unable to Open Forward for server %s", serverDetail.Id)
+			err = errors.Wrapf(err, "unable to Open Forward for server %s", serverInfo.Id)
 			return
 		}
 		clean = true
@@ -146,6 +171,21 @@ func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *c
 	return
 }
 
+func checkSshableForYunionCloudWithDetail(session *mcclient.ClientSession, serverDetail *comapi.ServerDetails) (sshable SSHable, clean bool, err error) {
+	if serverDetail.IPs == "" {
+		err = fmt.Errorf("empty ips for server %s", serverDetail.Id)
+		return
+	}
+	ips := strings.Split(serverDetail.IPs, ",")
+	ip := strings.TrimSpace(ips[0])
+	return checkSshableForYunionCloud(session, SServerInfo{
+		Ip:         ip,
+		Id:         serverDetail.Id,
+		Hypervisor: serverDetail.Hypervisor,
+		VpcId:      serverDetail.VpcId,
+	})
+}
+
 func CheckSSHable(session *mcclient.ClientSession, serverId string) (sshable SSHable, cleanFunc func() error, err error) {
 	params := jsonutils.NewDict()
 	params.Set("details", jsonutils.JSONTrue)
@@ -164,7 +204,7 @@ func CheckSSHable(session *mcclient.ClientSession, serverId string) (sshable SSH
 	// check sshable
 	var clean bool
 	if serverDetail.Hypervisor == comapi.HYPERVISOR_KVM || serverDetail.Hypervisor == comapi.HYPERVISOR_BAREMETAL {
-		sshable, clean, err = checkSshableForYunionCloud(session, &serverDetail)
+		sshable, clean, err = checkSshableForYunionCloudWithDetail(session, &serverDetail)
 		if err != nil {
 			return
 		}

pkg/util/logclient/consts.go

@@ -207,4 +207,6 @@ const (
 	ACT_CLOUDACCOUNT_SYNC_NETWORK = "sync_network"
 
 	ACT_MERGE_NETWORK = "merge_network"
+
+	ACT_RESTART_NETWORK = "restart_network"
 )