diff --git a/pkg/apis/compute/guest_const.go b/pkg/apis/compute/guest_const.go index 9d2731f675..754977156b 100644 --- a/pkg/apis/compute/guest_const.go +++ b/pkg/apis/compute/guest_const.go @@ -146,6 +146,9 @@ const ( VM_UPDATE_TAGS = "update_tags" VM_UPDATE_TAGS_FAILED = "update_tags_fail" + VM_RESTART_NETWORK = "restart_network" + VM_RESTART_NETWORK_FAILED = "restart_network_failed" + SHUTDOWN_STOP = "stop" SHUTDOWN_TERMINATE = "terminate" diff --git a/pkg/compute/models/guest_actions.go b/pkg/compute/models/guest_actions.go index f6150b52b4..b82f98c23c 100644 --- a/pkg/compute/models/guest_actions.go +++ b/pkg/compute/models/guest_actions.go @@ -720,8 +720,29 @@ func (self *SGuest) PerformAttachdisk(ctx context.Context, userCred mcclient.Tok return nil, self.GetDriver().StartGuestAttachDiskTask(ctx, userCred, self, taskData, "") } -func (self *SGuest) StartSyncTask(ctx context.Context, userCred mcclient.TokenCredential, firewallOnly bool, - parentTaskId string) error { +func (self *SGuest) StartRestartNetworkTask(ctx context.Context, userCred mcclient.TokenCredential, parentTaskId string, ip string) error { + data := jsonutils.NewDict() + data.Set("ip", jsonutils.NewString(ip)) + if task, err := taskman.TaskManager.NewTask(ctx, "GuestRestartNetworkTask", self, userCred, data, parentTaskId, "", nil); err != nil { + log.Errorln(err) + return err + } else { + task.ScheduleRun(nil) + } + return nil +} + +func (self *SGuest) startSyncTask(ctx context.Context, userCred mcclient.TokenCredential, firewallOnly bool, parentTaskId string, data *jsonutils.JSONDict) error { + if firewallOnly { + data.Add(jsonutils.JSONTrue, "fw_only") + } else if err := self.SetStatus(userCred, api.VM_SYNC_CONFIG, ""); err != nil { + log.Errorln(err) + return err + } + return self.doSyncTask(ctx, data, userCred, parentTaskId) +} + +func (self *SGuest) StartSyncTask(ctx context.Context, userCred mcclient.TokenCredential, firewallOnly bool, parentTaskId string) error { data := jsonutils.NewDict() if firewallOnly { @@ -2095,8 +2116,14 @@ func (self *SGuest) PerformChangeIpaddr(ctx context.Context, userCred mcclient.T } logclient.AddActionLogWithContext(ctx, self, logclient.ACT_VM_CHANGE_NIC, notes, userCred, true) - err = self.StartSyncTask(ctx, userCred, false, "") - return nil, err + restartNetwork, _ := data.Bool("restart_network") + + taskData := jsonutils.NewDict() + if self.Hypervisor == api.HYPERVISOR_KVM && restartNetwork { + taskData.Set("restart_network", jsonutils.JSONTrue) + taskData.Set("prev_ip", jsonutils.NewString(gn.IpAddr)) + } + return nil, self.startSyncTask(ctx, userCred, true, "", taskData) } func (self *SGuest) PerformDetachnetwork(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, input api.ServerDetachnetworkInput) (jsonutils.JSONObject, error) { diff --git a/pkg/compute/tasks/guest_restart_network_task.go b/pkg/compute/tasks/guest_restart_network_task.go new file mode 100644 index 0000000000..b1f2beb6a5 --- /dev/null +++ b/pkg/compute/tasks/guest_restart_network_task.go @@ -0,0 +1,209 @@ +package tasks + +import ( + "context" + "fmt" + "time" + + "yunion.io/x/jsonutils" + "yunion.io/x/log" + "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/tristate" + + ansible_api "yunion.io/x/onecloud/pkg/apis/ansible" + api "yunion.io/x/onecloud/pkg/apis/compute" + "yunion.io/x/onecloud/pkg/cloudcommon/db" + "yunion.io/x/onecloud/pkg/cloudcommon/db/taskman" + "yunion.io/x/onecloud/pkg/compute/models" + "yunion.io/x/onecloud/pkg/devtool/utils" + "yunion.io/x/onecloud/pkg/mcclient/auth" + ansible_modules "yunion.io/x/onecloud/pkg/mcclient/modules/ansible" + "yunion.io/x/onecloud/pkg/util/ansiblev2" + "yunion.io/x/onecloud/pkg/util/logclient" +) + +type GuestRestartNetworkTask struct { + SGuestBaseTask +} + +func init() { + taskman.RegisterTask(GuestRestartNetworkTask{}) +} + +func (self *GuestRestartNetworkTask) taskFailed(ctx context.Context, guest *models.SGuest, clean func() error, err error) { + guest.SetStatus(self.GetUserCred(), api.VM_RESTART_NETWORK_FAILED, err.Error()) + logclient.AddActionLogWithStartable(self, guest, logclient.ACT_RESTART_NETWORK, jsonutils.NewString(err.Error()), self.UserCred, false) + if clean != nil { + err := clean() + if err != nil { + log.Errorf("unable to clean: %s", err.Error()) + } + } + self.SetStageFailed(ctx, nil) +} + +func (self *GuestRestartNetworkTask) OnCloseIpMacSrcCheckComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.GetUserCred(), api.VM_RESTART_NETWORK, "restart network") + + vpc, err := guest.GetVpc() + if err != nil { + self.taskFailed(ctx, guest, nil, err) + return + } + ip, _ := self.Params.GetString("ip") + session := auth.GetAdminSession(ctx, "", "") + sshable, clean, err := utils.CheckSshableForYunionCloud(session, utils.SServerInfo{ + Id: guest.GetId(), + Ip: ip, + Hypervisor: guest.Hypervisor, + VpcId: vpc.GetId(), + }) + log.Infof("start to CheckSshableForYunionCloud") + if err != nil { + self.taskFailed(ctx, guest, clean, err) + return + } + log.Infof("ssable: %s", jsonutils.Marshal(sshable)) + if !sshable.Ok { + self.taskFailed(ctx, guest, clean, fmt.Errorf("guest %s is not sshable", guest.GetId())) + return + } + + playbook := `- hosts: all + become: true + + tasks: + - name: "restart network" + service: + name: network + state: restarted + async: 20 + poll: 0` + params := jsonutils.NewDict() + params.Set("playbook", jsonutils.NewString(playbook)) + + vars := map[string]interface{}{ + "ansible_port": fmt.Sprintf("%d", sshable.Port), + "ansible_user": sshable.User, + } + host := ansiblev2.NewHost() + host.Vars = vars + inv := ansiblev2.NewInventory() + inv.SetHost(sshable.Host, host) + + params.Set("inventory", jsonutils.NewString(inv.String())) + params.Set("generate_name", jsonutils.NewString(fmt.Sprintf("%s-restart-network", guest.Name))) + + apb, err := ansible_modules.AnsiblePlaybooksV2.Create(session, params) + if err != nil { + self.taskFailed(ctx, guest, clean, err) + return + } + id, _ := apb.GetString("id") + defer func() { + _, err := ansible_modules.AnsiblePlaybooksV2.Delete(session, id, nil) + if err != nil { + log.Errorf("unable to delete ansibleplaybook %s: %v", id, err) + } + }() + times, waitTimes := 0, time.Second +Loop: + for times < 10 { + time.Sleep(waitTimes) + times++ + waitTimes += time.Second * time.Duration(times) + apd, err := ansible_modules.AnsiblePlaybooksV2.GetSpecific(session, id, "status", nil) + if err != nil { + continue + } + status, _ := apd.GetString("status") + switch status { + case ansible_api.AnsiblePlaybookStatusInit, ansible_api.AnsiblePlaybookStatusRunning: + continue + case ansible_api.AnsiblePlaybookStatusFailed, ansible_api.AnsiblePlaybookStatusCanceled, ansible_api.AnsiblePlaybookStatusUnknown: + apd, err := ansible_modules.AnsiblePlaybooksV2.GetSpecific(session, id, "output", nil) + if err != nil { + self.taskFailed(ctx, guest, nil, errors.Wrapf(err, "ansibleplaybook %s exec failed and can't get its output", id)) + return + } + output, _ := apd.GetString("output") + self.taskFailed(ctx, guest, clean, fmt.Errorf("exec ansibleplaybook failed, its output:\n %s", output)) + return + case ansible_api.AnsiblePlaybookStatusSucceeded: + break Loop + } + } + + guest.SetStatus(self.GetUserCred(), api.VM_RUNNING, "") + logclient.AddActionLogWithStartable(self, guest, logclient.ACT_RESTART_NETWORK, "", self.UserCred, true) + if clean != nil { + err := clean() + if err != nil { + log.Errorf("unable to clean: %s", err.Error()) + } + } + + if !self.Params.Contains("src_ip_check") { + self.SetStageComplete(ctx, nil) + return + } + + srcIpCheck, _ := self.Params.Bool("src_ip_check") + srcMacCheck, _ := self.Params.Bool("src_mac_check") + _, err = db.Update(guest, func() error { + guest.SrcIpCheck = tristate.NewFromBool(srcIpCheck) + guest.SrcMacCheck = tristate.NewFromBool(srcMacCheck) + return nil + }) + if err != nil { + self.taskFailed(ctx, guest, nil, err) + return + } + self.SetStage("OnResumeIpMacSrcCheckComplete", nil) + err = guest.StartSyncTask(ctx, self.GetUserCred(), false, self.Id) + if err != nil { + self.taskFailed(ctx, guest, nil, err) + } +} + +func (self *GuestRestartNetworkTask) OnResumeIpMacSrcCheckComplete(ctx context.Context, obj db.IScopedResourceManager, data jsonutils.JSONObject) { + self.SetStageComplete(ctx, nil) +} + +func (self *GuestRestartNetworkTask) OnResumeIpMacSrcCheckCompleteFailed(ctx context.Context, obj db.IScopedResourceManager, data jsonutils.JSONObject) { + self.SetStageFailed(ctx, data) +} + +func (self *GuestRestartNetworkTask) OnCloseIpMacSrcCheckCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + guest.SetStatus(self.GetUserCred(), api.VM_RESTART_NETWORK_FAILED, data.String()) + logclient.AddActionLogWithStartable(self, guest, logclient.ACT_RESTART_NETWORK, data, self.UserCred, false) + self.SetStageFailed(ctx, nil) +} + +func (self *GuestRestartNetworkTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) { + guest := obj.(*models.SGuest) + if guest.SrcIpCheck.IsTrue() || guest.SrcMacCheck.IsTrue() { + data := jsonutils.NewDict() + data.Set("src_ip_check", jsonutils.NewBool(guest.SrcIpCheck.Bool())) + data.Set("src_mac_check", jsonutils.NewBool(guest.SrcMacCheck.Bool())) + _, err := db.Update(guest, func() error { + guest.SrcIpCheck = tristate.False + guest.SrcMacCheck = tristate.False + return nil + }) + if err != nil { + self.taskFailed(ctx, guest, nil, err) + return + } + self.SetStage("OnCloseIpMacSrcCheckComplete", data) + err = guest.StartSyncTask(ctx, self.GetUserCred(), false, self.Id) + if err != nil { + self.taskFailed(ctx, guest, nil, err) + return + } + } else { + self.OnCloseIpMacSrcCheckComplete(ctx, obj, data) + } +} diff --git a/pkg/compute/tasks/guest_sync_task.go b/pkg/compute/tasks/guest_sync_task.go index d8cc3ddb33..79df76a330 100644 --- a/pkg/compute/tasks/guest_sync_task.go +++ b/pkg/compute/tasks/guest_sync_task.go @@ -54,6 +54,17 @@ func (self *GuestSyncConfTask) OnSyncComplete(ctx context.Context, obj db.IStand guest := obj.(*models.SGuest) if fwOnly, _ := self.GetParams().Bool("fw_only"); fwOnly { db.OpsLog.LogEvent(guest, db.ACT_SYNC_CONF, nil, self.UserCred) + if restart, _ := self.Params.Bool("restart_network"); !restart { + self.SetStageComplete(ctx, nil) + return + } + prevIp, err := self.Params.GetString("prev_ip") + if err != nil { + log.Errorf("unable to get prev_ip when restart_network is true when sync guest") + self.SetStageComplete(ctx, nil) + return + } + guest.StartRestartNetworkTask(ctx, self.UserCred, "", prevIp) self.SetStageComplete(ctx, guest.GetShortDesc(ctx)) } else if data.Contains("task") { // XXX this is only applied to KVM, which will call task_complete twice diff --git a/pkg/devtool/utils/ssh.go b/pkg/devtool/utils/ssh.go index 361d0e07fe..d424e4ba50 100644 --- a/pkg/devtool/utils/ssh.go +++ b/pkg/devtool/utils/ssh.go @@ -85,14 +85,39 @@ func checkSshableForOtherCloud(session *mcclient.ClientSession, serverId string) return sshable, nil } -func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *comapi.ServerDetails) (sshable SSHable, clean bool, err error) { - if serverDetail.IPs == "" { - err = fmt.Errorf("empty ips for server %s", serverDetail.Id) +type SServerInfo struct { + Id string + Ip string + VpcId string + Hypervisor string +} + +func CheckSshableForYunionCloud(session *mcclient.ClientSession, serverInfo SServerInfo) (sshable SSHable, cleanFunc func() error, err error) { + sshable, clean, err := checkSshableForYunionCloud(session, serverInfo) + if err != nil { return } - ips := strings.Split(serverDetail.IPs, ",") - ip := strings.TrimSpace(ips[0]) - if serverDetail.Hypervisor == comapi.HYPERVISOR_BAREMETAL || serverDetail.VpcId == "" || serverDetail.VpcId == comapi.DEFAULT_VPC_ID { + if clean { + cleanFunc = func() error { + proxyAddr := sshable.Host + proxyPort := sshable.Port + params := jsonutils.NewDict() + params.Set("proto", jsonutils.NewString("tcp")) + params.Set("proxy_addr", jsonutils.NewString(proxyAddr)) + params.Set("proxy_port", jsonutils.NewInt(int64(proxyPort))) + _, err := modules.Servers.PerformAction(session, serverInfo.Id, "close-forward", params) + if err != nil { + return errors.Wrapf(err, "unable to close forward(addr %q, port %d, proto %q) for server %s", proxyAddr, proxyPort, "tcp", serverInfo.Id) + } + return nil + } + } + return +} + +func checkSshableForYunionCloud(session *mcclient.ClientSession, serverInfo SServerInfo) (sshable SSHable, clean bool, err error) { + ip := serverInfo.Ip + if serverInfo.Hypervisor == comapi.HYPERVISOR_BAREMETAL || serverInfo.VpcId == "" || serverInfo.VpcId == comapi.DEFAULT_VPC_ID { sshable = SSHable{ Ok: true, User: "cloudroot", @@ -105,9 +130,9 @@ func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *c lfParams.Set("proto", jsonutils.NewString("tcp")) lfParams.Set("port", jsonutils.NewInt(22)) lfParams.Set("addr", jsonutils.NewString(ip)) - data, err := modules.Servers.PerformAction(session, serverDetail.Id, "list-forward", lfParams) + data, err := modules.Servers.PerformAction(session, serverInfo.Id, "list-forward", lfParams) if err != nil { - err = errors.Wrapf(err, "unable to List Forward for server %s", serverDetail.Id) + err = errors.Wrapf(err, "unable to List Forward for server %s", serverInfo.Id) return } var openForward bool @@ -125,9 +150,9 @@ func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *c var forward jsonutils.JSONObject if openForward { - forward, err = modules.Servers.PerformAction(session, serverDetail.Id, "open-forward", lfParams) + forward, err = modules.Servers.PerformAction(session, serverInfo.Id, "open-forward", lfParams) if err != nil { - err = errors.Wrapf(err, "unable to Open Forward for server %s", serverDetail.Id) + err = errors.Wrapf(err, "unable to Open Forward for server %s", serverInfo.Id) return } clean = true @@ -146,6 +171,21 @@ func checkSshableForYunionCloud(session *mcclient.ClientSession, serverDetail *c return } +func checkSshableForYunionCloudWithDetail(session *mcclient.ClientSession, serverDetail *comapi.ServerDetails) (sshable SSHable, clean bool, err error) { + if serverDetail.IPs == "" { + err = fmt.Errorf("empty ips for server %s", serverDetail.Id) + return + } + ips := strings.Split(serverDetail.IPs, ",") + ip := strings.TrimSpace(ips[0]) + return checkSshableForYunionCloud(session, SServerInfo{ + Ip: ip, + Id: serverDetail.Id, + Hypervisor: serverDetail.Hypervisor, + VpcId: serverDetail.VpcId, + }) +} + func CheckSSHable(session *mcclient.ClientSession, serverId string) (sshable SSHable, cleanFunc func() error, err error) { params := jsonutils.NewDict() params.Set("details", jsonutils.JSONTrue) @@ -164,7 +204,7 @@ func CheckSSHable(session *mcclient.ClientSession, serverId string) (sshable SSH // check sshable var clean bool if serverDetail.Hypervisor == comapi.HYPERVISOR_KVM || serverDetail.Hypervisor == comapi.HYPERVISOR_BAREMETAL { - sshable, clean, err = checkSshableForYunionCloud(session, &serverDetail) + sshable, clean, err = checkSshableForYunionCloudWithDetail(session, &serverDetail) if err != nil { return } diff --git a/pkg/util/logclient/consts.go b/pkg/util/logclient/consts.go index 65cc833d5a..004e2746bd 100644 --- a/pkg/util/logclient/consts.go +++ b/pkg/util/logclient/consts.go @@ -207,4 +207,6 @@ const ( ACT_CLOUDACCOUNT_SYNC_NETWORK = "sync_network" ACT_MERGE_NETWORK = "merge_network" + + ACT_RESTART_NETWORK = "restart_network" )