From 168319f56bfecffbbf819b32c97d5fa487eaf691 Mon Sep 17 00:00:00 2001 From: Jiahui <4543bxy@gmail.com> Date: Fri, 14 Apr 2023 11:55:33 +0800 Subject: [PATCH] optimize reset node `ip link delete` command. (#2947) --- pkg/runtime/master.go | 16 ++++++++-------- pkg/runtime/node.go | 13 +++++++------ pkg/runtime/reset.go | 41 +++++++++++++++++++++++++---------------- pkg/runtime/utils.go | 14 +++++--------- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/pkg/runtime/master.go b/pkg/runtime/master.go index bba92559f..d53944d48 100644 --- a/pkg/runtime/master.go +++ b/pkg/runtime/master.go @@ -171,13 +171,13 @@ func (k *KubeadmRuntime) deleteMasters(masters []string) error { } func (k *KubeadmRuntime) deleteMaster(master string) error { - //remove master - masterIPs := strings.SliceRemoveStr(k.getMasterIPList(), master) - if len(masterIPs) > 0 { - if err := k.deleteKubeNode(master); err != nil { - return fmt.Errorf("delete master %s failed %v", master, err) + return k.resetNode(master, func() { + //remove master + masterIPs := strings.SliceRemoveStr(k.getMasterIPList(), master) + if len(masterIPs) > 0 { + if err := k.RemoveNodeFromK8sClient(master); err != nil { + logger.Warn(fmt.Errorf("delete master %s failed %v", master, err)) + } } - } - - return k.resetNode(master) + }) } diff --git a/pkg/runtime/node.go b/pkg/runtime/node.go index 83e723295..635e66d93 100644 --- a/pkg/runtime/node.go +++ b/pkg/runtime/node.go @@ -122,11 +122,12 @@ func (k *KubeadmRuntime) deleteNodes(nodes []string) error { } func (k *KubeadmRuntime) deleteNode(node string) error { - //remove node - if len(k.getMasterIPList()) > 0 { - if err := k.deleteKubeNode(node); err != nil { - return fmt.Errorf("delete node %s failed %v", node, err) + return k.resetNode(node, func() { + //remove node + if len(k.getMasterIPList()) > 0 { + if err := k.RemoveNodeFromK8sClient(node); err != nil { + logger.Warn(fmt.Errorf("delete node %s failed %v", node, err)) + } } - } - return k.resetNode(node) + }) } diff --git a/pkg/runtime/reset.go b/pkg/runtime/reset.go index c80295dc6..0ec6cb8f7 100644 --- a/pkg/runtime/reset.go +++ b/pkg/runtime/reset.go @@ -29,7 +29,7 @@ const ( remoteCleanMasterOrNode = `if which kubeadm;then kubeadm reset -f %s;fi && \ rm -rf /etc/kubernetes/ && \ rm -rf /etc/cni && rm -rf /opt/cni && \ -rm -rf %s && ip link delete kube-ipvs0 +rm -rf %s && (ip link delete kube-ipvs0 >/dev/null 2>&1 || true) ` ) @@ -45,7 +45,7 @@ func (k *KubeadmRuntime) resetNodes(nodes []string) { for _, node := range nodes { node := node eg.Go(func() error { - if err := k.resetNode(node); err != nil { + if err := k.resetNode(node, nil); err != nil { logger.Error("delete node %s failed %v", node, err) } return nil @@ -59,31 +59,40 @@ func (k *KubeadmRuntime) resetNodes(nodes []string) { func (k *KubeadmRuntime) resetMasters(nodes []string) { logger.Info("start to reset masters: %v", nodes) for _, node := range nodes { - if err := k.resetNode(node); err != nil { + if err := k.resetNode(node, nil); err != nil { logger.Error("delete master %s failed %v", node, err) } } } -func (k *KubeadmRuntime) resetNode(node string) error { +func (k *KubeadmRuntime) resetNode(node string, cleanHook func()) error { logger.Info("start to reset node: %s", node) resetCmd := fmt.Sprintf(remoteCleanMasterOrNode, vlogToStr(k.vlog), k.getEtcdDataDir()) - if err := k.sshCmdAsync(node, resetCmd); err != nil { - logger.Error("failed to clean node, exec command %s failed, %v", resetCmd, err) + removeKubeConfigErr := k.sshCmdAsync(node, removeKubeConfig) + resetCmdErr := k.sshCmdAsync(node, resetCmd) + ipvscleanErr := k.execIPVSClean(node) + hostsDeleteAPIServerErr := k.execHostsDelete(node, k.getAPIServerDomain()) + hostsDeleteLvscareErr := k.execHostsDelete(node, constants.DefaultLvscareDomain) + + if cleanHook != nil { + cleanHook() } - if err := k.sshCmdAsync(node, removeKubeConfig); err != nil { - logger.Error("failed to clean node, exec command %s failed, %v", removeKubeConfig, err) + + if resetCmdErr != nil { + logger.Error("failed to clean node, exec command %s failed, %v", resetCmd, resetCmdErr) } - if err := k.execIPVSClean(node); err != nil { - logger.Error("failed to clean node route and ipvs failed, %v", err) + if removeKubeConfigErr != nil { + logger.Error("failed to clean node, exec command %s failed, %v", removeKubeConfig, removeKubeConfigErr) } - err := k.execHostsDelete(node, k.getAPIServerDomain()) - if err != nil { - logger.Error("delete apiserver hosts failed %v", err) + if ipvscleanErr != nil { + logger.Error("failed to clean node route and ipvs failed, %v", ipvscleanErr) } - err = k.execHostsDelete(node, constants.DefaultLvscareDomain) - if err != nil { - return fmt.Errorf("add lvscare domain hosts failed %v", err) + if hostsDeleteAPIServerErr != nil { + logger.Error("delete apiserver hosts failed %v", hostsDeleteAPIServerErr) } + if hostsDeleteLvscareErr != nil { + return fmt.Errorf("add lvscare domain hosts failed %v", hostsDeleteLvscareErr) + } + return nil } diff --git a/pkg/runtime/utils.go b/pkg/runtime/utils.go index 042c4918c..94fbef311 100644 --- a/pkg/runtime/utils.go +++ b/pkg/runtime/utils.go @@ -122,28 +122,24 @@ func (k *KubeadmRuntime) sendFileToHosts(Hosts []string, src, dst string) error return eg.Wait() } -func (k *KubeadmRuntime) deleteKubeNode(ip string) error { - var err error +func (k *KubeadmRuntime) RemoveNodeFromK8sClient(ip string) error { logger.Info("start to remove node from k8s %s", ip) cli, err := kubernetes.NewKubernetesClient(k.getContentData().AdminFile(), k.getMaster0IPAPIServer()) if err != nil { - logger.Warn("kubernetes client get node %s failed %v,skip delete node", ip, err) - return nil + return fmt.Errorf("kubernetes client get node %s failed %v,skip delete node", ip, err) } ctx := context.Background() hostname, err := kubernetes.GetHostNameFromInternalIP(cli.Kubernetes(), ip) if err != nil { - logger.Warn("kubernetes client get hostname %s failed %v,skip delete node", ip, err) - return nil + return fmt.Errorf("kubernetes client get hostname %s failed %v,skip delete node", ip, err) } deletePropagation := v1.DeletePropagationBackground err = cli.Kubernetes().CoreV1().Nodes().Delete(ctx, hostname, v1.DeleteOptions{PropagationPolicy: &deletePropagation}) if err != nil { if errors.IsNotFound(err) { - logger.Warn("not find target delete node ip: %s", ip) - return nil + return fmt.Errorf("not find target delete node ip: %s", ip) } - logger.Warn("kubernetes client delete node %s failed %v,skip delete node", ip, err) + return fmt.Errorf("kubernetes client delete node %s failed %v,skip delete node", ip, err) } return nil }