From 62ea87efeccdb377e2752785d62ec04f203b6bc7 Mon Sep 17 00:00:00 2001 From: oldthreefeng Date: Sun, 13 Sep 2020 23:44:32 +0800 Subject: [PATCH] add sealos route cmd docs use github.com/vishvananda/netlink to set default route gateway instead of ip route command fix syntax refactor multi iface solution delete something fix to origin, fix docs stage and rebase --- cmd/route.go | 85 ++++++++++++++++++++++ docs/route.md | 149 ++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 3 + install/check.go | 17 +++++ install/clean.go | 12 +++ install/generator.go | 59 +++++++++++++++ install/generator_test.go | 33 +++++++++ install/init.go | 3 +- install/join.go | 29 ++++++++ install/route.go | 113 +++++++++++++++++++++++++++++ install/sealos.go | 4 +- install/send.go | 2 + install/vars.go | 3 - 14 files changed, 507 insertions(+), 6 deletions(-) create mode 100644 cmd/route.go create mode 100644 docs/route.md create mode 100644 install/route.go diff --git a/cmd/route.go b/cmd/route.go new file mode 100644 index 000000000..0554cc94a --- /dev/null +++ b/cmd/route.go @@ -0,0 +1,85 @@ +/* +Copyright © 2020 NAME HERE + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ +package cmd + +import ( + "github.com/fanux/sealos/install" + "github.com/spf13/cobra" +) + + + +var ( + host string + gatewayIp string +) + +func NewRouteCmd() *cobra.Command { + // routeCmd represents the route command + var cmd = &cobra.Command{ + Use: "route", + Short: "set default route gateway", + Run: RouteCmdFunc, + } + // check route for host + cmd.Flags().StringVar(&host, "host", "", "route host ip address for iFace") + cmd.AddCommand(NewDelRouteCmd()) + cmd.AddCommand(NewAddRouteCmd()) + return cmd +} + +func init() { + rootCmd.AddCommand(NewRouteCmd()) +} + +func NewAddRouteCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "add", + Short: "set route host via gateway", + Run: RouteAddCmdFunc, + } + // manually to set host via gateway + cmd.Flags().StringVar(&host, "host", "", "route host ,ex ip route add host via gateway") + cmd.Flags().StringVar(&gatewayIp, "gateway", "", "route gateway ,ex ip route add host via gateway") + return cmd +} + +func NewDelRouteCmd() *cobra.Command { + var cmd = &cobra.Command{ + Use: "del", + Short: "del route host via gateway, like ip route del host via gateway", + Run: RouteDelCmdFunc, + } + // manually to set host via gateway + cmd.Flags().StringVar(&host, "host", "", "route host ,ex ip route del host via gateway") + cmd.Flags().StringVar(&gatewayIp, "gateway", "", "route gateway ,ex ip route del host via gateway") + return cmd +} + +func RouteCmdFunc(cmd *cobra.Command, args []string) { + r := install.GetRouteFlag(host, gatewayIp) + r.CheckRoute() +} + +func RouteAddCmdFunc(cmd *cobra.Command, args []string) { + r := install.GetRouteFlag(host, gatewayIp) + r.SetRoute() +} + +func RouteDelCmdFunc(cmd *cobra.Command, args []string) { + r := install.GetRouteFlag(host, gatewayIp) + r.DelRoute() +} \ No newline at end of file diff --git a/docs/route.md b/docs/route.md new file mode 100644 index 000000000..46ea827fb --- /dev/null +++ b/docs/route.md @@ -0,0 +1,149 @@ +# `sealos route` 命令 + +## 使用`sealos route` + +> 这个命令产生的原因， 是在双网卡甚至多网卡的主机上， 使用`sealos init`安装 `kubernetes` 集群失败。 +> +> kubernetes 集群在使用`kubeadm init`安装的时候，会读取默认路由条目下的ip作为 `apiserver-advertise-address`和`etcd`的`advertise-address` +> +> 使用`localAPIEndpoint`来定义即可。 lvscare了创建 ipvs 在多网卡的情况下， +> node节点但是得添加一个 `10.103.97.2`到非默认路由 ip的 一个路由. 不然ipvs 不通 +> +> 对比目前ip和默认路由ip， 来判断是否需要进行添加路由操作。 + +``` +$ sealos route -h +set route gateway + +Usage: + sealos route [flags] + sealos route [command] + +Available Commands: + add set route host via gateway, like ip route add host via gateway + del del route host via gateway, like ip route del host via gateway + +Flags: + -h, --help help for route + --host string route host ip address for iFace + +Global Flags: + --config string config file (default is $HOME/.sealos/config.yaml) + +Use "sealos route [command] --help" for more information about a command. +``` + +说明一下选项 + +`sealos route --host` : 根据这个`--host ip`来比对当前的默认路由所绑定的`default ip`，来进行判断是否是默认路由的ip。 + + +like linux command `ip route add host via gateway` + +- `sealos route add --gateway --host`: 主机上的路由gateway ip， 如 `192.168.0.1`， 目标地址的host`192.168.0.23`， 这个gateway将通过`host`添加到路由条目， + + +like linux command `ip route del host via gateway` + +- `sealos route del --gateway --host`: 主机上的路由gateway ip， 如 `192.168.0.1`， 目标地址的host`192.168.0.23`， 将匹配这条路由删除掉。 +### 演示 + +演示主机默认路由为 `192.168.253.1` 。 + +名称|网卡1|网卡2 +---|---|--- +ip|192.168.160.243|192.168.253.129 +路由|192.168.160.1|192.168.253.1 + + +use `--host` to check default route gateway, only support ipv4 + +根据这个`--host ip`来比对当前的默认路由所绑定的`default ip`， + +``` +$ sealos route --host 192.168.253.129 + ok +$ sealos route --host 192.168.253.128 + failed +$ sealos route --host 192.168.160.243 + failed +``` + +use `add --gateway --host ` to set default route gateway, only support ipv4 + +``` +$ sealos route add --host 10.103.97.2 --gateway 192.168.253.129 +$ ip route show |grep 10.103.97.2 +10.103.97.2 via 192.168.253.129 dev ens32 + +## show nothing. when no error +$ sealos route del --host 10.103.97.2 --gateway 192.168.253.129 +$ ip route show |grep 10.103.97.2 + +## exec twice +$ sealos route del --host 10.103.97.2 --gateway 192.168.253.129 +delRouteGatewayViaHost err: no such process +``` + +## 实现原理 + +通过和默认的路由条目进行比对. 使用 "github.com/vishvananda/netlink"命令操作`linux`主机默认路由。 + + +获取默认路由`ip`， 使用的是`"k8s.io/apimachinery/pkg/util/net"`这个包里面的` ChooseHostInterface()`来获取. +像`apiserver`和`etcd`的`advertise-address`都是这个方法获取。 目前只支持ipv4. + +```go +func (r *RouteFlags) useHostCheckRoute() bool { + return k8s.IsIpv4(r.Host) && r.Gateway =="" +} + +func (r *RouteFlags) useGatewayManageRoute() bool { + return k8s.IsIpv4(r.Gateway) && k8s.IsIpv4(r.Host) +} + + + +// getDefaultRouteIp is get host ip by ChooseHostInterface() . +func getDefaultRouteIp() (ip string, err error) { + netIp, err := k8snet.ChooseHostInterface() + if err != nil { + return "", err + } + return netIp.String(), nil +} +``` + +具体详情请查看源码`"github.com/fanux/sealos/install/route.go"` + +### 配合和`sealos init`使用 + +这里简要描述一下init逻辑： + +1. check ， 检查主机名是否重复 +2. senSealos, 将最新的sealos复制到 /usr/sbin/ +3. sendPackage， 将压缩kube1.**.tar.gz包复制到各主机 +4. KubeadmConfigInstall, 生成`kubeadm-comfig` +5. GenerateCert, 生成`pki`证书 +6. CreateKubeconfig, 生成`kube-×.config`文件 +7. InstallMaster0, 安装第一个`master`节点。 +8. 安装其他子节点。 优先`master`， 再`node` 。 + + 8.1 安装master， 采用`kubeadm join --config=/root/kubeadm-join-config.yaml` + + 8.2 安装node节点，对当前的网卡进行判断， 如果是单网卡跳过， 如果是多网卡， 且安装的node ip 不是本机的默认路由ip， + 添加一条路由 例如`ip route add 10.103.97.2 via node ip` + +所以， 我们只需要在2.1的时候， 改变主机的默认路由即可， 这里需要增加`sealos init --gateway`参数。 +通过`--gateway` 是否存在来判断需要进行改变默认路由。用户未使用`--gateway`， 则跳过设置路由节点。 + +```go +// 如果不是默认路由， 则添加 vip 到 master的路由。 + cmdRoute := fmt.Sprintf("/usr/sbin/sealos route --host %s", IpFormat(node)) + status := SSHConfig.CmdToString(node, cmdRoute, "") + if status != "ok" { + // 以自己的ip作为路由网关 + addRouteCmd := fmt.Sprintf("/usr/sbin/sealos route add --host %s --gateway %s", VIP, IpFormat(node)) + SSHConfig.CmdToString(node, addRouteCmd, "") + } +``` diff --git a/go.mod b/go.mod index 9a3e2bee7..5e0ad3e01 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/satori/go.uuid v1.2.0 // indirect github.com/spf13/cobra v0.0.6 github.com/spf13/viper v1.6.2 + github.com/vishvananda/netlink v1.1.0 github.com/wonderivan/logger v1.0.0 go.etcd.io/etcd v0.0.0-20200716221620-18dfb9cca345 go.uber.org/zap v1.10.0 diff --git a/go.sum b/go.sum index 47b70ce51..e79133346 100644 --- a/go.sum +++ b/go.sum @@ -291,6 +291,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vishvananda/netlink v1.0.0 h1:bqNY2lgheFIu1meHUFSH3d7vG93AFyqg3oGbJCOJgSM= github.com/vishvananda/netlink v1.0.0/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= +github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936 h1:J9gO8RJCAFlln1jsvRba/CWVUnMHwObklfxxjErl1uk= github.com/vishvananda/netns v0.0.0-20171111001504-be1fbeda1936/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= @@ -357,6 +359,7 @@ golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456 h1:ng0gs1AKnRRuEMZoTLLlbOd+C17zUDepwGQBb/n+JVg= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7 h1:HmbHVPwrPEKPGLAcHSrMe6+hqSUlvZU0rab6x5EXfGU= diff --git a/install/check.go b/install/check.go index acf815704..bbc5bd92f 100644 --- a/install/check.go +++ b/install/check.go @@ -13,6 +13,23 @@ func SetHosts(hostip, hostName string) { } +func (s *SealosInstaller) CheckRoute() { + var hosts []string + hosts = append(hosts, s.Masters...) + for _, host := range hosts { + var cmd string + cmd = fmt.Sprintf("/usr/sbin/sealos route --host %s", IpFormat(host)) + status := SSHConfig.CmdToString(host, cmd, "") + if status != "ok" { + logger.Error(`check route err, please use --gateway to change your default gateway link your ip to solve. +more about multi interface to get from https://github.com/fanux/sealos/docs/route.md`) + os.Exit(1) + } + } + +} + + //CheckValid is func (s *SealosInstaller) CheckValid() { //hosts := append(Masters, Nodes...) diff --git a/install/clean.go b/install/clean.go index 8f86056f2..56e5f668d 100644 --- a/install/clean.go +++ b/install/clean.go @@ -111,6 +111,7 @@ func (s *SealosClean) Clean() { } func (s *SealosClean) cleanNode(node string) { + cleanRoute(node) clean(node) //remove node NodeIPs = SliceRemoveStr(NodeIPs, node) @@ -178,3 +179,14 @@ func clean(host string) { cmd = fmt.Sprint("rm -rf /usr/sbin/sealos") _ = SSHConfig.CmdAsync(host, cmd) } + +func cleanRoute(node string) { + // clean route + cmdRoute := fmt.Sprintf("/usr/sbin/sealos route --host %s", IpFormat(node)) + status := SSHConfig.CmdToString(node, cmdRoute, "") + if status != "ok" { + // 以自己的ip作为路由网关 + addRouteCmd := fmt.Sprintf("/usr/sbin/sealos route del --host %s --gateway %s", VIP, IpFormat(node)) + SSHConfig.CmdToString(node, addRouteCmd, "") + } +} \ No newline at end of file diff --git a/install/generator.go b/install/generator.go index 9e0335ead..d8de0f751 100644 --- a/install/generator.go +++ b/install/generator.go @@ -10,6 +10,12 @@ import ( ) const TemplateText = string(`apiVersion: kubeadm.k8s.io/v1beta1 +kind: InitConfiguration +localAPIEndpoint: + advertiseAddress: {{.Master0}} + bindPort: 6443 +--- +apiVersion: kubeadm.k8s.io/v1beta1 kind: ClusterConfiguration kubernetesVersion: {{.Version}} controlPlaneEndpoint: "{{.ApiServer}}:6443" @@ -64,17 +70,44 @@ ipvs: excludeCIDRs: - "{{.VIP}}/32"`) +const JoinCPTemplateText = string(`apiVersion: kubeadm.k8s.io/v1beta2 +caCertPath: /etc/kubernetes/pki/ca.crt +discovery: + bootstrapToken: + apiServerEndpoint: {{.Master0}}:6443 + token: {{.TokenDiscovery}} + caCertHashes: + - {{.TokenDiscoveryCAHash}} + timeout: 5m0s +kind: JoinConfiguration +controlPlane: + localAPIEndpoint: + advertiseAddress: {{.Master}} + bindPort: 6443`) + var ConfigType string func Config() { switch ConfigType { case "kubeadm": printlnKubeadmConfig() + case "join": + printlnJoinKubeadmConfig() default: printlnKubeadmConfig() } } +func joinKubeadmConfig() string { + var sb strings.Builder + sb.Write([]byte(JoinCPTemplateText)) + return sb.String() +} + +func printlnJoinKubeadmConfig() { + fmt.Println(joinKubeadmConfig()) +} + func kubeadmConfig() string { var sb strings.Builder sb.Write([]byte(TemplateText)) @@ -90,6 +123,31 @@ func Template() []byte { return TemplateFromTemplateContent(kubeadmConfig()) } +// JoinTemplate is generate JoinCP nodes configuration by master ip. +func JoinTemplate(ip string) []byte { + return JoinTemplateFromTemplateContent(joinKubeadmConfig(), ip) +} + +func JoinTemplateFromTemplateContent(templateContent, ip string) []byte { + tmpl, err := template.New("text").Parse(templateContent) + defer func() { + if r := recover(); r != nil { + logger.Error("join template parse failed:", err) + } + }() + if err != nil { + panic(1) + } + var envMap = make(map[string]interface{}) + envMap["Master0"] = IpFormat(MasterIPs[0]) + envMap["Master"] = IpFormat(ip) + envMap["TokenDiscovery"] = JoinToken + envMap["TokenDiscoveryCAHash"] = TokenCaCertHash + var buffer bytes.Buffer + _ = tmpl.Execute(&buffer, envMap) + return buffer.Bytes() +} + func TemplateFromTemplateContent(templateContent string) []byte { tmpl, err := template.New("text").Parse(templateContent) defer func() { @@ -114,6 +172,7 @@ func TemplateFromTemplateContent(templateContent string) []byte { envMap["PodCIDR"] = PodCIDR envMap["SvcCIDR"] = SvcCIDR envMap["Repo"] = Repo + envMap["Master0"] = IpFormat(MasterIPs[0]) var buffer bytes.Buffer _ = tmpl.Execute(&buffer, envMap) return buffer.Bytes() diff --git a/install/generator_test.go b/install/generator_test.go index bd9f139c5..cae528865 100644 --- a/install/generator_test.go +++ b/install/generator_test.go @@ -107,3 +107,36 @@ func TestKubeadmDataFromYaml(t *testing.T) { }) } } + +func TestJoinTemplate(t *testing.T) { + var masters = []string{"192.168.160.243:22"} + var vip = "10.103.97.1" + config := sshutil.SSH{ + User: "louis", + Password: "210010", + } + MasterIPs = masters + JoinToken = "1y6yyl.ramfafiy99vz3tbw" + TokenCaCertHash = "sha256:a68c79c87368ff794ae50c5fd6a8ce13fdb2778764f1080614ddfeaa0e2b9d14" + + + VIP = vip + config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate(masters[0]))+"\" > ~/aa") + t.Log(string(JoinTemplate(masters[0]))) +} + +var tepJoin = `apiVersion: kubeadm.k8s.io/v1beta2 +caCertPath: /etc/kubernetes/pki/ca.crt +discovery: + bootstrapToken: + apiServerEndpoint: {{.Master0}}:6443 + token: {{.TokenDiscovery}} + caCertHashes: + - {{.TokenDiscoveryCAHash}} + timeout: 5m0s +kind: JoinConfiguration +controlPlane: + localAPIEndpoint: + advertiseAddress: {{.Master}} + bindPort: 6443 +` \ No newline at end of file diff --git a/install/init.go b/install/init.go index 59277d24f..4f1906c62 100644 --- a/install/init.go +++ b/install/init.go @@ -67,7 +67,8 @@ func (s *SealosInstaller) KubeadmConfigInstall() { } templateData = string(TemplateFromTemplateContent(string(fileData))) } - cmd := "echo \"" + templateData + "\" > /root/kubeadm-config.yaml" + cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-config.yaml`, templateData) + //cmd := "echo \"" + templateData + "\" > /root/kubeadm-config.yaml" _ = SSHConfig.CmdAsync(s.Masters[0], cmd) //读取模板数据 kubeadm := KubeadmDataFromYaml(templateData) diff --git a/install/join.go b/install/join.go index 77ead34fd..6faba45b1 100644 --- a/install/join.go +++ b/install/join.go @@ -84,6 +84,21 @@ func getApiserverHost(ipAddr string) (host string) { return fmt.Sprintf("%s %s", ipAddr, ApiServer) } +// sendJoinCPConfig send join CP nodes configuration +func sendJoinCPConfig(joinMaster []string) { + var wg sync.WaitGroup + for _, master := range joinMaster { + wg.Add(1) + go func(master string) { + defer wg.Done() + templateData := string(JoinTemplate(master)) + cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`,templateData) + _ = SSHConfig.CmdAsync(master, cmd) + }(master) + } + wg.Wait() +} + //JoinMasters is func (s *SealosInstaller) JoinMasters(masters []string) { var wg sync.WaitGroup @@ -91,6 +106,10 @@ func (s *SealosInstaller) JoinMasters(masters []string) { s.sendCaAndKey(masters) s.SendKubeConfigs(masters, false) + + // send CP nodes configuration + sendJoinCPConfig(masters) + //join master do sth cmd := s.Command(Version, JoinMaster) for _, master := range masters { @@ -131,6 +150,16 @@ func (s *SealosInstaller) JoinNodes() { defer wg.Done() cmdHosts := fmt.Sprintf("echo %s %s >> /etc/hosts", VIP, ApiServer) _ = SSHConfig.CmdAsync(node, cmdHosts) + + // 如果不是默认路由， 则添加 vip 到 master的路由。 + cmdRoute := fmt.Sprintf("/usr/sbin/sealos route --host %s", IpFormat(node)) + status := SSHConfig.CmdToString(node, cmdRoute, "") + if status != "ok" { + // 以自己的ip作为路由网关 + addRouteCmd := fmt.Sprintf("/usr/sbin/sealos route add --host %s --gateway %s", VIP, IpFormat(node)) + SSHConfig.CmdToString(node, addRouteCmd, "") + } + _ = SSHConfig.CmdAsync(node, ipvsCmd) // create ipvs rules before we join node cmd := s.Command(Version, JoinNode) //create lvscare static pod diff --git a/install/route.go b/install/route.go new file mode 100644 index 000000000..4e7a7efb1 --- /dev/null +++ b/install/route.go @@ -0,0 +1,113 @@ +package install + +import ( + "fmt" + "github.com/fanux/sealos/k8s" + "github.com/vishvananda/netlink" + k8snet "k8s.io/apimachinery/pkg/util/net" + "net" +) + +type RouteFlags struct { + Host string + Gateway string +} + +func GetRouteFlag(host, gateway string) *RouteFlags { + return &RouteFlags{ + Host: host, + Gateway: gateway, + } + +} + +func (r *RouteFlags) useHostCheckRoute() bool { + return k8s.IsIpv4(r.Host) && r.Gateway == "" +} + +func (r *RouteFlags) useGatewayManageRoute() bool { + return k8s.IsIpv4(r.Gateway) && k8s.IsIpv4(r.Host) +} + +func (r *RouteFlags) CheckRoute() { + if r.useHostCheckRoute() { + if isDefaultRouteIp(r.Host) { + fmt.Println("ok") + return + } else { + fmt.Println("failed") + } + } +} + +func (r *RouteFlags) SetRoute() { + if r.useGatewayManageRoute() { + err := addRouteGatewayViaHost(r.Host, r.Gateway, 50) + if err != nil { + fmt.Println("addRouteGatewayViaHost err: ", err) + } + } +} + +func (r *RouteFlags) DelRoute() { + if r.useGatewayManageRoute() { + err := delRouteGatewayViaHost(r.Host, r.Gateway) + if err != nil { + fmt.Println("delRouteGatewayViaHost err: ", err) + } + } +} + + +// getDefaultRouteIp is get host ip by ChooseHostInterface() . +func getDefaultRouteIp() (ip string, err error) { + netIp, err := k8snet.ChooseHostInterface() + if err != nil { + return "", err + } + return netIp.String(), nil +} + +// isDefaultRouteIp return true if host equal default route ip host. +func isDefaultRouteIp(host string) bool { + ip, _ := getDefaultRouteIp() + return ip == host +} + + +// addRouteGatewayViaHost host: 10.103.97.2 gateway 192.168.253.129 +func addRouteGatewayViaHost(host, gateway string, priority int) error { + Dst := &net.IPNet{ + IP: net.ParseIP(host), + Mask: net.CIDRMask(32, 32), + } + r := netlink.Route{ + Dst: Dst, + Gw: net.ParseIP(gateway), + Priority: priority, + } + return netlink.RouteAdd(&r) +} + +// addRouteGatewayViaHost host: 10.103.97.2 gateway 192.168.253.129 +func delRouteGatewayViaHost(host, gateway string) error { + Dst := &net.IPNet{ + IP: net.ParseIP(host), + Mask: net.CIDRMask(32, 32), + } + r := netlink.Route{ + Dst: Dst, + Gw: net.ParseIP(gateway), + } + return netlink.RouteDel(&r) +} + +func getDefaultGatewayIp() string { + routes, _ := netlink.RouteList(nil, netlink.FAMILY_V4) + for _, r := range routes { + if r.Dst == nil { + return r.Gw.String() + } + } + return "" +} diff --git a/install/sealos.go b/install/sealos.go index 4c2871bcb..af4f7cb7d 100644 --- a/install/sealos.go +++ b/install/sealos.go @@ -56,14 +56,14 @@ func (s *SealosInstaller) Command(version string, name CommandType) (cmd string) cmds := make(map[CommandType]string) cmds = map[CommandType]string{ InitMaster: `kubeadm init --config=/root/kubeadm-config.yaml --experimental-upload-certs` + vlogToStr(), - JoinMaster: fmt.Sprintf("kubeadm join %s:6443 --token %s --discovery-token-ca-cert-hash %s --experimental-control-plane --certificate-key %s"+vlogToStr(), IpFormat(s.Masters[0]), JoinToken, TokenCaCertHash, CertificateKey), + JoinMaster: "kubeadm join %s:6443 --config=/root/kubeadm-join-config.yaml "+vlogToStr(), JoinNode: fmt.Sprintf("kubeadm join %s:6443 --token %s --discovery-token-ca-cert-hash %s"+vlogToStr(), VIP, JoinToken, TokenCaCertHash), } //other version //todo if VersionToInt(version) >= 115 { cmds[InitMaster] = `kubeadm init --config=/root/kubeadm-config.yaml --upload-certs` + vlogToStr() - cmds[JoinMaster] = fmt.Sprintf("kubeadm join %s:6443 --token %s --discovery-token-ca-cert-hash %s --control-plane --certificate-key %s"+vlogToStr(), IpFormat(s.Masters[0]), JoinToken, TokenCaCertHash, CertificateKey) + cmds[JoinMaster] = "kubeadm join --config=/root/kubeadm-join-config.yaml "+vlogToStr() } v, ok := cmds[name] diff --git a/install/send.go b/install/send.go index 39ca2d159..268dc7f47 100644 --- a/install/send.go +++ b/install/send.go @@ -16,6 +16,8 @@ func (s *SealosInstaller) SendPackage() { PkgUrl = SendPackage(PkgUrl, s.Hosts, "/root", nil, &kubeHook) + + // override sealos to avoid old version problem sealos := FetchSealosAbsPath() beforeHook := "ps -ef |grep -v 'grep'|grep sealos >/dev/null || rm -rf /usr/bin/sealos" diff --git a/install/vars.go b/install/vars.go index 1ebd08113..3dbf115f7 100644 --- a/install/vars.go +++ b/install/vars.go @@ -61,9 +61,6 @@ var ( SnapshotName string EtcdBackDir string RestorePath string - EtcdSnapshotSave bool - EtcdRestore bool - EtcdHealthCheck bool // oss OssEndpoint string