[WIP]Sealos kubeadm 1.23 v1beta3 (#673)

* fix #671

Signed-off-by: oldthreefeng <louisehong4168@gmail.com>

install/check.go

@@ -21,8 +21,7 @@ func (s *SealosInstaller) CheckValid() {
 	// 所有node节点
 	//nodes := append(Nodes, ParseIPs(NodeIPs)...)
 	//hosts := append(masters, nodes...)
-	var hosts []string
-	hosts = append(s.Masters, s.Nodes...)
+	var hosts = append(s.Masters, s.Nodes...)
 	if len(s.Hosts) == 0 && len(hosts) == 0 {
 		s.Print("Fail")
 		logger.Error("hosts not allow empty")

install/clean.go

@@ -155,7 +155,7 @@ func (s *SealosClean) cleanMaster(master string) {
 func clean(host string) {
 	cmd := "kubeadm reset -f " + vlogToStr()
 	_ = SSHConfig.CmdAsync(host, cmd)
-	cmd = fmt.Sprintf(`sed -i '/kubectl/d;/sealos/d' /root/.bashrc`)
+	cmd = `sed -i '/kubectl/d;/sealos/d' /root/.bashrc`
 	_ = SSHConfig.CmdAsync(host, cmd)
 	cmd = "modprobe -r ipip  && lsmod"
 	_ = SSHConfig.CmdAsync(host, cmd)
@@ -171,13 +171,13 @@ func clean(host string) {
 	_ = SSHConfig.CmdAsync(host, cmd)
 	cmd = fmt.Sprintf("sed -i \"/%s/d\" /etc/hosts ", ApiServer)
 	_ = SSHConfig.CmdAsync(host, cmd)
-	cmd = fmt.Sprint("rm -rf ~/kube")
+	cmd = "rm -rf ~/kube"
 	_ = SSHConfig.CmdAsync(host, cmd)
 	//clean pki certs
-	cmd = fmt.Sprint("rm -rf /etc/kubernetes/pki")
+	cmd = "rm -rf /etc/kubernetes/pki"
 	_ = SSHConfig.CmdAsync(host, cmd)
 	//clean sealos in /usr/bin/ except exec sealos
-	cmd = fmt.Sprint("ps -ef |grep -v 'grep'|grep sealos >/dev/null || rm -rf /usr/bin/sealos")
+	cmd = "ps -ef |grep -v 'grep'|grep sealos >/dev/null || rm -rf /usr/bin/sealos"
 	_ = SSHConfig.CmdAsync(host, cmd)
 }
 

install/constants.go

@@ -29,75 +29,47 @@ const (
 	DefaultContainerdCRISocket = "/run/containerd/containerd.sock"
 	DefaultCgroupDriver        = "cgroupfs"
 	DefaultSystemdCgroupDriver = "systemd"
+
+	KubeadmV1beta1   = "kubeadm.k8s.io/v1beta1"
+	KubeadmV1beta2   = "kubeadm.k8s.io/v1beta2"
+	KubeadmV1beta3   = "kubeadm.k8s.io/v1beta3"
+/*
+A list of changes since v1beta1:
+
+`certificateKey" field is added to InitConfiguration and JoinConfiguration.
+"ignorePreflightErrors" field is added to the NodeRegistrationOptions.
+The JSON "omitempty" tag is used in a more places where appropriate.
+The JSON "omitempty" tag of the "taints" field (inside NodeRegistrationOptions) is removed. See the Kubernetes 1.15 changelog for further details.
+
+
+A list of changes since v1beta2:
+
+The deprecated ClusterConfiguration.useHyperKubeImage field has been removed. Kubeadm no longer supports the hyperkube image.
+The ClusterConfiguration.dns.type field has been removed since CoreDNS is the only supported DNS server type by kubeadm.
+Include "datapolicy" tags on the fields that hold secrets. This would result in the field values to be omitted when API structures are printed with klog.
+Add InitConfiguration.skipPhases, JoinConfiguration.skipPhases to allow skipping a list of phases during kubeadm init/join command execution.
+Add InitConfiguration.nodeRegistration.imagePullPolicy" andJoinConfiguration.nodeRegistration.imagePullPolicy` to allow specifying the images pull policy during kubeadm "init" and "join". The value must be one of "Always", "Never" or "IfNotPresent". "IfNotPresent" is the default, which has been the existing behavior prior to this addition.
+Add InitConfiguration.patches.directory, JoinConfiguration.patches.directory to allow the user to configure a directory from which to take patches for components deployed by kubeadm.
+Move the BootstrapToken∗ API and related utilities out of the "kubeadm" API group to a new group "bootstraptoken". The kubeadm API version v1beta3 no longer contains the BootstrapToken∗ structures.
+
+*/
+	Bootstraptokenv1 = "bootstraptoken/v1"
 )
 
-const InitTemplateTextV1beta1 = string(`apiVersion: kubeadm.k8s.io/v1beta1
-kind: InitConfiguration
-localAPIEndpoint:
-  advertiseAddress: {{.Master0}}
-  bindPort: 6443
----
-apiVersion: kubeadm.k8s.io/v1beta1
-kind: ClusterConfiguration
-kubernetesVersion: {{.Version}}
-controlPlaneEndpoint: "{{.ApiServer}}:6443"
-imageRepository: {{.Repo}}
-networking:
-  # dnsDomain: cluster.local
-  podSubnet: {{.PodCIDR}}
-  serviceSubnet: {{.SvcCIDR}}
-apiServer:
-  certSANs:
-  - 127.0.0.1
-  - {{.ApiServer}}
-  {{range .Masters -}}
-  - {{.}}
-  {{end -}}
-  {{range .CertSANS -}}
-  - {{.}}
-  {{end -}}
-  - {{.VIP}}
-  extraArgs:
-    feature-gates: TTLAfterFinished=true
-  extraVolumes:
-  - name: localtime
-    hostPath: /etc/localtime
-    mountPath: /etc/localtime
-    readOnly: true
-    pathType: File
-controllerManager:
-  extraArgs:
-    feature-gates: TTLAfterFinished=true
-    experimental-cluster-signing-duration: 876000h
-{{- if eq .Network "cilium" }}
-    allocate-node-cidrs: \"true\"
-{{- end }}
-  extraVolumes:
-  - hostPath: /etc/localtime
-    mountPath: /etc/localtime
-    name: localtime
-    readOnly: true
-    pathType: File
-scheduler:
-  extraArgs:
-    feature-gates: TTLAfterFinished=true
-  extraVolumes:
-  - hostPath: /etc/localtime
-    mountPath: /etc/localtime
-    name: localtime
-    readOnly: true
-    pathType: File
----
-apiVersion: kubeproxy.config.k8s.io/v1alpha1
-kind: KubeProxyConfiguration
-mode: "ipvs"
-ipvs:
-  excludeCIDRs:
-  - "{{.VIP}}/32"
----
-` + kubeletConfigDefault)
+const (
+	InitTemplateText = string(InitConfigurationDefault +
+		ClusterConfigurationDefault +
+		kubeproxyConfigDefault +
+		kubeletConfigDefault)
+	JoinCPTemplateText = string(bootstrapTokenDefault +
+		JoinConfigurationDefault +
+		kubeletConfigDefault)
 
-const JoinCPTemplateTextV1beta2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
+	bootstrapTokenDefault = `{{- if eq .BootstrapApi "bootstraptoken/v1" }}
+apiVersion: {{.BootstrapApi}}
+  {{- else}}
+apiVersion: {{.KubeadmApi}}
+{{- end}}
 caCertPath: /etc/kubernetes/pki/ca.crt
 discovery:
   bootstrapToken:
@@ -110,6 +82,17 @@ discovery:
     caCertHashes:
     - {{.TokenDiscoveryCAHash}}
   timeout: 5m0s
+`
+	InitConfigurationDefault = `apiVersion: {{.KubeadmApi}}
+kind: InitConfiguration
+localAPIEndpoint:
+  advertiseAddress: {{.Master0}}
+  bindPort: 6443
+nodeRegistration:
+  criSocket: {{.CriSocket}}
+`
+
+	JoinConfigurationDefault = `
 kind: JoinConfiguration
 {{- if .Master }}
 controlPlane:
@@ -119,18 +102,10 @@ controlPlane:
 {{- end}}
 nodeRegistration:
   criSocket: {{.CriSocket}}
----
-` + kubeletConfigDefault)
+`
 
-const InitTemplateTextV1bate2 = string(`apiVersion: kubeadm.k8s.io/v1beta2
-kind: InitConfiguration
-localAPIEndpoint:
-  advertiseAddress: {{.Master0}}
-  bindPort: 6443
-nodeRegistration:
-  criSocket: /run/containerd/containerd.sock
----
-apiVersion: kubeadm.k8s.io/v1beta2
+	ClusterConfigurationDefault = `---
+apiVersion: {{.KubeadmApi}}
 kind: ClusterConfiguration
 kubernetesVersion: {{.Version}}
 controlPlaneEndpoint: "{{.ApiServer}}:6443"
@@ -180,6 +155,8 @@ scheduler:
     name: localtime
     readOnly: true
     pathType: File
+`
+	kubeproxyConfigDefault = `
 ---
 apiVersion: kubeproxy.config.k8s.io/v1alpha1
 kind: KubeProxyConfiguration
@@ -187,20 +164,10 @@ mode: "ipvs"
 ipvs:
   excludeCIDRs:
   - "{{.VIP}}/32"
+`
+	kubeletConfigDefault = `
 ---
-` + kubeletConfigDefault)
-
-const (
-	ContainerdShell = `if grep "SystemdCgroup = true"  /etc/containerd/config.toml &> /dev/null; then  
-driver=systemd
-else
-driver=cgroupfs
-fi
-echo ${driver}`
-	DockerShell = `driver=$(docker info -f "{{.CgroupDriver}}")
-	echo "${driver}"`
-
-  kubeletConfigDefault = `apiVersion: kubelet.config.k8s.io/v1beta1
+apiVersion: kubelet.config.k8s.io/v1beta1
 kind: KubeletConfiguration
 authentication:
   anonymous:
@@ -269,4 +236,13 @@ staticPodPath: /etc/kubernetes/manifests
 streamingConnectionIdleTimeout: 4h0m0s
 syncFrequency: 1m0s
 volumeStatsAggPeriod: 1m0s`
+
+	ContainerdShell = `if grep "SystemdCgroup = true"  /etc/containerd/config.toml &> /dev/null; then  
+driver=systemd
+else
+driver=cgroupfs
+fi
+echo ${driver}`
+	DockerShell = `driver=$(docker info -f "{{.CgroupDriver}}")
+	echo "${driver}"`
 )

install/etcd_save.go

@@ -193,12 +193,11 @@ func GetEtcdClient(ep []string) (*clientv3.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-	cli, err := clientv3.New(clientv3.Config{
+	return clientv3.New(clientv3.Config{
 		Endpoints:   ep,
 		DialTimeout: 5 * time.Second,
 		TLS:         clientTLS,
 	})
-	return cli, nil
 }
 
 type epHealth struct {

install/generator.go

@@ -11,6 +11,29 @@ import (
 
 var ConfigType string
 
+func setKubeadmApi(version string) {
+	major, _ := GetMajorMinorInt(version)
+	switch {
+	//
+	case major < 120:
+		KubeadmApi = KubeadmV1beta1
+		CriSocket = DefaultDockerCRISocket
+	case major < 123 && major >= 120:
+		KubeadmApi = KubeadmV1beta2
+		CriSocket = DefaultContainerdCRISocket
+	case major >= 123:
+		KubeadmApi = KubeadmV1beta3
+		CriSocket = DefaultContainerdCRISocket
+		BootstrapApi = Bootstraptokenv1
+	default:
+		KubeadmApi = KubeadmV1beta3
+		CriSocket = DefaultContainerdCRISocket
+		BootstrapApi = Bootstraptokenv1
+	}
+	logger.Debug("KubeadmApi: %s", KubeadmApi)
+	logger.Debug("CriSocket: %s", CriSocket)
+}
+
 func Config() {
 	switch ConfigType {
 	case "kubeadm":
@@ -24,7 +47,7 @@ func Config() {
 
 func joinKubeadmConfig() string {
 	var sb strings.Builder
-	sb.Write([]byte(JoinCPTemplateTextV1beta2))
+	sb.Write([]byte(JoinCPTemplateText))
 	return sb.String()
 }
 
@@ -34,13 +57,7 @@ func printlnJoinKubeadmConfig() {
 
 func kubeadmConfig() string {
 	var sb strings.Builder
-	// kubernetes gt 1.20, use Containerd instead of docker
-	if For120(Version) {
-		sb.Write([]byte(InitTemplateTextV1bate2))
-	} else {
-		sb.Write([]byte(InitTemplateTextV1beta1))
-	}
-
+	sb.Write([]byte(InitTemplateText))
 	return sb.String()
 }
 
@@ -59,6 +76,7 @@ func JoinTemplate(ip string, cgroup string) []byte {
 }
 
 func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte {
+	setKubeadmApi(Version)
 	tmpl, err := template.New("text").Parse(templateContent)
 	defer func() {
 		if r := recover(); r != nil {
@@ -74,11 +92,7 @@ func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte
 	envMap["TokenDiscovery"] = JoinToken
 	envMap["TokenDiscoveryCAHash"] = TokenCaCertHash
 	envMap["VIP"] = VIP
-	if For120(Version) {
-		CriSocket = DefaultContainerdCRISocket
-	} else {
-		CriSocket = DefaultDockerCRISocket
-	}
+	envMap["KubeadmApi"] = KubeadmApi
 	envMap["CriSocket"] = CriSocket
 	envMap["CgroupDriver"] = cgroup
 	var buffer bytes.Buffer
@@ -87,6 +101,7 @@ func JoinTemplateFromTemplateContent(templateContent, ip, cgroup string) []byte
 }
 
 func TemplateFromTemplateContent(templateContent string) []byte {
+	setKubeadmApi(Version)
 	tmpl, err := template.New("text").Parse(templateContent)
 	defer func() {
 		if r := recover(); r != nil {
@@ -113,6 +128,8 @@ func TemplateFromTemplateContent(templateContent string) []byte {
 	envMap["Master0"] = IpFormat(MasterIPs[0])
 	envMap["Network"] = Network
 	envMap["CgroupDriver"] = CgroupDriver
+	envMap["KubeadmApi"] = KubeadmApi
+	envMap["CriSocket"] = CriSocket
 	var buffer bytes.Buffer
 	_ = tmpl.Execute(&buffer, envMap)
 	return buffer.Bytes()

install/generator_test.go

@@ -32,6 +32,7 @@ func TestNetCiliumTemplate(t *testing.T) {
 	ApiServer = "apiserver.cluster.local"
 	Version = "1.20.5"
 	Network = "cilium"
+	CgroupDriver = DefaultCgroupDriver
 	t.Log(string(Template()))
 	Network = "calico"
 	t.Log(string(Template()))
@@ -39,6 +40,7 @@ func TestNetCiliumTemplate(t *testing.T) {
 	Network = "cilium"
 	t.Log(string(Template()))
 	Network = "calico"
+	CgroupDriver = DefaultSystemdCgroupDriver
 	t.Log(string(Template()))
 }
 
@@ -148,19 +150,3 @@ func TestJoinTemplate(t *testing.T) {
 	config.Cmd("127.0.0.1", "echo \""+string(JoinTemplate("", "systemd"))+"\" > ~/aa")
 	t.Log(string(JoinTemplate("", "cgroupfs")))
 }
-
-var tepJoin = `apiVersion: kubeadm.k8s.io/v1beta2
-caCertPath: /etc/kubernetes/pki/ca.crt
-discovery:
-  bootstrapToken: 
-    apiServerEndpoint: {{.Master0}}:6443
-    token: {{.TokenDiscovery}}
-    caCertHashes: 
-    - {{.TokenDiscoveryCAHash}}
-  timeout: 5m0s
-kind: JoinConfiguration
-controlPlane:
-  localAPIEndpoint:
-    advertiseAddress: {{.Master}}
-    bindPort: 6443
-`

install/init.go

@@ -193,7 +193,7 @@ func (s *SealosInstaller) InstallMaster0() {
 	decodeOutput(output)
 
 	cmd = `mkdir -p /root/.kube && cp /etc/kubernetes/admin.conf /root/.kube/config && chmod 600 /root/.kube/config`
-	output = SSHConfig.Cmd(s.Masters[0], cmd)
+	SSHConfig.Cmd(s.Masters[0], cmd)
 
 	if WithoutCNI {
 		logger.Info("--without-cni is true, so we not install calico or flannel, install it by yourself")
@@ -236,7 +236,7 @@ func (s *SealosInstaller) InstallMaster0() {
 	configYamlDir := filepath.Join(home, ".sealos", "cni.yaml")
 	ioutil.WriteFile(configYamlDir, []byte(netyaml), 0755)
 	SSHConfig.Copy(s.Masters[0], configYamlDir, "/tmp/cni.yaml")
-	output = SSHConfig.Cmd(s.Masters[0], "kubectl apply -f /tmp/cni.yaml")
+	SSHConfig.Cmd(s.Masters[0], "kubectl apply -f /tmp/cni.yaml")
 }
 
 //SendKubeConfigs

install/join.go

@@ -92,8 +92,7 @@ func (s *SealosInstaller) sendJoinCPConfig(joinMaster []string) {
 		wg.Add(1)
 		go func(master string) {
 			defer wg.Done()
-			var cgroup string
-			cgroup = s.getCgroupDriverFromShell(master)
+			cgroup := s.getCgroupDriverFromShell(master)
 			templateData := string(JoinTemplate(IpFormat(master), cgroup))
 			cmd := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
 			_ = SSHConfig.CmdAsync(master, cmd)
@@ -149,8 +148,7 @@ func (s *SealosInstaller) JoinNodes() {
 		go func(node string) {
 			defer wg.Done()
 			// send join node config
-			var cgroup string
-			cgroup = s.getCgroupDriverFromShell(node)
+			cgroup := s.getCgroupDriverFromShell(node)
 			templateData := string(JoinTemplate("", cgroup))
 			cmdJoinConfig := fmt.Sprintf(`echo "%s" > /root/kubeadm-join-config.yaml`, templateData)
 			_ = SSHConfig.CmdAsync(node, cmdJoinConfig)

install/upgrade.go

@@ -125,7 +125,7 @@ func (u *SealosUpgrade) upgradeNodes(hostnames []string, isMaster bool) {
 					os.Exit(1)
 				}
 			} else {
-				cmdUpgrade = fmt.Sprintf("kubeadm upgrade node --certificate-renewal=false")
+				cmdUpgrade = "kubeadm upgrade node --certificate-renewal=false"
 				err = SSHConfig.CmdAsync(ip, cmdUpgrade)
 				if err != nil {
 					logger.Error("kubeadm upgrade err: ", err)

install/utils.go

@@ -159,8 +159,7 @@ func IpFormat(host string) string {
 
 // RandString 生成随机字符串
 func RandString(len int) string {
-	var r *rand.Rand
-	r = rand.New(rand.NewSource(time.Now().Unix()))
+	var r *rand.Rand = rand.New(rand.NewSource(time.Now().Unix()))
 	bytes := make([]byte, len)
 	for i := 0; i < len; i++ {
 		b := r.Intn(26) + 65

install/vars.go

@@ -29,8 +29,10 @@ var (
 	EtcdKey      = cert.SealosConfigDir + "/pki/etcd/healthcheck-client.key"
 
 	//criSocket
-	CriSocket string
+	CriSocket    string
 	CgroupDriver string
+	KubeadmApi   string
+	BootstrapApi string
 
 	VIP     string
 	PkgUrl  string