feat: k3s runtime (#3787)

* style: remove getter functions in pkg/runtime and rename ssh functions names

* style: rename GetKubeadmConfig -> GetRuntimeConfig

Signed-off-by: cuisongliu <cuisongliu@qq.com>

# Conflicts:
#	pkg/guest/guest.go

* feat: k3s runtime implementation

Signed-off-by: cuisongliu <cuisongliu@qq.com>

* Update utils.go

---------

Signed-off-by: cuisongliu <cuisongliu@qq.com>
Co-authored-by: cuisongliu <cuisongliu@qq.com>

cmd/sealos/cmd/cert.go

@@ -71,13 +71,13 @@ func newCertCmd() *cobra.Command {
 			}
 
 			cf := clusterfile.NewClusterFile(clusterPath,
-				clusterfile.WithCustomKubeadmFiles([]string{kubeadmInitFilepath}),
+				clusterfile.WithCustomRuntimeConfigFiles([]string{kubeadmInitFilepath}),
 			)
 			if err = cf.Process(); err != nil {
 				return err
 			}
 			// TODO: using different runtime
-			rt, err := kubernetes.New(cluster, cf.GetKubeadmConfig())
+			rt, err := kubernetes.New(cluster, cf.GetRuntimeConfig())
 			if err != nil {
 				return fmt.Errorf("get default runtime failed, %v", err)
 			}

cmd/sealos/cmd/gen.go

@@ -49,6 +49,7 @@ func newGenCmd() *cobra.Command {
 		Cluster: &apply.Cluster{},
 		SSH:     &apply.SSH{},
 	}
+
 	var out string
 	var genCmd = &cobra.Command{
 		Use:     "gen",
@@ -78,6 +79,7 @@ func newGenCmd() *cobra.Command {
 	setRequireBuildahAnnotation(genCmd)
 	genArgs.RegisterFlags(genCmd.Flags())
 	genCmd.Flags().StringVarP(&out, "output", "o", "", "print output to named file")
+	genCmd.Flags().String("distribution", "kubernetes", "kubernetes distribution")
 	return genCmd
 }
 

controllers/cluster/controllers/cluster_controller.go

@@ -312,7 +312,7 @@ func getSSHclient(infra *infrav1.Infra) ssh.Interface {
 		User:   defaultUser,
 		PkData: infra.Spec.SSH.PkData,
 	}
-	c := ssh.NewSSHClient(s, true)
+	c := ssh.MustNewClient(s, true)
 
 	return c
 }

pkg/apply/gen.go

@@ -23,8 +23,7 @@ import (
 
 	"github.com/labring/sealos/pkg/apply/processor"
 	"github.com/labring/sealos/pkg/buildah"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	"github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/iputils"
 )
@@ -53,7 +52,13 @@ func NewClusterFromGenArgs(cmd *cobra.Command, args *RunArgs, imageNames []strin
 		return nil, fmt.Errorf("input first image %s is not kubernetes image", imageNames)
 	}
 	cluster.Status.Mounts = append(cluster.Status.Mounts, *img)
-	rt, err := kubernetes.New(cluster, types.NewKubeadmConfig())
+
+	distribution, _ := cmd.Flags().GetString("distribution")
+	cfg, err := factory.NewRuntimeConfig(distribution)
+	if err != nil {
+		return nil, err
+	}
+	rt, err := factory.New(distribution, cluster, cfg)
 	if err != nil {
 		return nil, err
 	}

pkg/apply/processor/create.go

@@ -29,7 +29,7 @@ import (
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
 	"github.com/labring/sealos/pkg/guest"
 	"github.com/labring/sealos/pkg/runtime"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/logger"
 	"github.com/labring/sealos/pkg/utils/maps"
@@ -101,7 +101,9 @@ func (c *CreateProcessor) preProcess(cluster *v2.Cluster) error {
 	for i := range cluster.Status.Mounts {
 		cluster.Status.Mounts[i].Env = maps.MergeMap(cluster.Status.Mounts[i].Env, c.ExtraEnvs)
 	}
-	rt, err := kubernetes.New(cluster, c.ClusterFile.GetKubeadmConfig())
+
+	distribution := cluster.GetDistribution()
+	rt, err := factory.New(distribution, cluster, c.ClusterFile.GetRuntimeConfig())
 	if err != nil {
 		return fmt.Errorf("failed to init runtime, %v", err)
 	}

pkg/apply/processor/delete.go

@@ -25,7 +25,7 @@ import (
 	"github.com/labring/sealos/pkg/clusterfile"
 	"github.com/labring/sealos/pkg/constants"
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	fileutil "github.com/labring/sealos/pkg/utils/file"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -85,7 +85,8 @@ func (d *DeleteProcessor) UndoBootstrap(cluster *v2.Cluster) error {
 }
 
 func (d *DeleteProcessor) Reset(cluster *v2.Cluster) error {
-	rt, err := kubernetes.New(cluster, d.ClusterFile.GetKubeadmConfig())
+	distribution := cluster.GetDistribution()
+	rt, err := factory.New(distribution, cluster, d.ClusterFile.GetRuntimeConfig())
 	if err != nil {
 		return fmt.Errorf("failed to delete runtime, %v", err)
 	}

pkg/apply/processor/install.go

@@ -28,7 +28,7 @@ import (
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
 	"github.com/labring/sealos/pkg/guest"
 	"github.com/labring/sealos/pkg/runtime"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/confirm"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -170,7 +170,8 @@ func (c *InstallProcessor) PreProcess(cluster *v2.Cluster) error {
 		cluster.SetMountImage(mount)
 		c.NewMounts = append(c.NewMounts, *mount)
 	}
-	rt, err := kubernetes.New(cluster, c.ClusterFile.GetKubeadmConfig())
+	distribution := cluster.GetDistribution()
+	rt, err := factory.New(distribution, cluster, c.ClusterFile.GetRuntimeConfig())
 	if err != nil {
 		return fmt.Errorf("failed to init runtime, %v", err)
 	}

pkg/apply/processor/interface.go

@@ -153,7 +153,7 @@ func ConfirmDeleteNodes() error {
 func MirrorRegistry(cluster *v2.Cluster, mounts []v2.MountImage) error {
 	registries := cluster.GetRegistryIPAndPortList()
 	logger.Debug("registry nodes is: %+v", registries)
-	sshClient := ssh.NewSSHByCluster(cluster, true)
+	sshClient := ssh.NewCacheClientFromCluster(cluster, true)
 	syncer := registry.New(constants.NewPathResolver(cluster.GetName()), sshClient, mounts)
 	return syncer.Sync(context.Background(), registries...)
 }

pkg/apply/processor/scale.go

@@ -29,7 +29,7 @@ import (
 	"github.com/labring/sealos/pkg/filesystem/rootfs"
 	"github.com/labring/sealos/pkg/guest"
 	"github.com/labring/sealos/pkg/runtime"
-	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/factory"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	fileutil "github.com/labring/sealos/pkg/utils/file"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -202,11 +202,13 @@ func (c *ScaleProcessor) preProcess(cluster *v2.Cluster) error {
 	if err = SyncClusterStatus(cluster, c.Buildah, false); err != nil {
 		return err
 	}
+	distribution := cluster.GetDistribution()
+
 	var rt runtime.Interface
 	if c.IsScaleUp {
-		rt, err = kubernetes.New(cluster, c.ClusterFile.GetKubeadmConfig())
+		rt, err = factory.New(distribution, cluster, c.ClusterFile.GetRuntimeConfig())
 	} else {
-		rt, err = kubernetes.New(c.ClusterFile.GetCluster(), c.ClusterFile.GetKubeadmConfig())
+		rt, err = factory.New(distribution, c.ClusterFile.GetCluster(), c.ClusterFile.GetRuntimeConfig())
 	}
 	if err != nil {
 		return fmt.Errorf("failed to init runtime: %v", err)

pkg/apply/reset.go

@@ -68,7 +68,7 @@ func (r *ClusterArgs) resetArgs(cmd *cobra.Command, args *ResetArgs) error {
 		nodes := stringsutil.SplitRemoveEmpty(args.Cluster.Nodes, ",")
 		r.hosts = []v2.Host{}
 
-		sshClient := ssh.NewSSHByCluster(r.cluster, true)
+		sshClient := ssh.NewCacheClientFromCluster(r.cluster, true)
 		r.setHostWithIpsPort(masters, []string{v2.MASTER, GetHostArch(sshClient, masters[0])})
 		if len(nodes) > 0 {
 			r.setHostWithIpsPort(nodes, []string{v2.NODE, GetHostArch(sshClient, nodes[0])})

pkg/apply/run.go

@@ -122,7 +122,7 @@ func (r *ClusterArgs) runArgs(cmd *cobra.Command, args *RunArgs, imageList []str
 	nodes := stringsutil.SplitRemoveEmpty(args.Cluster.Nodes, ",")
 	r.hosts = []v2.Host{}
 
-	sshClient := ssh.NewSSHByCluster(r.cluster, true)
+	sshClient := ssh.NewCacheClientFromCluster(r.cluster, true)
 	if len(masters) > 0 {
 		host, port := iputils.GetHostIPAndPortOrDefault(masters[0], defaultPort)
 		master0addr := net.JoinHostPort(host, port)

pkg/apply/run_test.go

@@ -167,7 +167,7 @@ func TestNewApplierFromArgs(t *testing.T) {
 					},
 					Spec: v2.ClusterSpec{
 						Hosts: []v2.Host{
-							{IPS: []string{iputils.LocalIP(addr) + ":22"}, Roles: []string{v2.MASTER, GetHostArch(ssh.NewSSHClient(&v2.SSH{}, true), iputils.LocalIP(addr)+":22")}},
+							{IPS: []string{iputils.LocalIP(addr) + ":22"}, Roles: []string{v2.MASTER, GetHostArch(ssh.MustNewClient(&v2.SSH{}, true), iputils.LocalIP(addr)+":22")}},
 						},
 						Image: []string{"labring/kubernetes:v1.24.0"},
 						SSH:   v2.SSH{},

pkg/apply/scale.go

@@ -171,7 +171,7 @@ func verifyAndSetNodes(cmd *cobra.Command, cluster *v2.Cluster, scaleArgs *Scale
 			global := cluster.Spec.SSH.DeepCopy()
 			ssh.OverSSHConfig(global, override)
 
-			sshClient := ssh.NewSSHClient(global, true)
+			sshClient := ssh.MustNewClient(global, true)
 
 			host := &v2.Host{
 				IPS:   addrs,

pkg/apply/utils.go

@@ -148,7 +148,7 @@ func CheckAndInitialize(cluster *v2.Cluster) {
 
 	if len(cluster.Spec.Hosts) == 0 {
 		clusterSSH := cluster.GetSSH()
-		sshClient := ssh.NewSSHClient(&clusterSSH, true)
+		sshClient := ssh.MustNewClient(&clusterSSH, true)
 
 		localIpv4 := iputils.GetLocalIpv4()
 		defaultPort := defaultSSHPort(cluster.Spec.SSH.Port)

pkg/bootstrap/context.go

@@ -61,7 +61,7 @@ func (ctx realContext) GetRemoter() remote.Interface {
 }
 
 func NewContextFrom(cluster *v2.Cluster) Context {
-	execer := ssh.NewSSHByCluster(cluster, true)
+	execer := ssh.NewCacheClientFromCluster(cluster, true)
 	envProcessor := env.NewEnvProcessor(cluster)
 	remoter := remote.New(cluster.GetName(), execer)
 

pkg/checker/crictl_checker.go

@@ -104,7 +104,7 @@ func (n *CRICtlChecker) Check(cluster *v2.Cluster, phase string) error {
 			pauseImage = mountImg.Env["sandboxImage"]
 		}
 	}
-	sshCtx := ssh.NewSSHByCluster(cluster, false)
+	sshCtx := ssh.NewCacheClientFromCluster(cluster, false)
 	root := constants.NewPathResolver(cluster.Name).RootFSPath()
 	regInfo := helpers.GetRegistryInfo(sshCtx, root, cluster.GetRegistryIPAndPort())
 

pkg/checker/host_checker.go

@@ -40,7 +40,7 @@ func (a HostChecker) Check(cluster *v2.Cluster, _ string) error {
 	if len(a.IPs) != 0 {
 		ipList = a.IPs
 	}
-	sshClient := ssh.NewSSHByCluster(cluster, false)
+	sshClient := ssh.NewCacheClientFromCluster(cluster, false)
 	if err := checkHostnameUnique(sshClient, ipList); err != nil {
 		return err
 	}

pkg/checker/registry_checker.go

@@ -87,7 +87,7 @@ func (n *RegistryChecker) Check(cluster *v2.Cluster, phase string) error {
 		}
 	}
 
-	sshCtx := ssh.NewSSHByCluster(cluster, false)
+	sshCtx := ssh.NewCacheClientFromCluster(cluster, false)
 	root := constants.NewPathResolver(cluster.Name).RootFSPath()
 	regInfo := helpers.GetRegistryInfo(sshCtx, root, cluster.GetRegistryIPAndPort())
 	status.Auth = fmt.Sprintf("%s:%s", regInfo.Username, regInfo.Password)

pkg/clusterfile/clusterfile.go

@@ -18,24 +18,25 @@ import (
 	"errors"
 	"sync"
 
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 )
 
 var ErrTypeNotFound = errors.New("no corresponding type structure was found")
 
 type ClusterFile struct {
-	path               string
-	customConfigFiles  []string
-	customKubeadmFiles []string
-	customValues       []string
-	customSets         []string
-	customEnvs         []string
-	setDefaults        bool
-	Cluster            *v2.Cluster
-	Configs            []v2.Config
-	KubeConfig         *types.KubeadmConfig
-	//Plugins    []v1.Plugin
+	path                     string
+	customConfigFiles        []string
+	customRuntimeConfigFiles []string
+	customValues             []string
+	customSets               []string
+	customEnvs               []string
+	setDefaults              bool
+
+	cluster       *v2.Cluster
+	configs       []v2.Config
+	runtimeConfig runtime.Config
+
 	once sync.Once
 }
 
@@ -43,19 +44,19 @@ type Interface interface {
 	PreProcessor
 	GetCluster() *v2.Cluster
 	GetConfigs() []v2.Config
-	GetKubeadmConfig() *types.KubeadmConfig
+	GetRuntimeConfig() runtime.Config
 }
 
 func (c *ClusterFile) GetCluster() *v2.Cluster {
-	return c.Cluster
+	return c.cluster
 }
 
 func (c *ClusterFile) GetConfigs() []v2.Config {
-	return c.Configs
+	return c.configs
 }
 
-func (c *ClusterFile) GetKubeadmConfig() *types.KubeadmConfig {
-	return c.KubeConfig
+func (c *ClusterFile) GetRuntimeConfig() runtime.Config {
+	return c.runtimeConfig
 }
 
 type OptionFunc func(*ClusterFile)
@@ -72,9 +73,9 @@ func WithCustomConfigFiles(files []string) OptionFunc {
 	}
 }
 
-func WithCustomKubeadmFiles(files []string) OptionFunc {
+func WithCustomRuntimeConfigFiles(files []string) OptionFunc {
 	return func(c *ClusterFile) {
-		c.customKubeadmFiles = files
+		c.customRuntimeConfigFiles = files
 	}
 }
 

pkg/clusterfile/clusterfile_test.go

@@ -21,7 +21,8 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 )
 
@@ -121,7 +122,7 @@ func Test_NewClusterFile(t *testing.T) {
 
 func Test_NoClusterFileWithSingleSchedule(t *testing.T) {
 	type args struct {
-		kubeadmConfig *types.KubeadmConfig
+		runtimeConfig runtime.Config
 	}
 	tests := []struct {
 		name    string
@@ -131,7 +132,7 @@ func Test_NoClusterFileWithSingleSchedule(t *testing.T) {
 		{
 			name: "run single with cluster file not exists",
 			args: args{
-				kubeadmConfig: &types.KubeadmConfig{
+				runtimeConfig: &types.KubeadmConfig{
 					InitConfiguration: kubeadm.InitConfiguration{
 						SkipPhases: []string{
 							"mark-control-plane",
@@ -156,7 +157,7 @@ func Test_NoClusterFileWithSingleSchedule(t *testing.T) {
 			if cf.GetConfigs() != nil {
 				t.Error("configs is not nil")
 			}
-			if !reflect.DeepEqual(cf.GetKubeadmConfig(), tt.args.kubeadmConfig) {
+			if !reflect.DeepEqual(cf.GetRuntimeConfig(), tt.args.runtimeConfig) {
 				t.Error("kubeadmConfig not equal")
 			}
 		})
@@ -167,7 +168,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 	type args struct {
 		cluster            *v2.Cluster
 		config             v2.Config
-		kubeadmConfig      *types.KubeadmConfig
+		runtimeConfig      runtime.Config
 		customEnv          []string
 		sets               []string
 		values             []string
@@ -223,7 +224,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 						Data: "test\n",
 					},
 				},
-				kubeadmConfig: &types.KubeadmConfig{
+				runtimeConfig: &types.KubeadmConfig{
 					InitConfiguration: kubeadm.InitConfiguration{
 						TypeMeta: metav1.TypeMeta{
 							APIVersion: "kubeadm.k8s.io/v1beta3",
@@ -255,7 +256,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 				WithCustomSets(tt.args.sets),
 				WithCustomValues(tt.args.values),
 				WithCustomConfigFiles(tt.args.customConfigs),
-				WithCustomKubeadmFiles(tt.args.customKubeadmFiles),
+				WithCustomRuntimeConfigFiles(tt.args.customKubeadmFiles),
 			)
 			err := cf.Process()
 			if (err != nil) != tt.wantErr {
@@ -269,7 +270,7 @@ func Test_NewClusterFileWithSingleSchedule(t *testing.T) {
 			if !reflect.DeepEqual(cf.GetConfigs()[0], tt.args.config) {
 				t.Error("config not equal")
 			}
-			if !reflect.DeepEqual(cf.GetKubeadmConfig(), tt.args.kubeadmConfig) {
+			if !reflect.DeepEqual(cf.GetRuntimeConfig(), tt.args.runtimeConfig) {
 				t.Error("kubeadmConfig not equal")
 			}
 		})

pkg/clusterfile/pre_process.go

@@ -25,7 +25,8 @@ import (
 
 	"github.com/labring/sealos/pkg/constants"
 	"github.com/labring/sealos/pkg/runtime/decode"
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime/k3s"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
 	"github.com/labring/sealos/pkg/template"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	fileutil "github.com/labring/sealos/pkg/utils/file"
@@ -38,10 +39,6 @@ type PreProcessor interface {
 	Process() error
 }
 
-func NewPreProcessor(path string) PreProcessor {
-	return &ClusterFile{path: path}
-}
-
 func (c *ClusterFile) Process() (err error) {
 	if !fileutil.IsExist(c.path) {
 		return ErrClusterFileNotExists
@@ -100,8 +97,8 @@ func (c *ClusterFile) loadClusterFile() ([]byte, error) {
 		out.Write(configData)
 	}
 
-	for i := range c.customKubeadmFiles {
-		configData, err := fileutil.ReadAll(c.customKubeadmFiles[i])
+	for i := range c.customRuntimeConfigFiles {
+		configData, err := fileutil.ReadAll(c.customRuntimeConfigFiles[i])
 		if err != nil {
 			return nil, err
 		}
@@ -124,7 +121,7 @@ func (c *ClusterFile) loadRenderValues() (map[string]interface{}, error) {
 
 func (c *ClusterFile) decode(data []byte) error {
 	for _, fn := range []func([]byte) error{
-		c.DecodeCluster, c.DecodeConfigs, c.DecodeKubeadmConfig,
+		c.DecodeCluster, c.DecodeConfigs, c.DecodeRuntimeConfig,
 	} {
 		if err := fn(data); err != nil && err != ErrTypeNotFound {
 			return err
@@ -141,7 +138,7 @@ func (c *ClusterFile) DecodeCluster(data []byte) error {
 	if cluster == nil {
 		return ErrTypeNotFound
 	}
-	c.Cluster = cluster
+	c.cluster = cluster
 	return nil
 }
 
@@ -154,18 +151,27 @@ func (c *ClusterFile) DecodeConfigs(data []byte) error {
 		return ErrTypeNotFound
 	}
 	cfgs := configs.([]v2.Config)
-	c.Configs = cfgs
+	c.configs = cfgs
 	return nil
 }
 
-func (c *ClusterFile) DecodeKubeadmConfig(data []byte) error {
-	kubeadmConfig, err := types.LoadKubeadmConfigs(string(data), c.setDefaults, decode.CRDFromString)
+func (c *ClusterFile) DecodeRuntimeConfig(data []byte) error {
+	// TODO: handling more types of runtime configuration
+	cfg, err := k3s.ParseConfig(data)
 	if err != nil {
 		return err
 	}
-	if kubeadmConfig == nil {
-		return ErrTypeNotFound
+	if cfg != nil {
+		c.runtimeConfig = cfg
+	} else {
+		kubeadmConfig, err := types.LoadKubeadmConfigs(string(data), c.setDefaults, decode.CRDFromString)
+		if err != nil {
+			return err
+		}
+		if kubeadmConfig == nil {
+			return ErrTypeNotFound
+		}
+		c.runtimeConfig = kubeadmConfig
 	}
-	c.KubeConfig = kubeadmConfig
 	return nil
 }

pkg/constants/consts.go

@@ -30,6 +30,12 @@ const (
 	DefaultHostsPath        = "/etc/hosts"
 )
 
+const (
+	DefaultAPIServerDomain = "apiserver.cluster.local"
+	DefaultDNSDomain       = "cluster.local"
+	DefaultAPIServerPort   = 6443
+)
+
 // CRD kind
 const (
 	Config  = "Config"

pkg/filesystem/rootfs/rootfs_default.go

@@ -51,7 +51,7 @@ func (f *defaultRootfs) getClusterName(cluster *v2.Cluster) string {
 }
 
 func (f *defaultRootfs) getSSH(cluster *v2.Cluster) ssh.Interface {
-	return ssh.NewSSHByCluster(cluster, true)
+	return ssh.NewCacheClientFromCluster(cluster, true)
 }
 
 func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error {

pkg/guest/guest.go

@@ -43,7 +43,7 @@ func NewGuestManager() (Interface, error) {
 
 func (d *Default) Apply(cluster *v2.Cluster, mounts []v2.MountImage, targetHosts []string) error {
 	envWrapper := env.NewEnvProcessor(cluster)
-	execer := ssh.NewSSHByCluster(cluster, true)
+	execer := ssh.NewCacheClientFromCluster(cluster, true)
 
 	for i, m := range mounts {
 		switch {

pkg/registry/password/apply.go

@@ -132,7 +132,7 @@ func (r *RegistryPasswdResults) Validate() (*v1beta1.Cluster, error) {
 
 func (r *RegistryPasswdResults) Apply(cluster *v1beta1.Cluster) error {
 	if r.execer == nil {
-		r.execer = ssh.NewSSHByCluster(cluster, true)
+		r.execer = ssh.NewCacheClientFromCluster(cluster, true)
 	}
 	if r.upgrade == nil {
 		r.upgrade = NewUpgrade(cluster.Name, r.execer)

pkg/runtime/factory/factory.go

@@ -0,0 +1,45 @@
+// Copyright © 2023 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package factory
+
+import (
+	"fmt"
+
+	"github.com/labring/sealos/pkg/runtime"
+	"github.com/labring/sealos/pkg/runtime/k3s"
+	"github.com/labring/sealos/pkg/runtime/kubernetes"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
+	"github.com/labring/sealos/pkg/types/v1beta1"
+)
+
+func New(distribution string, cluster *v1beta1.Cluster, cfg runtime.Config) (runtime.Interface, error) {
+	switch distribution {
+	case kubernetes.Distribution, "kubeadm", "":
+		return kubernetes.New(cluster, cfg)
+	case k3s.Distribution:
+		return k3s.New(cluster, cfg)
+	}
+	return nil, fmt.Errorf("unsupported distribution %s", distribution)
+}
+
+func NewRuntimeConfig(distribution string) (runtime.Config, error) {
+	switch distribution {
+	case kubernetes.Distribution, "kubeadm", "":
+		return types.NewKubeadmConfig(), nil
+	case k3s.Distribution:
+		return &k3s.Config{}, nil
+	}
+	return nil, fmt.Errorf("unsupported distribution %s", distribution)
+}

pkg/runtime/interface.go

@@ -32,3 +32,5 @@ type CertManager interface {
 	Renew() error
 	UpdateCertSANs(certSANs []string) error
 }
+
+type Config any

pkg/runtime/k3s/bootstrap.go

@@ -0,0 +1,256 @@
+// Copyright © 2023 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k3s
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/labring/sealos/pkg/ssh"
+	"github.com/labring/sealos/pkg/template"
+	"github.com/labring/sealos/pkg/utils/file"
+	"github.com/labring/sealos/pkg/utils/logger"
+	"github.com/labring/sealos/pkg/utils/rand"
+	"github.com/labring/sealos/pkg/utils/yaml"
+)
+
+func (k *K3s) initMaster0() error {
+	master0 := k.cluster.GetMaster0IPAndPort()
+	return k.runPipelines("init master0",
+		k.generateAndSendCerts,
+		func() error { return k.generateAndSendTokenFiles(master0, "token", "agent-token") },
+		k.generateAndSendInitConfig,
+		func() error { return k.generateAndEnableService(master0, serverMode) },
+		func() error { return k.createSymlinks(master0, defaultBinDir, Distribution, defaultSymlinks...) },
+	)
+}
+
+func (k *K3s) joinMasters(masters []string) error {
+	_, err := k.writeJoinConfigWithCallbacks(serverMode)
+	if err != nil {
+		return err
+	}
+	for _, master := range masters {
+		if err = k.joinMaster(master); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (k *K3s) writeJoinConfigWithCallbacks(runMode string, callbacks ...callback) (string, error) {
+	master0 := k.cluster.GetMaster0IP()
+	defaultCallbacks := []callback{
+		k.overrideConfig,
+		func(c *Config) { c.ServerURL = fmt.Sprintf("https://%s:%d", master0, c.HTTPSPort) },
+		// TODO: set --image-service-endpoint flag in c.ExtraKubeletArgs options
+	}
+	raw, err := k.getRawInitConfig(
+		append(defaultCallbacks, callbacks...)...,
+	)
+	if err != nil {
+		return "", err
+	}
+	var filename string
+	switch runMode {
+	case serverMode:
+		filename = defaultJoinMastersFilename
+	case agentMode:
+		filename = defaultJoinNodesFilename
+	}
+	path := filepath.Join(k.pathResolver.TmpPath(), filename)
+	return path, file.WriteFile(path, raw)
+}
+
+func (k *K3s) joinMaster(master string) error {
+	return k.runPipelines(fmt.Sprintf("join master %s", master),
+		func() error { return k.generateAndSendTokenFiles(master, "token") },
+		func() error {
+			return k.sshClient.Copy(master, filepath.Join(k.pathResolver.TmpPath(), defaultJoinMastersFilename), defaultConfigPath)
+		},
+		func() error { return k.generateAndEnableService(master, serverMode) },
+		func() error { return k.createSymlinks(master, defaultBinDir, Distribution, defaultSymlinks...) },
+	)
+}
+
+func (k *K3s) joinNodes(nodes []string) error {
+	if _, err := k.writeJoinConfigWithCallbacks(agentMode, func(c *Config) {
+		c.TokenFile = ""
+	}); err != nil {
+		return err
+	}
+	for i := range nodes {
+		if err := k.joinNode(nodes[i]); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (k *K3s) joinNode(node string) error {
+	return k.runPipelines(fmt.Sprintf("join node %s", node),
+		func() error { return k.generateAndSendTokenFiles(node, "agent-token") },
+		func() error {
+			return k.sshClient.Copy(node, filepath.Join(k.pathResolver.TmpPath(), defaultJoinNodesFilename), defaultConfigPath)
+		},
+		func() error { return k.generateAndEnableService(node, agentMode) },
+		func() error { return k.createSymlinks(node, defaultBinDir, Distribution, defaultSymlinks...) },
+	)
+}
+
+func (k *K3s) generateAndSendCerts() error {
+	logger.Debug("generate and send self-signed certificates")
+	// TODO: use self-signed certificates
+	return nil
+}
+
+func (k *K3s) generateRandomTokenFileIfNotExists(filename string) (string, error) {
+	fp := filepath.Join(k.pathResolver.EtcPath(), filepath.Base(filename))
+	if !file.IsExist(fp) {
+		logger.Debug("token file %s not exists, create new one", fp)
+		token, err := rand.CreateCertificateKey()
+		if err != nil {
+			return "", err
+		}
+		return fp, file.WriteFile(fp, []byte(token))
+	}
+	return fp, nil
+}
+
+func (k *K3s) generateAndSendTokenFiles(host string, filenames ...string) error {
+	for _, filename := range filenames {
+		src, err := k.generateRandomTokenFileIfNotExists(filename)
+		if err != nil {
+			return fmt.Errorf("generate token: %v", err)
+		}
+		dst := filepath.Join(k.pathResolver.ConfigsPath(), filename)
+		if err = k.sshClient.Copy(host, src, dst); err != nil {
+			return fmt.Errorf("copy token file: %v", err)
+		}
+	}
+	return nil
+}
+
+func (k *K3s) getRawInitConfig(callbacks ...callback) ([]byte, error) {
+	cfg, err := k.getInitConfig(callbacks...)
+	if err != nil {
+		return nil, err
+	}
+	return yaml.MarshalYamlConfigs(cfg)
+}
+
+func (k *K3s) generateAndSendInitConfig() error {
+	src := filepath.Join(k.pathResolver.TmpPath(), defaultInitFilename)
+	if !file.IsExist(src) {
+		raw, err := k.getRawInitConfig(k.overrideConfig, setClusterInit)
+		if err != nil {
+			return err
+		}
+		if err = file.WriteFile(src, raw); err != nil {
+			return err
+		}
+	}
+	return k.sshClient.Copy(k.cluster.GetMaster0IPAndPort(), src, defaultConfigPath)
+}
+
+func (k *K3s) generateAndEnableService(host string, runMode string) error {
+	logger.Debug("generate and enable service for %s running in %s mode", host, runMode)
+	sysType := verifySystem(k.sshClient, host)
+
+	data := map[string]string{
+		"binDir":  defaultBinDir,
+		"envFile": fmt.Sprintf("/etc/rancher/%s/%s.env", Distribution, runMode),
+		"exec":    fmt.Sprintf("%s --config=%s", runMode, defaultConfigPath),
+	}
+	serviceContent, err := getServiceContent(sysType, data)
+	if err != nil {
+		return err
+	}
+	src := filepath.Join(k.pathResolver.TmpPath(), host)
+	if err = file.WriteFile(src, serviceContent); err != nil {
+		return err
+	}
+	dst, err := getServiceFilePath(sysType, Distribution)
+	if err != nil {
+		return err
+	}
+	if err = k.sshClient.Copy(host, src, dst); err != nil {
+		return err
+	}
+	commands := getEnableAndStartServiceCommands(sysType, Distribution)
+	return k.sshClient.CmdAsync(host, commands...)
+}
+
+func (k *K3s) createSymlinks(host string, binDir string, command string, symlinks ...string) error {
+	script := fmt.Sprintf(`for cmd in %s; do ln -sf %s/%s %s/$cmd; done`, strings.Join(symlinks, " "), binDir, command, binDir)
+	return k.sshClient.CmdAsync(host, script)
+}
+
+func getEnableAndStartServiceCommands(sysType serviceType, svcName string) []string {
+	switch sysType {
+	case openRC:
+		return []string{
+			fmt.Sprintf("rc-update add %s default > /dev/null", svcName),
+			fmt.Sprintf("/etc/init.d/%s restart", svcName),
+		}
+	case systemd:
+		return []string{
+			"systemctl daemon-reload > /dev/null",
+			fmt.Sprintf("systemctl enable %s.service > /dev/null", svcName),
+			fmt.Sprintf("systemctl restart %s.service > /dev/null", svcName),
+		}
+	}
+	return []string{}
+}
+
+func verifySystem(sshClient ssh.Interface, target string) serviceType {
+	if _, err := sshClient.Cmd(target, "test -x /sbin/openrc-run"); err == nil {
+		return openRC
+	}
+	return systemd
+}
+
+func getServiceContent(sysType serviceType, data any) ([]byte, error) {
+	var tpl string
+	switch sysType {
+	case openRC:
+		tpl = openRCTpl
+	case systemd:
+		tpl = systemdTpl
+	}
+
+	t, err := template.Parse(tpl)
+	if err != nil {
+		return nil, err
+	}
+	var out bytes.Buffer
+	if err := t.Execute(&out, data); err != nil {
+		return nil, err
+	}
+	return out.Bytes(), nil
+}
+
+func getServiceFilePath(svcType serviceType, svcName string) (string, error) {
+	switch svcType {
+	case openRC:
+		return fmt.Sprintf("/etc/init.d/%s", svcName), nil
+	case systemd:
+		return fmt.Sprintf("/etc/systemd/system/%s.service", svcName), nil
+	}
+	return "", errors.New("unknown service type")
+}

pkg/runtime/k3s/config.go

@@ -0,0 +1,105 @@
+// Copyright © 2023 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k3s
+
+import (
+	"path/filepath"
+
+	"github.com/imdario/mergo"
+	"sigs.k8s.io/yaml"
+
+	"github.com/labring/sealos/pkg/constants"
+)
+
+var defaultMergeOpts = []func(*mergo.Config){
+	mergo.WithOverride,
+	mergo.WithAppendSlice,
+}
+
+var defaultConfig = &Config{
+	BindAddress:           "0.0.0.0",
+	HTTPSPort:             6443,
+	ClusterCIDR:           []string{"10.42.0.0/16"},
+	ServiceCIDR:           []string{"10.43.0.0/16"},
+	ClusterDNS:            []string{"10.43.0.10"},
+	ServiceNodePortRange:  "30000-32767",
+	ClusterDomain:         constants.DefaultDNSDomain,
+	DataDir:               "/var/lib/rancher/k3s",
+	DisableCCM:            true,
+	DisableHelmController: true,
+	DisableNPC:            true,
+	EgressSelectorMode:    "agent",
+	FlannelBackend:        "vxlan",
+	KubeConfigMode:        "0644",
+	PreferBundledBin:      true,
+	Disable:               []string{"servicelb", "traefik", "local-storage", "metrics-server"},
+	AgentConfig: &AgentConfig{
+		ExtraKubeletArgs:   []string{},
+		ExtraKubeProxyArgs: []string{},
+		PauseImage:         "docker.io/rancher/pause:3.1",
+		PrivateRegistry:    "/etc/rancher/k3s/registries.yaml",
+		Labels:             []string{"provisioner=k3s"},
+	},
+}
+
+type callback func(*Config)
+
+func merge(dst, src *Config) error {
+	if src == nil {
+		return nil
+	}
+	return mergo.Merge(dst, src, defaultMergeOpts...)
+}
+
+func setClusterInit(c *Config) {
+	c.ClusterInit = true
+}
+
+func (k *K3s) overrideConfig(cfg *Config) {
+	cfg.TokenFile = filepath.Join(k.pathResolver.ConfigsPath(), "token")
+	cfg.AgentTokenFile = filepath.Join(k.pathResolver.ConfigsPath(), "agent-token")
+	cfg.TLSSan = append(cfg.TLSSan, constants.DefaultAPIServerDomain)
+}
+
+func (k *K3s) getInitConfig(callbacks ...callback) (*Config, error) {
+	cfg := *defaultConfig
+	if err := merge(&cfg, k.config); err != nil {
+		return nil, err
+	}
+	for i := range callbacks {
+		callbacks[i](&cfg)
+	}
+	return &cfg, nil
+}
+
+// ParseConfig return nil if data structure is not matched
+func ParseConfig(data []byte) (*Config, error) {
+	var c Config
+	if err := yaml.Unmarshal(data, &c); err != nil {
+		return nil, err
+	}
+	out, err := yaml.Marshal(&c)
+	if err != nil {
+		return nil, err
+	}
+	var m map[string]interface{}
+	if err := yaml.Unmarshal(out, &m); err != nil {
+		return nil, err
+	}
+	if len(m) == 0 {
+		return nil, nil
+	}
+	return &c, nil
+}

pkg/runtime/k3s/consts.go

@@ -0,0 +1,103 @@
+// Copyright © 2023 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k3s
+
+const Distribution = "k3s"
+
+const (
+	defaultBinDir              = "/usr/local/bin"
+	defaultConfigPath          = "/etc/rancher/k3s/config.yaml"
+	defaultInitFilename        = "k3s-init.yaml"
+	defaultJoinMastersFilename = "k3s-join-master.yaml"
+	defaultJoinNodesFilename   = "k3s-join-master.yaml"
+)
+
+var defaultSymlinks = []string{"kubectl", "crictl", "ctr"}
+
+const (
+	serverMode = "server"
+	agentMode  = "agent"
+)
+
+type serviceType int
+
+const (
+	openRC serviceType = iota
+	systemd
+)
+
+const systemdTpl = `
+[Unit]
+Description=Lightweight Kubernetes
+Documentation=https://k3s.io
+Wants=network-online.target
+After=network-online.target
+
+[Install]
+WantedBy=multi-user.target
+
+[Service]
+Type=notify
+EnvironmentFile=-/etc/default/%N
+EnvironmentFile=-/etc/sysconfig/%N
+EnvironmentFile=-{{ .envFile }}
+KillMode=process
+Delegate=yes
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=1048576
+LimitNPROC=infinity
+LimitCORE=infinity
+TasksMax=infinity
+TimeoutStartSec=0
+Restart=always
+RestartSec=5s
+ExecStartPre=/bin/sh -xc '! /usr/bin/systemctl is-enabled --quiet nm-cloud-setup.service'
+ExecStartPre=-/sbin/modprobe br_netfilter
+ExecStartPre=-/sbin/modprobe overlay
+ExecStart={{ .binDir }}/k3s {{ .exec }}
+`
+
+const openRCTpl = `
+#!/sbin/openrc-run
+
+LOG_FILE=/var/log/k3s.log
+
+depend() {
+    after network-online
+    want cgroups
+}
+
+start_pre() {
+    rm -f /tmp/k3s.*
+}
+
+supervisor=supervise-daemon
+name=k3s
+command="{{ .binDir }}/k3s"
+command_args={{ .exec }}
+
+output_log=${LOG_FILE}
+error_log=${LOG_FILE}
+
+pidfile="/var/run/k3s.pid"
+respawn_delay=5
+respawn_max=0
+
+set -o allexport
+if [ -f /etc/environment ]; then . /etc/environment; fi
+if [ -f {{ .envFile }} ]; then . {{ .envFile }}; fi
+set +o allexport
+`

pkg/runtime/k3s/k3s.go

@@ -0,0 +1,104 @@
+// Copyright © 2023 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k3s
+
+import (
+	"fmt"
+
+	"github.com/labring/sealos/pkg/constants"
+	"github.com/labring/sealos/pkg/ssh"
+	v2 "github.com/labring/sealos/pkg/types/v1beta1"
+	"github.com/labring/sealos/pkg/utils/logger"
+	"github.com/labring/sealos/pkg/utils/yaml"
+)
+
+type K3s struct {
+	cluster *v2.Cluster
+	config  *Config
+
+	pathResolver constants.PathResolver
+	sshClient    ssh.Interface
+}
+
+func New(cluster *v2.Cluster, config any) (*K3s, error) {
+	sshClient := ssh.NewCacheClientFromCluster(cluster, true)
+	k := &K3s{
+		cluster:      cluster,
+		pathResolver: constants.NewPathResolver(cluster.GetName()),
+		sshClient:    sshClient,
+	}
+	if v, ok := config.(*Config); ok {
+		k.config = v
+	}
+	return k, nil
+}
+
+func (k *K3s) Init() error {
+	return k.initMaster0()
+}
+
+func (k *K3s) Reset() error {
+	if err := k.resetNodes(k.cluster.GetNodeIPAndPortList()); err != nil {
+		logger.Error("resetting nodes: %v", err)
+	}
+	if err := k.resetNodes(k.cluster.GetMasterIPAndPortList()); err != nil {
+		logger.Error("resetting masters: %v", err)
+	}
+	return nil
+}
+
+func (k *K3s) ScaleUp(masters []string, nodes []string) error {
+	if err := k.joinMasters(masters); err != nil {
+		return err
+	}
+	return k.joinNodes(nodes)
+}
+
+func (k *K3s) ScaleDown(masters []string, nodes []string) error {
+	if err := k.resetNodes(nodes); err != nil {
+		return err
+	}
+	return k.resetNodes(masters)
+}
+
+func (k *K3s) Upgrade(version string) error {
+	logger.Info("trying to upgrade to version %s", version)
+	return nil
+}
+
+func (k *K3s) GetRawConfig() ([]byte, error) {
+	cfg, err := k.getInitConfig(k.overrideConfig, setClusterInit)
+	if err != nil {
+		return nil, err
+	}
+	cluster := k.cluster.DeepCopy()
+	cluster.Status = v2.ClusterStatus{}
+	return yaml.MarshalYamlConfigs(cluster, cfg)
+}
+
+func (k *K3s) SyncNodeIPVS(_, _ []string) error {
+	logger.Error("not yet implemented, skip for testing")
+	return nil
+}
+
+func (k *K3s) runPipelines(phase string, pipelines ...func() error) error {
+	logger.Info("starting %s", phase)
+	for i := range pipelines {
+		if err := pipelines[i](); err != nil {
+			return fmt.Errorf("failed to %s: %v", phase, err)
+		}
+	}
+	return nil
+}

pkg/runtime/k3s/lifecycle.go

@@ -0,0 +1,45 @@
+// Copyright © 2023 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k3s
+
+import (
+	"context"
+	"fmt"
+
+	"golang.org/x/sync/errgroup"
+)
+
+func (k *K3s) resetNodes(nodes []string) error {
+	eg, _ := errgroup.WithContext(context.Background())
+	for i := range nodes {
+		node := nodes[i]
+		eg.Go(func() error {
+			if err := k.deleteNode(node); err != nil {
+				return err
+			}
+			return k.resetNode(node)
+		})
+	}
+	return eg.Wait()
+}
+
+func (k *K3s) resetNode(host string) error {
+	return k.sshClient.CmdAsync(host, fmt.Sprintf("%s/k3s-uninstall.sh", defaultBinDir))
+}
+
+// TODO: remove from API
+func (k *K3s) deleteNode(_ string) error {
+	return nil
+}

pkg/runtime/k3s/types.go

@@ -0,0 +1,132 @@
+// Copyright © 2023 sealos.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package k3s
+
+import "time"
+
+// from github.com/k3s-io/k3s/pkg/cli/cmds/server.go
+type Config struct {
+	ClusterCIDR          []string `json:"cluster-cidr,omitempty"`
+	AgentToken           string   `json:"agent-token,omitempty"`
+	AgentTokenFile       string   `json:"agent-token-file,omitempty"`
+	Token                string   `json:"token,omitempty"`
+	TokenFile            string   `json:"token-file,omitempty"`
+	ClusterSecret        string   `json:"-"`
+	ServiceCIDR          []string `json:"service-cidr,omitempty"`
+	ServiceNodePortRange string   `json:"service-node-port-range,omitempty"`
+	ClusterDNS           []string `json:"cluster-dns,omitempty"`
+	ClusterDomain        string   `json:"cluster-domain,omitempty"`
+	// The port which kubectl clients can access k8s
+	HTTPSPort int `json:"https-listen-port,omitempty"`
+	// The port which custom k3s API runs on
+	SupervisorPort int `json:"-"`
+	// The port which kube-apiserver runs on
+	APIServerPort            int           `json:"-"`
+	APIServerBindAddress     string        `json:"-"`
+	DataDir                  string        `json:"data-dir,omitempty"`
+	DisableAgent             bool          `json:"disable-agent,omitempty"`
+	PreferBundledBin         bool          `json:"prefer-bundled-bin,omitempty"`
+	KubeConfigOutput         string        `json:"write-kubeconfig,omitempty"`
+	KubeConfigMode           string        `json:"write-kubeconfig-mode,omitempty"`
+	HelmJobImage             string        `json:"helm-job-image,omitempty"`
+	TLSSan                   []string      `json:"tls-san,omitempty"`
+	BindAddress              string        `json:"bind-address,omitempty"`
+	EnablePProf              bool          `json:"enable-pprof,omitempty"`
+	ExtraAPIArgs             []string      `json:"kube-apiserver-arg,omitempty"`
+	ExtraEtcdArgs            []string      `json:"etcd-arg,omitempty"`
+	ExtraSchedulerArgs       []string      `json:"kube-scheduler-arg,omitempty"`
+	ExtraControllerArgs      []string      `json:"kube-controller-manager-arg,omitempty"`
+	ExtraCloudControllerArgs []string      `json:"kube-cloud-controller-manager-arg,omitempty"`
+	Rootless                 bool          `json:"rootless,omitempty"`
+	DatastoreEndpoint        string        `json:"datastore-endpoint,omitempty"`
+	DatastoreCAFile          string        `json:"datastore-cafile,omitempty"`
+	DatastoreCertFile        string        `json:"datastore-certfile,omitempty"`
+	DatastoreKeyFile         string        `json:"datastore-keyfile,omitempty"`
+	AdvertiseIP              string        `json:"advertise-address,omitempty"`
+	AdvertisePort            int           `json:"advertise-port,omitempty"`
+	DisableScheduler         bool          `json:"disable-scheduler,omitempty"`
+	ServerURL                string        `json:"server,omitempty"`
+	MultiClusterCIDR         bool          `json:"multi-cluster-cidr,omitempty"`
+	FlannelBackend           string        `json:"flannel-backend,omitempty"`
+	FlannelIPv6Masq          bool          `json:"flannel-ipv6-masq,omitempty"`
+	FlannelExternalIP        bool          `json:"flannel-external-ip,omitempty"`
+	EgressSelectorMode       string        `json:"egress-selector-mode,omitempty"`
+	DefaultLocalStoragePath  string        `json:"default-local-storage-path,omitempty"`
+	Disable                  []string      `json:"disable,omitempty"`
+	DisableCCM               bool          `json:"disable-cloud-controller,omitempty"`
+	DisableNPC               bool          `json:"disable-network-policy,omitempty"`
+	DisableHelmController    bool          `json:"disable-helm-controller,omitempty"`
+	DisableKubeProxy         bool          `json:"disable-kube-proxy,omitempty"`
+	DisableAPIServer         bool          `json:"disable-apiserver,omitempty"`
+	DisableControllerManager bool          `json:"disable-controller-manager,omitempty"`
+	DisableETCD              bool          `json:"disable-etcd,omitempty"`
+	ClusterInit              bool          `json:"cluster-init,omitempty"`
+	ClusterReset             bool          `json:"cluster-reset,omitempty"`
+	ClusterResetRestorePath  string        `json:"cluster-reset-restore-path,omitempty"`
+	EncryptSecrets           bool          `json:"secrets-encryption,omitempty"`
+	EncryptForce             bool          `json:"-"`
+	EncryptOutput            string        `json:"-"`
+	EncryptSkip              bool          `json:"-"`
+	SystemDefaultRegistry    string        `json:"system-default-registry,omitempty"`
+	EtcdSnapshotName         string        `json:"etcd-snapshot-name,omitempty"`
+	EtcdDisableSnapshots     bool          `json:"etcd-disable-snapshots,omitempty"`
+	EtcdExposeMetrics        bool          `json:"etcd-expose-metrics,omitempty"`
+	EtcdSnapshotDir          string        `json:"etcd-snapshot-dir,omitempty"`
+	EtcdSnapshotCron         string        `json:"etcd-snapshot-schedule-cron,omitempty"`
+	EtcdSnapshotRetention    int           `json:"etcd-snapshot-retention,omitempty"`
+	EtcdSnapshotCompress     bool          `json:"etcd-snapshot-compress,omitempty"`
+	EtcdS3                   bool          `json:"etcd-s3,omitempty"`
+	EtcdS3Endpoint           string        `json:"etcd-s3-endpoint,omitempty"`
+	EtcdS3EndpointCA         string        `json:"etcd-s3-endpoint-ca,omitempty"`
+	EtcdS3SkipSSLVerify      bool          `json:"etcd-s3-skip-ssl-verify,omitempty"`
+	EtcdS3AccessKey          string        `json:"etcd-s3-access-key,omitempty"`
+	EtcdS3SecretKey          string        `json:"etcd-s3-secret-key,omitempty"`
+	EtcdS3BucketName         string        `json:"etcd-s3-bucket,omitempty"`
+	EtcdS3Region             string        `json:"etcd-s3-region,omitempty"`
+	EtcdS3Folder             string        `json:"etcd-s3-folder,omitempty"`
+	EtcdS3Timeout            time.Duration `json:"etcd-s3-timeout,omitempty"`
+	EtcdS3Insecure           bool          `json:"etcd-s3-insecure,omitempty"`
+	ServiceLBNamespace       string        `json:"servicelb-namespace,omitempty"`
+	*AgentConfig
+}
+
+type AgentConfig struct {
+	ResolvConf               string   `json:"resolv-conf,omitempty"`
+	NodeIP                   []string `json:"node-ip,omitempty"`
+	NodeExternalIP           []string `json:"node-external-ip,omitempty"`
+	NodeName                 string   `json:"node-name,omitempty"`
+	PauseImage               string   `json:"pause-image,omitempty"`
+	Snapshotter              string   `json:"snapshotter,omitempty"`
+	Docker                   bool     `json:"docker,omitempty"`
+	ContainerRuntimeEndpoint string   `json:"container-runtime-endpoint,omitempty"`
+	FlannelIface             string   `json:"flannel-iface,omitempty"`
+	FlannelConf              string   `json:"flannel-conf,omitempty"`
+	FlannelCniConfFile       string   `json:"flannel-cni-conf,omitempty"`
+	VPNAuth                  string   `json:"vpn-auth,omitempty"`
+	VPNAuthFile              string   `json:"vpn-auth-file,omitempty"`
+	Debug                    bool     `json:"debug,omitempty"`
+	RootlessAlreadyUnshared  bool     `json:"-"`
+	WithNodeID               bool     `json:"with-node-id,omitempty"`
+	EnableSELinux            bool     `json:"selinux,omitempty"`
+	ProtectKernelDefaults    bool     `json:"protect-kernel-defaults,omitempty"`
+	PrivateRegistry          string   `json:"private-registry,omitempty"`
+	AirgapExtraRegistry      []string `json:"airgap-extra-registry,omitempty"`
+	ExtraKubeletArgs         []string `json:"kubelet-arg,omitempty"`
+	ExtraKubeProxyArgs       []string `json:"kube-proxy-arg,omitempty"`
+	Labels                   []string `json:"node-label,omitempty"`
+	Taints                   []string `json:"node-taint,omitempty"`
+	ImageCredProvBinDir      string   `json:"image-credential-provider-bin-dir,omitempty"`
+	ImageCredProvConfig      string   `json:"image-credential-provider-config,omitempty"`
+}

pkg/runtime/kubernetes/certs.go

@@ -88,14 +88,14 @@ func (k *KubeadmRuntime) saveNewKubeadmConfig() error {
 	if err != nil {
 		return err
 	}
-	certPath := path.Join(k.getContentData().EtcPath(), defaultUpdateKubeadmFileName)
+	certPath := path.Join(k.pathResolver.EtcPath(), defaultUpdateKubeadmFileName)
 	return yaml.MarshalYamlToFile(certPath, obj)
 }
 
 func (k *KubeadmRuntime) uploadConfigFromKubeadm() error {
 	logger.Info("start to upload kubeadm config for inCluster ...")
-	in := path.Join(k.getContentData().EtcPath(), defaultUpdateKubeadmFileName)
-	out := path.Join(k.getContentData().ConfigsPath(), defaultUpdateKubeadmFileName)
+	in := path.Join(k.pathResolver.EtcPath(), defaultUpdateKubeadmFileName)
+	out := path.Join(k.pathResolver.ConfigsPath(), defaultUpdateKubeadmFileName)
 	data, err := file.ReadAll(in)
 	if err != nil {
 		return err

pkg/runtime/kubernetes/commands.go

@@ -51,9 +51,9 @@ const (
 
 func (k *KubeadmRuntime) Command(version string, cmdType CommandType) (cmd string) {
 	initConfigPath := k.getInitMasterKubeadmConfigFilePath()
-	joinMasterConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinMasterKubeadmFileName)
-	joinNodeConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinNodeKubeadmFileName)
-	updateClusterConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultUpdateKubeadmFileName)
+	joinMasterConfigPath := path.Join(k.pathResolver.ConfigsPath(), defaultJoinMasterKubeadmFileName)
+	joinNodeConfigPath := path.Join(k.pathResolver.ConfigsPath(), defaultJoinNodeKubeadmFileName)
+	updateClusterConfigPath := path.Join(k.pathResolver.ConfigsPath(), defaultUpdateKubeadmFileName)
 
 	var discoveryTokens []string
 	for _, data := range k.getTokenCaCertHash() {

pkg/runtime/kubernetes/consts.go

@@ -14,6 +14,8 @@
 
 package kubernetes
 
+const Distribution = "kubernetes"
+
 const (
 	defaultInitKubeadmFileName       = "kubeadm-init.yaml"
 	defaultJoinMasterKubeadmFileName = "kubeadm-join-master.yaml"

pkg/runtime/kubernetes/init.go

@@ -28,8 +28,8 @@ func (k *KubeadmRuntime) InitKubeadmConfigToMaster0() error {
 	if err != nil {
 		return fmt.Errorf("generate init config error: %v", err)
 	}
-	initConfigPath := path.Join(k.getContentData().TmpPath(), defaultInitKubeadmFileName)
-	outConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultInitKubeadmFileName)
+	initConfigPath := path.Join(k.pathResolver.TmpPath(), defaultInitKubeadmFileName)
+	outConfigPath := path.Join(k.pathResolver.ConfigsPath(), defaultInitKubeadmFileName)
 	err = file.WriteFile(initConfigPath, data)
 	if err != nil {
 		return fmt.Errorf("failed to write tmp init kubeadm config: %v", err)
@@ -50,16 +50,16 @@ func (k *KubeadmRuntime) GenerateCert() error {
 		return fmt.Errorf("get hostname failed %v", err)
 	}
 
-	logger.Debug("GenerateCert param:", k.getContentData().PkiPath(),
-		k.getContentData().PkiEtcdPath(),
+	logger.Debug("GenerateCert param:", k.pathResolver.PkiPath(),
+		k.pathResolver.PkiEtcdPath(),
 		k.getCertSANs(),
 		k.getMaster0IP(),
 		hostName,
 		k.getServiceCIDR(),
 		k.getDNSDomain())
 	return cert.GenerateCert(
-		k.getContentData().PkiPath(),
-		k.getContentData().PkiEtcdPath(),
+		k.pathResolver.PkiPath(),
+		k.pathResolver.PkiEtcdPath(),
 		k.getCertSANs(),
 		k.getMaster0IP(),
 		hostName,
@@ -79,11 +79,11 @@ func (k *KubeadmRuntime) CreateKubeConfigFiles() error {
 		return fmt.Errorf("get hostname failed %v", err)
 	}
 	certConfig := cert.Config{
-		Path:     k.getContentData().PkiPath(),
+		Path:     k.pathResolver.PkiPath(),
 		BaseName: "ca",
 	}
 
-	err = cert.CreateJoinControlPlaneKubeConfigFiles(k.getContentData().EtcPath(),
+	err = cert.CreateJoinControlPlaneKubeConfigFiles(k.pathResolver.EtcPath(),
 		certConfig, hostName, k.getClusterAPIServer(), "kubernetes")
 	if err != nil {
 		return fmt.Errorf("failed to generate kubeconfig: %v", err)

pkg/runtime/kubernetes/kubeadm.go

@@ -28,7 +28,8 @@ import (
 	"k8s.io/apimachinery/pkg/util/json"
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/constants"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
 	fileutil "github.com/labring/sealos/pkg/utils/file"
 	"github.com/labring/sealos/pkg/utils/iputils"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -48,7 +49,7 @@ var (
 )
 
 const (
-	DefaultVIP = "10.103.97.2"
+	defaultVIP = "10.103.97.2"
 )
 
 // k.getKubeVersion can't be empty
@@ -100,7 +101,7 @@ func gte(v1, v2 *semver.Version) bool {
 }
 
 func (k *KubeadmRuntime) getCGroupDriver(node string) (string, error) {
-	driver, err := k.getRemoteInterface().CGroup(node)
+	driver, err := k.remoteUtil.CGroup(node)
 	if err != nil {
 		return "", err
 	}
@@ -154,7 +155,7 @@ func (k *KubeadmRuntime) validateVIP(ip string) error {
 }
 
 func (k *KubeadmRuntime) getDefaultKubeadmConfig() string {
-	return filepath.Join(k.getContentData().RootFSEtcPath(), defaultRootfsKubeadmFileName)
+	return filepath.Join(k.pathResolver.RootFSEtcPath(), defaultRootfsKubeadmFileName)
 }
 
 func (k *KubeadmRuntime) getVip() string {
@@ -163,7 +164,7 @@ func (k *KubeadmRuntime) getVip() string {
 
 func (k *KubeadmRuntime) getAPIServerPort() int32 {
 	if k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort == 0 {
-		k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort = types.DefaultAPIServerPort
+		k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort = constants.DefaultAPIServerPort
 	}
 	return k.kubeadmConfig.InitConfiguration.LocalAPIEndpoint.BindPort
 }
@@ -259,7 +260,7 @@ func (k *KubeadmRuntime) getServiceCIDR() string {
 
 func (k *KubeadmRuntime) getDNSDomain() string {
 	if k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain == "" {
-		k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain = types.DefaultDNSDomain
+		k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain = constants.DefaultDNSDomain
 	}
 	return k.kubeadmConfig.ClusterConfiguration.Networking.DNSDomain
 }
@@ -287,7 +288,7 @@ func (k *KubeadmRuntime) writeTokenFile(file string) error {
 func (k *KubeadmRuntime) setKubernetesToken() error {
 	if k.token == nil {
 		logger.Info("start to get kubernetes token...")
-		tokenFile := path.Join(k.getContentData().EtcPath(), defaultKubeadmTokenFileName)
+		tokenFile := path.Join(k.pathResolver.EtcPath(), defaultKubeadmTokenFileName)
 		if !fileutil.IsExist(tokenFile) {
 			err := k.writeTokenFile(tokenFile)
 			if err != nil {
@@ -426,7 +427,7 @@ func (k *KubeadmRuntime) getEtcdDataDir() string {
 }
 
 func (k *KubeadmRuntime) getCRISocket(node string) (string, error) {
-	criSocket, err := k.getRemoteInterface().Socket(node)
+	criSocket, err := k.remoteUtil.Socket(node)
 	if err != nil {
 		return "", err
 	}
@@ -445,7 +446,7 @@ var setCGroupDriverAndSocket = func(krt *KubeadmRuntime) error {
 }
 
 var setCertificateKey = func(krt *KubeadmRuntime) error {
-	certificateKeyFile := path.Join(krt.getContentData().EtcPath(), defaultCertificateKeyFileName)
+	certificateKeyFile := path.Join(krt.pathResolver.EtcPath(), defaultCertificateKeyFileName)
 	var key string
 	if !fileutil.IsExist(certificateKeyFile) {
 		key, _ = rand.CreateCertificateKey()

pkg/runtime/kubernetes/kubeconfig.go

@@ -21,7 +21,7 @@ import "path"
 const copyKubeAdminConfigCommand = `rm -rf $HOME/.kube/config && mkdir -p $HOME/.kube && cp /etc/kubernetes/admin.conf $HOME/.kube/config`
 
 func (k *KubeadmRuntime) copyKubeConfigFileToNodes(hosts []string) error {
-	src := k.getContentData().AdminFile()
+	src := k.pathResolver.AdminFile()
 	dst := path.Join(".kube", "config")
 	return k.sendFileToHosts(hosts, src, dst)
 }

pkg/runtime/kubernetes/master.go

@@ -66,8 +66,8 @@ func (k *KubeadmRuntime) ConfigJoinMasterKubeadmToMaster(master string) error {
 	if err != nil {
 		return fmt.Errorf("failed to generate join master kubeadm config: %s", err.Error())
 	}
-	joinConfigPath := path.Join(k.getContentData().TmpPath(), defaultJoinMasterKubeadmFileName)
-	outConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinMasterKubeadmFileName)
+	joinConfigPath := path.Join(k.pathResolver.TmpPath(), defaultJoinMasterKubeadmFileName)
+	outConfigPath := path.Join(k.pathResolver.ConfigsPath(), defaultJoinMasterKubeadmFileName)
 	err = file.WriteFile(joinConfigPath, data)
 	if err != nil {
 		return fmt.Errorf("write config join master kubeadm config error: %s", err.Error())
@@ -85,7 +85,7 @@ func (k *KubeadmRuntime) joinMasters(masters []string) error {
 	}
 	logger.Info("start to send manifests to masters...")
 	var err error
-	if err = ssh.WaitSSHReady(k.getSSHInterface(), 6, masters...); err != nil {
+	if err = ssh.WaitReady(k.sshClient, 6, masters...); err != nil {
 		return fmt.Errorf("join masters wait for ssh ready time out: %w", err)
 	}
 
@@ -176,7 +176,7 @@ func (k *KubeadmRuntime) deleteMaster(master string) error {
 		masterIPs := strings.SliceRemoveStr(k.getMasterIPList(), master)
 		if len(masterIPs) > 0 {
 			// TODO: do we need draining first?
-			if err := k.RemoveNodeFromK8sClient(master); err != nil {
+			if err := k.removeNode(master); err != nil {
 				logger.Warn(fmt.Errorf("delete master %s failed %v", master, err))
 			}
 		}

pkg/runtime/kubernetes/node.go

@@ -29,7 +29,7 @@ import (
 
 func (k *KubeadmRuntime) joinNodes(newNodesIPList []string) error {
 	var err error
-	if err = ssh.WaitSSHReady(k.getSSHInterface(), 6, newNodesIPList...); err != nil {
+	if err = ssh.WaitReady(k.sshClient, 6, newNodesIPList...); err != nil {
 		return fmt.Errorf("join nodes wait for ssh ready time out: %w", err)
 	}
 
@@ -85,8 +85,8 @@ func (k *KubeadmRuntime) copyKubeadmConfigToNode(node string) error {
 	if err != nil {
 		return fmt.Errorf("failed to generate join kubeadm config: %v", err)
 	}
-	joinConfigPath := path.Join(k.getContentData().TmpPath(), defaultJoinNodeKubeadmFileName)
-	outConfigPath := path.Join(k.getContentData().ConfigsPath(), defaultJoinNodeKubeadmFileName)
+	joinConfigPath := path.Join(k.pathResolver.TmpPath(), defaultJoinNodeKubeadmFileName)
+	outConfigPath := path.Join(k.pathResolver.ConfigsPath(), defaultJoinNodeKubeadmFileName)
 	err = file.WriteFile(joinConfigPath, data)
 	if err != nil {
 		return fmt.Errorf("write config join kubeadm config error: %s", err.Error())
@@ -121,7 +121,7 @@ func (k *KubeadmRuntime) deleteNode(node string) error {
 	return k.resetNode(node, func() {
 		//remove node
 		if len(k.getMasterIPList()) > 0 {
-			if err := k.RemoveNodeFromK8sClient(node); err != nil {
+			if err := k.removeNode(node); err != nil {
 				logger.Warn(fmt.Errorf("delete node %s failed %v", node, err))
 			}
 		}

pkg/runtime/kubernetes/runtime.go

@@ -24,7 +24,7 @@ import (
 	"github.com/labring/sealos/pkg/client-go/kubernetes"
 	"github.com/labring/sealos/pkg/constants"
 	"github.com/labring/sealos/pkg/remote"
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
 	"github.com/labring/sealos/pkg/ssh"
 	v2 "github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -38,12 +38,12 @@ type KubeadmRuntime struct {
 	token         *types.Token
 	kubeadmConfig *types.KubeadmConfig // a deep copy from config.KubeadmConfig or a new one
 
-	klogLevel     int
-	cli           kubernetes.Client
-	clusterClient ssh.Interface
-	pathResolver  constants.PathResolver
-	remoteUtil    remote.Interface
-	mu            sync.Mutex
+	klogLevel    int
+	cli          kubernetes.Client
+	sshClient    ssh.Interface
+	pathResolver constants.PathResolver
+	remoteUtil   remote.Interface
+	mu           sync.Mutex
 }
 
 func (k *KubeadmRuntime) Init() error {
@@ -65,13 +65,14 @@ func (k *KubeadmRuntime) GetRawConfig() ([]byte, error) {
 	if err := k.CompleteKubeadmConfig(); err != nil {
 		return nil, err
 	}
-	k.cluster.Status = v2.ClusterStatus{}
+	cluster := k.cluster.DeepCopy()
+	cluster.Status = v2.ClusterStatus{}
 
 	conversion, err := k.kubeadmConfig.ToConvertedKubeadmConfig()
 	if err != nil {
 		return nil, err
 	}
-	objects := []interface{}{k.cluster,
+	objects := []interface{}{cluster,
 		conversion.InitConfiguration,
 		conversion.ClusterConfiguration,
 		conversion.JoinConfiguration,
@@ -122,15 +123,15 @@ func (k *KubeadmRuntime) ScaleDown(deleteMastersIPList []string, deleteNodesIPLi
 }
 
 func newKubeadmRuntime(cluster *v2.Cluster, kubeadm *types.KubeadmConfig) (*KubeadmRuntime, error) {
-	sshClient := ssh.NewSSHByCluster(cluster, true)
+	sshClient := ssh.NewCacheClientFromCluster(cluster, true)
 	k := &KubeadmRuntime{
 		cluster: cluster,
 		config: &types.Config{
 			KubeadmConfig:   kubeadm,
-			APIServerDomain: types.DefaultAPIServerDomain,
+			APIServerDomain: constants.DefaultAPIServerDomain,
 		},
 		kubeadmConfig: types.NewKubeadmConfig(),
-		clusterClient: sshClient,
+		sshClient:     sshClient,
 		pathResolver:  constants.NewPathResolver(cluster.GetName()),
 		remoteUtil:    remote.New(cluster.GetName(), sshClient),
 	}
@@ -143,7 +144,11 @@ func newKubeadmRuntime(cluster *v2.Cluster, kubeadm *types.KubeadmConfig) (*Kube
 	return k, nil
 }
 
-func New(cluster *v2.Cluster, kubeadm *types.KubeadmConfig) (*KubeadmRuntime, error) {
+func New(cluster *v2.Cluster, config any) (*KubeadmRuntime, error) {
+	var kubeadm *types.KubeadmConfig
+	if v, ok := config.(*types.KubeadmConfig); ok {
+		kubeadm = v
+	}
 	return newKubeadmRuntime(cluster, kubeadm)
 }
 

pkg/runtime/kubernetes/runtime_getter.go

@@ -27,8 +27,6 @@ import (
 	"github.com/labring/sealos/pkg/client-go/kubernetes"
 	"github.com/labring/sealos/pkg/constants"
 	"github.com/labring/sealos/pkg/env"
-	"github.com/labring/sealos/pkg/remote"
-	"github.com/labring/sealos/pkg/ssh"
 	"github.com/labring/sealos/pkg/types/v1beta1"
 	"github.com/labring/sealos/pkg/utils/iputils"
 	"github.com/labring/sealos/pkg/utils/logger"
@@ -86,35 +84,37 @@ func (k *KubeadmRuntime) getMaster0IPAPIServer() string {
 	return fmt.Sprintf("https://%s:%d", master0, k.getAPIServerPort())
 }
 
-func (k *KubeadmRuntime) getLvscareImage() (string, error) {
-	labels := k.getImageLabels()
-	image := labels[v1beta1.ImageKubeLvscareImageKey]
-	if image == "" {
-		image = constants.DefaultLvsCareImage
+func (k *KubeadmRuntime) getLvscareImage() string {
+	img := k.cluster.GetRootfsImage()
+	if img != nil {
+		return getLabelValueOrDefault(img.Labels, v1beta1.ImageKubeLvscareImageKey, constants.DefaultLvsCareImage)
 	}
-	return image, nil
+	return constants.DefaultLvsCareImage
+}
+
+func getLabelValueOrDefault(m map[string]string, name string, defaultVal string) string {
+	if v, ok := m[name]; ok && v != "" {
+		return v
+	}
+	return defaultVal
 }
 
 func (k *KubeadmRuntime) getVIPFromImage() string {
-	labels := k.getImageLabels()
-	vip := labels[v1beta1.ImageVIPKey]
-	if vip == "" {
-		vip = DefaultVIP
-	} else {
-		envsInRootFsImage := k.cluster.GetRootfsImage().Env
-		envs := maps.MergeMap(envsInRootFsImage, k.getEnvInterface().Getenv(k.getMaster0IP()))
-		vip = stringsutil.RenderTextFromEnv(vip, envs)
+	img := k.cluster.GetRootfsImage()
+	if img != nil {
+		vip := getLabelValueOrDefault(img.Labels, v1beta1.ImageVIPKey, defaultVIP)
+		envs := maps.MergeMap(img.Env, k.getEnvInterface().Getenv(k.getMaster0IP()))
+		return stringsutil.RenderTextFromEnv(vip, envs)
 	}
-	logger.Debug("get vip is %s", vip)
-	return vip
+	return defaultVIP
 }
 
 func (k *KubeadmRuntime) execIPVS(ip string, masters []string) error {
-	return k.getRemoteInterface().IPVS(ip, k.getVipAndPort(), masters)
+	return k.remoteUtil.IPVS(ip, k.getVipAndPort(), masters)
 }
 
 func (k *KubeadmRuntime) execIPVSClean(ip string) error {
-	return k.getRemoteInterface().IPVSClean(ip, k.getVipAndPort())
+	return k.remoteUtil.IPVSClean(ip, k.getVipAndPort())
 }
 
 func (k *KubeadmRuntime) syncNodeIPVSYaml(masterIPs, nodesIPs []string) error {
@@ -139,24 +139,21 @@ func (k *KubeadmRuntime) syncNodeIPVSYaml(masterIPs, nodesIPs []string) error {
 }
 
 func (k *KubeadmRuntime) execIPVSPod(ip string, masters []string) error {
-	image, err := k.getLvscareImage()
-	if err != nil {
-		return err
-	}
-	return k.getRemoteInterface().StaticPod(ip, k.getVipAndPort(), constants.LvsCareStaticPodName, image, masters)
+	image := k.getLvscareImage()
+	return k.remoteUtil.StaticPod(ip, k.getVipAndPort(), constants.LvsCareStaticPodName, image, masters)
 }
 
 func (k *KubeadmRuntime) execToken(ip, certificateKey string) (string, error) {
-	return k.getRemoteInterface().Token(ip, k.getInitMasterKubeadmConfigFilePath(), certificateKey)
+	return k.remoteUtil.Token(ip, k.getInitMasterKubeadmConfigFilePath(), certificateKey)
 }
 
 func (k *KubeadmRuntime) execHostname(ip string) (string, error) {
-	hostname, err := k.getRemoteInterface().Hostname(ip)
+	hostname, err := k.remoteUtil.Hostname(ip)
 	return strings.ToLower(hostname), err
 }
 
 func (k *KubeadmRuntime) execHostsAppend(ip, host, domain string) error {
-	return k.getRemoteInterface().HostsAdd(ip, iputils.GetHostIP(host), domain)
+	return k.remoteUtil.HostsAdd(ip, iputils.GetHostIP(host), domain)
 }
 
 func (k *KubeadmRuntime) execCert(ip string) error {
@@ -164,54 +161,34 @@ func (k *KubeadmRuntime) execCert(ip string) error {
 	if err != nil {
 		return err
 	}
-	return k.getRemoteInterface().Cert(ip, k.getCertSANs(), iputils.GetHostIP(ip), hostname, k.getServiceCIDR(), k.getDNSDomain())
+	return k.remoteUtil.Cert(ip, k.getCertSANs(), iputils.GetHostIP(ip), hostname, k.getServiceCIDR(), k.getDNSDomain())
 }
 
 func (k *KubeadmRuntime) execHostsDelete(ip, domain string) error {
-	return k.getRemoteInterface().HostsDelete(ip, domain)
+	return k.remoteUtil.HostsDelete(ip, domain)
 }
 
 func (k *KubeadmRuntime) sshCmdAsync(host string, cmd ...string) error {
-	return k.getSSHInterface().CmdAsync(host, cmd...)
+	return k.sshClient.CmdAsync(host, cmd...)
 }
 
 func (k *KubeadmRuntime) sshCmdToString(host string, cmd string) (string, error) {
-	return k.getSSHInterface().CmdToString(host, cmd, "")
+	return k.sshClient.CmdToString(host, cmd, "")
 }
 
 func (k *KubeadmRuntime) sshCopy(host, srcFilePath, dstFilePath string) error {
-	if srcFilePath == dstFilePath {
-		logger.Info("src and dst is same path , skip copy %s", srcFilePath)
-		return nil
-	}
-	return k.getSSHInterface().Copy(host, srcFilePath, dstFilePath)
-}
-
-func (k *KubeadmRuntime) getImageLabels() map[string]string {
-	return k.cluster.GetImageLabels()
-}
-
-func (k *KubeadmRuntime) getSSHInterface() ssh.Interface {
-	return k.clusterClient
+	return k.sshClient.Copy(host, srcFilePath, dstFilePath)
 }
 
 func (k *KubeadmRuntime) getEnvInterface() env.Interface {
 	return env.NewEnvProcessor(k.cluster)
 }
 
-func (k *KubeadmRuntime) getRemoteInterface() remote.Interface {
-	return k.remoteUtil
-}
-
-func (k *KubeadmRuntime) getContentData() constants.PathResolver {
-	return k.pathResolver
-}
-
 func (k *KubeadmRuntime) getKubeInterface() (kubernetes.Client, error) {
 	if k.cli != nil {
 		return k.cli, nil
 	}
-	cli, err := kubernetes.NewKubernetesClient(k.getContentData().AdminFile(), k.getMaster0IPAPIServer())
+	cli, err := kubernetes.NewKubernetesClient(k.pathResolver.AdminFile(), k.getMaster0IPAPIServer())
 	if err != nil {
 		return nil, err
 	}
@@ -228,5 +205,5 @@ func (k *KubeadmRuntime) getKubeExpansion() (kubernetes.Expansion, error) {
 }
 
 func (k *KubeadmRuntime) getInitMasterKubeadmConfigFilePath() string {
-	return path.Join(k.getContentData().ConfigsPath(), defaultInitKubeadmFileName)
+	return path.Join(k.pathResolver.ConfigsPath(), defaultInitKubeadmFileName)
 }

pkg/runtime/kubernetes/static_files.go

@@ -50,7 +50,7 @@ func (k *KubeadmRuntime) CopyStaticFilesToMasters() error {
 func (k *KubeadmRuntime) copyStaticFiles(nodes []string) error {
 	logger.Info("start to copy static files to masters")
 	for _, file := range MasterStaticFiles {
-		staticFilePath := filepath.Join(k.getContentData().RootFSStaticsPath(), file.Name)
+		staticFilePath := filepath.Join(k.pathResolver.RootFSStaticsPath(), file.Name)
 		cmdLinkStatic := fmt.Sprintf(copyFileToDirCommand, file.DestinationDir, staticFilePath, filepath.Join(file.DestinationDir, file.Name))
 		eg, _ := errgroup.WithContext(context.Background())
 		for _, host := range nodes {

pkg/runtime/kubernetes/types/consts.go

@@ -18,8 +18,4 @@ const (
 	KubeadmV1beta1 = "kubeadm.k8s.io/v1beta1"
 	KubeadmV1beta2 = "kubeadm.k8s.io/v1beta2"
 	KubeadmV1beta3 = "kubeadm.k8s.io/v1beta3"
-
-	DefaultAPIServerDomain = "apiserver.cluster.local"
-	DefaultDNSDomain       = "cluster.local"
-	DefaultAPIServerPort   = 6443
 )

pkg/runtime/kubernetes/upgrade.go

@@ -25,7 +25,7 @@ import (
 	"k8s.io/kubernetes/cmd/kubeadm/app/apis/kubeadm"
 
 	"github.com/labring/sealos/pkg/runtime/decode"
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
 	"github.com/labring/sealos/pkg/utils/logger"
 	"github.com/labring/sealos/pkg/utils/yaml"
 )
@@ -75,13 +75,13 @@ func (k *KubeadmRuntime) upgradeMaster0(version string) error {
 			return err
 		}
 	}
-	master0Name, err := k.getRemoteInterface().Hostname(master0ip)
+	master0Name, err := k.remoteUtil.Hostname(master0ip)
 	if err != nil {
 		return err
 	}
 	//default nodeName in k8s is the lower case of their hostname because of DNS protocol.
 	master0Name = strings.ToLower(master0Name)
-	kubeBinaryPath := k.getContentData().RootFSBinPath()
+	kubeBinaryPath := k.pathResolver.RootFSBinPath()
 	//assure the connection to api-server succeed before executing upgrade cmds
 	if err = k.pingAPIServer(); err != nil {
 		return err
@@ -114,13 +114,13 @@ func (k *KubeadmRuntime) upgradeOtherNodes(ips []string, version string) error {
 				return err
 			}
 		}
-		nodename, err := k.getRemoteInterface().Hostname(ip)
+		nodename, err := k.remoteUtil.Hostname(ip)
 		if err != nil {
 			return err
 		}
 		//default nodeName in k8s is the lower case of their hostname because of DNS protocol.
 		nodename = strings.ToLower(nodename)
-		kubeBinaryPath := k.getContentData().RootFSBinPath()
+		kubeBinaryPath := k.pathResolver.RootFSBinPath()
 		//assure the connection to api-server succeed before executing upgrade cmds
 		if err = k.pingAPIServer(); err != nil {
 			return err

pkg/runtime/kubernetes/utils.go

@@ -77,13 +77,13 @@ func (k *KubeadmRuntime) ReplaceKubeConfigV1991V1992(masters []string) bool {
 
 func (k *KubeadmRuntime) sendKubeConfigFile(hosts []string, filename string) error {
 	dst := path.Join(kubernetesEtc, filename)
-	src := path.Join(k.getContentData().EtcPath(), filename)
+	src := path.Join(k.pathResolver.EtcPath(), filename)
 	return k.sendFileToHosts(hosts, src, dst)
 }
 
 func (k *KubeadmRuntime) sendNewCertAndKey(hosts []string) error {
 	logger.Info("start to copy etc pki files to masters")
-	return k.sendFileToHosts(hosts, k.getContentData().PkiPath(), kubernetesEtcPKI)
+	return k.sendFileToHosts(hosts, k.pathResolver.PkiPath(), kubernetesEtcPKI)
 }
 
 func (k *KubeadmRuntime) sendFileToHosts(Hosts []string, src, dst string) error {
@@ -100,25 +100,22 @@ func (k *KubeadmRuntime) sendFileToHosts(Hosts []string, src, dst string) error
 	return eg.Wait()
 }
 
-func (k *KubeadmRuntime) RemoveNodeFromK8sClient(ip string) error {
+func (k *KubeadmRuntime) removeNode(ip string) error {
 	logger.Info("start to remove node from k8s %s", ip)
-	ctx := context.Background()
 	client, err := k.getKubeInterface()
 	if err != nil {
 		return err
 	}
+	ctx := context.Background()
 	exp := kubernetes.NewKubeExpansion(client.Kubernetes())
 	hostname, err := exp.FetchHostNameFromInternalIP(ctx, ip)
 	if err != nil {
-		return fmt.Errorf("kubernetes client get hostname %s failed %v,skip delete node", ip, err)
+		return fmt.Errorf("cannot get node with ip address %s: %v", ip, err)
 	}
 	deletePropagation := v1.DeletePropagationBackground
 	err = client.Kubernetes().CoreV1().Nodes().Delete(ctx, hostname, v1.DeleteOptions{PropagationPolicy: &deletePropagation})
-	if err != nil {
-		if errors.IsNotFound(err) {
-			return fmt.Errorf("not find target delete node ip: %s", ip)
-		}
-		return fmt.Errorf("kubernetes client delete node %s failed %v,skip delete node", ip, err)
+	if err != nil && !errors.IsNotFound(err) {
+		return err
 	}
 	return nil
 }

pkg/runtime/utils/token.go

@@ -32,7 +32,7 @@ import (
 	"k8s.io/client-go/util/cert"
 	v1 "k8s.io/kubernetes/cmd/kubeadm/app/apis/bootstraptoken/v1"
 
-	"github.com/labring/sealos/pkg/runtime/types"
+	"github.com/labring/sealos/pkg/runtime/kubernetes/types"
 	"github.com/labring/sealos/pkg/utils/exec"
 	"github.com/labring/sealos/pkg/utils/file"
 	"github.com/labring/sealos/pkg/utils/logger"

pkg/ssh/exec.go

@@ -54,7 +54,7 @@ func NewExecCmdFromIPs(cluster *v2.Cluster, ips []string) (Exec, error) {
 }
 
 func (e *Exec) RunCmd(cmd string) error {
-	sshClient := NewSSHByCluster(e.cluster, true)
+	sshClient := NewCacheClientFromCluster(e.cluster, true)
 	eg, _ := errgroup.WithContext(context.Background())
 	for _, ipAddr := range e.ipList {
 		ip := ipAddr
@@ -69,7 +69,7 @@ func (e *Exec) RunCmd(cmd string) error {
 }
 
 func (e *Exec) RunCopy(srcFilePath, dstFilePath string) error {
-	sshClient := NewSSHByCluster(e.cluster, true)
+	sshClient := NewCacheClientFromCluster(e.cluster, true)
 	eg, _ := errgroup.WithContext(context.Background())
 	for _, ipAddr := range e.ipList {
 		ip := ipAddr

pkg/ssh/ssh.go

@@ -160,7 +160,7 @@ func newFromSSH(ssh *v2.SSH, isStdout bool) (Interface, error) {
 	return New(newOptionFromSSH(ssh, isStdout))
 }
 
-func NewSSHClient(ssh *v2.SSH, isStdout bool) Interface {
+func MustNewClient(ssh *v2.SSH, isStdout bool) Interface {
 	client, err := newFromSSH(ssh, isStdout)
 	if err != nil {
 		logger.Fatal("failed to create ssh client: %v", err)
@@ -168,7 +168,7 @@ func NewSSHClient(ssh *v2.SSH, isStdout bool) Interface {
 	return client
 }
 
-func NewSSHByCluster(cluster *v2.Cluster, isStdout bool) Interface {
+func NewCacheClientFromCluster(cluster *v2.Cluster, isStdout bool) Interface {
 	cc := &clusterClient{
 		cluster:  cluster,
 		isStdout: isStdout,
@@ -178,7 +178,7 @@ func NewSSHByCluster(cluster *v2.Cluster, isStdout bool) Interface {
 	return cc
 }
 
-func WaitSSHReady(client Interface, _ int, hosts ...string) error {
+func WaitReady(client Interface, _ int, hosts ...string) error {
 	eg, _ := errgroup.WithContext(context.Background())
 	for i := range hosts {
 		host := hosts[i]

pkg/template/template.go

@@ -29,6 +29,10 @@ func init() {
 		Funcs(funcMap())
 }
 
+func Parse(text string) (*template.Template, error) {
+	return defaultTpl.Parse(text)
+}
+
 func TryParse(text string) (*template.Template, bool, error) {
 	tmp, err := defaultTpl.Parse(text)
 	isFailed := err != nil && err.Error() == "text/template: cannot Parse after Execute"

pkg/types/v1beta1/utils.go

@@ -268,3 +268,11 @@ func (c *Cluster) GetRolesByIP(ip string) []string {
 	}
 	return routes
 }
+
+func (c *Cluster) GetDistribution() string {
+	root := c.GetRootfsImage()
+	if root != nil {
+		return root.Labels["sealos.io.distribution"]
+	}
+	return ""
+}

test/e2e/apply_infra_test.go

@@ -132,7 +132,7 @@ var _ = Describe("E2E_sealos_apply_infra_test", func() {
 
 			testApplier = &apply.Applier{EIp: eip, InfraDriver: infraDriver,
 				RemoteCmd: cmd2.Interface(&cmd2.RemoteCmd{Host: eip[0],
-					Interface: ssh.NewSSHClient(sshInterface, true)}),
+					Interface: ssh.MustNewClient(sshInterface, true)}),
 				LocalCmd: &cmd2.LocalCmd{}, Infra: infraCheck, SSH: sshInterface}
 			testApplier.Init()
 		})