From 95493cfa3ea4ec53bafbfb953618cb572dda1da5 Mon Sep 17 00:00:00 2001 From: fengxsong Date: Thu, 17 Aug 2023 21:23:04 +0800 Subject: [PATCH] Sweep: run guest command one by one and support run patch-type image (#3677) * refactor: run guest command one by one and support run patch-type image Signed-off-by: fengxsong * fix: ci lints and remove mounts parameter Signed-off-by: fengxsong * fix: test case Signed-off-by: fengxsong * fix: vip address rendering Signed-off-by: fengxsong * fix: test case Signed-off-by: fengxsong * feature(main): add test case Signed-off-by: cuisongliu * fix: inject rootfs env into bootstrap process Signed-off-by: fengxsong * feature(main): add join using guest process Signed-off-by: cuisongliu * fix: add targetHosts parameter as guest commands are not always applied to all hosts Signed-off-by: fengxsong * feature(main): add patch image Signed-off-by: cuisongliu * refactor: add context parameter and pass it along side to the end Signed-off-by: fengxsong * test(main): add test for patch image Signed-off-by: cuisongliu * test(main): add test for patch image Signed-off-by: cuisongliu * test(main): add test for patch image Signed-off-by: cuisongliu * test(main): add test for patch image Signed-off-by: cuisongliu --------- Signed-off-by: fengxsong Signed-off-by: cuisongliu Co-authored-by: cuisongliu --- .github/workflows/e2e_test_core.yml | 1 + cmd/sealos/cmd/apply.go | 2 +- cmd/sealos/cmd/gen.go | 2 +- cmd/sealos/cmd/reset.go | 2 +- cmd/sealos/cmd/run.go | 4 +- cmd/sealos/cmd/version.go | 5 +- docker/patch/Dockerfile | 3 +- docker/patch/save-cluster-images.sh | 2 +- docker/patch/upgrade.sh | 5 + pkg/apply/apply.go | 8 +- pkg/apply/apply_test.go | 10 +- .../applydrivers/apply_drivers_default.go | 15 ++- pkg/apply/args.go | 4 - pkg/apply/gen.go | 7 +- pkg/apply/gen_test.go | 6 +- pkg/apply/processor/context.go | 47 ++++++++ pkg/apply/processor/create.go | 11 +- pkg/apply/processor/error.go | 47 -------- pkg/apply/processor/errors.go | 30 +++++ pkg/apply/processor/install.go | 9 +- pkg/apply/processor/scale.go | 31 +++++ pkg/apply/reset.go | 29 ++--- pkg/apply/reset_test.go | 6 +- pkg/apply/run.go | 67 +++++------ pkg/apply/run_test.go | 11 +- pkg/apply/scale.go | 2 +- pkg/bootstrap/context.go | 15 ++- pkg/env/env.go | 90 ++++++++------- pkg/env/env_shim_test.go | 4 +- pkg/env/env_test.go | 6 +- pkg/filesystem/rootfs/rootfs_default.go | 53 +++++---- pkg/guest/guest.go | 106 +++++++++++------- pkg/guest/guest_test.go | 7 +- pkg/runtime/runtime_getter.go | 16 +-- pkg/types/v1beta1/cluster.go | 32 +++++- pkg/version/utils/utils.go | 65 +++++++++++ pkg/version/version.go | 46 -------- test/e2e/apply_docker_test.go | 2 +- test/e2e/apply_test.go | 2 +- test/e2e/filesystem_test.go | 2 +- test/e2e/images_buildrun_feature_test.go | 2 +- test/e2e/images_buildrun_test.go | 2 +- test/e2e/run_docker_test.go | 2 +- test/e2e/run_patch_test.go | 85 ++++++++++++++ test/e2e/run_test.go | 2 +- test/e2e/testhelper/config/patchfs.go | 90 +++++++++++++++ test/e2e/testhelper/config/patchfs_test.go | 59 ++++++++++ .../config/{dockerfile.go => rootfs.go} | 10 +- .../{dockerfile_test.go => rootfs_test.go} | 2 +- 49 files changed, 741 insertions(+), 325 deletions(-) create mode 100644 docker/patch/upgrade.sh create mode 100644 pkg/apply/processor/context.go delete mode 100644 pkg/apply/processor/error.go create mode 100644 pkg/version/utils/utils.go create mode 100644 test/e2e/run_patch_test.go create mode 100644 test/e2e/testhelper/config/patchfs.go create mode 100644 test/e2e/testhelper/config/patchfs_test.go rename test/e2e/testhelper/config/{dockerfile.go => rootfs.go} (89%) rename test/e2e/testhelper/config/{dockerfile_test.go => rootfs_test.go} (98%) diff --git a/.github/workflows/e2e_test_core.yml b/.github/workflows/e2e_test_core.yml index 830ed1ce0..4743ef067 100644 --- a/.github/workflows/e2e_test_core.yml +++ b/.github/workflows/e2e_test_core.yml @@ -44,6 +44,7 @@ jobs: E2E_sealos_apply_docker_test, E2E_sealos_apply_other_test, E2E_sealos_filesystem_test, + E2E_sealos_run_patchimage_test, E2E_sealos_runtime_version_119_test, E2E_sealos_runtime_version_120_test, E2E_sealos_runtime_version_121_test, diff --git a/cmd/sealos/cmd/apply.go b/cmd/sealos/cmd/apply.go index 7953a6e6f..92c2539bf 100644 --- a/cmd/sealos/cmd/apply.go +++ b/cmd/sealos/cmd/apply.go @@ -32,7 +32,7 @@ func newApplyCmd() *cobra.Command { Example: `sealos apply -f Clusterfile`, Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - applier, err := apply.NewApplierFromFile(clusterFile, applyArgs) + applier, err := apply.NewApplierFromFile(cmd, clusterFile, applyArgs) if err != nil { return err } diff --git a/cmd/sealos/cmd/gen.go b/cmd/sealos/cmd/gen.go index 70e0eeb8b..378170f83 100644 --- a/cmd/sealos/cmd/gen.go +++ b/cmd/sealos/cmd/gen.go @@ -56,7 +56,7 @@ func newGenCmd() *cobra.Command { Long: `generate a Clusterfile of the kubernetes cluster, which can be applied by 'sealos apply' command`, Example: exampleGen, RunE: func(cmd *cobra.Command, args []string) error { - data, err := apply.NewClusterFromGenArgs(args, genArgs) + data, err := apply.NewClusterFromGenArgs(cmd, genArgs, args) if err != nil { return err } diff --git a/cmd/sealos/cmd/reset.go b/cmd/sealos/cmd/reset.go index 08ec28ec8..c08506878 100644 --- a/cmd/sealos/cmd/reset.go +++ b/cmd/sealos/cmd/reset.go @@ -47,7 +47,7 @@ func newResetCmd() *cobra.Command { } return err } - applier, err := apply.NewApplierFromResetArgs(resetArgs) + applier, err := apply.NewApplierFromResetArgs(cmd, resetArgs) if err != nil { return err } diff --git a/cmd/sealos/cmd/run.go b/cmd/sealos/cmd/run.go index 2dab77743..c8c7e8610 100644 --- a/cmd/sealos/cmd/run.go +++ b/cmd/sealos/cmd/run.go @@ -70,7 +70,7 @@ func newRunCmd() *cobra.Command { return err } - applier, err := apply.NewApplierFromArgs(images, runArgs) + applier, err := apply.NewApplierFromArgs(cmd, runArgs, images) if err != nil { return err } @@ -87,7 +87,7 @@ func newRunCmd() *cobra.Command { runArgs.RegisterFlags(runCmd.Flags()) runCmd.Flags().BoolVar(new(bool), "single", false, "run cluster in single mode") if err := runCmd.Flags().MarkDeprecated("single", "it defaults to running cluster in single mode when there are no master and node"); err != nil { - logger.Error(err) + logger.Fatal(err) } runCmd.Flags().BoolVarP(&processor.ForceOverride, "force", "f", false, "force override app in this cluster") runCmd.Flags().StringVarP(&transport, "transport", "t", buildah.OCIArchive, diff --git a/cmd/sealos/cmd/version.go b/cmd/sealos/cmd/version.go index 359137e52..65128ef98 100644 --- a/cmd/sealos/cmd/version.go +++ b/cmd/sealos/cmd/version.go @@ -27,6 +27,7 @@ import ( "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/utils/logger" "github.com/labring/sealos/pkg/version" + versionutils "github.com/labring/sealos/pkg/version/utils" ) var shortPrint bool @@ -75,8 +76,8 @@ func PrintInfo() error { } return nil } - OutputInfo.KubernetesVersion = version.GetKubernetesVersion(cluster) - OutputInfo.CriRuntimeVersion = version.GetCriRuntimeVersion() + OutputInfo.KubernetesVersion = versionutils.GetKubernetesVersion(cluster) + OutputInfo.CriRuntimeVersion = versionutils.GetCriRuntimeVersion() err = PrintToStd(OutputInfo) if err != nil { diff --git a/docker/patch/Dockerfile b/docker/patch/Dockerfile index af16d4dc0..19d4ef5a3 100644 --- a/docker/patch/Dockerfile +++ b/docker/patch/Dockerfile @@ -1,7 +1,8 @@ FROM scratch -LABEL image="__lvscare__" LABEL sealos.io.type="patch" COPY images ./images COPY registry ./registry COPY sealctl ./opt/sealctl COPY image-cri-shim ./cri/image-cri-shim +COPY upgrade.sh ./upgrade.sh +CMD ["bash upgrade.sh"] diff --git a/docker/patch/save-cluster-images.sh b/docker/patch/save-cluster-images.sh index 5e0b3defc..d862669ec 100755 --- a/docker/patch/save-cluster-images.sh +++ b/docker/patch/save-cluster-images.sh @@ -22,7 +22,7 @@ mkdir -p images/shim echo "${IMAGE}" > images/shim/lvscareImage sed -i "s#__lvscare__#${IMAGE}#g" Dockerfile -sudo sealos build -t "${PATCH}" --platform linux/"${ARCH}" -f Dockerfile . +sudo sealos build -t "${PATCH}" --label=sealos.io.type=patch --label=image="${IMAGE}" --platform linux/"${ARCH}" -f Dockerfile . # save patch image cd - && sudo sealos save -o patch-"${ARCH}".tar "${PATCH}" diff --git a/docker/patch/upgrade.sh b/docker/patch/upgrade.sh new file mode 100644 index 000000000..1e85e8c50 --- /dev/null +++ b/docker/patch/upgrade.sh @@ -0,0 +1,5 @@ +#!/bin/bash +systemctl stop image-cri-shim +cp cri/image-cri-shim /usr/bin/image-cri-shim +systemctl start image-cri-shim +image-cri-shim -v diff --git a/pkg/apply/apply.go b/pkg/apply/apply.go index b6c7505cc..c5fc84aff 100644 --- a/pkg/apply/apply.go +++ b/pkg/apply/apply.go @@ -18,16 +18,17 @@ package apply import ( "fmt" - "os" "path/filepath" + "github.com/spf13/cobra" + "github.com/labring/sealos/pkg/apply/applydrivers" "github.com/labring/sealos/pkg/clusterfile" "github.com/labring/sealos/pkg/constants" ) -func NewApplierFromFile(path string, args *Args) (applydrivers.Interface, error) { +func NewApplierFromFile(cmd *cobra.Command, path string, args *Args) (applydrivers.Interface, error) { if !filepath.IsAbs(path) { pa, err := os.Getwd() if err != nil { @@ -59,7 +60,10 @@ func NewApplierFromFile(path string, args *Args) (applydrivers.Interface, error) } currentCluster := cf.GetCluster() + ctx := withCommonContext(cmd.Context(), cmd) + return &applydrivers.Applier{ + Context: ctx, ClusterDesired: cluster, ClusterFile: Clusterfile, ClusterCurrent: currentCluster, diff --git a/pkg/apply/apply_test.go b/pkg/apply/apply_test.go index 4a70cb9e5..1a695e1ac 100644 --- a/pkg/apply/apply_test.go +++ b/pkg/apply/apply_test.go @@ -20,6 +20,8 @@ import ( "reflect" "testing" + "github.com/spf13/cobra" + "github.com/labring/sealos/pkg/apply/applydrivers" ) @@ -74,7 +76,9 @@ func Test_NewApplierFromFile(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := NewApplierFromFile("../clusterfile/testdata/clusterfile.yaml", + _, err := NewApplierFromFile(&cobra.Command{ + Use: "mock", + }, "../clusterfile/testdata/clusterfile.yaml", &Args{ Values: tt.args.values, Sets: tt.args.sets, @@ -103,7 +107,9 @@ func TestNewApplierFromFile(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewApplierFromFile(tt.args.path, tt.args.args) + got, err := NewApplierFromFile(&cobra.Command{ + Use: "mock", + }, tt.args.path, tt.args.args) if (err != nil) != tt.wantErr { t.Errorf("NewApplierFromFile() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/apply/applydrivers/apply_drivers_default.go b/pkg/apply/applydrivers/apply_drivers_default.go index 56cc2bd08..d334d9608 100644 --- a/pkg/apply/applydrivers/apply_drivers_default.go +++ b/pkg/apply/applydrivers/apply_drivers_default.go @@ -15,12 +15,11 @@ package applydrivers import ( + "context" "errors" "fmt" "os" - "github.com/labring/sealos/pkg/utils/confirm" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/version" @@ -29,12 +28,13 @@ import ( "github.com/labring/sealos/pkg/clusterfile" "github.com/labring/sealos/pkg/constants" v2 "github.com/labring/sealos/pkg/types/v1beta1" + "github.com/labring/sealos/pkg/utils/confirm" "github.com/labring/sealos/pkg/utils/iputils" "github.com/labring/sealos/pkg/utils/logger" "github.com/labring/sealos/pkg/utils/yaml" ) -func NewDefaultApplier(cluster *v2.Cluster, cf clusterfile.Interface, images []string) (Interface, error) { +func NewDefaultApplier(ctx context.Context, cluster *v2.Cluster, cf clusterfile.Interface, images []string) (Interface, error) { if cluster.Name == "" { return nil, fmt.Errorf("cluster name cannot be empty") } @@ -47,6 +47,7 @@ func NewDefaultApplier(cluster *v2.Cluster, cf clusterfile.Interface, images []s } return &Applier{ + Context: ctx, ClusterDesired: cluster, ClusterFile: cf, ClusterCurrent: cf.GetCluster(), @@ -54,12 +55,13 @@ func NewDefaultApplier(cluster *v2.Cluster, cf clusterfile.Interface, images []s }, nil } -func NewDefaultScaleApplier(current, cluster *v2.Cluster) (Interface, error) { +func NewDefaultScaleApplier(ctx context.Context, current, cluster *v2.Cluster) (Interface, error) { if cluster.Name == "" { cluster.Name = current.Name } cFile := clusterfile.NewClusterFile(constants.Clusterfile(cluster.Name)) return &Applier{ + Context: ctx, ClusterDesired: cluster, ClusterFile: cFile, ClusterCurrent: current, @@ -67,6 +69,7 @@ func NewDefaultScaleApplier(current, cluster *v2.Cluster) (Interface, error) { } type Applier struct { + context.Context ClusterDesired *v2.Cluster ClusterCurrent *v2.Cluster ClusterFile clusterfile.Interface @@ -185,7 +188,7 @@ func (c *Applier) reconcileCluster() (clusterErr error, appErr error) { func (c *Applier) initCluster() error { logger.Info("Start to create a new cluster: master %s, worker %s, registry %s", c.ClusterDesired.GetMasterIPList(), c.ClusterDesired.GetNodeIPList(), c.ClusterDesired.GetRegistryIP()) - createProcessor, err := processor.NewCreateProcessor(c.ClusterDesired.Name, c.ClusterFile) + createProcessor, err := processor.NewCreateProcessor(c.Context, c.ClusterDesired.Name, c.ClusterFile) if err != nil { return err } @@ -205,7 +208,7 @@ func (c *Applier) installApp(images []string) error { if err != nil { return err } - installProcessor, err := processor.NewInstallProcessor(c.ClusterFile, images) + installProcessor, err := processor.NewInstallProcessor(c.Context, c.ClusterFile, images) if err != nil { return err } diff --git a/pkg/apply/args.go b/pkg/apply/args.go index 5e4139fd6..31b98bf99 100644 --- a/pkg/apply/args.go +++ b/pkg/apply/args.go @@ -60,7 +60,6 @@ type RunArgs struct { CustomEnv []string CustomCMD []string CustomConfigFiles []string - fs *pflag.FlagSet } func (arg *RunArgs) RegisterFlags(fs *pflag.FlagSet) { @@ -69,7 +68,6 @@ func (arg *RunArgs) RegisterFlags(fs *pflag.FlagSet) { fs.StringSliceVarP(&arg.CustomEnv, "env", "e", []string{}, "environment variables to set during command execution") fs.StringSliceVar(&arg.CustomCMD, "cmd", []string{}, "override CMD directive in images") fs.StringSliceVar(&arg.CustomConfigFiles, "config-file", []string{}, "path of custom config files, to use to replace the resource") - arg.fs = fs } type Args struct { @@ -89,13 +87,11 @@ func (arg *Args) RegisterFlags(fs *pflag.FlagSet) { type ResetArgs struct { *Cluster *SSH - fs *pflag.FlagSet } func (arg *ResetArgs) RegisterFlags(fs *pflag.FlagSet) { arg.Cluster.RegisterFlags(fs, "be reset", "reset") arg.SSH.RegisterFlags(fs) - arg.fs = fs } type ScaleArgs struct { diff --git a/pkg/apply/gen.go b/pkg/apply/gen.go index 9f31237fe..091aa6299 100644 --- a/pkg/apply/gen.go +++ b/pkg/apply/gen.go @@ -19,15 +19,16 @@ package apply import ( "fmt" - "github.com/labring/sealos/pkg/utils/iputils" + "github.com/spf13/cobra" "github.com/labring/sealos/pkg/apply/processor" "github.com/labring/sealos/pkg/buildah" "github.com/labring/sealos/pkg/runtime" "github.com/labring/sealos/pkg/types/v1beta1" + "github.com/labring/sealos/pkg/utils/iputils" ) -func NewClusterFromGenArgs(imageNames []string, args *RunArgs) ([]byte, error) { +func NewClusterFromGenArgs(cmd *cobra.Command, args *RunArgs, imageNames []string) ([]byte, error) { cluster := initCluster(args.ClusterName) c := &ClusterArgs{ clusterName: args.ClusterName, @@ -39,7 +40,7 @@ func NewClusterFromGenArgs(imageNames []string, args *RunArgs) ([]byte, error) { args.Cluster.Masters = localIpv4 } - if err := c.runArgs(imageNames, args); err != nil { + if err := c.runArgs(cmd, args, imageNames); err != nil { return nil, err } diff --git a/pkg/apply/gen_test.go b/pkg/apply/gen_test.go index a9a3da7d0..e4d448d3e 100644 --- a/pkg/apply/gen_test.go +++ b/pkg/apply/gen_test.go @@ -48,7 +48,6 @@ func TestNewClusterFromGenArgs(t *testing.T) { CustomEnv: nil, CustomCMD: nil, CustomConfigFiles: nil, - fs: nil, }, }, wantErr: false, @@ -67,7 +66,6 @@ func TestNewClusterFromGenArgs(t *testing.T) { CustomEnv: nil, CustomCMD: nil, CustomConfigFiles: nil, - fs: nil, }, }, wantErr: false, @@ -79,7 +77,9 @@ func TestNewClusterFromGenArgs(t *testing.T) { Short: "test", }) t.Run(tt.name, func(t *testing.T) { - got, _ := NewClusterFromGenArgs(tt.args.imageName, tt.args.args) + got, _ := NewClusterFromGenArgs(&cobra.Command{ + Use: "mock", + }, tt.args.args, tt.args.imageName) t.Logf("%s", string(got)) }) } diff --git a/pkg/apply/processor/context.go b/pkg/apply/processor/context.go new file mode 100644 index 000000000..99c0a9157 --- /dev/null +++ b/pkg/apply/processor/context.go @@ -0,0 +1,47 @@ +/* +Copyright 2023 fengxsong@outlook.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package processor + +import "context" + +var commandKey struct{} + +func WithCommands(ctx context.Context, commands []string) context.Context { + return context.WithValue(ctx, commandKey, commands) +} + +func GetCommands(ctx context.Context) []string { + v := ctx.Value(commandKey) + if v != nil { + return v.([]string) + } + return nil +} + +var envKey struct{} + +func WithEnvs(ctx context.Context, envs map[string]string) context.Context { + return context.WithValue(ctx, envKey, envs) +} + +func GetEnvs(ctx context.Context) map[string]string { + v := ctx.Value(commandKey) + if v != nil { + return v.(map[string]string) + } + return nil +} diff --git a/pkg/apply/processor/create.go b/pkg/apply/processor/create.go index 79012f59c..44c107224 100644 --- a/pkg/apply/processor/create.go +++ b/pkg/apply/processor/create.go @@ -31,6 +31,7 @@ import ( "github.com/labring/sealos/pkg/runtime" v2 "github.com/labring/sealos/pkg/types/v1beta1" "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/utils/maps" "github.com/labring/sealos/pkg/utils/yaml" ) @@ -39,6 +40,7 @@ type CreateProcessor struct { Buildah buildah.Interface Runtime runtime.Interface Guest guest.Interface + ExtraEnvs map[string]string // parsing from CLI arguments } func (c *CreateProcessor) Execute(cluster *v2.Cluster) error { @@ -94,6 +96,10 @@ func (c *CreateProcessor) preProcess(cluster *v2.Cluster) error { if err := MountClusterImages(c.Buildah, cluster, false); err != nil { return err } + // extra env must been set at the very first + for i := range cluster.Status.Mounts { + cluster.Status.Mounts[i].Env = maps.MergeMap(cluster.Status.Mounts[i].Env, c.ExtraEnvs) + } runTime, err := runtime.NewDefaultRuntime(cluster, c.ClusterFile.GetKubeadmConfig()) if err != nil { return fmt.Errorf("failed to init runtime, %v", err) @@ -162,14 +168,14 @@ func (c *CreateProcessor) Join(cluster *v2.Cluster) error { func (c *CreateProcessor) RunGuest(cluster *v2.Cluster) error { logger.Info("Executing pipeline RunGuest in CreateProcessor.") - err := c.Guest.Apply(cluster, cluster.Status.Mounts) + err := c.Guest.Apply(cluster, cluster.Status.Mounts, cluster.GetAllIPS()) if err != nil { return fmt.Errorf("%s: %w", RunGuestFailed, err) } return nil } -func NewCreateProcessor(name string, clusterFile clusterfile.Interface) (Interface, error) { +func NewCreateProcessor(ctx context.Context, name string, clusterFile clusterfile.Interface) (Interface, error) { bder, err := buildah.New(name) if err != nil { return nil, err @@ -183,5 +189,6 @@ func NewCreateProcessor(name string, clusterFile clusterfile.Interface) (Interfa ClusterFile: clusterFile, Buildah: bder, Guest: gs, + ExtraEnvs: GetEnvs(ctx), }, nil } diff --git a/pkg/apply/processor/error.go b/pkg/apply/processor/error.go deleted file mode 100644 index fe4be30ab..000000000 --- a/pkg/apply/processor/error.go +++ /dev/null @@ -1,47 +0,0 @@ -/* -Copyright 2022. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package processor - -type CheckError struct { - err error -} - -func (e *CheckError) Error() string { - return e.err.Error() -} - -func NewCheckError(err error) error { - if err != nil { - err = &CheckError{err: err} - } - return err -} - -type PreProcessError struct { - err error -} - -func (e *PreProcessError) Error() string { - return e.err.Error() -} - -func NewPreProcessError(err error) error { - if err != nil { - err = &PreProcessError{err: err} - } - return err -} diff --git a/pkg/apply/processor/errors.go b/pkg/apply/processor/errors.go index 1ea8addf5..c91e8830e 100644 --- a/pkg/apply/processor/errors.go +++ b/pkg/apply/processor/errors.go @@ -33,3 +33,33 @@ func IsRunGuestFailed(err error) bool { } var ErrCancelled = errors.New("cancelled") + +type CheckError struct { + err error +} + +func (e *CheckError) Error() string { + return e.err.Error() +} + +func NewCheckError(err error) error { + if err != nil { + err = &CheckError{err: err} + } + return err +} + +type PreProcessError struct { + err error +} + +func (e *PreProcessError) Error() string { + return e.err.Error() +} + +func NewPreProcessError(err error) error { + if err != nil { + err = &PreProcessError{err: err} + } + return err +} diff --git a/pkg/apply/processor/install.go b/pkg/apply/processor/install.go index 959774cc1..cc119daf3 100644 --- a/pkg/apply/processor/install.go +++ b/pkg/apply/processor/install.go @@ -31,6 +31,7 @@ import ( v2 "github.com/labring/sealos/pkg/types/v1beta1" "github.com/labring/sealos/pkg/utils/confirm" "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/utils/maps" "github.com/labring/sealos/pkg/utils/rand" ) @@ -43,6 +44,7 @@ type InstallProcessor struct { Guest guest.Interface NewMounts []v2.MountImage NewImages []string + ExtraEnvs map[string]string // parsing from CLI arguments imagesToOverride []string } @@ -162,6 +164,8 @@ func (c *InstallProcessor) PreProcess(cluster *v2.Cluster) error { if err = OCIToImageMount(c.Buildah, mount); err != nil { return err } + mount.Env = maps.MergeMap(mount.Env, c.ExtraEnvs) + cluster.SetMountImage(mount) c.NewMounts = append(c.NewMounts, *mount) } @@ -248,10 +252,10 @@ func (c *InstallProcessor) RunGuest(cluster *v2.Cluster) error { if len(c.NewMounts) == 0 { return nil } - return c.Guest.Apply(cluster, c.NewMounts) + return c.Guest.Apply(cluster, c.NewMounts, cluster.GetAllIPS()) } -func NewInstallProcessor(clusterFile clusterfile.Interface, images []string) (Interface, error) { +func NewInstallProcessor(ctx context.Context, clusterFile clusterfile.Interface, images []string) (Interface, error) { bder, err := buildah.New(clusterFile.GetCluster().Name) if err != nil { return nil, err @@ -267,5 +271,6 @@ func NewInstallProcessor(clusterFile clusterfile.Interface, images []string) (In Buildah: bder, Guest: gs, NewImages: images, + ExtraEnvs: GetEnvs(ctx), }, nil } diff --git a/pkg/apply/processor/scale.go b/pkg/apply/processor/scale.go index 3d2e50b1a..24733575b 100644 --- a/pkg/apply/processor/scale.go +++ b/pkg/apply/processor/scale.go @@ -18,6 +18,8 @@ import ( "context" "fmt" + "github.com/labring/sealos/pkg/guest" + "golang.org/x/sync/errgroup" "github.com/labring/sealos/pkg/bootstrap" @@ -44,6 +46,7 @@ type ScaleProcessor struct { NodesToJoin []string NodesToDelete []string IsScaleUp bool + Guest guest.Interface } func (c *ScaleProcessor) Execute(cluster *v2.Cluster) error { @@ -73,6 +76,7 @@ func (c *ScaleProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error c.Bootstrap, //s.GetPhasePluginFunc(plugin.PhasePreJoin), c.Join, + c.RunGuest, //s.GetPhasePluginFunc(plugin.PhasePostJoin), ) return todoList, nil @@ -89,6 +93,27 @@ func (c *ScaleProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error return todoList, nil } +func (c *ScaleProcessor) skipAppMounts(allMount []v2.MountImage) []v2.MountImage { + mounts := make([]v2.MountImage, 0) + for _, m := range allMount { + if !m.IsApplication() { + mount := m.DeepCopy() + mounts = append(mounts, *mount) + } + } + return mounts +} + +func (c *ScaleProcessor) RunGuest(cluster *v2.Cluster) error { + logger.Info("Executing pipeline RunGuest in ScaleProcessor.") + hosts := append(c.MastersToJoin, c.NodesToJoin...) + err := c.Guest.Apply(cluster, c.skipAppMounts(cluster.Status.Mounts), hosts) + if err != nil { + return fmt.Errorf("%s: %w", RunGuestFailed, err) + } + return nil +} + func (c *ScaleProcessor) Delete(cluster *v2.Cluster) error { logger.Info("Executing pipeline Delete in ScaleProcessor.") err := c.Runtime.DeleteMasters(c.MastersToDelete) @@ -275,6 +300,11 @@ func NewScaleProcessor(clusterFile clusterfile.Interface, name string, images v2 if err != nil { return nil, err } + gs, err := guest.NewGuestManager() + if err != nil { + return nil, err + } + return &ScaleProcessor{ MastersToDelete: masterToDelete, MastersToJoin: masterToJoin, @@ -284,5 +314,6 @@ func NewScaleProcessor(clusterFile clusterfile.Interface, name string, images v2 Buildah: bder, pullImages: images, IsScaleUp: len(masterToJoin) > 0 || len(nodeToJoin) > 0, + Guest: gs, }, nil } diff --git a/pkg/apply/reset.go b/pkg/apply/reset.go index 55b8aa56c..c3f35c10b 100644 --- a/pkg/apply/reset.go +++ b/pkg/apply/reset.go @@ -17,6 +17,8 @@ package apply import ( "fmt" + "github.com/spf13/cobra" + "github.com/labring/sealos/pkg/apply/applydrivers" "github.com/labring/sealos/pkg/clusterfile" "github.com/labring/sealos/pkg/constants" @@ -26,7 +28,7 @@ import ( stringsutil "github.com/labring/sealos/pkg/utils/strings" ) -func NewApplierFromResetArgs(args *ResetArgs) (applydrivers.Interface, error) { +func NewApplierFromResetArgs(cmd *cobra.Command, args *ResetArgs) (applydrivers.Interface, error) { clusterPath := constants.Clusterfile(args.ClusterName) cf := clusterfile.NewClusterFile(clusterPath) err := cf.Process() @@ -41,35 +43,22 @@ func NewApplierFromResetArgs(args *ResetArgs) (applydrivers.Interface, error) { clusterName: cluster.Name, cluster: cluster, } - if err = c.resetArgs(args); err != nil { + if err = c.resetArgs(cmd, args); err != nil { return nil, err } - return applydrivers.NewDefaultApplier(c.cluster, cf, nil) + return applydrivers.NewDefaultApplier(cmd.Context(), c.cluster, cf, nil) } -func (r *ClusterArgs) resetArgs(args *ResetArgs) error { +func (r *ClusterArgs) resetArgs(cmd *cobra.Command, args *ResetArgs) error { if args.Cluster.ClusterName == "" { return fmt.Errorf("cluster name can not be empty") } if err := PreProcessIPList(args.Cluster); err != nil { return err } - if args.fs != nil { - if args.fs.Changed("user") || r.cluster.Spec.SSH.User == "" { - r.cluster.Spec.SSH.User = args.SSH.User - } - if args.fs.Changed("pk") || r.cluster.Spec.SSH.Pk == "" { - r.cluster.Spec.SSH.Pk = args.SSH.Pk - } - if args.fs.Changed("pk-passwd") || r.cluster.Spec.SSH.PkPasswd == "" { - r.cluster.Spec.SSH.PkPasswd = args.SSH.PkPassword - } - if args.fs.Changed("port") || r.cluster.Spec.SSH.Port == 0 { - r.cluster.Spec.SSH.Port = args.SSH.Port - } - if args.fs.Changed("passwd") || r.cluster.Spec.SSH.Passwd == "" { - r.cluster.Spec.SSH.Passwd = args.SSH.Password - } + override := getSSHFromCommand(cmd) + if override != nil { + r.cluster.Spec.SSH = *override } if len(args.Cluster.Masters) > 0 { diff --git a/pkg/apply/reset_test.go b/pkg/apply/reset_test.go index 14da29c33..607387be1 100644 --- a/pkg/apply/reset_test.go +++ b/pkg/apply/reset_test.go @@ -18,6 +18,8 @@ package apply import ( "testing" + + "github.com/spf13/cobra" ) func TestNewApplierFromResetArgs(t *testing.T) { @@ -70,7 +72,9 @@ func TestNewApplierFromResetArgs(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - _, err := NewApplierFromResetArgs(tt.args.args) + _, err := NewApplierFromResetArgs(&cobra.Command{ + Use: "mock", + }, tt.args.args) if (err != nil) != tt.wantErr { t.Errorf("NewApplierFromResetArgs() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/apply/run.go b/pkg/apply/run.go index d62a7007e..a1d239cd1 100644 --- a/pkg/apply/run.go +++ b/pkg/apply/run.go @@ -15,17 +15,23 @@ package apply import ( + "context" + "errors" "fmt" "net" "strconv" + "github.com/spf13/cobra" + "github.com/labring/sealos/pkg/apply/applydrivers" + "github.com/labring/sealos/pkg/apply/processor" "github.com/labring/sealos/pkg/clusterfile" "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/ssh" v2 "github.com/labring/sealos/pkg/types/v1beta1" "github.com/labring/sealos/pkg/utils/iputils" "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/utils/maps" stringsutil "github.com/labring/sealos/pkg/utils/strings" ) @@ -35,7 +41,7 @@ type ClusterArgs struct { clusterName string } -func NewApplierFromArgs(imageName []string, args *RunArgs) (applydrivers.Interface, error) { +func NewApplierFromArgs(cmd *cobra.Command, args *RunArgs, imageName []string) (applydrivers.Interface, error) { clusterPath := constants.Clusterfile(args.ClusterName) cf := clusterfile.NewClusterFile(clusterPath, clusterfile.WithCustomConfigFiles(args.CustomConfigFiles), @@ -61,23 +67,38 @@ func NewApplierFromArgs(imageName []string, args *RunArgs) (applydrivers.Interfa clusterName: cluster.Name, cluster: cluster, } - if err = c.runArgs(imageName, args); err != nil { + if err = c.runArgs(cmd, args, imageName); err != nil { return nil, err } - return applydrivers.NewDefaultApplier(c.cluster, cf, imageName) + + ctx := withCommonContext(cmd.Context(), cmd) + + return applydrivers.NewDefaultApplier(ctx, c.cluster, cf, imageName) } -func (r *ClusterArgs) runArgs(imageList []string, args *RunArgs) error { - if args.Cluster.ClusterName == "" { - return fmt.Errorf("cluster name can not be empty") +func withCommonContext(ctx context.Context, cmd *cobra.Command) context.Context { + if flagChanged(cmd, "cmd") { + v, _ := cmd.Flags().GetStringSlice("cmd") + ctx = processor.WithCommands(ctx, v) } - //the first run check + if flagChanged(cmd, "env") { + v, _ := cmd.Flags().GetStringSlice("env") + ctx = processor.WithEnvs(ctx, maps.ListToMap(v)) + } + return ctx +} + +func (r *ClusterArgs) runArgs(cmd *cobra.Command, args *RunArgs, imageList []string) error { + if args.Cluster.ClusterName == "" { + return errors.New("cluster name can not be empty") + } + // the first run check if r.cluster.CreationTimestamp.IsZero() { if len(imageList) == 0 { - return fmt.Errorf("image can not be empty") + return errors.New("image can not be empty") } if len(args.Cluster.Masters) == 0 { - return fmt.Errorf("master ip(s) must specified") + return errors.New("master ip(s) must specified") } } else { if r.cluster.Status.Phase != v2.ClusterSuccess { @@ -88,30 +109,10 @@ func (r *ClusterArgs) runArgs(imageList []string, args *RunArgs) error { if err := PreProcessIPList(args.Cluster); err != nil { return err } - if args.fs != nil { - if args.fs.Changed("env") || len(r.cluster.Spec.Env) == 0 { - r.cluster.Spec.Env = make([]string, len(args.CustomEnv)) - copy(r.cluster.Spec.Env, args.CustomEnv) - } - if args.fs.Changed("cmd") || len(r.cluster.Spec.Command) == 0 { - r.cluster.Spec.Command = make([]string, len(args.CustomCMD)) - copy(r.cluster.Spec.Command, args.CustomCMD) - } - if args.fs.Changed("user") || r.cluster.Spec.SSH.User == "" { - r.cluster.Spec.SSH.User = args.SSH.User - } - if args.fs.Changed("pk") || r.cluster.Spec.SSH.Pk == "" { - r.cluster.Spec.SSH.Pk = args.SSH.Pk - } - if args.fs.Changed("pk-passwd") || r.cluster.Spec.SSH.PkPasswd == "" { - r.cluster.Spec.SSH.PkPasswd = args.SSH.PkPassword - } - if args.fs.Changed("port") || r.cluster.Spec.SSH.Port == 0 { - r.cluster.Spec.SSH.Port = args.SSH.Port - } - if args.fs.Changed("passwd") || r.cluster.Spec.SSH.Passwd == "" { - r.cluster.Spec.SSH.Passwd = args.SSH.Password - } + + override := getSSHFromCommand(cmd) + if override != nil { + r.cluster.Spec.SSH = *override } r.cluster.SetNewImages(imageList) diff --git a/pkg/apply/run_test.go b/pkg/apply/run_test.go index 809d6a8bb..55616a6fa 100644 --- a/pkg/apply/run_test.go +++ b/pkg/apply/run_test.go @@ -18,7 +18,7 @@ import ( "reflect" "testing" - "github.com/spf13/pflag" + "github.com/spf13/cobra" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "github.com/labring/sealos/pkg/clusterfile" @@ -79,7 +79,6 @@ func TestClusterArgs_SetClusterRunArgs(t *testing.T) { CustomCMD: []string{ "echo hello", }, - fs: pflag.NewFlagSet("test", pflag.ExitOnError), }, }, wantErr: false, @@ -92,7 +91,9 @@ func TestClusterArgs_SetClusterRunArgs(t *testing.T) { hosts: tt.fields.hosts, clusterName: tt.fields.clusterName, } - if err := r.runArgs(tt.args.imageList, tt.args.runArgs); (err != nil) != tt.wantErr { + if err := r.runArgs(&cobra.Command{ + Use: "mock", + }, tt.args.runArgs, tt.args.imageList); (err != nil) != tt.wantErr { t.Errorf("runArgs() error = %v, wantErr %v", err, tt.wantErr) } }) @@ -182,7 +183,9 @@ func TestNewApplierFromArgs(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - got, err := NewApplierFromArgs(tt.args.imageName, tt.args.args) + got, err := NewApplierFromArgs(&cobra.Command{ + Use: "mock", + }, tt.args.args, tt.args.imageName) if (err != nil) != tt.wantErr { t.Errorf("NewApplierFromArgs() error = %v, wantErr %v", err, tt.wantErr) return diff --git a/pkg/apply/scale.go b/pkg/apply/scale.go index 13d62b8ba..06e3b5a9a 100644 --- a/pkg/apply/scale.go +++ b/pkg/apply/scale.go @@ -65,7 +65,7 @@ func NewScaleApplierFromArgs(cmd *cobra.Command, scaleArgs *ScaleArgs) (applydri return nil, err } - return applydrivers.NewDefaultScaleApplier(curr, cluster) + return applydrivers.NewDefaultScaleApplier(cmd.Context(), curr, cluster) } func getSSHFromCommand(cmd *cobra.Command) *v2.SSH { diff --git a/pkg/bootstrap/context.go b/pkg/bootstrap/context.go index abaa27dcd..a7ac6ccdb 100644 --- a/pkg/bootstrap/context.go +++ b/pkg/bootstrap/context.go @@ -20,6 +20,8 @@ import ( "github.com/labring/sealos/pkg/remote" "github.com/labring/sealos/pkg/ssh" v2 "github.com/labring/sealos/pkg/types/v1beta1" + "github.com/labring/sealos/pkg/utils/maps" + stringsutil "github.com/labring/sealos/pkg/utils/strings" ) type Context interface { @@ -60,12 +62,21 @@ func (ctx realContext) GetRemoter() remote.Interface { func NewContextFrom(cluster *v2.Cluster) Context { execer := ssh.NewSSHByCluster(cluster, true) - envProcessor := env.NewEnvProcessor(cluster, cluster.Status.Mounts) + envProcessor := env.NewEnvProcessor(cluster) remoter := remote.New(cluster.GetName(), execer) + + rootfsImage := cluster.GetRootfsImage() + rootfsEnvs := v2.MergeEnvWithBuiltinKeys(rootfsImage.Env, *rootfsImage) + + // bootstrap process depends on the envs in the rootfs image + shellWrapper := func(host, shell string) string { + envs := maps.MergeMap(rootfsEnvs, envProcessor.Getenv(host)) + return stringsutil.RenderShellFromEnv(shell, envs) + } return &realContext{ cluster: cluster, execer: execer, - bash: constants.NewBash(cluster.GetName(), cluster.GetImageLabels(), envProcessor.WrapperShell), + bash: constants.NewBash(cluster.GetName(), cluster.GetImageLabels(), shellWrapper), data: constants.NewData(cluster.GetName()), remoter: remoter, } diff --git a/pkg/env/env.go b/pkg/env/env.go index 2415a2181..1fab00b20 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -22,53 +22,55 @@ import ( "path/filepath" "strings" - "github.com/labring/sealos/pkg/version" - "github.com/labring/sealos/pkg/template" "github.com/labring/sealos/pkg/types/v1beta1" fileutil "github.com/labring/sealos/pkg/utils/file" "github.com/labring/sealos/pkg/utils/logger" "github.com/labring/sealos/pkg/utils/maps" - strings2 "github.com/labring/sealos/pkg/utils/strings" + stringsutil "github.com/labring/sealos/pkg/utils/strings" ) const templateSuffix = ".tmpl" type Interface interface { - // WrapperShell :If host already set env like DATADISK=/data + // WrapShell :If host already set env like DATADISK=/data // This function add env to the shell, like: // Input shell: cat /etc/hosts // Output shell: DATADISK=/data cat /etc/hosts // So that you can get env values in you shell script - WrapperShell(host, shell string) string + WrapShell(host, shell string) string // RenderAll :render env to all the files in dir - RenderAll(host, dir string) error - WrapperEnv(host string) map[string]string + RenderAll(host, dir string, envs map[string]string) error + Getenv(host string) map[string]string } type processor struct { *v1beta1.Cluster - //types.ImageListOCIV1 - mounts []v1beta1.MountImage + cache map[string]map[string]string } -func NewEnvProcessor(cluster *v1beta1.Cluster, mounts []v1beta1.MountImage) Interface { - return &processor{Cluster: cluster, mounts: mounts} +func NewEnvProcessor(cluster *v1beta1.Cluster) Interface { + return &processor{ + Cluster: cluster, + cache: make(map[string]map[string]string), + } } -func (p *processor) WrapperEnv(host string) map[string]string { + +func (p *processor) Getenv(host string) map[string]string { env := make(map[string]string) - envs := p.getHostEnv(host) + envs := p.getHostEnvInCache(host) for k, v := range envs { env[k] = v } return env } -func (p *processor) WrapperShell(host, shell string) string { - envs := p.getHostEnv(host) - return strings2.RenderShellFromEnv(shell, envs) + +func (p *processor) WrapShell(host, shell string) string { + envs := p.getHostEnvInCache(host) + return stringsutil.RenderShellFromEnv(shell, envs) } -func (p *processor) RenderAll(host, dir string) error { +func (p *processor) RenderAll(host, dir string, envs map[string]string) error { return filepath.Walk(dir, func(path string, info os.FileInfo, errIn error) error { if errIn != nil { return errIn @@ -100,19 +102,30 @@ func (p *processor) RenderAll(host, dir string) error { return fmt.Errorf("failed to create template: %s %v", path, err) } if host != "" { - if err := t.Execute(writer, p.getHostEnv(host)); err != nil { + data := maps.MergeMap(envs, p.getHostEnvInCache(host)) + if err := t.Execute(writer, data); err != nil { return fmt.Errorf("failed to render env template: %s %v", path, err) } } } else { - return errors.New("convert template failed") + return errors.New("parse template failed") } return nil }) } +func (p *processor) getHostEnvInCache(hostIP string) map[string]string { + if v, ok := p.cache[hostIP]; ok { + return v + } + v := p.getHostEnv(hostIP) + p.cache[hostIP] = v + return v +} + // Merge the host ENV and global env, the host env will overwrite cluster.Spec.Env func (p *processor) getHostEnv(hostIP string) map[string]string { + // TODO: what if hostIP not found? var hostEnv []string for _, host := range p.Spec.Hosts { for _, ip := range host.IPS { @@ -121,30 +134,31 @@ func (p *processor) getHostEnv(hostIP string) map[string]string { } } } + hostEnvMap := maps.ListToMap(hostEnv) specEnvMap := maps.ListToMap(p.Spec.Env) - var imageEnvMap map[string]string - for _, img := range p.mounts { - imageEnvMap = maps.MergeMap(imageEnvMap, img.Env) - if img.Type == v1beta1.RootfsImage { - imageEnvMap[v1beta1.ImageKubeVersionEnvSysKey] = img.Labels[v1beta1.ImageKubeVersionKey] - imageEnvMap[v1beta1.ImageSealosVersionEnvSysKey] = version.Get().GitVersion + excludeSysEnv := func(m map[string]string) map[string]string { + m, exclude := ExcludeKeysWithPrefix(m, "SEALOS_SYS") + if len(exclude) > 0 { + logger.Warn("skip %s cause envs with prefix SEALOS_SYS are sealos system only", strings.Join(exclude, ", ")) } + return m } - filterSysEnv := func(env map[string]string) map[string]string { - outEnv := make(map[string]string, 0) - for k, v := range env { - if strings.HasPrefix(k, "SEALOS_SYS") { - logger.Warn("skip %s env , SEALOS_SYS prefix env is sealos system env", k) - } else { - outEnv[k] = v - } - } - return outEnv - } - - envs := maps.MergeMap(imageEnvMap, filterSysEnv(specEnvMap), filterSysEnv(hostEnvMap)) + envs := maps.MergeMap(excludeSysEnv(specEnvMap), excludeSysEnv(hostEnvMap)) return envs } + +func ExcludeKeysWithPrefix(m map[string]string, prefix string) (map[string]string, []string) { + out := make(map[string]string, 0) + var exclude []string + for k, v := range m { + if strings.HasPrefix(k, prefix) { + exclude = append(exclude, k) + } else { + out[k] = v + } + } + return out, exclude +} diff --git a/pkg/env/env_shim_test.go b/pkg/env/env_shim_test.go index d0e038fce..55e98beea 100644 --- a/pkg/env/env_shim_test.go +++ b/pkg/env/env_shim_test.go @@ -145,9 +145,9 @@ func Test_shim_RenderAll(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &processor{ Cluster: tt.fields.Cluster, - mounts: tt.fields.Cluster.Status.Mounts, + cache: make(map[string]map[string]string), } - if err := p.RenderAll(tt.args.host, tt.args.dir); (err != nil) != tt.wantErr { + if err := p.RenderAll(tt.args.host, tt.args.dir, map[string]string{}); (err != nil) != tt.wantErr { t.Errorf("RenderAll() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pkg/env/env_test.go b/pkg/env/env_test.go index 44147b562..d1896a944 100644 --- a/pkg/env/env_test.go +++ b/pkg/env/env_test.go @@ -70,8 +70,9 @@ func Test_processor_WrapperShell(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &processor{ Cluster: tt.fields.Cluster, + cache: make(map[string]map[string]string), } - got := p.WrapperShell(tt.args.host, tt.args.shell) + got := p.WrapShell(tt.args.host, tt.args.shell) for _, want := range tt.want { if !strings.Contains(got, want) { t.Errorf("WrapperShell() = %v, want %v", got, want) @@ -110,8 +111,9 @@ func Test_processor_RenderAll(t *testing.T) { t.Run(tt.name, func(t *testing.T) { p := &processor{ Cluster: tt.fields.Cluster, + cache: make(map[string]map[string]string), } - if err := p.RenderAll(tt.args.host, tt.args.dir); (err != nil) != tt.wantErr { + if err := p.RenderAll(tt.args.host, tt.args.dir, nil); (err != nil) != tt.wantErr { t.Errorf("RenderAll() error = %v, wantErr %v", err, tt.wantErr) } }) diff --git a/pkg/filesystem/rootfs/rootfs_default.go b/pkg/filesystem/rootfs/rootfs_default.go index 31b5f4dcd..a2e55076d 100644 --- a/pkg/filesystem/rootfs/rootfs_default.go +++ b/pkg/filesystem/rootfs/rootfs_default.go @@ -58,7 +58,7 @@ func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error target := constants.NewData(f.getClusterName(cluster)).RootFSPath() ctx := context.Background() eg, _ := errgroup.WithContext(ctx) - envProcessor := env.NewEnvProcessor(cluster, f.mounts) + envProcessor := env.NewEnvProcessor(cluster) for _, mount := range f.mounts { src := mount eg.Go(func() error { @@ -68,7 +68,8 @@ func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error } // TODO: if we are planing to support rendering templates for each host, // then move this rendering process before ssh.CopyDir and do it one by one. - err := renderTemplatesWithEnv(src.MountPoint, ipList, envProcessor) + envs := v2.MergeEnvWithBuiltinKeys(src.Env, src) + err := renderTemplatesWithEnv(src.MountPoint, ipList, envProcessor, envs) if err != nil { return fmt.Errorf("failed to render env: %w", err) } @@ -93,47 +94,45 @@ func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error notRegistryDirFilter := func(entry fs.DirEntry) bool { return !constants.IsRegistryDir(entry) } + copyFn := func(m v2.MountImage, targetHost, targetDir string) error { + logger.Debug("send mount image, target: %s, image: %s, type: %s", targetHost, m.ImageName, m.Type) + if err := ssh.CopyDir(sshClient, targetHost, m.MountPoint, targetDir, notRegistryDirFilter); err != nil { + logger.Error("error occur while sending mount image %s: %v", m.Name, err) + return err + } + return nil + } + for idx := range ipList { ip := ipList[idx] eg.Go(func() error { - egg, _ := errgroup.WithContext(ctx) - for idj := range f.mounts { - mount := f.mounts[idj] - egg.Go(func() error { - switch mount.Type { - case v2.RootfsImage, v2.PatchImage: - logger.Debug("send mount image, ip: %s, image name: %s, image type: %s", ip, mount.ImageName, mount.Type) - err := ssh.CopyDir(sshClient, ip, mount.MountPoint, target, notRegistryDirFilter) - if err != nil { - return fmt.Errorf("failed to copy %s %s: %v", mount.Type, mount.Name, err) - } + for i := range f.mounts { + if f.mounts[i].IsRootFs() || f.mounts[i].IsPatch() { + // contents in rootfs/patch type images cannot be replicated asynchronously + if err := copyFn(f.mounts[i], ip, target); err != nil { + return err } - return nil - }) + } } - return egg.Wait() + return nil }) } if err := eg.Wait(); err != nil { return err } - endEg, _ := errgroup.WithContext(ctx) + eg, _ = errgroup.WithContext(ctx) master0 := cluster.GetMaster0IPAndPort() for idx := range f.mounts { mountInfo := f.mounts[idx] - endEg.Go(func() error { - if mountInfo.Type == v2.AppImage { - logger.Debug("send app mount images, ip: %s, image name: %s, image type: %s", master0, mountInfo.ImageName, mountInfo.Type) - err := ssh.CopyDir(sshClient, master0, mountInfo.MountPoint, constants.GetAppWorkDir(cluster.Name, mountInfo.Name), notRegistryDirFilter) - if err != nil { - return fmt.Errorf("failed to copy %s %s: %v", mountInfo.Type, mountInfo.Name, err) - } + eg.Go(func() error { + if mountInfo.IsApplication() { + return copyFn(mountInfo, master0, constants.GetAppWorkDir(cluster.Name, mountInfo.Name)) } return nil }) } - return endEg.Wait() + return eg.Wait() } func (f *defaultRootfs) unmountRootfs(cluster *v2.Cluster, ipList []string) error { @@ -152,7 +151,7 @@ func (f *defaultRootfs) unmountRootfs(cluster *v2.Cluster, ipList []string) erro return eg.Wait() } -func renderTemplatesWithEnv(mountDir string, ipList []string, p env.Interface) error { +func renderTemplatesWithEnv(mountDir string, ipList []string, p env.Interface, envs map[string]string) error { var ( renderEtc = path.Join(mountDir, constants.EtcDirName) renderScripts = path.Join(mountDir, constants.ScriptsDirName) @@ -163,7 +162,7 @@ func renderTemplatesWithEnv(mountDir string, ipList []string, p env.Interface) e for _, dir := range []string{renderEtc, renderScripts, renderManifests} { logger.Debug("render env dir: %s", dir) if file.IsExist(dir) { - err := p.RenderAll(ipList[0], dir) + err := p.RenderAll(ipList[0], dir, envs) if err != nil { return err } diff --git a/pkg/guest/guest.go b/pkg/guest/guest.go index a9656b9f9..f53afbd67 100644 --- a/pkg/guest/guest.go +++ b/pkg/guest/guest.go @@ -15,10 +15,13 @@ package guest import ( + "context" "fmt" "path/filepath" "strings" + "golang.org/x/sync/errgroup" + "github.com/labring/sealos/fork/golang/expansion" "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/env" @@ -26,27 +29,22 @@ import ( "github.com/labring/sealos/pkg/ssh" v2 "github.com/labring/sealos/pkg/types/v1beta1" fileutil "github.com/labring/sealos/pkg/utils/file" - "github.com/labring/sealos/pkg/utils/logger" "github.com/labring/sealos/pkg/utils/maps" + stringsutil "github.com/labring/sealos/pkg/utils/strings" ) type Interface interface { - Apply(cluster *v2.Cluster, mounts []v2.MountImage) error + Apply(cluster *v2.Cluster, mounts []v2.MountImage, targetHosts []string) error Delete(cluster *v2.Cluster) error } -type Default struct { -} +type Default struct{} func NewGuestManager() (Interface, error) { return &Default{}, nil } -func (d *Default) Apply(cluster *v2.Cluster, mounts []v2.MountImage) error { - envInterface := env.NewEnvProcessor(cluster, cluster.Status.Mounts) - envs := envInterface.WrapperEnv(cluster.GetMaster0IP()) //clusterfile - guestCMD := d.getGuestCmd(envs, cluster, mounts) - +func (d *Default) Apply(cluster *v2.Cluster, mounts []v2.MountImage, targetHosts []string) error { kubeConfig := filepath.Join(constants.GetHomeDir(), ".kube", "config") if !fileutil.IsExist(kubeConfig) { adminFile := constants.NewData(cluster.Name).AdminFile() @@ -63,50 +61,74 @@ func (d *Default) Apply(cluster *v2.Cluster, mounts []v2.MountImage) error { _ = fileutil.CleanFiles(kubeConfig) }() } - sshInterface := ssh.NewSSHByCluster(cluster, true) - for _, cmd := range guestCMD { - logger.Debug("exec guest command: %s", cmd) - if err := sshInterface.CmdAsync(cluster.GetMaster0IPAndPort(), envInterface.WrapperShell(cluster.GetMaster0IP(), cmd)); err != nil { - return err + + envWrapper := env.NewEnvProcessor(cluster) + execer := ssh.NewSSHByCluster(cluster, true) + + for i, m := range mounts { + switch { + case m.IsRootFs(), m.IsPatch(): + eg, ctx := errgroup.WithContext(context.Background()) + for j := range targetHosts { + node := targetHosts[j] + envs := envWrapper.Getenv(node) + cmds := formalizeImageCommands(cluster, i, m, envs) + eg.Go(func() error { + return execer.CmdAsyncWithContext(ctx, node, + stringsutil.RenderShellFromEnv(strings.Join(cmds, "; "), envs), + ) + }) + } + if err := eg.Wait(); err != nil { + return err + } + case m.IsApplication(): + // on run on the first master + envs := envWrapper.Getenv(cluster.GetMaster0IP()) + cmds := formalizeImageCommands(cluster, i, m, envs) + if err := execer.CmdAsync(cluster.GetMaster0IPAndPort(), + stringsutil.RenderShellFromEnv(strings.Join(cmds, "; "), envs), + ); err != nil { + return err + } } } - logger.Debug("finish to exec guest commands: %v", guestCMD) return nil } -func (d *Default) getGuestCmd(envs map[string]string, cluster *v2.Cluster, mounts []v2.MountImage) []string { - command := make([]string, 0) - overrideCmd := cluster.Spec.Command - workCmd := func(applicationName, cmd string, t v2.ImageType) string { - if t == v2.RootfsImage { - return fmt.Sprintf(constants.CdAndExecCmd, constants.GetRootWorkDir(cluster.Name), cmd) - } - return fmt.Sprintf(constants.CdAndExecCmd, constants.GetAppWorkDir(cluster.Name, applicationName), cmd) +func formalizeWorkingCommand(clusterName string, imageName string, t v2.ImageType, cmd string) string { + if cmd == "" { + return "" } - for idx, i := range mounts { - if i.Type != v2.AppImage && i.Type != v2.RootfsImage { - continue - } - mergeENV := maps.MergeMap(i.Env, envs) - mapping := expansion.MappingFuncFor(mergeENV) - for _, cmd := range i.Entrypoint { - command = append(command, workCmd(i.Name, expansion.Expand(cmd, mapping), i.Type)) - } + switch t { + case v2.RootfsImage, v2.PatchImage: + return fmt.Sprintf(constants.CdAndExecCmd, constants.GetRootWorkDir(clusterName), cmd) + case v2.AppImage, "": + return fmt.Sprintf(constants.CdAndExecCmd, constants.GetAppWorkDir(clusterName, imageName), cmd) + } + return "" +} - // if --cmd is specified, only the CMD of the first MountImage will be overridden - if idx == 0 && len(overrideCmd) > 0 { - for _, cmd := range overrideCmd { - command = append(command, workCmd(i.Name, expansion.Expand(cmd, mapping), i.Type)) - } - continue - } +func formalizeImageCommands(cluster *v2.Cluster, index int, m v2.MountImage, extraEnvs map[string]string) []string { + envs := maps.MergeMap(m.Env, extraEnvs) + envs = v2.MergeEnvWithBuiltinKeys(envs, m) + mapping := expansion.MappingFuncFor(envs) - for _, cmd := range i.Cmd { - command = append(command, workCmd(i.Name, expansion.Expand(cmd, mapping), i.Type)) + cmds := make([]string, 0) + for i := range m.Entrypoint { + cmds = append(cmds, formalizeWorkingCommand(cluster.Name, m.Name, m.Type, expansion.Expand(m.Entrypoint[i], mapping))) + } + if index == 0 && len(cluster.Spec.Command) > 0 { + for i := range cluster.Spec.Command { + cmds = append(cmds, formalizeWorkingCommand(cluster.Name, m.Name, m.Type, expansion.Expand(cluster.Spec.Command[i], mapping))) + } + } else { + for i := range m.Cmd { + cmds = append(cmds, formalizeWorkingCommand(cluster.Name, m.Name, m.Type, expansion.Expand(m.Cmd[i], mapping))) } } - return command + return cmds } func (d Default) Delete(_ *v2.Cluster) error { diff --git a/pkg/guest/guest_test.go b/pkg/guest/guest_test.go index 908e47316..d63e9e3ff 100644 --- a/pkg/guest/guest_test.go +++ b/pkg/guest/guest_test.go @@ -127,9 +127,10 @@ func TestDefault_getGuestCmd(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &Default{} - if got := d.getGuestCmd(tt.args.envs, tt.args.cluster, tt.args.mounts); !reflect.DeepEqual(got, tt.want) { - t.Errorf("getGuestCmd() = %v, want %v", got, tt.want) + for i := range tt.args.mounts { + if got := formalizeImageCommands(tt.args.cluster, i, tt.args.mounts[i], tt.args.envs); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getGuestCmd() = %v, want %v", got, tt.want) + } } }) } diff --git a/pkg/runtime/runtime_getter.go b/pkg/runtime/runtime_getter.go index a96c61653..5194611d7 100644 --- a/pkg/runtime/runtime_getter.go +++ b/pkg/runtime/runtime_getter.go @@ -21,12 +21,9 @@ import ( "fmt" "strings" - "github.com/labring/sealos/pkg/client-go/kubernetes" - - strings2 "github.com/labring/sealos/pkg/utils/strings" - "golang.org/x/sync/errgroup" + "github.com/labring/sealos/pkg/client-go/kubernetes" "github.com/labring/sealos/pkg/constants" "github.com/labring/sealos/pkg/env" "github.com/labring/sealos/pkg/remote" @@ -34,6 +31,8 @@ import ( "github.com/labring/sealos/pkg/types/v1beta1" "github.com/labring/sealos/pkg/utils/iputils" "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/utils/maps" + stringsutil "github.com/labring/sealos/pkg/utils/strings" ) func (k *KubeadmRuntime) getKubeVersion() string { @@ -101,8 +100,9 @@ func (k *KubeadmRuntime) getVIPFromImage() string { if vip == "" { vip = DefaultVIP } else { - envs := k.getENVInterface().WrapperEnv(k.getMaster0IP()) - vip = strings2.RenderTextFromEnv(vip, envs) + envsInRootFsImage := k.Cluster.GetRootfsImage().Env + envs := maps.MergeMap(envsInRootFsImage, k.getEnvInterface().Getenv(k.getMaster0IP())) + vip = stringsutil.RenderTextFromEnv(vip, envs) } logger.Debug("get vip is %s", vip) return vip @@ -193,8 +193,8 @@ func (k *KubeadmRuntime) getSSHInterface() ssh.Interface { return k.ClusterClient } -func (k *KubeadmRuntime) getENVInterface() env.Interface { - return env.NewEnvProcessor(k.Cluster, k.Cluster.Status.Mounts) +func (k *KubeadmRuntime) getEnvInterface() env.Interface { + return env.NewEnvProcessor(k.Cluster) } func (k *KubeadmRuntime) getRemoteInterface() remote.Interface { diff --git a/pkg/types/v1beta1/cluster.go b/pkg/types/v1beta1/cluster.go index 637e58fd0..57fafa987 100644 --- a/pkg/types/v1beta1/cluster.go +++ b/pkg/types/v1beta1/cluster.go @@ -18,6 +18,8 @@ import ( v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/yaml" + + "github.com/labring/sealos/pkg/version" ) // +kubebuilder:object:root=true @@ -78,11 +80,35 @@ type MountImage struct { Entrypoint []string `json:"entrypoint,omitempty"` } -func (img *MountImage) KubeVersion() string { - if img.Type != RootfsImage || img.Labels == nil { +func (m *MountImage) KubeVersion() string { + if m.Type != RootfsImage || m.Labels == nil { return "" } - return img.Labels[ImageKubeVersionKey] + return m.Labels[ImageKubeVersionKey] +} + +func (m *MountImage) IsApplication() bool { + return m.Type == "" || m.Type == AppImage +} + +func (m *MountImage) IsRootFs() bool { + return m.Type == RootfsImage +} + +func (m *MountImage) IsPatch() bool { + return m.Type == PatchImage +} + +func MergeEnvWithBuiltinKeys(src map[string]string, m MountImage) map[string]string { + out := make(map[string]string, len(src)) + for k, v := range src { + out[k] = v + } + if m.IsRootFs() { + out[ImageKubeVersionEnvSysKey] = m.Labels[ImageKubeVersionKey] + out[ImageSealosVersionEnvSysKey] = version.Get().GitVersion + } + return out } type ClusterPhase string diff --git a/pkg/version/utils/utils.go b/pkg/version/utils/utils.go new file mode 100644 index 000000000..4efd7f9d1 --- /dev/null +++ b/pkg/version/utils/utils.go @@ -0,0 +1,65 @@ +/* +Copyright 2023 sealos. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "fmt" + + "gopkg.in/yaml.v2" + + "github.com/labring/sealos/pkg/constants" + v2 "github.com/labring/sealos/pkg/types/v1beta1" + "github.com/labring/sealos/pkg/utils/exec" + "github.com/labring/sealos/pkg/utils/logger" + "github.com/labring/sealos/pkg/version" +) + +func GetKubernetesVersion(cluster *v2.Cluster) *version.KubernetesVersion { + var cmd string + if cluster == nil { + cmd = "kubectl version -o yaml" + } else { + data := constants.NewData(cluster.Name) + cmd = fmt.Sprintf("kubectl version --kubeconfig %s -o yaml", data.AdminFile()) + } + serverVersion, err := exec.RunBashCmd(cmd) + if err != nil || serverVersion == "" { + logger.Debug(err, "get kubernetes version failed") + return nil + } + var Unmarshaled version.KubernetesVersion + if err = yaml.Unmarshal([]byte(serverVersion), &Unmarshaled); err != nil { + logger.Debug(err, "unmarshal kubernetes version failed") + return nil + } + return &Unmarshaled +} + +func GetCriRuntimeVersion() *version.CriRuntimeVersion { + cmd := "crictl version" + criRuntimeVersion, err := exec.RunBashCmd(cmd) + if err != nil || criRuntimeVersion == "" { + logger.Debug(err, "get cri runtime version failed") + return nil + } + var Unmarshaled version.CriRuntimeVersion + if err = yaml.Unmarshal([]byte(criRuntimeVersion), &Unmarshaled); err != nil { + logger.Debug(err, "unmarshal cri runtime version failed") + return nil + } + return &Unmarshaled +} diff --git a/pkg/version/version.go b/pkg/version/version.go index c29a44673..a0cb2f60e 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -19,16 +19,6 @@ package version import ( "fmt" "runtime" - - "github.com/labring/sealos/pkg/utils/logger" - - "gopkg.in/yaml.v2" - - "github.com/labring/sealos/pkg/utils/exec" - - "github.com/labring/sealos/pkg/constants" - - v2 "github.com/labring/sealos/pkg/types/v1beta1" ) // Get returns the overall codebase version. It's for detecting @@ -45,39 +35,3 @@ func Get() Info { Platform: fmt.Sprintf("%s/%s", runtime.GOOS, runtime.GOARCH), } } - -func GetKubernetesVersion(cluster *v2.Cluster) *KubernetesVersion { - var cmd string - if cluster == nil { - cmd = "kubectl version -o yaml" - } else { - data := constants.NewData(cluster.Name) - cmd = fmt.Sprintf("kubectl version --kubeconfig %s -o yaml", data.AdminFile()) - } - serverVersion, err := exec.RunBashCmd(cmd) - if err != nil || serverVersion == "" { - logger.Debug(err, "get kubernetes version failed") - return nil - } - var Unmarshaled KubernetesVersion - if err = yaml.Unmarshal([]byte(serverVersion), &Unmarshaled); err != nil { - logger.Debug(err, "unmarshal kubernetes version failed") - return nil - } - return &Unmarshaled -} - -func GetCriRuntimeVersion() *CriRuntimeVersion { - cmd := "crictl version" - criRuntimeVersion, err := exec.RunBashCmd(cmd) - if err != nil || criRuntimeVersion == "" { - logger.Debug(err, "get cri runtime version failed") - return nil - } - var Unmarshaled CriRuntimeVersion - if err = yaml.Unmarshal([]byte(criRuntimeVersion), &Unmarshaled); err != nil { - logger.Debug(err, "unmarshal cri runtime version failed") - return nil - } - return &Unmarshaled -} diff --git a/test/e2e/apply_docker_test.go b/test/e2e/apply_docker_test.go index c1e440579..29c4c9422 100644 --- a/test/e2e/apply_docker_test.go +++ b/test/e2e/apply_docker_test.go @@ -78,7 +78,7 @@ networking: serviceSubnet: "100.55.0.0/16" podSubnet: "10.160.0.0/12" ` - dFile := config.Dockerfile{ + dFile := config.RootfsDockerfile{ KubeadmYaml: kubeadm, BaseImage: "labring/kubernetes-docker:v1.25.0", } diff --git a/test/e2e/apply_test.go b/test/e2e/apply_test.go index 11a78af95..b0dda1cc3 100644 --- a/test/e2e/apply_test.go +++ b/test/e2e/apply_test.go @@ -78,7 +78,7 @@ networking: serviceSubnet: "100.55.0.0/16" podSubnet: "10.160.0.0/12" ` - dFile := config.Dockerfile{ + dFile := config.RootfsDockerfile{ KubeadmYaml: kubeadm, BaseImage: "labring/kubernetes:v1.25.0", } diff --git a/test/e2e/filesystem_test.go b/test/e2e/filesystem_test.go index 995954093..b1a3b566f 100644 --- a/test/e2e/filesystem_test.go +++ b/test/e2e/filesystem_test.go @@ -43,7 +43,7 @@ var _ = Describe("E2E_sealos_filesystem_test", func() { Context("sealos filesystem suit", func() { BeforeEach(func() { By("build rootfs") - dFile := config.Dockerfile{ + dFile := config.RootfsDockerfile{ BaseImage: "labring/kubernetes:v1.25.0", Copys: []string{"sealctl opt/"}, } diff --git a/test/e2e/images_buildrun_feature_test.go b/test/e2e/images_buildrun_feature_test.go index 735c7c8dc..50f40c201 100644 --- a/test/e2e/images_buildrun_feature_test.go +++ b/test/e2e/images_buildrun_feature_test.go @@ -38,7 +38,7 @@ var _ = Describe("E2E_sealos_images_buildrun_feature_test", func() { It("images build image running cluster", func() { By("write dockerfile") _ = os.Setenv("SEALOS_REGISTRY_SYNC_EXPERIMENTAL", "true") - dFile := config.Dockerfile{ + dFile := config.RootfsDockerfile{ Images: []string{"docker.io/altinity/clickhouse-operator:0.18.4", "docker.io/altinity/metrics-exporter:0.18.4"}, BaseImage: "labring/kubernetes:v1.25.0", Copys: []string{"sealctl opt/"}, diff --git a/test/e2e/images_buildrun_test.go b/test/e2e/images_buildrun_test.go index 677a4b2cd..7542190f5 100644 --- a/test/e2e/images_buildrun_test.go +++ b/test/e2e/images_buildrun_test.go @@ -35,7 +35,7 @@ var _ = Describe("E2E_sealos_images_buildrun_test", func() { It("images build image running cluster", func() { By("write dockerfile") - dFile := config.Dockerfile{ + dFile := config.RootfsDockerfile{ Images: []string{"docker.io/altinity/clickhouse-operator:0.18.4", "docker.io/altinity/metrics-exporter:0.18.4"}, BaseImage: "labring/kubernetes:v1.25.0", Copys: []string{"sealctl opt/"}, diff --git a/test/e2e/run_docker_test.go b/test/e2e/run_docker_test.go index fc883f135..319f9df58 100644 --- a/test/e2e/run_docker_test.go +++ b/test/e2e/run_docker_test.go @@ -60,7 +60,7 @@ kind: ClusterConfiguration networking: serviceSubnet: "100.55.0.0/16" ` - dFile := config.Dockerfile{ + dFile := config.RootfsDockerfile{ KubeadmYaml: kubeadm, BaseImage: "labring/kubernetes-docker:v1.25.0", } diff --git a/test/e2e/run_patch_test.go b/test/e2e/run_patch_test.go new file mode 100644 index 000000000..fd3472aca --- /dev/null +++ b/test/e2e/run_patch_test.go @@ -0,0 +1,85 @@ +/* +Copyright 2023 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "path" + + "github.com/labring/sealos/test/e2e/testhelper/config" + "github.com/labring/sealos/test/e2e/testhelper/utils" + + "github.com/labring/sealos/test/e2e/suites/operators" + + . "github.com/onsi/ginkgo/v2" + + "github.com/labring/sealos/test/e2e/suites/checkers" +) + +var _ = Describe("E2E_sealos_run_patchimage_test", func() { + var ( + fakeClient *operators.FakeClient + err error + fakeCheckInterface checkers.FakeInterface + ) + fakeClient = operators.NewFakeClient("") + Context("sealos run suit", func() { + AfterEach(func() { + err = fakeClient.Cluster.Reset() + utils.CheckErr(err, fmt.Sprintf("failed to reset cluster for earch cluster: %v", err)) + }) + It("sealos run patch image containerd", func() { + By("write dockerfile") + dFile := config.PatchDockerfile{ + Images: []string{"nginx"}, + Copys: []string{"sealctl opt/sealctl", "image-cri-shim cri/image-cri-shim"}, + Cmds: []string{"systemctl stop image-cri-shim", "cp cri/image-cri-shim /usr/bin/image-cri-shim", "systemctl start image-cri-shim", "image-cri-shim -v"}, + } + tmpdir, err := dFile.Write() + utils.CheckErr(err, fmt.Sprintf("failed to create dockerfile: %v", err)) + + By("copy sealctl to rootfs") + err = fakeClient.CmdInterface.Copy("/tmp/sealctl", path.Join(tmpdir, "sealctl")) + utils.CheckErr(err, fmt.Sprintf("failed to copy sealctl to rootfs: %v", err)) + + By("copy image-cri-shim to rootfs") + err = fakeClient.CmdInterface.Copy("/tmp/image-cri-shim", path.Join(tmpdir, "image-cri-shim")) + utils.CheckErr(err, fmt.Sprintf("failed to copy image-cri-shim to rootfs: %v", err)) + + By("build image") + err = fakeClient.Image.BuildImage("test-build-image:patch-upgrade", tmpdir, operators.BuildOptions{ + MaxPullProcs: 5, + }) + utils.CheckErr(err) + + images := []string{"labring/kubernetes:v1.25.0", "labring/helm:v3.8.2"} + err = fakeClient.Image.PullImage(images...) + utils.CheckErr(err, fmt.Sprintf("failed to pull image: %v", err)) + err = fakeClient.Cluster.Run(images...) + utils.CheckErr(err, fmt.Sprintf("failed to Run new cluster for single using tar: %v", err)) + newImages := []string{"labring/kubernetes:v1.25.0", "labring/helm:v3.8.2"} + fakeCheckInterface, err = checkers.NewFakeGroupClient("default", &checkers.FakeOpts{Images: newImages}) + utils.CheckErr(err, fmt.Sprintf("failed to get cluster interface: %v", err)) + err = fakeCheckInterface.Verify() + utils.CheckErr(err, fmt.Sprintf("failed to verify cluster for single: %v", err)) + patchImage := "test-build-image:patch-upgrade" + err = fakeClient.Cluster.Run(patchImage) + utils.CheckErr(err, fmt.Sprintf("failed to Run patch image for single using tar: %v", err)) + }) + }) + +}) diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index ca19a0195..ed66a2de1 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -62,7 +62,7 @@ kind: ClusterConfiguration networking: serviceSubnet: "100.55.0.0/16" ` - dFile := config.Dockerfile{ + dFile := config.RootfsDockerfile{ KubeadmYaml: kubeadm, BaseImage: "labring/kubernetes:v1.25.0", } diff --git a/test/e2e/testhelper/config/patchfs.go b/test/e2e/testhelper/config/patchfs.go new file mode 100644 index 000000000..5d4889376 --- /dev/null +++ b/test/e2e/testhelper/config/patchfs.go @@ -0,0 +1,90 @@ +/* +Copyright 2023 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import ( + "bytes" + "fmt" + "os" + "path" + + "github.com/labring/sealos/pkg/utils/logger" + + "github.com/labring/sealos/test/e2e/testhelper/template" + "github.com/labring/sealos/test/e2e/testhelper/utils" + + "github.com/pkg/errors" +) + +const ( + // ImageDockerfile = `FROM scratch + //COPY . .` + PatchfsTemplateDockerfile = `FROM scratch +MAINTAINER labring +LABEL sealos.io.type="patch" +{{- if .Copys }} +{{- range .Copys }} +COPY {{.}} +{{- end }} +{{- end }} +{{- if .Cmds }} +CMD [{{- range $index, $cmd := .Cmds }}{{if $index}}, {{end}}"{{ $cmd }}"{{- end }}] +{{- end }}` +) + +type PatchDockerfile struct { + Images []string + dockerfileContent string + Copys []string + Cmds []string +} + +func (d *PatchDockerfile) Write() (string, error) { + tmpdir, err := utils.MkTmpdir("") + if err != nil { + return "", errors.WithMessage(err, "create tmpdir failed") + } + if len(d.Images) != 0 { + if err := os.MkdirAll(path.Join(tmpdir, "images", "shim"), 0755); err != nil { + return "", errors.WithMessage(err, "create images dir failed") + } + for i, image := range d.Images { + if err := os.WriteFile(path.Join(tmpdir, "images", "shim", fmt.Sprintf("image%d", i)), []byte(image), 0644); err != nil { + return "", errors.WithMessage(err, "write shim image failed") + } + } + d.Copys = append(d.Copys, "registry registry") + } + + t, _, err := template.TryParse(PatchfsTemplateDockerfile) + if err != nil { + return "", err + } + out := bytes.NewBuffer(nil) + if err = t.Execute(out, d); err != nil { + return "", err + } + d.dockerfileContent = out.String() + if d.dockerfileContent == "" { + return "", errors.New("dockerfile content is not set") + } + logger.Info("dockerfile content: %s", d.dockerfileContent) + if err := os.WriteFile(tmpdir+"/Dockerfile", []byte(d.dockerfileContent), 0644); err != nil { + return "", errors.WithMessage(err, "write RootfsDockerfile failed") + } + return tmpdir, nil +} diff --git a/test/e2e/testhelper/config/patchfs_test.go b/test/e2e/testhelper/config/patchfs_test.go new file mode 100644 index 000000000..debb7071e --- /dev/null +++ b/test/e2e/testhelper/config/patchfs_test.go @@ -0,0 +1,59 @@ +/* +Copyright 2023 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package config + +import "testing" + +func TestPatchDockerfile_Write(t *testing.T) { + type fields struct { + Images []string + dockerfileContent string + Copys []string + Cmds []string + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "default", + fields: fields{ + Images: []string{"nginx"}, + Copys: []string{"/tmp/sealctl /opt/"}, + Cmds: []string{"bash upgrade.sh", "bash init.sh"}, + }, + wantErr: false, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + d := &PatchDockerfile{ + Images: tt.fields.Images, + dockerfileContent: tt.fields.dockerfileContent, + Copys: tt.fields.Copys, + Cmds: tt.fields.Cmds, + } + got, err := d.Write() + if (err != nil) != tt.wantErr { + t.Errorf("Write() error = %v, wantErr %v", err, tt.wantErr) + return + } + t.Logf("out dir is %s", got) + }) + } +} diff --git a/test/e2e/testhelper/config/dockerfile.go b/test/e2e/testhelper/config/rootfs.go similarity index 89% rename from test/e2e/testhelper/config/dockerfile.go rename to test/e2e/testhelper/config/rootfs.go index 456418f71..970b83545 100644 --- a/test/e2e/testhelper/config/dockerfile.go +++ b/test/e2e/testhelper/config/rootfs.go @@ -31,7 +31,7 @@ import ( const ( // ImageDockerfile = `FROM scratch //COPY . .` - TemplateDockerfile = `FROM {{ .BaseImage }} + RootfsTemplateDockerfile = `FROM {{ .BaseImage }} MAINTAINER labring {{- if .Copys }} {{- range .Copys }} @@ -40,7 +40,7 @@ COPY {{.}} {{- end }}` ) -type Dockerfile struct { +type RootfsDockerfile struct { Images []string KubeadmYaml string BaseImage string @@ -48,7 +48,7 @@ type Dockerfile struct { Copys []string } -func (d *Dockerfile) Write() (string, error) { +func (d *RootfsDockerfile) Write() (string, error) { tmpdir, err := utils.MkTmpdir("") if err != nil { return "", errors.WithMessage(err, "create tmpdir failed") @@ -75,7 +75,7 @@ func (d *Dockerfile) Write() (string, error) { d.Copys = append(d.Copys, "kubeadm.yml etc/") } - t, _, err := template.TryParse(TemplateDockerfile) + t, _, err := template.TryParse(RootfsTemplateDockerfile) if err != nil { return "", err } @@ -88,7 +88,7 @@ func (d *Dockerfile) Write() (string, error) { return "", errors.New("dockerfile content is not set") } if err := os.WriteFile(tmpdir+"/Dockerfile", []byte(d.dockerfileContent), 0644); err != nil { - return "", errors.WithMessage(err, "write Dockerfile failed") + return "", errors.WithMessage(err, "write RootfsDockerfile failed") } return tmpdir, nil } diff --git a/test/e2e/testhelper/config/dockerfile_test.go b/test/e2e/testhelper/config/rootfs_test.go similarity index 98% rename from test/e2e/testhelper/config/dockerfile_test.go rename to test/e2e/testhelper/config/rootfs_test.go index eeefe6ee8..c2507b989 100644 --- a/test/e2e/testhelper/config/dockerfile_test.go +++ b/test/e2e/testhelper/config/rootfs_test.go @@ -61,7 +61,7 @@ func TestDockerfile_Write(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - d := &Dockerfile{ + d := &RootfsDockerfile{ Images: tt.fields.Images, KubeadmYaml: tt.fields.KubeadmYaml, BaseImage: tt.fields.BaseImage,