From c747d2dea7bfc9076b3f77427793bcdf146e4ecc Mon Sep 17 00:00:00 2001 From: cuisongliu Date: Thu, 7 Apr 2022 15:34:22 +0800 Subject: [PATCH] refactor(master): add shim online for 0.0.7 version (#916) * refactor(master): add auto-build * refactor(master): add auto-build * refactor(master): add shim online for 0.0.7 version * refactor(master): add shim online for 0.0.7 version * refactor(master): add shim online for 0.0.7 version --- Makefile | 4 +- applications/Kubefile | 19 --------- applications/calico/Dockerfile | 2 +- applications/registry/Clusterfile | 46 ---------------------- pkg/apply/processor/delete.go | 8 +++- pkg/apply/utils.go | 2 +- pkg/filesystem/rootfs/rootfs_default.go | 4 +- pkg/runtime/image_shim.go | 52 +++++++++++++++++++++++++ pkg/runtime/init.go | 4 +- pkg/runtime/registry.go | 7 ++-- pkg/runtime/reset.go | 2 +- pkg/utils/contants/bash.go | 2 + pkg/utils/contants/data.go | 2 + pkg/utils/strings/strings.go | 11 ------ 14 files changed, 76 insertions(+), 89 deletions(-) delete mode 100644 applications/Kubefile delete mode 100644 applications/registry/Clusterfile create mode 100644 pkg/runtime/image_shim.go diff --git a/Makefile b/Makefile index 01d3c8b8b..f4a9b42fc 100644 --- a/Makefile +++ b/Makefile @@ -68,9 +68,9 @@ install-ossutil: ## check license if not exist install go-lint tools push-oss:install-ossutil build - $(OSSUTIL_BIN) cp -f dist/sealos_linux_amd64/sealos oss://sealyun-temp/sealos/${COMMIT_ID}/sealos + $(OSSUTIL_BIN) cp -f dist/sealos_linux_amd64/sealos oss://sealyun-temp/sealos/${COMMIT_ID}/sealos-amd64 $(OSSUTIL_BIN) cp -f dist/sealos_linux_arm64/sealos oss://sealyun-temp/sealos/${COMMIT_ID}/sealos-arm64 - $(OSSUTIL_BIN) cp -f dist/sealctl_linux_amd64/sealctl oss://sealyun-temp/sealos/${COMMIT_ID}/sealctl + $(OSSUTIL_BIN) cp -f dist/sealctl_linux_amd64/sealctl oss://sealyun-temp/sealos/${COMMIT_ID}/sealctl-amd64 $(OSSUTIL_BIN) cp -f dist/sealctl_linux_arm64/sealctl oss://sealyun-temp/sealos/${COMMIT_ID}/sealctl-arm64 generator-contributors: diff --git a/applications/Kubefile b/applications/Kubefile deleted file mode 100644 index 94c467364..000000000 --- a/applications/Kubefile +++ /dev/null @@ -1,19 +0,0 @@ -FROM scratch -MAINTAINER sealyun -LABEL init="init.sh $criData $registryDomain $registryPort $registryUsername $registryPassword" -LABEL version="v1.22.8" -LABEL image="ghcr.io/sealyun/lvscare:v1.1.3-beta.2" -LABEL clean="clean.sh $criData" -LABEL check="check.sh" -LABEL init-registry="init-registry.sh $registryPort $registryData $registryConfig" -LABEL clean-registry="clean-registry.sh $registryData $registryConfig" -LABEL auth="auth.sh" -LABEL type="kubernetes" -ENV criData=/var/lib/containerd -ENV registryData=/var/lib/registry -ENV registryConfig=/etc/registry -ENV registryDomain=sealos.hub -ENV registryPort=5000 -ENV registryUsername=admin -ENV registryPassword=passw0rd -COPY . . diff --git a/applications/calico/Dockerfile b/applications/calico/Dockerfile index 8f16de43f..3563618a6 100644 --- a/applications/calico/Dockerfile +++ b/applications/calico/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/cuisongliu/oci-kubernetes-without-cni:1.22.8 +FROM registry.cn-hongkong.aliyuncs.com/sealyun/oci-kubernetes:1.22.8 COPY cni ./cni COPY registry ./registry CMD ["kubectl apply -f cni/tigera-operator.yaml","kubectl apply -f cni/custom-resources.yaml"] diff --git a/applications/registry/Clusterfile b/applications/registry/Clusterfile deleted file mode 100644 index a523f68b2..000000000 --- a/applications/registry/Clusterfile +++ /dev/null @@ -1,46 +0,0 @@ -apiVersion: apps.sealyun.com/v1beta1 -kind: Cluster -metadata: - creationTimestamp: null - name: default -spec: - env: - - criData=/var/lib/containerd - - registryData=/var/lib/registry - - registryConfig=/etc/registry - - registryDomain=sealos.hub - - registryPort=5000 - - registryUsername=admin - - registryPassword=passw0rd - hosts: - - ips: - - 192.168.64.15 - roles: - - master - - amd64 - - ips: - - 192.168.64.17 - roles: - - node - - amd64 - ssh: - pk: /Users/cuisongliu/.ssh/id_rsa - port: 22 - user: root - ---- -apiVersion: apps.sealyun.com/v1beta1 -kind: Config -metadata: - creationTimestamp: null - name: registry -spec: - data: | - domain: sealos.hub - ip: 192.168.64.15 - password: passw0rd - port: 5000 - username: admin - data: /var/lib/registry - path: etc/registry.yml - strategy: merge diff --git a/pkg/apply/processor/delete.go b/pkg/apply/processor/delete.go index 213ae9ea9..29bfd3023 100644 --- a/pkg/apply/processor/delete.go +++ b/pkg/apply/processor/delete.go @@ -17,6 +17,8 @@ package processor import ( "fmt" + "github.com/fanux/sealos/pkg/utils/logger" + "github.com/fanux/sealos/pkg/clusterfile" "github.com/fanux/sealos/pkg/filesystem" "github.com/fanux/sealos/pkg/image" @@ -40,7 +42,7 @@ type DeleteProcessor struct { func (d DeleteProcessor) Execute(cluster *v2.Cluster) (err error) { d.cManifest, err = d.ClusterManager.Inspect(cluster.Name) if err != nil { - return fmt.Errorf("failed to inspect cluster, %v", err) + logger.Warn("delete process failed to inspect cluster, %v", err) } d.img, err = d.ImageManager.Inspect(cluster.Spec.Image) if err != nil { @@ -82,6 +84,10 @@ func (d DeleteProcessor) GetPipeLine() ([]func(cluster *v2.Cluster) error, error func (d DeleteProcessor) UnMountRootfs(cluster *v2.Cluster) error { hosts := append(cluster.GetMasterIPList(), cluster.GetNodeIPList()...) + if d.cManifest == nil { + logger.Warn("delete process unmount rootfs skip is cluster not mount rootfs") + return nil + } fs, err := filesystem.NewRootfsMounter(d.cManifest, d.img) if err != nil { return err diff --git a/pkg/apply/utils.go b/pkg/apply/utils.go index da448b90c..763cc2014 100644 --- a/pkg/apply/utils.go +++ b/pkg/apply/utils.go @@ -54,5 +54,5 @@ func PreProcessIPList(joinArgs *RunArgs) error { } func removeIPListDuplicatesAndEmpty(ipList []string) []string { - return strings2.DedupeStrSlice(strings2.RemoveStrSlice(ipList, []string{""})) + return strings2.RemoveDuplicate(strings2.RemoveStrSlice(ipList, []string{""})) } diff --git a/pkg/filesystem/rootfs/rootfs_default.go b/pkg/filesystem/rootfs/rootfs_default.go index ae882e710..6eec80e47 100644 --- a/pkg/filesystem/rootfs/rootfs_default.go +++ b/pkg/filesystem/rootfs/rootfs_default.go @@ -23,6 +23,8 @@ import ( "path" "path/filepath" + "github.com/fanux/sealos/pkg/runtime" + "github.com/fanux/sealos/pkg/utils/logger" "github.com/fanux/sealos/pkg/image/types" @@ -89,7 +91,7 @@ func (f *defaultRootfs) mountRootfs(cluster *v2.Cluster, ipList []string) error if checkBash == "" { return nil } - if err = f.getSSH(cluster).CmdAsync(ip, envProcessor.WrapperShell(ip, check.CheckBash())); err != nil { + if err = f.getSSH(cluster).CmdAsync(ip, envProcessor.WrapperShell(ip, check.CheckBash()), runtime.ApplyImageShimCMD(target)); err != nil { return err } if err = f.getSSH(cluster).CmdAsync(ip, envProcessor.WrapperShell(ip, check.InitBash())); err != nil { diff --git a/pkg/runtime/image_shim.go b/pkg/runtime/image_shim.go new file mode 100644 index 000000000..95a2322c2 --- /dev/null +++ b/pkg/runtime/image_shim.go @@ -0,0 +1,52 @@ +/* +Copyright 2022 cuisongliu@qq.com. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package runtime + +import ( + "fmt" + "path" + + "github.com/fanux/sealos/pkg/utils/contants" + "github.com/fanux/sealos/pkg/utils/logger" + "github.com/fanux/sealos/pkg/utils/yaml" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +var defaultRootDirectory = "/var/lib/image-cri-shim" + +//GetImageShim default dir is /var/lib/image-cri-shim +func GetImageShim(rootfs string) string { + const imageCustomConfig = "image-cri-shim.yaml" + etcPath := path.Join(rootfs, contants.EtcDirName, imageCustomConfig) + registryConfig, err := yaml.Unmarshal(etcPath) + if err != nil { + logger.Debug("use default registry config") + return defaultRootDirectory + } + image, _, _ := unstructured.NestedString(registryConfig, "image") + logger.Debug("show image shim info, image dir : %s ", image) + return image +} + +func ApplyImageShimCMD(rootfs string) string { + shimData := GetImageShim(rootfs) + return fmt.Sprintf(contants.DefaultCPFmt, shimData, path.Join(rootfs, contants.ImagesDirName, contants.ImageShimDirName), shimData) +} + +func DeleteImageShimCMD(rootfs string) string { + return fmt.Sprintf("rm -rf %s", GetImageShim(rootfs)) +} diff --git a/pkg/runtime/init.go b/pkg/runtime/init.go index c66ce4705..b5aa327ae 100644 --- a/pkg/runtime/init.go +++ b/pkg/runtime/init.go @@ -28,13 +28,13 @@ func (k *KubeadmRuntime) ConfigInitKubeadmToMaster0() error { logger.Info("start to copy kubeadm config to master0") data, err := k.generateInitConfigs() if err != nil { - return fmt.Errorf("generator Config init kubeadm Config error: %s", err.Error()) + return fmt.Errorf("generator config init kubeadm Config error: %s", err.Error()) } initConfigPath := path.Join(k.getContantData().TmpPath(), contants.DefaultInitKubeadmFileName) outConfigPath := path.Join(k.getContantData().EtcPath(), contants.DefaultInitKubeadmFileName) err = file.WriteFile(initConfigPath, data) if err != nil { - return fmt.Errorf("write Config init kubeadm Config error: %s", err.Error()) + return fmt.Errorf("write Config init kubeadm config error: %s", err.Error()) } err = k.sshCopy(k.getMaster0IP(), initConfigPath, outConfigPath) if err != nil { diff --git a/pkg/runtime/registry.go b/pkg/runtime/registry.go index ebcd07d6b..3cb58b382 100644 --- a/pkg/runtime/registry.go +++ b/pkg/runtime/registry.go @@ -28,8 +28,6 @@ import ( "github.com/fanux/sealos/pkg/utils/logger" ) -const DefaultLnFmt = "ln -s %s %s" //ln -s src dest - func GetRegistry(rootfs, defaultRegistry string) *RegistryConfig { const registryCustomConfig = "registry.yml" var DefaultConfig = &RegistryConfig{ @@ -72,19 +70,20 @@ func GetRegistry(rootfs, defaultRegistry string) *RegistryConfig { } func (k *KubeadmRuntime) htpasswd() error { - htpasswdPath := path.Join(k.getContantData().EtcPath(), "registry_htpasswd") + htpasswdPath := path.Join(k.getContantData().RootFSEtcPath(), "registry_htpasswd") registry := k.getRegistry() if registry.Username == "" && registry.Password == "" { return nil } data := passwd.Htpasswd(registry.Username, registry.Password) + logger.Debug("write htpasswd file: %s,data: %s", htpasswdPath, data) return file.WriteFile(htpasswdPath, []byte(data)) } func (k *KubeadmRuntime) ApplyRegistry() error { logger.Info("start to apply registry") registry := k.getRegistry() - err := k.sshCmdAsync(registry.IP, fmt.Sprintf(DefaultLnFmt, k.getContantData().RootFSRegistryPath(), registry.Data)) + err := k.sshCmdAsync(registry.IP, fmt.Sprintf(contants.DefaultLnFmt, k.getContantData().RootFSRegistryPath(), registry.Data)) if err != nil { return fmt.Errorf("copy registry data failed %v", err) } diff --git a/pkg/runtime/reset.go b/pkg/runtime/reset.go index 84f398816..b43404a2e 100644 --- a/pkg/runtime/reset.go +++ b/pkg/runtime/reset.go @@ -70,7 +70,7 @@ func (k *KubeadmRuntime) resetMasters(nodes []string) { func (k *KubeadmRuntime) resetNode(node string) error { logger.Info("start to reset node: %s", node) if err := k.sshCmdAsync(node, fmt.Sprintf(RemoteCleanMasterOrNode, vlogToStr(k.vlog), k.getEtcdDataDir()), - RemoveKubeConfig); err != nil { + RemoveKubeConfig, DeleteImageShimCMD(k.getContantData().RootFSPath())); err != nil { return fmt.Errorf("exec node clean in sealos failed %v", err) } err := k.execClean(node) diff --git a/pkg/utils/contants/bash.go b/pkg/utils/contants/bash.go index bf836cb6a..7bf93ec89 100644 --- a/pkg/utils/contants/bash.go +++ b/pkg/utils/contants/bash.go @@ -20,6 +20,8 @@ import "fmt" const ( DefaultBashFmt = "cd %s && bash %s" + DefaultCPFmt = "mkdir -p %s && cp -rf %s/* %s/" + DefaultLnFmt = "ln -s %s %s" CdAndExecCmd = "cd %s && %s" renderInit = "init" renderClean = "clean" diff --git a/pkg/utils/contants/data.go b/pkg/utils/contants/data.go index 6fe6b54f1..f711dcb09 100644 --- a/pkg/utils/contants/data.go +++ b/pkg/utils/contants/data.go @@ -30,6 +30,8 @@ const ( ChartsDirName = "charts" ManifestsDirName = "manifests" RegistryDirName = "registry" + ImagesDirName = "images" + ImageShimDirName = "shim" PkiDirName = "pki" PkiEtcdDirName = "etcd" ScriptsDirName = "scripts" diff --git a/pkg/utils/strings/strings.go b/pkg/utils/strings/strings.go index 7ec2e9731..36f347037 100644 --- a/pkg/utils/strings/strings.go +++ b/pkg/utils/strings/strings.go @@ -165,17 +165,6 @@ func RemoveDuplicate(list []string) []string { func WrapExecResult(host, command string, output []byte, err error) error { return fmt.Errorf("failed to execute command(%s) on host(%s): output(%s), error(%v)", command, host, output, err) } -func DedupeStrSlice(in []string) []string { - m := make(map[string]struct{}) - var res []string - for _, s := range in { - if _, ok := m[s]; !ok { - res = append(res, s) - m[s] = struct{}{} - } - } - return res -} // RemoveStrSlice remove dst element from src slice func RemoveStrSlice(src, dst []string) []string {