diff --git a/cmd/etcd.go b/cmd/etcd.go index 4a7de97bd..ae0998c28 100644 --- a/cmd/etcd.go +++ b/cmd/etcd.go @@ -16,6 +16,7 @@ limitations under the License. package cmd import ( + "fmt" "github.com/fanux/sealos/install" "github.com/spf13/cobra" "github.com/wonderivan/logger" @@ -26,6 +27,10 @@ import ( var exampleCmd = ` # snapshot save the etcd, the backupPath is on etcd nodes. not on the sealos init machine. sealos etcd save --name snapshot --backupPath /opt/sealos/ectd-backup + + # save the snapshot to aliyun oss bucket. this recommend to use in cronjob + sealos etcd save --aliId youraliyunkeyid --aliKey youraliyunkeysecrets --ep oss-cn-hangzhou.aliyuncs.com \ + --bucket etcdbackup --objectPath /sealos/ --docker # snapshot restore the etcd sealos etcd restore --name snapshot --backupPath /opt/sealos/ectd-backup @@ -34,6 +39,8 @@ var exampleCmd = ` sealos etcd health ` +var force bool + func init() { rootCmd.AddCommand(NewEtcdCommand()) // Here you will define your flags and configuration settings. @@ -73,7 +80,7 @@ func NewEtcdSaveCommand() *cobra.Command { Short: "Stores an etcd node backend snapshot to a given file", Run: EtcdSaveCmdFunc, } - cmd.Flags().BoolVar(&install.InDocker, "docker", false, "snapshot your kubernets etcd in container") + cmd.Flags().BoolVar(&install.InDocker, "docker", false, "snapshot your kubernets etcd in container, will add unix timestamp to snapshot name") cmd.Flags().StringVar(&install.SnapshotName, "name", install.ETCDSNAPSHOTDEFAULTNAME, "Specify snapshot name") cmd.Flags().StringVar(&install.EtcdBackDir, "backupPath", install.ETCDDEFAULTBACKUPDIR, "Specify snapshot backup dir") cmd.Flags().StringVar(&install.BucketName, "bucket","","oss bucketName to save snapshot") @@ -94,6 +101,7 @@ func NewEtcdRestoreCommand() *cobra.Command { cmd.Flags().StringVar(&install.SnapshotName, "name", install.ETCDSNAPSHOTDEFAULTNAME, "Specify snapshot name") cmd.Flags().StringVar(&install.EtcdBackDir, "backupPath", install.ETCDDEFAULTBACKUPDIR, "Specify snapshot backup dir") cmd.Flags().StringVar(&install.RestorePath, "restorePath", install.ETCDDEFAULTRESTOREDIR, "Specify snapshot restore dir") + cmd.Flags().BoolVarP(&force, "force","f", false, "restore need interactive to confirm") return cmd } @@ -114,6 +122,15 @@ func EtcdSaveCmdFunc(cmd *cobra.Command, args []string) { func EtcdRestoreCmdFunc(cmd *cobra.Command, args []string) { e := install.GetRestoreFlags() + // restore need interactive to confirm + if !force { + prompt := fmt.Sprintf("restore cmd will stop your kubernetes cluster immediately and restore etcd from your backup %s file (y/n)?", e.Name) + result := install.Confirm(prompt) + if !result { + logger.Info("restore %s file is skip, Exit", e.Name) + os.Exit(-1) + } + } e.RestoreAll() //logger.Info("Restore Success! Check Your Restore Dir: %s", e.RestoreDir) time.Sleep(time.Second * 10) diff --git a/docs/etcdbackup.md b/docs/etcdbackup.md new file mode 100644 index 000000000..719a8010f --- /dev/null +++ b/docs/etcdbackup.md @@ -0,0 +1,266 @@ +# `sealos etcd` 命令 + +## 使用`sealos etcd save` + +备份落盘位置会在执行sealos的主机上. 同时, restore的时候, 也会恢复在`执行sealos的主机上`, `sealos save` 逻辑是**调用 `etcd clientv3` 生成`snapshot`备份文件.** + +``` +$ sealos etcd save -h +Flags: + --aliId string aliyun accessKeyId to save snapshot + --aliKey string aliyun accessKeySecrets to save snapshot + --backupPath string Specify snapshot backup dir (default "/opt/sealos/ectd-backup") + --bucket string oss bucketName to save snapshot + --docker snapshot your kubernets etcd in container, will add unix timestamp to snapshot name + --ep string aliyun endpoints to save snapshot + -h, --help help for save + --name string Specify snapshot name (default "snapshot") + --objectPath string aliyun oss objectPath to save snapshot, like: /sealos/snapshots/ +``` + +说明一下选项 + +- `--aliId`: 阿里云的`accessKeyId` +- `--aliKey`: 阿里云的`accessKeySecrets` +- `--backupPath`: 在执行sealos的主机以及master主机上的备份路径. 默认为`/opt/sealos/ectd-backup` +- `--bucket`: 阿里云的`oss bucketName` +- `--ep`: 阿里云`oss endpoint`. 例如: `oss-cn-hangzhou.aliyuncs.com` +- `--name`: 备份文件的名字, 默认为`snapshot` +- `--objectPath`: 阿里云`oss objectPath`. 例如: `/sealos/snapshots/` + +这里, 上传阿里云oss. 使用命令行或者编辑配置文件均可. + +``` +$ sealos etcd save --docker \ + --aliId youraliyunkeyid \ + --aliKey youraliyunkeysecrets \ + --ep oss-cn-hangzhou.aliyuncs.com \ + --bucket etcdbackup \ + --objectPath /sealos/ +``` + +或者使用`vim` 编辑 `.sealos/config.yaml `文件. 均可 + +``` +$ cat .sealos/config.yaml +masters: +- 192.168.0.31:22 +nodes: +- 192.168.0.30:22 +- 192.168.0.88:22 +- 192.168.0.65:22 +dnsdomain: cluster.local +apiservercertsans: +- 127.0.0.1 +- apiserver.cluster.local +- 192.168.0.31 +- 10.103.97.2 +user: root +passwd: "" +privatekey: /root/.ssh/id_rsa +pkpassword: "" +apiserverdomian: apiserver.cluster.local +vip: 10.103.97.2 +pkgurl: /root/kube1.18.0.tar.gz +version: v1.18.0 +repo: k8s.gcr.io +podcidr: 100.64.0.0/10 +svccidr: 10.96.0.0/12 +certpath: /root/.sealos/pki +certetcdpath: /root/.sealos/pki/etcd +lvscarename: fanux/lvscare +lvscaretag: latest +alioss: + ossendpoint: oss-cn-hangzhou.aliyuncs.com + accesskeyid: ***** + accesskeysecrets: **** + bucketname: etcdbackup + objectpath: /sealos/ +``` + +### 在`kubernetes `使用`cronjob`备份 + +依赖 `~/.sealos/config.yaml` +依赖 `etcd cacert，cert， key`， 目前读取的文件是 `~/.sealos/pki/etcd/`的证书。 + +首先. 只需要挂载`~/.sealos/`到容器`~/.sealos/`. 挂载这个目录的目的是， 使用其中四个文件. 给四个文件创建`secret`感觉有点臃肿. 故而采用目录挂载. 我们假设`master-01`是您执行`sealos init`的主机. + +> 如果在集群外执行`init`. 请使用`scp`. 例如`scp -rf ~/.sealos master-01:/root/`. + +``` +$ kubectl label nodes master-01 name=sealos --overwrite +``` + +如果你通过ssh连接到各master的方式如果为秘钥验证, 则需要添加一个`secret`保存秘钥即可. 为了安全，我这边使用在`kube-system`中创建的。 + +``` +$ kubectl create secret generic pk-sealos --from-file=/root/.ssh/id_rsa -n kube-system +``` + +我们编辑`crontjob.yaml` + +``` +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: backup-etcd + namespace: kube-system ## you can change to any namespace +spec: #当没有指定pod选择器时它将根据pod模板中的标签创建 + schedule: "0 21 * * *" # backup everyday on 21：00 + successfulJobsHistoryLimit: 3 + jobTemplate: #创建新pod所使用的pod模板 + spec: + template: # 此CronJob创建Job资源会用到的模板 + spec: + restartPolicy: OnFailure # Job不能使用Always作为默认的重新启动策略 + nodeSelector: + name: sealos ## this is your sealos init machine. so we can use hostPath. + volumes: + - hostPath: + path: /root/.sealos + type: DirectoryOrCreate + name: sealos-home + # if use password to auth, just remove this secret + # before use secret, you must create it. + - secret: + defaultMode: 420 + items: + - key: id_rsa + path: id_rsa + secretName: pk-sealos + name: pk-sealos + containers: + - name: sealos + image: louisehong/sealos:3.3.10 + args: + - etcd + - save + - --docker + volumeMounts: + - mountPath: /root/.sealos + name: sealos-home + # if use password to auth, just remove this volumeMounts + - mountPath: /root/.ssh/id_rsa + name: pk-sealos + subPath: id_rsa +``` + +然后执行`apply`即可. + +``` +$ kubectl apply -f crontjob.yaml +$ kubectl get cronjobs.batch -n kube-system +NAME SCHEDULE SUSPEND ACTIVE LAST SCHEDULE AGE +backup-etcd 0 21 * * * False 0 20h 9d +``` + +备份后, 可以查看具体的日志. + +``` +$ $ kubectl get pods -n kube-system| grep backup +backup-etcd-1598662800-69sbh 0/1 Completed 0 2d8h +backup-etcd-1598706000-vwhpz 0/1 Completed 0 44h +backup-etcd-1598792400-m5z5z 0/1 Completed 0 20h +$ kubectl logs -f backup-etcd-1598792400-m5z5z -n kube-system +... +13:00:12 [INFO] [etcd_save.go:120] Finished saving/uploading snapshot [snapshot-1598792407] on aliyun oss [etcdbackup] bucket +13:00:12 [INFO] [etcd.go:111] Finished saving/uploading snapshot [snapshot-1598792407] +13:00:12 [INFO] [etcd_save.go:259] health check for etcd: [{192.168.0.31:2379 true 18.571896ms }] +``` + +## 使用`sealos etcd health` + +很简单的调用`etcd clientv3`的sdk. 执行健康检查. 每次 `save` 也会进行健康检查. + +``` +$ sealos etcd health +17:14:06 [INFO] [etcd_save.go:255] health check for etcd: [{192.168.0.31:2379 true 10.493436ms }] +``` + +## 使用`sealos etcd restore` + +`restore`的逻辑是(如果有高手有好的恢复逻辑, 请单独联系我): + +1. 手动交互确认. 如果使用了`-f 或者 --force` 则跳过确认. +2. 通过`save`下来的文件, 进行`restore`操作. 会在执行`sealos etcd restore`的主机下生成目录`restorePath + hostname`. +3. 停止 `kubernetes kube-apiserver kube-controller-manager kube-scheduler etcd `. 并备份`/var/lib/etcd/` +4. 将2生成的备份目录打包成`tar`包, 复制到各`etcd`节点(一般是`master`节点). 然后在各节点分别解压到` /var/lib/etcd `. +5. 启动`kubernetes kube-apiserver kube-controller-manager kube-scheduler etcd` +6. 最后做一次健康检查(60s). +7. 如果其中有错误发生. 则恢复3步骤的备份. + +``` +$ sealos etcd restore -h +Restores an etcd member snapshot to an etcd directory + +Usage: + sealos etcd restore [flags] + +Flags: + --backupPath string Specify snapshot backup dir (default "/opt/sealos/ectd-backup") + -f, --force restore need interactive to confirm + -h, --help help for restore + --name string Specify snapshot name (default "snapshot") + --restorePath string Specify snapshot restore dir (default "/opt/sealos/ectd-restore") + +Global Flags: + --config string config file (default is $HOME/.sealos/config.yaml) +``` + +与`sealos save` 相反的操作. 但是此操作比较危险. 所以添加了交互确认. + +选项说明 + +- `--backupPath`: 存储备份的文件夹. 默认和`save`下来的一致, `/opt/sealos/ectd-backup` + +- `--restorePath`: 恢复文件夹. 默认`/opt/sealos/ectd-restore`. 配合主机名. 避免多`master`导致恢复文件夹重复. +- `-f, --force`: 交互式确认是否要执行restore. +- `--name`: 备份的文件名字. 默认和`save`下来的一致. `snapshot` + +### 单机测试restore恢复 + +如下: + +``` +[root@dev-k8s-master ~]# ./sealos etcd restore +restore cmd will stop your kubernetes cluster immediately and restore etcd from your backup snapshot file (y/n)?y +17:34:17 [INFO] [ssh.go:12] [ssh][192.168.160.243] hostname +17:34:17 [DEBG] [ssh.go:24] [ssh][192.168.160.243]command result is: dev-k8s-master + +17:34:17 [INFO] [ssh.go:105] [ssh][192.168.160.243] cd /tmp && rm -rf /opt/sealos/ectd-restore-dev-k8s-master +17:34:17 [INFO] [ssh.go:12] [ssh][192.168.160.243] hostname +17:34:18 [DEBG] [ssh.go:24] [ssh][192.168.160.243]command result is: dev-k8s-master + +{"level":"info","ts":1598866458.160008,"caller":"snapshot/v3_snapshot.go:296","msg":"restoring snapshot","path":"/opt/sealos/ectd-backup/snapshot","wal-dir":"/opt/sealos/ectd-restore-dev-k8s-master/member/wal","data-dir":"/opt/sealos/ectd-restore-dev-k8s-master","snap-dir":"/opt/sealos/ectd-restore-dev-k8s-master/member/snap"} +{"level":"info","ts":1598866458.1982617,"caller":"mvcc/kvstore.go:380","msg":"restored last compact revision","meta-bucket-name":"meta","meta-bucket-name-key":"finishedCompactRev","restored-compact-revision":970469} +{"level":"info","ts":1598866458.2281547,"caller":"membership/cluster.go:392","msg":"added member","cluster-id":"d12074ddc55c9483","local-member-id":"0","added-peer-id":"5dfe17d3cf203a7e","added-peer-peer-urls":["https://192.168.160.243:2380"]} +{"level":"info","ts":1598866458.235216,"caller":"snapshot/v3_snapshot.go:309","msg":"restored snapshot","path":"/opt/sealos/ectd-backup/snapshot","wal-dir":"/opt/sealos/ectd-restore-dev-k8s-master/member/wal","data-dir":"/opt/sealos/ectd-restore-dev-k8s-master","snap-dir":"/opt/sealos/ectd-restore-dev-k8s-master/member/snap"} +17:34:28 [INFO] [ssh.go:105] [ssh][192.168.160.243] cd /tmp && mv /etc/kubernetes/manifests /etc/kubernetes/manifestslezSCljV +17:34:28 [INFO] [ssh.go:105] [ssh][192.168.160.243] cd /tmp && mv /var/lib/etcd /var/lib/etcdlezSCljV +17:34:38 [INFO] [etcd.go:136] send restore file to etcd master node and start etcd +17:34:38 [INFO] [ssh.go:12] [ssh][192.168.160.243] hostname +17:34:38 [DEBG] [ssh.go:24] [ssh][192.168.160.243]command result is: dev-k8s-master + +17:34:39 [INFO] [etcd_restore.go:140] compress file +17:34:39 [INFO] [ssh.go:57] [ssh][192.168.160.243] mkdir -p /var/lib || true +17:34:39 [DEBG] [download.go:29] [192.168.160.243]please wait for mkDstDir +17:34:39 [INFO] [ssh.go:12] [ssh][192.168.160.243] ls -l /var/lib/ectd-restore-dev-k8s-master.tar 2>/dev/null |wc -l +17:34:39 [DEBG] [ssh.go:24] [ssh][192.168.160.243]command result is: 0 + +17:34:39 [DEBG] [scp.go:24] [ssh]source file md5 value is bc76f9bb1aea210fb815a43aed27aa29 +17:34:40 [ALRT] [scp.go:98] [ssh][192.168.160.243]transfer total size is: 1244.01KB ;speed is 1MB +17:34:40 [INFO] [ssh.go:12] [ssh][192.168.160.243] md5sum /var/lib/ectd-restore-dev-k8s-master.tar | cut -d" " -f1 +17:34:40 [DEBG] [ssh.go:24] [ssh][192.168.160.243]command result is: bc76f9bb1aea210fb815a43aed27aa29 + +17:34:40 [DEBG] [scp.go:27] [ssh]host: 192.168.160.243 , remote md5: bc76f9bb1aea210fb815a43aed27aa29 +17:34:40 [INFO] [scp.go:31] [ssh]md5 validate true +17:34:40 [INFO] [download.go:38] [192.168.160.243]copy file md5 validate success +17:34:40 [DEBG] [download.go:44] [192.168.160.243]please wait for after hook +17:34:40 [INFO] [ssh.go:57] [ssh][192.168.160.243] tar xf /var/lib/ectd-restore-dev-k8s-master.tar -C /var/lib/ && mv /var/lib/ectd-restore-dev-k8s-master /var/lib/etcd && rm -rf /var/lib/ectd-restore-dev-k8s-master.tar +17:34:41 [INFO] [etcd.go:145] Start kube-apiserver kube-controller-manager kube-scheduler +17:34:41 [INFO] [ssh.go:105] [ssh][192.168.160.243] cd /tmp && mv /etc/kubernetes/manifestslezSCljV /etc/kubernetes/manifests +17:34:41 [INFO] [etcd.go:148] Wait 60s to health check for etcd +17:35:41 [INFO] [etcd_save.go:259] health check for etcd: [{192.168.160.243:2379 true 6.206351ms }] +17:35:41 [INFO] [etcd.go:151] restore kubernetes yourself glad~ +``` + diff --git a/pkg/appmanager/deleteapp.go b/pkg/appmanager/deleteapp.go index e2e102adb..2fb2b8eef 100644 --- a/pkg/appmanager/deleteapp.go +++ b/pkg/appmanager/deleteapp.go @@ -39,7 +39,7 @@ func DeleteApp(flag *DeleteFlags) error { pkgConfig.Workspace = fmt.Sprintf("%s/%s", flag.WorkDir, pkgConfig.Name) if !flag.CleanForce { - prompt := fmt.Sprintf("deletew command will del your installed %s App , continue delete (y/n)?", pkgConfig.Name) + prompt := fmt.Sprintf("delete command will del your installed %s App , continue delete (y/n)?", pkgConfig.Name) result := install.Confirm(prompt) if !result { logger.Info("delete %s App is skip, Exit", pkgConfig.Name) diff --git a/test/cronjob.yaml.example b/test/cronjob.yaml.example new file mode 100644 index 000000000..ea1e9e7bd --- /dev/null +++ b/test/cronjob.yaml.example @@ -0,0 +1,43 @@ +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: backup-etcd + namespace: kube-system ## you can change to any namespace +spec: #当没有指定pod选择器时它将根据pod模板中的标签创建 + schedule: "0 21 * * *" # backup everyday on 21：00 + successfulJobsHistoryLimit: 3 + jobTemplate: #创建新pod所使用的pod模板 + spec: + template: # 此CronJob创建Job资源会用到的模板 + spec: + restartPolicy: OnFailure # Job不能使用Always作为默认的重新启动策略 + nodeSelector: + name: sealos ## this is your sealos init machine. so we can use hostPath. + volumes: + - hostPath: + path: /root/.sealos + type: DirectoryOrCreate + name: sealos-home + # if use password to auth, just remove this secret + # before use secret, you must create it. + - secret: + defaultMode: 420 + items: + - key: id_rsa + path: id_rsa + secretName: pk-sealos + name: pk-sealos + containers: + - name: sealos + image: louisehong/sealos:3.3.10 + args: + - etcd + - save + - --docker + volumeMounts: + - mountPath: /root/.sealos + name: sealos-home + # if use password to auth, just remove this volumeMounts + - mountPath: /root/.ssh/id_rsa + name: pk-sealos + subPath: id_rsa \ No newline at end of file