From 06c9d5472c574f88214d1eca420e5715266d6bc1 Mon Sep 17 00:00:00 2001 From: cuisongliu Date: Wed, 25 Mar 2020 01:14:24 +0800 Subject: [PATCH 1/3] feat(develop): fix download feature --- go.mod | 3 +- go.sum | 4 +- install/download.go | 147 ++++++++++++++++++ install/init.go | 2 +- install/install.go | 4 +- install/join.go | 4 +- install/send.go | 11 +- install/utils.go | 132 ---------------- install/utils_test.go | 26 ++-- .../cuisongliu/sshcmd/pkg/cmd/cmd.go | 35 +++++ .../cuisongliu/sshcmd/pkg/cmd/watch.go | 58 +++++++ .../cuisongliu/sshcmd/pkg/md5sum/md5sum.go | 4 + .../cuisongliu/sshcmd/pkg/sshutil/connect.go | 7 +- .../cuisongliu/sshcmd/pkg/sshutil/types.go | 3 + vendor/modules.txt | 3 +- 15 files changed, 285 insertions(+), 158 deletions(-) create mode 100644 install/download.go create mode 100644 vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go create mode 100644 vendor/github.com/cuisongliu/sshcmd/pkg/cmd/watch.go diff --git a/go.mod b/go.mod index 6fd35fcc6..c4b3c4bcb 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,13 @@ module github.com/fanux/sealos go 1.13 require ( - github.com/cuisongliu/sshcmd v0.0.0-20200324095132-757019718a50 + github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4 github.com/fanux/sealgate v0.0.5 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.8.1 github.com/spf13/cobra v0.0.5 github.com/spf13/viper v1.5.0 github.com/wonderivan/logger v1.0.0 - golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586 gopkg.in/yaml.v2 v2.2.8 k8s.io/api v0.17.3 k8s.io/apimachinery v0.17.3 diff --git a/go.sum b/go.sum index ffc895c96..7c3163c25 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cuisongliu/sshcmd v0.0.0-20200324095132-757019718a50 h1:KvJ/qwYCUlYUiHJxaqKxF1obxqI+dvYFAVbjqj9U2RA= -github.com/cuisongliu/sshcmd v0.0.0-20200324095132-757019718a50/go.mod h1:jAEBZhFW5kNey9B+59lR8mfQaCQsyp8b8QaI7RhESxE= +github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4 h1:uUJMoSd1bMQOISCE0T1N6ig0qFH3f1vLvkRsnyq4Wuc= +github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4/go.mod h1:jAEBZhFW5kNey9B+59lR8mfQaCQsyp8b8QaI7RhESxE= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/install/download.go b/install/download.go new file mode 100644 index 000000000..139375877 --- /dev/null +++ b/install/download.go @@ -0,0 +1,147 @@ +package install + +import ( + "fmt" + "github.com/cuisongliu/sshcmd/pkg/cmd" + "github.com/cuisongliu/sshcmd/pkg/filesize" + "github.com/cuisongliu/sshcmd/pkg/md5sum" + "github.com/wonderivan/logger" + "net/url" + "path" + "strings" + "sync" +) + +//location : url +//md5 +//dst: /root +//hook: cd /root && rm -rf kube && tar zxvf %s && cd /root/kube/shell && sh init.sh +func SendPackage(location string, hosts []string, dst, hook string) { + location, md5 := downloadFile(location) + pkg := path.Base(location) + fullPath := fmt.Sprintf("%s/%s", dst, pkg) + mkDstDir := fmt.Sprintf("mkdir -p %s || true", dst) + var wm sync.WaitGroup + for _, host := range hosts { + wm.Add(1) + go func(host string) { + defer wm.Done() + SSHConfig.Cmd(host, mkDstDir) + logger.Debug("[%s]please wait for mkDstDir", host) + if SSHConfig.IsFilExist(host, fullPath) { + logger.Warn("[%s]SendPackage: file is exist", host) + } else { + if ok := SSHConfig.CopyForMD5(host, location, fullPath, md5); ok { + logger.Info("[%s]copy file md5 validate success", host) + } else { + logger.Error("[%s]copy file md5 validate failed", host) + } + } + if hook != "" { + logger.Debug("[%s]please wait for hook", host) + SSHConfig.Cmd(host, hook) + } + }(host) + } + wm.Wait() +} + +// FetchPackage if url exist wget it, or scp the local package to hosts +// dst is the remote offline path like /root +func FetchPackage(url string, hosts []string, dst string) { + pkg := path.Base(url) + fullDst := fmt.Sprintf("%s/%s", dst, pkg) + mkdstdir := fmt.Sprintf("mkdir -p %s || true", dst) + + //only http + isHttp := strings.HasPrefix(url, "http") + wgetCommand := "" + if isHttp { + wgetParam := "" + if strings.HasPrefix(url, "https") { + wgetParam = "--no-check-certificate" + } + wgetCommand = fmt.Sprintf(" wget %s ", wgetParam) + } + remoteCmd := fmt.Sprintf("cd %s && %s %s", dst, wgetCommand, url) + + var wm sync.WaitGroup + for _, host := range hosts { + wm.Add(1) + go func(host string) { + defer wm.Done() + logger.Debug("[%s]please wait for copy offline package", host) + SSHConfig.Cmd(host, mkdstdir) + if SSHConfig.IsFilExist(host, fullDst) { + logger.Warn("[%s]SendPackage: [%s] file is exist", host, fullDst) + } else { + if isHttp { + go SSHConfig.LoggerFileSize(host, fullDst, int(filesize.Do(url))) + SSHConfig.Cmd(host, remoteCmd) + rMD5 := SSHConfig.Md5Sum(host, fullDst) //获取已经上传文件的md5 + uMd5 := "urlGetMD5(url) " //获取url的md5值 + logger.Debug("[%s] remote file local %s, md5 is %s", host, fullDst, rMD5) + logger.Debug("[%s] url is %s, md5 is %s", host, url, uMd5) + if strings.TrimSpace(rMD5) == strings.TrimSpace(uMd5) { + logger.Info("[%s]file md5 validate success", host) + } else { + logger.Error("[%s]copy file md5 validate failed", host) + } + } else { + if !SSHConfig.CopyForMD5(host, url, fullDst, "") { + logger.Error("[%s]copy file md5 validate failed", host) + } else { + logger.Info("[%s]file md5 validate success", host) + } + } + } + }(host) + } + wm.Wait() +} + +// +func downloadFile(location string) (filePATH, md5 string) { + if _, isUrl := isUrl(location); isUrl { + absPATH := "/tmp/sealos/" + path.Base(location) + if cmd.IsFilExist(absPATH) { + //logs + logger.Warn("[%s] file is exist", absPATH) + location = absPATH + goto end + } + //generator download cmd + dwnCmd := downloadCmd(location) + //go func watch filesize + go cmd.LoggerFileSize(absPATH, int(filesize.Do(location))) + //os exec download command + cmd.Cmd("/bin/sh", "-c", "mkdir -p /tmp/sealos && cd /tmp/sealos && "+dwnCmd) + location = absPATH + } +end: + //file md5 + md5 = md5sum.FromLocal(location) + return location, md5 +} + +//根据url 获取command +func downloadCmd(url string) string { + //only http + u, isHttp := isUrl(url) + var c = "" + if isHttp { + param := "" + if u.Scheme == "https" { + param = "--no-check-certificate" + } + c = fmt.Sprintf(" wget %s %s", param, url) + } + return c +} + +func isUrl(u string) (url.URL, bool) { + if uu, err := url.Parse(u); err == nil && uu != nil && uu.Host != "" { + return *uu, true + } + return url.URL{}, false +} diff --git a/install/init.go b/install/init.go index a09b9aec0..e4fadaed2 100644 --- a/install/init.go +++ b/install/init.go @@ -23,7 +23,7 @@ func BuildInit() { } i.CheckValid() i.Print() - i.SendPackage("kube") + i.SendPackage() i.Print("SendPackage") i.KubeadmConfigInstall() i.Print("SendPackage", "KubeadmConfigInstall") diff --git a/install/install.go b/install/install.go index 698cab808..67cc2549f 100644 --- a/install/install.go +++ b/install/install.go @@ -189,7 +189,7 @@ func (r *RunOnEveryNodes) Run(config SealConfig, url, pkgName string) { workspace := fmt.Sprintf("/root/%s", pkgName) nodes := append(config.Masters, config.Nodes...) - FetchPackage(url, nodes, workspace) + SendPackage(url, nodes, workspace, "") for _, node := range nodes { wg.Add(1) go func(node string) { @@ -210,7 +210,7 @@ type RunOnMaster struct { func (r *RunOnMaster) Run(config SealConfig, url, pkgName string) { workspace := fmt.Sprintf("/root/%s", pkgName) - FetchPackage(url, []string{config.Masters[0]}, workspace) + SendPackage(url, []string{config.Masters[0]}, workspace, "") tarCmd := fmt.Sprintf("tar xvf %s.tar", pkgName) CmdWorkSpace(config.Masters[0], tarCmd, workspace) for _, cmd := range r.Cmd { diff --git a/install/join.go b/install/join.go index 3301c0245..2bd66ae35 100644 --- a/install/join.go +++ b/install/join.go @@ -27,7 +27,7 @@ func joinMastersFunc(joinMasters []string) { Nodes: nodes, } i.CheckValid() - i.SendPackage("kube") + i.SendPackage() i.GeneratorCerts() i.JoinMasters(joinMasters) //master join to MasterIPs @@ -46,7 +46,7 @@ func joinNodesFunc(joinNodes []string) { Nodes: nodes, } i.CheckValid() - i.SendPackage("kube") + i.SendPackage() i.GeneratorToken() i.JoinNodes() //node join to NodeIPs diff --git a/install/send.go b/install/send.go index e05a641bc..fdacc3980 100644 --- a/install/send.go +++ b/install/send.go @@ -1,6 +1,13 @@ package install +import ( + "fmt" + "path" +) + //SendPackage is -func (s *SealosInstaller) SendPackage(packName string) { - SendPackage(PkgUrl, s.Hosts, packName) +func (s *SealosInstaller) SendPackage() { + pkg := path.Base(PkgUrl) + hook := fmt.Sprintf("cd /root && rm -rf kube && tar zxvf %s && cd /root/kube/shell && sh init.sh", pkg) + SendPackage(PkgUrl, s.Hosts, "/root", hook) } diff --git a/install/utils.go b/install/utils.go index 50d006526..d076fcf72 100644 --- a/install/utils.go +++ b/install/utils.go @@ -2,22 +2,16 @@ package install import ( "fmt" - "github.com/cuisongliu/sshcmd/pkg/filesize" "github.com/wonderivan/logger" "math/big" "math/rand" "net" - "net/url" "os" - "path" "strconv" "strings" - "sync" "time" ) -const oneMBByte = 1024 * 1024 - //VersionToInt v1.15.6 => 115 func VersionToInt(version string) int { // v1.15.6 => 1.15.6 @@ -38,119 +32,6 @@ func IpFormat(host string) string { return ipAndPort[0] } -func SendPackage(url string, hosts []string, packName string) { - pkg := path.Base(url) - //only http - isHttp := strings.HasPrefix(url, "http") - wgetCommand := "" - if isHttp { - wgetParam := "" - if strings.HasPrefix(url, "https") { - wgetParam = "--no-check-certificate" - } - wgetCommand = fmt.Sprintf(" wget %s ", wgetParam) - } - remoteCmd := fmt.Sprintf("cd /root && %s %s && tar zxvf %s", wgetCommand, url, pkg) - localCmd := fmt.Sprintf("cd /root && rm -rf %s && tar zxvf %s ", packName, pkg) - kubeLocal := fmt.Sprintf("/root/%s", pkg) - var kubeCmd string - if packName == "kube" { - kubeCmd = "cd /root/kube/shell && sh init.sh" - } else { - kubeCmd = fmt.Sprintf("cd /root/%s && docker load -i images.tar", packName) - } - - var wm sync.WaitGroup - for _, host := range hosts { - wm.Add(1) - go func(host string) { - defer wm.Done() - logger.Debug("[%s]please wait for tar zxvf exec", host) - if SSHConfig.IsFilExist(host, kubeLocal) { - logger.Warn("[%s]SendPackage: file is exist", host) - SSHConfig.Cmd(host, localCmd) - } else { - if isHttp { - go SSHConfig.LoggerFileSize(host, kubeLocal, int(filesize.Do(url))) - SSHConfig.Cmd(host, remoteCmd) - rMD5 := SSHConfig.Md5Sum(host, kubeLocal) //获取已经上传文件的md5 - uMd5 := UrlGetMd5(url) //获取url的md5值 - logger.Debug("[%s] remote file local %s, md5 is %s", host, kubeLocal, rMD5) - logger.Debug("[%s] url is %s, md5 is %s", host, url, uMd5) - if strings.TrimSpace(rMD5) == strings.TrimSpace(uMd5) { - logger.Info("[%s]file md5 validate success", host) - } else { - logger.Error("[%s]copy file md5 validate failed", host) - } - } else { - if ok := SSHConfig.CopyForMD5(host, url, kubeLocal, ""); ok { - SSHConfig.Cmd(host, localCmd) - logger.Info("[%s]file md5 validate success", host) - } else { - logger.Error("[%s]file md5 validate failed", host) - } - } - } - SSHConfig.Cmd(host, kubeCmd) - }(host) - } - wm.Wait() -} - -// FetchPackage if url exist wget it, or scp the local package to hosts -// dst is the remote offline path like /root -func FetchPackage(url string, hosts []string, dst string) { - pkg := path.Base(url) - fullDst := fmt.Sprintf("%s/%s", dst, pkg) - mkdstdir := fmt.Sprintf("mkdir -p %s || true", dst) - - //only http - isHttp := strings.HasPrefix(url, "http") - wgetCommand := "" - if isHttp { - wgetParam := "" - if strings.HasPrefix(url, "https") { - wgetParam = "--no-check-certificate" - } - wgetCommand = fmt.Sprintf(" wget %s ", wgetParam) - } - remoteCmd := fmt.Sprintf("cd %s && %s %s", dst, wgetCommand, url) - - var wm sync.WaitGroup - for _, host := range hosts { - wm.Add(1) - go func(host string) { - defer wm.Done() - logger.Debug("[%s]please wait for copy offline package", host) - SSHConfig.Cmd(host, mkdstdir) - if SSHConfig.IsFilExist(host, fullDst) { - logger.Warn("[%s]SendPackage: [%s] file is exist", host, fullDst) - } else { - if isHttp { - go SSHConfig.LoggerFileSize(host, fullDst, int(filesize.Do(url))) - SSHConfig.Cmd(host, remoteCmd) - rMD5 := SSHConfig.Md5Sum(host, fullDst) //获取已经上传文件的md5 - uMd5 := UrlGetMd5(url) //获取url的md5值 - logger.Debug("[%s] remote file local %s, md5 is %s", host, fullDst, rMD5) - logger.Debug("[%s] url is %s, md5 is %s", host, url, uMd5) - if strings.TrimSpace(rMD5) == strings.TrimSpace(uMd5) { - logger.Info("[%s]file md5 validate success", host) - } else { - logger.Error("[%s]copy file md5 validate failed", host) - } - } else { - if !SSHConfig.CopyForMD5(host, url, fullDst, "") { - logger.Error("[%s]copy file md5 validate failed", host) - } else { - logger.Info("[%s]file md5 validate success", host) - } - } - } - }(host) - } - wm.Wait() -} - // RandString 生成随机字符串 func RandString(len int) string { var r *rand.Rand @@ -272,19 +153,6 @@ func SliceRemoveStr(ss []string, s string) (result []string) { return } -func UrlGetMd5(downloadUrl string) string { - u, err := url.Parse(downloadUrl) - if err == nil { - p := u.Path - if paths := strings.Split(p, "/"); len(paths) > 2 { - if paths = strings.Split(paths[1], "-"); len(paths) > 1 { - return paths[0] - } - } - } - return "" -} - //判断当前host的hostname func isHostName(master, host string) string { hostString := SSHConfig.CmdToString(master, "kubectl get nodes | grep -v NAME | awk '{print $1}'", ",") diff --git a/install/utils_test.go b/install/utils_test.go index 1ea2c64d7..6abdcc8df 100644 --- a/install/utils_test.go +++ b/install/utils_test.go @@ -18,6 +18,13 @@ func TestProcess(t *testing.T) { //bar(100, 1, 0) } +func TestNotReadyNode(t *testing.T) { + SSHConfig.User = "root" + SSHConfig.Password = "PaaS@123" + ss := isHostName("172.27.139.74", "172.27.139.126") + print(ss) +} + func TestPrint(t *testing.T) { //User = "root" //Passwd = "admin" @@ -29,12 +36,7 @@ func TestPrint(t *testing.T) { } func TestVersionToInt(t *testing.T) { - t.Log(ParseIPs([]string{"1.1.1.1-1.1.1.5"})) -} - -func TestUrlGetMd5(t *testing.T) { - aa := UrlGetMd5("https://sealyun.oss-cn-beijing.aliyuncs.com/37374d999dbadb788ef0461844a70151-1.16.0/kube1.16.0.tar.gz") - t.Log(aa) + t.Log(ParseIPs([]string{"172.26.13.133-172.26.13.136:2222"})) } func TestSliceRemoveStr(t *testing.T) { @@ -55,17 +57,17 @@ func TestParseIPs(t *testing.T) { { "test multiple ips", args{[]string{"192.168.0.2-192.168.0.6"}}, - []string{"192.168.0.2","192.168.0.3","192.168.0.4","192.168.0.5","192.168.0.6"}, + []string{"192.168.0.2", "192.168.0.3", "192.168.0.4", "192.168.0.5", "192.168.0.6"}, }, { "test multiple ips", - args{[]string{"192.168.0.2-192.168.0.3","192.168.0.5-192.168.0.6"}}, - []string{"192.168.0.2","192.168.0.3","192.168.0.5","192.168.0.6"}, + args{[]string{"192.168.0.2-192.168.0.3", "192.168.0.5-192.168.0.6"}}, + []string{"192.168.0.2", "192.168.0.3", "192.168.0.5", "192.168.0.6"}, }, { "test multiple ips", - args{[]string{"192.168.0.2-192.168.0.4","192.168.0.8"}}, - []string{"192.168.0.2","192.168.0.3","192.168.0.4","192.168.0.8"}, + args{[]string{"192.168.0.2-192.168.0.4", "192.168.0.8"}}, + []string{"192.168.0.2", "192.168.0.3", "192.168.0.4", "192.168.0.8"}, }, } for _, tt := range tests { @@ -75,4 +77,4 @@ func TestParseIPs(t *testing.T) { } }) } -} \ No newline at end of file +} diff --git a/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go b/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go new file mode 100644 index 000000000..549e1c4eb --- /dev/null +++ b/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go @@ -0,0 +1,35 @@ +package cmd + +import ( + "bytes" + "github.com/wonderivan/logger" + "os" + "os/exec" +) + +//Cmd is exec on os ,no return +func Cmd(name string, arg ...string) { + cmd := exec.Command(name, arg[:]...) + cmd.Stdin = os.Stdin + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + err := cmd.Run() + if err != nil { + logger.Error("调用执行失败。", err) + } +} + +//CmdToString is exec on os , return result +func CmdToString(name string, arg ...string) string { + cmd := exec.Command(name, arg[:]...) + cmd.Stdin = os.Stdin + var b bytes.Buffer + cmd.Stdout = &b + cmd.Stderr = &b + err := cmd.Run() + if err != nil { + logger.Error("调用执行失败。", err) + return "" + } + return b.String() +} diff --git a/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/watch.go b/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/watch.go new file mode 100644 index 000000000..908c5c416 --- /dev/null +++ b/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/watch.go @@ -0,0 +1,58 @@ +package cmd + +import ( + "fmt" + "github.com/wonderivan/logger" + "path" + "strconv" + "strings" + "time" +) + +const oneMBByte = 1024 * 1024 + +//WatchFileSize is +func LoggerFileSize(filename string, size int) { + t := time.NewTicker(3 * time.Second) //every 3s check file + defer t.Stop() + for { + select { + case <-t.C: + length := CmdToString("/bin/sh", "-c", "ls -l "+filename+" | awk '{print $5}'", "") + length = strings.Replace(length, "\n", "", -1) + length = strings.Replace(length, "\r", "", -1) + lengthByte, _ := strconv.Atoi(length) + if lengthByte == size { + t.Stop() + } + lengthFloat := float64(lengthByte) + value, _ := strconv.ParseFloat(fmt.Sprintf("%.2f", lengthFloat/oneMBByte), 64) + logger.Alert("[%s]transfer total size is: %.2f%s", filename, value, "MB") + } + } +} + +//RemoteFilExist is +func IsFilExist(filepath string) bool { + // if remote file is + // ls -l | grep aa | wc -l + fileName := path.Base(filepath) // aa + fileDirName := path.Dir(filepath) + fileCommand := fmt.Sprintf("ls -l %s | grep %s | wc -l", fileDirName, fileName) + data := strings.Replace(CmdToString("/bin/sh", "-c", fileCommand), "\r", "", -1) + data = strings.Replace(data, "\n", "", -1) + count, err := strconv.Atoi(strings.TrimSpace(data)) + defer func() { + if r := recover(); r != nil { + logger.Error("[%s]RemoteFilExist:%s", filepath, err) + } + }() + if err != nil { + panic(1) + } + if count == 0 { + return false + } else { + return true + } +} diff --git a/vendor/github.com/cuisongliu/sshcmd/pkg/md5sum/md5sum.go b/vendor/github.com/cuisongliu/sshcmd/pkg/md5sum/md5sum.go index ae82f5ae3..541cafe77 100644 --- a/vendor/github.com/cuisongliu/sshcmd/pkg/md5sum/md5sum.go +++ b/vendor/github.com/cuisongliu/sshcmd/pkg/md5sum/md5sum.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/wonderivan/logger" "os/exec" + "strings" ) func FromLocal(localPath string) string { @@ -14,5 +15,8 @@ func FromLocal(localPath string) string { logger.Error(err) } md5 := string(out) + md5 = strings.ReplaceAll(md5, "\n", "") + md5 = strings.ReplaceAll(md5, "\r", "") + return md5 } diff --git a/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/connect.go b/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/connect.go index 29d824120..dd257f56d 100644 --- a/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/connect.go +++ b/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/connect.go @@ -19,11 +19,14 @@ func (ss *SSH) Connect(host string) (*ssh.Session, error) { config := ssh.Config{ Ciphers: []string{"aes128-ctr", "aes192-ctr", "aes256-ctr", "aes128-gcm@openssh.com", "arcfour256", "arcfour128", "aes128-cbc", "3des-cbc", "aes192-cbc", "aes256-cbc"}, } - + DefaultTimeout := time.Duration(1) * time.Minute + if ss.Timeout == nil { + ss.Timeout = &DefaultTimeout + } clientConfig := &ssh.ClientConfig{ User: ss.User, Auth: auth, - Timeout: time.Duration(1) * time.Minute, + Timeout: *ss.Timeout, Config: config, HostKeyCallback: func(hostname string, remote net.Addr, key ssh.PublicKey) error { return nil diff --git a/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/types.go b/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/types.go index d0ad82eac..18880a73b 100644 --- a/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/types.go +++ b/vendor/github.com/cuisongliu/sshcmd/pkg/sshutil/types.go @@ -1,7 +1,10 @@ package sshutil +import "time" + type SSH struct { User string Password string PkFile string + Timeout *time.Duration } diff --git a/vendor/modules.txt b/vendor/modules.txt index 578b51980..50082446b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -10,7 +10,8 @@ github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses github.com/aliyun/alibaba-cloud-sdk-go/sdk/utils github.com/aliyun/alibaba-cloud-sdk-go/services/ecs -# github.com/cuisongliu/sshcmd v0.0.0-20200324095132-757019718a50 +# github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4 +github.com/cuisongliu/sshcmd/pkg/cmd github.com/cuisongliu/sshcmd/pkg/filesize github.com/cuisongliu/sshcmd/pkg/md5sum github.com/cuisongliu/sshcmd/pkg/sshutil From 5e62452801b6f25b6d166dfa6d695d7b8004cc25 Mon Sep 17 00:00:00 2001 From: cuisongliu Date: Wed, 25 Mar 2020 01:19:58 +0800 Subject: [PATCH 2/3] feat(develop): fix download feature --- go.mod | 2 +- go.sum | 4 +- install/download.go | 55 ------------------- .../cuisongliu/sshcmd/pkg/cmd/cmd.go | 6 +- vendor/modules.txt | 2 +- 5 files changed, 8 insertions(+), 61 deletions(-) diff --git a/go.mod b/go.mod index c4b3c4bcb..91e996633 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/fanux/sealos go 1.13 require ( - github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4 + github.com/cuisongliu/sshcmd v0.0.0-20200324171807-1615e61773b1 github.com/fanux/sealgate v0.0.5 github.com/mitchellh/go-homedir v1.1.0 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 7c3163c25..05f5f6ad3 100644 --- a/go.sum +++ b/go.sum @@ -30,8 +30,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= -github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4 h1:uUJMoSd1bMQOISCE0T1N6ig0qFH3f1vLvkRsnyq4Wuc= -github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4/go.mod h1:jAEBZhFW5kNey9B+59lR8mfQaCQsyp8b8QaI7RhESxE= +github.com/cuisongliu/sshcmd v0.0.0-20200324171807-1615e61773b1 h1:VhGG7tHUJ8sud9OgypR5VmiUrsWB+kq7h/ivOf1Tfyg= +github.com/cuisongliu/sshcmd v0.0.0-20200324171807-1615e61773b1/go.mod h1:jAEBZhFW5kNey9B+59lR8mfQaCQsyp8b8QaI7RhESxE= github.com/davecgh/go-spew v0.0.0-20151105211317-5215b55f46b2/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= diff --git a/install/download.go b/install/download.go index 139375877..2a633b365 100644 --- a/install/download.go +++ b/install/download.go @@ -8,7 +8,6 @@ import ( "github.com/wonderivan/logger" "net/url" "path" - "strings" "sync" ) @@ -46,60 +45,6 @@ func SendPackage(location string, hosts []string, dst, hook string) { wm.Wait() } -// FetchPackage if url exist wget it, or scp the local package to hosts -// dst is the remote offline path like /root -func FetchPackage(url string, hosts []string, dst string) { - pkg := path.Base(url) - fullDst := fmt.Sprintf("%s/%s", dst, pkg) - mkdstdir := fmt.Sprintf("mkdir -p %s || true", dst) - - //only http - isHttp := strings.HasPrefix(url, "http") - wgetCommand := "" - if isHttp { - wgetParam := "" - if strings.HasPrefix(url, "https") { - wgetParam = "--no-check-certificate" - } - wgetCommand = fmt.Sprintf(" wget %s ", wgetParam) - } - remoteCmd := fmt.Sprintf("cd %s && %s %s", dst, wgetCommand, url) - - var wm sync.WaitGroup - for _, host := range hosts { - wm.Add(1) - go func(host string) { - defer wm.Done() - logger.Debug("[%s]please wait for copy offline package", host) - SSHConfig.Cmd(host, mkdstdir) - if SSHConfig.IsFilExist(host, fullDst) { - logger.Warn("[%s]SendPackage: [%s] file is exist", host, fullDst) - } else { - if isHttp { - go SSHConfig.LoggerFileSize(host, fullDst, int(filesize.Do(url))) - SSHConfig.Cmd(host, remoteCmd) - rMD5 := SSHConfig.Md5Sum(host, fullDst) //获取已经上传文件的md5 - uMd5 := "urlGetMD5(url) " //获取url的md5值 - logger.Debug("[%s] remote file local %s, md5 is %s", host, fullDst, rMD5) - logger.Debug("[%s] url is %s, md5 is %s", host, url, uMd5) - if strings.TrimSpace(rMD5) == strings.TrimSpace(uMd5) { - logger.Info("[%s]file md5 validate success", host) - } else { - logger.Error("[%s]copy file md5 validate failed", host) - } - } else { - if !SSHConfig.CopyForMD5(host, url, fullDst, "") { - logger.Error("[%s]copy file md5 validate failed", host) - } else { - logger.Info("[%s]file md5 validate success", host) - } - } - } - }(host) - } - wm.Wait() -} - // func downloadFile(location string) (filePATH, md5 string) { if _, isUrl := isUrl(location); isUrl { diff --git a/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go b/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go index 549e1c4eb..1efad75c1 100644 --- a/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go +++ b/vendor/github.com/cuisongliu/sshcmd/pkg/cmd/cmd.go @@ -9,18 +9,20 @@ import ( //Cmd is exec on os ,no return func Cmd(name string, arg ...string) { + logger.Info("[os]exec cmd is : ", name, arg) cmd := exec.Command(name, arg[:]...) cmd.Stdin = os.Stdin cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout err := cmd.Run() if err != nil { - logger.Error("调用执行失败。", err) + logger.Error("os call error.", err) } } //CmdToString is exec on os , return result func CmdToString(name string, arg ...string) string { + logger.Info("[os]exec cmd is : ", name, arg) cmd := exec.Command(name, arg[:]...) cmd.Stdin = os.Stdin var b bytes.Buffer @@ -28,7 +30,7 @@ func CmdToString(name string, arg ...string) string { cmd.Stderr = &b err := cmd.Run() if err != nil { - logger.Error("调用执行失败。", err) + logger.Error("os call error.", err) return "" } return b.String() diff --git a/vendor/modules.txt b/vendor/modules.txt index 50082446b..901e53cbb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -10,7 +10,7 @@ github.com/aliyun/alibaba-cloud-sdk-go/sdk/requests github.com/aliyun/alibaba-cloud-sdk-go/sdk/responses github.com/aliyun/alibaba-cloud-sdk-go/sdk/utils github.com/aliyun/alibaba-cloud-sdk-go/services/ecs -# github.com/cuisongliu/sshcmd v0.0.0-20200324165749-69d4a2b195b4 +# github.com/cuisongliu/sshcmd v0.0.0-20200324171807-1615e61773b1 github.com/cuisongliu/sshcmd/pkg/cmd github.com/cuisongliu/sshcmd/pkg/filesize github.com/cuisongliu/sshcmd/pkg/md5sum From f164723f523777be1b5db5374a43e14d9ff72067 Mon Sep 17 00:00:00 2001 From: cuisongliu Date: Wed, 25 Mar 2020 21:30:52 +0800 Subject: [PATCH 3/3] feat(develop): fix download feature --- cmd/init.go | 2 +- install/download.go | 1 + install/install.go | 26 +------------------------- 3 files changed, 3 insertions(+), 26 deletions(-) diff --git a/cmd/init.go b/cmd/init.go index f88835e65..67f895c79 100644 --- a/cmd/init.go +++ b/cmd/init.go @@ -26,8 +26,8 @@ var initCmd = &cobra.Command{ Long: `sealos init --master 192.168.0.2 --master 192.168.0.3 --master 192.168.0.4 --node 192.168.0.5 --user root --passwd your-server-password`, Run: func(cmd *cobra.Command, args []string) { c := &install.SealConfig{} - c.Dump("") install.BuildInit() + c.Dump("") }, } diff --git a/install/download.go b/install/download.go index 2a633b365..f27406113 100644 --- a/install/download.go +++ b/install/download.go @@ -17,6 +17,7 @@ import ( //hook: cd /root && rm -rf kube && tar zxvf %s && cd /root/kube/shell && sh init.sh func SendPackage(location string, hosts []string, dst, hook string) { location, md5 := downloadFile(location) + PkgUrl = location pkg := path.Base(location) fullPath := fmt.Sprintf("%s/%s", dst, pkg) mkDstDir := fmt.Sprintf("mkdir -p %s || true", dst) diff --git a/install/install.go b/install/install.go index 67cc2549f..a47fa82d3 100644 --- a/install/install.go +++ b/install/install.go @@ -7,7 +7,6 @@ import ( "github.com/wonderivan/logger" "io" "os" - "os/exec" "path" "strings" "sync" @@ -52,29 +51,6 @@ func AppInstall(url string) { Exec(pkgConfig, *c) } -func LoadRemoteFile(url string) string { - isHttp := strings.HasPrefix(url, "http") - if !isHttp { - logger.Info("using local package %s", url) - return url - } - logger.Info("wait for wget app package...") - wgetParam := "" - if strings.HasPrefix(url, "https") { - wgetParam = "--no-check-certificate" - } - wgetCommand := fmt.Sprintf(" wget %s ", wgetParam) - cmd := fmt.Sprintf("%s %s", wgetCommand, url) - c := exec.Command("sh", "-c", cmd) - out, err := c.CombinedOutput() - if err != nil { - logger.Error(err) - } - logger.Info("%s", out) - - return path.Base(url) -} - // LoadConfig from tar package /* kube.tar @@ -90,7 +66,7 @@ STOP systemctl top APPLY kubectl apply -f */ func LoadConfig(packageFile string) (*PkgConfig, error) { - filename := LoadRemoteFile(packageFile) + filename, _ := downloadFile(packageFile) file, err := os.Open(filename) if err != nil {