From 58e81bec86d0aa3aba83cf9f750c3f7a37b899a8 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Tue, 25 Dec 2018 11:53:52 +0800 Subject: [PATCH] move util to pkg/utils --- pkg/cloudcommon/fstabutils/doc.go | 1 - pkg/cloudcommon/qemuimg/qemuimg.go | 422 ------------------ pkg/cloudcommon/qemutils/doc.go | 1 - pkg/cloudcommon/qemutils/qemutils.go | 111 ----- pkg/cloudcommon/version/doc.go | 1 - pkg/cloudcommon/version/version.go | 46 -- pkg/hostman/client.go | 13 +- pkg/hostman/guestfs/guestfs.go | 6 +- pkg/hostman/guestfs/linux.go | 30 +- pkg/hostman/guestfs/localfs.go | 9 +- pkg/hostman/guestman/guestman.go | 14 +- pkg/hostman/guestman/guesttasks.go | 4 +- pkg/hostman/guestman/qemu-kvm.go | 15 +- pkg/hostman/hostinfo/host_cpu.go | 6 +- pkg/hostman/hostinfo/hostinfo.go | 12 +- pkg/hostman/storageman/diskutils.go | 2 +- pkg/hostman/storageman/nbdman.go | 5 +- pkg/mcclient/modules/mod_tasks.go | 6 +- pkg/util/fileutils2/cheksum.go | 33 ++ .../utils.go => util/fileutils2/fileutils.go} | 155 +------ .../fileutils2/fileutils_test.go} | 2 +- .../fstabutils/fstabutils.go | 0 pkg/util/netutils2/netutils.go | 119 +++++ pkg/util/qemuimg/consts.go | 9 + pkg/util/qemuimg/init.go | 9 + pkg/util/qemuimg/qemuimg_test.go | 3 + .../qemutils/qemutils_test.go | 0 pkg/util/timeutils2/timeutils.go | 21 + .../version/version_test.go | 0 29 files changed, 269 insertions(+), 786 deletions(-) delete mode 100644 pkg/cloudcommon/fstabutils/doc.go delete mode 100644 pkg/cloudcommon/qemuimg/qemuimg.go delete mode 100644 pkg/cloudcommon/qemutils/doc.go delete mode 100644 pkg/cloudcommon/qemutils/qemutils.go delete mode 100644 pkg/cloudcommon/version/doc.go delete mode 100644 pkg/cloudcommon/version/version.go rename pkg/{cloudcommon/utils.go => util/fileutils2/fileutils.go} (59%) rename pkg/{cloudcommon/utils_test.go => util/fileutils2/fileutils_test.go} (96%) rename pkg/{cloudcommon => util}/fstabutils/fstabutils.go (100%) create mode 100644 pkg/util/netutils2/netutils.go rename pkg/{cloudcommon => util}/qemutils/qemutils_test.go (100%) create mode 100644 pkg/util/timeutils2/timeutils.go rename pkg/{cloudcommon => util}/version/version_test.go (100%) diff --git a/pkg/cloudcommon/fstabutils/doc.go b/pkg/cloudcommon/fstabutils/doc.go deleted file mode 100644 index 2b06110b68..0000000000 --- a/pkg/cloudcommon/fstabutils/doc.go +++ /dev/null @@ -1 +0,0 @@ -package fstabutils // import "yunion.io/x/onecloud/pkg/cloudcommon/fstabutils" diff --git a/pkg/cloudcommon/qemuimg/qemuimg.go b/pkg/cloudcommon/qemuimg/qemuimg.go deleted file mode 100644 index 5316b31b29..0000000000 --- a/pkg/cloudcommon/qemuimg/qemuimg.go +++ /dev/null @@ -1,422 +0,0 @@ -package qemuimg - -import ( - "bytes" - "errors" - "fmt" - "io" - "os" - "os/exec" - "strconv" - "strings" - - "yunion.io/x/log" - "yunion.io/x/pkg/utils" - - "yunion.io/x/onecloud/pkg/cloudcommon/qemutils" - "yunion.io/x/onecloud/pkg/cloudprovider" - "yunion.io/x/onecloud/pkg/compute/models" - "yunion.io/x/onecloud/pkg/util/fileutils2" -) - -var ( - ErrUnsupportedFormat = errors.New("unsupported format") -) - -type SQemuImage struct { - Path string - Password string - Format TImageFormat - SizeBytes int64 - ActualSizeBytes int64 - ClusterSize int - BackFilePath string - Compat string - Encryption bool - Subformat string -} - -func NewQemuImage(path string) (*SQemuImage, error) { - return NewEncryptedQemuImage(path, "") -} - -func NewEncryptedQemuImage(path string, password string) (*SQemuImage, error) { - qemuImg := SQemuImage{Path: path, Password: password} - err := qemuImg.parse() - if err != nil { - return nil, err - } - return &qemuImg, nil -} - -func (img *SQemuImage) parse() error { - if len(img.Path) == 0 { - return fmt.Errorf("empty image path") - } - if strings.HasPrefix(img.Path, "nbd") { - // nbd TCP -> nbd:: - // nbd Unix Domain Sockets -> nbd:unix: - img.ActualSizeBytes = 0 - } else if strings.HasPrefix(img.Path, "iscsi") { - // iSCSI LUN -> iscsi://[:]// - return cloudprovider.ErrNotImplemented - } else if strings.HasPrefix(img.Path, "sheepdog") { - // sheepdog -> sheepdog[+tcp|+unix]://[host:port]/vdiname[?socket=path][#snapid|#tag] - return cloudprovider.ErrNotImplemented - } else if strings.HasPrefix(img.Path, models.STORAGE_RBD) { - img.ActualSizeBytes = 0 - } else { - fileInfo, err := os.Stat(img.Path) - if err != nil { - if !os.IsNotExist(err) { - return err - } else { - return nil - } - } else { - img.ActualSizeBytes = fileInfo.Size() - } - } - cmd := exec.Command(qemutils.GetQemuImg(), "info", img.Path) - if len(img.Password) > 0 { - cmd.Stdin = bytes.NewBuffer([]byte(img.Password)) - } - var out bytes.Buffer - cmd.Stdout = &out - err := cmd.Run() - if err != nil { - log.Errorf("qemu-img info %s fail %s", img.Path, err) - return fmt.Errorf("qemu-img info error %s", err) - } - for { - line, err := out.ReadString('\n') - if len(line) > 0 { - line = strings.TrimSpace(line) - switch { - case strings.HasPrefix(line, "file format:"): - img.Format = TImageFormat(line[strings.LastIndexByte(line, ' ')+1:]) - case strings.HasPrefix(line, "virtual size:"): - if img.SizeBytes == 0 { - sizeStr := line[strings.LastIndexByte(line, '(')+1 : strings.LastIndexByte(line, ' ')] - size, err := strconv.ParseInt(sizeStr, 10, 64) - if err != nil { - return fmt.Errorf("invalid size str %s: %s", sizeStr, err) - } - img.SizeBytes = size - } - case strings.HasPrefix(line, "cluster_size:"): - sizeStr := line[strings.LastIndexByte(line, ' ')+1:] - size, err := strconv.ParseInt(sizeStr, 10, 64) - if err != nil { - return fmt.Errorf("invalid cluster size str %s", sizeStr) - } - img.ClusterSize = int(size) - case strings.HasPrefix(line, "backing file:"): - img.BackFilePath = line[strings.LastIndexByte(line, ' ')+1:] - case strings.HasPrefix(line, "compat:"): - img.Compat = line[strings.LastIndexByte(line, ' ')+1:] - case strings.HasPrefix(line, "encrypted:"): - if line[strings.LastIndexByte(line, ' ')+1:] == "yes" { - img.Encryption = true - } - case strings.HasPrefix(line, "create type:"): - img.Subformat = line[strings.LastIndexByte(line, ' ')+1:] - } - } - if err != nil { - if err == io.EOF { - break - } else { - log.Errorf("read output fail %s", err) - return fmt.Errorf("read output fail %s", err) - } - } - } - if img.Format == RAW { - // test if it is an ISO - blkType := fileutils2.GetBlkidType(img.Path) - if blkType == "iso9660" { - img.Format = ISO - } - } - return nil -} - -func (img *SQemuImage) IsValid() bool { - return len(img.Format) > 0 -} - -func (img *SQemuImage) IsChained() bool { - return len(img.BackFilePath) > 0 -} - -func (img *SQemuImage) doConvert(name string, format TImageFormat, options []string, compact bool, password string) error { - if !img.IsValid() { - return fmt.Errorf("self is not valid") - } - cmdline := []string{"convert"} - if compact { - cmdline = append(cmdline, "-c") - } - cmdline = append(cmdline, "-f", img.Format.String(), "-O", format.String()) - if len(password) > 0 { - if options == nil { - options = make([]string, 0) - } - options = append(options, "encryption=on") - } - if len(options) > 0 { - cmdline = append(cmdline, "-o", strings.Join(options, ",")) - } - cmdline = append(cmdline, img.Path, name) - cmd := exec.Command(qemutils.GetQemuImg(), cmdline...) - if len(img.Password) > 0 || len(password) > 0 { - input := "" - if len(img.Password) > 0 { - input = fmt.Sprintf("%s%s\r", input, img.Password) - } - if len(password) > 0 { - input = fmt.Sprintf("%s%s\r", input, password) - } - cmd.Stdin = bytes.NewBuffer([]byte(input)) - } - err := cmd.Run() - if err != nil { - log.Errorf("clone fail %s", err) - os.Remove(name) - return err - } - return nil -} - -func (img *SQemuImage) Clone(name string, format TImageFormat, compact bool) (*SQemuImage, error) { - switch format { - case QCOW2: - return img.CloneQcow2(name, compact) - case VMDK: - return img.CloneVmdk(name, compact) - case RAW: - return img.CloneRaw(name) - default: - return nil, ErrUnsupportedFormat - } -} - -func (img *SQemuImage) clone(name string, format TImageFormat, options []string, compact bool, password string) (*SQemuImage, error) { - err := img.doConvert(name, format, options, compact, password) - if err != nil { - return nil, err - } - return NewQemuImage(name) -} - -func (img *SQemuImage) convert(format TImageFormat, options []string, compact bool, password string) error { - tmpPath := fmt.Sprintf("%s.%s", img.Path, utils.GenRequestId(36)) - err := img.doConvert(tmpPath, format, options, compact, password) - if err != nil { - return err - } - cmd := exec.Command("mv", "-f", tmpPath, img.Path) - err = cmd.Run() - if err != nil { - log.Errorf("convert move temp file error %s", err) - os.Remove(tmpPath) - return err - } - img.Password = password - return img.parse() -} - -func (img *SQemuImage) Copy(name string) (*SQemuImage, error) { - if !img.IsValid() { - return nil, fmt.Errorf("self is not valid") - } - cmd := exec.Command("cp", "--sparse=always", img.Path, name) - err := cmd.Run() - if err != nil { - log.Errorf("copy fail %s", err) - os.Remove(name) - return nil, err - } - return NewQemuImage(name) -} - -func (img *SQemuImage) Convert2Qcow2(compact bool) error { - options := make([]string, 0) - // if len(backPath) > 0 { - // options = append(options, fmt.Sprintf("backing_file=%s", backPath)) - //} else - if !compact { - sparseOpts := qcow2SparseOptions() - options = append(options, sparseOpts...) - } - return img.convert(QCOW2, options, compact, "") -} - -func (img *SQemuImage) Convert2Vmdk(compact bool) error { - return img.convert(VMDK, vmdkOptions(compact), false, "") -} - -func (img *SQemuImage) Convert2Raw() error { - return img.convert(RAW, nil, false, "") -} - -func (img *SQemuImage) IsRaw() bool { - return img.Format == RAW -} - -func (img *SQemuImage) IsSparseQcow2() bool { - return img.Format == QCOW2 && img.ClusterSize >= 1024*1024*2 -} - -func (img *SQemuImage) IsSparseVmdk() bool { - return img.Format == VMDK && img.Subformat != "streamOptimized" -} - -func (img *SQemuImage) IsSparse() bool { - return img.IsRaw() || img.IsSparseQcow2() || img.IsSparseVmdk() -} - -func (img *SQemuImage) Expand() error { - if img.IsSparse() { - return nil - } - return img.Convert2Qcow2(false) -} - -func (img *SQemuImage) CloneQcow2(name string, compact bool) (*SQemuImage, error) { - options := make([]string, 0) - //if len(backPath) > 0 { - // options = append(options, fmt.Sprintf("backing_file=%s", backPath)) - //} else - if !compact { - sparseOpts := qcow2SparseOptions() - options = append(options, sparseOpts...) - } - return img.clone(name, QCOW2, options, compact, "") -} - -func vmdkOptions(compact bool) []string { - if compact { - return []string{"subformat=streamOptimized"} - } else { - return []string{"subformat=monolithicSparse"} - } -} - -func (img *SQemuImage) CloneVmdk(name string, compact bool) (*SQemuImage, error) { - return img.clone(name, VMDK, vmdkOptions(compact), compact, "") -} - -func (img *SQemuImage) CloneRaw(name string) (*SQemuImage, error) { - return img.clone(name, RAW, nil, false, "") -} - -func (img *SQemuImage) create(sizeMB int, format TImageFormat, options []string) error { - if img.IsValid() { - return fmt.Errorf("create: the image is valid??? %s", img.Format) - } - args := []string{"create", "-f", format.String()} - if len(options) > 0 { - args = append(args, "-o", strings.Join(options, ",")) - } - args = append(args, img.Path) - if sizeMB > 0 { - args = append(args, fmt.Sprintf("%dM", sizeMB)) - } - cmd := exec.Command(qemutils.GetQemuImg(), args...) - err := cmd.Run() - if err != nil { - log.Errorf("create error %s", err) - return err - } - return img.parse() -} - -func (img *SQemuImage) CreateQcow2(sizeMB int, compact bool, backPath string) error { - options := make([]string, 0) - if len(backPath) > 0 { - options = append(options, fmt.Sprintf("backing_file=%s", backPath)) - if !compact { - options = append(options, "cluster_size=2M") - } - } else if !compact { - sparseOpts := qcow2SparseOptions() - options = append(options, sparseOpts...) - } - return img.create(sizeMB, QCOW2, options) -} - -func (img *SQemuImage) CreateVmdk(sizeMB int, compact bool) error { - return img.create(sizeMB, VMDK, vmdkOptions(compact)) -} - -func (img *SQemuImage) CreateRaw(sizeMB int) error { - return img.create(sizeMB, RAW, nil) -} - -func (img *SQemuImage) GetSizeMB() int { - return int(img.SizeBytes / 1024 / 1024) -} - -func (img *SQemuImage) GetActualSizeMB() int { - return int(img.ActualSizeBytes / 1024 / 1024) -} - -func (img *SQemuImage) Resize(sizeMB int) error { - if !img.IsValid() { - return fmt.Errorf("self is not valid") - } - cmd := exec.Command(qemutils.GetQemuImg(), "resize", img.Path, fmt.Sprintf("%dM", sizeMB)) - err := cmd.Run() - if err != nil { - log.Errorf("resize fail %s", err) - return err - } - return img.parse() -} - -func (img *SQemuImage) Rebase(backPath string, force bool) error { - if !img.IsValid() { - return fmt.Errorf("self is not valid") - } - args := []string{"rebase"} - if force { - args = append(args, "-u") - } - args = append(args, "-b", backPath, img.Path) - cmd := exec.Command(qemutils.GetQemuImg(), args...) - err := cmd.Run() - if err != nil { - log.Errorf("rebase fail %s", err) - return err - } - return img.parse() -} - -func (img *SQemuImage) Delete() error { - if !img.IsValid() { - return nil - } - err := os.Remove(img.Path) - if err != nil { - log.Errorf("delete fail %s", err) - return err - } - img.Format = "" - img.ActualSizeBytes = 0 - img.SizeBytes = 0 - return nil -} - -func (img *SQemuImage) Fallocate() error { - if !img.IsValid() { - return fmt.Errorf("self is not valid") - } - cmd := exec.Command("fallocate", "-l", fmt.Sprintf("%dm", img.GetSizeMB()), img.Path) - return cmd.Run() -} - -func (img *SQemuImage) String() string { - return fmt.Sprintf("Qemu %s %d(%d) %s", img.Format, img.GetSizeMB(), img.GetActualSizeMB(), img.Path) -} diff --git a/pkg/cloudcommon/qemutils/doc.go b/pkg/cloudcommon/qemutils/doc.go deleted file mode 100644 index 7f89fc9866..0000000000 --- a/pkg/cloudcommon/qemutils/doc.go +++ /dev/null @@ -1 +0,0 @@ -package qemutils // import "yunion.io/x/onecloud/pkg/cloudcommon/qemutils" diff --git a/pkg/cloudcommon/qemutils/qemutils.go b/pkg/cloudcommon/qemutils/qemutils.go deleted file mode 100644 index 496e3be1d9..0000000000 --- a/pkg/cloudcommon/qemutils/qemutils.go +++ /dev/null @@ -1,111 +0,0 @@ -package qemutils - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "regexp" - "sort" - "strings" - - "yunion.io/x/onecloud/pkg/cloudcommon/version" -) - -const ( - USER_LOCAL_BIN = "/usr/local/bin" - USER_BIN = "/usr/bin" -) - -func GetQemu(version string) string { - return getQemuCmd("qemu-system-x86_64", version) -} - -func GetQemuNbd() string { - return getQemuCmd("qemu-nbd", "") -} - -func GetQemuImg() string { - return getQemuCmd("qemu-img", "") -} - -func getQemuCmd(cmd, version string) string { - if len(version) > 0 { - return getQemuCmdByVersion(cmd, version) - } else { - return getQemuDefaultCmd(cmd) - } -} - -func getQemuCmdByVersion(cmd, version string) string { - p := path.Join(fmt.Sprintf("/usr/local/qemu-%s/bin", version), cmd) - if _, err := os.Stat(p); !os.IsNotExist(err) { - return p - } - cmd = cmd + "_" + version - p = path.Join(USER_LOCAL_BIN, cmd) - if _, err := os.Stat(p); !os.IsNotExist(err) { - return p - } - p = path.Join(USER_BIN, cmd) - if _, err := os.Stat(p); !os.IsNotExist(err) { - return p - } - return "" -} - -func getQemuVersion(verString string) string { - s := regexp.MustCompile(`qemu-(?P\d+(\.\d+)+)$`).FindString(verString) - if len(s) == 0 { - return "" - } - return s[len("qemu-"):] -} - -func getCmdVersion(cmd string) string { - s := regexp.MustCompile(`_(?P\d+(\.\d+)+)$`).FindString(cmd) - if len(s) == 0 { - return "" - } - return s[1:] -} - -func getQemuDefaultCmd(cmd string) string { - var qemus = make([]string, 0) - if files, err := ioutil.ReadDir("/usr/local"); err == nil { - for i := 0; i < len(files); i++ { - if strings.HasPrefix(files[i].Name(), "qemu-") { - qemus = append(qemus, files[i].Name()) - } - } - if len(qemus) > 0 { - sort.Slice(qemus, func(i, j int) bool { - return version.LT(getQemuVersion(qemus[i]), - getQemuVersion(qemus[j])) - }) - p := fmt.Sprintf("/usr/local/%s/bin/%s", qemus[len(qemus)-1], cmd) - if _, err := os.Stat(p); !os.IsNotExist(err) { - return p - } - } - } - cmds := make([]string, 0) - if files, err := ioutil.ReadDir(USER_LOCAL_BIN); err == nil { - for i := 0; i < len(files); i++ { - if strings.HasPrefix(files[i].Name(), cmd) { - cmds = append(cmds, files[i].Name()) - } - } - if len(cmds) > 0 { - sort.Slice(cmds, func(i, j int) bool { - return version.LT(getCmdVersion(cmds[i]), - getCmdVersion(cmds[j])) - }) - p := path.Join(USER_LOCAL_BIN, cmds[len(cmds)-1]) - if _, err := os.Stat(p); !os.IsNotExist(err) { - return p - } - } - } - return "" -} diff --git a/pkg/cloudcommon/version/doc.go b/pkg/cloudcommon/version/doc.go deleted file mode 100644 index 315032e76c..0000000000 --- a/pkg/cloudcommon/version/doc.go +++ /dev/null @@ -1 +0,0 @@ -package version // import "yunion.io/x/onecloud/pkg/cloudcommon/version" diff --git a/pkg/cloudcommon/version/version.go b/pkg/cloudcommon/version/version.go deleted file mode 100644 index d42061b73a..0000000000 --- a/pkg/cloudcommon/version/version.go +++ /dev/null @@ -1,46 +0,0 @@ -package version - -import ( - "strconv" - "strings" -) - -func less(v1Str, v2Str string) (bool, bool) { - v1 := strings.Split(v1Str, ".") - v2 := strings.Split(v2Str, ".") - var i = 0 - for ; i < len(v2); i++ { - if i >= len(v1) { - return true, false - } - v, _ := strconv.ParseInt(v2[i], 10, 0) - compareV, _ := strconv.ParseInt(v1[i], 10, 0) - if v < compareV { - return false, false - } else if compareV < v { - return true, false - } - } - if i < len(v1)-1 { - return false, false - } - return true, true -} - -func LE(v1Str, v2Str string) bool { - l, _ := less(v1Str, v2Str) - return l -} - -func LT(v1Str, v2Str string) bool { - l, e := less(v1Str, v2Str) - return l && !e -} - -func GT(v1Str, v2Str string) bool { - return LT(v2Str, v1Str) -} - -func GE(v1Str, v2Str string) bool { - return LE(v2Str, v1Str) -} diff --git a/pkg/hostman/client.go b/pkg/hostman/client.go index 579d0d675a..6531bf3359 100644 --- a/pkg/hostman/client.go +++ b/pkg/hostman/client.go @@ -5,6 +5,7 @@ import ( "fmt" "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/appctx" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/mcclient" @@ -12,19 +13,13 @@ import ( "yunion.io/x/onecloud/pkg/mcclient/modules" ) -var session *mcclient.ClientSession - -func GetComputeSession() *mcclient.ClientSession { - return session -} - -func init() { - session = auth.GetAdminSession(options.HostOptions.Region, "v2") +func GetComputeSession(ctx context.Context) *mcclient.ClientSession { + return auth.GetAdminSession(ctx, options.HostOptions.Region, "v2") } func TaskFailed(ctx context.Context, reason string) error { if taskId := ctx.Value(appctx.APP_CONTEXT_KEY_TASK_ID); taskId != nil { - modules.ComputeTasks.TaskFailed(ctx, taskId.(string), reason) + modules.ComputeTasks.TaskFailed2(GetComputeSession(ctx), taskId.(string), reason) return nil } else { log.Errorln("Reqeuest task failed missing task id, with reason(%s)", reason) diff --git a/pkg/hostman/guestfs/guestfs.go b/pkg/hostman/guestfs/guestfs.go index 70ceb2a4c0..c0f2f44d5c 100644 --- a/pkg/hostman/guestfs/guestfs.go +++ b/pkg/hostman/guestfs/guestfs.go @@ -8,7 +8,7 @@ import ( "time" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/pkg/utils" ) @@ -24,14 +24,14 @@ func NewKVMGuestDiskPartition(devPath string) *SKVMGuestDiskPartition { var res = new(SKVMGuestDiskPartition) res.partDev = devPath res.fs = res.getFsFormat() - cloudcommon.CleanFailedMountpoints() + fileutils2.CleanFailedMountpoints() mountPath := fmt.Sprintf("/tmp/%s", strings.Replace(devPath, "/", "_", -1)) res.SLocalGuestFS = NewLocalGuestFS(mountPath) return res } func (p *SKVMGuestDiskPartition) getFsFormat() string { - return cloudcommon.GetFsFormat(p.partDev) + return fileutils2.GetFsFormat(p.partDev) } func (p *SKVMGuestDiskPartition) Mount() bool { diff --git a/pkg/hostman/guestfs/linux.go b/pkg/hostman/guestfs/linux.go index 8297f53b70..057c6c0e81 100644 --- a/pkg/hostman/guestfs/linux.go +++ b/pkg/hostman/guestfs/linux.go @@ -10,13 +10,15 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" - "yunion.io/x/onecloud/pkg/cloudcommon/fstabutils" + "yunion.io/x/pkg/utils" + "yunion.io/x/onecloud/pkg/cloudcommon/sshkeys" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/fstabutils" + "yunion.io/x/onecloud/pkg/util/netutils2" "yunion.io/x/onecloud/pkg/util/seclib2" - "yunion.io/x/pkg/utils" ) type SLinuxRootFs struct { @@ -44,7 +46,7 @@ func (l *SLinuxRootFs) DeployHost(hn, domain string, ips []string) error { } oldHostFile = string(oldhf) } - hf := make(cloudcommon.HostsFile, 0) + hf := make(fileutils2.HostsFile, 0) hf.Parse(oldHostFile) hf.Add("127.0.0.1", "localhost") for _, ip := range ips { @@ -355,7 +357,7 @@ func (d *SDebianLikeRootFs) DeployNetworkingScripts(nics []jsonutils.JSONObject) cmds := "" cmds += "auto lo\n" cmds += "iface lo inet loopback\n\n" - mainNic, err := cloudcommon.GetMainNic(nics) + mainNic, err := netutils2.GetMainNic(nics) if err != nil { return err } @@ -376,11 +378,11 @@ func (d *SDebianLikeRootFs) DeployNetworkingScripts(nics []jsonutils.JSONObject) cmds += fmt.Sprintf("auto eth%d\n", nicIdx) if jsonutils.QueryBoolean(nic, "virtual", false) { cmds += fmt.Sprintf("iface eth%d inet static\n", nicIdx) - cmds += fmt.Sprintf(" address %s\n", cloudcommon.PSEUDO_VIP) + cmds += fmt.Sprintf(" address %s\n", netutils2.PSEUDO_VIP) cmds += " netmask 255.255.255.255\n" cmds += "\n" } else if jsonutils.QueryBoolean(nic, "manual", false) { - netmask := cloudcommon.Netlen2Mask(nicDesc.Masklen) + netmask := netutils2.Netlen2Mask(nicDesc.Masklen) ip, err := nic.GetString("ip") if err != nil { return err @@ -392,12 +394,12 @@ func (d *SDebianLikeRootFs) DeployNetworkingScripts(nics []jsonutils.JSONObject) cmds += fmt.Sprintf(" gateway %s\n", nicDesc.Gateway) } var routes = make([][]string, 0) - cloudcommon.AddNicRoutes(&routes, nicDesc, mainIp, len(nics), options.HostOptions.PrivatePrefixes) + netutils2.AddNicRoutes(&routes, nicDesc, mainIp, len(nics), options.HostOptions.PrivatePrefixes) for _, r := range routes { cmds += fmt.Sprintf(" up route add -net %s gw %s || true\n", r[0], r[1]) cmds += fmt.Sprintf(" down route del -net %s gw %s || true\n", r[0], r[1]) } - dnslist := cloudcommon.GetNicDns(nicDesc) + dnslist := netutils2.GetNicDns(nicDesc) if len(dnslist) > 0 { cmds += fmt.Sprintf(" dns-nameservers %s\n", strings.Join(dnslist, " ")) cmds += fmt.Sprintf(" dns-search %s\n", nicDesc.Domain) @@ -514,7 +516,7 @@ func (r *SRedhatLikeRootFs) DeployNetworkingScripts(nics []jsonutils.JSONObject) if err != nil { return err } - mainNic, err := cloudcommon.GetMainNic(nics) + mainNic, err := netutils2.GetMainNic(nics) if err != nil { return err } @@ -535,10 +537,10 @@ func (r *SRedhatLikeRootFs) DeployNetworkingScripts(nics []jsonutils.JSONObject) if nicdesc.Virtual { cmds += "BOOTPROTO=none\n" cmds += "NETMASK=255.255.255.255\n" - cmds += fmt.Sprintf("IPADDR=%s\n", cloudcommon.PSEUDO_VIP) + cmds += fmt.Sprintf("IPADDR=%s\n", netutils2.PSEUDO_VIP) cmds += "USERCTL=no\n" } else if nicdesc.Manual { - netmask := cloudcommon.Netlen2Mask(nicdesc.Masklen) + netmask := netutils2.Netlen2Mask(nicdesc.Masklen) cmds += "BOOTPROTO=none\n" cmds += fmt.Sprintf("NETMASK=%s\n", netmask) cmds += fmt.Sprintf("IPADDR=%s\n", nicdesc.Ip) @@ -548,7 +550,7 @@ func (r *SRedhatLikeRootFs) DeployNetworkingScripts(nics []jsonutils.JSONObject) } var routes = make([][]string, 0) var rtbl string - cloudcommon.AddNicRoutes(&routes, nicdesc, mainIp, len(nics), options.HostOptions.PrivatePrefixes) + netutils2.AddNicRoutes(&routes, nicdesc, mainIp, len(nics), options.HostOptions.PrivatePrefixes) for _, r := range routes { rtbl += fmt.Sprintf("%s via %s dev eth%d\n", r[0], r[1], nicdesc.Index) } @@ -558,7 +560,7 @@ func (r *SRedhatLikeRootFs) DeployNetworkingScripts(nics []jsonutils.JSONObject) return err } } - dnslist := cloudcommon.GetNicDns(nicdesc) + dnslist := netutils2.GetNicDns(nicdesc) if len(dnslist) > 0 { cmds += "PEERDNS=yes\n" for i := 0; i < len(dnslist); i++ { diff --git a/pkg/hostman/guestfs/localfs.go b/pkg/hostman/guestfs/localfs.go index d080d6576d..0a5439d228 100644 --- a/pkg/hostman/guestfs/localfs.go +++ b/pkg/hostman/guestfs/localfs.go @@ -12,8 +12,9 @@ import ( "syscall" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/cloudcommon/sshkeys" + "yunion.io/x/onecloud/pkg/util/fileutils2" ) type SLocalGuestFS struct { @@ -117,7 +118,7 @@ func (f *SLocalGuestFS) Listdir(sPath string, caseInsensitive bool) []string { func (f *SLocalGuestFS) Cleandir(dir string, keepdir, caseInsensitive bool) error { sPath := f.getLocalPath(dir, caseInsensitive) if len(sPath) > 0 { - return cloudcommon.Cleandir(sPath, keepdir) + return fileutils2.Cleandir(sPath, keepdir) } return fmt.Errorf("No such file %s", sPath) } @@ -126,7 +127,7 @@ func (f *SLocalGuestFS) Cleandir(dir string, keepdir, caseInsensitive bool) erro func (f *SLocalGuestFS) Zerofiles(dir string, caseInsensitive bool) error { sPath := f.getLocalPath(dir, caseInsensitive) if len(sPath) > 0 { - return cloudcommon.Zerofiles(sPath) + return fileutils2.Zerofiles(sPath) } return fmt.Errorf("No such file %s", sPath) } @@ -313,7 +314,7 @@ func (f *SLocalGuestFS) FilePutContents(sPath, content string, modAppend, caseIn } } if len(sPath) > 0 { - return cloudcommon.FilePutContents(sPath, content, modAppend) + return fileutils2.FilePutContents(sPath, content, modAppend) } else { return fmt.Errorf("Cann't put content") } diff --git a/pkg/hostman/guestman/guestman.go b/pkg/hostman/guestman/guestman.go index ddae2bc4b2..b77fcaada5 100644 --- a/pkg/hostman/guestman/guestman.go +++ b/pkg/hostman/guestman/guestman.go @@ -13,14 +13,15 @@ import ( "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/pkg/util/regutils" + "yunion.io/x/pkg/util/seclib" + "yunion.io/x/onecloud/pkg/cloudcommon/httpclients" "yunion.io/x/onecloud/pkg/cloudcommon/sshkeys" "yunion.io/x/onecloud/pkg/cloudcommon/workmanager" "yunion.io/x/onecloud/pkg/hostman/guestfs" "yunion.io/x/onecloud/pkg/httperrors" - "yunion.io/x/pkg/util/regutils" - "yunion.io/x/pkg/util/seclib" + "yunion.io/x/onecloud/pkg/util/timeutils2" ) const VNC_PORT_BASE = 5900 @@ -88,7 +89,7 @@ func (m *SGuestManager) VerifyExistingGuests(pendingDelete bool) { func (m *SGuestManager) OnVerifyExistingGuestsFail(err error, pendingDelete bool) { log.Errorf("OnVerifyExistingGuestFail: %s, try again 30 seconds later", err.Error()) - cloudcommon.AddTimeout(30*time.Second, func() { m.VerifyExistingGuests(false) }) + timeutils2.AddTimeout(30*time.Second, func() { m.VerifyExistingGuests(false) }) } func (m *SGuestManager) OnVerifyExistingGuestsSucc(res jsonutils.JSONObject, pendingDelete bool) { @@ -328,8 +329,9 @@ func (m *SGuestManager) GetFreeVncPort() int64 { } var port = 1 for { - if _, ok := vncPorts[port]; !ok && !cloudcommon.IsTcpPortUsed("0.0.0.0", VNC_PORT_BASE+port) && - !cloudcommon.IsTcpPortUsed("0.0.0.0", MONITOR_PORT_BASE+port) { + // TODO: IsTcpPortUsed + if _, ok := vncPorts[port]; !ok && !netutils2.IsTcpPortUsed("0.0.0.0", VNC_PORT_BASE+port) && + !netutils2.IsTcpPortUsed("0.0.0.0", MONITOR_PORT_BASE+port) { break } else { port += 1 diff --git a/pkg/hostman/guestman/guesttasks.go b/pkg/hostman/guestman/guesttasks.go index 2b3dbae6f3..8fee67d5b2 100644 --- a/pkg/hostman/guestman/guesttasks.go +++ b/pkg/hostman/guestman/guesttasks.go @@ -4,8 +4,8 @@ import ( "context" "time" - "yunion.io/x/onecloud/pkg/cloudcommon" "yunion.io/x/onecloud/pkg/cloudcommon/httpclients" + "yunion.io/x/onecloud/pkg/util/timeutils2" ) type SGuestStopTask struct { @@ -49,5 +49,5 @@ func (s *SGuestStopTask) checkGuestRunning() { } func (s *SGuestStopTask) CheckGuestRunningLater() { - cloudcommon.AddTimeout(time.Second*1, s.checkGuestRunning()) + timeutils2.AddTimeout(time.Second*1, s.checkGuestRunning()) } diff --git a/pkg/hostman/guestman/qemu-kvm.go b/pkg/hostman/guestman/qemu-kvm.go index 6e20771bb5..693a5ddc8d 100644 --- a/pkg/hostman/guestman/qemu-kvm.go +++ b/pkg/hostman/guestman/qemu-kvm.go @@ -16,13 +16,14 @@ import ( "yunion.io/x/log" "yunion.io/x/pkg/util/regutils" - "yunion.io/x/onecloud/pkg/cloudcommon" "yunion.io/x/onecloud/pkg/cloudcommon/httpclients" "yunion.io/x/onecloud/pkg/cloudcommon/storagetypes" "yunion.io/x/onecloud/pkg/hostman/guestfs" "yunion.io/x/onecloud/pkg/hostman/hostinfo" "yunion.io/x/onecloud/pkg/hostman/monitor" "yunion.io/x/onecloud/pkg/hostman/options" + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/timeutils2" ) const ( @@ -215,7 +216,7 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa return nil, nil } else { log.Infof("Async start server %s failed: %s!!!", s.GetName(), err) - cloudcommon.AddTimeout(100*time.Millisecond, s.SyncStatus()) + timeutils2.AddTimeout(100*time.Millisecond, s.SyncStatus()) return nil, err } } @@ -225,14 +226,14 @@ func (s *SKVMGuestInstance) saveScripts(data *jsonutils.JSONDict) error { if err != nil { return err } - if err := cloudcommon.FilePutContents(s.GetStartScriptPath, startScript, false); err != nil { + if err := fileutils2.FilePutContents(s.GetStartScriptPath, startScript, false); err != nil { return err } stopScript, err := s.generateStopScript(data) if err != nil { return err } - return cloudcommon.FilePutContents(s.GetStopScriptPath, stopScript, false) + return fileutils2.FilePutContents(s.GetStopScriptPath, stopScript, false) } func (s *SKVMGuestInstance) GetStartScriptPath() string { @@ -303,7 +304,7 @@ func (s *SKVMGuestInstance) ListStateFilePaths() []string { } func (s *SKVMGuestInstance) StartMonitor(ctx context.Context) { - cloudcommon.AddTimeout(100*time.Millisecond, func() { s.delayStartMonitor(ctx) }) + timeutils2.AddTimeout(100*time.Millisecond, func() { s.delayStartMonitor(ctx) }) } func (s *SKVMGuestInstance) delayStartMonitor(ctx context.Context) { @@ -374,7 +375,7 @@ func (s *SKVMGuestInstance) GetVncPort() int { } func (s *SKVMGuestInstance) saveVncPort(port int64) error { - return cloudcommon.FilePutContents(s.GetVncFilePath(), fmt.Sprintf("%d", port), false) + return fileutils2.FilePutContents(s.GetVncFilePath(), fmt.Sprintf("%d", port), false) } func (s *SKVMGuestInstance) SyncStatus() { @@ -394,7 +395,7 @@ func (s *SKVMGuestInstance) SaveDesc(desc jsonutils.JSONObject) error { // bw_info = self._get_bw_info() // netmon_info = self._get_netmon_info() s.Desc = desc.(*jsonutils.JSONDict) - if err := cloudcommon.FilePutContents(s.GetDescFilePath(), desc.String()); err != nil { + if err := fileutils2.FilePutContents(s.GetDescFilePath(), desc.String()); err != nil { log.Errorln(err) } // TODO diff --git a/pkg/hostman/hostinfo/host_cpu.go b/pkg/hostman/hostinfo/host_cpu.go index 486d40f4f1..3a6856e0b6 100644 --- a/pkg/hostman/hostinfo/host_cpu.go +++ b/pkg/hostman/hostinfo/host_cpu.go @@ -8,10 +8,10 @@ import ( "strings" "github.com/shirou/gopsutil/cpu" - "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" + "yunion.io/x/onecloud/pkg/cloudcommon/types" + "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/sysutils" ) @@ -39,7 +39,7 @@ func DetectCpuInfo() (*SCPUInfo, err) { return nil, err } cpu.Percent(interval, percpu) - ret, err := cloudcommon.FileGetContents("/proc/cpuinfo") + ret, err := fileutils2.FileGetContents("/proc/cpuinfo") if err != nil { log.Errorln(err) return nil, err diff --git a/pkg/hostman/hostinfo/hostinfo.go b/pkg/hostman/hostinfo/hostinfo.go index 381611f546..277e81b951 100644 --- a/pkg/hostman/hostinfo/hostinfo.go +++ b/pkg/hostman/hostinfo/hostinfo.go @@ -8,10 +8,12 @@ import ( "strings" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" - "yunion.io/x/onecloud/pkg/cloudcommon/qemutils" + "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/storageman" + "yunion.io/x/onecloud/pkg/util/fileutils2" + "yunion.io/x/onecloud/pkg/util/qemutils" + "yunion.io/x/onecloud/pkg/util/timeutils2" ) type SHostInfo struct { @@ -91,7 +93,7 @@ func (h *SHostInfo) prepareEnv() error { ioParams["queue/iosched/group_idle"] = "0" ioParams["queue/iosched/quantum"] = "32" } - cloudcommon.ChangeAllBlkdevsParams(ioParams) + fileutils2.ChangeAllBlkdevsParams(ioParams) _, err = exec.Command("modprobe", "tun").Output() if err != nil { return fmt.Errorf("Failed to activate tun/tap device") @@ -209,7 +211,7 @@ func (h *SHostInfo) EnableNativeHugepages() error { h.setSysConfig(k, v) } preAllocPagesNum := h.GetMemory()/h.Memory.GetHugepagesizeMb() + 1 - cmd := cloudcommon.CommandWithTimeout(1, "sh", "-c", fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", preAllocPagesNum)) + cmd := timeutils2.CommandWithTimeout(1, "sh", "-c", fmt.Sprintf("echo %d > /proc/sys/vm/nr_hugepages", preAllocPagesNum)) _, err := cmd.Output() if err != nil { log.Errorln(err) @@ -228,7 +230,7 @@ func (h *SHostInfo) setSysConfig(path, val string) bool { if _, err := os.Stat(path); !os.IsNotExist(err) { oval, _ := ioutil.ReadFile(path) if string(oval) != val { - err = cloudcommon.FilePutContents(path, val, false) + err = fileutils2.FilePutContents(path, val, false) if err == nil { return true } diff --git a/pkg/hostman/storageman/diskutils.go b/pkg/hostman/storageman/diskutils.go index 1ad02da48b..ba61d00ee3 100644 --- a/pkg/hostman/storageman/diskutils.go +++ b/pkg/hostman/storageman/diskutils.go @@ -11,8 +11,8 @@ import ( "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/qemutils" "yunion.io/x/onecloud/pkg/hostman/guestfs" + "yunion.io/x/onecloud/pkg/util/qemutils" ) const MAX_TRIES = 3 diff --git a/pkg/hostman/storageman/nbdman.go b/pkg/hostman/storageman/nbdman.go index 13be225a93..0d51f85d78 100644 --- a/pkg/hostman/storageman/nbdman.go +++ b/pkg/hostman/storageman/nbdman.go @@ -6,7 +6,8 @@ import ( "sync" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon" + + "yunion.io/x/onecloud/pkg/util/fileutils2" ) type SNBDManager struct { @@ -51,7 +52,7 @@ func (m *SNBDManager) AcquireNbddev() string { defer m.nbdLock.Unlock() m.nbdLock.Lock() for nbdDev := range m.nbdDevs { - if cloudcommon.IsBlockDeviceUsed(nbdDev) { + if fileutils2.IsBlockDeviceUsed(nbdDev) { m.nbdDevs[nbdDev] = true } if !m.nbdDevs[nbdDev] { diff --git a/pkg/mcclient/modules/mod_tasks.go b/pkg/mcclient/modules/mod_tasks.go index 8dbfc433ae..1717a96ca8 100644 --- a/pkg/mcclient/modules/mod_tasks.go +++ b/pkg/mcclient/modules/mod_tasks.go @@ -45,8 +45,12 @@ func (man ComputeTasksManager) TaskComplete(session *mcclient.ClientSession, tas } func (man ComputeTasksManager) TaskFailed(session *mcclient.ClientSession, taskId string, err error) { + man.TaskFailed2(session, taskId, err.Error()) +} + +func (man ComputeTasksManager) TaskFailed2(session *mcclient.ClientSession, taskId string, reason string) { params := jsonutils.NewDict() params.Add(jsonutils.NewString("error"), "__status__") - params.Add(jsonutils.NewString(err.Error()), "__reason__") + params.Add(jsonutils.NewString(reason), "__reason__") man.TaskComplete(session, taskId, params) } diff --git a/pkg/util/fileutils2/cheksum.go b/pkg/util/fileutils2/cheksum.go index 84d4053bd2..4cbfdb6dd9 100644 --- a/pkg/util/fileutils2/cheksum.go +++ b/pkg/util/fileutils2/cheksum.go @@ -10,11 +10,18 @@ import ( "crypto/sha1" "crypto/sha256" "crypto/sha512" +<<<<<<< HEAD "yunion.io/x/log" ) func FileHash(filename string, hash []hash.Hash) ([][]byte, error) { +======= + "yunion.io/x/log" +) + +func FileHash(filename string, hash []hash.Hash) ([]string, error) { +>>>>>>> move util to pkg/utils fp, err := os.Open(filename) if err != nil { log.Errorf("open file for hash fail %s", err) @@ -38,19 +45,33 @@ func FileHash(filename string, hash []hash.Hash) ([][]byte, error) { return nil, err } } +<<<<<<< HEAD sums := make([][]byte, len(hash)) for i := 0; i < len(hash); i += 1 { sums[i] = hash[i].Sum(nil) +======= + sums := make([]string, len(hash)) + for i := 0; i < len(hash); i += 1 { + sums[i] = fmt.Sprintf("%x", hash[i].Sum(nil)) +>>>>>>> move util to pkg/utils } return sums, nil } +<<<<<<< HEAD func MD5(filename string) (string, error) { +======= +func Md5(filename string) (string, error) { +>>>>>>> move util to pkg/utils sums, err := FileHash(filename, []hash.Hash{md5.New()}) if err != nil { return "", err } +<<<<<<< HEAD return fmt.Sprintf("%x", sums[0]), nil +======= + return sums[0], nil +>>>>>>> move util to pkg/utils } func SHA1(filename string) (string, error) { @@ -58,7 +79,11 @@ func SHA1(filename string) (string, error) { if err != nil { return "", err } +<<<<<<< HEAD return fmt.Sprintf("%x", sums[0]), nil +======= + return sums[0], nil +>>>>>>> move util to pkg/utils } func SHA256(filename string) (string, error) { @@ -66,7 +91,11 @@ func SHA256(filename string) (string, error) { if err != nil { return "", err } +<<<<<<< HEAD return fmt.Sprintf("%x", sums[0]), nil +======= + return sums[0], nil +>>>>>>> move util to pkg/utils } func SHA512(filename string) (string, error) { @@ -74,5 +103,9 @@ func SHA512(filename string) (string, error) { if err != nil { return "", err } +<<<<<<< HEAD return fmt.Sprintf("%x", sums[0]), nil +======= + return sums[0], nil +>>>>>>> move util to pkg/utils } diff --git a/pkg/cloudcommon/utils.go b/pkg/util/fileutils2/fileutils.go similarity index 59% rename from pkg/cloudcommon/utils.go rename to pkg/util/fileutils2/fileutils.go index c21ac07dc2..cf98d3ba1e 100644 --- a/pkg/cloudcommon/utils.go +++ b/pkg/util/fileutils2/fileutils.go @@ -1,4 +1,4 @@ -package cloudcommon +package fileutils2 import ( "bufio" @@ -9,33 +9,10 @@ import ( "path" "regexp" "strings" - "time" - "yunion.io/x/jsonutils" "yunion.io/x/log" - "yunion.io/x/onecloud/pkg/cloudcommon/types" - "yunion.io/x/pkg/util/netutils" ) -// timer utils - -func AddTimeout(second time.Duration, callback func()) { - go func() { - <-time.NewTimer(second).C - callback() - }() -} - -func CommandWithTimeout(timeout int, cmds ...string) *exec.Cmd { - if timeout > 0 { - cmds = append([]string{"timeout", "--signal=KILL", fmt.Sprintf("%ds", timeout)}, cmds...) - } - return exec.Command(cmds[0], cmds[1:]...) -} - -// file utils - -// TODO: test func Cleandir(sPath string, keepdir bool) error { if f, _ := os.Lstat(sPath); f == nil || f.Mode()&os.ModeSymlink == os.ModeSymlink { return nil @@ -164,21 +141,19 @@ func ChangeBlkdevParameter(dev, key, value string) { } } -/* func PathNotExists(path string) bool { - if _, err := os.Stat(path); os.IsNotExist(err) { - return true - } - return false + if _, err := os.Stat(path); os.IsNotExist(err) { + return true + } + return false } func PathExists(path string) bool { - if _, err := os.Stat(path); !os.IsNotExist(err) { - return true - } - return false + if _, err := os.Stat(path); !os.IsNotExist(err) { + return true + } + return false } -*/ func FileGetContents(file string) (string, error) { content, err := ioutil.ReadFile(file) @@ -251,115 +226,3 @@ func (hf HostsFile) String() string { } return ret } - -//net utils -var PSEUDO_VIP = "169.254.169.231" -var MASKS = []string{"0", "128", "192", "224", "240", "248", "252", "254", "255"} - -var PRIVATE_PREFIXES = []string{ - "10.0.0.0/8", - "172.16.0.0/12", - "192.168.0.0/16", -} - -func GetPrivatePrefixes(privatePrefixes []string) []string { - if privatePrefixes != nil { - return privatePrefixes - } else { - return PRIVATE_PREFIXES - } -} - -func GetMainNic(nics []jsonutils.JSONObject) (jsonutils.JSONObject, error) { - var mainIp netutils.IPV4Addr - var mainNic jsonutils.JSONObject - for _, n := range nics { - if n.Contains("gateway") { - ip, _ := n.GetString("ip") - ipInt, err := netutils.NewIPV4Addr(ip) - if err != nil { - return nil, err - } - if mainIp > 0 { - mainIp = ipInt - mainNic = n - } else if !netutils.IsPrivate(ipInt) && netutils.IsPrivate(mainIp) { - mainIp = ipInt - mainNic = n - } - } - } - return mainNic, nil -} - -func Netlen2Mask(netmasklen int) string { - var mask = "" - var segCnt = 0 - for netmasklen > 0 { - var m string - if netmasklen > 8 { - m = MASKS[8] - netmasklen -= 8 - } else { - m = MASKS[netmasklen] - } - if mask != "" { - mask += "." - } - mask += m - segCnt += 1 - } - for i := 0; i < (4 - segCnt); i++ { - if mask != "" { - mask += "." - } - mask += "0" - } - return mask -} - -func addRoute(routes *[][]string, net, gw string) { - for _, rt := range *routes { - if rt[0] == net { - return - } - } - *routes = append(*routes, []string{net, gw}) -} - -func extendRoutes(routes *[][]string, nicRoutes []types.Route) error { - for i := 0; i < len(nicRoutes); i++ { - addRoute(routes, nicRoutes[i][0], nicRoutes[i][1]) - } - return nil -} - -func isExitAddress(ip string) bool { - ipv4, err := netutils.NewIPV4Addr(ip) - if err != nil { - return false - } - return !netutils.IsPrivate(ipv4) || netutils.IsHostLocal(ipv4) || netutils.IsLinkLocal(ipv4) -} - -func AddNicRoutes(routes *[][]string, nicDesc *types.ServerNic, mainIp string, nicCnt int, privatePrefixes []string) { - if mainIp == nicDesc.Ip { - return - } - if len(nicDesc.Routes) > 0 { - extendRoutes(routes, nicDesc.Routes) - } else if len(nicDesc.Gateway) > 0 && isExitAddress(nicDesc.Ip) && - nicCnt == 2 && nicDesc.Ip != mainIp && isExitAddress(mainIp) { - for _, pref := range GetPrivatePrefixes(privatePrefixes) { - addRoute(routes, pref, nicDesc.Gateway) - } - } -} - -func GetNicDns(nicdesc *types.ServerNic) []string { - dnslist := []string{} - if len(nicdesc.Dns) > 0 { - dnslist = append(dnslist, nicdesc.Dns) - } - return dnslist -} diff --git a/pkg/cloudcommon/utils_test.go b/pkg/util/fileutils2/fileutils_test.go similarity index 96% rename from pkg/cloudcommon/utils_test.go rename to pkg/util/fileutils2/fileutils_test.go index a3b4a23b34..a5ca9afadd 100644 --- a/pkg/cloudcommon/utils_test.go +++ b/pkg/util/fileutils2/fileutils_test.go @@ -1,4 +1,4 @@ -package cloudcommon +package fileutils2 import "testing" diff --git a/pkg/cloudcommon/fstabutils/fstabutils.go b/pkg/util/fstabutils/fstabutils.go similarity index 100% rename from pkg/cloudcommon/fstabutils/fstabutils.go rename to pkg/util/fstabutils/fstabutils.go diff --git a/pkg/util/netutils2/netutils.go b/pkg/util/netutils2/netutils.go new file mode 100644 index 0000000000..ff7c444f4c --- /dev/null +++ b/pkg/util/netutils2/netutils.go @@ -0,0 +1,119 @@ +package netutils2 + +import ( + "yunion.io/x/jsonutils" + "yunion.io/x/pkg/util/netutils" + + "yunion.io/x/onecloud/pkg/cloudcommon/types" +) + +var PSEUDO_VIP = "169.254.169.231" +var MASKS = []string{"0", "128", "192", "224", "240", "248", "252", "254", "255"} + +var PRIVATE_PREFIXES = []string{ + "10.0.0.0/8", + "172.16.0.0/12", + "192.168.0.0/16", +} + +func GetPrivatePrefixes(privatePrefixes []string) []string { + if privatePrefixes != nil { + return privatePrefixes + } else { + return PRIVATE_PREFIXES + } +} + +func GetMainNic(nics []jsonutils.JSONObject) (jsonutils.JSONObject, error) { + var mainIp netutils.IPV4Addr + var mainNic jsonutils.JSONObject + for _, n := range nics { + if n.Contains("gateway") { + ip, _ := n.GetString("ip") + ipInt, err := netutils.NewIPV4Addr(ip) + if err != nil { + return nil, err + } + if mainIp > 0 { + mainIp = ipInt + mainNic = n + } else if !netutils.IsPrivate(ipInt) && netutils.IsPrivate(mainIp) { + mainIp = ipInt + mainNic = n + } + } + } + return mainNic, nil +} + +func Netlen2Mask(netmasklen int) string { + var mask = "" + var segCnt = 0 + for netmasklen > 0 { + var m string + if netmasklen > 8 { + m = MASKS[8] + netmasklen -= 8 + } else { + m = MASKS[netmasklen] + } + if mask != "" { + mask += "." + } + mask += m + segCnt += 1 + } + for i := 0; i < (4 - segCnt); i++ { + if mask != "" { + mask += "." + } + mask += "0" + } + return mask +} + +func addRoute(routes *[][]string, net, gw string) { + for _, rt := range *routes { + if rt[0] == net { + return + } + } + *routes = append(*routes, []string{net, gw}) +} + +func extendRoutes(routes *[][]string, nicRoutes []types.Route) error { + for i := 0; i < len(nicRoutes); i++ { + addRoute(routes, nicRoutes[i][0], nicRoutes[i][1]) + } + return nil +} + +func isExitAddress(ip string) bool { + ipv4, err := netutils.NewIPV4Addr(ip) + if err != nil { + return false + } + return !netutils.IsPrivate(ipv4) || netutils.IsHostLocal(ipv4) || netutils.IsLinkLocal(ipv4) +} + +func AddNicRoutes(routes *[][]string, nicDesc *types.ServerNic, mainIp string, nicCnt int, privatePrefixes []string) { + if mainIp == nicDesc.Ip { + return + } + if len(nicDesc.Routes) > 0 { + extendRoutes(routes, nicDesc.Routes) + } else if len(nicDesc.Gateway) > 0 && isExitAddress(nicDesc.Ip) && + nicCnt == 2 && nicDesc.Ip != mainIp && isExitAddress(mainIp) { + for _, pref := range GetPrivatePrefixes(privatePrefixes) { + addRoute(routes, pref, nicDesc.Gateway) + } + } +} + +func GetNicDns(nicdesc *types.ServerNic) []string { + dnslist := []string{} + if len(nicdesc.Dns) > 0 { + dnslist = append(dnslist, nicdesc.Dns) + } + return dnslist +} diff --git a/pkg/util/qemuimg/consts.go b/pkg/util/qemuimg/consts.go index d708eda21d..69a3526fab 100644 --- a/pkg/util/qemuimg/consts.go +++ b/pkg/util/qemuimg/consts.go @@ -1,10 +1,13 @@ package qemuimg +<<<<<<< HEAD import ( "strings" // "yunion.io/x/log" ) +======= +>>>>>>> move util to pkg/utils type TImageFormat string const ( @@ -15,6 +18,7 @@ const ( RAW = TImageFormat("raw") ) +<<<<<<< HEAD var supportedImageFormats = []TImageFormat{ QCOW2, VMDK, VHD, ISO, RAW, } @@ -28,6 +32,8 @@ func IsSupportedImageFormat(fmtStr string) bool { return false } +======= +>>>>>>> move util to pkg/utils func (fmt TImageFormat) String() string { switch string(fmt) { case "vhd": @@ -36,6 +42,7 @@ func (fmt TImageFormat) String() string { return string(fmt) } } +<<<<<<< HEAD func String2ImageFormat(fmt string) TImageFormat { switch strings.ToLower(fmt) { @@ -53,3 +60,5 @@ func String2ImageFormat(fmt string) TImageFormat { // log.Fatalf("unknown image format!!! %s", fmt) return TImageFormat(fmt) } +======= +>>>>>>> move util to pkg/utils diff --git a/pkg/util/qemuimg/init.go b/pkg/util/qemuimg/init.go index 259b139193..8c6ca67560 100644 --- a/pkg/util/qemuimg/init.go +++ b/pkg/util/qemuimg/init.go @@ -1,12 +1,21 @@ package qemuimg import ( +<<<<<<< HEAD "fmt" "os/exec" "regexp" "yunion.io/x/log" +======= + "os/exec" + + "fmt" + "regexp" + + "yunion.io/x/log" +>>>>>>> move util to pkg/utils "yunion.io/x/onecloud/pkg/util/qemutils" "yunion.io/x/onecloud/pkg/util/version" ) diff --git a/pkg/util/qemuimg/qemuimg_test.go b/pkg/util/qemuimg/qemuimg_test.go index d77ec5c64b..d5a927f131 100644 --- a/pkg/util/qemuimg/qemuimg_test.go +++ b/pkg/util/qemuimg/qemuimg_test.go @@ -110,6 +110,7 @@ func TestQcow2(t *testing.T) { img4.Delete() img.Delete() } +<<<<<<< HEAD func TestVhd(t *testing.T) { img, err := NewQemuImage("test") if err != nil { @@ -147,6 +148,8 @@ func TestVhd(t *testing.T) { t.Logf("%s %v", img, img.IsSparse()) img.Delete() } +======= +>>>>>>> move util to pkg/utils func TestVmdk(t *testing.T) { img, err := NewQemuImage("test") diff --git a/pkg/cloudcommon/qemutils/qemutils_test.go b/pkg/util/qemutils/qemutils_test.go similarity index 100% rename from pkg/cloudcommon/qemutils/qemutils_test.go rename to pkg/util/qemutils/qemutils_test.go diff --git a/pkg/util/timeutils2/timeutils.go b/pkg/util/timeutils2/timeutils.go new file mode 100644 index 0000000000..fb2e1b7d8c --- /dev/null +++ b/pkg/util/timeutils2/timeutils.go @@ -0,0 +1,21 @@ +package timeutils2 + +import ( + "fmt" + "os/exec" + "time" +) + +func AddTimeout(second time.Duration, callback func()) { + go func() { + <-time.NewTimer(second).C + callback() + }() +} + +func CommandWithTimeout(timeout int, cmds ...string) *exec.Cmd { + if timeout > 0 { + cmds = append([]string{"timeout", "--signal=KILL", fmt.Sprintf("%ds", timeout)}, cmds...) + } + return exec.Command(cmds[0], cmds[1:]...) +} diff --git a/pkg/cloudcommon/version/version_test.go b/pkg/util/version/version_test.go similarity index 100% rename from pkg/cloudcommon/version/version_test.go rename to pkg/util/version/version_test.go