From f31605ca72ea58f4f1dceca9029305d286d72687 Mon Sep 17 00:00:00 2001 From: wanyaoqi Date: Thu, 13 Dec 2018 20:30:53 +0800 Subject: [PATCH] 2018-12-13 --- pkg/hostman/guestfs/core.go | 66 ++++++++++++++ pkg/hostman/guestfs/guestfs.go | 115 +++++++++++++++++++++++ pkg/hostman/storageman/core.go | 13 ++- pkg/hostman/storageman/diskutils.go | 137 ++++++++++++++++++++++++++++ pkg/hostman/storageman/nbdman.go | 71 ++++++++++++++ pkg/hostman/utils.go | 83 +++++++++++++---- pkg/hostman/utils_test.go | 32 +++++++ 7 files changed, 498 insertions(+), 19 deletions(-) create mode 100644 pkg/hostman/guestfs/guestfs.go create mode 100644 pkg/hostman/storageman/diskutils.go create mode 100644 pkg/hostman/storageman/nbdman.go create mode 100644 pkg/hostman/utils_test.go diff --git a/pkg/hostman/guestfs/core.go b/pkg/hostman/guestfs/core.go index 1a037d2c13..2a1fb7dd34 100644 --- a/pkg/hostman/guestfs/core.go +++ b/pkg/hostman/guestfs/core.go @@ -1,8 +1,16 @@ package guestfs import ( + "fmt" + "io/ioutil" + "math/rand" + "os" + "strings" + "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudcommon/sshkeys" + "yunion.io/x/onecloud/pkg/hostman" ) type SDeployInfo struct { @@ -11,3 +19,61 @@ type SDeployInfo struct { password string isInit bool } + +type SLocalGuestFS struct { + mountPath string + readOnly bool +} + +func (f *SLocalGuestFS) isReadonly() bool { + log.Infof("Test if read-only fs ...") + var filename = fmt.Sprint("./%f", rand.Float32()) + if err := hostman.FilePutContents(filename, fmt.Sprint("%f", rand.Float32()), false); err == nil { + f.Remove(filename, false) + return false + } else { + log.Errorf("File system is readonly: %s", err) + f.readOnly = true + return true + } +} + +func (f *SLocalGuestFS) getLocalPath(path string, caseInsensitive bool) string { + var fullPath = f.mountPath + pathSegs := strings.Split(path, "/") + for i := 0; i < len(pathSegs); i++ { + if len(pathSegs[i]) > 0 { + var readSeg string + files, _ := ioutil.ReadDir(fullPath) + for _, file := range files { + if file.Name() == pathSegs[i] + || (caseInsensitive && strings.ToLower(file.Name())) == strings.ToLower(pathSegs[i]) { + + } + } + } + } +} + +func (f *SLocalGuestFS) Remove(path string, caseInsensitive bool) { + path = f.getLocalPath(path, caseInsensitive) + if len(path) > 0 { + os.Remove(path) + } +} + +func NewLocalGuestFS(mountPath string) *SLocalGuestFS { + var ret = new(SLocalGuestFS) + ret.mountPath = mountPath + return ret +} + +type IRootFsDriver interface { +} + +var rootfsDrivers map[string]IRootFsDriver + +func DetectRootFs(part *SKVMGuestDiskPartition) IRootFsDriver { + //TODO + return nil +} diff --git a/pkg/hostman/guestfs/guestfs.go b/pkg/hostman/guestfs/guestfs.go new file mode 100644 index 0000000000..b118973ca6 --- /dev/null +++ b/pkg/hostman/guestfs/guestfs.go @@ -0,0 +1,115 @@ +package guestfs + +import ( + "fmt" + "os/exec" + "strings" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/hostman" + "yunion.io/x/pkg/utils" +) + +type SKVMGuestDiskPartition struct { + *SLocalGuestFS + partDev string + fs string + + readonly bool +} + +func NewKVMGuestDiskPartition(devPath string) *SKVMGuestDiskPartition { + var res = new(SKVMGuestDiskPartition) + res.partDev = devPath + res.fs = res.getFsFormat() + hostman.CleanFailedMountpoints() + mountPath := fmt.Sprintf("/tmp/%s", strings.Replace(devPath, "/", "_", -1)) + res.SLocalGuestFS = NewLocalGuestFS(mountPath) + return res +} + +func (p *SKVMGuestDiskPartition) getFsFormat() string { + return hostman.GetFsFormat(p.partDev) +} + +func (p *SKVMGuestDiskPartition) Mount() bool { + if len(p.fs) == 0 || utils.IsInStringArray(p.fs, []string{"swap", "btrfs"}) { + return false + } + err := p.fsck() + if err != nil { + log.Errorf("SKVMGuestDiskPartition fsck error: %s", err) + return false + } + err = p.mount(false) + if err != nil { + log.Errorf("SKVMGuestDiskPartition mount error: %s", err) + return false + } + if p.isReadonly() { + log.Errorf("SKVMGuestDiskPartition %s is readonly, try mount as ro", p.partDev) + p.Umount() + err = p.mount(true) + if err != nil { + log.Errorf("SKVMGuestDiskPartition mount as ro error %s", err) + return false + } else { + p.readonly = true + } + } + return true +} + +func (p *SKVMGuestDiskPartition) mount(readonly bool) error { + exec.Command("mkdir", "-p", p.mountPath) + var cmds = []string{"mount", "-t"} + var opt, fsType string + if readonly { + opt = "ro" + } + fsType = p.fs + if fsType == "ntfs" { + fsType = "ntfs-3g" + if !readonly { + opt = "recover,remove_hiberfile,noatime,windows_names" + } + } else if fsType == "hfsplus" && !readonly { + opt = "force,rw" + } + cmds = append(cmds, fsType) + if len(opt) > 0 { + cmds = append(cmds, "-o", opt) + } + cmds = append(cmds, p.partDev, p.mountPath) + return exec.Command(cmds[0], cmds[1:]...).Run() +} + +func (p *SKVMGuestDiskPartition) fsck() error { + var checkCmd, fixCmd []string + switch p.fs { + case "hfsplus": + checkCmd = []string{"fsck.hfsplus", "-q", p.partDev} + fixCmd = []string{"fsck.hfsplus", "-fpy", p.partDev} + case "ext2", "ext3", "ext4": + checkCmd = []string{"e2fsck", "-n", p.partDev} + fixCmd = []string{"e2fsck", "-fp", p.partDev} + case "ntfs": + checkCmd = []string{"ntfsfix", "-n", p.partDev} + fixCmd = []string{"ntfsfix", p.partDev} + } + if len(checkCmd) > 0 { + _, err := exec.Command(checkCmd[0], checkCmd[1:]...).Output() + if err != nil { + log.Warningf("FS %s dirty, try to repair ...", p.partDev) + for i := 0; i < 3; i++ { + _, err := exec.Command(fixCmd[0], fixCmd[1:]...).Output() + if err == nil { + break + } else { + return err + } + } + } + } + return nil +} diff --git a/pkg/hostman/storageman/core.go b/pkg/hostman/storageman/core.go index b1dd36bb55..b0741dabd4 100644 --- a/pkg/hostman/storageman/core.go +++ b/pkg/hostman/storageman/core.go @@ -1,10 +1,12 @@ package storageman import ( + "fmt" "path" "sync" "yunion.io/x/jsonutils" + "yunion.io/x/log" "yunion.io/x/onecloud/pkg/hostman/guestfs" ) @@ -63,8 +65,15 @@ func (d *SBaseDisk) DeployGuestFs( deployInfo *guestfs.SDeployInfo) (jsonutils.JSONObject, error) { // TODO var kvmDisk = NewKVMGuestDisk(d.getPath()) - defer kvmDisk.Disconnect() - kvmDisk.Connect() + if kvmDisk.Connect() { + defer kvmDisk.Disconnect() + log.Infof("Kvm Disk Connect Success !!") + if part := kvmDisk.Mount(); part != nil { + defer kvmDisk.Umount(part) + return part.DeployGuestFs(guestdesc, deployInfo) + } + } + return nil, fmt.Errorf("Kvm disk connect or mount error") } func NewBaseDisk(storage IStorage, id string) *SBaseDisk { diff --git a/pkg/hostman/storageman/diskutils.go b/pkg/hostman/storageman/diskutils.go new file mode 100644 index 0000000000..920a3b9eb4 --- /dev/null +++ b/pkg/hostman/storageman/diskutils.go @@ -0,0 +1,137 @@ +package storageman + +import ( + "fmt" + "io/ioutil" + "os/exec" + "path" + "path/filepath" + "strings" + "time" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/cloudcommon/qemutils" + "yunion.io/x/onecloud/pkg/hostman/guestfs" +) + +const MAX_TRIES = 3 + +type SKVMGuestDisk struct { + imagePath string + nbdDev string + partitions []*guestfs.SKVMGuestDiskPartition +} + +func NewKVMGuestDisk(imagePath string) *SKVMGuestDisk { + var ret = new(SKVMGuestDisk) + ret.imagePath = imagePath + ret.partitions = make([]string, 0) + return ret +} + +func (d *SKVMGuestDisk) Connect() bool { + d.nbdDev = nbdManager.AcquireNbddev() + if len(d.nbdDev) == 0 { + log.Errorln("Cannot get nbd device") + return false + } + + var cmd []string + if strings.HasPrefix(d.imagePath, "rbd:") || d.getImageFormat() == "raw" { + cmd = []string{qemutils.GetQemuNbd(), "-c", d.nbdDev, "-f", "raw", d.imagePath} + } else { + cmd = []string{qemutils.GetQemuNbd(), "-c", d.nbdDev, d.imagePath} + } + _, err := exec.Command(cmd[0], cmd[1:]...).Output() + if err != nil { + log.Errorln(err.Error()) + return false + } + + var tried uint = 0 + for len(d.partitions) == 0 && tried < MAX_TRIES { + time.Sleep((1 << tried) * time.Second) + err = d.findPartitions() + if err != nil { + log.Errorln(err.Error()) + return false + } + tried += 1 + } + d.setupLVMS() + return true +} + +func (d *SKVMGuestDisk) getImageFormat() string { + lines, err := exec.Command(qemutils.GetQemuImg(), "info", d.imagePath).Output() + if err != nil { + return "" + } + imgStr := strings.Split(string(lines), "\n") + for i := 0; i < len(imgStr); i++ { + if strings.HasPrefix(imgStr[i], "file format: ") { + return imgStr[i][len("file format: "):] + } + } + return "" +} + +func (d *SKVMGuestDisk) findPartitions() error { + if len(d.nbdDev) == 0 { + return fmt.Errorf("Want find partitions but dosen't have nbd dev") + } + dev := filepath.Base(d.nbdDev) + devpath := filepath.Dir(d.nbdDev) + files, err := ioutil.ReadDir(devpath) + if err != nil { + return err + } + var partitions []*guestfs.SKVMGuestDiskPartition + for i := 0; i < len(files); i++ { + if files[i].Name() != dev && strings.HasPrefix(files[i].Name(), dev+"p") { + var part = guestfs.NewKVMGuestDiskPartition(path.Join(devpath, files[i].Name())) + d.partitions = append(d.partitions, part) + } + } + return nil +} + +func (d *SKVMGuestDisk) setupLVMS() error { + //TODO?? 可能不需要开发这里 +} + +func (d *SKVMGuestDisk) DisConnect() bool { + if len(d.nbdDev) > 0 { + // TODO?? PutdownLVMS ?? + _, err := exec.Command(qemutils.GetQemuNbd(), "-d", d.nbdDev) + if err != nil { + log.Errorln(err.Error()) + return false + } + nbdManager.ReleaseNbddev(d.nbdDev) + d.nbdDev = "" + d.partitions = d.partitions[len(d.partitions):] + return true + } else { + return false + } +} + +func (d *SKVMGuestDisk) Mount() guestfs.IRootFsDriver { + for i := 0; i < d.partitions; i++ { + if d.partitions[i].Mount() { + if fs := guestfs.DetectRootFs(d.partitions[i]); fs != nil { + return fs + } else { + d.partitions[i].Umount() + } + } + } + return nil +} + +func (d *SKVMGuestDisk) Umount(fd guestfs.IRootFsDriver) { + if part := fd.GetPartition(); part != nil { + part.Umount() + } +} diff --git a/pkg/hostman/storageman/nbdman.go b/pkg/hostman/storageman/nbdman.go new file mode 100644 index 0000000000..5924d5832b --- /dev/null +++ b/pkg/hostman/storageman/nbdman.go @@ -0,0 +1,71 @@ +package storageman + +import ( + "fmt" + "os" + "sync" + + "yunion.io/x/log" + "yunion.io/x/onecloud/pkg/hostman" +) + +type SNBDManager struct { + // find local /dev/nbdx device, map key is devs, value is it in use + nbdDevs map[string]bool + nbdLock *sync.Mutex +} + +var nbdManager *SNBDManager + +func init() { + nbdManager = NewNBDManager() +} + +func GetNBDManager() *SNBDManager { + return nbdManager +} + +func NewNBDManager() *SNBDManager { + var ret = new(SNBDManager) + ret.nbdDevs = make(map[string]bool, 0) + ret.nbdLock = new(sync.Mutex) + ret.findNbdDevices() + return ret +} + +func (m *SNBDManager) findNbdDevices() { + var i = 0 + for { + _, err := os.Stat(fmt.Sprintf("/dev/nbd%d", i)) + if !os.IsNotExist(err) { + m.nbdDevs[fmt.Sprintf("/dev/nbd%d", i)] = false + i++ + } else { + break + } + } + log.Infof("NBD_DEVS: %#v", m.nbdDevs) +} + +func (m *SNBDManager) AcquireNbddev() string { + defer m.nbdLock.Unlock() + m.nbdLock.Lock() + for nbdDev := range m.nbdDevs { + if hostman.IsBlockDeviceUsed(nbdDev) { + m.nbdDevs[nbdDev] = true + } + if !m.nbdDevs[nbdDev] { + m.nbdDevs[nbdDev] = true + return nbdDev + } + } + return "" +} + +func (m *SNBDManager) ReleaseNbddev(nbddev string) { + if _, ok := m.nbdDevs[nbddev]; ok { + defer m.nbdLock.Unlock() + m.nbdLock.Lock() + m.nbdDevs[nbddev] = false + } +} diff --git a/pkg/hostman/utils.go b/pkg/hostman/utils.go index 2a329bc625..bfc5240ac2 100644 --- a/pkg/hostman/utils.go +++ b/pkg/hostman/utils.go @@ -1,6 +1,7 @@ package hostman import ( + "bufio" "fmt" "io/ioutil" "os" @@ -12,6 +13,24 @@ import ( "yunion.io/x/log" ) +// timer utils + +func AddTimeout(second time.Duration, callback func()) { + go func() { + <-time.NewTimer(second).C + callback() + }() +} + +func CommandWithTimeout(timeout int, cmds ...string) *exec.Cmd { + if timeout > 0 { + cmds = append([]string{"timeout", "--signal=KILL", fmt.Sprintf("%ds", timeout)}, cmds...) + } + return exec.Command(cmds[0], cmds[1:]...) +} + +// file utils + func FilePutContents(filename string, context string, modAppend bool) error { var mode = os.O_WRONLY | os.O_CREATE if modAppend { @@ -26,14 +45,6 @@ func FilePutContents(filename string, context string, modAppend bool) error { return err } -func CommandWithTimeout(timeout int, cmds ...string) *exec.Cmd { - if timeout > 0 { - cmds = append([]string{"timeout", "--signal=KILL", fmt.Sprintf("%ds", timeout)}, cmds...) - } - return exec.Command(cmds[0], cmds[1:]...) - -} - func IsBlockDevMounted(dev string) bool { devPath := "/dev/" + dev mounts, err := exec.Command("mount").Output() @@ -48,6 +59,18 @@ func IsBlockDevMounted(dev string) bool { return false } +func IsBlockDeviceUsed(dev string) bool { + if strings.HasPrefix(dev, "/dev/") { + dev = dev[strings.LastIndex(dev, "/")+1:] + } + devStr := fmt.Sprint(" %s\n", dev) + devs, _ := exec.Command("cat", "/proc/partitions").Output() + if idx := strings.Index(string(devs), devStr); idx > 0 { + return false + } + return true +} + func ChangeAllBlkdevsParams(params map[string]string) { if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) { blockDevs, err := ioutil.ReadDir("/sys/block") @@ -76,15 +99,6 @@ func ChangeBlkdevParameter(dev, key, value string) { } } -// timer utils - -func AddTimeout(second time.Duration, callback func()) { - go func() { - <-time.NewTimer(second).C - callback() - }() -} - /* func PathNotExists(path string) bool { if _, err := os.Stat(path); os.IsNotExist(err) { @@ -108,3 +122,38 @@ func FileGetContents(file string) (string, error) { } return string(content), nil } + +func GetFsFormat(diskPath string) string { + ret, err := exec.Command("blkid", "-o", "value", "-s", "TYPE", diskPath).Output() + if err != nil { + return "" + } + var res string + for _, line := range strings.Split(string(ret), "\n") { + res += line + } + return res +} + +func CleanFailedMountpoints() { + var mtfile = "/etc/mtab" + if _, err := os.Stat(mtfile); os.IsNotExist(err) { + mtfile = "/proc/mounts" + } + f, err := os.Open(mtfile) + if err != nil { + log.Errorf("CleanFailedMountpoints error: %s", err) + } + reader := bufio.NewReader(f) + line, _, err := reader.ReadLine() + for err != nil { + m := strings.Split(string(line), " ") + if len(m) > 1 { + mp := m[1] + if _, err := os.Stat(mp); os.IsNotExist(err) { + log.Warningf("Mount point %s not exists", mp) + } + exec.Command("umount", mp).Run() + } + } +} diff --git a/pkg/hostman/utils_test.go b/pkg/hostman/utils_test.go new file mode 100644 index 0000000000..e13db0f8bb --- /dev/null +++ b/pkg/hostman/utils_test.go @@ -0,0 +1,32 @@ +package hostman + +import "testing" + +func TestIsBlockDeviceUsed(t *testing.T) { + type args struct { + dev string + } + tests := []struct { + name string + args args + want bool + }{ + { + name: "nbd1", + args: {"/dev/nbd1"}, + want: false, + }, + { + name: "sda", + args: {"/dev/sda"}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := IsBlockDeviceUsed(tt.args.dev); got != tt.want { + t.Errorf("IsBlockDeviceUsed() = %v, want %v", got, tt.want) + } + }) + } +}