2018-12-13

pkg/hostman/guestfs/core.go

@@ -1,8 +1,16 @@
 package guestfs
 
 import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"strings"
+
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/cloudcommon/sshkeys"
+	"yunion.io/x/onecloud/pkg/hostman"
 )
 
 type SDeployInfo struct {
@@ -11,3 +19,61 @@ type SDeployInfo struct {
 	password  string
 	isInit    bool
 }
+
+type SLocalGuestFS struct {
+	mountPath string
+	readOnly  bool
+}
+
+func (f *SLocalGuestFS) isReadonly() bool {
+	log.Infof("Test if read-only fs ...")
+	var filename = fmt.Sprint("./%f", rand.Float32())
+	if err := hostman.FilePutContents(filename, fmt.Sprint("%f", rand.Float32()), false); err == nil {
+		f.Remove(filename, false)
+		return false
+	} else {
+		log.Errorf("File system is readonly: %s", err)
+		f.readOnly = true
+		return true
+	}
+}
+
+func (f *SLocalGuestFS) getLocalPath(path string, caseInsensitive bool) string {
+	var fullPath = f.mountPath
+	pathSegs := strings.Split(path, "/")
+	for i := 0; i < len(pathSegs); i++ {
+		if len(pathSegs[i]) > 0 {
+			var readSeg string
+			files, _ := ioutil.ReadDir(fullPath)
+			for _, file := range files {
+				if file.Name() == pathSegs[i] 
+                    || (caseInsensitive && strings.ToLower(file.Name())) == strings.ToLower(pathSegs[i]) {
+
+				}
+			}
+		}
+	}
+}
+
+func (f *SLocalGuestFS) Remove(path string, caseInsensitive bool) {
+	path = f.getLocalPath(path, caseInsensitive)
+	if len(path) > 0 {
+		os.Remove(path)
+	}
+}
+
+func NewLocalGuestFS(mountPath string) *SLocalGuestFS {
+	var ret = new(SLocalGuestFS)
+	ret.mountPath = mountPath
+	return ret
+}
+
+type IRootFsDriver interface {
+}
+
+var rootfsDrivers map[string]IRootFsDriver
+
+func DetectRootFs(part *SKVMGuestDiskPartition) IRootFsDriver {
+	//TODO
+	return nil
+}

pkg/hostman/guestfs/guestfs.go

@@ -0,0 +1,115 @@
+package guestfs
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/hostman"
+	"yunion.io/x/pkg/utils"
+)
+
+type SKVMGuestDiskPartition struct {
+	*SLocalGuestFS
+	partDev string
+	fs      string
+
+	readonly bool
+}
+
+func NewKVMGuestDiskPartition(devPath string) *SKVMGuestDiskPartition {
+	var res = new(SKVMGuestDiskPartition)
+	res.partDev = devPath
+	res.fs = res.getFsFormat()
+	hostman.CleanFailedMountpoints()
+	mountPath := fmt.Sprintf("/tmp/%s", strings.Replace(devPath, "/", "_", -1))
+	res.SLocalGuestFS = NewLocalGuestFS(mountPath)
+	return res
+}
+
+func (p *SKVMGuestDiskPartition) getFsFormat() string {
+	return hostman.GetFsFormat(p.partDev)
+}
+
+func (p *SKVMGuestDiskPartition) Mount() bool {
+	if len(p.fs) == 0 || utils.IsInStringArray(p.fs, []string{"swap", "btrfs"}) {
+		return false
+	}
+	err := p.fsck()
+	if err != nil {
+		log.Errorf("SKVMGuestDiskPartition fsck error: %s", err)
+		return false
+	}
+	err = p.mount(false)
+	if err != nil {
+		log.Errorf("SKVMGuestDiskPartition mount error: %s", err)
+		return false
+	}
+	if p.isReadonly() {
+		log.Errorf("SKVMGuestDiskPartition %s is readonly, try mount as ro", p.partDev)
+		p.Umount()
+		err = p.mount(true)
+		if err != nil {
+			log.Errorf("SKVMGuestDiskPartition mount as ro error %s", err)
+			return false
+		} else {
+			p.readonly = true
+		}
+	}
+	return true
+}
+
+func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
+	exec.Command("mkdir", "-p", p.mountPath)
+	var cmds = []string{"mount", "-t"}
+	var opt, fsType string
+	if readonly {
+		opt = "ro"
+	}
+	fsType = p.fs
+	if fsType == "ntfs" {
+		fsType = "ntfs-3g"
+		if !readonly {
+			opt = "recover,remove_hiberfile,noatime,windows_names"
+		}
+	} else if fsType == "hfsplus" && !readonly {
+		opt = "force,rw"
+	}
+	cmds = append(cmds, fsType)
+	if len(opt) > 0 {
+		cmds = append(cmds, "-o", opt)
+	}
+	cmds = append(cmds, p.partDev, p.mountPath)
+	return exec.Command(cmds[0], cmds[1:]...).Run()
+}
+
+func (p *SKVMGuestDiskPartition) fsck() error {
+	var checkCmd, fixCmd []string
+	switch p.fs {
+	case "hfsplus":
+		checkCmd = []string{"fsck.hfsplus", "-q", p.partDev}
+		fixCmd = []string{"fsck.hfsplus", "-fpy", p.partDev}
+	case "ext2", "ext3", "ext4":
+		checkCmd = []string{"e2fsck", "-n", p.partDev}
+		fixCmd = []string{"e2fsck", "-fp", p.partDev}
+	case "ntfs":
+		checkCmd = []string{"ntfsfix", "-n", p.partDev}
+		fixCmd = []string{"ntfsfix", p.partDev}
+	}
+	if len(checkCmd) > 0 {
+		_, err := exec.Command(checkCmd[0], checkCmd[1:]...).Output()
+		if err != nil {
+			log.Warningf("FS %s dirty, try to repair ...", p.partDev)
+			for i := 0; i < 3; i++ {
+				_, err := exec.Command(fixCmd[0], fixCmd[1:]...).Output()
+				if err == nil {
+					break
+				} else {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

pkg/hostman/storageman/core.go

@@ -1,10 +1,12 @@
 package storageman
 
 import (
+	"fmt"
 	"path"
 	"sync"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs"
 )
 
@@ -63,8 +65,15 @@ func (d *SBaseDisk) DeployGuestFs(
 	deployInfo *guestfs.SDeployInfo) (jsonutils.JSONObject, error) {
 	// TODO
 	var kvmDisk = NewKVMGuestDisk(d.getPath())
-	defer kvmDisk.Disconnect()
-	kvmDisk.Connect()
+	if kvmDisk.Connect() {
+		defer kvmDisk.Disconnect()
+		log.Infof("Kvm Disk Connect Success !!")
+		if part := kvmDisk.Mount(); part != nil {
+			defer kvmDisk.Umount(part)
+			return part.DeployGuestFs(guestdesc, deployInfo)
+		}
+	}
+	return nil, fmt.Errorf("Kvm disk connect or mount error")
 }
 
 func NewBaseDisk(storage IStorage, id string) *SBaseDisk {

pkg/hostman/storageman/diskutils.go

@@ -0,0 +1,137 @@
+package storageman
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/cloudcommon/qemutils"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+)
+
+const MAX_TRIES = 3
+
+type SKVMGuestDisk struct {
+	imagePath  string
+	nbdDev     string
+	partitions []*guestfs.SKVMGuestDiskPartition
+}
+
+func NewKVMGuestDisk(imagePath string) *SKVMGuestDisk {
+	var ret = new(SKVMGuestDisk)
+	ret.imagePath = imagePath
+	ret.partitions = make([]string, 0)
+	return ret
+}
+
+func (d *SKVMGuestDisk) Connect() bool {
+	d.nbdDev = nbdManager.AcquireNbddev()
+	if len(d.nbdDev) == 0 {
+		log.Errorln("Cannot get nbd device")
+		return false
+	}
+
+	var cmd []string
+	if strings.HasPrefix(d.imagePath, "rbd:") || d.getImageFormat() == "raw" {
+		cmd = []string{qemutils.GetQemuNbd(), "-c", d.nbdDev, "-f", "raw", d.imagePath}
+	} else {
+		cmd = []string{qemutils.GetQemuNbd(), "-c", d.nbdDev, d.imagePath}
+	}
+	_, err := exec.Command(cmd[0], cmd[1:]...).Output()
+	if err != nil {
+		log.Errorln(err.Error())
+		return false
+	}
+
+	var tried uint = 0
+	for len(d.partitions) == 0 && tried < MAX_TRIES {
+		time.Sleep((1 << tried) * time.Second)
+		err = d.findPartitions()
+		if err != nil {
+			log.Errorln(err.Error())
+			return false
+		}
+		tried += 1
+	}
+	d.setupLVMS()
+	return true
+}
+
+func (d *SKVMGuestDisk) getImageFormat() string {
+	lines, err := exec.Command(qemutils.GetQemuImg(), "info", d.imagePath).Output()
+	if err != nil {
+		return ""
+	}
+	imgStr := strings.Split(string(lines), "\n")
+	for i := 0; i < len(imgStr); i++ {
+		if strings.HasPrefix(imgStr[i], "file format: ") {
+			return imgStr[i][len("file format: "):]
+		}
+	}
+	return ""
+}
+
+func (d *SKVMGuestDisk) findPartitions() error {
+	if len(d.nbdDev) == 0 {
+		return fmt.Errorf("Want find partitions but dosen't have nbd dev")
+	}
+	dev := filepath.Base(d.nbdDev)
+	devpath := filepath.Dir(d.nbdDev)
+	files, err := ioutil.ReadDir(devpath)
+	if err != nil {
+		return err
+	}
+	var partitions []*guestfs.SKVMGuestDiskPartition
+	for i := 0; i < len(files); i++ {
+		if files[i].Name() != dev && strings.HasPrefix(files[i].Name(), dev+"p") {
+			var part = guestfs.NewKVMGuestDiskPartition(path.Join(devpath, files[i].Name()))
+			d.partitions = append(d.partitions, part)
+		}
+	}
+	return nil
+}
+
+func (d *SKVMGuestDisk) setupLVMS() error {
+	//TODO?? 可能不需要开发这里
+}
+
+func (d *SKVMGuestDisk) DisConnect() bool {
+	if len(d.nbdDev) > 0 {
+		// TODO?? PutdownLVMS ??
+		_, err := exec.Command(qemutils.GetQemuNbd(), "-d", d.nbdDev)
+		if err != nil {
+			log.Errorln(err.Error())
+			return false
+		}
+		nbdManager.ReleaseNbddev(d.nbdDev)
+		d.nbdDev = ""
+		d.partitions = d.partitions[len(d.partitions):]
+		return true
+	} else {
+		return false
+	}
+}
+
+func (d *SKVMGuestDisk) Mount() guestfs.IRootFsDriver {
+	for i := 0; i < d.partitions; i++ {
+		if d.partitions[i].Mount() {
+			if fs := guestfs.DetectRootFs(d.partitions[i]); fs != nil {
+				return fs
+			} else {
+				d.partitions[i].Umount()
+			}
+		}
+	}
+	return nil
+}
+
+func (d *SKVMGuestDisk) Umount(fd guestfs.IRootFsDriver) {
+	if part := fd.GetPartition(); part != nil {
+		part.Umount()
+	}
+}

pkg/hostman/storageman/nbdman.go

@@ -0,0 +1,71 @@
+package storageman
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	"yunion.io/x/log"
+	"yunion.io/x/onecloud/pkg/hostman"
+)
+
+type SNBDManager struct {
+	// find local /dev/nbdx device, map key is devs, value is it in use
+	nbdDevs map[string]bool
+	nbdLock *sync.Mutex
+}
+
+var nbdManager *SNBDManager
+
+func init() {
+	nbdManager = NewNBDManager()
+}
+
+func GetNBDManager() *SNBDManager {
+	return nbdManager
+}
+
+func NewNBDManager() *SNBDManager {
+	var ret = new(SNBDManager)
+	ret.nbdDevs = make(map[string]bool, 0)
+	ret.nbdLock = new(sync.Mutex)
+	ret.findNbdDevices()
+	return ret
+}
+
+func (m *SNBDManager) findNbdDevices() {
+	var i = 0
+	for {
+		_, err := os.Stat(fmt.Sprintf("/dev/nbd%d", i))
+		if !os.IsNotExist(err) {
+			m.nbdDevs[fmt.Sprintf("/dev/nbd%d", i)] = false
+			i++
+		} else {
+			break
+		}
+	}
+	log.Infof("NBD_DEVS: %#v", m.nbdDevs)
+}
+
+func (m *SNBDManager) AcquireNbddev() string {
+	defer m.nbdLock.Unlock()
+	m.nbdLock.Lock()
+	for nbdDev := range m.nbdDevs {
+		if hostman.IsBlockDeviceUsed(nbdDev) {
+			m.nbdDevs[nbdDev] = true
+		}
+		if !m.nbdDevs[nbdDev] {
+			m.nbdDevs[nbdDev] = true
+			return nbdDev
+		}
+	}
+	return ""
+}
+
+func (m *SNBDManager) ReleaseNbddev(nbddev string) {
+	if _, ok := m.nbdDevs[nbddev]; ok {
+		defer m.nbdLock.Unlock()
+		m.nbdLock.Lock()
+		m.nbdDevs[nbddev] = false
+	}
+}

pkg/hostman/utils.go

@@ -1,6 +1,7 @@
 package hostman
 
 import (
+	"bufio"
 	"fmt"
 	"io/ioutil"
 	"os"
@@ -12,6 +13,24 @@ import (
 	"yunion.io/x/log"
 )
 
+// timer utils
+
+func AddTimeout(second time.Duration, callback func()) {
+	go func() {
+		<-time.NewTimer(second).C
+		callback()
+	}()
+}
+
+func CommandWithTimeout(timeout int, cmds ...string) *exec.Cmd {
+	if timeout > 0 {
+		cmds = append([]string{"timeout", "--signal=KILL", fmt.Sprintf("%ds", timeout)}, cmds...)
+	}
+	return exec.Command(cmds[0], cmds[1:]...)
+}
+
+// file utils
+
 func FilePutContents(filename string, context string, modAppend bool) error {
 	var mode = os.O_WRONLY | os.O_CREATE
 	if modAppend {
@@ -26,14 +45,6 @@ func FilePutContents(filename string, context string, modAppend bool) error {
 	return err
 }
 
-func CommandWithTimeout(timeout int, cmds ...string) *exec.Cmd {
-	if timeout > 0 {
-		cmds = append([]string{"timeout", "--signal=KILL", fmt.Sprintf("%ds", timeout)}, cmds...)
-	}
-	return exec.Command(cmds[0], cmds[1:]...)
-
-}
-
 func IsBlockDevMounted(dev string) bool {
 	devPath := "/dev/" + dev
 	mounts, err := exec.Command("mount").Output()
@@ -48,6 +59,18 @@ func IsBlockDevMounted(dev string) bool {
 	return false
 }
 
+func IsBlockDeviceUsed(dev string) bool {
+	if strings.HasPrefix(dev, "/dev/") {
+		dev = dev[strings.LastIndex(dev, "/")+1:]
+	}
+	devStr := fmt.Sprint(" %s\n", dev)
+	devs, _ := exec.Command("cat", "/proc/partitions").Output()
+	if idx := strings.Index(string(devs), devStr); idx > 0 {
+		return false
+	}
+	return true
+}
+
 func ChangeAllBlkdevsParams(params map[string]string) {
 	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
 		blockDevs, err := ioutil.ReadDir("/sys/block")
@@ -76,15 +99,6 @@ func ChangeBlkdevParameter(dev, key, value string) {
 	}
 }
 
-// timer utils
-
-func AddTimeout(second time.Duration, callback func()) {
-	go func() {
-		<-time.NewTimer(second).C
-		callback()
-	}()
-}
-
 /*
 func PathNotExists(path string) bool {
     if _, err := os.Stat(path); os.IsNotExist(err) {
@@ -108,3 +122,38 @@ func FileGetContents(file string) (string, error) {
 	}
 	return string(content), nil
 }
+
+func GetFsFormat(diskPath string) string {
+	ret, err := exec.Command("blkid", "-o", "value", "-s", "TYPE", diskPath).Output()
+	if err != nil {
+		return ""
+	}
+	var res string
+	for _, line := range strings.Split(string(ret), "\n") {
+		res += line
+	}
+	return res
+}
+
+func CleanFailedMountpoints() {
+	var mtfile = "/etc/mtab"
+	if _, err := os.Stat(mtfile); os.IsNotExist(err) {
+		mtfile = "/proc/mounts"
+	}
+	f, err := os.Open(mtfile)
+	if err != nil {
+		log.Errorf("CleanFailedMountpoints error: %s", err)
+	}
+	reader := bufio.NewReader(f)
+	line, _, err := reader.ReadLine()
+	for err != nil {
+		m := strings.Split(string(line), " ")
+		if len(m) > 1 {
+			mp := m[1]
+			if _, err := os.Stat(mp); os.IsNotExist(err) {
+				log.Warningf("Mount point %s not exists", mp)
+			}
+			exec.Command("umount", mp).Run()
+		}
+	}
+}

pkg/hostman/utils_test.go

@@ -0,0 +1,32 @@
+package hostman
+
+import "testing"
+
+func TestIsBlockDeviceUsed(t *testing.T) {
+	type args struct {
+		dev string
+	}
+	tests := []struct {
+		name string
+		args args
+		want bool
+	}{
+		{
+			name: "nbd1",
+			args: {"/dev/nbd1"},
+			want: false,
+		},
+		{
+			name: "sda",
+			args: {"/dev/sda"},
+			want: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := IsBlockDeviceUsed(tt.args.dev); got != tt.want {
+				t.Errorf("IsBlockDeviceUsed() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}