feat(host-deployer): add deploy driver libguestfs

Add drvier libguestfs to support deploy more operator system,
you can use it by edit host config file add 'image_deploy_driver: libguestfs'.
By the default host-deploy support run three guestfish at the same time.
And host deployer will recycle free guestfish every ten minitues.

cherrypick: release/3.6

Makefile

@@ -199,7 +199,7 @@ mod:
 	go mod vendor -v
 
 
-DOCKER_CENTOS_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-1
+DOCKER_CENTOS_BUILD_IMAGE?=registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-2
 
 define dockerCentOSBuildCmd
 set -o xtrace

build/docker/Dockerfile.host-deployer

@@ -1,4 +1,4 @@
-FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:v0.5
+FROM registry.cn-beijing.aliyuncs.com/yunionio/host-deployer-base:0.6
 
 MAINTAINER "Yaoqi Wan wanyaoqi@yunionyun.com"
 

pkg/cloudcommon/options/options.go

@@ -116,6 +116,13 @@ type CommonOptions struct {
 	BaseOptions
 }
 
+type HostCommonOptions struct {
+	CommonOptions
+
+	ExecutorSocketPath     string `help:"Executor socket path" default:"/var/run/onecloud/exec.sock"`
+	DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"`
+}
+
 type DBOptions struct {
 	SqlConnection string `help:"SQL connection string" alias:"connection"`
 

pkg/esxi/options/options.go

@@ -33,7 +33,7 @@ type EsxiOptions struct {
 	WindowsDefaultAdminUser  bool   `help:"Default account for Windows system is Administrator" default:"true"`
 	DefaultImageSaveFormat   string `help:"Default image save format, default is vmdk, canbe qcow2" default:"vmdk"`
 	Zone                     string `help:"Zone where the agent locates"`
-	DeployServerSocketPath   string `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
+	DeployServerSocketPath   string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"`
 	HostDelayTaskWorkerCount int    `default:"8" help:"Host delay worker thread count, default is 8"`
 	esxi.EsxiOptions
 }

pkg/hostman/diskutils/fsutils/doc.go

@@ -0,0 +1 @@
+package fsutils // import "yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils"

pkg/hostman/diskutils/fsutils/fsutils.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package diskutils
+package fsutils
 
 import (
 	"fmt"

pkg/hostman/diskutils/interface.go

@@ -34,14 +34,26 @@ type DiskParams struct {
 	VddkInfo   *apis.VDDKConInfo
 }
 
-func GetIDisk(params DiskParams) IDisk {
+func GetIDisk(params DiskParams, driver string) IDisk {
 	hypervisor := params.Hypervisor
 	switch hypervisor {
 	case comapi.HYPERVISOR_KVM:
-		return NewKVMGuestDisk(params.DiskPath)
+		return NewKVMGuestDisk(params.DiskPath, driver)
 	case comapi.HYPERVISOR_ESXI:
-		return NewVDDKDisk(params.VddkInfo, params.DiskPath)
+		return NewVDDKDisk(params.VddkInfo, params.DiskPath, driver)
 	default:
-		return NewKVMGuestDisk(params.DiskPath)
+		return NewKVMGuestDisk(params.DiskPath, driver)
 	}
 }
+
+type IDeployer interface {
+	Connect() error
+	Disconnect() error
+
+	GetPartitions() []fsdriver.IDiskPartition
+	IsLVMPartition() bool
+	Zerofree()
+	ResizePartition() error
+	FormatPartition(fs, uuid string) error
+	MakePartition(fs string) error
+}

pkg/hostman/diskutils/kvm.go

@@ -0,0 +1,139 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package diskutils
+
+import (
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts"
+)
+
+type SKVMGuestDisk struct {
+	deployer IDeployer
+}
+
+func NewKVMGuestDisk(imagePath, driver string) *SKVMGuestDisk {
+	return &SKVMGuestDisk{
+		deployer: newDeployer(imagePath, driver),
+	}
+}
+
+func newDeployer(imagePath, driver string) IDeployer {
+	switch driver {
+	case consts.DEPLOY_DRIVER_NBD:
+		return nbd.NewNBDDriver(imagePath)
+	case consts.DEPLOY_DRIVER_LIBGUESTFS:
+		return libguestfs.NewLibguestfsDriver(imagePath)
+	default:
+		return nbd.NewNBDDriver(imagePath)
+	}
+
+}
+
+func (d *SKVMGuestDisk) IsLVMPartition() bool {
+	return d.deployer.IsLVMPartition()
+}
+
+func (d *SKVMGuestDisk) Connect() error {
+	return d.deployer.Connect()
+}
+
+func (d *SKVMGuestDisk) Disconnect() error {
+	return d.deployer.Disconnect()
+}
+
+func (d *SKVMGuestDisk) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool {
+	partitions := d.deployer.GetPartitions()
+	for i := 0; i < len(partitions); i++ {
+		if partitions[i].IsMounted() {
+			if rootfs.DetectIsUEFISupport(partitions[i]) {
+				return true
+			}
+		} else {
+			if partitions[i].Mount() {
+				support := rootfs.DetectIsUEFISupport(partitions[i])
+				partitions[i].Umount()
+				if support {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func (d *SKVMGuestDisk) MountRootfs() fsdriver.IRootFsDriver {
+	return d.MountKvmRootfs()
+}
+
+func (d *SKVMGuestDisk) MountKvmRootfs() fsdriver.IRootFsDriver {
+	return d.mountKvmRootfs(false)
+}
+func (d *SKVMGuestDisk) mountKvmRootfs(readonly bool) fsdriver.IRootFsDriver {
+	partitions := d.deployer.GetPartitions()
+	for i := 0; i < len(partitions); i++ {
+		mountFunc := partitions[i].Mount
+		if readonly {
+			mountFunc = partitions[i].MountPartReadOnly
+		}
+		if mountFunc() {
+			if fs := guestfs.DetectRootFs(partitions[i]); fs != nil {
+				log.Infof("Use rootfs %s, partition %s",
+					fs, partitions[i].GetPartDev())
+				return fs
+			} else {
+				partitions[i].Umount()
+			}
+		}
+	}
+	return nil
+}
+
+func (d *SKVMGuestDisk) MountKvmRootfsReadOnly() fsdriver.IRootFsDriver {
+	return d.mountKvmRootfs(true)
+}
+
+func (d *SKVMGuestDisk) UmountKvmRootfs(fd fsdriver.IRootFsDriver) {
+	if part := fd.GetPartition(); part != nil {
+		part.Umount()
+	}
+}
+
+func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) {
+	if fd == nil {
+		return
+	}
+	d.UmountKvmRootfs(fd)
+}
+
+func (d *SKVMGuestDisk) MakePartition(fs string) error {
+	return d.deployer.MakePartition(fs)
+}
+
+func (d *SKVMGuestDisk) FormatPartition(fs, uuid string) error {
+	return d.deployer.FormatPartition(fs, uuid)
+}
+
+func (d *SKVMGuestDisk) ResizePartition() error {
+	return d.deployer.ResizePartition()
+}
+
+func (d *SKVMGuestDisk) Zerofree() {
+	d.deployer.Zerofree()
+}

pkg/hostman/diskutils/libguestfs/doc.go

@@ -0,0 +1 @@
+package libguestfs // import "yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs"

pkg/hostman/diskutils/libguestfs/driver.go

@@ -0,0 +1,246 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package libguestfs
+
+import (
+	"fmt"
+	"io/ioutil"
+	"math/rand"
+	"path"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/sortedmap"
+
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs/guestfish"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/guestfishpart"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+)
+
+const (
+	DiskLabelLength = 6
+	letterBytes     = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+)
+
+func RandStringBytes(n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}
+
+func init() {
+	rand.Seed(time.Now().UnixNano())
+}
+
+type SLibguestfsDriver struct {
+	imagePath string
+	nbddev    string
+	diskLabel string
+	lvmParts  []string
+	fsmap     *sortedmap.SSortedMap
+	fish      *guestfish.Guestfish
+	device    string
+
+	parts []fsdriver.IDiskPartition
+}
+
+func NewLibguestfsDriver(imagePath string) *SLibguestfsDriver {
+	return &SLibguestfsDriver{
+		imagePath: imagePath,
+	}
+}
+
+func (d *SLibguestfsDriver) Connect() error {
+	fish, err := guestfsManager.AcquireFish()
+	if err != nil {
+		return err
+	}
+	d.fish = fish
+
+	d.nbddev = nbd.GetNBDManager().AcquireNbddev()
+	if err != nil {
+		return errors.Errorf("Cannot get nbd device")
+	}
+	log.Debugf("acquired device %s", d.nbddev)
+
+	err = nbd.QemuNbdConnect(d.imagePath, d.nbddev)
+	if err != nil {
+		return err
+	}
+
+	lable := RandStringBytes(DiskLabelLength)
+	err = fish.AddDrive(d.nbddev, lable, false)
+	if err != nil {
+		return err
+	}
+	d.diskLabel = lable
+
+	if err = fish.LvmClearFilter(); err != nil {
+		return err
+	}
+
+	devices, err := fish.ListDevices()
+	if err != nil {
+		return err
+	}
+	if len(devices) == 0 {
+		return errors.Errorf("fish list devices no device found")
+	}
+	d.device = devices[0]
+
+	fsmap, err := fish.ListFilesystems()
+	if err != nil {
+		return err
+	}
+	d.fsmap = fsmap
+	log.Debugf("fsmap output %#v", d.fsmap)
+
+	lvs, err := fish.Lvs()
+	if err != nil {
+		return err
+	}
+	d.lvmParts = lvs
+
+	keys := d.fsmap.Keys()
+	for i := 0; i < len(keys); i++ {
+		partDev := keys[i]
+		ifs, _ := d.fsmap.Get(keys[i])
+		fs := ifs.(string)
+		log.Debugf("new partition %s %s %s", d.device, partDev, fs)
+
+		/* guestfish run ntfs mount to host is too slow
+		 * use host nbd partition replace */
+		if fs == "ntfs" && len(d.lvmParts) == 0 {
+			log.Infof("has ntfs, use nbd parts")
+			d.parts, err = d.findNbdPartitions()
+			if err != nil {
+				return err
+			}
+			if err = guestfsManager.ReleaseFish(d.fish); err != nil {
+				log.Errorf("release fish failed %s", err)
+			}
+			d.diskLabel = ""
+			d.fish = nil
+			break
+		}
+		part := guestfishpart.NewGuestfishDiskPartition(d.device, partDev, fs, fish)
+		d.parts = append(d.parts, part)
+	}
+
+	return nil
+}
+
+func (d *SLibguestfsDriver) findNbdPartitions() ([]fsdriver.IDiskPartition, error) {
+	if len(d.nbddev) == 0 {
+		return nil, fmt.Errorf("Want find partitions but dosen't have nbd dev")
+	}
+	dev := filepath.Base(d.nbddev)
+	devpath := filepath.Dir(d.nbddev)
+	files, err := ioutil.ReadDir(devpath)
+	if err != nil {
+		return nil, errors.Wrapf(err, "read dir %s", devpath)
+	}
+
+	parts := make([]fsdriver.IDiskPartition, 0)
+	for i := 0; i < len(files); i++ {
+		if files[i].Name() != dev && strings.HasPrefix(files[i].Name(), dev+"p") {
+			var part = kvmpart.NewKVMGuestDiskPartition(path.Join(devpath, files[i].Name()), "", false)
+			parts = append(parts, part)
+		}
+	}
+	return parts, nil
+}
+
+func (d *SLibguestfsDriver) Disconnect() error {
+	if len(d.diskLabel) > 0 {
+		if err := guestfsManager.ReleaseFish(d.fish); err != nil {
+			log.Errorf("release fish failed %s", err)
+		}
+		d.diskLabel = ""
+		d.fish = nil
+	}
+	if len(d.nbddev) > 0 {
+		if err := nbd.QemuNbdDisconnect(d.nbddev); err != nil {
+			return err
+		}
+		nbd.GetNBDManager().ReleaseNbddev(d.nbddev)
+	}
+	return nil
+}
+
+func (d *SLibguestfsDriver) GetPartitions() []fsdriver.IDiskPartition {
+	return d.parts
+}
+
+func (d *SLibguestfsDriver) IsLVMPartition() bool {
+	return len(d.lvmParts) > 0
+}
+
+func (d *SLibguestfsDriver) Zerofree() {
+	startTime := time.Now()
+	for _, part := range d.parts {
+		part.Zerofree()
+	}
+	log.Infof("libguestfs zerofree %d partitions takes %f seconds",
+		len(d.parts), time.Now().Sub(startTime).Seconds())
+}
+
+func (d *SLibguestfsDriver) ResizePartition() error {
+	return fsutils.ResizeDiskFs(d.nbddev, 0)
+}
+
+func (d *SLibguestfsDriver) FormatPartition(fs, uuid string) error {
+	return fsutils.FormatPartition(fmt.Sprintf("%sp1", d.nbddev), fs, uuid)
+}
+
+func (d *SLibguestfsDriver) MakePartition(fsFormat string) error {
+	return fsutils.Mkpartition(d.nbddev, fsFormat)
+}
+
+func (d *SLibguestfsDriver) FormatPartition2(fs, uuid string) error {
+	partDev := fmt.Sprintf("%s1", d.device)
+	switch fs {
+	case "swap":
+		return d.fish.Mkswap(partDev, uuid, "")
+	case "ext2", "ext3", "ext4", "xfs", "fat":
+		return d.fish.Mkfs(partDev, fs)
+	}
+	return errors.Errorf("Unknown fs %s", fs)
+}
+
+func (d *SLibguestfsDriver) MakePartition2(fsFormat string) error {
+	var (
+		labelType = "gpt"
+		diskType  = fileutils2.FsFormatToDiskType(fsFormat)
+	)
+	if len(diskType) == 0 {
+		return errors.Errorf("Unknown fsFormat %s", fsFormat)
+	}
+
+	err := d.fish.PartDisk(d.device, labelType)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/hostman/diskutils/libguestfs/guestfish/doc.go

@@ -0,0 +1 @@
+package guestfish // import "yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs/guestfish"

pkg/hostman/diskutils/libguestfs/guestfish/guestfish.go

@@ -0,0 +1,341 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package guestfish
+
+import (
+	"bufio"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"runtime/debug"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/kr/pty"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/sortedmap"
+)
+
+type Guestfish struct {
+	*exec.Cmd
+
+	stdoutScanner *bufio.Scanner
+	stderrScanner *bufio.Scanner
+
+	pty *os.File
+
+	lock  sync.Mutex
+	label string
+
+	stdout  *os.File
+	stdoutW *os.File
+	stderr  *os.File
+	stderrW *os.File
+
+	alive bool
+}
+
+const guestFishToken = "><fs>"
+
+func NewGuestfish() (*Guestfish, error) {
+	gf := &Guestfish{Cmd: exec.Command("guestfish")}
+	stdout, stdoutW, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+	stderr, stderrW, err := os.Pipe()
+	if err != nil {
+		return nil, err
+	}
+
+	gf.stdout = stdout
+	gf.stdoutW = stdoutW
+
+	gf.stderr = stderr
+	gf.stderrW = stderrW
+
+	gf.Cmd.Stdout = stdoutW
+	gf.Cmd.Stderr = stderrW
+
+	gf.stdoutScanner = bufio.NewScanner(stdout)
+	gf.stderrScanner = bufio.NewScanner(stderr)
+
+	pty, err := pty.Start(gf.Cmd)
+	if err != nil {
+		return nil, errors.Wrap(err, "start Guestfish")
+	}
+
+	/* pty handle stdin */
+	gf.pty = pty
+
+	/* exec guestfish run command */
+	if err = gf.Run(); err != nil {
+		return nil, err
+	}
+
+	gf.alive = true
+	return gf, nil
+}
+
+func (fish *Guestfish) IsAlive() bool {
+	return fish.alive
+}
+
+func (fish *Guestfish) execute(cmd string) ([]string, error) {
+	fish.lock.Lock()
+	defer fish.lock.Unlock()
+	log.Debugf("exec command: %s", cmd)
+	_, err := fish.pty.WriteString(cmd + "\n\n")
+	if err != nil {
+		return nil, errors.Wrapf(err, "exec cmd %s", cmd)
+	}
+	return fish.fetch()
+}
+
+func (fish *Guestfish) fetch() ([]string, error) {
+	var (
+		stdout = make([]string, 0)
+	)
+
+	var tokenMeeted = false
+	for fish.stdoutScanner.Scan() {
+		line := fish.stdoutScanner.Text()
+		log.Debugf("Guestfish stdoutScanner: %s", line)
+		if strings.HasPrefix(line, guestFishToken) {
+			if !tokenMeeted {
+				tokenMeeted = true
+				continue
+			}
+			log.Debugf("fetch success")
+			break
+		}
+		stdout = append(stdout, line)
+	}
+	if err := fish.stdoutScanner.Err(); err != nil {
+		log.Errorf("scan guestfish stdoutScanner error %s", err)
+		fish.Quit()
+		return nil, err
+	}
+
+	fish.stderr.SetReadDeadline(time.Now().Add(time.Second * 1))
+	output, err := ioutil.ReadAll(fish.stderr)
+	if err != nil && !strings.Contains(err.Error(), "i/o timeout") {
+		log.Errorf("scan guestfish stderrScanner error %s", err)
+		fish.Quit()
+		return nil, err
+	}
+	var stderrErr error
+	if len(output) > 0 {
+		stderrErr = errors.Errorf(string(output))
+	}
+	return stdout, stderrErr
+}
+
+/* Fetch error message from stderrScanner, until got ><fs> from stdoutScanner */
+func (fish *Guestfish) fetchError() error {
+	_, err := fish.fetch()
+	return err
+}
+
+func (fish *Guestfish) Run() error {
+	_, err := fish.execute("run")
+	return err
+}
+
+func (fish *Guestfish) Quit() error {
+	checkError := func(err error) {
+		if err != nil {
+			log.Errorln(err)
+		}
+	}
+
+	defer func() {
+		if r := recover(); r != nil {
+			log.Errorf("fish quit faild %s \nstack: %s", r, debug.Stack())
+		}
+	}()
+
+	fish.lock.Lock()
+	defer fish.lock.Unlock()
+	log.Debugf("exec command: %s", "quit")
+	_, err := fish.pty.WriteString("quit\n\n")
+	if err != nil {
+		return errors.Wrap(err, "exec cmd kill-subprocess and quit")
+	}
+
+	fish.alive = false
+	if err := fish.Cmd.Wait(); err != nil {
+		log.Errorf("failed wait guestfish %s", err)
+	}
+
+	checkError(fish.stdout.Close())
+	checkError(fish.stderr.Close())
+	checkError(fish.stdoutW.Close())
+	checkError(fish.stderrW.Close())
+	checkError(fish.pty.Close())
+	return err
+}
+
+func (fish *Guestfish) AddDrive(path, label string, readonly bool) error {
+	cmd := fmt.Sprintf("add-drive %s label:%s", path, label)
+	if readonly {
+		cmd += " readonly:true"
+	}
+	_, err := fish.execute(cmd)
+	if err != nil {
+		return err
+	}
+	fish.label = label
+	return nil
+}
+
+func (fish *Guestfish) RemoveDrive() error {
+	if len(fish.label) == 0 {
+		return errors.Errorf("no drive add")
+	}
+	_, err := fish.execute(fmt.Sprintf("remove-drive %s", fish.label))
+	if err != nil {
+		return err
+	}
+	fish.label = ""
+	return err
+}
+
+func (fish *Guestfish) ListFilesystems() (*sortedmap.SSortedMap, error) {
+	output, err := fish.execute("list-filesystems")
+	if err != nil {
+		return nil, err
+	}
+	return fish.parseListFilesystemsOutput(output), nil
+}
+
+func (fish *Guestfish) parseListFilesystemsOutput(output []string) *sortedmap.SSortedMap {
+	/* /dev/sda1: xfs
+	   /dev/centos/root: xfs
+	   /dev/centos/swap: swap */
+	res := sortedmap.SSortedMap{}
+	for i := 0; i < len(output); i++ {
+		line := output[i]
+		log.Debugf("line %s", line)
+		segs := strings.Split(line, ":")
+		log.Debugf("parse line of list filesystems: %#v", segs)
+		if len(segs) != 2 {
+			log.Warningf("Guestfish: parse list filesystem got unwanted line: %s", line)
+		}
+		res = sortedmap.Add(res, strings.TrimSpace(segs[0]), strings.TrimSpace(segs[1]))
+	}
+	return &res
+}
+
+func (fish *Guestfish) ListDevices() ([]string, error) {
+	return fish.execute("list-devices")
+}
+
+func (fish *Guestfish) Mount(partition string) error {
+	_, err := fish.execute(fmt.Sprintf("mount %s /", partition))
+	return err
+}
+
+func (fish *Guestfish) MountLocal(localmountpoint string, readonly bool) error {
+	cmd := fmt.Sprintf("mount-local %s", localmountpoint)
+	if readonly {
+		cmd += " readonly:true"
+	}
+	_, err := fish.execute(cmd)
+	return err
+}
+
+func (fish *Guestfish) Umount(partition string) error {
+	_, err := fish.execute("umount")
+	return err
+}
+
+func (fish *Guestfish) UmountLocal() error {
+	_, err := fish.execute("umount-local")
+	return err
+}
+
+/* This should only be called after "mount_local" returns successfully.
+ * The call will not return until the filesystem is unmounted. */
+func (fish *Guestfish) MountLocalRun() error {
+	_, err := fish.execute("mount-local-run")
+	return err
+}
+
+/* Clears the LVM cache and performs a volume group scan. */
+func (fish *Guestfish) LvmClearFilter() error {
+	_, err := fish.execute("lvm-clear-filter")
+	return err
+}
+
+func (fish *Guestfish) Lvs() ([]string, error) {
+	return fish.execute("lvs")
+}
+
+func (fish *Guestfish) SfdiskL(dev string) ([]string, error) {
+	return fish.execute(fmt.Sprintf("sfdisk-l %s", dev))
+}
+
+func (fish *Guestfish) Fsck(dev, fs string) error {
+	out, err := fish.execute(fmt.Sprintf("fsck %s %s", fs, dev))
+	log.Infof("FSCK ret code: %v", out)
+	return err
+}
+
+func (fish *Guestfish) Ntfsfix(dev string) error {
+	out, err := fish.execute(fmt.Sprintf("ntfsfix %s", dev))
+	log.Infof("NTFSFIX ret code: %v", out)
+	return err
+}
+
+func (fish *Guestfish) Zerofree(dev string) error {
+	_, err := fish.execute(fmt.Sprintf("zerofree %s", dev))
+	return err
+}
+
+func (fish *Guestfish) ZeroFreeSpace(dir string) error {
+	_, err := fish.execute(fmt.Sprintf("zero-free-space %s", dir))
+	return err
+}
+
+func (fish *Guestfish) Blkid(partDev string) ([]string, error) {
+	return fish.execute(fmt.Sprintf("blkid %s", partDev))
+}
+
+func (fish *Guestfish) Mkswap(partDev, uuid, label string) error {
+	cmd := fmt.Sprintf("mkswap %s", partDev)
+	if len(uuid) > 0 {
+		cmd += fmt.Sprintf(" uuid:%s", uuid)
+	}
+	if len(label) > 0 {
+		cmd += fmt.Sprintf(" label:%s", label)
+	}
+	_, err := fish.execute(cmd)
+	return err
+}
+
+func (fish *Guestfish) Mkfs(dev, fs string) error {
+	_, err := fish.execute(fmt.Sprintf("mkfs %s %s", fs, dev))
+	return err
+}
+
+func (fish *Guestfish) PartDisk(dev, diskType string) error {
+	_, err := fish.execute(fmt.Sprintf("part-disk %s %s", dev, diskType))
+	return err
+}

pkg/hostman/diskutils/libguestfs/manager.go

@@ -0,0 +1,198 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package libguestfs
+
+import (
+	"sync"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs/guestfish"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type ErrorFish string
+
+func (e ErrorFish) Error() string {
+	return string(e)
+}
+
+const (
+	ErrFishsMismatch = ErrorFish("fishs mismatch")
+	ErrFishsWorking  = ErrorFish("fishs working")
+	ErrFishsDied     = ErrorFish("fishs died")
+)
+
+var guestfsManager *GuestfsManager
+
+func Init(count int) error {
+	if guestfsManager == nil {
+		if err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", "dm-mod").Run(); err != nil {
+			return errors.Wrap(err, "modprobe dm-mod")
+		}
+		log.Infof("guestfish count %v", count)
+		guestfsManager = NewGuestfsManager(count)
+		time.AfterFunc(time.Minute*3, guestfsManager.fishsRecycle)
+	}
+	return nil
+}
+
+type GuestfsManager struct {
+	fishMaximum      int
+	happyFishCount   int
+	workingFishCount int
+
+	fishs    map[*guestfish.Guestfish]bool
+	fishChan chan *guestfish.Guestfish
+	fishlock sync.Mutex
+
+	lastTimeFishing time.Time
+}
+
+func NewGuestfsManager(count int) *GuestfsManager {
+	if count < 1 {
+		count = 1
+	}
+	return &GuestfsManager{
+		fishMaximum: count,
+		fishs:       make(map[*guestfish.Guestfish]bool, count),
+		fishChan:    make(chan *guestfish.Guestfish, count),
+	}
+}
+
+func (m *GuestfsManager) AcquireFish() (*guestfish.Guestfish, error) {
+	fisher := func() (*guestfish.Guestfish, error) {
+		m.fishlock.Lock()
+		defer m.fishlock.Unlock()
+
+		m.lastTimeFishing = time.Now()
+		fish, err := m.acquireFish()
+		if err == ErrFishsWorking {
+			return nil, err
+		} else if err != nil {
+			return nil, err
+		}
+		if !fish.IsAlive() {
+			go func() {
+				if e := m.ReleaseFish(fish); e != nil {
+					log.Errorf("release fish failed %s", e)
+				}
+			}()
+			return nil, nil
+		}
+		return fish, nil
+	}
+	for i := 0; i < 3; i++ {
+		fish, err := fisher()
+		if err == ErrFishsWorking {
+			return m.waitingFishFinish(), nil
+		}
+		if err != nil {
+			return nil, err
+		}
+		if fish != nil {
+			return fish, nil
+		}
+	}
+	return nil, ErrFishsDied
+}
+
+func (m *GuestfsManager) acquireFish() (*guestfish.Guestfish, error) {
+	if m.happyFishCount == 0 {
+		if m.workingFishCount < m.fishMaximum {
+			fish, err := guestfish.NewGuestfish()
+			if err != nil {
+				return nil, err
+			}
+			m.fishs[fish] = true
+			m.workingFishCount++
+			return fish, nil
+		} else {
+			return nil, ErrFishsWorking
+		}
+	} else {
+		for fish, working := range m.fishs {
+			if !working {
+				m.fishs[fish] = true
+				m.workingFishCount++
+				m.happyFishCount--
+				return fish, nil
+			}
+		}
+		return nil, ErrFishsMismatch
+	}
+}
+
+func (m *GuestfsManager) waitingFishFinish() *guestfish.Guestfish {
+	select {
+	case fish := <-m.fishChan:
+		return fish
+	}
+}
+
+func (m *GuestfsManager) ReleaseFish(fish *guestfish.Guestfish) error {
+	err := m.washfish(fish)
+	m.fishlock.Lock()
+	defer m.fishlock.Unlock()
+	if err != nil {
+		errQ := fish.Quit()
+		if errQ != nil {
+			log.Errorf("fish quit failed: %s", errQ)
+		}
+		delete(m.fishs, fish)
+		m.workingFishCount--
+	}
+	m.fishChan <- fish
+	return err
+}
+
+func (m *GuestfsManager) washfish(fish *guestfish.Guestfish) error {
+	return fish.RemoveDrive()
+}
+
+func (m *GuestfsManager) fishsRecycle() {
+	defer time.AfterFunc(time.Minute*10, m.fishsRecycle)
+	m.fishlock.Lock()
+	defer m.fishlock.Unlock()
+	if m.lastTimeFishing.IsZero() || time.Now().Sub(m.lastTimeFishing) < time.Minute*10 {
+		return
+	}
+	log.Infof("start release fishs ...")
+
+Loop:
+	for {
+		select {
+		case fish := <-m.fishChan:
+			m.fishs[fish] = false
+			m.happyFishCount++
+			m.workingFishCount--
+		default:
+			break Loop
+		}
+	}
+
+	for fish, working := range m.fishs {
+		if !working {
+			err := fish.Quit()
+			if err != nil {
+				log.Errorf("fish quit failed: %s", err)
+			}
+			m.happyFishCount--
+			delete(m.fishs, fish)
+		}
+	}
+}

pkg/hostman/diskutils/nbd/driver.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package diskutils
+package nbd
 
 import (
 	"fmt"
@@ -26,9 +26,9 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
-	"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
-	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
 	"yunion.io/x/onecloud/pkg/util/qemutils"
@@ -36,86 +36,53 @@ import (
 
 const MAX_TRIES = 3
 
+type NBDDriver struct {
+	partitions            []fsdriver.IDiskPartition
+	lvms                  []*SKVMGuestLVMPartition
+	imageRootBackFilePath string
+	imagePath             string
+	acquiredLvm           bool
+	nbdDev                string
+}
+
+func NewNBDDriver(imagePath string) *NBDDriver {
+	return &NBDDriver{
+		imagePath:  imagePath,
+		partitions: make([]fsdriver.IDiskPartition, 0),
+	}
+}
+
 var lvmTool *SLVMImageConnectUniqueToolSet
 
 func init() {
 	lvmTool = NewLVMImageConnectUniqueToolSet()
 }
 
-type SKVMGuestDisk struct {
-	imagePath   string
-	nbdDev      string
-	partitions  []*guestfs.SKVMGuestDiskPartition
-	lvms        []*SKVMGuestLVMPartition
-	acquiredLvm bool
+func (d *NBDDriver) Connect() error {
+	pathType := lvmTool.GetPathType(d.rootImagePath())
+	if pathType == LVM_PATH || pathType == PATH_TYPE_UNKNOWN {
+		lvmTool.Acquire(d.rootImagePath())
+		d.acquiredLvm = true
+	}
 
-	imageRootBackFilePath string
-}
-
-func NewKVMGuestDisk(imagePath string) *SKVMGuestDisk {
-	var ret = new(SKVMGuestDisk)
-	ret.imagePath = imagePath
-	ret.partitions = make([]*guestfs.SKVMGuestDiskPartition, 0)
-	return ret
-}
-
-func (d *SKVMGuestDisk) IsLVMPartition() bool {
-	return len(d.lvms) > 0
-}
-
-func (d *SKVMGuestDisk) connect() error {
-	d.nbdDev = nbd.GetNBDManager().AcquireNbddev()
+	d.nbdDev = GetNBDManager().AcquireNbddev()
 	if len(d.nbdDev) == 0 {
 		return errors.Errorf("Cannot get nbd device")
 	}
-
-	var cmd []string
-	if strings.HasPrefix(d.imagePath, "rbd:") || d.getImageFormat() == "raw" {
-		//qemu-nbd 连接ceph时 /etc/ceph/ceph.conf 必须存在
-		if strings.HasPrefix(d.imagePath, "rbd:") {
-			err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", "/etc/ceph").Run()
-			if err != nil {
-				log.Errorf("Failed to mkdir /etc/ceph: %s", err)
-				return errors.Wrap(err, "Failed to mkdir /etc/ceph: %s")
-			}
-			err = procutils.NewRemoteCommandAsFarAsPossible("test", "-f", "/etc/ceph/ceph.conf").Run()
-			if err != nil {
-				err = procutils.NewRemoteCommandAsFarAsPossible("touch", "/etc/ceph/ceph.conf").Run()
-				if err != nil {
-					log.Errorf("failed to create /etc/ceph/ceph.conf: %s", err)
-					return errors.Wrap(err, "failed to create /etc/ceph/ceph.conf")
-				}
-			}
-		}
-		cmd = []string{qemutils.GetQemuNbd(), "-c", d.nbdDev, "-f", "raw", d.imagePath}
-	} else {
-		cmd = []string{qemutils.GetQemuNbd(), "-c", d.nbdDev, d.imagePath}
-	}
-	output, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output()
-	if err != nil {
-		log.Errorf("qemu-nbd connect failed %s %s", output, err.Error())
-		return errors.Wrapf(err, "qemu-nbd connect failed %s", output)
+	if err := QemuNbdConnect(d.imagePath, d.nbdDev); err != nil {
+		return err
 	}
 
 	var tried uint = 0
 	for len(d.partitions) == 0 && tried < MAX_TRIES {
 		time.Sleep((1 << tried) * time.Second)
-		err = d.findPartitions()
+		err := d.findPartitions()
 		if err != nil {
 			log.Errorln(err.Error())
 			return err
 		}
 		tried += 1
 	}
-	return nil
-}
-
-func (d *SKVMGuestDisk) Connect() error {
-	pathType := d.connectionPrecheck()
-
-	if err := d.connect(); err != nil {
-		return errors.Wrap(err, "disk.connect")
-	}
 
 	if pathType == LVM_PATH {
 		if _, err := d.setupLVMS(); err != nil {
@@ -132,25 +99,10 @@ func (d *SKVMGuestDisk) Connect() error {
 			d.cacheNonLVMImagePath()
 		}
 	}
-
 	return nil
 }
 
-func (d *SKVMGuestDisk) getImageFormat() string {
-	lines, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuImg(), "info", d.imagePath).Output()
-	if err != nil {
-		return ""
-	}
-	imgStr := strings.Split(string(lines), "\n")
-	for i := 0; i < len(imgStr); i++ {
-		if strings.HasPrefix(imgStr[i], "file format: ") {
-			return imgStr[i][len("file format: "):]
-		}
-	}
-	return ""
-}
-
-func (d *SKVMGuestDisk) findPartitions() error {
+func (d *NBDDriver) findPartitions() error {
 	if len(d.nbdDev) == 0 {
 		return fmt.Errorf("Want find partitions but dosen't have nbd dev")
 	}
@@ -162,23 +114,15 @@ func (d *SKVMGuestDisk) findPartitions() error {
 	}
 	for i := 0; i < len(files); i++ {
 		if files[i].Name() != dev && strings.HasPrefix(files[i].Name(), dev+"p") {
-			var part = guestfs.NewKVMGuestDiskPartition(path.Join(devpath, files[i].Name()), "", false)
+			var part = kvmpart.NewKVMGuestDiskPartition(path.Join(devpath, files[i].Name()), "", false)
 			d.partitions = append(d.partitions, part)
 		}
 	}
 
-	// XXX: HACK reverse partitions
-	// for i, j := 0, len(d.partitions)-1; i < j; i, j = i+1, j-1 {
-	// 	d.partitions[i], d.partitions[j] = d.partitions[j], d.partitions[i]
-	// }
 	return nil
 }
 
-func (d *SKVMGuestDisk) findLVMPartitions(partDev string) string {
-	return findVgname(partDev)
-}
-
-func (d *SKVMGuestDisk) rootImagePath() string {
+func (d *NBDDriver) rootImagePath() string {
 	if len(d.imageRootBackFilePath) > 0 {
 		return d.imageRootBackFilePath
 	}
@@ -199,37 +143,16 @@ func (d *SKVMGuestDisk) rootImagePath() string {
 	return d.imageRootBackFilePath
 }
 
-func (d *SKVMGuestDisk) isNonLvmImagePath() bool {
+func (d *NBDDriver) isNonLvmImagePath() bool {
 	pathType := lvmTool.GetPathType(d.rootImagePath())
 	return pathType == NON_LVM_PATH
 }
 
-func (d *SKVMGuestDisk) cacheNonLVMImagePath() {
+func (d *NBDDriver) cacheNonLVMImagePath() {
 	lvmTool.CacheNonLvmImagePath(d.rootImagePath())
 }
 
-func (d *SKVMGuestDisk) connectionPrecheck() int {
-	pathType := lvmTool.GetPathType(d.rootImagePath())
-	if pathType == LVM_PATH || pathType == PATH_TYPE_UNKNOWN {
-		lvmTool.Acquire(d.rootImagePath())
-		d.acquiredLvm = true
-	}
-	return pathType
-}
-
-func (d *SKVMGuestDisk) LvmDisconnectNotify() {
-	defer func() {
-		if r := recover(); r != nil {
-			log.Errorf("Catch panic on LvmDisconnectNotify %v \n %s", r, debug.Stack())
-		}
-	}()
-	pathType := lvmTool.GetPathType(d.rootImagePath())
-	if d.acquiredLvm || pathType != NON_LVM_PATH {
-		lvmTool.Release(d.rootImagePath())
-	}
-}
-
-func (d *SKVMGuestDisk) setupLVMS() (bool, error) {
+func (d *NBDDriver) setupLVMS() (bool, error) {
 	// Scan all devices and send the metadata to lvmetad
 	output, err := procutils.NewCommand("pvscan", "--cache").Output()
 	if err != nil {
@@ -237,7 +160,7 @@ func (d *SKVMGuestDisk) setupLVMS() (bool, error) {
 		return false, err
 	}
 
-	lvmPartitions := []*guestfs.SKVMGuestDiskPartition{}
+	lvmPartitions := []fsdriver.IDiskPartition{}
 	for _, part := range d.partitions {
 		vgname := d.findLVMPartitions(part.GetPartDev())
 		if len(vgname) > 0 {
@@ -245,7 +168,9 @@ func (d *SKVMGuestDisk) setupLVMS() (bool, error) {
 			d.lvms = append(d.lvms, lvm)
 			if lvm.SetupDevice() {
 				if subparts := lvm.FindPartitions(); len(subparts) > 0 {
-					lvmPartitions = append(lvmPartitions, subparts...)
+					for i := 0; i < len(subparts); i++ {
+						lvmPartitions = append(lvmPartitions, subparts[i])
+					}
 				}
 			}
 		}
@@ -259,114 +184,126 @@ func (d *SKVMGuestDisk) setupLVMS() (bool, error) {
 	}
 }
 
-func (d *SKVMGuestDisk) PutdownLVMs() {
-	for _, lvm := range d.lvms {
-		lvm.PutdownDevice()
-	}
-	d.lvms = []*SKVMGuestLVMPartition{}
+func (d *NBDDriver) findLVMPartitions(partDev string) string {
+	return findVgname(partDev)
 }
 
-func (d *SKVMGuestDisk) Disconnect() error {
+func (d *NBDDriver) Disconnect() error {
 	if len(d.nbdDev) > 0 {
-		defer d.LvmDisconnectNotify()
-		d.PutdownLVMs()
+		defer d.lvmDisconnectNotify()
+		d.putdownLVMs()
 		return d.disconnect()
 	} else {
 		return nil
 	}
 }
 
-func (d *SKVMGuestDisk) disconnect() error {
-	output, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuNbd(), "-d", d.nbdDev).Output()
-	if err != nil {
-		log.Errorln(err.Error())
-		return errors.Wrapf(err, "qemu-nbd disconnect %s", output)
+func (d *NBDDriver) lvmDisconnectNotify() {
+	defer func() {
+		if r := recover(); r != nil {
+			log.Errorf("Catch panic on LvmDisconnectNotify %v \n %s", r, debug.Stack())
+		}
+	}()
+	pathType := lvmTool.GetPathType(d.rootImagePath())
+	if d.acquiredLvm || pathType != NON_LVM_PATH {
+		lvmTool.Release(d.rootImagePath())
 	}
-	nbd.GetNBDManager().ReleaseNbddev(d.nbdDev)
+}
+
+func (d *NBDDriver) disconnect() error {
+	if err := QemuNbdDisconnect(d.nbdDev); err != nil {
+		return err
+	}
+	GetNBDManager().ReleaseNbddev(d.nbdDev)
 	d.nbdDev = ""
 	d.partitions = d.partitions[len(d.partitions):]
 	return nil
-
 }
 
-func (d *SKVMGuestDisk) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool {
-	for i := 0; i < len(d.partitions); i++ {
-		if d.partitions[i].IsMounted() {
-			if rootfs.DetectIsUEFISupport(d.partitions[i]) {
-				return true
-			}
-		} else {
-			if d.partitions[i].Mount() {
-				support := rootfs.DetectIsUEFISupport(d.partitions[i])
-				d.partitions[i].Umount()
-				if support {
-					return true
-				}
-			}
-		}
+func (d *NBDDriver) putdownLVMs() {
+	for _, lvm := range d.lvms {
+		lvm.PutdownDevice()
 	}
-	return false
+	d.lvms = []*SKVMGuestLVMPartition{}
 }
 
-func (d *SKVMGuestDisk) MountRootfs() fsdriver.IRootFsDriver {
-	return d.MountKvmRootfs()
+func (d *NBDDriver) GetPartitions() []fsdriver.IDiskPartition {
+	return d.partitions
 }
 
-func (d *SKVMGuestDisk) MountKvmRootfs() fsdriver.IRootFsDriver {
-	return d.mountKvmRootfs(false)
-}
-func (d *SKVMGuestDisk) mountKvmRootfs(readonly bool) fsdriver.IRootFsDriver {
-	for i := 0; i < len(d.partitions); i++ {
-		mountFunc := d.partitions[i].Mount
-		if readonly {
-			mountFunc = d.partitions[i].MountPartReadOnly
-		}
-		if mountFunc() {
-			if fs := guestfs.DetectRootFs(d.partitions[i]); fs != nil {
-				log.Infof("Use rootfs %s, partition %s",
-					fs, d.partitions[i].GetPartDev())
-				return fs
-			} else {
-				d.partitions[i].Umount()
-			}
-		}
-	}
-	return nil
+func (d *NBDDriver) MakePartition(fs string) error {
+	return fsutils.Mkpartition(d.nbdDev, fs)
 }
 
-func (d *SKVMGuestDisk) MountKvmRootfsReadOnly() fsdriver.IRootFsDriver {
-	return d.mountKvmRootfs(true)
+func (d *NBDDriver) FormatPartition(fs, uuid string) error {
+	return fsutils.FormatPartition(fmt.Sprintf("%sp1", d.nbdDev), fs, uuid)
 }
 
-func (d *SKVMGuestDisk) UmountKvmRootfs(fd fsdriver.IRootFsDriver) {
-	if part := fd.GetPartition(); part != nil {
-		part.Umount()
-	}
+func (d *NBDDriver) ResizePartition() error {
+	return fsutils.ResizeDiskFs(d.nbdDev, 0)
 }
 
-func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) {
-	if fd == nil {
-		return
-	}
-	d.UmountKvmRootfs(fd)
-}
-
-func (d *SKVMGuestDisk) MakePartition(fs string) error {
-	return Mkpartition(d.nbdDev, fs)
-}
-
-func (d *SKVMGuestDisk) FormatPartition(fs, uuid string) error {
-	return FormatPartition(fmt.Sprintf("%sp1", d.nbdDev), fs, uuid)
-}
-
-func (d *SKVMGuestDisk) ResizePartition() error {
-	return ResizeDiskFs(d.nbdDev, 0)
-}
-
-func (d *SKVMGuestDisk) Zerofree() {
+func (d *NBDDriver) Zerofree() {
 	startTime := time.Now()
 	for _, part := range d.partitions {
 		part.Zerofree()
 	}
 	log.Infof("Zerofree %d partitions takes %f seconds", len(d.partitions), time.Now().Sub(startTime).Seconds())
 }
+
+func (d *NBDDriver) IsLVMPartition() bool {
+	return len(d.lvms) > 0
+}
+
+func QemuNbdConnect(imagePath, nbddev string) error {
+	var cmd []string
+	if strings.HasPrefix(imagePath, "rbd:") || getImageFormat(imagePath) == "raw" {
+		//qemu-nbd 连接ceph时 /etc/ceph/ceph.conf 必须存在
+		if strings.HasPrefix(imagePath, "rbd:") {
+			err := procutils.NewRemoteCommandAsFarAsPossible("mkdir", "-p", "/etc/ceph").Run()
+			if err != nil {
+				log.Errorf("Failed to mkdir /etc/ceph: %s", err)
+				return errors.Wrap(err, "Failed to mkdir /etc/ceph: %s")
+			}
+			err = procutils.NewRemoteCommandAsFarAsPossible("test", "-f", "/etc/ceph/ceph.conf").Run()
+			if err != nil {
+				err = procutils.NewRemoteCommandAsFarAsPossible("touch", "/etc/ceph/ceph.conf").Run()
+				if err != nil {
+					log.Errorf("failed to create /etc/ceph/ceph.conf: %s", err)
+					return errors.Wrap(err, "failed to create /etc/ceph/ceph.conf")
+				}
+			}
+		}
+		cmd = []string{qemutils.GetQemuNbd(), "-c", nbddev, "-f", "raw", imagePath}
+	} else {
+		cmd = []string{qemutils.GetQemuNbd(), "-c", nbddev, imagePath}
+	}
+	output, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output()
+	if err != nil {
+		log.Errorf("qemu-nbd connect failed %s %s", output, err.Error())
+		return errors.Wrapf(err, "qemu-nbd connect failed %s", output)
+	}
+	return nil
+}
+
+func getImageFormat(imagePath string) string {
+	lines, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuImg(), "info", imagePath).Output()
+	if err != nil {
+		return ""
+	}
+	imgStr := strings.Split(string(lines), "\n")
+	for i := 0; i < len(imgStr); i++ {
+		if strings.HasPrefix(imgStr[i], "file format: ") {
+			return imgStr[i][len("file format: "):]
+		}
+	}
+	return ""
+}
+
+func QemuNbdDisconnect(nbddev string) error {
+	output, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuNbd(), "-d", nbddev).Output()
+	if err != nil {
+		return errors.Wrapf(err, "qemu-nbd disconnect %s", output)
+	}
+	return nil
+}

pkg/hostman/diskutils/nbd/lvmutils.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package diskutils
+package nbd
 
 import (
 	"fmt"
@@ -24,7 +24,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/stringutils"
 
-	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 )
@@ -128,7 +128,7 @@ func (p *SKVMGuestLVMPartition) SetupDevice() bool {
 	return false
 }
 
-func (p *SKVMGuestLVMPartition) FindPartitions() []*guestfs.SKVMGuestDiskPartition {
+func (p *SKVMGuestLVMPartition) FindPartitions() []*kvmpart.SKVMGuestDiskPartition {
 	if !p.isVgActive() {
 		return nil
 	}
@@ -139,10 +139,10 @@ func (p *SKVMGuestLVMPartition) FindPartitions() []*guestfs.SKVMGuestDiskPartiti
 		return nil
 	}
 
-	parts := []*guestfs.SKVMGuestDiskPartition{}
+	parts := []*kvmpart.SKVMGuestDiskPartition{}
 	for _, f := range files {
 		partPath := fmt.Sprintf("/dev/%s/%s", p.vgname, f.Name())
-		part := guestfs.NewKVMGuestDiskPartition(partPath, p.partDev, true)
+		part := kvmpart.NewKVMGuestDiskPartition(partPath, p.partDev, true)
 		parts = append(parts, part)
 	}
 	return parts

pkg/hostman/diskutils/vddk.go

@@ -37,6 +37,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/hostman/guestfs"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 )
 
@@ -61,17 +62,19 @@ type VDDKDisk struct {
 	Proc     *Command
 	Pid      int
 
-	kvmDisk *SKVMGuestDisk
+	kvmDisk      *SKVMGuestDisk
+	deployDriver string
 }
 
-func NewVDDKDisk(vddkInfo *apis.VDDKConInfo, diskPath string) *VDDKDisk {
+func NewVDDKDisk(vddkInfo *apis.VDDKConInfo, diskPath, deployDriver string) *VDDKDisk {
 	return &VDDKDisk{
-		Host:     vddkInfo.Host,
-		Port:     int(vddkInfo.Port),
-		User:     vddkInfo.User,
-		Passwd:   vddkInfo.Passwd,
-		VmRef:    vddkInfo.Vmref,
-		DiskPath: diskPath,
+		Host:         vddkInfo.Host,
+		Port:         int(vddkInfo.Port),
+		User:         vddkInfo.User,
+		Passwd:       vddkInfo.Passwd,
+		VmRef:        vddkInfo.Vmref,
+		DiskPath:     diskPath,
+		deployDriver: deployDriver,
 	}
 }
 
@@ -139,7 +142,7 @@ func (vd *VDDKDisk) Connect() error {
 	if err != nil {
 		return err
 	}
-	vd.kvmDisk = NewKVMGuestDisk(flatFile)
+	vd.kvmDisk = NewKVMGuestDisk(flatFile, vd.deployDriver)
 	return vd.kvmDisk.Connect()
 }
 
@@ -428,7 +431,7 @@ func (vd *VDDKDisk) ResizePartition() error {
 }
 
 type VDDKPartition struct {
-	*guestfs.SLocalGuestFS
+	*kvmpart.SLocalGuestFS
 }
 
 func (vp *VDDKPartition) Mount() bool {
@@ -455,6 +458,14 @@ func (vp *VDDKPartition) GetPhysicalPartitionType() string {
 	return ""
 }
 
-func newVDDKPartition(mntPath string) *VDDKPartition {
-	return &VDDKPartition{guestfs.NewLocalGuestFS(mntPath)}
+func (vp *VDDKPartition) GetPartDev() string {
+	return ""
+}
+
+func (vp *VDDKPartition) IsMounted() bool {
+	return true
+}
+
+func (vp *VDDKPartition) Zerofree() {
+	return
 }

pkg/hostman/guestfs/core.go

@@ -24,7 +24,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/netutils"
 
-	fsdriver "yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 )
 
@@ -42,13 +42,13 @@ func testRootfs(d fsdriver.IRootFsDriver) bool {
 	caseInsensitive := d.IsFsCaseInsensitive()
 	for _, rd := range d.RootSignatures() {
 		if !d.GetPartition().Exists(rd, caseInsensitive) {
-			log.Infof("[%s] test root fs: %s not exists", d, rd)
+			log.Debugf("[%s] test root fs: %s not exists", d, rd)
 			return false
 		}
 	}
 	for _, rd := range d.RootExcludeSignatures() {
 		if d.GetPartition().Exists(rd, caseInsensitive) {
-			log.Infof("[%s] test root fs: %s exists, test failed", d, rd)
+			log.Debugf("[%s] test root fs: %s exists, test failed", d, rd)
 			return false
 		}
 	}

pkg/hostman/guestfs/fsdriver/interface.go

@@ -46,12 +46,15 @@ type IDiskPartition interface {
 	Zerofiles(dir string, caseInsensitive bool) error
 	SupportSerialPorts() bool
 
+	GetPartDev() string
+	IsMounted() bool
 	Mount() bool
 	MountPartReadOnly() bool
 	Umount() bool
 	GetMountPath() string
 	IsReadonly() bool
 	GetPhysicalPartitionType() string
+	Zerofree()
 }
 
 type IRootFsDriver interface {

pkg/hostman/guestfs/guestfishpart/doc.go

@@ -0,0 +1 @@
+package guestfishpart // import "yunion.io/x/onecloud/pkg/hostman/guestfs/guestfishpart"

pkg/hostman/guestfs/guestfishpart/guestfishpart.go

@@ -0,0 +1,253 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package guestfishpart
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs/guestfish"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+type SGuestfishDiskPartition struct {
+	*kvmpart.SLocalGuestFS
+
+	fish *guestfish.Guestfish
+	// device name in guest fish
+	dev string
+	// one of part in guest fish filesystems
+	partDev string
+	// guest fish detected filesystem type
+	fs string
+	// is partition mounted on host filesystem
+	mounted bool
+	// mount as readonly
+	readonly bool
+
+	mountErrorChan chan error
+}
+
+var _ fsdriver.IDiskPartition = &SGuestfishDiskPartition{}
+
+/* dev like /dev/sda, partDev like /dev/sda1 */
+func NewGuestfishDiskPartition(
+	dev, partDev, fs string, fish *guestfish.Guestfish,
+) *SGuestfishDiskPartition {
+	mountPath := fmt.Sprintf("/tmp/%s", strings.Replace(partDev, "/", "_", -1))
+	return &SGuestfishDiskPartition{
+		SLocalGuestFS:  kvmpart.NewLocalGuestFS(mountPath),
+		dev:            dev,
+		partDev:        partDev,
+		fs:             fs,
+		fish:           fish,
+		mountErrorChan: make(chan error, 1),
+	}
+}
+
+func (d *SGuestfishDiskPartition) GetPartDev() string {
+	return d.partDev
+}
+
+func (d *SGuestfishDiskPartition) IsMounted() bool {
+	return d.mounted
+}
+
+func (d *SGuestfishDiskPartition) fsck() error {
+	switch d.fs {
+	case "hfsplus", "ext2", "ext3", "ext4":
+		return d.fish.Fsck(d.partDev, d.fs)
+	case "ntfs":
+		return d.fish.Ntfsfix(d.partDev)
+	}
+	return nil
+}
+
+func (d *SGuestfishDiskPartition) Mount() bool {
+	err := d.fsck()
+	if err != nil {
+		log.Errorf("fsck error: %s", err)
+		return false
+	}
+	err = d.mount(false)
+	if err != nil {
+		log.Errorf("mount error：%s", err)
+		return false
+	}
+	return true
+}
+
+func (d *SGuestfishDiskPartition) mount(readonly bool) error {
+	if output, err := procutils.NewCommand("mkdir", "-p", d.GetMountPath()).Output(); err != nil {
+		return errors.Wrapf(err, "mkdir %s failed: %s", d.GetMountPath(), output)
+	}
+	err := d.fish.Mount(d.partDev)
+	if err != nil {
+		return err
+	}
+
+	err = d.fish.MountLocal(d.GetMountPath(), readonly)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		d.mountErrorChan <- d.fish.MountLocalRun()
+	}()
+
+	select {
+	case err = <-d.mountErrorChan:
+		return err
+	case <-time.After(time.Second * 1):
+		cmd := exec.Command("ls", d.GetMountPath())
+		accessibleChan := make(chan error)
+		go func() {
+			accessibleChan <- cmd.Run()
+		}()
+		select {
+		case err = <-accessibleChan:
+			log.Infof("mount filesystem %s accessable: %v", d.GetMountPath(), err)
+		case <-time.After(3 * time.Second):
+			log.Errorf("mount filesystem %s not accessable", d.GetMountPath())
+			err = cmd.Process.Kill()
+			if err != nil {
+				log.Errorf("failed kill ls process %s", err)
+			}
+			return errors.Errorf("mount filesystem %s not accessable", d.GetMountPath())
+		}
+
+		log.Infof("may be mount success")
+	}
+
+	d.mounted = true
+	return nil
+}
+
+func (d *SGuestfishDiskPartition) MountPartReadOnly() bool {
+	if len(d.fs) == 0 || utils.IsInStringArray(d.fs, []string{"swap", "btrfs"}) {
+		return false
+	}
+	err := d.mount(true)
+	if err != nil {
+		log.Errorf("SGuestfishDiskPartition mount as readonly error: %s", err)
+		return false
+	}
+	d.readonly = true
+	return true
+}
+
+func (d *SGuestfishDiskPartition) Umount() bool {
+	if d.IsMounted() {
+		var tries = 0
+		for tries < 10 {
+			tries += 1
+			output, err := procutils.NewCommand("umount", d.GetMountPath()).Output()
+			if err != nil {
+				log.Errorf("failed umount %s: %s %s", d.GetMountPath(), output, err)
+				time.Sleep(time.Second * 1)
+			} else {
+				d.mounted = false
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func (d *SGuestfishDiskPartition) IsReadonly() bool {
+	return d.readonly
+}
+
+func (d *SGuestfishDiskPartition) GetPhysicalPartitionType() string {
+	ret, err := d.fish.SfdiskL(d.dev)
+	if err != nil {
+		log.Errorf("failed sfdisk-l %s: %s", d.dev, err)
+		return ""
+	}
+	var partType string
+	for i := 0; i < len(ret); i++ {
+		if idx := strings.Index(ret[i], "Disk label type:"); idx > 0 {
+			partType = strings.TrimSpace(string(ret[i])[idx+len("Disk label type:"):])
+		}
+	}
+	if partType == "dos" {
+		return "mbr"
+	} else {
+		return partType
+	}
+}
+
+func (d *SGuestfishDiskPartition) Zerofree() {
+	if !d.IsMounted() {
+		switch d.fs {
+		case "swap":
+			d.zerofreeSwap()
+		case "ext2", "ext3", "ext4":
+			d.zerofreeExt()
+		case "xfs", "ntfs":
+			d.zerofreeSpace()
+		}
+	}
+}
+
+func (d *SGuestfishDiskPartition) zerofreeSwap() {
+	res, err := d.fish.Blkid(d.partDev)
+	if err != nil {
+		log.Errorf("failed get blkid %s", err)
+		return
+	}
+	var label, uuid string
+	for i := 0; i < len(res); i++ {
+		if strings.HasPrefix(res[i], "UUID:") {
+			uuid = strings.TrimSpace(strings.Split(res[i], "")[1])
+		} else if strings.HasPrefix(res[i], "LABEL:") {
+			label = strings.TrimSpace(strings.Split(res[i], "")[1])
+		}
+	}
+	if len(uuid) == 0 {
+		log.Warningf("zerofree swap missing uuid")
+		return
+	}
+	err = d.fish.Mkswap(d.partDev, uuid, label)
+	if err != nil {
+		log.Errorf("mkswap failed %s", err)
+	}
+}
+
+func (d *SGuestfishDiskPartition) zerofreeExt() {
+	err := d.fish.Zerofree(d.partDev)
+	if err != nil {
+		log.Errorf("zerofree %s failed %s", d.partDev, err)
+	}
+}
+
+// mount and zero-free-space
+func (d *SGuestfishDiskPartition) zerofreeSpace() {
+	if err := d.fish.Mount(d.partDev); err != nil {
+		log.Errorf("failed mount partDev %s", err)
+		return
+	}
+	if err := d.fish.ZeroFreeSpace("/"); err != nil {
+		log.Errorf("guestfish zero free space failed %s", err)
+	}
+}

pkg/hostman/guestfs/kvmpart/doc.go

@@ -0,0 +1 @@
+package kvmpart // import "yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"

pkg/hostman/guestfs/kvmpart/kvmpart.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package guestfs
+package kvmpart
 
 import (
 	"fmt"
@@ -24,6 +24,8 @@ import (
 	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
+	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 )
@@ -38,6 +40,8 @@ type SKVMGuestDiskPartition struct {
 	IsLVMPart bool
 }
 
+var _ fsdriver.IDiskPartition = &SKVMGuestDiskPartition{}
+
 func NewKVMGuestDiskPartition(devPath, sourceDev string, isLVM bool) *SKVMGuestDiskPartition {
 	var res = new(SKVMGuestDiskPartition)
 	res.partDev = devPath
@@ -79,7 +83,7 @@ func (p *SKVMGuestDiskPartition) GetPartDev() string {
 }
 
 func (p *SKVMGuestDiskPartition) IsReadonly() bool {
-	return IsPartitionReadonly(p)
+	return guestfs.IsPartitionReadonly(p)
 }
 
 func (p *SKVMGuestDiskPartition) getFsFormat() string {

pkg/hostman/guestfs/kvmpart/localfs.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package guestfs
+package kvmpart
 
 import (
 	"fmt"

pkg/hostman/guestfs/kvmpart/utils.go

@@ -0,0 +1,54 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package kvmpart
+
+import (
+	"sync"
+
+	"yunion.io/x/log"
+)
+
+func LockXfsPartition(uuid string) {
+	log.Infof("xfs lock %s", uuid)
+
+	var (
+		xfsLock *sync.Mutex
+		ok      bool
+	)
+
+	mapLock.Lock()
+	xfsLock, ok = xfsMountUniqueTool[uuid]
+	if !ok {
+		xfsLock = new(sync.Mutex)
+		xfsMountUniqueTool[uuid] = xfsLock
+	}
+	mapLock.Unlock()
+
+	xfsLock.Lock()
+}
+
+func UnlockXfsPartition(uuid string) {
+	log.Infof("xfs unlock %s", uuid)
+	mapLock.Lock()
+	xfsLock := xfsMountUniqueTool[uuid]
+	mapLock.Unlock()
+
+	xfsLock.Unlock()
+}
+
+var (
+	mapLock            = sync.Mutex{}
+	xfsMountUniqueTool = map[string]*sync.Mutex{}
+)

pkg/hostman/guestfs/sshpart/sshpart.go

@@ -44,6 +44,8 @@ type SSHPartition struct {
 	part      *disktool.Partition
 }
 
+var _ fsdriver.IDiskPartition = &SSHPartition{}
+
 func NewSSHPartition(term *ssh.Client, part *disktool.Partition) *SSHPartition {
 	p := new(SSHPartition)
 	p.term = term
@@ -200,6 +202,10 @@ func (p *SSHPartition) IsMounted() bool {
 	return true
 }
 
+func (p *SSHPartition) GetPartDev() string {
+	return ""
+}
+
 func (p *SSHPartition) Chmod(sPath string, mode uint32, caseI bool) error {
 	sPath = p.GetLocalPath(sPath, caseI)
 	if sPath != "" {
@@ -529,6 +535,10 @@ func (p *SSHPartition) Cleandir(dir string, keepdir, caseInsensitive bool) error
 	return nil
 }
 
+func (p *SSHPartition) Zerofree() {
+	log.Warningf("zerofree should not called in ssh partition")
+}
+
 func MountSSHRootfs(term *ssh.Client, layouts []baremetal.Layout) (*SSHPartition, fsdriver.IRootFsDriver, error) {
 	tool := disktool.NewSSHPartitionTool(term)
 	tool.FetchDiskConfs(baremetal.GetDiskConfigurations(layouts))

pkg/hostman/guestfs/utils.go

@@ -1,40 +0,0 @@
-package guestfs
-
-import (
-	"sync"
-
-	"yunion.io/x/log"
-)
-
-func LockXfsPartition(uuid string) {
-	log.Infof("xfs lock %s", uuid)
-
-	var (
-		xfsLock *sync.Mutex
-		ok      bool
-	)
-
-	mapLock.Lock()
-	xfsLock, ok = xfsMountUniqueTool[uuid]
-	if !ok {
-		xfsLock = new(sync.Mutex)
-		xfsMountUniqueTool[uuid] = xfsLock
-	}
-	mapLock.Unlock()
-
-	xfsLock.Lock()
-}
-
-func UnlockXfsPartition(uuid string) {
-	log.Infof("xfs unlock %s", uuid)
-	mapLock.Lock()
-	xfsLock := xfsMountUniqueTool[uuid]
-	mapLock.Unlock()
-
-	xfsLock.Unlock()
-}
-
-var (
-	mapLock            = sync.Mutex{}
-	xfsMountUniqueTool = map[string]*sync.Mutex{}
-)

pkg/hostman/guestman/guestman.go

@@ -32,18 +32,18 @@ import (
 	"yunion.io/x/pkg/util/regutils"
 	"yunion.io/x/pkg/util/seclib"
 
-	compute "yunion.io/x/onecloud/pkg/apis/compute"
-	appsrv "yunion.io/x/onecloud/pkg/appsrv"
+	"yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/appsrv"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
-	hostutils "yunion.io/x/onecloud/pkg/hostman/hostutils"
-	options "yunion.io/x/onecloud/pkg/hostman/options"
-	storageman "yunion.io/x/onecloud/pkg/hostman/storageman"
-	httperrors "yunion.io/x/onecloud/pkg/httperrors"
-	modules "yunion.io/x/onecloud/pkg/mcclient/modules"
-	cgrouputils "yunion.io/x/onecloud/pkg/util/cgrouputils"
-	fileutils2 "yunion.io/x/onecloud/pkg/util/fileutils2"
-	netutils2 "yunion.io/x/onecloud/pkg/util/netutils2"
-	timeutils2 "yunion.io/x/onecloud/pkg/util/timeutils2"
+	"yunion.io/x/onecloud/pkg/hostman/hostutils"
+	"yunion.io/x/onecloud/pkg/hostman/options"
+	"yunion.io/x/onecloud/pkg/hostman/storageman"
+	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/mcclient/modules"
+	"yunion.io/x/onecloud/pkg/util/cgrouputils"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/netutils2"
+	"yunion.io/x/onecloud/pkg/util/timeutils2"
 )
 
 var (

pkg/hostman/hostdeployer/apis/deploy.pb.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // source: deploy.proto
 

pkg/hostman/hostdeployer/consts/consts.go

@@ -0,0 +1,20 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package consts
+
+const (
+	DEPLOY_DRIVER_NBD        = "nbd"
+	DEPLOY_DRIVER_LIBGUESTFS = "libguestfs"
+)

pkg/hostman/hostdeployer/consts/doc.go

@@ -0,0 +1 @@
+package consts // import "yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts"

pkg/hostman/hostdeployer/deployserver/deployserver.go

@@ -30,13 +30,14 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	comapi "yunion.io/x/onecloud/pkg/apis/compute"
-	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
 	"yunion.io/x/onecloud/pkg/cloudcommon/service"
 	"yunion.io/x/onecloud/pkg/hostman/diskutils"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs"
 	"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/sysutils"
@@ -66,7 +67,7 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 		Hypervisor: req.GuestDesc.Hypervisor,
 		DiskPath:   req.DiskPath,
 		VddkInfo:   req.VddkInfo,
-	})
+	}, DeployOption.ImageDeployDriver)
 	if len(req.GuestDesc.Hypervisor) == 0 {
 		req.GuestDesc.Hypervisor = comapi.HYPERVISOR_KVM
 	}
@@ -109,7 +110,7 @@ func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsPara
 		Hypervisor: req.Hypervisor,
 		DiskPath:   req.DiskPath,
 		VddkInfo:   req.VddkInfo,
-	})
+	}, DeployOption.ImageDeployDriver)
 	defer disk.Disconnect()
 	if err := disk.Connect(); err != nil {
 		return new(deployapi.Empty), errors.Wrap(err, "disk connect failed")
@@ -128,7 +129,7 @@ func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsPara
 
 func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsParams) (*deployapi.Empty, error) {
 	log.Infof("********* Format fs on %s", req.DiskPath)
-	gd := diskutils.NewKVMGuestDisk(req.DiskPath)
+	gd := diskutils.NewKVMGuestDisk(req.DiskPath, DeployOption.ImageDeployDriver)
 	defer gd.Disconnect()
 	if err := gd.Connect(); err == nil {
 		if err := gd.MakePartition(req.FsFormat); err == nil {
@@ -148,7 +149,7 @@ func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsPara
 func (*DeployerServer) SaveToGlance(ctx context.Context, req *deployapi.SaveToGlanceParams) (*deployapi.SaveToGlanceResponse, error) {
 	log.Infof("********* %s save to glance", req.DiskPath)
 	var (
-		kvmDisk = diskutils.NewKVMGuestDisk(req.DiskPath)
+		kvmDisk = diskutils.NewKVMGuestDisk(req.DiskPath, DeployOption.ImageDeployDriver)
 		osInfo  string
 		relInfo *deployapi.ReleaseInfo
 	)
@@ -198,7 +199,7 @@ func getImageInfo(kvmDisk *diskutils.SKVMGuestDisk, rootfs fsdriver.IRootFsDrive
 
 func (*DeployerServer) ProbeImageInfo(ctx context.Context, req *deployapi.ProbeImageInfoPramas) (*deployapi.ImageInfo, error) {
 	log.Infof("********* %s probe image info", req.DiskPath)
-	kvmDisk := diskutils.NewKVMGuestDisk(req.DiskPath)
+	kvmDisk := diskutils.NewKVMGuestDisk(req.DiskPath, DeployOption.ImageDeployDriver)
 	defer kvmDisk.Disconnect()
 	if err := kvmDisk.Connect(); err != nil {
 		log.Infof("Failed to connect kvm disk %s: %s", req.DiskPath, err)
@@ -229,7 +230,7 @@ func (*DeployerServer) ConnectEsxiDisks(
 	)
 	ret.Disks = make([]*deployapi.EsxiDiskInfo, len(req.AccessInfo))
 	for i := 0; i < len(req.AccessInfo); i++ {
-		disk := diskutils.NewVDDKDisk(req.VddkInfo, req.AccessInfo[i].DiskPath)
+		disk := diskutils.NewVDDKDisk(req.VddkInfo, req.AccessInfo[i].DiskPath, DeployOption.ImageDeployDriver)
 		flatFilePath, err = disk.ConnectBlockDevice()
 		if err != nil {
 			err = errors.Wrapf(err, "disk %s connect block device", req.AccessInfo[i].DiskPath)
@@ -355,10 +356,9 @@ func (s *SDeployService) PrepareEnv() error {
 }
 
 func (s *SDeployService) InitService() {
-	common_options.ParseOptions(&DeployOption, os.Args, "host.conf", "deploy-server")
-	log.Infof("exec socket path: %s", DeployOption.ExecSocketPath)
+	log.Infof("exec socket path: %s", DeployOption.ExecutorSocketPath)
 	if DeployOption.EnableRemoteExecutor {
-		execlient.Init(DeployOption.ExecSocketPath)
+		execlient.Init(DeployOption.ExecutorSocketPath)
 		procutils.SetRemoteExecutor()
 	}
 
@@ -368,6 +368,11 @@ func (s *SDeployService) InitService() {
 	if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil {
 		log.Fatalln(err)
 	}
+	if DeployOption.ImageDeployDriver == consts.DEPLOY_DRIVER_LIBGUESTFS {
+		if err := libguestfs.Init(3); err != nil {
+			log.Fatalln(err)
+		}
+	}
 	s.O = &DeployOption.BaseOptions
 	if len(DeployOption.DeployServerSocketPath) == 0 {
 		log.Fatalf("missing deploy server socket path")

pkg/hostman/hostdeployer/deployserver/options.go

@@ -14,17 +14,39 @@
 
 package deployserver
 
-import "yunion.io/x/onecloud/pkg/cloudcommon/options"
+import (
+	"os"
+
+	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
+)
 
 type SDeployOptions struct {
-	options.BaseOptions
+	common_options.HostCommonOptions
 
-	DeployServerSocketPath string   `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
-	PrivatePrefixes        []string `help:"IPv4 private prefixes"`
-	ChntpwPath             string   `help:"path to chntpw tool" default:"/usr/local/bin/chntpw.static"`
-	EnableRemoteExecutor   bool     `help:"Enable remote executor" default:"false"`
-	ExecSocketPath         string   `help:"Exec socket paht" default:"/var/run/exec.sock"`
-	CloudrootDir           string   `help:"User cloudroot home dir" default:"/opt"`
+	PrivatePrefixes      []string `help:"IPv4 private prefixes"`
+	ChntpwPath           string   `help:"path to chntpw tool" default:"/usr/local/bin/chntpw.static"`
+	EnableRemoteExecutor bool     `help:"Enable remote executor" default:"false"`
+	CloudrootDir         string   `help:"User cloudroot home dir" default:"/opt"`
+	ImageDeployDriver    string   `help:"Image deploy driver" default:"nbd" choices:"nbd|libguestfs"`
+	CommonConfigFile     string   `help:"common config file for container"`
 }
 
 var DeployOption SDeployOptions
+
+func Parse() (hostOpts SDeployOptions) {
+	common_options.ParseOptions(&hostOpts, os.Args, "host.conf", "host")
+	if len(hostOpts.CommonConfigFile) > 0 {
+		commonCfg := &common_options.HostCommonOptions{}
+		commonCfg.Config = hostOpts.CommonConfigFile
+		common_options.ParseOptions(commonCfg, []string{os.Args[0]}, "common.conf", "host")
+		baseOpt := hostOpts.BaseOptions.BaseOptions
+		hostOpts.HostCommonOptions = *commonCfg
+		// keep base options
+		hostOpts.BaseOptions.BaseOptions = baseOpt
+	}
+	return hostOpts
+}
+
+func init() {
+	DeployOption = Parse()
+}

pkg/hostman/hostinfo/hostconsts/doc.go

@@ -1 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package hostconsts // import "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostconsts"

pkg/hostman/hostinfo/hostconsts/hostconsts.go

@@ -1,3 +1,17 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
 package hostconsts
 
 const (

pkg/hostman/options/options.go

@@ -21,7 +21,7 @@ import (
 )
 
 type SHostOptions struct {
-	common_options.CommonOptions
+	common_options.HostCommonOptions
 	common_options.EtcdOptions
 
 	HostType        string   `help:"Host server type, either hypervisor or kubelet" default:"hypervisor"`
@@ -110,11 +110,9 @@ type SHostOptions struct {
 
 	MaxReservedMemory int `default:"10240" help:"host reserved memory"`
 
-	DeployServerSocketPath    string `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
-	DefaultRequestWorkerCount int    `default:"8" help:"default request worker count"`
-	EnableRemoteExecutor      bool   `help:"Enable remote executor" default:"false"`
-	ExecutorSocketPath        string `help:"Executor socket path" default:"/var/run/exec.sock"`
-	CommonConfigFile          string `help:"common config file for container"`
+	DefaultRequestWorkerCount int `default:"8" help:"default request worker count"`
+
+	CommonConfigFile string `help:"common config file for container"`
 
 	AllowSwitchVMs bool `help:"allow machines run as switch (spoof mac)" default:"true"`
 	AllowRouterVMs bool `help:"allow machines run as router (spoof ip)" default:"true"`
@@ -131,10 +129,11 @@ type SHostOptions struct {
 	OvnEipBridge              string `help:"name of bridge for eip traffic management" default:"$HOST_OVN_EIP_BRIDGE|breip"`
 	OvnUnderlayMtu            int    `help:"mtu of ovn underlay network" default:"1500"`
 
-	EnableHealthChecker bool   `help:"enable host health checker" default:"true"`
-	HealthDriver        string `help:"Component save host health state" default:"etcd"`
-	HostHealthTimeout   int    `help:"host health timeout" default:"30"`
-	HostLeaseTimeout    int    `help:"lease timeout" default:"10"`
+	EnableRemoteExecutor bool   `help:"Enable remote executor" default:"false"`
+	EnableHealthChecker  bool   `help:"enable host health checker" default:"true"`
+	HealthDriver         string `help:"Component save host health state" default:"etcd"`
+	HostHealthTimeout    int    `help:"host health timeout" default:"30"`
+	HostLeaseTimeout     int    `help:"lease timeout" default:"10"`
 
 	SyncStorageInfoDurationSecond int  `help:"sync storage size duration, unit is second" default:"60"`
 	StartHostIgnoreSysError       bool `help:"start host agent ignore sys error" default:"false"`
@@ -147,11 +146,11 @@ var (
 func Parse() (hostOpts SHostOptions) {
 	common_options.ParseOptions(&hostOpts, os.Args, "host.conf", "host")
 	if len(hostOpts.CommonConfigFile) > 0 {
-		commonCfg := &common_options.CommonOptions{}
+		commonCfg := &common_options.HostCommonOptions{}
 		commonCfg.Config = hostOpts.CommonConfigFile
 		common_options.ParseOptions(commonCfg, []string{os.Args[0]}, "common.conf", "host")
 		baseOpt := hostOpts.BaseOptions.BaseOptions
-		hostOpts.CommonOptions = *commonCfg
+		hostOpts.HostCommonOptions = *commonCfg
 		// keep base options
 		hostOpts.BaseOptions.BaseOptions = baseOpt
 	}

pkg/image/options/options.go

@@ -40,7 +40,7 @@ type SImageOptions struct {
 
 	TorrentClientPath string `help:"path to torrent executable" default:"/opt/yunion/bin/torrent"`
 
-	DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/deploy.sock"`
+	DeployServerSocketPath string `help:"Deploy server listen socket path" default:"/var/run/onecloud/deploy.sock"`
 }
 
 var (

scripts/docker_push.sh

@@ -55,6 +55,14 @@ build_bin() {
                 registry.cn-beijing.aliyuncs.com/yunionio/alpine-build:1.0-5 \
                 /bin/sh -c "set -ex; cd /root/go/src/yunion.io/x/onecloud; $BUILD_ARCH $BUILD_CGO GOOS=linux make cmd/$1 cmd/*cli; chown -R $(id -u):$(id -g) _output"
             ;;
+        host-deployer)
+            docker run --rm \
+                -v $SRC_DIR:/root/go/src/yunion.io/x/onecloud \
+                -v $SRC_DIR/_output/alpine-build:/root/go/src/yunion.io/x/onecloud/_output \
+                -v $SRC_DIR/_output/alpine-build/_cache:/root/.cache \
+                registry.cn-beijing.aliyuncs.com/yunionio/centos-build:1.1-2 \
+                /bin/sh -c "set -ex; cd /root/go/src/yunion.io/x/onecloud; $BUILD_ARCH $BUILD_CGO GOOS=linux make cmd/$1; chown -R $(id -u):$(id -g) _output"
+            ;;
         *)
             docker run --rm \
                 -v $SRC_DIR:/root/go/src/yunion.io/x/onecloud \