Automated cherry pick of #18822: Automated cherry pick of #18821: Feat/deploy guest (#18823)

* feat(region): add guest rescue api && download initrd from baremetal tftp server && add baremetal agent ip finder & update pic address & support ip kernel config.

* feat(host-deployer): use qemu-kvm as default deploy dirver

---------

Co-authored-by: boironic@gmail.com <boironic@gmail.com>

pkg/hostman/diskutils/deploy_iface/deployer.go

@@ -0,0 +1,42 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deploy_iface
+
+import (
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
+)
+
+type IDeployer interface {
+	Connect(desc *apis.GuestDesc) error
+	Disconnect() error
+
+	GetPartitions() []fsdriver.IDiskPartition
+	IsLVMPartition() bool
+	Zerofree()
+	ResizePartition() error
+	FormatPartition(fs, uuid string) error
+	MakePartition(fs string) error
+
+	MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error)
+	UmountRootfs(fd fsdriver.IRootFsDriver) error
+	DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool
+
+	DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error)
+	ResizeFs() (res *apis.Empty, err error)
+	FormatFs(req *apis.FormatFsParams) (*apis.Empty, error)
+	SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error)
+	ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error)
+}

pkg/hostman/diskutils/deploy_iface/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deploy_iface // import "yunion.io/x/onecloud/pkg/hostman/diskutils/deploy_iface"

pkg/hostman/diskutils/fsutils/fsutils.go

@@ -23,11 +23,14 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/pkg/errors"
-
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
 	"yunion.io/x/pkg/utils"
 
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/deploy_iface"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/regutils2"
@@ -44,7 +47,7 @@ func IsPartedFsString(fsstr string) bool {
 	})
 }
 
-func ParseDiskPartition(dev string, lines []byte) ([][]string, string) {
+func ParseDiskPartition(dev string, lines []string) ([][]string, string) {
 	var (
 		parts        = [][]string{}
 		label        string
@@ -52,7 +55,7 @@ func ParseDiskPartition(dev string, lines []byte) ([][]string, string) {
 		partten      = regexp.MustCompile(`(?P<idx>\d+)\s+(?P<start>\d+)s\s+(?P<end>\d+)s\s+(?P<count>\d+)s`)
 	)
 
-	for _, line := range strings.Split(string(lines), "\n") {
+	for _, line := range lines {
 		if len(label) == 0 {
 			m := regutils2.GetParams(labelPartten, line)
 			if len(m) > 0 {
@@ -135,7 +138,7 @@ func ResizeDiskFs(diskPath string, sizeMb int) error {
 		log.Errorf("resize disk fs fail, output: %s , error: %s", lines, err)
 		return err
 	}
-	parts, label := ParseDiskPartition(diskPath, lines)
+	parts, label := ParseDiskPartition(diskPath, strings.Split(string(lines), "\n"))
 	log.Infof("Parts: %v label: %s", parts, label)
 	maxSector := GetDevSector512Count(path.Base(diskPath))
 	if label == "gpt" {
@@ -199,7 +202,7 @@ func ResizeDiskFs(diskPath string, sizeMb int) error {
 		if label == "msdos" && end >= 4294967296 {
 			end = 4294967295
 		}
-		if isSupportResizeFs(part[6]) {
+		if IsSupportResizeFs(part[6]) {
 			cmds := []string{"parted", "-a", "none", "-s", diskPath, "--", "resizepart", part[0], fmt.Sprintf("%ds", end)}
 			log.Infof("resize disk partition: %s", cmds)
 			output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output()
@@ -215,7 +218,7 @@ func ResizeDiskFs(diskPath string, sizeMb int) error {
 	return nil
 }
 
-func isSupportResizeFs(fs string) bool {
+func IsSupportResizeFs(fs string) bool {
 	if strings.HasPrefix(fs, "linux-swap") {
 		return true
 	} else if strings.HasPrefix(fs, "ext") {
@@ -406,3 +409,187 @@ func FormatPartition(path, fs, uuid string) error {
 	}
 	return fmt.Errorf("Unknown fs %s", fs)
 }
+
+func DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver, partitions []fsdriver.IDiskPartition) bool {
+	for i := 0; i < len(partitions); i++ {
+		if partitions[i].IsMounted() {
+			if rootfs.DetectIsUEFISupport(partitions[i]) {
+				return true
+			}
+		} else {
+			if partitions[i].Mount() {
+				support := rootfs.DetectIsUEFISupport(partitions[i])
+				if err := partitions[i].Umount(); err != nil {
+					log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err)
+				}
+				if support {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+func MountRootfs(readonly bool, partitions []fsdriver.IDiskPartition) (fsdriver.IRootFsDriver, error) {
+	errs := []error{}
+	for i := 0; i < len(partitions); i++ {
+		log.Infof("detect partition %s", partitions[i].GetPartDev())
+		mountFunc := partitions[i].Mount
+		if readonly {
+			mountFunc = partitions[i].MountPartReadOnly
+		}
+		if mountFunc() {
+			fs, err := guestfs.DetectRootFs(partitions[i])
+			if err == nil {
+				log.Infof("Use rootfs %s, partition %s", fs, partitions[i].GetPartDev())
+				return fs, nil
+			}
+			errs = append(errs, err)
+			if err := partitions[i].Umount(); err != nil {
+				log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err)
+			}
+		}
+	}
+	if len(partitions) == 0 {
+		return nil, errors.Wrap(errors.ErrNotFound, "not found any partition")
+	}
+	var err error = errors.ErrNotFound
+	if len(errs) > 0 {
+		err = errors.Wrapf(errors.ErrNotFound, errors.NewAggregate(errs).Error())
+	}
+	return nil, err
+}
+
+func DeployGuestfs(d deploy_iface.IDeployer, req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) {
+	root, err := d.MountRootfs(false)
+	if err != nil {
+		if errors.Cause(err) == errors.ErrNotFound && req.DeployInfo.IsInit {
+			// if init deploy, ignore no partition error
+			log.Errorf("disk.MountRootfs not found partition, not init, quit")
+			return nil, nil
+		}
+		log.Errorf("Failed mounting rootfs for %s disk: %s", req.GuestDesc.Hypervisor, err)
+		return nil, err
+	}
+	defer d.UmountRootfs(root)
+	ret, err := guestfs.DoDeployGuestFs(root, req.GuestDesc, req.DeployInfo)
+	if err != nil {
+		log.Errorf("guestfs.DoDeployGuestFs fail %s", err)
+		return nil, err
+	}
+	if ret == nil {
+		log.Errorf("guestfs.DoDeployGuestFs return empty results")
+		return nil, nil
+	}
+	return ret, nil
+}
+
+func ResizeFs(d deploy_iface.IDeployer) (*apis.Empty, error) {
+	unmount := func(root fsdriver.IRootFsDriver) error {
+		err := d.UmountRootfs(root)
+		if err != nil {
+			return errors.Wrap(err, "unmount rootfs")
+		}
+		return nil
+	}
+
+	root, err := d.MountRootfs(false)
+	if err != nil {
+		if errors.Cause(err) == errors.ErrNotFound {
+			return new(apis.Empty), nil
+		}
+		return new(apis.Empty), errors.Wrapf(err, "disk.MountRootfs")
+	}
+	if !root.IsResizeFsPartitionSupport() {
+		err := unmount(root)
+		if err != nil {
+			return new(apis.Empty), err
+		}
+		return new(apis.Empty), errors.ErrNotSupported
+	}
+
+	// must umount rootfs before resize partition
+	err = unmount(root)
+	if err != nil {
+		return new(apis.Empty), err
+	}
+	err = d.ResizePartition()
+	if err != nil {
+		return new(apis.Empty), errors.Wrap(err, "resize disk partition")
+	}
+	return new(apis.Empty), nil
+}
+
+func FormatFs(d deploy_iface.IDeployer, req *apis.FormatFsParams) (*apis.Empty, error) {
+	err := d.MakePartition(req.FsFormat)
+	if err != nil {
+		return new(apis.Empty), errors.Wrap(err, "MakePartition")
+	}
+	err = d.FormatPartition(req.FsFormat, req.Uuid)
+	if err != nil {
+		return new(apis.Empty), errors.Wrap(err, "FormatPartition")
+	}
+	return new(apis.Empty), nil
+}
+
+func SaveToGlance(d deploy_iface.IDeployer, req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
+	var (
+		osInfo  string
+		relInfo *apis.ReleaseInfo
+	)
+
+	ret := &apis.SaveToGlanceResponse{
+		OsInfo:      osInfo,
+		ReleaseInfo: relInfo,
+	}
+
+	root, err := d.MountRootfs(false)
+	if err != nil {
+		if errors.Cause(err) == errors.ErrNotFound {
+			return ret, nil
+		}
+		return ret, errors.Wrapf(err, "MountKvmRootfs")
+	}
+	defer d.UmountRootfs(root)
+
+	osInfo = root.GetOs()
+	relInfo = root.GetReleaseInfo(root.GetPartition())
+	if req.Compress {
+		err = root.PrepareFsForTemplate(root.GetPartition())
+		if err != nil {
+			log.Errorf("PrepareFsForTemplate %s", err)
+		}
+	}
+	if req.Compress {
+		d.Zerofree()
+	}
+	return ret, err
+}
+
+func ProbeImageInfo(d deploy_iface.IDeployer) (*apis.ImageInfo, error) {
+	// Fsck is executed during mount
+	rootfs, err := d.MountRootfs(false)
+	if err != nil {
+		if errors.Cause(err) == errors.ErrNotFound {
+			return new(apis.ImageInfo), nil
+		}
+		return new(apis.ImageInfo), errors.Wrapf(err, "d.MountKvmRootfs")
+	}
+	partition := rootfs.GetPartition()
+	imageInfo := &apis.ImageInfo{
+		OsInfo:               rootfs.GetReleaseInfo(partition),
+		OsType:               rootfs.GetOs(),
+		IsLvmPartition:       d.IsLVMPartition(),
+		IsReadonly:           partition.IsReadonly(),
+		IsInstalledCloudInit: rootfs.IsCloudinitInstall(),
+	}
+	d.UmountRootfs(rootfs)
+
+	// In case of deploy driver is guestfish, we can't mount
+	// multi partition concurrent, so we need umount rootfs first
+	imageInfo.IsUefiSupport = d.DetectIsUEFISupport(rootfs)
+	imageInfo.PhysicalPartitionType = partition.GetPhysicalPartitionType()
+	log.Infof("ProbeImageInfo response %s", imageInfo)
+	return imageInfo, nil
+}

pkg/hostman/diskutils/interface.go

@@ -22,12 +22,17 @@ import (
 )
 
 type IDisk interface {
-	Connect() error
+	Connect(desc *apis.GuestDesc) error
 	Disconnect() error
 	MountRootfs() (fsdriver.IRootFsDriver, error)
 	UmountRootfs(driver fsdriver.IRootFsDriver) error
-	ResizePartition() error
 	Cleanup()
+
+	DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error)
+	ResizeFs() (res *apis.Empty, err error)
+	FormatFs(req *apis.FormatFsParams) (*apis.Empty, error)
+	SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error)
+	ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error)
 }
 
 type DiskParams struct {
@@ -48,15 +53,3 @@ func GetIDisk(params DiskParams, driver string, readOnly bool) (IDisk, error) {
 		return NewKVMGuestDisk(params.DiskInfo, driver, readOnly)
 	}
 }
-
-type IDeployer interface {
-	Connect() error
-	Disconnect() error
-
-	GetPartitions() []fsdriver.IDiskPartition
-	IsLVMPartition() bool
-	Zerofree()
-	ResizePartition() error
-	FormatPartition(fs, uuid string) error
-	MakePartition(fs string) error
-}

pkg/hostman/diskutils/kvm.go

@@ -23,10 +23,12 @@ import (
 	"yunion.io/x/pkg/errors"
 
 	cloudconsts "yunion.io/x/onecloud/pkg/cloudcommon/consts"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/deploy_iface"
 	"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs"
 	"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
-	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/qemu_kvm"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
 )
@@ -35,7 +37,7 @@ type SKVMGuestDisk struct {
 	readOnly     bool
 	kvmImagePath string
 	topImagePath string
-	deployer     IDeployer
+	deployer     deploy_iface.IDeployer
 }
 
 func NewKVMGuestDisk(imageInfo qemuimg.SImageInfo, driver string, readOnly bool) (*SKVMGuestDisk, error) {
@@ -78,12 +80,19 @@ func (d *SKVMGuestDisk) Cleanup() {
 	}
 }
 
-func newDeployer(imageInfo qemuimg.SImageInfo, driver string) IDeployer {
+var _ deploy_iface.IDeployer = (*qemu_kvm.QemuKvmDriver)(nil)
+var _ deploy_iface.IDeployer = (*qemu_kvm.LocalDiskDriver)(nil)
+
+func newDeployer(imageInfo qemuimg.SImageInfo, driver string) deploy_iface.IDeployer {
 	switch driver {
 	case consts.DEPLOY_DRIVER_NBD:
 		return nbd.NewNBDDriver(imageInfo)
 	case consts.DEPLOY_DRIVER_LIBGUESTFS:
 		return libguestfs.NewLibguestfsDriver(imageInfo)
+	case consts.DEPLOY_DRIVER_QEMU_KVM:
+		return qemu_kvm.NewQemuKvmDriver(imageInfo)
+	case consts.DEPLOY_DRIVER_LOCAL_DISK:
+		return qemu_kvm.NewLocalDiskDriver()
 	default:
 		return nbd.NewNBDDriver(imageInfo)
 	}
@@ -93,36 +102,14 @@ func (d *SKVMGuestDisk) IsLVMPartition() bool {
 	return d.deployer.IsLVMPartition()
 }
 
-func (d *SKVMGuestDisk) Connect() error {
-	return d.deployer.Connect()
+func (d *SKVMGuestDisk) Connect(guestDesc *apis.GuestDesc) error {
+	return d.deployer.Connect(guestDesc)
 }
 
 func (d *SKVMGuestDisk) Disconnect() error {
 	return d.deployer.Disconnect()
 }
 
-func (d *SKVMGuestDisk) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool {
-	partitions := d.deployer.GetPartitions()
-	for i := 0; i < len(partitions); i++ {
-		if partitions[i].IsMounted() {
-			if rootfs.DetectIsUEFISupport(partitions[i]) {
-				return true
-			}
-		} else {
-			if partitions[i].Mount() {
-				support := rootfs.DetectIsUEFISupport(partitions[i])
-				if err := partitions[i].Umount(); err != nil {
-					log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err)
-				}
-				if support {
-					return true
-				}
-			}
-		}
-	}
-	return false
-}
-
 func (d *SKVMGuestDisk) MountRootfs() (fsdriver.IRootFsDriver, error) {
 	return d.MountKvmRootfs()
 }
@@ -132,34 +119,7 @@ func (d *SKVMGuestDisk) MountKvmRootfs() (fsdriver.IRootFsDriver, error) {
 }
 
 func (d *SKVMGuestDisk) mountKvmRootfs(readonly bool) (fsdriver.IRootFsDriver, error) {
-	partitions := d.deployer.GetPartitions()
-	errs := []error{}
-	for i := 0; i < len(partitions); i++ {
-		log.Infof("detect partition %s", partitions[i].GetPartDev())
-		mountFunc := partitions[i].Mount
-		if readonly {
-			mountFunc = partitions[i].MountPartReadOnly
-		}
-		if mountFunc() {
-			fs, err := guestfs.DetectRootFs(partitions[i])
-			if err == nil {
-				log.Infof("Use rootfs %s, partition %s", fs, partitions[i].GetPartDev())
-				return fs, nil
-			}
-			errs = append(errs, err)
-			if err := partitions[i].Umount(); err != nil {
-				log.Errorf("failed umount %s: %s", partitions[i].GetPartDev(), err)
-			}
-		}
-	}
-	if len(partitions) == 0 {
-		return nil, errors.Wrap(errors.ErrNotFound, "not found any partition")
-	}
-	var err error = errors.ErrNotFound
-	if len(errs) > 0 {
-		err = errors.Wrapf(errors.ErrNotFound, errors.NewAggregate(errs).Error())
-	}
-	return nil, err
+	return d.deployer.MountRootfs(readonly)
 }
 
 func (d *SKVMGuestDisk) MountKvmRootfsReadOnly() (fsdriver.IRootFsDriver, error) {
@@ -180,18 +140,22 @@ func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) error {
 	return d.UmountKvmRootfs(fd)
 }
 
-func (d *SKVMGuestDisk) MakePartition(fs string) error {
-	return d.deployer.MakePartition(fs)
+func (d *SKVMGuestDisk) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) {
+	return d.deployer.DeployGuestfs(req)
 }
 
-func (d *SKVMGuestDisk) FormatPartition(fs, uuid string) error {
-	return d.deployer.FormatPartition(fs, uuid)
+func (d *SKVMGuestDisk) ResizeFs() (*apis.Empty, error) {
+	return d.deployer.ResizeFs()
 }
 
-func (d *SKVMGuestDisk) ResizePartition() error {
-	return d.deployer.ResizePartition()
+func (d *SKVMGuestDisk) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) {
+	return d.deployer.FormatFs(req)
 }
 
-func (d *SKVMGuestDisk) Zerofree() {
-	d.deployer.Zerofree()
+func (d *SKVMGuestDisk) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
+	return d.deployer.SaveToGlance(req)
+}
+
+func (d *SKVMGuestDisk) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) {
+	return d.deployer.ProbeImageInfo(req)
 }

pkg/hostman/diskutils/libguestfs/driver.go

@@ -33,6 +33,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/guestfishpart"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
 )
@@ -73,7 +74,7 @@ func NewLibguestfsDriver(imageInfo qemuimg.SImageInfo) *SLibguestfsDriver {
 	}
 }
 
-func (d *SLibguestfsDriver) Connect() error {
+func (d *SLibguestfsDriver) Connect(*apis.GuestDesc) error {
 	fish, err := guestfsManager.AcquireFish()
 	if err != nil {
 		return err
@@ -250,3 +251,38 @@ func (d *SLibguestfsDriver) MakePartition2(fsFormat string) error {
 	}
 	return nil
 }
+
+func (d *SLibguestfsDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool {
+	return fsutils.DetectIsUEFISupport(rootfs, d.GetPartitions())
+}
+
+func (d *SLibguestfsDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) {
+	return fsutils.MountRootfs(readonly, d.GetPartitions())
+}
+
+func (d *SLibguestfsDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error {
+	if part := fd.GetPartition(); part != nil {
+		return part.Umount()
+	}
+	return nil
+}
+
+func (d *SLibguestfsDriver) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) {
+	return fsutils.DeployGuestfs(d, req)
+}
+
+func (d *SLibguestfsDriver) ResizeFs() (*apis.Empty, error) {
+	return fsutils.ResizeFs(d)
+}
+
+func (d *SLibguestfsDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
+	return fsutils.SaveToGlance(d, req)
+}
+
+func (d *SLibguestfsDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) {
+	return fsutils.FormatFs(d, req)
+}
+
+func (d *SLibguestfsDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) {
+	return fsutils.ProbeImageInfo(d)
+}

pkg/hostman/diskutils/nbd/driver.go

@@ -30,6 +30,7 @@ import (
 	"yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/util/procutils"
 	"yunion.io/x/onecloud/pkg/util/qemuimg"
 	"yunion.io/x/onecloud/pkg/util/qemutils"
@@ -58,7 +59,7 @@ func init() {
 	lock = new(sync.Mutex)
 }
 
-func (d *NBDDriver) Connect() error {
+func (d *NBDDriver) Connect(*apis.GuestDesc) error {
 	d.nbdDev = GetNBDManager().AcquireNbddev()
 	if len(d.nbdDev) == 0 {
 		return errors.Errorf("Cannot get nbd device")
@@ -262,6 +263,41 @@ func (d *NBDDriver) IsLVMPartition() bool {
 	return len(d.lvms) > 0
 }
 
+func (d *NBDDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool {
+	return fsutils.DetectIsUEFISupport(rootfs, d.GetPartitions())
+}
+
+func (d *NBDDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) {
+	return fsutils.MountRootfs(readonly, d.GetPartitions())
+}
+
+func (d *NBDDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error {
+	if part := fd.GetPartition(); part != nil {
+		return part.Umount()
+	}
+	return nil
+}
+
+func (d *NBDDriver) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) {
+	return fsutils.DeployGuestfs(d, req)
+}
+
+func (d *NBDDriver) ResizeFs() (*apis.Empty, error) {
+	return fsutils.ResizeFs(d)
+}
+
+func (d *NBDDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
+	return fsutils.SaveToGlance(d, req)
+}
+
+func (d *NBDDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) {
+	return fsutils.FormatFs(d, req)
+}
+
+func (d *NBDDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) {
+	return fsutils.ProbeImageInfo(d)
+}
+
 func getQemuNbdVersion() (string, error) {
 	output, err := procutils.NewRemoteCommandAsFarAsPossible(qemutils.GetQemuNbd(), "--version").Output()
 	if err != nil {

pkg/hostman/diskutils/nbd/nbdman.go

@@ -148,15 +148,15 @@ func (m *SNBDManager) findNbdDevices() error {
 	for {
 		nbddev := fmt.Sprintf("/dev/nbd%d", i)
 		if fileutils2.Exists(nbddev) {
+			i++
 			if fileutils2.IsBlockDeviceUsed(nbddev) {
 				continue
 			}
 			m.nbdDevs[nbddev] = false
 			// https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-bdi
-			nbdBdi := fmt.Sprintf("/sys/block/nbd%d/bdi/", i)
+			nbdBdi := fmt.Sprintf("/sys/block/nbd%d/bdi/", i-1)
 			sysutils.SetSysConfig(nbdBdi+"max_ratio", "0")
 			sysutils.SetSysConfig(nbdBdi+"min_ratio", "0")
-			i++
 		} else {
 			break
 		}

pkg/hostman/diskutils/qemu_kvm/doc.go

@@ -0,0 +1,15 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu_kvm // import "yunion.io/x/onecloud/pkg/hostman/diskutils/qemu_kvm"

pkg/hostman/diskutils/qemu_kvm/driver.go

@@ -0,0 +1,617 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu_kvm
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/util/stringutils"
+
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
+	"yunion.io/x/onecloud/pkg/hostman/monitor"
+	"yunion.io/x/onecloud/pkg/util/fileutils2"
+	"yunion.io/x/onecloud/pkg/util/netutils2"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+	"yunion.io/x/onecloud/pkg/util/qemuimg"
+	"yunion.io/x/onecloud/pkg/util/ssh"
+	"yunion.io/x/onecloud/pkg/util/sysutils"
+)
+
+var BASE_SSH_PORT = 22222
+
+var (
+	QEMU_KVM_PATH = "/usr/libexec/qemu-kvm"
+
+	OS_ARCH_AARCH64 = "aarch64"
+	ARM_INITRD_PATH = "/yunionos/aarch64/initramfs"
+	ARM_KERNEL_PATH = "/yunionos/aarch64/kernel"
+	X86_INITRD_PATH = "/yunionos/x86_64/initramfs"
+	X86_KERNEL_PATH = "/yunionos/x86_64/kernel"
+
+	DEPLOY_ISO      = "/opt/cloud/host_deployer_v1.iso"
+	DEPLOYER_BIN    = "/opt/yunion/bin/host-deployer"
+	YUNIONOS_PASSWD = "mosbaremetal"
+)
+
+type QemuDeployManager struct {
+	cpuArch         string
+	hugepage        bool
+	hugepageSizeKB  int
+	portsInUse      *sync.Map
+	lastUsedSshPort int
+
+	c chan struct{}
+}
+
+func (m *QemuDeployManager) Acquire() {
+	log.Infof("acquire QemuDeployManager")
+	m.c <- struct{}{}
+}
+
+func (m *QemuDeployManager) Release() {
+	log.Infof("release QemuDeployManager")
+	<-m.c
+}
+
+func (m *QemuDeployManager) GetFreePortByBase(basePort int) int {
+	var port = 1
+	for {
+		if netutils2.IsTcpPortUsed("0.0.0.0", basePort+port) {
+			port += 1
+		} else {
+			if !m.checkAndSetPort(basePort + port) {
+				continue
+			}
+			break
+		}
+	}
+	return port + basePort
+}
+
+func (m *QemuDeployManager) checkAndSetPort(port int) bool {
+	_, loaded := m.portsInUse.LoadOrStore(port, struct{}{})
+	return !loaded
+}
+
+func (m *QemuDeployManager) unsetPort(port int) {
+	m.portsInUse.Delete(port)
+}
+
+func (m *QemuDeployManager) GetSshFreePort() int {
+	port := m.GetFreePortByBase(BASE_SSH_PORT + m.lastUsedSshPort)
+	m.lastUsedSshPort = port - BASE_SSH_PORT
+	if m.lastUsedSshPort > 10000 {
+		m.lastUsedSshPort = 0
+	}
+	return port
+}
+
+var manager *QemuDeployManager
+
+func InitQemuDeployManager(cpuArch string, hugepage bool, hugepageSizeKB int, deployConcurrent int) {
+	if deployConcurrent <= 0 {
+		deployConcurrent = 10
+	}
+
+	if manager == nil {
+		manager = &QemuDeployManager{
+			cpuArch:        cpuArch,
+			hugepage:       hugepage,
+			hugepageSizeKB: hugepageSizeKB,
+			portsInUse:     new(sync.Map),
+			c:              make(chan struct{}, deployConcurrent),
+		}
+	}
+}
+
+type QemuKvmDriver struct {
+	imageInfo      qemuimg.SImageInfo
+	qemuArchDriver IQemuArchDriver
+	sshClient      *ssh.Client
+
+	partitions    []fsdriver.IDiskPartition
+	lvmPartitions []fsdriver.IDiskPartition
+}
+
+func NewQemuKvmDriver(imageInfo qemuimg.SImageInfo) *QemuKvmDriver {
+	return &QemuKvmDriver{
+		imageInfo: imageInfo,
+	}
+}
+
+func (d *QemuKvmDriver) Connect(guestDesc *apis.GuestDesc) error {
+	manager.Acquire()
+	d.qemuArchDriver = NewCpuArchDriver(manager.cpuArch)
+	err := d.connect(guestDesc)
+	if err != nil {
+		d.qemuArchDriver.CleanGuest()
+		return err
+	}
+
+	return nil
+}
+
+func (d *QemuKvmDriver) connect(guestDesc *apis.GuestDesc) error {
+	var (
+		ncpu      = 2
+		memSizeMB = 1024 // yunionos acquire least 1g mem
+		disks     = make([]string, 0)
+	)
+
+	var sshport = manager.GetSshFreePort()
+	defer manager.unsetPort(sshport)
+
+	if guestDesc != nil {
+		for i := range guestDesc.Disks {
+			disks = append(disks, guestDesc.Disks[i].Path)
+		}
+	} else {
+		disks = append(disks, d.imageInfo.Path)
+	}
+
+	err := d.qemuArchDriver.StartGuest(sshport, ncpu, memSizeMB, manager.hugepage, manager.hugepageSizeKB, disks)
+	if err != nil {
+		return err
+	}
+	log.Infof("guest started ....")
+
+	cli, err := ssh.NewClient("localhost", sshport, "root", YUNIONOS_PASSWD, "")
+	if err != nil {
+		return errors.Wrap(err, "new ssh client")
+	}
+	d.sshClient = cli
+	log.Infof("guest ssh connected")
+
+	out, err := d.sshRun("mount /dev/sr0 /opt")
+	if err != nil {
+		return errors.Wrapf(err, "failed mount iso /dev/sr0: %v", out)
+	}
+	return nil
+}
+
+func (d *QemuKvmDriver) Disconnect() error {
+	d.sshClient.Close()
+	d.qemuArchDriver.CleanGuest()
+	d.qemuArchDriver = nil
+	return nil
+}
+
+func (d *QemuKvmDriver) GetPartitions() []fsdriver.IDiskPartition {
+	return nil
+}
+
+func (d *QemuKvmDriver) IsLVMPartition() bool {
+	return false
+}
+
+func (d *QemuKvmDriver) Zerofree() {}
+
+func (d *QemuKvmDriver) ResizePartition() error {
+	return nil
+}
+
+func (d *QemuKvmDriver) FormatPartition(fs, uuid string) error {
+	return nil
+}
+
+func (d *QemuKvmDriver) MakePartition(fs string) error {
+	return nil
+}
+
+func (d *QemuKvmDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool {
+	return false
+}
+
+func (d *QemuKvmDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) {
+	return nil, nil
+}
+
+func (d *QemuKvmDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error {
+	return nil
+}
+
+func (d *QemuKvmDriver) sshRun(cmd string) ([]string, error) {
+	log.Infof("QemuKvmDriver start command %s", cmd)
+	return d.sshClient.Run(cmd)
+}
+
+func (d *QemuKvmDriver) DeployGuestfs(req *apis.DeployParams) (*apis.DeployGuestFsResponse, error) {
+	defer func() {
+		logStr, _ := d.sshRun("test -f /log && cat /log")
+		log.Infof("DeployGuestfs log: %v", strings.Join(logStr, "\n"))
+	}()
+
+	params, _ := json.Marshal(req)
+	cmd := fmt.Sprintf("%s --deploy-action deploy_guest_fs --deploy-params '%s'", DEPLOYER_BIN, params)
+	out, err := d.sshRun(cmd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "run deploy_guest_fs failed %s", out)
+	}
+	log.Infof("DeployGuestfs log: %s", strings.Join(out, "\n"))
+
+	responseStrs, err := d.sshRun("test -f /response && cat /response")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var res = new(apis.DeployGuestFsResponse)
+	if len(responseStrs[0]) > 0 {
+		err := json.Unmarshal([]byte(responseStrs[0]), res)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed unmarshal deploy response %s", responseStrs[0])
+		}
+		return res, nil
+	}
+
+	errStrs, err := d.sshRun("test -f /error && cat /error")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var retErr error = nil
+	if len(errStrs[0]) > 0 {
+		retErr = errors.Errorf(errStrs[0])
+	}
+	return res, retErr
+}
+
+func (d *QemuKvmDriver) ResizeFs() (*apis.Empty, error) {
+	defer func() {
+		logStr, _ := d.sshRun("test -f /log && cat /log")
+		log.Infof("ResizeFs log: %v", strings.Join(logStr, "\n"))
+	}()
+
+	cmd := fmt.Sprintf("%s --deploy-action resize_fs", DEPLOYER_BIN)
+	out, err := d.sshRun(cmd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "run resize_fs failed %s", out)
+	}
+	log.Infof("ResizeFs log: %s", strings.Join(out, "\n"))
+
+	errStrs, err := d.sshRun("test -f /error && cat /error")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var retErr error = nil
+	if len(errStrs[0]) > 0 {
+		retErr = errors.Errorf(errStrs[0])
+	}
+	return new(apis.Empty), retErr
+}
+
+func (d *QemuKvmDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) {
+	defer func() {
+		logStr, _ := d.sshRun("test -f /log && cat /log")
+		log.Infof("FormatFs log: %v", strings.Join(logStr, "\n"))
+	}()
+
+	params, _ := json.Marshal(req)
+	cmd := fmt.Sprintf("%s --deploy-action format_fs --deploy-params '%s'", DEPLOYER_BIN, params)
+	out, err := d.sshRun(cmd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "run format_fs failed %s", out)
+	}
+	log.Infof("FormatFs log: %s", strings.Join(out, "\n"))
+
+	errStrs, err := d.sshRun("test -f /error && cat /error")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var retErr error = nil
+	if len(errStrs[0]) > 0 {
+		retErr = errors.Errorf(errStrs[0])
+	}
+	return new(apis.Empty), retErr
+}
+
+func (d *QemuKvmDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
+	defer func() {
+		logStr, _ := d.sshRun("test -f /log && cat /log")
+		log.Infof("SaveToGlance log: %s", strings.Join(logStr, "\n"))
+	}()
+
+	params, _ := json.Marshal(req)
+	cmd := fmt.Sprintf("%s --deploy-action save_to_glance --deploy-params '%s'", DEPLOYER_BIN, params)
+	out, err := d.sshRun(cmd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "run save_to_glance failed %s", out)
+	}
+	log.Infof("SaveToGlance log: %s", strings.Join(out, "\n"))
+
+	responseStrs, err := d.sshRun("test -f /response && cat /response")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var res = new(apis.SaveToGlanceResponse)
+	if len(responseStrs[0]) > 0 {
+		err := json.Unmarshal([]byte(responseStrs[0]), res)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed unmarshal deploy response %s", responseStrs[0])
+		}
+		return res, nil
+	}
+
+	errStrs, err := d.sshRun("test -f /error && cat /error")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var retErr error = nil
+	if len(errStrs[0]) > 0 {
+		retErr = errors.Errorf(errStrs[0])
+	}
+	return res, retErr
+}
+
+func (d *QemuKvmDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) {
+	defer func() {
+		logStr, _ := d.sshRun("test -f /log && cat /log")
+		log.Infof("ProbeImageInfo log: %v", strings.Join(logStr, "\n"))
+	}()
+
+	params, _ := json.Marshal(req)
+	cmd := fmt.Sprintf("%s --deploy-action probe_image_info --deploy-params '%s'", DEPLOYER_BIN, params)
+	out, err := d.sshRun(cmd)
+	if err != nil {
+		return nil, errors.Wrapf(err, "run probe_image_info failed %s", out)
+	}
+	log.Infof("ProbeImageInfo log: %s", strings.Join(out, "\n"))
+
+	responseStrs, err := d.sshRun("test -f /response && cat /response")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var res = new(apis.ImageInfo)
+	if len(responseStrs[0]) > 0 {
+		err := json.Unmarshal([]byte(responseStrs[0]), res)
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed unmarshal deploy response %s", responseStrs[0])
+		}
+		return res, nil
+	}
+
+	errStrs, err := d.sshRun("test -f /error && cat /error")
+	if err != nil {
+		return nil, errors.Wrapf(err, "ssh gather errors failed")
+	}
+	var retErr error = nil
+	if len(errStrs[0]) > 0 {
+		retErr = errors.Errorf(errStrs[0])
+	}
+	return res, retErr
+}
+
+// wrap strings
+func __(v string, vs ...interface{}) string {
+	return fmt.Sprintf(" "+v, vs...)
+}
+
+type QemuBaseDriver struct {
+	proc *procutils.Command
+	outb io.ReadCloser
+	errb io.ReadCloser
+
+	hmp          *monitor.HmpMonitor
+	hugepagePath string
+	pidPath      string
+}
+
+func (d *QemuBaseDriver) CleanGuest() {
+	defer manager.Release()
+
+	if d.hmp != nil {
+		d.hmp.IsConnected()
+		d.hmp.Quit(func(string) {})
+		d.hmp = nil
+	}
+
+	if d.pidPath != "" && fileutils2.Exists(d.pidPath) {
+		pid, _ := fileutils2.FileGetContents(d.pidPath)
+		if len(pid) > 0 {
+			out, err := procutils.NewCommand("kill", pid).Output()
+			log.Infof("kill  process %s %v", out, err)
+		}
+	}
+
+	if d.hugepagePath != "" {
+		err, out := procutils.NewCommand("umount", d.hugepagePath).Output()
+		if err != nil {
+			log.Errorf("failed umount %s %s : %s", d.hugepagePath, err, out)
+		} else {
+			procutils.NewCommand("rm", "-rf", d.hugepagePath).Run()
+		}
+		d.hugepagePath = ""
+	}
+}
+
+type QemuX86Driver struct {
+	QemuBaseDriver
+}
+
+func (d *QemuX86Driver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error {
+	uuid := stringutils.UUID4()
+	socketPath := fmt.Sprintf("/tmp/hmp_%s.socket", uuid)
+
+	cmd := QEMU_KVM_PATH
+	if sysutils.IsKvmSupport() {
+		cmd += __("-enable-kvm")
+	}
+
+	cmd += __("-cpu host")
+	cmd += __("-M pc")
+
+	d.pidPath = fmt.Sprintf("/tmp/%s.pid", uuid)
+	cmd += __("-nodefaults")
+	cmd += __("-daemonize")
+	cmd += __("-monitor unix:%s,server,nowait", socketPath)
+	cmd += __("-pidfile %s", d.pidPath)
+	cmd += __("-vnc none")
+	cmd += __("-smp %d", ncpu)
+
+	//if hugePage {
+	//	if pageSizeKB/1024 > memSizeMB {
+	//		memSizeMB = pageSizeKB / 1024
+	//	}
+	//
+	//	hugepagePath := fmt.Sprintf("/dev/hugepages/host-deployer/%s", uuid)
+	//	out, err := procutils.NewCommand("mkdir", "-p", hugepagePath).Output()
+	//	if err != nil {
+	//		return errors.Wrapf(err, "mkdir %s failed: %s", hugepagePath, out)
+	//	}
+	//	d.hugepagePath = hugepagePath
+	//
+	//	mountCmd := fmt.Sprintf("mount -t hugetlbfs -o pagesize=%dK,size=%dM hugetlbfs-%s %s", pageSizeKB, memSizeMB, uuid, hugepagePath)
+	//	out, err = procutils.NewCommand("bash", "-c", mountCmd).Output()
+	//	if err != nil {
+	//		return errors.Wrapf(err, "mount %s failed: %s", mountCmd, out)
+	//	}
+	//
+	//	cmd += __("-m %dM", memSizeMB)
+	//	cmd += __("-object memory-backend-file,id=mem,prealloc=on,mem-path=%s,size=%dM,share=on", hugepagePath, memSizeMB)
+	//	cmd += __("-numa node,memdev=mem")
+	//} else {
+	cmd += __("-m %dM", memSizeMB)
+	//}
+
+	cmd += __("-initrd %s", X86_INITRD_PATH)
+	cmd += __("-kernel %s", X86_KERNEL_PATH)
+	cmd += __("-device VGA")
+	cmd += __("-device virtio-serial-pci")
+	cmd += __("-netdev user,id=hostnet0,hostfwd=tcp::%d-:22", sshPort)
+	cmd += __("-device virtio-net-pci,netdev=hostnet0")
+	cmd += __("-device virtio-scsi-pci,id=scsi")
+	for i, diskPath := range disks {
+		cmd += __("-drive file=%s,if=none,id=drive_%d,cache=none,aio=native,file.locking=off", diskPath, i)
+		cmd += __("-device scsi-hd,drive=drive_%d,bus=scsi.0,id=drive_%d", i, i)
+	}
+	cmd += __("-drive id=ide0-cd0,if=none,media=cdrom,file=%s", DEPLOY_ISO)
+	cmd += __("-device ide-cd,drive=ide0-cd0,bus=ide.1")
+
+	log.Infof("start guest %s", cmd)
+	out, err := procutils.NewCommand("bash", "-c", cmd).Output()
+	if err != nil {
+		log.Errorf("failed start guest %s: %s", out, err)
+		return errors.Wrapf(err, "failed start guest %s", out)
+	}
+
+	var c = make(chan error)
+	onMonitorConnected := func() {
+		log.Infof("monitor connected")
+		c <- nil
+	}
+	onMonitorDisConnect := func(e error) {
+		log.Errorf("monitor disconnect %s", e)
+	}
+	onMonitorConnectFailed := func(e error) {
+		log.Errorf("monitor connect failed %s", e)
+		c <- e
+	}
+	m := monitor.NewHmpMonitor("", "", onMonitorDisConnect, onMonitorConnectFailed, onMonitorConnected)
+	if err = m.ConnectWithSocket(socketPath); err != nil {
+		return errors.Wrapf(err, "connect socket %s failed", socketPath)
+	}
+	if err = <-c; err != nil {
+		return errors.Wrap(err, "monitor connect failed")
+	}
+	d.hmp = m
+	return nil
+}
+
+type QemuARMDriver struct {
+	QemuBaseDriver
+}
+
+func (d *QemuARMDriver) StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error {
+	uuid := stringutils.UUID4()
+	socketPath := fmt.Sprintf("/tmp/hmp_%s.socket", uuid)
+
+	cmd := QEMU_KVM_PATH
+	if sysutils.IsKvmSupport() {
+		cmd += __("-enable-kvm")
+	}
+
+	cmd += __("-cpu host")
+	cmd += __("-M virt")
+
+	d.pidPath = fmt.Sprintf("/tmp/%s.pid", uuid)
+	cmd += __("-nodefaults")
+	cmd += __("-daemonize")
+	cmd += __("-monitor unix:%s,server,nowait", socketPath)
+	cmd += __("-pidfile %s", d.pidPath)
+	cmd += __("-vnc none")
+	cmd += __("-smp %d", ncpu)
+
+	cmd += __("-m %dM", memSizeMB)
+	cmd += __("-initrd %s", ARM_INITRD_PATH)
+	cmd += __("-kernel %s", ARM_KERNEL_PATH)
+	cmd += __("-drive if=pflash,format=raw,unit=0,file=/opt/cloud/contrib/OVMF.fd,readonly=on")
+
+	cmd += __("-device virtio-serial-pci")
+	cmd += __("-netdev user,id=hostnet0,hostfwd=tcp::%d-:22", sshPort)
+	cmd += __(" -device virtio-net-pci,netdev=hostnet0")
+	cmd += __("-device virtio-scsi-pci,id=scsi")
+	for i, diskPath := range disks {
+		cmd += __("-drive file=%s,if=none,id=drive_%d,cache=none,aio=native,file.locking=off", diskPath, i)
+		cmd += __("-device scsi-hd,drive=drive_%d,bus=scsi.0,id=drive_%d", i, i)
+	}
+	cmd += __("-drive if=none,file=%s,id=cd0,media=cdrom", DEPLOY_ISO)
+	cmd += __("-device scsi-cd,drive=cd0,share-rw=true")
+
+	log.Infof("start guest %s", cmd)
+	out, err := procutils.NewCommand("bash", "-c", cmd).Output()
+	if err != nil {
+		log.Errorf("failed start guest %s: %s", out, err)
+		return errors.Wrapf(err, "failed start guest %s", out)
+	}
+
+	var c = make(chan error)
+	onMonitorConnected := func() {
+		log.Infof("monitor connected")
+		c <- nil
+	}
+	onMonitorDisConnect := func(e error) {
+		log.Errorf("monitor disconnect %s", e)
+	}
+	onMonitorConnectFailed := func(e error) {
+		log.Errorf("monitor connect failed %s", e)
+		c <- e
+	}
+	m := monitor.NewHmpMonitor("", "", onMonitorDisConnect, onMonitorConnectFailed, onMonitorConnected)
+	if err = m.ConnectWithSocket(socketPath); err != nil {
+		return errors.Wrapf(err, "connect socket %s failed", socketPath)
+	}
+	if err = <-c; err != nil {
+		return errors.Wrap(err, "monitor connect failed")
+	}
+	d.hmp = m
+	return nil
+}
+
+type IQemuArchDriver interface {
+	StartGuest(sshPort, ncpu, memSizeMB int, hugePage bool, pageSizeKB int, disks []string) error
+	CleanGuest()
+}
+
+func NewCpuArchDriver(cpuArch string) IQemuArchDriver {
+	if cpuArch == OS_ARCH_AARCH64 {
+		return &QemuARMDriver{}
+	}
+
+	return &QemuX86Driver{}
+}

pkg/hostman/diskutils/qemu_kvm/local_driver.go

@@ -0,0 +1,140 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu_kvm
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/fsutils"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+	"yunion.io/x/onecloud/pkg/util/stringutils2"
+)
+
+type LocalDiskDriver struct {
+	partitions    []fsdriver.IDiskPartition
+	lvmPartitions []fsdriver.IDiskPartition
+}
+
+func NewLocalDiskDriver() *LocalDiskDriver {
+	return &LocalDiskDriver{
+		partitions:    make([]fsdriver.IDiskPartition, 0),
+		lvmPartitions: make([]fsdriver.IDiskPartition, 0),
+	}
+}
+
+func (d *LocalDiskDriver) Connect(desc *apis.GuestDesc) error {
+	out, err := procutils.NewCommand("sh", "-c", "cat /proc/partitions | grep -v name | awk '{print $4}'").Output()
+	if err != nil {
+		return errors.Wrap(err, "cat proc partitions")
+	}
+	lines := strings.Split(string(out), "\n")
+	partDevs := make([]string, 0)
+	for _, line := range lines {
+		line = strings.TrimSpace(line)
+		if strings.HasPrefix(line, "sd") {
+			partDevs = append(partDevs, line)
+		}
+	}
+	sortPartDevs := stringutils2.NewSortedStrings(partDevs)
+	for _, partDev := range sortPartDevs {
+		part := kvmpart.NewKVMGuestDiskPartition(fmt.Sprintf("/dev/%s", partDev), "", false)
+		d.partitions = append(d.partitions, part)
+		log.Infof("found part dev %s", part.GetPartDev())
+	}
+	d.setupLVMS()
+	if len(d.lvmPartitions) > 0 {
+		d.partitions = append(d.partitions, d.lvmPartitions...)
+	}
+	return nil
+}
+
+func (d *LocalDiskDriver) Disconnect() error {
+	return nil
+}
+
+func (d *LocalDiskDriver) GetPartitions() []fsdriver.IDiskPartition {
+	return d.partitions
+}
+
+func (d *LocalDiskDriver) IsLVMPartition() bool {
+	return len(d.lvmPartitions) > 0
+}
+
+func (d *LocalDiskDriver) Zerofree() {
+	startTime := time.Now()
+	for _, part := range d.partitions {
+		part.Zerofree()
+	}
+	log.Infof("Zerofree %d partitions takes %f seconds", len(d.partitions), time.Now().Sub(startTime).Seconds())
+}
+
+func (d *LocalDiskDriver) ResizePartition() error {
+	if d.IsLVMPartition() {
+		// do not resize LVM partition
+		return nil
+	}
+	return fsutils.ResizeDiskFs("/dev/sda", 0)
+}
+
+func (d *LocalDiskDriver) FormatPartition(fs, uuid string) error {
+	return fsutils.FormatPartition("/dev/sda1", fs, uuid)
+}
+
+func (d *LocalDiskDriver) MakePartition(fs string) error {
+	return fsutils.Mkpartition("/dev/sda", fs)
+}
+
+func (d *LocalDiskDriver) DetectIsUEFISupport(rootfs fsdriver.IRootFsDriver) bool {
+	return fsutils.DetectIsUEFISupport(rootfs, d.GetPartitions())
+}
+
+func (d *LocalDiskDriver) MountRootfs(readonly bool) (fsdriver.IRootFsDriver, error) {
+	return fsutils.MountRootfs(readonly, d.GetPartitions())
+}
+
+func (d *LocalDiskDriver) UmountRootfs(fd fsdriver.IRootFsDriver) error {
+	if part := fd.GetPartition(); part != nil {
+		return part.Umount()
+	}
+	return nil
+}
+
+func (d *LocalDiskDriver) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) {
+	return fsutils.DeployGuestfs(d, req)
+}
+
+func (d *LocalDiskDriver) ResizeFs() (*apis.Empty, error) {
+	return fsutils.ResizeFs(d)
+}
+
+func (d *LocalDiskDriver) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
+	return fsutils.SaveToGlance(d, req)
+}
+
+func (d *LocalDiskDriver) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) {
+	return fsutils.FormatFs(d, req)
+}
+
+func (d *LocalDiskDriver) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) {
+	return fsutils.ProbeImageInfo(d)
+}

pkg/hostman/diskutils/qemu_kvm/lvm.go

@@ -0,0 +1,138 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu_kvm
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"yunion.io/x/log"
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
+	"yunion.io/x/onecloud/pkg/util/procutils"
+)
+
+func (d *LocalDiskDriver) setupLVMS() {
+	for _, part := range d.partitions {
+		vg, err := d.findVg(part.GetPartDev())
+		if err != nil {
+			log.Infof("failed find vg %s", err)
+			continue
+		}
+		if vg == nil {
+			continue
+		}
+
+		log.Infof("found vg %s from %s", vg.VgName, part.GetPartDev())
+		err = d.vgActive(vg.VgName)
+		if err != nil {
+			log.Infof("failed active vg %s: %s", vg.VgName, err)
+			continue
+		}
+		lvs, err := d.getVgLvs(vg.VgName)
+		if err != nil {
+			log.Infof("failed get vg lvs %s: %s", vg.VgName, err)
+			continue
+		}
+		log.Infof("found lvs %v from vg %s", lvs, vg.VgName)
+		for _, lv := range lvs {
+			lvmpart := kvmpart.NewKVMGuestDiskPartition(lv.LvPath, "", true)
+			d.lvmPartitions = append(d.lvmPartitions, lvmpart)
+			log.Infof("found lvm part dev %v", lvmpart.GetPartDev())
+		}
+	}
+}
+
+func (d *LocalDiskDriver) vgActive(vgname string) error {
+	_, err := procutils.NewCommand("vgchange", "-ay", vgname).Output()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+type LvProps struct {
+	LvName string
+	LvPath string
+}
+
+type LvNames struct {
+	Report []struct {
+		LV []struct {
+			LVName string `json:"lv_name"`
+			LVPath string `json:"lv_path"`
+		} `json:"lv"`
+	} `json:"report"`
+}
+
+func (d *LocalDiskDriver) getVgLvs(vg string) ([]LvProps, error) {
+	cmd := fmt.Sprintf("lvs --reportformat json -o lv_name,lv_path %s 2>/dev/null", vg)
+	out, err := procutils.NewCommand("sh", "-c", cmd).Output()
+	if err != nil {
+		return nil, errors.Wrap(err, "find vg lvs")
+	}
+	var res LvNames
+	err = json.Unmarshal(out, &res)
+	if err != nil {
+		return nil, errors.Wrap(err, "unmarshal lvs")
+	}
+	if len(res.Report) != 1 {
+		return nil, errors.Errorf("unexpect res %v", res)
+	}
+	lvs := make([]LvProps, len(res.Report[0].LV))
+	for i := 0; i < len(res.Report[0].LV); i++ {
+		if res.Report[0].LV[i].LVName == "" {
+			continue
+		}
+		lvs = append(lvs, LvProps{res.Report[0].LV[i].LVName, res.Report[0].LV[i].LVPath})
+	}
+	return lvs, nil
+}
+
+type VgProps struct {
+	VgName string
+	VgUuid string
+}
+
+type VgReports struct {
+	Report []struct {
+		VG []struct {
+			VgName string `json:"vg_name"`
+			VgUuid string `json:"vg_uuid"`
+		} `json:"vg"`
+	} `json:"report"`
+}
+
+func (d *LocalDiskDriver) findVg(partDev string) (*VgProps, error) {
+	cmd := fmt.Sprintf("vgs --reportformat json -o vg_name,vg_uuid --devices %s 2>/dev/null", partDev)
+	out, err := procutils.NewCommand("sh", "-c", cmd).Output()
+	if err != nil {
+		return nil, errors.Wrap(err, "find vg lvs")
+	}
+	var vgReports VgReports
+	err = json.Unmarshal(out, &vgReports)
+	if err != nil {
+		return nil, errors.Wrapf(err, "unmarshal vgprops %s", out)
+	}
+	if len(vgReports.Report) == 1 && len(vgReports.Report[0].VG) == 1 {
+		var vgProps VgProps
+		vgProps.VgName = vgReports.Report[0].VG[0].VgName
+		vgProps.VgUuid = vgReports.Report[0].VG[0].VgUuid
+		return &vgProps, nil
+	}
+
+	return nil, nil
+}

pkg/hostman/diskutils/vddk.go

@@ -147,7 +147,7 @@ func (vd *VDDKDisk) Cleanup() {
 	}
 }
 
-func (vd *VDDKDisk) Connect() error {
+func (vd *VDDKDisk) Connect(*apis.GuestDesc) error {
 	flatFile, err := vd.ConnectBlockDevice()
 	if err != nil {
 		return errors.Wrap(err, "ConnectBlockDevice")
@@ -156,7 +156,7 @@ func (vd *VDDKDisk) Connect() error {
 	if err != nil {
 		return errors.Wrap(err, "NewKVMGuestDisk")
 	}
-	return vd.kvmDisk.Connect()
+	return vd.kvmDisk.Connect(nil)
 }
 
 func (vd *VDDKDisk) Disconnect() error {
@@ -441,8 +441,24 @@ func (vd *VDDKDisk) DisconnectBlockDevice() error {
 	return fmt.Errorf("vddk disk has not connected")
 }
 
-func (vd *VDDKDisk) ResizePartition() error {
-	return vd.kvmDisk.ResizePartition()
+func (vd *VDDKDisk) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) {
+	return vd.kvmDisk.DeployGuestfs(req)
+}
+
+func (d *VDDKDisk) ResizeFs() (*apis.Empty, error) {
+	return d.kvmDisk.ResizeFs()
+}
+
+func (d *VDDKDisk) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) {
+	return d.kvmDisk.FormatFs(req)
+}
+
+func (d *VDDKDisk) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
+	return d.kvmDisk.SaveToGlance(req)
+}
+
+func (d *VDDKDisk) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) {
+	return d.kvmDisk.ProbeImageInfo(req)
 }
 
 type VDDKPartition struct {

pkg/hostman/guestfs/fsdriver/linux.go

@@ -497,6 +497,9 @@ func (l *sLinuxRootFs) GetArch(rootFs IDiskPartition) string {
 				if fileInfo.IsDir() {
 					continue
 				}
+				if fileInfo.Mode()&os.ModeSymlink != 0 {
+					continue
+				}
 				rp, err := filepath.EvalSymlinks(p)
 				if err != nil {
 					log.Errorf("readlink of %s: %s", p, err)

pkg/hostman/guestfs/kvmpart/kvmpart.go

@@ -154,7 +154,7 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
 	if output, err := procutils.NewCommand("mkdir", "-p", p.mountPath).Output(); err != nil {
 		return errors.Wrapf(err, "mkdir %s failed: %s", p.mountPath, output)
 	}
-	var cmds = []string{"mount", "-t"}
+	var cmds = []string{"mount"}
 	var opt, fsType string
 	if readonly {
 		opt = "ro"
@@ -168,7 +168,9 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
 	} else if fsType == "hfsplus" && !readonly {
 		opt = "force,rw"
 	}
-	cmds = append(cmds, fsType)
+	if len(fsType) > 0 {
+		cmds = append(cmds, "-t", fsType)
+	}
 	if len(opt) > 0 {
 		cmds = append(cmds, "-o", opt)
 	}
@@ -189,13 +191,35 @@ func (p *SKVMGuestDiskPartition) mount(readonly bool) error {
 	}
 
 	retrier := func(utils.FibonacciRetrier) (bool, error) {
-		output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output()
-		if err == nil {
+		var errChan = make(chan error)
+		go func() {
+			output, err := procutils.NewCommand(cmds[0], cmds[1:]...).Output()
+			if err == nil {
+				errChan <- nil
+			} else {
+				log.Errorf("mount fail: %s %s", err, output)
+				time.Sleep(time.Millisecond * time.Duration(100+rand.Intn(400)))
+				errChan <- err
+			}
+		}()
+		select {
+		case err := <-errChan:
+			if err != nil {
+				return false, err
+			}
 			return true, nil
-		} else {
-			log.Errorf("mount fail: %s %s", err, output)
-			time.Sleep(time.Millisecond * time.Duration(100+rand.Intn(400)))
-			return false, errors.Wrap(err, "")
+		case <-time.After(time.Second * 3):
+			if fsType == "ntfs-3g" {
+				if err = procutils.NewCommand("mountpoint", p.mountPath).Run(); err != nil {
+					// mountpath is not a mountpoint
+					return false, err
+				} else {
+					// ntfs already mounted, but maybe in buildroot yunion os Failed to daemonize.
+					return true, nil
+				}
+			} else {
+				return false, errors.Errorf("mount timeout")
+			}
 		}
 	}
 	_, err = utils.NewFibonacciRetrierMaxTries(3, retrier).Start(context.Background())

pkg/hostman/guestfs/kvmpart/localfs.go

@@ -224,15 +224,51 @@ func (f *SLocalGuestFS) Chmod(sPath string, mode uint32, caseInsensitive bool) e
 	return nil
 }
 
+func (f *SLocalGuestFS) updateUserEtcShadow(username string) error {
+	sPath := f.GetLocalPath("/etc/shadow", false)
+	if !fileutils2.Exists(sPath) {
+		return nil
+	}
+	content, err := fileutils2.FileGetContents(sPath)
+	if err != nil {
+		return errors.Wrap(err, "read /etc/shadow")
+	}
+
+	var (
+		minimumDays = "0"     // -m 0
+		maximumDays = "99999" // -M 99999
+	)
+
+	lines := strings.Split(string(content), "\n")
+	for i, line := range lines {
+		fields := strings.Split(line, ":")
+		if len(fields) >= 7 && fields[0] == username {
+			fields[3] = minimumDays
+			fields[4] = maximumDays
+			fields[5] = "" // password warning period
+			fields[6] = "" // password inactivity period
+			fields[7] = "" // account expiration date
+			line = strings.Join(fields, ":")
+			lines[i] = line
+			break
+		}
+	}
+	newContent := strings.Join(lines, "\n")
+	err = fileutils2.FilePutContents(sPath, newContent, false)
+	if err != nil {
+		return errors.Wrapf(err, "read %s, put %s to /etc/shadow", content, newContent)
+	}
+
+	return nil
+}
+
 func (f *SLocalGuestFS) CheckOrAddUser(user, homeDir string, isSys bool) (realHomeDir string, err error) {
 	var exist bool
 	if exist, realHomeDir, err = f.checkUser(user); err != nil || exist {
 		if exist {
-			cmd := []string{"chage", "-R", f.mountPath, "-E", "-1", "-m", "0", "-M", "99999", "-I", "-1", user}
-			command := procutils.NewCommand(cmd[0], cmd[1:]...)
-			_, err = command.Output()
+			err = f.updateUserEtcShadow(user)
 			if err != nil {
-				err = errors.Wrap(err, "chage")
+				err = errors.Wrap(err, "updateUserEtcShadow")
 				return
 			}
 			if !f.Exists(realHomeDir, false) {

pkg/hostman/guestfs/sshpart/sshpart.go

@@ -40,17 +40,17 @@ type SSHPartition struct {
 	term      *ssh.Client
 	partDev   string
 	mountPath string
-	part      *disktool.Partition
+	isLVM     bool
 }
 
 var _ fsdriver.IDiskPartition = &SSHPartition{}
 
-func NewSSHPartition(term *ssh.Client, part *disktool.Partition) *SSHPartition {
+func NewSSHPartition(term *ssh.Client, partDev string, isLVM bool) *SSHPartition {
 	p := new(SSHPartition)
 	p.term = term
-	p.partDev = part.GetDev()
+	p.partDev = partDev
+	p.isLVM = isLVM
 	p.mountPath = fmt.Sprintf("/tmp/%s", strings.Replace(p.partDev, "/", "_", -1))
-	p.part = part
 	return p
 }
 
@@ -616,7 +616,7 @@ func MountSSHRootfs(tool *disktool.SSHPartitionTool, term *ssh.Client, layouts [
 		return nil, nil, fmt.Errorf("Not found root disk partitions")
 	}
 	for _, part := range parts {
-		dev := NewSSHPartition(term, part)
+		dev := NewSSHPartition(term, part.GetDev(), false)
 		if !dev.Mount() {
 			continue
 		}

pkg/hostman/guestman/desc/desc.go

@@ -343,7 +343,7 @@ type SGuestControlDesc struct {
 
 	EncryptKeyId string
 
-	RescueMode bool // rescue mode
+	LightMode bool // light mode
 }
 
 type SGuestMetaDesc struct {

pkg/hostman/guestman/qemu-kvm.go

@@ -787,7 +787,7 @@ func (s *SKVMGuestInstance) StartMonitorWithImportGuestSocketFile(ctx context.Co
 		}, // on monitor connected
 		s.onReceiveQMPEvent, // on reveive qmp event
 	)
-	return mon.ConnectWithSocket(socketFile)
+	return mon.ConnectWithSocket(socketFile, 0)
 }
 
 func (s *SKVMGuestInstance) StartMonitor(ctx context.Context, cb func()) error {
@@ -1651,13 +1651,13 @@ func (s *SKVMGuestInstance) SaveSourceDesc(guestDesc *desc.SGuestDesc) error {
 	}
 
 	// Save rescue desc if exist
-	if s.SourceDesc.RescueMode {
-		err := s.GetRescueDesc()
-		if err != nil {
-			log.Errorf("get rescue desc failed %s", err)
-			return errors.Wrap(err, "get rescue desc")
-		}
-	}
+	//if s.SourceDesc.LightMode {
+	//	err := s.GetRescueDesc()
+	//	if err != nil {
+	//		log.Errorf("get rescue desc failed %s", err)
+	//		return errors.Wrap(err, "get rescue desc")
+	//	}
+	//}
 
 	if err := fileutils2.FilePutContents(
 		s.GetSourceDescFilePath(), jsonutils.Marshal(s.SourceDesc).String(), false,
@@ -1679,29 +1679,29 @@ func (s *SKVMGuestInstance) GetVpcNIC() *desc.SGuestNetwork {
 	return nil
 }
 
-func (s *SKVMGuestInstance) GetRescueDesc() error {
-	if !s.SourceDesc.RescueMode {
-		return errors.Errorf("guest %s not in rescue mode", s.Id)
-	}
-
-	s.SourceDesc.RescueInitdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS)
-	s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL)
-	s.SourceDesc.RescueDiskPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_SYS_DISK_NAME)
-	if s.manager.GetHost().IsAarch64() {
-		s.SourceDesc.RescueInitdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS_ARM64)
-		s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL_ARM64)
-	}
-
-	// Address
-	bus, slot, found := s.findUnusedSlotForController(desc.CONTROLLER_TYPE_PCI_ROOT, 0)
-	if !found {
-		return errors.Errorf("no valid pci address found ?")
-	}
-	s.SourceDesc.RescueDiskDeviceBus = uint(bus)
-	s.SourceDesc.RescueDiskDeviceSlot = uint(slot)
-
-	return nil
-}
+//func (s *SKVMGuestInstance) GetRescueDesc() error {
+//	if !s.SourceDesc.LightMode {
+//		return errors.Errorf("guest %s not in rescue mode", s.Id)
+//	}
+//
+//	s.SourceDesc.RescueInitrdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS)
+//	s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL)
+//	s.SourceDesc.RescueDiskPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_SYS_DISK_NAME)
+//	if s.manager.GetHost().IsAarch64() {
+//		s.SourceDesc.RescueInitrdPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS_ARM64)
+//		s.SourceDesc.RescueKernelPath = path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL_ARM64)
+//	}
+//
+//	// Address
+//	bus, slot, found := s.findUnusedSlotForController(desc.CONTROLLER_TYPE_PCI_ROOT, 0)
+//	if !found {
+//		return errors.Errorf("no valid pci address found ?")
+//	}
+//	s.SourceDesc.RescueDiskDeviceBus = uint(bus)
+//	s.SourceDesc.RescueDiskDeviceSlot = uint(slot)
+//
+//	return nil
+//}
 
 type guestStartTask struct {
 	s *SKVMGuestInstance

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -558,12 +558,9 @@ function nic_mtu() {
 	}
 
 	// set rescue flag to input
-	if s.Desc.RescueMode {
-		input.RescueInitdPath = s.SourceDesc.RescueInitdPath
-		input.RescueKernelPath = s.SourceDesc.RescueKernelPath
-		input.RescueDiskPath = s.SourceDesc.RescueDiskPath
-		input.RescueDiskDeviceBus = s.SourceDesc.RescueDiskDeviceBus
-		input.RescueDiskDeviceSlot = s.SourceDesc.RescueDiskDeviceSlot
+	if s.Desc.LightMode {
+		input.RescueInitrdPath = s.getRescueInitrdPath()
+		input.RescueKernelPath = s.getRescueKernelPath()
 	}
 
 	qemuOpts, err := qemu.GenerateStartOptions(input)
@@ -578,6 +575,20 @@ function nic_mtu() {
 	return cmd, nil
 }
 
+func (s *SKVMGuestInstance) getRescueInitrdPath() string {
+	if s.manager.GetHost().IsAarch64() {
+		return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS_ARM64)
+	}
+	return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_INITRAMFS)
+}
+
+func (s *SKVMGuestInstance) getRescueKernelPath() string {
+	if s.manager.GetHost().IsAarch64() {
+		return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL_ARM64)
+	}
+	return path.Join(s.GetRescueDirPath(), api.GUEST_RESCUE_KERNEL)
+}
+
 func (s *SKVMGuestInstance) slaveDiskPrepare(input *qemu.GenerateStartOptionsInput, diskUri string) error {
 	for i := 0; i < len(input.GuestDesc.Disks); i++ {
 		diskPath := input.GuestDesc.Disks[i].Path

pkg/hostman/guestman/qemu/generate.go

@@ -219,25 +219,10 @@ func generateScsiOptions(scsi *desc.SGuestVirtioScsi) string {
 	return opt
 }
 
-func generateInitrdOptions(drvOpt QemuOptions, initrdPath, kernel, sys_img string, rescueDiskDeviceBus, rescueDiskDeviceSlot uint, nics []*desc.SGuestNetwork) []string {
+func generateInitrdOptions(drvOpt QemuOptions, initrdPath, kernel string) []string {
 	opts := make([]string, 0)
-	opts = append(opts, fmt.Sprintf("-initrd %s", initrdPath))
-	opts = append(opts, fmt.Sprintf("-kernel %s", kernel))
-
-	// create temp disk info
-	driveString := fmt.Sprintf("file=%s,if=none,id=initrd,cache=none,aio=native,file.locking=off", sys_img)
-	opts = append(opts, drvOpt.Drive(driveString))
-	deviceString := fmt.Sprintf("virtio-blk-pci,drive=initrd,iothread=iothread0,bus=pci.%d,addr=0x%02x,id=initrd,bootindex=1", rescueDiskDeviceBus, rescueDiskDeviceSlot)
-	opts = append(opts, drvOpt.Device(deviceString))
-
-	// add ip config
-	//var ips []string
-	//for _, nic := range nics {
-	//	ips = append(ips, fmt.Sprintf("ip=%s:%s:%s:%s:%s:%s:off,", nic.Ip, "", nic.Gateway, netutils.Masklen2Mask(nic.Masklen).String(), "", nic.Ifname))
-	//}
-	//appendIps := strings.Join(ips, ",")
-	//
-	//opts = append(opts, fmt.Sprintf("-append %s", appendIps))
+	opts = append(opts, drvOpt.Initrd(initrdPath))
+	opts = append(opts, drvOpt.Kernel(kernel))
 
 	return opts
 }
@@ -659,11 +644,8 @@ type GenerateStartOptionsInput struct {
 
 	EncryptKeyPath string
 
-	RescueInitdPath      string // rescue initramfs path
-	RescueKernelPath     string // rescue kernel path
-	RescueDiskPath       string // rescue disk path
-	RescueDiskDeviceBus  uint
-	RescueDiskDeviceSlot uint
+	RescueInitrdPath string // rescue initramfs path
+	RescueKernelPath string // rescue kernel path
 }
 
 func (input *GenerateStartOptionsInput) HasBootIndex() bool {
@@ -788,15 +770,11 @@ func GenerateStartOptions(
 	}
 
 	// generate initrd and kernel options
-	if input.RescueInitdPath != "" {
+	if input.GuestDesc.LightMode {
 		opts = append(opts, generateInitrdOptions(
 			drvOpt,
-			input.RescueInitdPath,
+			input.RescueInitrdPath,
 			input.RescueKernelPath,
-			input.RescueDiskPath,
-			input.RescueDiskDeviceBus,
-			input.RescueDiskDeviceSlot,
-			input.GuestDesc.Nics,
 		)...)
 	}
 
@@ -835,7 +813,7 @@ func GenerateStartOptions(
 	}
 
 	// isolated devices
-	if len(input.GuestDesc.IsolatedDevices) > 0 {
+	if len(input.GuestDesc.IsolatedDevices) > 0 && !input.GuestDesc.LightMode {
 		opts = append(opts, generateIsolatedDeviceOptions(input.GuestDesc)...)
 	}
 

pkg/hostman/guestman/qemu/qemu.go

@@ -106,6 +106,8 @@ type QemuOptions interface {
 	Pidfile(file string) string
 	USB() string
 	VNC(port uint, usePasswd bool) string
+	Initrd(initrdPath string) string
+	Kernel(kernelPath string) string
 }
 
 var (
@@ -328,6 +330,14 @@ func (o baseOptions) USB() string {
 	return "-usb"
 }
 
+func (o baseOptions) Initrd(initrdPath string) string {
+	return "-initrd " + initrdPath
+}
+
+func (o baseOptions) Kernel(kernelPath string) string {
+	return "-kernel " + kernelPath
+}
+
 func (o baseOptions) VNC(port uint, usePasswd bool) string {
 	opt := fmt.Sprintf("-vnc :%d", port)
 	if usePasswd {

pkg/hostman/hostdeployer/consts/consts.go

@@ -17,4 +17,6 @@ package consts
 const (
 	DEPLOY_DRIVER_NBD        = "nbd"
 	DEPLOY_DRIVER_LIBGUESTFS = "libguestfs"
+	DEPLOY_DRIVER_QEMU_KVM   = "qemu-kvm"
+	DEPLOY_DRIVER_LOCAL_DISK = "local"
 )

pkg/hostman/hostdeployer/deployserver/deployserver.go

@@ -16,6 +16,7 @@ package deployserver
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"net"
 	"os"
@@ -29,11 +30,12 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
+	commonconsts "yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	"yunion.io/x/onecloud/pkg/cloudcommon/service"
 	"yunion.io/x/onecloud/pkg/hostman/diskutils"
 	"yunion.io/x/onecloud/pkg/hostman/diskutils/libguestfs"
 	"yunion.io/x/onecloud/pkg/hostman/diskutils/nbd"
-	"yunion.io/x/onecloud/pkg/hostman/guestfs"
+	"yunion.io/x/onecloud/pkg/hostman/diskutils/qemu_kvm"
 	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
 	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
 	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts"
@@ -92,32 +94,20 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 	}
 	defer disk.Cleanup()
 
-	if err := disk.Connect(); err != nil {
+	if err := disk.Connect(req.GuestDesc); err != nil {
 		log.Errorf("Failed to connect %s disk: %s", req.GuestDesc.Hypervisor, err)
 		return new(deployapi.DeployGuestFsResponse), errors.Wrap(err, "Connect")
 	}
 	defer disk.Disconnect()
-	root, err := disk.MountRootfs()
-	if err != nil {
-		if errors.Cause(err) == errors.ErrNotFound && req.DeployInfo.IsInit {
-			// if init deploy, ignore no partition error
-			log.Errorf("disk.MountRootfs not found partition, not init, quit")
-			return new(deployapi.DeployGuestFsResponse), nil
-		}
-		log.Errorf("Failed mounting rootfs for %s disk: %s", req.GuestDesc.Hypervisor, err)
-		return new(deployapi.DeployGuestFsResponse), err
-	}
-	defer disk.UmountRootfs(root)
-	ret, err := guestfs.DoDeployGuestFs(root, req.GuestDesc, req.DeployInfo)
-	if err != nil {
-		log.Errorf("guestfs.DoDeployGuestFs fail %s", err)
-		return new(deployapi.DeployGuestFsResponse), err
-	}
+
+	ret, err := disk.DeployGuestfs(req)
 	if ret == nil {
-		log.Errorf("guestfs.DoDeployGuestFs return empty results")
-		return new(deployapi.DeployGuestFsResponse), nil
+		ret = new(deployapi.DeployGuestFsResponse)
 	}
-	return ret, nil
+	if err != nil {
+		log.Errorf("failed deploy guest fs: %s", err)
+	}
+	return ret, err
 }
 
 func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsParams) (res *deployapi.Empty, err error) {
@@ -142,44 +132,12 @@ func (*DeployerServer) ResizeFs(ctx context.Context, req *deployapi.ResizeFsPara
 		return new(deployapi.Empty), errors.Wrap(err, "GetIDisk fail")
 	}
 	defer disk.Cleanup()
-	if err := disk.Connect(); err != nil {
+	if err := disk.Connect(nil); err != nil {
 		return new(deployapi.Empty), errors.Wrap(err, "disk connect failed")
 	}
 	defer disk.Disconnect()
 
-	unmount := func(root fsdriver.IRootFsDriver) error {
-		err := disk.UmountRootfs(root)
-		if err != nil {
-			return errors.Wrap(err, "unmount rootfs")
-		}
-		return nil
-	}
-
-	root, err := disk.MountRootfs()
-	if err != nil {
-		if errors.Cause(err) == errors.ErrNotFound {
-			return new(deployapi.Empty), nil
-		}
-		return new(deployapi.Empty), errors.Wrapf(err, "disk.MountRootfs")
-	}
-	if !root.IsResizeFsPartitionSupport() {
-		err := unmount(root)
-		if err != nil {
-			return new(deployapi.Empty), err
-		}
-		return new(deployapi.Empty), errors.ErrNotSupported
-	}
-
-	// must umount rootfs before resize partition
-	err = unmount(root)
-	if err != nil {
-		return new(deployapi.Empty), err
-	}
-	err = disk.ResizePartition()
-	if err != nil {
-		return new(deployapi.Empty), errors.Wrap(err, "resize disk partition")
-	}
-	return new(deployapi.Empty), nil
+	return disk.ResizeFs()
 }
 
 func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsParams) (*deployapi.Empty, error) {
@@ -190,16 +148,9 @@ func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsPara
 	}
 	defer gd.Cleanup()
 
-	if err := gd.Connect(); err == nil {
+	if err := gd.Connect(nil); err == nil {
 		defer gd.Disconnect()
-		if err := gd.MakePartition(req.FsFormat); err == nil {
-			err = gd.FormatPartition(req.FsFormat, req.Uuid)
-			if err != nil {
-				return new(deployapi.Empty), errors.Wrap(err, "FormatPartition")
-			}
-		} else {
-			return new(deployapi.Empty), errors.Wrap(err, "MakePartition")
-		}
+		return gd.FormatFs(req)
 	} else {
 		log.Errorf("failed connect kvm disk %#v: %s", apiDiskInfo(req.DiskInfo), err)
 	}
@@ -208,75 +159,20 @@ func (*DeployerServer) FormatFs(ctx context.Context, req *deployapi.FormatFsPara
 
 func (*DeployerServer) SaveToGlance(ctx context.Context, req *deployapi.SaveToGlanceParams) (*deployapi.SaveToGlanceResponse, error) {
 	log.Infof("********* %#v save to glance", apiDiskInfo(req.DiskInfo))
-	var (
-		osInfo  string
-		relInfo *deployapi.ReleaseInfo
-	)
+
 	kvmDisk, err := diskutils.NewKVMGuestDisk(apiDiskInfo(req.GetDiskInfo()), DeployOption.ImageDeployDriver, false)
 	if err != nil {
 		return new(deployapi.SaveToGlanceResponse), errors.Wrap(err, "NewKVMGuestDisk")
 	}
 	defer kvmDisk.Cleanup()
 
-	ret := &deployapi.SaveToGlanceResponse{
-		OsInfo:      osInfo,
-		ReleaseInfo: relInfo,
-	}
-
-	err = kvmDisk.Connect()
+	err = kvmDisk.Connect(nil)
 	if err != nil {
-		return ret, errors.Wrapf(err, "kvmDisk.Connect %#v", apiDiskInfo(req.DiskInfo))
+		return new(deployapi.SaveToGlanceResponse), errors.Wrapf(err, "kvmDisk.Connect %#v", apiDiskInfo(req.DiskInfo))
 	}
 	defer kvmDisk.Disconnect()
 
-	root, err := kvmDisk.MountKvmRootfs()
-	if err != nil {
-		if errors.Cause(err) == errors.ErrNotFound {
-			return ret, nil
-		}
-		return ret, errors.Wrapf(err, "MountKvmRootfs")
-	}
-	defer kvmDisk.UmountKvmRootfs(root)
-
-	osInfo = root.GetOs()
-	relInfo = root.GetReleaseInfo(root.GetPartition())
-	if req.Compress {
-		err = root.PrepareFsForTemplate(root.GetPartition())
-		if err != nil {
-			log.Errorf("PrepareFsForTemplate %s", err)
-		}
-	}
-	if req.Compress {
-		kvmDisk.Zerofree()
-	}
-	return ret, err
-}
-
-func (*DeployerServer) getImageInfo(kvmDisk *diskutils.SKVMGuestDisk) (*deployapi.ImageInfo, error) {
-	// Fsck is executed during mount
-	rootfs, err := kvmDisk.MountKvmRootfs()
-	if err != nil {
-		if errors.Cause(err) == errors.ErrNotFound {
-			return new(deployapi.ImageInfo), nil
-		}
-		return new(deployapi.ImageInfo), errors.Wrapf(err, "kvmDisk.MountKvmRootfs")
-	}
-	partition := rootfs.GetPartition()
-	imageInfo := &deployapi.ImageInfo{
-		OsInfo:               rootfs.GetReleaseInfo(partition),
-		OsType:               rootfs.GetOs(),
-		IsLvmPartition:       kvmDisk.IsLVMPartition(),
-		IsReadonly:           partition.IsReadonly(),
-		IsInstalledCloudInit: rootfs.IsCloudinitInstall(),
-	}
-	kvmDisk.UmountKvmRootfs(rootfs)
-
-	// In case of deploy driver is guestfish, we can't mount
-	// multi partition concurrent, so we need umount rootfs first
-	imageInfo.IsUefiSupport = kvmDisk.DetectIsUEFISupport(rootfs)
-	imageInfo.PhysicalPartitionType = partition.GetPhysicalPartitionType()
-	log.Infof("ProbeImageInfo response %s", imageInfo)
-	return imageInfo, nil
+	return kvmDisk.SaveToGlance(req)
 }
 
 func (s *DeployerServer) ProbeImageInfo(ctx context.Context, req *deployapi.ProbeImageInfoPramas) (*deployapi.ImageInfo, error) {
@@ -287,13 +183,13 @@ func (s *DeployerServer) ProbeImageInfo(ctx context.Context, req *deployapi.Prob
 	}
 	defer kvmDisk.Cleanup()
 
-	if err := kvmDisk.Connect(); err != nil {
+	if err := kvmDisk.Connect(nil); err != nil {
 		log.Errorf("Failed to connect kvm disk %#v: %s", apiDiskInfo(req.DiskInfo), err)
 		return new(deployapi.ImageInfo), errors.Wrap(err, "Disk connector failed to connect image")
 	}
 	defer kvmDisk.Disconnect()
 
-	return s.getImageInfo(kvmDisk)
+	return kvmDisk.ProbeImageInfo(req)
 }
 
 var connectedEsxiDisks = map[string]*diskutils.VDDKDisk{}
@@ -365,6 +261,22 @@ func NewDeployService() *SDeployService {
 }
 
 func (s *SDeployService) RunService() {
+	if DeployOption.DeployAction != "" {
+		res, err := StartLocalDeploy(DeployOption.DeployAction)
+		if err != nil {
+			if e := fileutils2.FilePutContents("/error", err.Error(), false); e != nil {
+				log.Errorf("failed put errors to file: %s", e)
+			}
+		}
+		if res != nil {
+			resStr, _ := json.Marshal(res)
+			if e := fileutils2.FilePutContents("/response", string(resStr), false); e != nil {
+				log.Errorf("failed put response to file: %s", e)
+			}
+		}
+		return
+	}
+
 	s.grpcServer = grpc.NewServer()
 	deployapi.RegisterDeployAgentServer(s.grpcServer, &DeployerServer{})
 	if fileutils2.Exists(DeployOption.DeployServerSocketPath) {
@@ -399,12 +311,71 @@ func (s *SDeployService) FixPathEnv() error {
 }
 
 func (s *SDeployService) PrepareEnv() error {
+	if !fileutils2.Exists(DeployOption.DeployTempDir) {
+		err := os.MkdirAll(DeployOption.DeployTempDir, 0755)
+		if err != nil {
+			return errors.Errorf("fail to create %s: %s", DeployOption.DeployTempDir, err)
+		}
+	}
+	commonconsts.SetDeployTempDir(DeployOption.DeployTempDir)
+	commonconsts.SetAllowVmSELinux(DeployOption.AllowVmSELinux)
+
 	if err := s.FixPathEnv(); err != nil {
 		return err
 	}
 
-	if err := nbd.Init(); err != nil {
-		return errors.Wrap(err, "nbd.Init")
+	if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil {
+		log.Fatalln(err)
+	}
+	if DeployOption.ImageDeployDriver == consts.DEPLOY_DRIVER_LIBGUESTFS {
+		if err := libguestfs.Init(3); err != nil {
+			log.Fatalln(err)
+		}
+	}
+
+	if DeployOption.ImageDeployDriver != consts.DEPLOY_DRIVER_QEMU_KVM {
+		if err := nbd.Init(); err != nil {
+			return errors.Wrap(err, "nbd.Init")
+		}
+	} else {
+		// prepare for yunionos don't have but necessary files
+		out, err := procutils.NewCommand("cp", "-rf", "/usr/bin/chntpw.static", "/opt/yunion/bin/chntpw.static").Output()
+		if err != nil {
+			return errors.Wrapf(err, "cp files failed %s", out)
+		}
+		out, err = procutils.NewCommand("cp", "-rf", "/usr/bin/.chntpw.static.bin", "/opt/yunion/bin/.chntpw.static.bin").Output()
+		if err != nil {
+			return errors.Wrapf(err, "cp files failed %s", out)
+		}
+		out, err = procutils.NewCommand("mkdir", "-p", "/opt/yunion/bin/bundles").Output()
+		if err != nil {
+			return errors.Wrapf(err, "cp files failed %s", out)
+		}
+		out, err = procutils.NewCommand("cp", "-rf", "/usr/bin/bundles/chntpw.static", "/opt/yunion/bin/bundles/chntpw.static").Output()
+		if err != nil {
+			return errors.Wrapf(err, "cp files failed %s", out)
+		}
+		out, err = procutils.NewCommand("cp", "-rf", "/usr/sbin/zerofree", "/opt/yunion/bin/zerofree").Output()
+		if err != nil {
+			return errors.Wrapf(err, "cp files failed %s", out)
+		}
+
+		cmd := fmt.Sprintf("mkisofs -l -J -L -R -r -v -hide-rr-moved -o %s -graft-points vmware-vddk=/opt/vmware-vddk yunion=/opt/yunion", qemu_kvm.DEPLOY_ISO)
+		out, err = procutils.NewCommand("bash", "-c", cmd).Output()
+		if err != nil {
+			return errors.Wrapf(err, "mkisofs failed %s", out)
+		}
+
+		cpuArch, err := procutils.NewCommand("uname", "-m").Output()
+		if err != nil {
+			return errors.Wrap(err, "get cpu architecture")
+		}
+		qemu_kvm.InitQemuDeployManager(
+			strings.TrimSpace(string(cpuArch)),
+			DeployOption.HugepagesOption == "native",
+			DeployOption.HugepageSizeMb*1024,
+			DeployOption.DeployConcurrent,
+		)
 	}
 
 	// create /dev/lvm_remote
@@ -454,6 +425,14 @@ func (s *SDeployService) InitService() {
 			}
 		}
 	})
+
+	if DeployOption.DeployAction != "" {
+		if err := LocalInitEnv(); err != nil {
+			log.Fatalf("local init env %s", err)
+		}
+		return
+	}
+
 	log.Infof("exec socket path: %s", DeployOption.ExecutorSocketPath)
 	if DeployOption.EnableRemoteExecutor {
 		execlient.Init(DeployOption.ExecutorSocketPath)
@@ -464,14 +443,7 @@ func (s *SDeployService) InitService() {
 	if err := s.PrepareEnv(); err != nil {
 		log.Fatalln(err)
 	}
-	if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil {
-		log.Fatalln(err)
-	}
-	if DeployOption.ImageDeployDriver == consts.DEPLOY_DRIVER_LIBGUESTFS {
-		if err := libguestfs.Init(3); err != nil {
-			log.Fatalln(err)
-		}
-	}
+
 	s.O = &DeployOption.BaseOptions
 	if len(DeployOption.DeployServerSocketPath) == 0 {
 		log.Fatalf("missing deploy server socket path")

pkg/hostman/hostdeployer/deployserver/localdeploy.go

@@ -0,0 +1,152 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package deployserver
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+
+	"github.com/sirupsen/logrus"
+
+	"yunion.io/x/pkg/errors"
+
+	"yunion.io/x/onecloud/pkg/hostman/diskutils"
+	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
+	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
+	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/consts"
+	"yunion.io/x/onecloud/pkg/util/qemuimg"
+	"yunion.io/x/onecloud/pkg/util/winutils"
+)
+
+type LocalDeploy struct{}
+
+func (d *LocalDeploy) DeployGuestFs(req *deployapi.DeployParams) (res *deployapi.DeployGuestFsResponse, err error) {
+	localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "new local disk")
+	}
+	if err := localDisk.Connect(req.GuestDesc); err != nil {
+		return nil, errors.Wrapf(err, "local disk connect")
+	}
+	defer localDisk.Disconnect()
+	return localDisk.DeployGuestfs(req)
+}
+
+func (d *LocalDeploy) ResizeFs(req *deployapi.ResizeFsParams) (res *deployapi.Empty, err error) {
+	localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "new local disk")
+	}
+	if err := localDisk.Connect(nil); err != nil {
+		return nil, errors.Wrapf(err, "local disk connect")
+	}
+	defer localDisk.Disconnect()
+	return localDisk.ResizeFs()
+}
+
+func (d *LocalDeploy) FormatFs(req *deployapi.FormatFsParams) (*deployapi.Empty, error) {
+	localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "new local disk")
+	}
+	if err := localDisk.Connect(nil); err != nil {
+		return nil, errors.Wrapf(err, "local disk connect")
+	}
+	defer localDisk.Disconnect()
+	return localDisk.FormatFs(req)
+}
+
+func (d *LocalDeploy) SaveToGlance(req *deployapi.SaveToGlanceParams) (*deployapi.SaveToGlanceResponse, error) {
+	localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "new local disk")
+	}
+	if err := localDisk.Connect(nil); err != nil {
+		return nil, errors.Wrapf(err, "local disk connect")
+	}
+	defer localDisk.Disconnect()
+	return localDisk.SaveToGlance(req)
+}
+
+func (d *LocalDeploy) ProbeImageInfo(req *deployapi.ProbeImageInfoPramas) (*deployapi.ImageInfo, error) {
+	localDisk, err := diskutils.NewKVMGuestDisk(qemuimg.SImageInfo{}, consts.DEPLOY_DRIVER_LOCAL_DISK, false)
+	if err != nil {
+		return nil, errors.Wrap(err, "new local disk")
+	}
+	if err := localDisk.Connect(nil); err != nil {
+		return nil, errors.Wrapf(err, "local disk connect")
+	}
+	defer localDisk.Disconnect()
+	return localDisk.ProbeImageInfo(req)
+}
+
+func LocalInitEnv() error {
+	f, _ := os.OpenFile("/log", os.O_CREATE|os.O_WRONLY, 0777)
+	logrus.SetOutput(f)
+
+	// /bin:/sbin:/usr/bin:/usr/sbin
+	var paths = []string{
+		"/bin",
+		"/sbin",
+		"/usr/bin",
+		"/usr/sbin",
+		"/opt/yunion/bin", // for zerofree command
+	}
+	err := os.Setenv("PATH", strings.Join(paths, ":"))
+	if err != nil {
+		return errors.Wrap(err, "set env path")
+	}
+
+	if err := fsdriver.Init(DeployOption.PrivatePrefixes, DeployOption.CloudrootDir); err != nil {
+		return errors.Wrap(err, "init fsdriver")
+	}
+	winutils.SetChntpwPath("/opt/yunion/bin/chntpw.static")
+	return nil
+}
+
+func StartLocalDeploy(deployAction string) (interface{}, error) {
+	localDeployer := LocalDeploy{}
+	switch deployAction {
+	case "deploy_guest_fs":
+		params := new(deployapi.DeployParams)
+		if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil {
+			return nil, errors.Wrap(err, "unmarshal params")
+		}
+		return localDeployer.DeployGuestFs(params)
+	case "resize_fs":
+		return localDeployer.ResizeFs(nil)
+	case "format_fs":
+		params := new(deployapi.FormatFsParams)
+		if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil {
+			return nil, errors.Wrap(err, "unmarshal params")
+		}
+		return localDeployer.FormatFs(params)
+	case "save_to_glance":
+		params := new(deployapi.SaveToGlanceParams)
+		if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil {
+			return nil, errors.Wrap(err, "unmarshal params")
+		}
+		return localDeployer.SaveToGlance(params)
+	case "probe_image_info":
+		params := new(deployapi.ProbeImageInfoPramas)
+		if err := json.Unmarshal([]byte(DeployOption.DeployParams), params); err != nil {
+			return nil, errors.Wrap(err, "unmarshal params")
+		}
+		return localDeployer.ProbeImageInfo(params)
+	default:
+		return nil, errors.Errorf("unknown deploy action")
+	}
+}

pkg/hostman/hostdeployer/deployserver/options.go

@@ -17,11 +17,7 @@ package deployserver
 import (
 	"os"
 
-	"yunion.io/x/log"
-
-	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
 	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
-	"yunion.io/x/onecloud/pkg/util/fileutils2"
 )
 
 type SDeployOptions struct {
@@ -31,12 +27,19 @@ type SDeployOptions struct {
 	ChntpwPath      string   `help:"path to chntpw tool" default:"/usr/local/bin/chntpw.static"`
 
 	CloudrootDir      string `help:"User cloudroot home dir" default:"/opt"`
-	ImageDeployDriver string `help:"Image deploy driver" default:"nbd" choices:"nbd|libguestfs"`
+	ImageDeployDriver string `help:"Image deploy driver" default:"qemu-kvm" choices:"qemu-kvm|nbd|libguestfs"`
 	CommonConfigFile  string `help:"common config file for container"`
 
 	DeployTempDir string `help:"temp dir for deployer" default:"/opt/cloud/workspace/run/deploy"`
 
 	AllowVmSELinux bool `help:"turn off vm selinux" default:"false" json:"allow_vm_selinux"`
+
+	HugepagesOption string `help:"Hugepages option: disable|native|transparent" default:"transparent"`
+	HugepageSizeMb  int    `help:"hugepage size mb default 1G" default:"1024"`
+
+	DeployAction     string `help:"local deploy action"`
+	DeployParams     string `help:"params for deploy action"`
+	DeployConcurrent int    `help:"qemu-kvm deploy driver concurrent" default:"3"`
 }
 
 var DeployOption SDeployOptions
@@ -52,15 +55,6 @@ func Parse() (hostOpts SDeployOptions) {
 		// keep base options
 		hostOpts.BaseOptions.BaseOptions = baseOpt
 	}
-	if !fileutils2.Exists(hostOpts.DeployTempDir) {
-		err := os.MkdirAll(hostOpts.DeployTempDir, 0755)
-		if err != nil {
-			log.Fatalf("fail to create %s: %s", hostOpts.DeployTempDir, err)
-			return
-		}
-	}
-	consts.SetDeployTempDir(hostOpts.DeployTempDir)
-	consts.SetAllowVmSELinux(hostOpts.AllowVmSELinux)
 	return hostOpts
 }
 

pkg/hostman/monitor/monitor.go

@@ -186,7 +186,7 @@ func (self *BlockJob) CalcOffset(preOffset int64) {
 
 type Monitor interface {
 	Connect(host string, port int) error
-	ConnectWithSocket(address string) error
+	ConnectWithSocket(address string, timeout time.Duration) error
 	Disconnect()
 	IsConnected() bool
 
@@ -307,6 +307,12 @@ func (m *SBaseMonitor) onConnectSuccess(conn net.Conn) {
 	m.rwc = conn
 }
 
+func (m *SBaseMonitor) SetReadDeadlineTimeout(duration time.Duration) {
+	if m.rwc != nil {
+		m.rwc.SetReadDeadline(time.Now().Add(duration))
+	}
+}
+
 func (m *SBaseMonitor) Connect(host string, port int) error {
 	return m.connect("tcp", fmt.Sprintf("%s:%d", host, port))
 }

pkg/hostman/monitor/qmp.go

@@ -311,7 +311,7 @@ func (m *QmpMonitor) Query(cmd *Command, cb qmpMonitorCallBack) {
 	}
 }
 
-func (m *QmpMonitor) ConnectWithSocket(address string) error {
+func (m *QmpMonitor) ConnectWithSocket(address string, timeout time.Duration) error {
 	err := m.SBaseMonitor.connect("unix", address)
 	if err != nil {
 		return err