Merge pull request #12901 from zexi/qemu-4.0.1

feat(host): qemu 4.0.1 supported

pkg/hostman/guestman/guesttasks.go

@@ -29,6 +29,7 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	"yunion.io/x/onecloud/pkg/appctx"
+	"yunion.io/x/onecloud/pkg/hostman/guestman/qemu"
 	"yunion.io/x/onecloud/pkg/hostman/hostinfo"
 	"yunion.io/x/onecloud/pkg/hostman/hostutils"
 	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
@@ -310,7 +311,7 @@ func (d *SGuestDiskSyncTask) onAddDiskSucc(disk jsonutils.JSONObject, results st
 	var (
 		diskIndex, _  = disk.Int("index")
 		diskDirver, _ = disk.GetString("driver")
-		dev           = d.guest.GetDiskDeviceModel(diskDirver)
+		dev           = qemu.GetDiskDeviceModel(diskDirver)
 	)
 
 	var params = map[string]interface{}{

pkg/hostman/guestman/qemu-arm.go

@@ -1,297 +0,0 @@
-// Copyright 2019 Yunion
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package guestman
-
-import (
-	"fmt"
-
-	"yunion.io/x/jsonutils"
-	"yunion.io/x/pkg/errors"
-
-	api "yunion.io/x/onecloud/pkg/apis/compute"
-	options "yunion.io/x/onecloud/pkg/hostman/options"
-	"yunion.io/x/onecloud/pkg/util/fileutils2"
-	"yunion.io/x/onecloud/pkg/util/qemutils"
-)
-
-func (s *SKVMGuestInstance) getArmMachine() string {
-	return "virt-2.12"
-}
-
-// arm cpu unsupport hackintosh
-func (s *SKVMGuestInstance) generateArmStartScript(data *jsonutils.JSONDict) (string, error) {
-	var (
-		uuid, _ = s.Desc.GetString("uuid")
-		mem, _  = s.Desc.Int("mem")
-		cpu, _  = s.Desc.Int("cpu")
-		name, _ = s.Desc.GetString("name")
-		nics, _ = s.Desc.GetArray("nics")
-		osname  = s.getOsname()
-		cmd     = ""
-	)
-	disks := make([]api.GuestdiskJsonDesc, 0)
-	s.Desc.Unmarshal(&disks, "disks")
-
-	vncPort, _ := data.Int("vnc_port")
-
-	qemuVersion := options.HostOptions.DefaultQemuVersion
-	if data.Contains("qemu_version") {
-		qemuVersion, _ = data.GetString("qemu_version")
-	}
-	if qemuVersion == "latest" {
-		qemuVersion = ""
-	}
-
-	var devAddrs = []string{}
-	isolatedParams, _ := s.Desc.GetArray("isolated_devices")
-	for _, params := range isolatedParams {
-		devAddr, _ := params.GetString("addr")
-		devAddrs = append(devAddrs, devAddr)
-	}
-	isolatedDevsParams := s.manager.GetHost().GetIsolatedDeviceManager().GetQemuParams(devAddrs)
-
-	for _, nic := range nics {
-		downscript := s.getNicDownScriptPath(nic)
-		ifname, _ := nic.GetString("ifname")
-		cmd += fmt.Sprintf("%s %s\n", downscript, ifname)
-	}
-
-	if options.HostOptions.HugepagesOption == "native" {
-		cmd += fmt.Sprintf("mkdir -p /dev/hugepages/%s\n", uuid)
-		cmd += fmt.Sprintf("mount -t hugetlbfs -o size=%dM hugetlbfs-%s /dev/hugepages/%s\n",
-			mem, uuid, uuid)
-	}
-
-	cmd += "sleep 1\n"
-	cmd += fmt.Sprintf("echo %d > %s\n", vncPort, s.GetVncFilePath())
-
-	diskScripts, err := s.generateDiskSetupScripts(disks)
-	if err != nil {
-		return "", errors.Wrap(err, "generateDiskSetupScripts")
-	}
-	cmd += diskScripts
-
-	cmd += fmt.Sprintf("STATE_FILE=`ls -d %s* | head -n 1`\n", s.getStateFilePathRootPrefix())
-	cmd += fmt.Sprintf("PID_FILE=%s\n", s.GetPidFilePath())
-
-	var qemuCmd = qemutils.GetQemu(qemuVersion)
-	if len(qemuCmd) == 0 {
-		qemuCmd = qemutils.GetQemu("")
-	}
-
-	cmd += fmt.Sprintf("DEFAULT_QEMU_CMD='%s'\n", qemuCmd)
-	cmd += "if [ -n \"$STATE_FILE\" ]; then\n"
-	cmd += "    QEMU_VER=`echo $STATE_FILE" +
-		` | grep -o '_[[:digit:]]\+\.[[:digit:]]\+.*'` + "`\n"
-	cmd += "    QEMU_CMD=\"qemu-system-x86_64\"\n"
-	cmd += "    QEMU_LOCAL_PATH=\"/usr/local/bin/$QEMU_CMD\"\n"
-	cmd += "    QEMU_LOCAL_PATH_VER=\"/usr/local/qemu-$QEMU_VER/bin/$QEMU_CMD\"\n"
-	cmd += "    QEMU_BIN_PATH=\"/usr/bin/$QEMU_CMD\"\n"
-	cmd += "    if [ -f \"$QEMU_LOCAL_PATH_VER\" ]; then\n"
-	cmd += "        QEMU_CMD=$QEMU_LOCAL_PATH_VER\n"
-	cmd += "    elif [ -f \"$QEMU_LOCAL_PATH\" ]; then\n"
-	cmd += "        QEMU_CMD=$QEMU_LOCAL_PATH\n"
-	cmd += "    elif [ -f \"$QEMU_BIN_PATH\" ]; then\n"
-	cmd += "        QEMU_CMD=$QEMU_BIN_PATH\n"
-	cmd += "    fi\n"
-	cmd += "else\n"
-	cmd += "    QEMU_CMD=$DEFAULT_QEMU_CMD\n"
-	cmd += "fi\n"
-	if s.IsKvmSupport() && !options.HostOptions.DisableKVM {
-		cmd += "QEMU_CMD_KVM_ARG=-enable-kvm\n"
-	} else {
-		cmd += "QEMU_CMD_KVM_ARG=\n"
-	}
-	cmd += `
-function nic_speed() {
-    $QEMU_CMD $QEMU_CMD_KVM_ARG -device virtio-net-pci,help 2>&1 | grep -q "\<speed="
-    if [ "$?" -eq "0" ]; then
-        echo ",speed=$1"
-    fi
-}
-
-function nic_mtu() {
-    local bridge="$1"; shift
-
-    $QEMU_CMD $QEMU_CMD_KVM_ARG -device virtio-net-pci,help 2>&1 | grep -q '\<host_mtu='
-    if [ "$?" -eq "0" ]; then
-        local origmtu="$(<"/sys/class/net/$bridge/mtu")"
-        if [ -n "$origmtu" -a "$origmtu" -gt 576 ]; then
-            echo ",host_mtu=$(($origmtu - ` + api.VpcOvnEncapCostStr() + `))"
-        fi
-    fi
-}
-`
-
-	// Generate Start VM script
-	cmd += `CMD="$QEMU_CMD $QEMU_CMD_KVM_ARG`
-	var accel, cpuType string
-	if s.IsKvmSupport() && !options.HostOptions.DisableKVM {
-		accel = "kvm"
-		if options.HostOptions.HostCpuPassthrough {
-			cpuType = "host"
-		} else {
-			// * under KVM, -cpu max is the same as -cpu host
-			// * under TCG, -cpu max means "emulate with as many features as possible"
-			cpuType = "max"
-		}
-	} else {
-		accel = "tcg"
-		cpuType = "max"
-	}
-
-	cmd += fmt.Sprintf(" -cpu %s", cpuType)
-	if options.HostOptions.LogLevel == "debug" {
-		cmd += fmt.Sprintf(" -D %s -d all", s.getQemuLogPath())
-	}
-
-	cmd += s.getMonitorDesc("hmqmon", s.GetHmpMonitorPort(int(vncPort)), MODE_READLINE)
-	if options.HostOptions.EnableQmpMonitor {
-		cmd += s.getMonitorDesc("qmqmon", s.GetQmpMonitorPort(int(vncPort)), MODE_CONTROL)
-	}
-
-	cmd += " -rtc base=utc,clock=host,driftfix=none"
-	cmd += " -daemonize"
-	cmd += " -nodefaults -nodefconfig"
-	cmd += fmt.Sprintf(" -machine %s,accel=%s,gic-version=3", s.getArmMachine(), accel)
-	cmd += " -k en-us"
-
-	// warning: Number of hotpluggable cpus requested (128)
-	// exceeds the recommended cpus supported by KVM (32)
-	cmd += fmt.Sprintf(" -smp cpus=%d,sockets=2,cores=32,maxcpus=64", cpu)
-	cmd += fmt.Sprintf(" -name %s", name)
-	if options.HostOptions.EnableVmUuid {
-		cmd += fmt.Sprintf(" -uuid %s", uuid)
-	}
-	cmd += fmt.Sprintf(" -m %dM,slots=4,maxmem=262144M", mem)
-
-	if options.HostOptions.HugepagesOption == "native" {
-		cmd += fmt.Sprintf(" -mem-prealloc -mem-path %s", fmt.Sprintf("/dev/hugepages/%s", uuid))
-	}
-
-	bootOrder, _ := s.Desc.GetString("boot_order")
-	cmd += fmt.Sprintf(" -boot order=%s", bootOrder)
-
-	// arm unsupport bios
-	cmd += fmt.Sprintf(" -bios %s", options.HostOptions.OvmfPath)
-
-	cmd += " -device qemu-xhci,p2=8,p3=8,id=usb"
-	cmd += " -device usb-tablet,id=input0,bus=usb.0,port=1"
-	cmd += " -device usb-kbd,id=input1,bus=usb.0,port=2"
-	cmd += " -device virtio-gpu-pci,id=video0,max_outputs=1"
-
-	if s.IsVdiSpice() {
-		cmd += s.generateSpiceArgs(vncPort)
-	} else {
-		cmd += fmt.Sprintf(" -vnc :%d", vncPort)
-		if options.HostOptions.SetVncPassword {
-			cmd += ",password"
-		}
-	}
-
-	cmd += " -object iothread,id=iothread0"
-
-	cmd += s.generateDiskParams(disks, true)
-
-	cdrom, _ := s.Desc.Get("cdrom")
-	if cdrom != nil && cdrom.Contains("path") {
-		cdromPath, _ := cdrom.GetString("path")
-		if len(cdromPath) > 0 {
-			cmd += " -device virtio-scsi-device -device scsi-cd,drive=cd0,share-rw=true"
-			cmd += fmt.Sprintf(" -drive if=none,file=%s,id=cd0,media=cdrom", cdromPath)
-		}
-	}
-
-	for i := 0; i < len(nics); i++ {
-		if osname == OS_NAME_VMWARE {
-			nics[i].(*jsonutils.JSONDict).Set("driver", jsonutils.NewString("vmxnet3"))
-		}
-		nicCmd, err := s.getNetdevDesc(nics[i])
-		if err != nil {
-			return "", err
-		} else {
-			cmd += nicCmd
-		}
-		// aarch64 with addr lead to:
-		// virtio_net: probe of virtioN failed with error -22
-		cmd += s.getVnicDesc(nics[i], false)
-	}
-
-	if isolatedDevsParams != nil {
-		for _, each := range isolatedDevsParams.Devices {
-			cmd += each
-		}
-	}
-
-	cmd += fmt.Sprintf(" -pidfile %s", s.GetPidFilePath())
-	cmd += s.extraOptions()
-
-	// cmd += s.getQgaDesc()
-	if options.HostOptions.EnableVirtioRngDevice && fileutils2.Exists("/dev/random") {
-		cmd += " -object rng-random,filename=/dev/random,id=rng0"
-		cmd += " -device virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000"
-	}
-
-	if jsonutils.QueryBoolean(data, "need_migrate", false) {
-		migratePort := s.manager.GetFreePortByBase(LIVE_MIGRATE_PORT_BASE)
-		s.Desc.Set("live_migrate_dest_port", jsonutils.NewInt(int64(migratePort)))
-		cmd += fmt.Sprintf(" -incoming tcp:0:%d", migratePort)
-	} else if jsonutils.QueryBoolean(s.Desc, "is_slave", false) {
-		cmd += fmt.Sprintf(" -incoming tcp:0:%d",
-			s.manager.GetFreePortByBase(LIVE_MIGRATE_PORT_BASE))
-	} else if jsonutils.QueryBoolean(s.Desc, "is_master", false) {
-		cmd += " -S"
-	}
-	// cmd += fmt.Sprintf(" -D %s", path.Join(s.HomeDir(), "log"))
-
-	// -device pvpanic: 'pvpanic' is not a valid device model name
-	// if !s.disablePvpanicDev() {
-	// 	cmd += " -device pvpanic"
-	// }
-
-	cmd += "\"\n"
-	cmd += "if [ ! -z \"$STATE_FILE\" ] && [ -d \"$STATE_FILE\" ] && [ -f \"$STATE_FILE/content\" ]; then\n"
-	cmd += "    $CMD --incoming \"exec: cat $STATE_FILE/content\"\n"
-	cmd += "elif [ ! -z \"$STATE_FILE\" ] && [ -f $STATE_FILE ]; then\n"
-	cmd += "    $CMD --incoming \"exec: cat $STATE_FILE\"\n"
-	cmd += "else\n"
-	cmd += "    $CMD\n"
-	cmd += "fi\n"
-
-	return cmd, nil
-}
-
-func (s *SKVMGuestInstance) generateSpiceArgs(vncPort int64) string {
-	cmd := ""
-	cmd += " -device qxl-vga,id=video0,ram_size=141557760,vram_size=141557760"
-	cmd += " -device intel-hda,id=sound0"
-	cmd += " -device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0"
-	cmd += fmt.Sprintf(" -spice port=%d,password=87654312,seamless-migration=on", 5900+vncPort)
-	// # ,streaming-video=all,playback-compression=on,jpeg-wan-compression=always,zlib-glz-wan-compression=always,image-compression=glz" % (5900+vnc_port)
-	cmd += fmt.Sprintf(" -device virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=%s", s.GetPciBus())
-	cmd += " -chardev spicevmc,name=vdagent,id=vdagent"
-	cmd += " -device virtserialport,nr=1,bus=virtio-serial0.0,chardev=vdagent,name=com.redhat.spice.0"
-
-	// # usb redirect
-	cmd += " -device ich9-usb-ehci1,id=usbspice"
-	cmd += " -device ich9-usb-uhci1,masterbus=usbspice.0,firstport=0,multifunction=on"
-	cmd += " -device ich9-usb-uhci2,masterbus=usbspice.0,firstport=2"
-	cmd += " -device ich9-usb-uhci3,masterbus=usbspice.0,firstport=4"
-	cmd += " -chardev spicevmc,name=usbredir,id=usbredirchardev1"
-	cmd += " -device usb-redir,chardev=usbredirchardev1,id=usbredirdev1"
-	cmd += " -chardev spicevmc,name=usbredir,id=usbredirchardev2"
-	cmd += " -device usb-redir,chardev=usbredirchardev2,id=usbredirdev2"
-	return cmd
-}

pkg/hostman/guestman/qemu-kvm.go

@@ -288,7 +288,7 @@ func (s *SKVMGuestInstance) asyncScriptStart(ctx context.Context, params interfa
 		s.StartMonitor(ctx, nil)
 		return nil, nil
 	}
-	log.Infof("Async start server %s failed: %s!!!", s.GetName(), err)
+	log.Errorf("Async start server %s failed: %s!!!", s.GetName(), err)
 	if ctx != nil && len(appctx.AppContextTaskId(ctx)) >= 0 {
 		hostutils.TaskFailed(ctx, fmt.Sprintf("Async start server failed: %s", err))
 	}
@@ -1787,35 +1787,6 @@ func (s *SKVMGuestInstance) generateDiskSetupScripts(disks []api.GuestdiskJsonDe
 	return cmd, nil
 }
 
-func (s *SKVMGuestInstance) generateDiskParams(disks []api.GuestdiskJsonDesc, isArm bool) string {
-	cmd := " "
-	firstDriver := make(map[string]bool)
-	for _, disk := range disks {
-		driver := disk.Driver
-		if isArm && (driver == DISK_DRIVER_IDE || driver == DISK_DRIVER_SATA) {
-			// unsupported configuration: IDE controllers are unsupported
-			driver = DISK_DRIVER_SCSI
-		}
-		if driver == DISK_DRIVER_SCSI || driver == DISK_DRIVER_PVSCSI {
-			if _, ok := firstDriver[driver]; !ok {
-				switch driver {
-				case DISK_DRIVER_SCSI:
-					// FIXME: iothread will make qemu-monitor hang
-					// REF: https://www.mail-archive.com/qemu-devel@nongnu.org/msg592729.html
-					// cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5"
-					cmd += " -device virtio-scsi-pci,id=scsi,num_queues=4,vectors=5"
-				case DISK_DRIVER_PVSCSI:
-					cmd += " -device pvscsi,id=scsi"
-				}
-				firstDriver[driver] = true
-			}
-		}
-		cmd += s.getDriveDesc(disk, isArm)
-		cmd += s.getVdiskDesc(disk, isArm)
-	}
-	return cmd
-}
-
 func (s *SKVMGuestInstance) getQemuCmdline() (string, error) {
 	content, err := fileutils2.FileGetContents(s.GetStartScriptPath())
 	if err != nil {

pkg/hostman/guestman/qemu-kvmhelper.go

@@ -31,6 +31,7 @@ import (
 	"yunion.io/x/pkg/utils"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/hostman/guestman/qemu"
 	"yunion.io/x/onecloud/pkg/hostman/options"
 	"yunion.io/x/onecloud/pkg/util/fileutils2"
 	"yunion.io/x/onecloud/pkg/util/qemutils"
@@ -38,22 +39,22 @@ import (
 )
 
 const (
-	OS_NAME_LINUX   = "Linux"
-	OS_NAME_WINDOWS = "Windows"
-	OS_NAME_MACOS   = "macOS"
-	OS_NAME_ANDROID = "Android"
-	OS_NAME_VMWARE  = "VMWare"
-	OS_NAME_CIRROS  = "Cirros"
-	OS_NAME_OPENWRT = "OpenWrt"
+	OS_NAME_LINUX   = qemu.OS_NAME_LINUX
+	OS_NAME_WINDOWS = qemu.OS_NAME_WINDOWS
+	OS_NAME_MACOS   = qemu.OS_NAME_MACOS
+	OS_NAME_ANDROID = qemu.OS_NAME_ANDROID
+	OS_NAME_VMWARE  = qemu.OS_NAME_VMWARE
+	OS_NAME_CIRROS  = qemu.OS_NAME_CIRROS
+	OS_NAME_OPENWRT = qemu.OS_NAME_OPENWRT
 
-	MODE_READLINE = "readline"
-	MODE_CONTROL  = "control"
+	MODE_READLINE = qemu.MODE_READLINE
+	MODE_CONTROL  = qemu.MODE_CONTROL
 
-	DISK_DRIVER_VIRTIO = "virtio"
-	DISK_DRIVER_SCSI   = "scsi"
-	DISK_DRIVER_PVSCSI = "pvscsi"
-	DISK_DRIVER_IDE    = "ide"
-	DISK_DRIVER_SATA   = "sata"
+	DISK_DRIVER_VIRTIO = qemu.DISK_DRIVER_VIRTIO
+	DISK_DRIVER_SCSI   = qemu.DISK_DRIVER_SCSI
+	DISK_DRIVER_PVSCSI = qemu.DISK_DRIVER_PVSCSI
+	DISK_DRIVER_IDE    = qemu.DISK_DRIVER_IDE
+	DISK_DRIVER_SATA   = qemu.DISK_DRIVER_SATA
 )
 
 func (s *SKVMGuestInstance) IsKvmSupport() bool {
@@ -65,15 +66,6 @@ func (s *SKVMGuestInstance) IsVdiSpice() bool {
 	return vdi == "spice"
 }
 
-func (s *SKVMGuestInstance) getMonitorDesc(idstr string, port int, mode string) string {
-	var cmd = ""
-	cmd += fmt.Sprintf(" -chardev socket,id=%sdev", idstr)
-	cmd += fmt.Sprintf(",port=%d", port)
-	cmd += ",host=127.0.0.1,nodelay,server,nowait"
-	cmd += fmt.Sprintf(" -mon chardev=%sdev,id=%s,mode=%s", idstr, idstr, mode)
-	return cmd
-}
-
 func (s *SKVMGuestInstance) getOsname() string {
 	osName, err := s.Desc.GetString("metadata", "os_name")
 	if err != nil {
@@ -164,101 +156,8 @@ func (s *SKVMGuestInstance) disablePvpanicDev() bool {
 	return val == "true"
 }
 
-func isLocalStorage(disk api.GuestdiskJsonDesc) bool {
-	if disk.StorageType == api.STORAGE_LOCAL || len(disk.StorageType) == 0 {
-		return true
-	} else {
-		return false
-	}
-}
-
-func (s *SKVMGuestInstance) getDriveDesc(disk api.GuestdiskJsonDesc, isArm bool) string {
-	format := disk.Format
-	diskIndex := disk.Index
-	cacheMode := disk.CacheMode
-	aioMode := disk.AioMode
-
-	cmd := " -drive"
-	cmd += fmt.Sprintf(" file=$DISK_%d", diskIndex)
-	cmd += ",if=none"
-	cmd += fmt.Sprintf(",id=drive_%d", diskIndex)
-	if len(format) == 0 || format == "qcow2" {
-		// pass    # qemu will automatically detect image format
-	} else if format == "raw" {
-		cmd += ",format=raw"
-	}
-	cmd += fmt.Sprintf(",cache=%s", cacheMode)
-	if isLocalStorage(disk) {
-		cmd += fmt.Sprintf(",aio=%s", aioMode)
-	}
-	if len(disk.Url) > 0 { // # a remote file backed image
-		cmd += ",copy-on-read=on"
-	}
-	if isLocalStorage(disk) {
-		cmd += ",file.locking=off"
-	}
-	// #cmd += ",media=disk"
-	return cmd
-}
-
 func (s *SKVMGuestInstance) GetDiskAddr(idx int) int {
-	var base = 5
-	if s.IsVdiSpice() {
-		base += 10
-	}
-	return base + idx
-}
-
-func (s *SKVMGuestInstance) GetDiskDeviceModel(driver string) string {
-	if driver == DISK_DRIVER_VIRTIO {
-		return "virtio-blk-pci"
-	} else if utils.IsInStringArray(driver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) {
-		return "scsi-hd"
-	} else if driver == DISK_DRIVER_IDE {
-		return "ide-hd"
-	} else if driver == DISK_DRIVER_SATA {
-		return "ide-drive"
-	} else {
-		return "None"
-	}
-}
-
-func (s *SKVMGuestInstance) getVdiskDesc(disk api.GuestdiskJsonDesc, isArm bool) string {
-	diskIndex := disk.Index
-	diskDriver := disk.Driver
-	numQueues := disk.NumQueues
-	isSsd := disk.IsSSD
-
-	if numQueues == 0 {
-		numQueues = 4
-	}
-
-	if isArm && (diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA) {
-		// unsupported configuration: IDE controllers are unsupported
-		// for this QEMU binary or machine type
-		// replace with scsi
-		diskDriver = DISK_DRIVER_SCSI
-	}
-
-	var cmd = ""
-	cmd += fmt.Sprintf(" -device %s", s.GetDiskDeviceModel(diskDriver))
-	cmd += fmt.Sprintf(",drive=drive_%d", diskIndex)
-	if diskDriver == DISK_DRIVER_VIRTIO {
-		// virtio-blk
-		cmd += fmt.Sprintf(",bus=%s,addr=0x%x", s.GetPciBus(), s.GetDiskAddr(int(diskIndex)))
-		cmd += fmt.Sprintf(",num-queues=%d,vectors=%d,iothread=iothread0", numQueues, numQueues+1)
-	} else if utils.IsInStringArray(diskDriver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) {
-		cmd += ",bus=scsi.0"
-	} else if diskDriver == DISK_DRIVER_IDE {
-		cmd += fmt.Sprintf(",bus=ide.%d,unit=%d", diskIndex/2, diskIndex%2)
-	} else if diskDriver == DISK_DRIVER_SATA {
-		cmd += fmt.Sprintf(",bus=ide.%d", diskIndex)
-	}
-	cmd += fmt.Sprintf(",id=drive_%d", diskIndex)
-	if isSsd {
-		cmd += ",rotation_rate=1"
-	}
-	return cmd
+	return qemu.GetDiskAddr(idx, s.IsVdiSpice())
 }
 
 func (s *SKVMGuestInstance) getNicUpScriptPath(nic jsonutils.JSONObject) string {
@@ -281,102 +180,22 @@ func (s *SKVMGuestInstance) generateNicScripts(nic jsonutils.JSONObject) error {
 	}
 	isSlave := s.IsSlave()
 	if err := dev.GenerateIfupScripts(s.getNicUpScriptPath(nic), nic, isSlave); err != nil {
-		log.Errorln(err)
-		return err
+		return errors.Wrap(err, "GenerateIfupScripts")
 	}
 	if err := dev.GenerateIfdownScripts(s.getNicDownScriptPath(nic), nic, isSlave); err != nil {
-		log.Errorln(err)
-		return err
+		return errors.Wrap(err, "GenerateIfdownScripts")
 	}
 	return nil
 }
 
-func (s *SKVMGuestInstance) getNetdevDesc(nic jsonutils.JSONObject) (string, error) {
-	ifname, _ := nic.GetString("ifname")
-	driver, _ := nic.GetString("driver")
-
-	if err := s.generateNicScripts(nic); err != nil {
-		return "", err
-	}
-	upscript := s.getNicUpScriptPath(nic)
-	downscript := s.getNicDownScriptPath(nic)
-	cmd := " -netdev type=tap"
-	cmd += fmt.Sprintf(",id=%s", ifname)
-	cmd += fmt.Sprintf(",ifname=%s", ifname)
-	if driver == "virtio" && s.IsKvmSupport() {
-		cmd += ",vhost=on,vhostforce=off"
-	}
-	cmd += fmt.Sprintf(",script=%s", upscript)
-	cmd += fmt.Sprintf(",downscript=%s", downscript)
-	return cmd, nil
-}
-
 func (s *SKVMGuestInstance) getNicDeviceModel(name string) string {
-	if name == "virtio" {
-		return "virtio-net-pci"
-	} else if name == "e1000" {
-		return "e1000-82545em"
-	} else {
-		return name
-	}
+	return qemu.GetNicDeviceModel(name)
 }
 
 func (s *SKVMGuestInstance) getNicAddr(index int) int {
-	var pciBase = 10
 	disks, _ := s.Desc.GetArray("disks")
-	if len(disks) > 10 {
-		pciBase = 20
-	}
 	isolatedDevices, _ := s.Desc.GetArray("isolated_devices")
-	if len(isolatedDevices) > 0 {
-		pciBase += 10
-	}
-	return s.GetDiskAddr(pciBase + index)
-}
-
-func (s *SKVMGuestInstance) getVnicDesc(nic jsonutils.JSONObject, withAddr bool) string {
-	bridge, _ := nic.GetString("bridge")
-	ifname, _ := nic.GetString("ifname")
-	driver, _ := nic.GetString("driver")
-	mac, _ := nic.GetString("mac")
-	index, _ := nic.Int("index")
-	vectors, _ := nic.Int("vectors")
-	bw, _ := nic.Int("bw")
-
-	cmd := fmt.Sprintf(" -device %s", s.getNicDeviceModel(driver))
-	cmd += fmt.Sprintf(",id=netdev-%s", ifname)
-	cmd += fmt.Sprintf(",netdev=%s", ifname)
-	cmd += fmt.Sprintf(",mac=%s", mac)
-
-	if withAddr {
-		cmd += fmt.Sprintf(",addr=0x%x", s.getNicAddr(int(index)))
-	}
-	if driver == "virtio" {
-		if nic.Contains("vectors") {
-			cmd += fmt.Sprintf(",vectors=%d", vectors)
-		}
-		cmd += fmt.Sprintf("$(nic_speed %d)", bw)
-		if bridge == options.HostOptions.OvnIntegrationBridge {
-			cmd += fmt.Sprintf("$(nic_mtu %q)", bridge)
-		}
-	}
-	return cmd
-}
-
-func (s *SKVMGuestInstance) getQgaDesc() string {
-	cmd := " -chardev socket,path="
-	cmd += path.Join(s.HomeDir(), "qga.sock")
-	cmd += ",server,nowait,id=qga0"
-	cmd += " -device virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"
-	return cmd
-}
-
-func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (string, error) {
-	if s.manager.host.IsAarch64() {
-		return s.generateArmStartScript(data)
-	} else {
-		return s._generateStartScript(data)
-	}
+	return qemu.GetNicAddr(index, len(disks), len(isolatedDevices), s.IsVdiSpice())
 }
 
 func (s *SKVMGuestInstance) extraOptions() string {
@@ -396,7 +215,8 @@ func (s *SKVMGuestInstance) extraOptions() string {
 	return cmd
 }
 
-func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (string, error) {
+func (s *SKVMGuestInstance) generateStartScript(data *jsonutils.JSONDict) (string, error) {
+	// initial data
 	var (
 		uuid, _ = s.Desc.GetString("uuid")
 		mem, _  = s.Desc.Int("mem")
@@ -404,18 +224,40 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri
 		name, _ = s.Desc.GetString("name")
 		nics, _ = s.Desc.GetArray("nics")
 		osname  = s.getOsname()
-		cmd     = ""
+		input   = &qemu.GenerateStartOptionsInput{
+			UUID:                 uuid,
+			Mem:                  uint64(mem),
+			Cpu:                  uint(cpu),
+			Name:                 name,
+			OsName:               osname,
+			Nics:                 nics,
+			OVNIntegrationBridge: options.HostOptions.OvnIntegrationBridge,
+			HomeDir:              s.HomeDir(),
+			HugepagesEnabled:     s.manager.host.IsHugepagesEnabled(),
+			PidFilePath:          s.GetPidFilePath(),
+		}
 	)
+
+	cmd := ""
+
+	// inject disks
 	disks := make([]api.GuestdiskJsonDesc, 0)
 	s.Desc.Unmarshal(&disks, "disks")
+	input.Disks = disks
 
-	if osname == OS_NAME_MACOS {
-		s.Desc.Set("machine", jsonutils.NewString("q35"))
-		s.Desc.Set("bios", jsonutils.NewString("UEFI"))
+	// inject machine and bios
+	if input.OsName == OS_NAME_MACOS {
+		s.Desc.Set("machine", jsonutils.NewString(api.VM_MACHINE_TYPE_Q35))
+		input.Machine = api.VM_MACHINE_TYPE_Q35
+		s.Desc.Set("bios", jsonutils.NewString(qemu.BIOS_UEFI))
+		input.BIOS = qemu.BIOS_UEFI
 	}
 
+	// inject vncPort
 	vncPort, _ := data.Int("vnc_port")
+	input.VNCPort = uint(vncPort)
 
+	// inject qemu version and arch
 	qemuVersion := options.HostOptions.DefaultQemuVersion
 	if data.Contains("qemu_version") {
 		qemuVersion, _ = data.GetString("qemu_version")
@@ -423,7 +265,15 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri
 	if qemuVersion == "latest" {
 		qemuVersion = ""
 	}
+	input.QemuVersion = qemu.Version(qemuVersion)
+	// inject qemu arch
+	if s.manager.host.IsAarch64() {
+		input.QemuArch = qemu.Arch_aarch64
+	} else {
+		input.QemuArch = qemu.Arch_x86_64
+	}
 
+	// inject isolatedDevices
 	var devAddrs = []string{}
 	isolatedParams, _ := s.Desc.GetArray("isolated_devices")
 	for _, params := range isolatedParams {
@@ -431,32 +281,33 @@ func (s *SKVMGuestInstance) _generateStartScript(data *jsonutils.JSONDict) (stri
 		devAddrs = append(devAddrs, devAddr)
 	}
 	isolatedDevsParams := s.manager.GetHost().GetIsolatedDeviceManager().GetQemuParams(devAddrs)
+	input.IsolatedDevicesParams = isolatedDevsParams
 
-	for _, nic := range nics {
+	for _, nic := range input.Nics {
 		downscript := s.getNicDownScriptPath(nic)
 		ifname, _ := nic.GetString("ifnam")
 		cmd += fmt.Sprintf("%s %s\n", downscript, ifname)
 	}
 
-	if s.manager.host.IsHugepagesEnabled() {
-		cmd += fmt.Sprintf("mkdir -p /dev/hugepages/%s\n", uuid)
+	if input.HugepagesEnabled {
+		cmd += fmt.Sprintf("mkdir -p /dev/hugepages/%s\n", input.UUID)
 		cmd += fmt.Sprintf("mount -t hugetlbfs -o size=%dM hugetlbfs-%s /dev/hugepages/%s\n",
-			mem, uuid, uuid)
+			input.Mem, input.UUID, input.UUID)
 	}
 
 	cmd += "sleep 1\n"
-	cmd += fmt.Sprintf("echo %d > %s\n", vncPort, s.GetVncFilePath())
+	cmd += fmt.Sprintf("echo %d > %s\n", input.VNCPort, s.GetVncFilePath())
 
-	diskScripts, err := s.generateDiskSetupScripts(disks)
+	diskScripts, err := s.generateDiskSetupScripts(input.Disks)
 	if err != nil {
 		return "", errors.Wrap(err, "generateDiskSetupScripts")
 	}
 	cmd += diskScripts
 
 	// cmd += fmt.Sprintf("STATE_FILE=`ls -d %s* | head -n 1`\n", s.getStateFilePathRootPrefix())
-	cmd += fmt.Sprintf("PID_FILE=%s\n", s.GetPidFilePath())
+	cmd += fmt.Sprintf("PID_FILE=%s\n", input.PidFilePath)
 
-	var qemuCmd = qemutils.GetQemu(qemuVersion)
+	var qemuCmd = qemutils.GetQemu(string(input.QemuVersion))
 	if len(qemuCmd) == 0 {
 		qemuCmd = qemutils.GetQemu("")
 	}
@@ -507,182 +358,118 @@ function nic_mtu() {
 
 	// Generate Start VM script
 	cmd += `CMD="$QEMU_CMD $QEMU_CMD_KVM_ARG`
-	var accel, cpuType string
+
+	// inject cpu info
 	if s.IsKvmSupport() && !options.HostOptions.DisableKVM {
-		accel = "kvm"
-		cpuType = ""
-		if osname == OS_NAME_MACOS {
-			cpuType = "Penryn,vendor=GenuineIntel"
-		} else if options.HostOptions.HostCpuPassthrough {
-			cpuType = "host"
-			// https://unix.stackexchange.com/questions/216925/nmi-received-for-unknown-reason-20-do-you-have-a-strange-power-saving-mode-ena
-			cpuType += ",+kvm_pv_eoi"
-		} else {
-			cpuType = "qemu64"
-			cpuType += ",+kvm_pv_eoi"
-			if sysutils.IsProcessorIntel() {
-				cpuType += ",+vmx"
-				cpuType += ",+ssse3,+sse4.1,+sse4.2,-x2apic,+aes,+avx"
-				cpuType += ",+vme,+pat,+ss,+pclmulqdq,+xsave"
-				cpuType += ",level=13"
-			} else if sysutils.IsProcessorAmd() {
-				cpuType += ",+svm"
-			}
-		}
-
-		if !guestManager.GetHost().IsNestedVirtualization() {
-			cpuType += ",kvm=off"
-		}
-
-		if isolatedDevsParams != nil && len(isolatedDevsParams.Cpu) > 0 {
-			cpuType = isolatedDevsParams.Cpu
-		}
-	} else {
-		accel = "tcg"
-		cpuType = "qemu64"
+		input.EnableKVM = true
+		input.HostCPUPassthrough = options.HostOptions.HostCpuPassthrough
+		input.IsCPUIntel = sysutils.IsProcessorIntel()
+		input.IsCPUAMD = sysutils.IsProcessorAmd()
+		input.EnableNested = guestManager.GetHost().IsNestedVirtualization()
 	}
 
-	cmd += fmt.Sprintf(" -cpu %s", cpuType)
-
-	if options.HostOptions.LogLevel == "debug" {
-		cmd += fmt.Sprintf(" -D %s -d all", s.getQemuLogPath())
+	// inject monitor
+	input.HMPMonitor = &qemu.Monitor{
+		Id:   "hmqmon",
+		Port: uint(s.GetHmpMonitorPort(int(input.VNCPort))),
+		Mode: MODE_READLINE,
 	}
-
-	// TODO hmp - -
-	cmd += s.getMonitorDesc("hmqmon", s.GetHmpMonitorPort(int(vncPort)), MODE_READLINE)
 	if options.HostOptions.EnableQmpMonitor {
-		cmd += s.getMonitorDesc("qmqmon", s.GetQmpMonitorPort(int(vncPort)), MODE_CONTROL)
+		input.QMPMonitor = &qemu.Monitor{
+			Id:   "qmqmon",
+			Port: uint(s.GetQmpMonitorPort(int(input.VNCPort))),
+			Mode: MODE_CONTROL,
+		}
 	}
 
-	cmd += " -rtc base=utc,clock=host,driftfix=none"
-	cmd += " -daemonize"
-	cmd += " -nodefaults -nodefconfig"
-	cmd += " -no-kvm-pit-reinjection"
-	cmd += " -global kvm-pit.lost_tick_policy=discard"
-	cmd += fmt.Sprintf(" -machine %s,accel=%s", s.getMachine(), accel)
-	cmd += " -k en-us"
-	// #cmd += " -g 800x600"
-	cmd += fmt.Sprintf(" -smp cpus=%d,sockets=2,cores=64,maxcpus=128", cpu)
-	cmd += fmt.Sprintf(" -name %s", name)
-	if options.HostOptions.EnableVmUuid {
-		cmd += fmt.Sprintf(" -uuid %s", uuid)
-	}
-	cmd += fmt.Sprintf(" -m %dM,slots=4,maxmem=524288M", mem)
-
-	if s.manager.host.IsHugepagesEnabled() {
-		cmd += fmt.Sprintf(" -mem-prealloc -mem-path %s", fmt.Sprintf("/dev/hugepages/%s", uuid))
-	}
+	input.EnableUUID = options.HostOptions.EnableVmUuid
+	// inject machine
+	input.Machine = s.getMachine()
 
+	// inject bootOrder and cdrom
 	bootOrder, _ := s.Desc.GetString("boot_order")
-	cmd += fmt.Sprintf(" -boot order=%s", bootOrder)
+	input.BootOrder = bootOrder
 	cdrom, _ := s.Desc.Get("cdrom")
 	if cdrom != nil && cdrom.Contains("path") {
-		cmd += ",menu=on"
+		cdromPath, _ := cdrom.GetString("path")
+		input.CdromPath = cdromPath
 	}
 
-	if s.getBios() == "UEFI" {
-		cmd += fmt.Sprintf(" -bios %s", options.HostOptions.OvmfPath)
+	// UEFI ovmf file path
+	if s.getBios() == qemu.BIOS_UEFI || input.QemuArch == qemu.Arch_aarch64 {
+		input.BIOS = qemu.BIOS_UEFI
+		input.OVMFPath = options.HostOptions.OvmfPath
 	}
 
-	if osname == OS_NAME_MACOS {
-		cmd += " -device isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc"
-		for i := 0; i < len(disks); i++ {
+	// inject nic and disks
+	if input.OsName == OS_NAME_MACOS {
+		for i := 0; i < len(input.Disks); i++ {
 			disks[i].Driver = DISK_DRIVER_SATA
 		}
-		for i := 0; i < len(nics); i++ {
+		for i := 0; i < len(input.Nics); i++ {
 			nic := nics[i].(*jsonutils.JSONDict)
 			nic.Set("vectors", jsonutils.NewInt(0))
 			nic.Set("driver", jsonutils.NewString("e1000"))
 		}
-	} else if osname == OS_NAME_ANDROID {
-		if len(nics) > 1 {
-			s.Desc.Set("nics", jsonutils.NewArray(nics[0]))
+	} else if input.OsName == OS_NAME_ANDROID {
+		if len(input.Nics) > 1 {
+			s.Desc.Set("nics", jsonutils.NewArray(input.Nics[0]))
 		}
 		nics, _ = s.Desc.GetArray("nics")
+		input.Nics = nics
 	}
 
-	cmd += " -device virtio-serial"
-	cmd += " -usb"
-	if !utils.IsInStringArray(s.getOsDistribution(), []string{OS_NAME_OPENWRT, OS_NAME_CIRROS}) &&
-		!s.isOldWindows() && !s.isWindows10() &&
-		!s.disableUsbKbd() {
-		cmd += " -device usb-kbd"
-	}
-	if osname == OS_NAME_ANDROID {
-		cmd += " -device usb-mouse"
-	} else if !s.isOldWindows() {
-		cmd += " -device usb-tablet"
-	}
-
-	if s.IsVdiSpice() {
-		cmd += s.generateSpiceArgs(vncPort)
+	// inject devices
+	if input.QemuArch == qemu.Arch_aarch64 {
+		input.Devices = append(input.Devices,
+			"qemu-xhci,p2=8,p3=8,id=usb",
+			"usb-tablet,id=input0,bus=usb.0,port=1",
+			"usb-kbd,id=input1,bus=usb.0,port=2",
+			"virtio-gpu-pci,id=video0,max_outputs=1",
+		)
 	} else {
-		if isolatedDevsParams != nil && len(isolatedDevsParams.Vga) > 0 {
-			cmd += isolatedDevsParams.Vga
-		} else {
-			vga, err := s.Desc.GetString("vga")
-			if err != nil {
-				vga = "std"
-			}
-			cmd += fmt.Sprintf(" -vga %s", vga)
+		if !utils.IsInStringArray(s.getOsDistribution(), []string{OS_NAME_OPENWRT, OS_NAME_CIRROS}) &&
+			!s.isOldWindows() && !s.isWindows10() &&
+			!s.disableUsbKbd() {
+			input.Devices = append(input.Devices, "usb-kbd")
 		}
-		cmd += fmt.Sprintf(" -vnc :%d", vncPort)
-		if options.HostOptions.SetVncPassword {
-			cmd += ",password"
+		if osname == OS_NAME_ANDROID {
+			input.Devices = append(input.Devices, "usb-mouse")
+		} else if !s.isOldWindows() {
+			input.Devices = append(input.Devices, "usb-tablet")
 		}
 	}
 
-	cmd += " -object iothread,id=iothread0"
-
-	cmd += s.generateDiskParams(disks, false)
-
-	if osname != OS_NAME_MACOS {
-		cmd += " -device ide-cd,drive=ide0-cd0,bus=ide.1"
-		if !s.isQ35() {
-			cmd += ",unit=1"
-		}
-		cmd += " -drive id=ide0-cd0,media=cdrom,if=none"
-	}
-
-	if cdrom != nil && cdrom.Contains("path") {
-		cdromPath, _ := cdrom.GetString("path")
-		if len(cdromPath) > 0 {
-			if osname != OS_NAME_MACOS {
-				cmd += fmt.Sprintf(",file=%s", cdromPath)
-			} else {
-				cmd += " -device ide-drive,drive=MacDVD"
-				cmd += fmt.Sprintf(",bus=ide.%d", len(disks))
-				cmd += " -drive id=MacDVD,if=none,snapshot=on"
-				cmd += fmt.Sprintf(",file=%s", cdromPath)
-			}
-		}
-	}
-
-	for i := 0; i < len(nics); i++ {
-		if osname == OS_NAME_VMWARE {
-			nics[i].(*jsonutils.JSONDict).Set("driver", jsonutils.NewString("vmxnet3"))
-		}
-		nicCmd, err := s.getNetdevDesc(nics[i])
+	// inject spice and vnc display
+	input.IsVdiSpice = s.IsVdiSpice()
+	input.SpicePort = uint(5900 + vncPort)
+	input.PCIBus = s.GetPciBus()
+	if input.QemuArch != qemu.Arch_aarch64 {
+		vga, err := s.Desc.GetString("vga")
 		if err != nil {
-			return "", err
-		} else {
-			cmd += nicCmd
+			vga = "std"
 		}
-		cmd += s.getVnicDesc(nics[i], true)
+		input.VGA = vga
+	}
+	input.VNCPassword = options.HostOptions.SetVncPassword
+
+	// reinject nics
+	input.IsKVMSupport = s.IsKvmSupport()
+	for i := 0; i < len(input.Nics); i++ {
+		if input.OsName == OS_NAME_VMWARE {
+			input.Nics[i].(*jsonutils.JSONDict).Set("driver", jsonutils.NewString("vmxnet3"))
+		}
+		if err := s.generateNicScripts(input.Nics[i]); err != nil {
+			return "", errors.Wrapf(err, "generateNicScripts for nic: %s", input.Nics[i])
+		}
+		upscript := s.getNicUpScriptPath(input.Nics[i])
+		downscript := s.getNicDownScriptPath(input.Nics[i])
+		input.Nics[i].(*jsonutils.JSONDict).Set("upscript_path", jsonutils.NewString(upscript))
+		input.Nics[i].(*jsonutils.JSONDict).Set("downscript_path", jsonutils.NewString(downscript))
 	}
 
-	// USB 3.0
-	cmd += " -device qemu-xhci,id=usb"
-	if isolatedDevsParams != nil {
-		for _, each := range isolatedDevsParams.Devices {
-			cmd += each
-		}
-	}
+	input.ExtraOptions = append(input.ExtraOptions, s.extraOptions())
 
-	cmd += fmt.Sprintf(" -pidfile %s", s.GetPidFilePath())
-	cmd += s.extraOptions()
-
-	cmd += s.getQgaDesc()
 	/*
 		QIU Jian
 		virtio-rng device may cause live migration failure
@@ -690,31 +477,36 @@ function nic_mtu() {
 		qemu-system-x86_64: load of migration failed: Invalid argument
 	*/
 	if options.HostOptions.EnableVirtioRngDevice && fileutils2.Exists("/dev/random") {
-		cmd += " -object rng-random,filename=/dev/random,id=rng0"
-		cmd += " -device virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000"
+		input.EnableRNGRandom = true
 	}
 
 	// add serial device
 	if !s.disableIsaSerialDev() {
-		cmd += " -chardev pty,id=charserial0"
-		cmd += " -device isa-serial,chardev=charserial0,id=serial0"
+		input.EnableSerialDevice = true
 	}
 
 	if jsonutils.QueryBoolean(data, "need_migrate", false) {
+		input.NeedMigrate = true
 		migratePort := s.manager.GetFreePortByBase(LIVE_MIGRATE_PORT_BASE)
 		s.Desc.Set("live_migrate_dest_port", jsonutils.NewInt(int64(migratePort)))
-		cmd += fmt.Sprintf(" -incoming tcp:0:%d", migratePort)
+		input.LiveMigratePort = uint(migratePort)
 	} else if jsonutils.QueryBoolean(s.Desc, "is_slave", false) {
-		cmd += fmt.Sprintf(" -incoming tcp:0:%d",
-			s.manager.GetFreePortByBase(LIVE_MIGRATE_PORT_BASE))
+		input.IsSlave = true
+		input.LiveMigratePort = uint(s.manager.GetFreePortByBase(LIVE_MIGRATE_PORT_BASE))
 	} else if jsonutils.QueryBoolean(s.Desc, "is_master", false) {
-		cmd += " -S"
+		input.IsMaster = true
 	}
 	// cmd += fmt.Sprintf(" -D %s", path.Join(s.HomeDir(), "log"))
 	if !s.disablePvpanicDev() {
-		cmd += " -device pvpanic"
+		input.EnablePvpanic = true
 	}
 
+	qemuOpts, err := qemu.GenerateStartOptions(input)
+	if err != nil {
+		return "", errors.Wrap(err, "GenerateStartCommand")
+	}
+	cmd = fmt.Sprintf("%s %s", cmd, qemuOpts)
+
 	cmd += "\"\n"
 	// cmd += "if [ ! -z \"$STATE_FILE\" ] && [ -d \"$STATE_FILE\" ] && [ -f \"$STATE_FILE/content\" ]; then\n"
 	// cmd += "    $CMD --incoming \"exec: cat $STATE_FILE/content\"\n"

pkg/hostman/guestman/qemu/doc.go

@@ -0,0 +1 @@
+package qemu // import "yunion.io/x/onecloud/pkg/hostman/guestman/qemu"

pkg/hostman/guestman/qemu/generate.go

@@ -0,0 +1,486 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu
+
+import (
+	"fmt"
+	"strings"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"
+	"yunion.io/x/pkg/utils"
+
+	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/hostman/isolated_device"
+)
+
+type Monitor struct {
+	Id   string
+	Port uint
+	Mode string
+}
+
+type GenerateStartOptionsInput struct {
+	QemuVersion Version
+	QemuArch    Arch
+
+	CPUOption
+
+	EnableUUID            bool
+	UUID                  string
+	Mem                   uint64
+	Cpu                   uint
+	Name                  string
+	OsName                string
+	HugepagesEnabled      bool
+	IsQ35                 bool
+	BootOrder             string
+	CdromPath             string
+	Nics                  []jsonutils.JSONObject
+	OVNIntegrationBridge  string
+	Disks                 []api.GuestdiskJsonDesc
+	Devices               []string
+	Machine               string
+	BIOS                  string
+	OVMFPath              string
+	VNCPort               uint
+	VNCPassword           bool
+	IsolatedDevicesParams *isolated_device.QemuParams
+	HMPMonitor            *Monitor
+	QMPMonitor            *Monitor
+	IsVdiSpice            bool
+	SpicePort             uint
+	PCIBus                string
+	VGA                   string
+	PidFilePath           string
+	HomeDir               string
+	ExtraOptions          []string
+	EnableRNGRandom       bool
+	EnableSerialDevice    bool
+	NeedMigrate           bool
+	LiveMigratePort       uint
+	IsSlave               bool
+	IsMaster              bool
+	EnablePvpanic         bool
+}
+
+func GenerateStartOptions(
+	input *GenerateStartOptionsInput,
+) (string, error) {
+	drv, ok := GetCommand(input.QemuVersion, input.QemuArch)
+	if !ok {
+		return "", errors.Errorf("Qemu comand driver %s %s not registered", input.QemuVersion, input.QemuArch)
+	}
+	drvOpt := drv.GetOptions()
+
+	opts := []string{}
+
+	if input.IsolatedDevicesParams != nil && len(input.IsolatedDevicesParams.Cpu) > 0 {
+		input.CPUOption.IsolatedDeviceCPU = input.IsolatedDevicesParams.Cpu
+	}
+
+	cpuOpt, accel, err := drvOpt.CPU(input.CPUOption, input.OsName)
+	if err != nil {
+		return "", errors.Wrap(err, "Get CPU option")
+	}
+
+	opts = append(opts, cpuOpt)
+
+	// TODO hmp - -
+	opts = append(opts, getMonitorOptions(drvOpt, input.HMPMonitor)...)
+	if input.QMPMonitor != nil {
+		opts = append(opts, getMonitorOptions(drvOpt, input.QMPMonitor)...)
+	}
+
+	opts = append(opts,
+		drvOpt.RTC(),
+		drvOpt.Daemonize(),
+		drvOpt.Nodefaults(),
+		drvOpt.Nodefconfig(),
+		drvOpt.NoKVMPitReinjection(),
+		drvOpt.Global(),
+		drvOpt.Machine(input.Machine, accel),
+		drvOpt.KeyboardLayoutLanguage("en-us"),
+		drvOpt.SMP(input.Cpu),
+		drvOpt.Name(input.Name),
+		drvOpt.UUID(input.EnableUUID, input.UUID),
+		drvOpt.Memory(input.Mem),
+	)
+
+	if input.HugepagesEnabled {
+		opts = append(opts,
+			drvOpt.MemPrealloc(),
+			drvOpt.MemPath(fmt.Sprintf("/dev/hugepages/%s", input.UUID)),
+		)
+	}
+
+	// bootOrder
+	enableMenu := false
+	if input.CdromPath != "" {
+		enableMenu = true
+	}
+	opts = append(opts, drvOpt.Boot(input.BootOrder, enableMenu))
+
+	// bios
+	if input.BIOS == BIOS_UEFI {
+		if input.OVMFPath == "" {
+			return "", errors.Errorf("input OVMF path is empty")
+		}
+		opts = append(opts, drvOpt.BIOS(input.OVMFPath))
+	}
+
+	if input.OsName == OS_NAME_MACOS {
+		opts = append(opts, drvOpt.Device("isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc"))
+	}
+
+	opts = append(opts, drvOpt.Device("virtio-serial"))
+	// enable USB emulation
+	opts = append(opts, drvOpt.USB())
+	for _, device := range input.Devices {
+		opts = append(opts, drvOpt.Device(device))
+	}
+
+	// vdi spice
+	if input.IsVdiSpice {
+		opts = append(opts, drvOpt.VdiSpice(input.SpicePort, input.PCIBus)...)
+	} else {
+		if input.IsolatedDevicesParams != nil && len(input.IsolatedDevicesParams.Vga) > 0 {
+			opts = append(opts, drvOpt.VGA("", input.IsolatedDevicesParams.Vga))
+		} else {
+			if input.VGA != "" {
+				opts = append(opts, drvOpt.VGA(input.VGA, ""))
+			}
+		}
+		opts = append(opts, drvOpt.VNC(input.VNCPort, input.VNCPassword))
+	}
+
+	// iothread object
+	opts = append(opts, drvOpt.Object("iothread", map[string]string{"id": "iothread0"}))
+
+	// genereate disk options
+	opts = append(opts, generateDisksOptions(drvOpt, input.Disks, input.PCIBus, input.IsVdiSpice)...)
+
+	// cdrom
+	opts = append(opts, drvOpt.Cdrom(input.CdromPath, input.OsName, input.IsQ35, len(input.Disks))...)
+
+	// genereate nics
+	nicOpts, err := generateNicOptions(drvOpt, input)
+	if err != nil {
+		return "", errors.Wrap(err, "generateNicOptions")
+	}
+	opts = append(opts, nicOpts...)
+
+	// isolated devices
+	// USB 3.0
+	opts = append(opts, drvOpt.Device("qemu-xhci,id=usb"))
+	if input.IsolatedDevicesParams != nil {
+		for _, each := range input.IsolatedDevicesParams.Devices {
+			opts = append(opts, each)
+		}
+	}
+
+	// pidfile
+	opts = append(opts, drvOpt.Pidfile(input.PidFilePath))
+
+	// extra options
+	if len(input.ExtraOptions) != 0 {
+		opts = append(opts, input.ExtraOptions...)
+	}
+
+	// qga
+	opts = append(opts, drvOpt.QGA(input.HomeDir)...)
+
+	// random device
+	if input.EnableRNGRandom {
+		opts = append(opts, getRNGRandomOptions(drvOpt)...)
+	}
+
+	// serial device
+	if input.EnableSerialDevice {
+		opts = append(opts, drvOpt.SerialDevice()...)
+	}
+
+	// migrate options
+	opts = append(opts, getMigrateOptions(drvOpt, input)...)
+
+	// pvpanic device
+	opts = append(opts, drvOpt.PvpanicDevice())
+
+	return strings.Join(opts, " "), nil
+}
+
+func getMonitorOptions(drvOpt QemuOptions, input *Monitor) []string {
+	if input == nil {
+		return nil
+	}
+	idDev := input.Id + "dev"
+	opts := []string{
+		drvOpt.MonitorChardev(idDev, input.Port, "127.0.0.1"),
+		drvOpt.Mon(idDev, input.Id, input.Mode),
+	}
+	return opts
+}
+
+func generateDisksOptions(drvOpt QemuOptions, disks []api.GuestdiskJsonDesc, pciBus string, isVdiSpice bool) []string {
+	opts := []string{}
+	isArm := drvOpt.IsArm()
+	firstDriver := make(map[string]bool)
+	for _, disk := range disks {
+		driver := disk.Driver
+		if isArm && (driver == DISK_DRIVER_IDE || driver == DISK_DRIVER_SATA) {
+			// unsupported configuration: IDE controllers are unsupported
+			driver = DISK_DRIVER_SCSI
+		}
+		if driver == DISK_DRIVER_SCSI || driver == DISK_DRIVER_PVSCSI {
+			if _, ok := firstDriver[driver]; !ok {
+				switch driver {
+				case DISK_DRIVER_SCSI:
+					// FIXME: iothread will make qemu-monitor hang
+					// REF: https://www.mail-archive.com/qemu-devel@nongnu.org/msg592729.html
+					// cmd += " -device virtio-scsi-pci,id=scsi,iothread=iothread0,num_queues=4,vectors=5"
+					opts = append(opts, drvOpt.Device("virtio-scsi-pci,id=scsi,num_queues=4,vectors=5"))
+				case DISK_DRIVER_PVSCSI:
+					opts = append(opts, drvOpt.Device("pvscsi,id=scsi"))
+				}
+				firstDriver[driver] = true
+			}
+		}
+		opts = append(opts,
+			getDiskDriveOption(drvOpt, disk, isArm),
+			getDiskDeviceOption(drvOpt, disk, isArm, pciBus, isVdiSpice),
+		)
+	}
+	return opts
+}
+
+func getDiskDriveOption(drvOpt QemuOptions, disk api.GuestdiskJsonDesc, isArm bool) string {
+	format := disk.Format
+	diskIndex := disk.Index
+	cacheMode := disk.CacheMode
+	aioMode := disk.AioMode
+
+	opt := fmt.Sprintf("file=$DISK_%d", diskIndex)
+	opt += ",if=none"
+	opt += fmt.Sprintf(",id=drive_%d", diskIndex)
+	if len(format) == 0 || format == "qcow2" {
+		// pass    # qemu will automatically detect image format
+	} else if format == "raw" {
+		opt += ",format=raw"
+	}
+	opt += fmt.Sprintf(",cache=%s", cacheMode)
+	if isLocalStorage(disk) {
+		opt += fmt.Sprintf(",aio=%s", aioMode)
+	}
+	if len(disk.Url) > 0 { // # a remote file backed image
+		opt += ",copy-on-read=on"
+	}
+	if isLocalStorage(disk) {
+		opt += ",file.locking=off"
+	}
+	// #opt += ",media=disk"
+	return drvOpt.Drive(opt)
+}
+
+func isLocalStorage(disk api.GuestdiskJsonDesc) bool {
+	if disk.StorageType == api.STORAGE_LOCAL || len(disk.StorageType) == 0 {
+		return true
+	} else {
+		return false
+	}
+}
+
+func getDiskDeviceOption(optDrv QemuOptions, disk api.GuestdiskJsonDesc, isArm bool, pciBus string, isVdiSpice bool) string {
+	diskIndex := disk.Index
+	diskDriver := disk.Driver
+	numQueues := disk.NumQueues
+	isSsd := disk.IsSSD
+
+	if numQueues == 0 {
+		numQueues = 4
+	}
+
+	if isArm && (diskDriver == DISK_DRIVER_IDE || diskDriver == DISK_DRIVER_SATA) {
+		// unsupported configuration: IDE controllers are unsupported
+		// for this QEMU binary or machine type
+		// replace with scsi
+		diskDriver = DISK_DRIVER_SCSI
+	}
+
+	var opt = ""
+	opt += GetDiskDeviceModel(diskDriver)
+	opt += fmt.Sprintf(",drive=drive_%d", diskIndex)
+	if diskDriver == DISK_DRIVER_VIRTIO {
+		// virtio-blk
+		opt += fmt.Sprintf(",bus=%s,addr=0x%x", pciBus, GetDiskAddr(int(diskIndex), isVdiSpice))
+		opt += fmt.Sprintf(",num-queues=%d,vectors=%d,iothread=iothread0", numQueues, numQueues+1)
+	} else if utils.IsInStringArray(diskDriver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) {
+		opt += ",bus=scsi.0"
+	} else if diskDriver == DISK_DRIVER_IDE {
+		opt += fmt.Sprintf(",bus=ide.%d,unit=%d", diskIndex/2, diskIndex%2)
+	} else if diskDriver == DISK_DRIVER_SATA {
+		opt += fmt.Sprintf(",bus=ide.%d", diskIndex)
+	}
+	opt += fmt.Sprintf(",id=drive_%d", diskIndex)
+	if isSsd {
+		opt += ",rotation_rate=1"
+	}
+	return optDrv.Device(opt)
+
+}
+
+func GetNicAddr(index int, disksLen int, isoDevsLen int, isVdiSpice bool) int {
+	var pciBase = 10
+	if disksLen > 10 {
+		pciBase = 20
+	}
+	if isoDevsLen > 0 {
+		pciBase += 10
+	}
+	return GetDiskAddr(pciBase+index, isVdiSpice)
+}
+
+func GetDiskAddr(idx int, isVdiSpice bool) int {
+	var base = 5
+	if isVdiSpice {
+		base += 10
+	}
+	return base + idx
+}
+
+func GetDiskDeviceModel(driver string) string {
+	if driver == DISK_DRIVER_VIRTIO {
+		return "virtio-blk-pci"
+	} else if utils.IsInStringArray(driver, []string{DISK_DRIVER_SCSI, DISK_DRIVER_PVSCSI}) {
+		return "scsi-hd"
+	} else if driver == DISK_DRIVER_IDE {
+		return "ide-hd"
+	} else if driver == DISK_DRIVER_SATA {
+		return "ide-drive"
+	} else {
+		return "None"
+	}
+}
+
+func generateNicOptions(drvOpt QemuOptions, input *GenerateStartOptionsInput) ([]string, error) {
+	opts := []string{}
+	nics := input.Nics
+	withAddr := true
+	if drvOpt.IsArm() {
+		withAddr = false
+	}
+	for idx := range nics {
+		netDevOpt, err := getNicNetdevOption(drvOpt, nics[idx], input.IsKVMSupport)
+		if err != nil {
+			return nil, errors.Wrapf(err, "getNicNetdevOption %s", nics[idx])
+		}
+		opts = append(opts,
+			netDevOpt,
+			// aarch64 with addr lead to:
+			// virtio_net: probe of virtioN failed with error -22
+			getNicDeviceOption(drvOpt, nics[idx], input, withAddr))
+	}
+	return opts, nil
+}
+
+func getNicNetdevOption(drvOpt QemuOptions, nic jsonutils.JSONObject, isKVMSupport bool) (string, error) {
+	ifname, _ := nic.GetString("ifname")
+	if ifname == "" {
+		return "", errors.Error("ifname is empty")
+	}
+	driver, _ := nic.GetString("driver")
+	upscript, _ := nic.GetString("upscript_path")
+	if upscript == "" {
+		return "", errors.Error("upscript_path is empty")
+	}
+	downscript, _ := nic.GetString("downscript_path")
+	if downscript == "" {
+		return "", errors.Error("downscript_path is empty")
+	}
+	opt := "-netdev type=tap"
+	opt += fmt.Sprintf(",id=%s", ifname)
+	opt += fmt.Sprintf(",ifname=%s", ifname)
+	if driver == "virtio" && isKVMSupport {
+		opt += ",vhost=on,vhostforce=off"
+	}
+	opt += fmt.Sprintf(",script=%s", upscript)
+	opt += fmt.Sprintf(",downscript=%s", downscript)
+	return opt, nil
+}
+
+func getNicDeviceOption(drvOpt QemuOptions, nic jsonutils.JSONObject, input *GenerateStartOptionsInput, withAddr bool) string {
+	bridge, _ := nic.GetString("bridge")
+	ifname, _ := nic.GetString("ifname")
+	driver, _ := nic.GetString("driver")
+	mac, _ := nic.GetString("mac")
+	index, _ := nic.Int("index")
+	vectors, _ := nic.Int("vectors")
+	bw, _ := nic.Int("bw")
+
+	cmd := fmt.Sprintf("-device %s", GetNicDeviceModel(driver))
+	cmd += fmt.Sprintf(",id=netdev-%s", ifname)
+	cmd += fmt.Sprintf(",netdev=%s", ifname)
+	cmd += fmt.Sprintf(",mac=%s", mac)
+
+	if withAddr {
+		disksLen := len(input.Disks)
+		isoDevsLen := 0
+		if input.IsolatedDevicesParams != nil {
+			isoDevsLen = len(input.IsolatedDevicesParams.Devices)
+		}
+		cmd += fmt.Sprintf(",addr=0x%x", GetNicAddr(int(index), disksLen, isoDevsLen, input.IsVdiSpice))
+	}
+	if driver == "virtio" {
+		if nic.Contains("vectors") {
+			cmd += fmt.Sprintf(",vectors=%d", vectors)
+		}
+		cmd += fmt.Sprintf("$(nic_speed %d)", bw)
+		if bridge == input.OVNIntegrationBridge {
+			cmd += fmt.Sprintf("$(nic_mtu %q)", bridge)
+		}
+	}
+	return cmd
+}
+
+func GetNicDeviceModel(name string) string {
+	if name == "virtio" {
+		return "virtio-net-pci"
+	} else if name == "e1000" {
+		return "e1000-82545em"
+	} else {
+		return name
+	}
+}
+
+func getRNGRandomOptions(drvOpt QemuOptions) []string {
+	return []string{
+		drvOpt.Object("rng-random", map[string]string{
+			"filename": "/dev/random",
+			"id":       "rng0",
+		}),
+		drvOpt.Device("virtio-rng-pci,rng=rng0,max-bytes=1024,period=1000"),
+	}
+}
+
+func getMigrateOptions(drvOpt QemuOptions, input *GenerateStartOptionsInput) []string {
+	opts := []string{}
+	if input.NeedMigrate {
+		opts = append(opts, fmt.Sprintf("-incoming tcp:0:%d", input.LiveMigratePort))
+	} else if input.IsSlave {
+		opts = append(opts, fmt.Sprintf("-incoming tcp:0:%d", input.LiveMigratePort))
+	} else if input.IsMaster {
+		opts = append(opts, "-S")
+	}
+	return opts
+}

pkg/hostman/guestman/qemu/generate_test.go

@@ -0,0 +1,44 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"yunion.io/x/log"
+)
+
+func TestGenerateStartCommand(t *testing.T) {
+	assert := assert.New(t)
+	log.Errorf("---%v", assert)
+	input := &GenerateStartOptionsInput{
+		QemuVersion:          Version_2_12_1,
+		QemuArch:             Arch_x86_64,
+		UUID:                 "uuid-xxxx-xxxx",
+		Mem:                  1024,
+		Cpu:                  2,
+		Name:                 "test-vm",
+		OsName:               OS_NAME_LINUX,
+		OVNIntegrationBridge: "brvpc",
+		HomeDir:              "/opt/cloud/workspace/servers/sid",
+		HugepagesEnabled:     true,
+		PidFilePath:          "/opt/cloud/workspace/servers/sid/pid",
+	}
+	cmd, err := GenerateStartOptions(input)
+	log.Errorf("cmd: %s", cmd)
+	log.Errorf("error: %s", err)
+}

pkg/hostman/guestman/qemu/qemu.go

@@ -0,0 +1,523 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu
+
+import (
+	"fmt"
+	"path"
+	"strings"
+	"sync"
+
+	"yunion.io/x/log"
+
+	"yunion.io/x/onecloud/pkg/apis/compute"
+)
+
+type Version string
+
+const (
+	Version_4_0_1  Version = "4.0.1"
+	Version_2_12_1 Version = "2.12.1"
+)
+
+type Arch string
+
+const (
+	Arch_x86_64  Arch = "x86_64"
+	Arch_aarch64 Arch = "aarch64"
+)
+
+const (
+	OS_NAME_LINUX   = "Linux"
+	OS_NAME_WINDOWS = "Windows"
+	OS_NAME_MACOS   = "macOS"
+	OS_NAME_ANDROID = "Android"
+	OS_NAME_VMWARE  = "VMWare"
+	OS_NAME_CIRROS  = "Cirros"
+	OS_NAME_OPENWRT = "OpenWrt"
+
+	MODE_READLINE = "readline"
+	MODE_CONTROL  = "control"
+
+	DISK_DRIVER_VIRTIO = "virtio"
+	DISK_DRIVER_SCSI   = "scsi"
+	DISK_DRIVER_PVSCSI = "pvscsi"
+	DISK_DRIVER_IDE    = "ide"
+	DISK_DRIVER_SATA   = "sata"
+
+	BIOS_UEFI = "UEFI"
+)
+
+type QemuCommand interface {
+	GetVersion() Version
+	GetArch() Arch
+	GetOptions() QemuOptions
+}
+
+type CPUOption struct {
+	EnableKVM          bool
+	IsKVMSupport       bool
+	HostCPUPassthrough bool
+	IsCPUIntel         bool
+	IsCPUAMD           bool
+	EnableNested       bool
+	IsolatedDeviceCPU  string
+}
+
+type QemuOptions interface {
+	IsArm() bool
+	CPU(opt CPUOption, osName string) (string, string, error)
+	Log(enable bool, qemuLogPath string) string
+	RTC() string
+	Daemonize() string
+	Nodefaults() string
+	Nodefconfig() string
+	NoKVMPitReinjection() string
+	Global() string
+	Machine(machineType string, accel string) string
+	KeyboardLayoutLanguage(lang string) string
+	SMP(cpus uint) string
+	Name(name string) string
+	UUID(enable bool, uuid string) string
+	Memory(sizeMB uint64) string
+	MemPrealloc() string
+	MemPath(p string) string
+	Boot(order string, enableMenu bool) string
+	BIOS(file string) string
+	Device(devStr string) string
+	Drive(driveStr string) string
+	Spice(port uint, password string) string
+	Chardev(backend string, id string, name string) string
+	MonitorChardev(id string, port uint, host string) string
+	Mon(chardev string, id string, mode string) string
+	Object(typeName string, props map[string]string) string
+	Pidfile(file string) string
+	USB() string
+	VdiSpice(spicePort uint, pciBus string) []string
+	VNC(port uint, usePasswd bool) string
+	VGA(vType string, alterOpt string) string
+	Cdrom(cdromPath string, osName string, isQ35 bool, disksLen int) []string
+	SerialDevice() []string
+	QGA(homeDir string) []string
+	PvpanicDevice() string
+}
+
+var (
+	cmdDrivers = new(sync.Map)
+)
+
+func getCommandKey(version Version, arch Arch) string {
+	return fmt.Sprintf("%s-%s", version, arch)
+}
+
+func RegisterCmd(cmds ...QemuCommand) {
+	for _, cmd := range cmds {
+		_, ok := GetCommand(cmd.GetVersion(), cmd.GetArch())
+		if ok {
+			log.Fatalf("cmd 'version %s' 'arch %s' already registered!", cmd.GetVersion(), cmd.GetArch())
+		}
+		cmdDrivers.Store(getCommandKey(cmd.GetVersion(), cmd.GetArch()), cmd)
+	}
+}
+
+func GetCommand(version Version, arch Arch) (QemuCommand, bool) {
+	key := getCommandKey(version, arch)
+	val, ok := cmdDrivers.Load(key)
+	if !ok {
+		return nil, false
+	}
+	return val.(QemuCommand), true
+}
+
+func EnsureGetCommand(version Version, arch Arch) QemuCommand {
+	cmd, ok := GetCommand(version, arch)
+	if !ok {
+		log.Fatalf("cmd %s %s not registered!", version, arch)
+	}
+	return cmd
+}
+
+type baseCommand struct {
+	version Version
+	arch    Arch
+	options QemuOptions
+}
+
+func newBaseCommand(version Version, arch Arch, opts QemuOptions) QemuCommand {
+	return &baseCommand{
+		version: version,
+		arch:    arch,
+		options: opts,
+	}
+}
+
+func (c *baseCommand) GetVersion() Version {
+	return c.version
+}
+
+func (c *baseCommand) GetArch() Arch {
+	return c.arch
+}
+
+func (c *baseCommand) GetOptions() QemuOptions {
+	return c.options
+}
+
+type baseOptions struct {
+	arch Arch
+}
+
+func newBaseOptions(arch Arch) *baseOptions {
+	return &baseOptions{
+		arch: arch,
+	}
+}
+
+func (o baseOptions) IsArm() bool {
+	return o.arch == Arch_aarch64
+}
+
+func (o baseOptions) Log(enable bool, qemuLogPath string) string {
+	if !enable {
+		return ""
+	}
+	return fmt.Sprintf("-D %s -d all", qemuLogPath)
+}
+
+func (o baseOptions) RTC() string {
+	return "-rtc base=utc,clock=host,driftfix=none"
+}
+
+func (o baseOptions) Daemonize() string {
+	return "-daemonize"
+}
+
+func (o baseOptions) Nodefaults() string {
+	return "-nodefaults"
+}
+
+func (o baseOptions) Nodefconfig() string {
+	return "-nodefconfig"
+}
+
+func (o baseOptions) NoKVMPitReinjection() string {
+	return "-no-kvm-pit-reinjection"
+}
+
+func (o baseOptions) Global() string {
+	return "-global kvm-pit.lost_tick_policy=discard"
+}
+
+func (o baseOptions) KeyboardLayoutLanguage(lang string) string {
+	return "-k " + lang
+}
+
+func (o baseOptions) Name(name string) string {
+	return "-name " + name
+}
+
+func (o baseOptions) UUID(enable bool, uuid string) string {
+	if !enable {
+		return ""
+	}
+	return "-uuid " + uuid
+}
+
+func (o baseOptions) MemPrealloc() string {
+	return "-mem-prealloc"
+}
+
+func (o baseOptions) MemPath(p string) string {
+	return "-mem-path " + p
+}
+
+func (o baseOptions) Boot(order string, enableMenu bool) string {
+	opt := "-boot order=" + order
+	if enableMenu {
+		opt += ",menu=on"
+	}
+	return opt
+}
+
+func (o baseOptions) BIOS(file string) string {
+	return "-bios " + file
+}
+
+func (o baseOptions) Device(devStr string) string {
+	return "-device " + devStr
+}
+
+func (o baseOptions) Drive(driveStr string) string {
+	return "-drive " + driveStr
+}
+
+func (o baseOptions) Spice(port uint, password string) string {
+	return fmt.Sprintf("-spice port=%d,password=%s,seamless-migration=on", port, password)
+}
+
+func (o baseOptions) Chardev(backend string, id string, name string) string {
+	opt := fmt.Sprintf("-chardev %s,id=%s", backend, id)
+	if name != "" {
+		opt = fmt.Sprintf("%s,name=%s", opt, name)
+	}
+	return opt
+}
+
+func (o baseOptions) MonitorChardev(id string, port uint, host string) string {
+	opt := o.Chardev("socket", id, "")
+	return fmt.Sprintf("%s,port=%d,host=%s,nodelay,server,nowait", opt, port, host)
+}
+
+func (o baseOptions) Mon(chardev string, id string, mode string) string {
+	return fmt.Sprintf("-mon chardev=%s,id=%s,mode=%s", chardev, id, mode)
+}
+
+func (o baseOptions) Object(typeName string, props map[string]string) string {
+	propStrs := []string{}
+	for k, v := range props {
+		propStrs = append(propStrs, fmt.Sprintf("%s=%s", k, v))
+	}
+	opt := fmt.Sprintf("-object %s", typeName)
+	if len(propStrs) > 0 {
+		opt = opt + "," + strings.Join(propStrs, ",")
+	}
+	return opt
+}
+
+func (o baseOptions) Pidfile(file string) string {
+	return "-pidfile " + file
+}
+
+func (o baseOptions) USB() string {
+	return "-usb"
+}
+
+func (o baseOptions) VdiSpice(spicePort uint, pciBus string) []string {
+	return []string{
+		o.Device("qxl-vga,id=video0,ram_size=141557760,vram_size=141557760"),
+		o.Device("intel-hda,id=sound0"),
+		o.Device("hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0"),
+		fmt.Sprintf("-spice port=%d,password=87654312,seamless-migration=on", spicePort),
+		// # ,streaming-video=all,playback-compression=on,jpeg-wan-compression=always,zlib-glz-wan-compression=always,image-compression=glz" % (5900+vnc_port)
+		o.Device(fmt.Sprintf("virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=%s", pciBus)),
+		o.Chardev("spicevmc", "vdagent", "vdagent"),
+		o.Device("virtserialport,nr=1,bus=virtio-serial0.0,chardev=vdagent,name=com.redhat.spice.0"),
+
+		// usb redirect
+		o.Device("ich9-usb-ehci1,id=usbspice"),
+		o.Device("ich9-usb-uhci1,masterbus=usbspice.0,firstport=0,multifunction=on"),
+		o.Device("ich9-usb-uhci2,masterbus=usbspice.0,firstport=2"),
+		o.Device("ich9-usb-uhci3,masterbus=usbspice.0,firstport=4"),
+		o.Chardev("spicevmc", "usbredirchardev1", "usbredir"),
+		o.Device("usb-redir,chardev=usbredirchardev1,id=usbredirdev1"),
+		o.Chardev("spicevmc", "usbredirchardev2", "usbredir"),
+		o.Device("usb-redir,chardev=usbredirchardev2,id=usbredirdev2"),
+	}
+}
+
+func (o baseOptions) VNC(port uint, usePasswd bool) string {
+	opt := fmt.Sprintf("-vnc :%d", port)
+	if usePasswd {
+		opt += ",password"
+	}
+	return opt
+}
+
+func (o baseOptions) VGA(vType string, alternativeOpt string) string {
+	if alternativeOpt != "" {
+		return alternativeOpt
+	}
+	return "-vga " + vType
+}
+
+type baseOptions_x86_64 struct {
+	*baseOptions
+}
+
+func newBaseOptions_x86_64() *baseOptions_x86_64 {
+	return &baseOptions_x86_64{
+		baseOptions: newBaseOptions(Arch_x86_64),
+	}
+}
+
+func (o *baseOptions_x86_64) CPU(input CPUOption, osName string) (string, string, error) {
+	var accel, cpuType string
+	if input.EnableKVM {
+		accel = "kvm"
+		cpuType = ""
+		if osName == OS_NAME_MACOS {
+			cpuType = "Penryn,vendor=GenuineIntel"
+		} else if input.HostCPUPassthrough {
+			cpuType = "host"
+			// https://unix.stackexchange.com/questions/216925/nmi-received-for-unknown-reason-20-do-you-have-a-strange-power-saving-mode-ena
+			cpuType += ",+kvm_pv_eoi"
+		} else {
+			cpuType = "qemu64"
+			cpuType += ",+kvm_pv_eoi"
+			if input.IsCPUIntel {
+				cpuType += ",+vmx"
+				cpuType += ",+ssse3,+sse4.1,+sse4.2,-x2apic,+aes,+avx"
+				cpuType += ",+vme,+pat,+ss,+pclmulqdq,+xsave"
+				cpuType += ",level=13"
+			} else if input.IsCPUAMD {
+				cpuType += ",+svm"
+			}
+		}
+
+		if !input.EnableNested {
+			cpuType += ",kvm=off"
+		}
+
+		if len(input.IsolatedDeviceCPU) > 0 {
+			cpuType = input.IsolatedDeviceCPU
+		}
+	} else {
+		accel = "tcg"
+		cpuType = "qemu64"
+	}
+	return fmt.Sprintf("-cpu %s", cpuType), accel, nil
+}
+
+func (o baseOptions_x86_64) Machine(mType string, accel string) string {
+	return fmt.Sprintf("-machine %s,accel=%s", mType, accel)
+}
+
+func (o baseOptions_x86_64) SMP(cpus uint) string {
+	return fmt.Sprintf("-smp cpus=%d,sockets=2,cores=64,maxcpus=128", cpus)
+}
+
+func (o baseOptions_x86_64) Memory(sizeMB uint64) string {
+	return fmt.Sprintf("-m %dM,slots=4,maxmem=524288M", sizeMB)
+}
+
+func (o baseOptions_x86_64) Cdrom(cdromPath string, osName string, isQ35 bool, disksLen int) []string {
+	opts := []string{}
+	cdromDrv := "id=ide0-cd0,media=cdrom,if=none"
+	if osName != OS_NAME_MACOS {
+		tmpOpt := "ide-cd,drive=ide0-cd0,bus=ide.1"
+		if isQ35 {
+			tmpOpt += ",unit=1"
+		}
+		opts = append(opts, o.Device(tmpOpt))
+	}
+	if cdromPath != "" {
+		if osName != OS_NAME_MACOS {
+			cdromDrv += fmt.Sprintf(",file=%s", cdromPath)
+		} else {
+			opts = append(opts,
+				o.Device(fmt.Sprintf("ide-drive,drive=MacDVD,bus=ide.%d", disksLen)),
+				o.Drive(fmt.Sprintf("id=MacDVD,if=none,snapshot=on,file=%s", cdromPath)))
+		}
+	}
+	if osName != OS_NAME_MACOS {
+		opts = append(opts, o.Drive(cdromDrv))
+	}
+	return opts
+}
+
+func (o baseOptions_x86_64) SerialDevice() []string {
+	return []string{
+		o.Chardev("pty", "charserial0", ""),
+		o.Device("isa-serial,chardev=charserial0,id=serial0"),
+	}
+}
+
+func (o baseOptions_x86_64) QGA(homeDir string) []string {
+	return []string{
+		fmt.Sprintf("-chardev socket,path=%s,server,nowait,id=qga0", path.Join(homeDir, "qga.sock")),
+		o.Device("virtserialport,chardev=qga0,name=org.qemu.guest_agent.0"),
+	}
+}
+
+func (o baseOptions_x86_64) PvpanicDevice() string {
+	return o.Device("pvpanic")
+}
+
+type baseOptions_aarch64 struct {
+	*baseOptions
+}
+
+func newBaseOptions_aarch64() *baseOptions_aarch64 {
+	return &baseOptions_aarch64{
+		baseOptions: newBaseOptions(Arch_aarch64),
+	}
+}
+
+func (o *baseOptions_aarch64) CPU(input CPUOption, osName string) (string, string, error) {
+	var accel, cpuType string
+	if input.EnableKVM {
+		accel = "kvm"
+		if input.HostCPUPassthrough {
+			cpuType = "host"
+		} else {
+			// * under KVM, -cpu max is the same as -cpu host
+			// * under TCG, -cpu max means "emulate with as many features as possible"
+			cpuType = "max"
+		}
+	} else {
+		accel = "tcg"
+		cpuType = "max"
+	}
+	return fmt.Sprintf("-cpu %s", cpuType), accel, nil
+}
+
+func (o baseOptions_aarch64) Machine(mType string, accel string) string {
+	// TODO: fix machine type on region controller side
+	if mType == "" || mType == compute.VM_MACHINE_TYPE_PC || mType == compute.VM_MACHINE_TYPE_Q35 {
+		mType = "virt-2.12"
+	}
+	return fmt.Sprintf("-machine %s,accel=%s,gic-version=3", mType, accel)
+}
+
+func (o baseOptions_aarch64) NoKVMPitReinjection() string {
+	return ""
+}
+
+func (o baseOptions_aarch64) Global() string {
+	return ""
+}
+
+func (o baseOptions_aarch64) SMP(cpus uint) string {
+	// warning: Number of hotpluggable cpus requested (128)
+	// exceeds the recommended cpus supported by KVM (32)
+	return fmt.Sprintf("-smp cpus=%d,sockets=2,cores=32,maxcpus=64", cpus)
+}
+
+func (o baseOptions_aarch64) Memory(sizeMB uint64) string {
+	return fmt.Sprintf("-m %dM,slots=4,maxmem=262144M", sizeMB)
+}
+
+func (o baseOptions_aarch64) Cdrom(cdromPath string, osName string, isQ35 bool, disksLen int) []string {
+	opts := []string{}
+	if len(cdromPath) > 0 {
+		opts = append(opts,
+			o.Device("virtio-scsi-device -device scsi-cd,drive=cd0,share-rw=true"),
+			o.Drive(fmt.Sprintf("if=none,file=%s,id=cd0,media=cdrom", cdromPath)))
+	}
+	return opts
+}
+
+func (o baseOptions_aarch64) SerialDevice() []string {
+	return nil
+}
+
+func (o baseOptions_aarch64) QGA(_ string) []string {
+	return nil
+}
+
+func (o baseOptions_aarch64) PvpanicDevice() string {
+	// -device pvpanic: 'pvpanic' is not a valid device model name
+	return ""
+}

pkg/hostman/guestman/qemu/qemu_test.go

@@ -0,0 +1,70 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func Test_baseOptions(t *testing.T) {
+	opt := newBaseOptions_x86_64()
+	assert := assert.New(t)
+
+	// test object
+	assert.Equal("-object iothread,id=iothread0", opt.Object("iothread", map[string]string{"id": "iothread0"}))
+	// test chardev
+	assert.Equal("-chardev socket,id=test", opt.Chardev("socket", "test", ""))
+	assert.Equal("-chardev socket,id=test,name=tname", opt.Chardev("socket", "test", "tname"))
+	assert.Equal("-chardev socket,id=test,port=1234,host=127.0.0.1,nodelay,server,nowait", opt.MonitorChardev("test", 1234, "127.0.0.1"))
+	assert.Equal([]string{
+		"-chardev socket,id=testdev,port=1234,host=127.0.0.1,nodelay,server,nowait",
+		"-mon chardev=testdev,id=test,mode=readline",
+	}, getMonitorOptions(opt, &Monitor{
+		Id:   "test",
+		Port: 1234,
+		Mode: "readline",
+	}))
+	// test memory
+	assert.Equal("-m 1024M,slots=4,maxmem=524288M", opt.Memory(1024))
+	// test device
+	assert.Equal("-device isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc", opt.Device("isa-applesmc,osk=ourhardworkbythesewordsguardedpleasedontsteal(c)AppleComputerInc"))
+	// test vdi spice
+	assert.Equal([]string{
+		"-device qxl-vga,id=video0,ram_size=141557760,vram_size=141557760",
+		"-device intel-hda,id=sound0",
+		"-device hda-duplex,id=sound0-codec0,bus=sound0.0,cad=0",
+		"-spice port=5910,password=87654312,seamless-migration=on",
+		"-device virtio-serial-pci,id=virtio-serial0,max_ports=16,bus=pcie.0",
+		"-chardev spicevmc,id=vdagent,name=vdagent",
+		"-device virtserialport,nr=1,bus=virtio-serial0.0,chardev=vdagent,name=com.redhat.spice.0",
+		"-device ich9-usb-ehci1,id=usbspice",
+		"-device ich9-usb-uhci1,masterbus=usbspice.0,firstport=0,multifunction=on",
+		"-device ich9-usb-uhci2,masterbus=usbspice.0,firstport=2",
+		"-device ich9-usb-uhci3,masterbus=usbspice.0,firstport=4",
+		"-chardev spicevmc,id=usbredirchardev1,name=usbredir",
+		"-device usb-redir,chardev=usbredirchardev1,id=usbredirdev1",
+		"-chardev spicevmc,id=usbredirchardev2,name=usbredir",
+		"-device usb-redir,chardev=usbredirchardev2,id=usbredirdev2",
+	},
+		opt.VdiSpice(5910, "pcie.0"))
+	// test vnc
+	assert.Equal("-vnc :5900,password", opt.VNC(5900, true))
+	assert.Equal("-vnc :5900", opt.VNC(5900, false))
+	// test vga
+	assert.Equal("-vga std", opt.VGA("std", ""))
+	assert.Equal("-vga x", opt.VGA("std", "-vga x"))
+}

pkg/hostman/guestman/qemu/v2_12_1.go

@@ -0,0 +1,48 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu
+
+func init() {
+	RegisterCmd(
+		newCmd_2_12_1_x86_64(),
+		newCmd_2_12_1_aarch64(),
+	)
+}
+
+func newCmd_2_12_1_x86_64() QemuCommand {
+	return newBaseCommand(
+		Version_2_12_1,
+		Arch_x86_64,
+		newOpt_2_12_1_x86_64(),
+	)
+}
+
+type opt_2121_x86_64 struct {
+	*baseOptions_x86_64
+}
+
+func newOpt_2_12_1_x86_64() QemuOptions {
+	return &opt_2121_x86_64{
+		baseOptions_x86_64: newBaseOptions_x86_64(),
+	}
+}
+
+func newCmd_2_12_1_aarch64() QemuCommand {
+	return newBaseCommand(
+		Version_2_12_1,
+		Arch_aarch64,
+		newBaseOptions_aarch64(),
+	)
+}

pkg/hostman/guestman/qemu/v4_0_1.go

@@ -0,0 +1,70 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package qemu
+
+func init() {
+	RegisterCmd(
+		newCmd_4_0_1_x86_64(),
+		newCmd_4_0_1_aarch64(),
+	)
+}
+
+func newCmd_4_0_1_x86_64() QemuCommand {
+	return newBaseCommand(
+		Version_4_0_1,
+		Arch_x86_64,
+		newOpt_4_0_1_x86_64())
+}
+
+type opt_401_x86_64 struct {
+	*baseOptions_x86_64
+}
+
+func newOpt_4_0_1_x86_64() QemuOptions {
+	return &opt_401_x86_64{
+		baseOptions_x86_64: newBaseOptions_x86_64(),
+	}
+}
+
+func (o opt_401_x86_64) Nodefconfig() string {
+	return "-no-user-config"
+}
+
+func (o opt_401_x86_64) NoKVMPitReinjection() string {
+	// https://www.qemu.org/docs/master/about/removed-features.html#no-kvm-pit-reinjection-removed-in-3-0
+	// -no-kvm-pit-reinjection (removed in 3.0)
+	return ""
+}
+
+func newCmd_4_0_1_aarch64() QemuCommand {
+	return newBaseCommand(
+		Version_4_0_1,
+		Arch_aarch64,
+		newOpt_4_0_1_aarch64())
+}
+
+type opt_401_aarch64 struct {
+	*baseOptions_aarch64
+}
+
+func newOpt_4_0_1_aarch64() QemuOptions {
+	return &opt_401_aarch64{
+		baseOptions_aarch64: newBaseOptions_aarch64(),
+	}
+}
+
+func (o opt_401_aarch64) Nodefconfig() string {
+	return "-no-user-config"
+}

pkg/hostman/hostinfo/hostbridge/hostbridge.go

@@ -337,8 +337,7 @@ func (d *SBaseBridgeDriver) saveFileExecutable(scriptPath, script string) error
 func (d *SBaseBridgeDriver) generateIfdownScripts(driver IBridgeDriver, scriptPath string, nic jsonutils.JSONObject, isSlave bool) error {
 	script, err := driver.getDownScripts(nic, isSlave)
 	if err != nil {
-		log.Errorln(err)
-		return err
+		return errors.Wrap(err, "getDownScripts")
 	}
 	return d.saveFileExecutable(scriptPath, script)
 }