Fix problem of esxi (#7661)

* fix(esxi): Fetch full templates of datastoreImageCache

Datastore 和 DatastoreImageCache 是一对一的关系。
每一个 Datastore 可能有多个对应的 Host，所以 GetIIMages 里面获取
Templates 的逻辑应该尝试去获取所有 Host 的 Templates。

* fix(host-deployer): Exit correctly after mounting

MountRootfs的返回值为nil时，还是需要Umount的操作来正确退出（比如VDDK）。
VDDK的Mount分为两步，第一步会挂在remote
disk到fuse下，然后挂载fuse/xxx/flat
到块设备上。第一步成功，第二部失败也应该正确退出。

* fix(host-deployer): Kill process

* opt(host-deployer): Increase timeout of waiting mount

* opt(host-deployer): Add debug log print command to mount

* fix(esxi): Support multi agent in different zone

pkg/cloudcommon/agent/agent.go

@@ -239,6 +239,7 @@ func (agent *SBaseAgent) createOrUpdateBaremetalAgent(session *mcclient.ClientSe
 	if agent.IAgent().GetAgentType() != string(api.AgentTypeEsxi) {
 		params.Add(jsonutils.NewString(naccessIP.String()), "access_ip")
 	}
+	params.Add(jsonutils.NewString(agent.IAgent().GetZoneName()), "zone_id")
 	params.Add(jsonutils.NewString(agent.IAgent().GetAgentType()), "agent_type")
 	ret, err := modules.Baremetalagents.List(session, params)
 	if err != nil {

pkg/hostman/diskutils/diskutils.go

@@ -347,6 +347,9 @@ func (d *SKVMGuestDisk) UmountKvmRootfs(fd fsdriver.IRootFsDriver) {
 }
 
 func (d *SKVMGuestDisk) UmountRootfs(fd fsdriver.IRootFsDriver) {
+	if fd == nil {
+		return
+	}
 	d.UmountKvmRootfs(fd)
 }
 

pkg/hostman/diskutils/vddk.go

@@ -154,13 +154,6 @@ func (vd *VDDKDisk) MountRootfs() fsdriver.IRootFsDriver {
 	if err != nil {
 		log.Errorf("VDDKDisk Mount failed: %s", err)
 	}
-	// something is wrong
-	if vd.Proc != nil {
-		err := vd.Proc.Kill()
-		if err != nil {
-			log.Errorf("unable to kill proc: %s", err.Error())
-		}
-	}
 	return nil
 }
 
@@ -275,6 +268,7 @@ func (vd *VDDKDisk) ExecProg() error {
 	}
 	cmd := NewCommand(execpath(), "-info", "-host", vd.Host, "-port", strconv.Itoa(vd.Port), "-user", vd.User,
 		"-password", vd.Passwd, "-mode", "nbd", "-thumb", thumb, "-vm", fmt.Sprintf("moref=%s", vd.VmRef), vd.DiskPath)
+	log.Debugf("command to mount: %s", cmd)
 	env := os.Environ()
 	env = append(env, fmt.Sprintf("LD_LIBRARY_PATH=%s", libdir()))
 	cmd.Env = env
@@ -321,7 +315,7 @@ func (vd *VDDKDisk) getServerCertThumbSha1(addr string) (string, error) {
 
 func (vd *VDDKDisk) WaitMounted() error {
 	endStr := []byte("Do you want to procede to unmount the volume")
-	timeout := 30 * time.Second
+	timeout := 300 * time.Second
 	endClock := time.After(timeout)
 	isEnd := false
 
@@ -349,13 +343,16 @@ Loop:
 	}
 	if vd.Proc.Exited() {
 		retCode := vd.Proc.ProcessState.ExitCode()
-		// ignore the error
-		vd.Proc.Kill()
-		vd.Proc = nil
+		err := vd.Proc.Kill()
+		if err != nil {
+			log.Errorf("unable to kill process '%d'", vd.Proc.Process.Pid)
+		}
 		return errors.Error(fmt.Sprintf("VDDKDisk prog exit error(%d): %s", retCode, backup))
 	} else if !isEnd {
-		// timeout
-		vd.Proc.Kill()
+		err := vd.Proc.Kill()
+		if err != nil {
+			log.Errorf("unable to kill process '%d'", vd.Proc.Process.Pid)
+		}
 		return errors.Error(fmt.Sprintf("VDDKDisk read timeout, program blocked"))
 	}
 	return nil

pkg/hostman/hostdeployer/deployserver/deployserver.go

@@ -74,11 +74,11 @@ func (*DeployerServer) DeployGuestFs(ctx context.Context, req *deployapi.DeployP
 		return new(deployapi.DeployGuestFsResponse), nil
 	}
 	root := disk.MountRootfs()
+	defer disk.UmountRootfs(root)
 	if root == nil {
 		log.Infof("Failed mounting rootfs for %s disk", req.GuestDesc.Hypervisor)
 		return new(deployapi.DeployGuestFsResponse), nil
 	}
-	defer disk.UmountRootfs(root)
 
 	ret, err := guestfs.DoDeployGuestFs(root, req.GuestDesc, req.DeployInfo)
 	if err != nil {

pkg/multicloud/esxi/storagecache.go

@@ -81,17 +81,25 @@ func (self *SDatastoreImageCache) GetIImages() ([]cloudprovider.ICloudImage, err
 	ret := make([]cloudprovider.ICloudImage, 0, 2)
 
 	// get vm template with only one disk
-	tems, err := self.host.GetTemplateVMs()
+	ihosts, err := self.datastore.GetAttachedHosts()
 	if err != nil {
-		log.Errorf("fail to get templateVMs of host '%s' in SDatastoreImageCache.GetIImages", self.host.GetName())
-		return ret, nil
+		return nil, errors.Wrap(err, "SDatastore.GetAttachedHosts")
 	}
-	for _, tem := range tems {
-		// for now, add vm template with only one disk as cachedimage
-		if len(tem.vdisks) != 1 {
-			continue
+
+	for _, ihost := range ihosts {
+		host := ihost.(*SHost)
+		tems, err := host.GetTemplateVMs()
+		if err != nil {
+			log.Errorf("fail to get templateVMs of host '%s' in SDatastoreImageCache.GetIImages", self.host.GetName())
+			return ret, nil
+		}
+		for _, tem := range tems {
+			// for now, add vm template with only one disk as cachedimage
+			if len(tem.vdisks) != 1 {
+				continue
+			}
+			ret = append(ret, NewVMTemplate(tem, self))
 		}
-		ret = append(ret, NewVMTemplate(tem, self))
 	}
 
 	files, err := self.datastore.ListDir(ctx, IMAGE_CACHE_DIR_NAME)

pkg/multicloud/esxi/template.go

@@ -19,6 +19,7 @@ import (
 	"time"
 
 	"yunion.io/x/jsonutils"
+	"yunion.io/x/log"
 	"yunion.io/x/pkg/errors"
 
 	api "yunion.io/x/onecloud/pkg/apis/compute"
@@ -56,11 +57,22 @@ func (t *SVMTemplate) GetGlobalId() string {
 }
 
 func (t *SVMTemplate) GetStatus() string {
-	_, err := t.cache.host.GetTemplateVMById(t.uuid)
-	if errors.Cause(err) == cloudprovider.ErrNotFound {
+	ihosts, err := t.cache.datastore.GetAttachedHosts()
+	if err != nil {
 		return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 	}
-	return api.CACHED_IMAGE_STATUS_READY
+	for _, ihost := range ihosts {
+		host := ihost.(*SHost)
+		_, err := host.GetTemplateVMById(t.uuid)
+		if err == nil {
+			return api.CACHED_IMAGE_STATUS_READY
+		}
+		if errors.Cause(err) != cloudprovider.ErrNotFound {
+			log.Errorf("fail to find templatevm %q: %v", t.uuid, err)
+			return api.CACHED_IMAGE_STATUS_CACHE_FAILED
+		}
+	}
+	return api.CACHED_IMAGE_STATUS_CACHE_FAILED
 }
 
 func (t *SVMTemplate) Refresh() error {