Merge pull request #865 in YUNIONIO/onecloud from ~QIUJIAN/onecloud:feature/qj-esxiagent-support to release/2.5.0

* commit '1061b47afd0a060091a5fbd5526b295d05db4e60':
  修正：镜像通过PUT接口上传后，需要启动image convert tasks
  支持esxi agent，创建主机缺省用vmdk格式的镜像

pkg/compute/guestdrivers/esxi.go

@@ -3,7 +3,6 @@ package guestdrivers
 import (
 	"context"
 	"fmt"
-	"net/http"
 	"time"
 
 	"yunion.io/x/jsonutils"
@@ -13,6 +12,7 @@ import (
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/util/billing"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 type SESXiGuestDriver struct {
@@ -101,11 +101,7 @@ func (self *SESXiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gues
 	config := guest.GetDeployConfigOnHost(ctx, host, task.GetParams())
 	log.Debugf("RequestDeployGuestOnHost: %s", config)
 
-	agent, err := host.GetEsxiAgentHost()
-	if err != nil {
-		return err
-	}
-	if agent == nil {
+	if !host.IsEsxiAgentReady() {
 		return fmt.Errorf("No ESXi agent host")
 	}
 
@@ -132,11 +128,9 @@ func (self *SESXiGuestDriver) RequestDeployGuestOnHost(ctx context.Context, gues
 	body := jsonutils.NewDict()
 	body.Add(config, "disk")
 
-	header := http.Header{}
-	header.Add("X-Task-Id", task.GetTaskId())
-	header.Add("X-Region-Version", "v2")
+	header := task.GetTaskRequestHeader()
 
-	_, err = agent.Request(ctx, task.GetUserCred(), "POST", url, header, body)
+	_, err = host.EsxiRequest(ctx, httputils.POST, url, header, body)
 	return err
 }
 

pkg/compute/hostdrivers/esxi.go

@@ -2,12 +2,12 @@ package hostdrivers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 
-	"github.com/golang-plus/errors"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
 	"yunion.io/x/onecloud/pkg/compute/models"
 	"yunion.io/x/onecloud/pkg/util/httputils"
@@ -61,7 +61,8 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo
 	content.ImageId = imageId
 	content.HostId = host.Id
 	content.HostIp = host.AccessIp
-	content.Format = cacheImage.GetFormat()
+	// format force VMDK
+	content.Format = "vmdk" // cacheImage.GetFormat()
 
 	storage := host.GetStorageByFilePath(storageCache.Path)
 	if storage == nil {
@@ -95,17 +96,11 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo
 		content.SrcDatastore = accessInfo
 	}
 
-	agent, err := host.GetEsxiAgentHost()
-	if err != nil {
-		log.Errorf("find ESXi agent fail: %s", err)
-		return err
-	}
-
-	if agent == nil {
+	if !host.IsEsxiAgentReady() {
 		return fmt.Errorf("fail to find valid ESXi agent")
 	}
 
-	url := fmt.Sprintf("%s/disks/image_cache", agent.ManagerUri)
+	url := "/disks/image_cache"
 
 	if isForce {
 		content.IsForce = true
@@ -117,7 +112,7 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo
 
 	header := task.GetTaskRequestHeader()
 
-	_, _, err = httputils.JSONRequest(httputils.GetDefaultClient(), ctx, "POST", url, header, body, false)
+	_, err = host.EsxiRequest(ctx, httputils.POST, url, header, body)
 	if err != nil {
 		return err
 	}
@@ -125,13 +120,7 @@ func (self *SESXiHostDriver) CheckAndSetCacheImage(ctx context.Context, host *mo
 }
 
 func (self *SESXiHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, host *models.SHost, storage *models.SStorage, disk *models.SDisk, task taskman.ITask, content *jsonutils.JSONDict) error {
-	agent, err := host.GetEsxiAgentHost()
-	if err != nil {
-		log.Errorf("find ESXi agent fail: %s", err)
-		return err
-	}
-
-	if agent == nil {
+	if !host.IsEsxiAgentReady() {
 		return fmt.Errorf("fail to find valid ESXi agent")
 	}
 
@@ -158,18 +147,12 @@ func (self *SESXiHostDriver) RequestAllocateDiskOnStorage(ctx context.Context, h
 
 	header := task.GetTaskRequestHeader()
 
-	_, err = agent.Request(ctx, task.GetUserCred(), "POST", url, header, body)
+	_, err = host.EsxiRequest(ctx, httputils.POST, url, header, body)
 	return err
 }
 
 func (self *SESXiHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, host *models.SHost, disk *models.SDisk, imageId string, task taskman.ITask) error {
-	agent, err := host.GetEsxiAgentHost()
-	if err != nil {
-		log.Errorf("find ESXi agent fail: %s", err)
-		return err
-	}
-
-	if agent == nil {
+	if !host.IsEsxiAgentReady() {
 		return fmt.Errorf("fail to find valid ESXi agent")
 	}
 
@@ -218,7 +201,7 @@ func (self *SESXiHostDriver) RequestPrepareSaveDiskOnHost(ctx context.Context, h
 	url := fmt.Sprintf("/disks/agent/save-prepare/%s", disk.Id)
 	header := task.GetTaskRequestHeader()
 
-	_, err = agent.Request(ctx, task.GetUserCred(), "POST", url, header, body)
+	_, err = host.EsxiRequest(ctx, httputils.POST, url, header, body)
 	return err
 }
 
@@ -228,15 +211,15 @@ func (self *SESXiHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, h
 	if len(imagePath) == 0 {
 		return fmt.Errorf("missing parameter backup")
 	}
-	agentId, _ := data.GetString("agent_id")
-	if len(agentId) == 0 {
-		return fmt.Errorf("missing parameter agent_id")
-	}
+	// agentId, _ := data.GetString("agent_id")
+	// if len(agentId) == 0 {
+	// 	return fmt.Errorf("missing parameter agent_id")
+	// }
 
-	agent := models.HostManager.FetchHostById(agentId)
-	if agent == nil {
-		return fmt.Errorf("cannot find host with id %s", agentId)
-	}
+	// agent := models.HostManager.FetchHostById(agentId)
+	// if agent == nil {
+	// 	return fmt.Errorf("cannot find host with id %s", agentId)
+	// }
 
 	storage := disk.GetStorage()
 
@@ -245,7 +228,7 @@ func (self *SESXiHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, h
 		ImageId        string
 		StorageId      string
 		StoragecacheId string
-		Compress       bool `json:",allowempty"`
+		Compress       bool `json:",allowfalse"`
 	}
 
 	spec := specStruct{}
@@ -262,6 +245,6 @@ func (self *SESXiHostDriver) RequestSaveUploadImageOnHost(ctx context.Context, h
 
 	header := task.GetTaskRequestHeader()
 
-	_, err := agent.Request(ctx, task.GetUserCred(), "POST", url, header, body)
+	_, err := host.EsxiRequest(ctx, httputils.POST, url, header, body)
 	return err
 }

pkg/compute/models/baremetalagents.go

@@ -28,6 +28,8 @@ type SBaremetalagent struct {
 	AccessIp   string `width:"16" charset:"ascii" nullable:"false" list:"admin" update:"admin" create:"admin_required"`
 	ManagerUri string `width:"256" charset:"ascii" nullable:"true" list:"admin" update:"admin" create:"admin_required"`
 	ZoneId     string `width:"128" charset:"ascii" nullable:"false" list:"admin" update:"admin" create:"admin_required"`
+
+	AgentType string `width:"32" charset:"ascii" nullable:"true" default:"baremetal" list:"admin" update:"admin" create:"admin_optional"`
 }
 
 var BaremetalagentManager *SBaremetalagentManager
@@ -94,11 +96,11 @@ func (manager *SBaremetalagentManager) ValidateCreateData(ctx context.Context, u
 	if count > 0 {
 		return nil, httperrors.NewDuplicateResourceError("Duplicate manager_uri %s", mangerUri)
 	}
-	accessIp, _ := data.GetString("access_ip")
-	count = manager.Query().Equals("access_ip", accessIp).Count()
-	if count > 0 {
-		return nil, httperrors.NewDuplicateResourceError("Duplicate access_ip %s", accessIp)
-	}
+	//accessIp, _ := data.GetString("access_ip")
+	//count = manager.Query().Equals("access_ip", accessIp).Count()
+	//if count > 0 {
+	//	return nil, httperrors.NewDuplicateResourceError("Duplicate access_ip %s", accessIp)
+	//}
 	return manager.SStandaloneResourceBaseManager.ValidateCreateData(ctx, userCred, ownerProjId, query, data)
 }
 
@@ -137,7 +139,7 @@ func (self *SBaremetalagent) AllowPerformOnline(ctx context.Context, userCred mc
 }
 
 func (self *SBaremetalagent) PerformOnline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.Status != BAREMETAL_AGENT_OFFLINE {
+	if self.Status == BAREMETAL_AGENT_OFFLINE {
 		self.GetModelManager().TableSpec().Update(self, func() error {
 			self.Status = BAREMETAL_AGENT_ENABLED
 			return nil
@@ -152,7 +154,7 @@ func (self *SBaremetalagent) AllowPerformOffline(ctx context.Context, userCred m
 }
 
 func (self *SBaremetalagent) PerformOffline(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
-	if self.Status != BAREMETAL_AGENT_ENABLED {
+	if self.Status == BAREMETAL_AGENT_ENABLED {
 		self.GetModelManager().TableSpec().Update(self, func() error {
 			self.Status = BAREMETAL_AGENT_OFFLINE
 			return nil

pkg/compute/models/hosts.go

@@ -11,8 +11,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/serialx/hashring"
-
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/tristate"
@@ -2218,16 +2216,6 @@ func (self *SHost) GetLocalStoragecache() *SStoragecache {
 	return nil
 }
 
-func (self *SHost) IsBaremetalAgentReady() bool {
-	url, err := auth.GetServiceURL("baremetal", options.Options.Region, self.GetZone().GetName(), "")
-	if err != nil {
-		log.Errorln("is baremetal agent ready: false")
-		return false
-	}
-	log.Infof("baremetal url:%s", url)
-	return true
-}
-
 func (self *SHost) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data jsonutils.JSONObject) {
 	self.SEnabledStatusStandaloneResourceBase.PostCreate(ctx, userCred, ownerProjId, query, data)
 	kwargs := data.(*jsonutils.JSONDict)
@@ -2611,6 +2599,16 @@ func (self *SHost) StartBaremetalUnmaintenanceTask(ctx context.Context, userCred
 	return nil
 }
 
+func (self *SHost) IsBaremetalAgentReady() bool {
+	url, err := auth.GetServiceURL("baremetal", options.Options.Region, self.GetZone().GetName(), "")
+	if err != nil {
+		log.Errorln("is baremetal agent ready: false")
+		return false
+	}
+	log.Infof("baremetal url:%s", url)
+	return true
+}
+
 func (self *SHost) BaremetalSyncRequest(ctx context.Context, method httputils.THttpMethod, url string, headers http.Header, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
 	serviceUrl, err := auth.GetServiceURL("baremetal", options.Options.Region, self.GetZone().GetName(), "")
 	if err != nil {
@@ -3508,43 +3506,63 @@ func (host *SHost) SyncHostExternalNics(ctx context.Context, userCred mcclient.T
 	return result
 }
 
-func (manager *SHostManager) GetEsxiAgentHostId(key string) (string, error) {
-	q := HostManager.Query("id")
-	q = q.Equals("host_status", HOST_ONLINE)
-	q = q.Equals("host_type", HOST_TYPE_HYPERVISOR)
-	q = q.IsTrue("enabled")
+// func (manager *SHostManager) GetEsxiAgentHostId(key string) (string, error) {
+// 	q := HostManager.Query("id")
+// 	q = q.Equals("host_status", HOST_ONLINE)
+// 	q = q.Equals("host_type", HOST_TYPE_HYPERVISOR)
+// 	q = q.IsTrue("enabled")
+//
+// 	rows, err := q.Rows()
+// 	if err != nil {
+// 		return "", err
+// 	}
+// 	defer rows.Close()
+//
+// 	var hostId string
+// 	hostIds := make([]string, 0)
+// 	for rows.Next() {
+// 		err = rows.Scan(&hostId)
+// 		if err != nil {
+// 			return "", err
+// 		}
+// 		hostIds = append(hostIds, hostId)
+// 	}
+//
+// 	ring := hashring.New(hostIds)
+// 	ret, _ := ring.GetNode(key)
+// 	return ret, nil
+// }
+//
+// func (manager *SHostManager) GetEsxiAgentHost(key string) (*SHost, error) {
+// 	hostId, err := manager.GetEsxiAgentHostId(key)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+// 	return manager.FetchHostById(hostId), nil
+// }
+//
+// func (host *SHost) GetEsxiAgentHost() (*SHost, error) {
+// 	return HostManager.GetEsxiAgentHost(host.Id)
+// }
 
-	rows, err := q.Rows()
+func (self *SHost) IsEsxiAgentReady() bool {
+	url, err := auth.GetServiceURL("esxiagent", options.Options.Region, self.GetZone().GetName(), "")
 	if err != nil {
-		return "", err
+		log.Errorln("is esxi agent ready: false")
+		return false
 	}
-	defer rows.Close()
-
-	var hostId string
-	hostIds := make([]string, 0)
-	for rows.Next() {
-		err = rows.Scan(&hostId)
-		if err != nil {
-			return "", err
-		}
-		hostIds = append(hostIds, hostId)
-	}
-
-	ring := hashring.New(hostIds)
-	ret, _ := ring.GetNode(key)
-	return ret, nil
+	log.Infof("esxi agent url:%s", url)
+	return true
 }
 
-func (manager *SHostManager) GetEsxiAgentHost(key string) (*SHost, error) {
-	hostId, err := manager.GetEsxiAgentHostId(key)
+func (self *SHost) EsxiRequest(ctx context.Context, method httputils.THttpMethod, url string, headers http.Header, body *jsonutils.JSONDict) (jsonutils.JSONObject, error) {
+	serviceUrl, err := auth.GetServiceURL("esxiagent", options.Options.Region, self.GetZone().GetName(), "")
 	if err != nil {
 		return nil, err
 	}
-	return manager.FetchHostById(hostId), nil
-}
-
-func (host *SHost) GetEsxiAgentHost() (*SHost, error) {
-	return HostManager.GetEsxiAgentHost(host.Id)
+	url = serviceUrl + url
+	_, data, err := httputils.JSONRequest(httputils.GetDefaultClient(), ctx, method, url, headers, body, false)
+	return data, err
 }
 
 func (manager *SHostManager) GetHostByIp(hostIp string) (*SHost, error) {

pkg/image/models/images.go

@@ -461,6 +461,7 @@ func (self *SImage) ValidateUpdateData(ctx context.Context, userCred mcclient.To
 				}
 				self.OnSaveSuccess(ctx, userCred, "update upload success")
 				data.Remove("status")
+				self.StartImageConvertTask(ctx, userCred, "", true)
 			} else {
 				copyFrom := appParams.Request.Header.Get(modules.IMAGE_META_COPY_FROM)
 				if len(copyFrom) > 0 {

pkg/mcclient/modules/mod_baremetalagents.go

@@ -8,7 +8,7 @@ func init() {
 	Baremetalagents = NewComputeManager(
 		"baremetalagent",
 		"baremetalagents",
-		[]string{"ID", "Name", "Access_ip", "Manager_URI", "Status"},
+		[]string{"ID", "Name", "Access_ip", "Manager_URI", "Status", "agent_type"},
 		[]string{},
 	)
 	registerCompute(&Baremetalagents)