Merge pull request #1942 from tb365/bugfix/tb-huawei-0723

华为 & ucloud fix

cmd/climc/shell/cloudproviders.go

@@ -28,6 +28,7 @@ func init() {
 
 	type CloudproviderListOptions struct {
 		options.BaseListOptions
+		Usable bool `help:"Vpc & Network usable"`
 	}
 	R(&CloudproviderListOptions{}, "cloud-provider-list", "List cloud providers", func(s *mcclient.ClientSession, args *CloudproviderListOptions) error {
 		var params *jsonutils.JSONDict
@@ -37,6 +38,10 @@ func init() {
 			if err != nil {
 				return err
 			}
+
+			if args.Usable {
+				params.Add(jsonutils.NewBool(true), "usable")
+			}
 		}
 		result, err := modules.Cloudproviders.List(s, params)
 		if err != nil {

pkg/apis/compute/vpcs_const.go

@@ -17,6 +17,7 @@ package compute
 const (
 	VPC_STATUS_PENDING       = "pending"
 	VPC_STATUS_AVAILABLE     = "available"
+	VPC_STATUS_UNAVAILABLE   = "unavailable"
 	VPC_STATUS_FAILED        = "failed"
 	VPC_STATUS_START_DELETE  = "start_delete"
 	VPC_STATUS_DELETING      = "deleting"

pkg/compute/models/cloudproviders.go

@@ -938,6 +938,38 @@ func (manager *SCloudproviderManager) ListItemFilter(ctx context.Context, q *sql
 		}
 		q = q.Equals("cloudaccount_id", accountObj.GetId())
 	}
+
+	if jsonutils.QueryBoolean(query, "usable", false) {
+		providers := CloudproviderManager.Query().SubQuery()
+		networks := NetworkManager.Query().SubQuery()
+		wires := WireManager.Query().SubQuery()
+		vpcs := VpcManager.Query().SubQuery()
+
+		sq := providers.Query(sqlchemy.DISTINCT("id", providers.Field("id")))
+		sq = sq.Join(vpcs, sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id")))
+		sq = sq.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
+		sq = sq.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id")))
+		sq = sq.Filter(sqlchemy.Equals(networks.Field("status"), api.NETWORK_STATUS_AVAILABLE))
+		sq = sq.Filter(sqlchemy.IsTrue(providers.Field("enabled")))
+		sq = sq.Filter(sqlchemy.In(providers.Field("status"), api.CLOUD_PROVIDER_VALID_STATUS))
+		sq = sq.Filter(sqlchemy.In(providers.Field("health_status"), api.CLOUD_PROVIDER_VALID_HEALTH_STATUS))
+		sq = sq.Filter(sqlchemy.Equals(vpcs.Field("status"), api.VPC_STATUS_AVAILABLE))
+
+
+		sq2 := providers.Query(sqlchemy.DISTINCT("id", providers.Field("id")))
+		sq2 = sq2.Join(vpcs, sqlchemy.Equals(vpcs.Field("manager_id"), providers.Field("id")))
+		sq2 = sq2.Join(wires, sqlchemy.Equals(vpcs.Field("id"), wires.Field("vpc_id")))
+		sq2 = sq2.Join(networks, sqlchemy.Equals(wires.Field("id"), networks.Field("wire_id")))
+		sq2 = sq2.Filter(sqlchemy.Equals(networks.Field("status"), api.NETWORK_STATUS_AVAILABLE))
+		sq2 = sq2.Filter(sqlchemy.IsNullOrEmpty(vpcs.Field("manager_id")))
+		sq2 = sq2.Filter(sqlchemy.Equals(vpcs.Field("status"), api.VPC_STATUS_AVAILABLE))
+
+		q = q.Filter(sqlchemy.OR(
+			sqlchemy.In(q.Field("id"), sq.SubQuery()),
+			sqlchemy.In(q.Field("id"), sq2.SubQuery()),
+		))
+	}
+
 	q, err := manager.SEnabledStatusStandaloneResourceBaseManager.ListItemFilter(ctx, q, userCred, query)
 	if err != nil {
 		return nil, err

pkg/compute/models/guest_actions.go

@@ -1891,6 +1891,10 @@ func (self *SGuest) PerformChangeConfig(ctx context.Context, userCred mcclient.T
 			return nil, err
 		}
 
+		if self.GetDriver().GetProvider() == api.CLOUD_PROVIDER_UCLOUD && !strings.HasPrefix(self.InstanceType, sku.InstanceTypeFamily) {
+			return nil, httperrors.NewInputParameterError("Cannot change config with different instance family")
+		}
+
 		if sku.GetName() != self.InstanceType {
 			confs.Add(jsonutils.NewString(sku.GetName()), "instance_type")
 			confs.Add(jsonutils.NewInt(int64(sku.CpuCoreCount)), "vcpu_count")

pkg/compute/models/snapshots.go

@@ -475,6 +475,7 @@ func (self *SSnapshot) AllowDeleteItem(ctx context.Context, userCred mcclient.To
 func (self *SSnapshot) StartSnapshotDeleteTask(ctx context.Context, userCred mcclient.TokenCredential, reloadDisk bool, parentTaskId string) error {
 	params := jsonutils.NewDict()
 	params.Set("reload_disk", jsonutils.NewBool(reloadDisk))
+	self.SetStatus(userCred, api.SNAPSHOT_DELETING, "")
 	task, err := taskman.TaskManager.NewTask(ctx, "SnapshotDeleteTask", self, userCred, params, parentTaskId, "", nil)
 	if err != nil {
 		log.Errorln(err)

pkg/util/aws/instance.go

@@ -439,6 +439,17 @@ func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd s
 		cloudconfig.MergeUser(loginUser)
 	}
 
+	// compare sysSizeGB
+	image, err := self.host.zone.region.GetImage(imageId)
+	if err != nil {
+		return "", err
+	} else {
+		minSizeGB := image.GetMinOsDiskSizeGb()
+		if minSizeGB > sysSizeGB {
+			sysSizeGB = minSizeGB
+		}
+	}
+
 	diskId, err := self.host.zone.region.ReplaceSystemDisk(ctx, self.InstanceId, imageId, sysSizeGB, cloudconfig.UserDataBase64())
 	if err != nil {
 		return "", err

pkg/util/aws/region.go

@@ -170,6 +170,7 @@ func (self *SRegion) fetchIVpcs() error {
 			RegionId:  self.RegionId,
 			Status:    *vpc.State,
 			VpcId:     *vpc.VpcId,
+			InstanceTenancy: *vpc.InstanceTenancy,
 		})
 	}
 

pkg/util/aws/vpc.go

@@ -24,6 +24,7 @@ import (
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/secrules"
 
+	api "yunion.io/x/onecloud/pkg/apis/compute"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 )
@@ -40,13 +41,14 @@ type SVpc struct {
 	iwires    []cloudprovider.ICloudWire
 	secgroups []cloudprovider.ICloudSecurityGroup
 
-	RegionId  string
-	VpcId     string
-	VpcName   string
-	CidrBlock string
-	IsDefault bool
-	Status    string
-	Tags      map[string]string // 名称、描述等
+	RegionId        string
+	VpcId           string
+	VpcName         string
+	CidrBlock       string
+	IsDefault       bool
+	Status          string
+	InstanceTenancy string
+	Tags            map[string]string // 名称、描述等
 }
 
 func (self *SVpc) addWire(wire *SWire) {
@@ -72,6 +74,10 @@ func (self *SVpc) GetGlobalId() string {
 }
 
 func (self *SVpc) GetStatus() string {
+	// 目前不支持专用主机
+	if self.InstanceTenancy == "dedicated" {
+		return api.VPC_STATUS_UNAVAILABLE
+	}
 	return strings.ToLower(self.Status)
 }
 
@@ -321,12 +327,13 @@ func (self *SRegion) GetVpcs(vpcId []string, offset int, limit int) ([]SVpc, int
 		vpcs = append(vpcs, SVpc{
 			region: self,
 			// secgroups: nil,
-			RegionId:  self.RegionId,
-			VpcId:     *item.VpcId,
-			VpcName:   *item.VpcId,
-			CidrBlock: *item.CidrBlock,
-			IsDefault: *item.IsDefault,
-			Status:    *item.State,
+			RegionId:        self.RegionId,
+			VpcId:           *item.VpcId,
+			VpcName:         *item.VpcId,
+			CidrBlock:       *item.CidrBlock,
+			IsDefault:       *item.IsDefault,
+			Status:          *item.State,
+			InstanceTenancy: *item.InstanceTenancy,
 			// Tags:      *item.Tags,
 		})
 	}

pkg/util/huawei/client/client.go

@@ -50,6 +50,7 @@ type Client struct {
 	SecurityGroups     *modules.SSecurityGroupManager
 	NovaSecurityGroups *modules.SSecurityGroupManager
 	Servers            *modules.SServerManager
+	ServersV2          *modules.SServerManager
 	NovaServers        *modules.SServerManager
 	Snapshots          *modules.SSnapshotManager
 	OsSnapshots        *modules.SSnapshotManager
@@ -97,6 +98,7 @@ func (self *Client) InitWithAccessKey(regionId, projectId, accessKey, secretKey
 func (self *Client) initManagers() {
 	if !self.init {
 		self.Servers = modules.NewServerManager(self.regionId, self.projectId, self.signer, self.debug)
+		self.ServersV2 = modules.NewServerV2Manager(self.regionId, self.projectId, self.signer, self.debug)
 		self.NovaServers = modules.NewNovaServerManager(self.regionId, self.projectId, self.signer, self.debug)
 		self.Snapshots = modules.NewSnapshotManager(self.regionId, self.projectId, self.signer, self.debug)
 		self.OsSnapshots = modules.NewOsSnapshotManager(self.regionId, self.projectId, self.signer, self.debug)

pkg/util/huawei/client/modules/mod_servers.go

@@ -107,3 +107,18 @@ func NewNovaServerManager(regionId, projectId string, signer auth.Signer, debug
 		ResourceKeyword: "servers",
 	}}
 }
+
+// 重装弹性云服务器操作系统（安装Cloud-init）,请用这个manager
+func NewServerV2Manager(regionId, projectId string, signer auth.Signer, debug bool) *SServerManager {
+	return &SServerManager{SResourceManager: SResourceManager{
+		SBaseManager:  NewBaseManager(signer, debug),
+		ServiceName:   ServiceNameECS,
+		Region:        regionId,
+		ProjectId:     projectId,
+		version:       "v2",
+		Keyword:       "server",
+		KeywordPlural: "servers",
+
+		ResourceKeyword: "cloudservers",
+	}}
+}

pkg/util/huawei/instance.go

@@ -22,6 +22,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
+
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/osprofile"
@@ -29,6 +31,7 @@ import (
 
 	billing_api "yunion.io/x/onecloud/pkg/apis/billing"
 	api "yunion.io/x/onecloud/pkg/apis/compute"
+	"yunion.io/x/onecloud/pkg/util/cloudinit"
 	"yunion.io/x/onecloud/pkg/cloudprovider"
 	"yunion.io/x/onecloud/pkg/multicloud"
 	"yunion.io/x/onecloud/pkg/util/billing"
@@ -524,13 +527,22 @@ func (self *SInstance) UpdateUserData(userData string) error {
 func (self *SInstance) RebuildRoot(ctx context.Context, imageId string, passwd string, publicKey string, sysSizeGB int) (string, error) {
 	var err error
 	var jobId string
+
+	publicKeyName := ""
+	if len(publicKey) > 0 {
+		publicKeyName, err = self.host.zone.region.syncKeypair(publicKey)
+		if err != nil {
+			return "", err
+		}
+	}
+
 	if self.Metadata.MeteringImageID == imageId {
-		jobId, err = self.host.zone.region.RebuildRoot(ctx, self.GetId(), passwd, publicKey)
+		jobId, err = self.host.zone.region.RebuildRoot(ctx, self.UserID, self.GetId(), passwd, publicKeyName, self.OSEXTSRVATTRUserData)
 		if err != nil {
 			return "", err
 		}
 	} else {
-		jobId, err = self.host.zone.region.ChangeRoot(ctx, self.GetId(), imageId, passwd, publicKey)
+		jobId, err = self.host.zone.region.ChangeRoot(ctx, self.UserID, self.GetId(), imageId, passwd, publicKeyName, self.OSEXTSRVATTRUserData)
 		if err != nil {
 			return "", err
 		}
@@ -738,7 +750,6 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st
 	params.ImageRef = imageId
 	params.KeyName = keypair
 	params.AdminPass = passwd
-	params.UserData = userData
 	params.Description = desc
 	params.Count = 1
 	params.Nics = []NIC{{SubnetID: SubnetId, IpAddress: ipAddr}}
@@ -775,6 +786,14 @@ func (self *SRegion) CreateInstance(name string, imageId string, instanceType st
 		params.Extendparam.ChargingMode = POST_PAID
 	}
 
+	// https://support.huaweicloud.com/api-ecs/zh-cn_topic_0020212668.html#ZH-CN_TOPIC_0020212668__table761103195216
+	udata, err := updateUserData(userData, "root", params.AdminPass)
+	if err != nil {
+		return "", errors.Wrap(err, "region.CreateInstance.UpdateUserData")
+	}
+
+	params.UserData = udata
+
 	serverObj := jsonutils.Marshal(params)
 	createParams := jsonutils.NewDict()
 	createParams.Add(serverObj, "server")
@@ -991,11 +1010,21 @@ func (self *SRegion) UpdateVM(instanceId, name string) error {
 
 // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876349.html
 // 返回job id
-func (self *SRegion) RebuildRoot(ctx context.Context, instanceId, passwd, publicKeyName string) (string, error) {
+func (self *SRegion) RebuildRoot(ctx context.Context, userId, instanceId, passwd, publicKeyName, userData string) (string, error) {
 	params := jsonutils.NewDict()
 	reinstallObj := jsonutils.NewDict()
-	// meta := jsonutils.NewDict()
-	// meta.Add(jsonutils.NewString(""), "user_data")
+
+	if len(userData) > 0 {
+		udata, err := updateUserData(userData, "root", passwd)
+		if err != nil {
+			return "", errors.Wrap(err, "region.RebuildRoot.UpdateUserData")
+		}
+
+		meta := jsonutils.NewDict()
+		meta.Add(jsonutils.NewString(udata), "user_data")
+		reinstallObj.Add(meta, "metadata")
+	}
+
 	if len(passwd) > 0 {
 		reinstallObj.Add(jsonutils.NewString(passwd), "adminpass")
 	} else if len(publicKeyName) > 0 {
@@ -1004,8 +1033,12 @@ func (self *SRegion) RebuildRoot(ctx context.Context, instanceId, passwd, public
 		return "", fmt.Errorf("both password and publicKey are empty.")
 	}
 
+	if len(userId) > 0 {
+		reinstallObj.Add(jsonutils.NewString(userId), "userid")
+	}
+
 	params.Add(reinstallObj, "os-reinstall")
-	ret, err := self.ecsClient.Servers.PerformAction2("reinstallos", instanceId, params, "")
+	ret, err := self.ecsClient.ServersV2.PerformAction2("reinstallos", instanceId, params, "")
 	if err != nil {
 		return "", err
 	}
@@ -1015,11 +1048,20 @@ func (self *SRegion) RebuildRoot(ctx context.Context, instanceId, passwd, public
 
 // https://support.huaweicloud.com/api-ecs/zh-cn_topic_0067876971.html
 // 返回job id
-func (self *SRegion) ChangeRoot(ctx context.Context, instanceId, imageId, passwd, publicKeyName string) (string, error) {
+func (self *SRegion) ChangeRoot(ctx context.Context, userId, instanceId, imageId, passwd, publicKeyName, userData string) (string, error) {
 	params := jsonutils.NewDict()
 	changeOsObj := jsonutils.NewDict()
-	// meta := jsonutils.NewDict()
-	// meta.Add(jsonutils.NewString(""), "user_data")
+	if len(userData) > 0 {
+		udata, err := updateUserData(userData, "root", passwd)
+		if err != nil {
+			return "", errors.Wrap(err, "region.ChangeRoot.UpdateUserData")
+		}
+
+		meta := jsonutils.NewDict()
+		meta.Add(jsonutils.NewString(udata), "user_data")
+		changeOsObj.Add(meta, "metadata")
+	}
+
 	if len(passwd) > 0 {
 		changeOsObj.Add(jsonutils.NewString(passwd), "adminpass")
 	} else if len(publicKeyName) > 0 {
@@ -1028,10 +1070,14 @@ func (self *SRegion) ChangeRoot(ctx context.Context, instanceId, imageId, passwd
 		return "", fmt.Errorf("both password and publicKey are empty.")
 	}
 
+	if len(userId) > 0 {
+		changeOsObj.Add(jsonutils.NewString(userId), "userid")
+	}
+
 	changeOsObj.Add(jsonutils.NewString(imageId), "imageid")
 	params.Add(changeOsObj, "os-change")
 
-	ret, err := self.ecsClient.Servers.PerformAction2("changeos", instanceId, params, "")
+	ret, err := self.ecsClient.ServersV2.PerformAction2("changeos", instanceId, params, "")
 	if err != nil {
 		return "", err
 	}
@@ -1235,3 +1281,18 @@ func (self *SInstance) GetProjectId() string {
 func (self *SInstance) GetError() error {
 	return nil
 }
+
+func updateUserData(userData, username, password string) (string, error) {
+	config, err := cloudinit.ParseUserDataBase64(userData)
+	if err != nil {
+		return "", fmt.Errorf("invalid userdata %s", userData)
+	}
+
+	if len(password) > 0 {
+		user := cloudinit.NewUser(username)
+		user.Password(password)
+		config.MergeUser(user)
+	}
+
+	return config.UserDataBase64(), nil
+}

pkg/util/huawei/shell/instance.go

@@ -116,14 +116,16 @@ func init() {
 
 	type InstanceRebuildRootOptions struct {
 		ID        string `help:"instance ID"`
+		UserId    string `help:"instance user ID"`
 		Image     string `help:"Image ID"`
 		Password  string `help:"admin password"`
 		PublicKey string `help:"public key name"`
+		UserData  string `help:"cloud-init user data"`
 	}
 
 	shellutils.R(&InstanceRebuildRootOptions{}, "instance-rebuild-root", "Reinstall virtual server system image", func(cli *huawei.SRegion, args *InstanceRebuildRootOptions) error {
 		ctx := context.Background()
-		jobId, err := cli.ChangeRoot(ctx, args.ID, args.Image, args.Password, args.PublicKey)
+		jobId, err := cli.ChangeRoot(ctx, args.UserId,args.ID, args.Image, args.Password, args.PublicKey, args.UserData)
 		if err != nil {
 			return err
 		}

pkg/util/ucloud/disk.go

@@ -242,7 +242,7 @@ func (self *SDisk) CreateISnapshot(ctx context.Context, name string, desc string
 }
 
 func (self *SDisk) getSnapshot(snapshotId string) (*SSnapshot, error) {
-	snapshot, err := self.storage.zone.region.GetSnapshotById(snapshotId)
+	snapshot, err := self.storage.zone.region.GetSnapshotById(self.Zone, snapshotId)
 	return &snapshot, err
 }
 

pkg/util/ucloud/region.go

@@ -269,23 +269,26 @@ func (self *SRegion) GetISnapshots() ([]cloudprovider.ICloudSnapshot, error) {
 }
 
 func (self *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
+	if len(snapshotId) == 0 {
+		return nil, cloudprovider.ErrNotFound
+	}
+
 	params := NewUcloudParams()
-	params.Set("SnapshotId", snapshotId)
 	snapshots := make([]SSnapshot, 0)
 	err := self.DoListAll("DescribeUDiskSnapshot", params, &snapshots)
 	if err != nil {
 		return nil, err
 	}
 
-	if len(snapshots) == 1 {
-		snapshot := snapshots[0]
-		snapshot.region = self
-		return &snapshot, nil
-	} else if len(snapshots) == 0 {
-		return nil, cloudprovider.ErrNotFound
-	} else {
-		return nil, fmt.Errorf("GetISnapshotById %s %d snapshot found", snapshotId, len(snapshots))
+	for i := range snapshots {
+		if snapshots[i].SnapshotID == snapshotId {
+			snapshot := snapshots[i]
+			snapshot.region = self
+			return &snapshot, nil
+		}
 	}
+
+	return nil, cloudprovider.ErrNotFound
 }
 
 func (self *SRegion) GetIHosts() ([]cloudprovider.ICloudHost, error) {

pkg/util/ucloud/snapshot.go

@@ -77,7 +77,12 @@ func (self *SSnapshot) GetStatus() string {
 }
 
 func (self *SSnapshot) Refresh() error {
-	snapshot, err := self.region.GetSnapshotById(self.GetId())
+	disk, err := self.region.GetDisk(self.UDiskID)
+	if err != nil {
+		return err
+	}
+
+	snapshot, err := self.region.GetSnapshotById(disk.Zone, self.GetId())
 	if err != nil {
 		return err
 	}
@@ -124,7 +129,7 @@ func (self *SSnapshot) Delete() error {
 	return self.region.DeleteSnapshot(self.GetId(), idisk.Zone)
 }
 
-func (self *SRegion) GetSnapshotById(snapshotId string) (SSnapshot, error) {
+func (self *SRegion) GetSnapshotById(zoneId string, snapshotId string) (SSnapshot, error) {
 	snapshots, err := self.GetSnapshots("", snapshotId)
 	if err != nil {
 		return SSnapshot{}, err
@@ -142,10 +147,20 @@ func (self *SRegion) GetSnapshotById(snapshotId string) (SSnapshot, error) {
 func (self *SRegion) GetSnapshots(diskId string, snapshotId string) ([]SSnapshot, error) {
 	params := NewUcloudParams()
 	if len(diskId) > 0 {
+		disk, err := self.GetDisk(diskId)
+		if err != nil {
+			return nil, err
+		}
+
 		params.Set("UDiskId", diskId)
+		params.Set("Zone", disk.Zone)
 	}
 
 	if len(snapshotId) > 0 {
+		if len(diskId) == 0 {
+			return nil, fmt.Errorf("GetSnapshots required parameter diskId.")
+		}
+
 		params.Set("SnapshotId", snapshotId)
 	}