rebase

pkg/cloudcommon/policy/policy.go

@@ -203,20 +203,10 @@ func (manager *SPolicyManager) Allow(isAdmin bool, userCred mcclient.TokenCreden
 	return currentPriv
 }
 
-func (manager *SPolicyManager) explainPolicy(userCred mcclient.TokenCredential, policyReq jsonutils.JSONObject) (rbacutils.TRbacResult, error) {
+func (manager *SPolicyManager) explainPolicy(userCred mcclient.TokenCredential, policyReq jsonutils.JSONObject) ([]string, rbacutils.TRbacResult, error) {
 	policySeq, err := policyReq.GetArray()
 	if err != nil {
-		return rbacutils.Deny, httperrors.NewInputParameterError("invalid format")
-	}
-	isAdmin, _ := policySeq[0].Bool()
-	if !consts.IsRbacEnabled() {
-		if !isAdmin {
-			return rbacutils.OwnerAllow, nil
-		} else if isAdmin && userCred.IsSystemAdmin() {
-			return rbacutils.AdminAllow, nil
-		} else {
-			return rbacutils.Deny, httperrors.NewForbiddenError("operation not allowed")
-		}
+		return nil, rbacutils.Deny, httperrors.NewInputParameterError("invalid format")
 	}
 	service := rbacutils.WILD_MATCH
 	resource := rbacutils.WILD_MATCH
@@ -233,11 +223,27 @@ func (manager *SPolicyManager) explainPolicy(userCred mcclient.TokenCredential,
 	}
 	if len(policySeq) > 4 {
 		for i := 4; i < len(policySeq); i += 1 {
-			extra[i-4], _ = policySeq[i].GetString()
+			ev, _ := policySeq[i].GetString()
+			extra = append(extra, ev)
 		}
 	}
 
-	return manager.Allow(isAdmin, userCred, service, resource, action, extra...), nil
+	reqStrs := []string{service, resource, action}
+	if len(extra) > 0 {
+		reqStrs = append(reqStrs, extra...)
+	}
+
+	isAdmin, _ := policySeq[0].Bool()
+	if !consts.IsRbacEnabled() {
+		if !isAdmin {
+			return reqStrs, rbacutils.OwnerAllow, nil
+		} else if isAdmin && userCred.IsSystemAdmin() {
+			return reqStrs, rbacutils.AdminAllow, nil
+		} else {
+			return reqStrs, rbacutils.Deny, httperrors.NewForbiddenError("operation not allowed")
+		}
+	}
+	return reqStrs, manager.Allow(isAdmin, userCred, service, resource, action, extra...), nil
 }
 
 func (manager *SPolicyManager) ExplainRpc(userCred mcclient.TokenCredential, params jsonutils.JSONObject) (jsonutils.JSONObject, error) {
@@ -247,11 +253,12 @@ func (manager *SPolicyManager) ExplainRpc(userCred mcclient.TokenCredential, par
 	}
 	ret := jsonutils.NewDict()
 	for key, policyReq := range paramDict {
-		result, err := manager.explainPolicy(userCred, policyReq)
+		reqStrs, result, err := manager.explainPolicy(userCred, policyReq)
 		if err != nil {
 			return nil, err
 		}
-		ret.Add(jsonutils.NewString(string(result)), key)
+		reqStrs = append(reqStrs, string(result))
+		ret.Add(jsonutils.NewStringArray(reqStrs), key)
 	}
 	return ret, nil
 }

pkg/cloudcommon/validators/errors.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"yunion.io/x/onecloud/pkg/httperrors"
+	"yunion.io/x/onecloud/pkg/util/httputils"
 )
 
 var returnHttpError = true
@@ -54,65 +55,72 @@ func (ve *ValidateError) Error() string {
 
 // TODO let each validator provide the error
 func newMissingKeyError(key string) error {
-	msg := fmt.Sprintf("missing %q", key)
-	return newError(ERR_MISSING_KEY, msg)
+	return newError(ERR_MISSING_KEY, "missing %q", key)
 }
 
 func newGeneralError(key string, err error) error {
-	msg := fmt.Sprintf("general error for %q: %s", key, err)
-	return newError(ERR_GENERAL, msg)
+	return newError(ERR_GENERAL, "general error for %q: %s", key, err)
 }
 
 func newInvalidTypeError(key string, typ string, err error) error {
-	msg := fmt.Sprintf("expecting %s type for %q: %s", typ, key, err)
-	return newError(ERR_INVALID_TYPE, msg)
+	return newError(ERR_INVALID_TYPE, "expecting %s type for %q: %s", typ, key, err)
 }
 
 func newInvalidChoiceError(key string, choices Choices, choice string) error {
-	msg := fmt.Sprintf("invalid %q, want %s, got %s", key, choices, choice)
-	return newError(ERR_INVALID_CHOICE, msg)
+	return newError(ERR_INVALID_CHOICE, "invalid %q, want %s, got %s", key, choices, choice)
 }
 
 func newNotInRangeError(key string, value, lower, upper int64) error {
-	msg := fmt.Sprintf("invalid %q: %d, want [%d,%d]", key, value, lower, upper)
-	return newError(ERR_NOT_IN_RANGE, msg)
+	return newError(ERR_NOT_IN_RANGE, "invalid %q: %d, want [%d,%d]", key, value, lower, upper)
 }
 
 func newInvalidValueError(key string, value string) error {
-	msg := fmt.Sprintf("invalid %q: %s", key, value)
-	return newError(ERR_INVALID_VALUE, msg)
+	return newError(ERR_INVALID_VALUE, "invalid %q: %s", key, value)
+}
+
+func newInvalidStructError(key string, err error) error {
+	errFmt := "invalid %q: "
+	params := []interface{}{key}
+	jsonClientErr, ok := err.(*httputils.JSONClientError)
+	if ok {
+		errFmt += jsonClientErr.Data.Id
+		for _, f := range jsonClientErr.Data.Fields {
+			params = append(params, f)
+		}
+	}
+	return newError(ERR_INVALID_VALUE, errFmt, params...)
 }
 
 func newModelManagerError(modelKeyword string) error {
-	msg := fmt.Sprintf("internal error: getting model manager for %q failed",
-		modelKeyword)
-	return newError(ERR_MODEL_MANAGER, msg)
+	return newError(ERR_MODEL_MANAGER, "failed getting model manager for %q", modelKeyword)
 }
 
 func newModelNotFoundError(modelKeyword, idOrName string, err error) error {
-	msg := fmt.Sprintf("cannot find %q with id/name %q",
-		modelKeyword, idOrName)
+	errFmt := "cannot find %q with id/name %q"
+	params := []interface{}{modelKeyword, idOrName}
 	if err != sql.ErrNoRows {
-		msg += ": " + err.Error()
+		errFmt += ": %s"
+		params = append(params, err.Error())
 	}
-	return newError(ERR_MODEL_NOT_FOUND, msg)
+	return newError(ERR_MODEL_NOT_FOUND, errFmt, params...)
 }
 
-func newError(typ ErrType, msg string) error {
-	err := &ValidateError{
-		ErrType: typ,
-		Msg:     msg,
-	}
+func newError(typ ErrType, errFmt string, params ...interface{}) error {
+	errFmt = fmt.Sprintf("%s: %s", typ, errFmt)
 	if returnHttpError {
 		switch typ {
 		case ERR_SUCCESS:
 			return nil
 		case ERR_GENERAL, ERR_MODEL_MANAGER:
-			return httperrors.NewInternalServerError(msg)
+			return httperrors.NewInternalServerError(errFmt, params...)
 		default:
-			return httperrors.NewInputParameterError(msg)
+			return httperrors.NewInputParameterError(errFmt, params...)
 		}
 	}
+	err := &ValidateError{
+		ErrType: typ,
+		Msg:     fmt.Sprintf(errFmt, params...),
+	}
 	return err
 }
 

pkg/cloudcommon/validators/validators.go

@@ -511,7 +511,7 @@ func (v *ValidatorStruct) Validate(data *jsonutils.JSONDict) error {
 	if valueValidator, ok := v.Value.(IValidatorBase); ok {
 		err = valueValidator.Validate(data)
 		if err != nil {
-			return newInvalidValueError(v.Key, err.Error())
+			return newInvalidStructError(v.Key, err)
 		}
 	}
 	data.Set(v.Key, jsonutils.Marshal(v.Value))

pkg/compute/models/cloudproviders.go

@@ -130,7 +130,7 @@ func (self *SCloudprovider) ValidateUpdateData(ctx context.Context, userCred mcc
 }
 
 func (self *SCloudproviderManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerProjId string, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
-	return nil, httperrors.NewUnsupportOperationError("Not support create cloudprovider, please considir create cloudaccount")
+	return nil, httperrors.NewUnsupportOperationError("Directly creating cloudprovider is not supported, create cloudaccount instead")
 }
 
 func (self *SCloudprovider) getPassword() (string, error) {

pkg/compute/models/loadbalanceracls.go

@@ -14,6 +14,7 @@ import (
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
 	"yunion.io/x/onecloud/pkg/cloudcommon/validators"
+	"yunion.io/x/onecloud/pkg/httperrors"
 	"yunion.io/x/onecloud/pkg/mcclient"
 )
 
@@ -33,16 +34,16 @@ func (aclEntry *SLoadbalancerAclEntry) Validate(data *jsonutils.JSONDict) error
 	} else {
 		ip := net.ParseIP(aclEntry.Cidr).To4()
 		if ip == nil {
-			return fmt.Errorf("invalid addr %s", aclEntry.Cidr)
+			return httperrors.NewInputParameterError("invalid addr %s", aclEntry.Cidr)
 		}
 	}
 	if commentLimit := 128; len(aclEntry.Comment) > commentLimit {
-		return fmt.Errorf("comment too long (%d>=%d)",
+		return httperrors.NewInputParameterError("comment too long (%d>=%d)",
 			len(aclEntry.Comment), commentLimit)
 	}
 	for _, r := range aclEntry.Comment {
 		if !unicode.IsPrint(r) {
-			return fmt.Errorf("comment contains non-printable char: %v", r)
+			return httperrors.NewInputParameterError("comment contains non-printable char: %v", r)
 		}
 	}
 	return nil
@@ -68,7 +69,7 @@ func (aclEntries *SLoadbalancerAclEntries) Validate(data *jsonutils.JSONDict) er
 		}
 		if _, ok := found[aclEntry.Cidr]; ok {
 			// error so that the user has a chance to deal with comments
-			return fmt.Errorf("acl cidr duplicate %s", aclEntry.Cidr)
+			return httperrors.NewInputParameterError("acl cidr duplicate %s", aclEntry.Cidr)
 		}
 		found[aclEntry.Cidr] = true
 	}

pkg/compute/models/loadbalanceragents.go

@@ -154,7 +154,7 @@ func (p *SLoadbalancerAgentParamsTelegraf) Validate(data *jsonutils.JSONDict) er
 	if p.InfluxDbOutputUrl != "" {
 		_, err := url.Parse(p.InfluxDbOutputUrl)
 		if err != nil {
-			return err
+			return httperrors.NewInputParameterError("telegraf params: invalid influxdb url: %s", err)
 		}
 	}
 	if p.HaproxyInputInterval <= 0 {

pkg/compute/models/snapshots.go

@@ -9,7 +9,6 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/pkg/util/compare"
-	"yunion.io/x/pkg/utils"
 	"yunion.io/x/sqlchemy"
 
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -348,29 +347,20 @@ func (self *SSnapshot) CustomizeDelete(ctx context.Context, userCred mcclient.To
 	if self.Status == SNAPSHOT_DELETING {
 		return fmt.Errorf("Cannot delete snapshot in status %s", self.Status)
 	}
-	if self.Status == SNAPSHOT_UNKNOWN {
-		return self.RealDelete(ctx, userCred)
-	}
 	if len(self.ExternalId) == 0 {
-		if utils.IsInStringArray(self.Status, []string{SNAPSHOT_FAILED}) {
-			return self.RealDelete(ctx, userCred)
-		}
 		if self.CreatedBy == MANUAL {
 			if !self.FakeDeleted {
 				return self.FakeDelete()
-			} else {
-				_, err := SnapshotManager.GetConvertSnapshot(self)
-				if err != nil {
-					return fmt.Errorf("Cannot delete snapshot: %s, disk need at least one of snapshot as backing file", err.Error())
-				}
-				return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
 			}
-		} else {
-			return fmt.Errorf("Cannot delete snapshot created by %s", self.CreatedBy)
+			_, err := SnapshotManager.GetConvertSnapshot(self)
+			if err != nil {
+				return fmt.Errorf("Cannot delete snapshot: %s, disk need at least one of snapshot as backing file", err.Error())
+			}
+			return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
 		}
-	} else {
-		return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
+		return fmt.Errorf("Cannot delete snapshot created by %s", self.CreatedBy)
 	}
+	return self.StartSnapshotDeleteTask(ctx, userCred, false, "")
 }
 
 func (self *SSnapshot) AllowPerformDeleted(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool {

pkg/compute/tasks/guest_create_task.go

@@ -52,6 +52,7 @@ func (self *GuestCreateTask) OnDiskPreparedFailed(ctx context.Context, obj db.IS
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
 	logclient.AddActionLog(guest, logclient.ACT_ALLOCATE, data, self.UserCred, false)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DISK_FAILED, data.String())
+	self.SetStageFailed(ctx, data.String())
 }
 
 func (self *GuestCreateTask) OnDiskPrepared(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
@@ -80,6 +81,7 @@ func (self *GuestCreateTask) OnCdromPreparedFailed(ctx context.Context, obj db.I
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
 	logclient.AddActionLog(guest, logclient.ACT_ALLOCATE, data, self.UserCred, false)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DISK_FAILED, fmt.Sprintf("cdrom_failed %s", data))
+	self.SetStageFailed(ctx, fmt.Sprintf("cdrom_failed %s", data))
 }
 
 func (self *GuestCreateTask) StartDeployGuest(ctx context.Context, guest *models.SGuest) {
@@ -107,6 +109,7 @@ func (self *GuestCreateTask) OnDeployGuestDescCompleteFailed(ctx context.Context
 	db.OpsLog.LogEvent(guest, db.ACT_ALLOCATE_FAIL, data, self.UserCred)
 	logclient.AddActionLog(guest, logclient.ACT_ALLOCATE, data, self.UserCred, false)
 	notifyclient.NotifySystemError(guest.Id, guest.Name, models.VM_DEPLOY_FAILED, data.String())
+	self.SetStageFailed(ctx, data.String())
 }
 
 func (self *GuestCreateTask) OnAutoStartGuest(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {

pkg/compute/tasks/guest_disk_snapshot_task.go

@@ -162,12 +162,16 @@ func (self *SnapshotDeleteTask) deleteExternalSnapshot(ctx context.Context, snap
 	}
 	cloudSnapshot, err := cloudRegion.GetISnapshotById(snapshot.ExternalId)
 	if err != nil {
+		if err == cloudprovider.ErrNotFound {
+			return nil
+		}
 		log.Errorln(err, cloudSnapshot)
 		return err
 	}
-	cloudSnapshot.Delete()
-	err = cloudprovider.WaitDeleted(cloudSnapshot, 10*time.Second, 300*time.Second)
-	return err
+	if err := cloudSnapshot.Delete(); err != nil {
+		return err
+	}
+	return cloudprovider.WaitDeleted(cloudSnapshot, 10*time.Second, 300*time.Second)
 }
 
 func (self *SnapshotDeleteTask) StartReloadDisk(ctx context.Context, snapshot *models.SSnapshot, guest *models.SGuest) {

pkg/mcclient/modules/managers.go

@@ -25,6 +25,15 @@ func NewMonitorManager(keyword, keywordPlural string, columns, adminColumns []st
 		Keyword: keyword, KeywordPlural: keywordPlural}
 }
 
+func NewCloudmonManager(keyword, keywordPlural string, columns, adminColumns []string) ResourceManager {
+	return ResourceManager{
+		BaseManager: BaseManager{columns: columns,
+			adminColumns: adminColumns,
+			version:      "v1",
+			serviceType:  "cloudmon"},
+		Keyword: keyword, KeywordPlural: keywordPlural}
+}
+
 func NewNotifyManager(keyword, keywordPlural string, columns, adminColumns []string) ResourceManager {
 	return ResourceManager{
 		BaseManager: BaseManager{columns: columns,

pkg/mcclient/modules/mod_underutilizedinstances.go

@@ -0,0 +1,13 @@
+package modules
+
+var (
+	UnderutilizedInstances ResourceManager
+)
+
+func init() {
+	UnderutilizedInstances = NewCloudmonManager("underutilizedinstance", "underutilizedinstances",
+		[]string{"id", "vm_id", "vm_name", "datetime_str", "vm_cpu", "vm_disk", "vm_memory", "vm_provider", "cpu_usage_threshold", "netio_rx_bps_threshold", "netio_tx_bps_threshold", "stastics_details"},
+		[]string{})
+
+	register(&UnderutilizedInstances)
+}

pkg/mcclient/options/k8s/repo.go

@@ -16,13 +16,17 @@ type RepoGetOptions struct {
 
 type RepoCreateOptions struct {
 	RepoGetOptions
-	URL string `help:"Repository url"`
+	URL    string `help:"Repository url"`
+	Public bool   `help:"Make repostitory public"`
 }
 
 func (o RepoCreateOptions) Params() *jsonutils.JSONDict {
 	params := jsonutils.NewDict()
 	params.Add(jsonutils.NewString(o.NAME), "name")
 	params.Add(jsonutils.NewString(o.URL), "url")
+	if o.Public {
+		params.Add(jsonutils.JSONTrue, "is_public")
+	}
 	return params
 }
 

pkg/util/aliyun/snapshot.go

@@ -160,13 +160,16 @@ func (self *SRegion) GetSnapshots(instanceId string, diskId string, snapshotName
 }
 
 func (self *SRegion) GetISnapshotById(snapshotId string) (cloudprovider.ICloudSnapshot, error) {
-	if snapshots, total, err := self.GetSnapshots("", "", "", []string{snapshotId}, 0, 1); err != nil {
+	snapshots, total, err := self.GetSnapshots("", "", "", []string{snapshotId}, 0, 1)
+	if err != nil {
 		return nil, err
-	} else if total != 1 {
-		return nil, cloudprovider.ErrNotFound
-	} else {
-		return &snapshots[0], nil
 	}
+	if total == 0 {
+		return nil, cloudprovider.ErrNotFound
+	} else if total > 1 {
+		return nil, cloudprovider.ErrDuplicateId
+	}
+	return &snapshots[0], nil
 }
 
 func (self *SRegion) DeleteSnapshot(snapshotId string) error {

pkg/util/azure/classic_disk.go

@@ -101,7 +101,9 @@ func (self *SRegion) GetClassicDisks() ([]SClassicDisk, error) {
 }
 
 func (self *SClassicDisk) GetMetadata() *jsonutils.JSONDict {
-	return nil
+	data := jsonutils.NewDict()
+	data.Add(jsonutils.NewString(models.HYPERVISOR_AZURE), "hypervisor")
+	return data
 }
 
 func (self *SClassicDisk) CreateISnapshot(name, desc string) (cloudprovider.ICloudSnapshot, error) {

pkg/util/azure/classic_instance.go

@@ -113,7 +113,8 @@ type SClassicInstance struct {
 
 func (self *SClassicInstance) GetMetadata() *jsonutils.JSONDict {
 	data := jsonutils.NewDict()
-	data.Add(jsonutils.NewString(self.Properties.HardwareProfile.Size), "price_key")
+	priceKey := fmt.Sprintf("%s::%s", self.Properties.HardwareProfile.Size, self.host.zone.region.Name)
+	data.Add(jsonutils.NewString(priceKey), "price_key")
 	if self.Properties.NetworkProfile.NetworkSecurityGroup != nil {
 		data.Add(jsonutils.NewString(self.Properties.NetworkProfile.NetworkSecurityGroup.ID), "secgroupId")
 	}

pkg/util/azure/instance.go

@@ -229,7 +229,8 @@ func (self *SInstance) GetMetadata() *jsonutils.JSONDict {
 		data.Add(jsonutils.NewString(loginKey), "login_key")
 	}
 
-	data.Add(jsonutils.NewString(self.Properties.HardwareProfile.VMSize), "price_key")
+	priceKey := fmt.Sprintf("%s::%s", self.Properties.HardwareProfile.VMSize, self.host.zone.region.Name)
+	data.Add(jsonutils.NewString(priceKey), "price_key")
 	if nics, err := self.getNics(); err == nil {
 		for _, nic := range nics {
 			if nic.Properties.NetworkSecurityGroup != nil {