opensource/cloudpods
3
0
Fork 0
mirror of https://github.com/yunionio/cloudpods.git synced 2026-09-24 16:03:43 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #2956 from wanyaoqi/bugfix/wyq/20190912

Browse Source 
optimize batch create validate
This commit is contained in:
yunion-ci-robot
2019-09-19 21:19:28 +08:00
committed by GitHub
parent d955e5d8f3 c648b823b0
commit ed9bbd17c8
10 changed files with 146 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/appsrv/handlerinfo.go
+7 -6
View File
@@ -46,14 +46,15 @@ type SHandlerInfo struct {
}
func (this *SHandlerInfo) FetchProcessTimeout(r *http.Request) time.Duration {
var tm time.Duration
if this.processTimeoutCallback != nil {
tm = this.processTimeoutCallback(this, r)
tm := this.processTimeoutCallback(this, r)
if tm < this.processTimeout {
tm = this.processTimeout
}
return tm
} else {
return this.processTimeout
}
if tm < this.processTimeout {
tm = this.processTimeout
}
return tm
}
func (this *SHandlerInfo) SetProcessTimeoutCallback(callback TProcessTimeoutCallback) {
pkg/baremetal/tasks/bm_register.go
+1 -1
View File
@@ -88,7 +88,7 @@ func (s *sBaremetalRegisterTask) CreateBaremetal() error {
}
}
if s.accessNic == nil {
return fmt.Errorf("Register baremeatl failed: access nic not found ???")
s.accessNic = nicinfo[0]
}
params := jsonutils.NewDict()
pkg/cloudcommon/db/db_dispatcher.go
+42 -5
View File
@@ -965,7 +965,24 @@ func DoCreate(manager IModelManager, ctx context.Context, userCred mcclient.Toke
return doCreateItem(manager, ctx, userCred, ownerId, nil, data)
}
func doCreateItem(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) (IModel, error) {
func doCreateItem(
manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential,
ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) (IModel, error) {
return _doCreateItem(manager, ctx, userCred, ownerId, query, data, false, 1)
}
func batchCreateDoCreateItem(
manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
query jsonutils.JSONObject, data jsonutils.JSONObject, baseIndex int) (IModel, error) {
return _doCreateItem(manager, ctx, userCred, ownerId, query, data, true, baseIndex)
}
func _doCreateItem(
manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
query jsonutils.JSONObject, data jsonutils.JSONObject, batchCreate bool, baseIndex int) (IModel, error) {
dataDict, ok := data.(*jsonutils.JSONDict)
if !ok {
log.Errorf("doCreateItem: fail to decode json data %s", data)
@@ -976,7 +993,7 @@ func doCreateItem(manager IModelManager, ctx context.Context, userCred mcclient.
generateName, _ := dataDict.GetString("generate_name")
if len(generateName) > 0 {
dataDict.Remove("generate_name")
newName, err := GenerateName(manager, ownerId, generateName)
newName, err := GenerateName2(manager, ownerId, generateName, nil, baseIndex)
if err != nil {
return nil, err
}
@@ -991,7 +1008,12 @@ func doCreateItem(manager IModelManager, ctx context.Context, userCred mcclient.
}
}*/
dataDict, err = manager.ValidateCreateData(ctx, userCred, ownerId, query, dataDict)
if batchCreate {
dataDict, err = manager.BatchCreateValidateCreateData(ctx, userCred, ownerId, query, dataDict)
} else {
dataDict, err = manager.ValidateCreateData(ctx, userCred, ownerId, query, dataDict)
}
if err != nil {
return nil, httperrors.NewGeneralError(err)
}
@@ -1142,7 +1164,11 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
err error
}
var multiData []jsonutils.JSONObject
var (
multiData []jsonutils.JSONObject
onBatchCreateFail func()
validateError error
)
createResults, err := func() ([]sCreateResult, error) {
lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
@@ -1152,9 +1178,20 @@ func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query json
if err != nil {
return nil, err
}
ret := make([]sCreateResult, len(multiData))
for i, cdata := range multiData {
model, err := doCreateItem(manager, ctx, userCred, ownerId, query, cdata)
if i == 0 {
onBatchCreateFail, validateError = manager.BatchPreValidate(
ctx, userCred, ownerId, query, cdata.(*jsonutils.JSONDict), len(multiData))
if validateError != nil {
return nil, err
}
}
model, err := batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, cdata, i)
if err != nil && onBatchCreateFail != nil {
onBatchCreateFail()
}
ret[i] = sCreateResult{model: model, err: err}
}
return ret, nil
pkg/cloudcommon/db/interface.go
+3
View File
@@ -78,8 +78,11 @@ type IModelManager interface {
// create hooks
AllowCreateItem(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error)
OnCreateComplete(ctx context.Context, items []IModel, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject)
BatchPreValidate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
query jsonutils.JSONObject, data *jsonutils.JSONDict, count int) (func(), error)
// allow perform action
AllowPerformAction(ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) bool
pkg/cloudcommon/db/modelbase.go
+11
View File
@@ -353,6 +353,17 @@ func (manager *SModelBaseManager) GetPropertyDistinctField(ctx context.Context,
return res, nil
}
func (manager *SModelBaseManager) BatchPreValidate(
ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
query jsonutils.JSONObject, data *jsonutils.JSONDict, count int,
) (func(), error) {
return nil, nil
}
func (manager *SModelBaseManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
return nil, nil
}
func (model *SModelBase) GetId() string {
return ""
}
pkg/cloudcommon/db/namevalidator.go
+6 -7
View File
@@ -80,18 +80,17 @@ func alterNameValidator(model IModel, name string) error {
}
func GenerateName(manager IModelManager, ownerId mcclient.IIdentityProvider, hint string) (string, error) {
return GenerateName2(manager, ownerId, hint, nil)
return GenerateName2(manager, ownerId, hint, nil, 1)
}
func GenerateName2(manager IModelManager, ownerId mcclient.IIdentityProvider, hint string, model IModel) (string, error) {
func GenerateName2(manager IModelManager, ownerId mcclient.IIdentityProvider, hint string, model IModel, baseIndex int) (string, error) {
_, pattern, patternLen := stringutils.ParseNamePattern(hint)
var name string
idx := 1
if patternLen == 0 {
name = hint
} else {
name = fmt.Sprintf(pattern, idx)
idx += 1
name = fmt.Sprintf(pattern, baseIndex)
baseIndex += 1
}
for {
var uniq bool
@@ -107,7 +106,7 @@ func GenerateName2(manager IModelManager, ownerId mcclient.IIdentityProvider, hi
if uniq {
return name, nil
}
name = fmt.Sprintf(pattern, idx)
idx += 1
name = fmt.Sprintf(pattern, baseIndex)
baseIndex += 1
}
}
pkg/compute/models/guests.go
+66 -10
View File
@@ -817,7 +817,46 @@ func (self *SGuest) ValidateUpdateData(ctx context.Context, userCred mcclient.To
return self.SVirtualResourceBase.ValidateUpdateData(ctx, userCred, query, data)
}
func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
func (manager *SGuestManager) BatchPreValidate(
ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
query jsonutils.JSONObject, data *jsonutils.JSONDict, count int,
) (func(), error) {
input, err := manager.validateCreateData(ctx, userCred, ownerId, query, data)
if err != nil {
return nil, err
}
if !input.IsSystem {
reqQuota, err := manager.checkCreateQuota(ctx, userCred, ownerId, input, input.Backup, count)
if err != nil {
return nil, err
}
quota := &SQuota{
Cpu: reqQuota.Cpu / count,
Memory: reqQuota.Memory / count,
Storage: reqQuota.Storage / count,
Port: reqQuota.Port / count,
Eport: reqQuota.Eport / count,
Bw: reqQuota.Bw / count,
Ebw: reqQuota.Ebw / count,
IsolatedDevice: reqQuota.IsolatedDevice / count,
Eip: reqQuota.Eip / count,
}
quotaPlatform := make([]string, 0)
if len(input.Hypervisor) > 0 {
quotaPlatform = GetDriver(input.Hypervisor).GetQuotaPlatformID()
}
return func() {
QuotaManager.CancelPendingUsage(
ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, quota, quota)
}, nil
}
return nil, nil
}
func (manager *SGuestManager) validateCreateData(
ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
query jsonutils.JSONObject, data *jsonutils.JSONDict) (*api.ServerCreateInput, error) {
// TODO: 定义 api.ServerCreateInput 的 Unmarshal 函数，直接通过 data.Unmarshal(input) 解析参数
input, err := cmdline.FetchServerCreateInputByJSON(data)
if err != nil {
@@ -1122,17 +1161,31 @@ func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred m
return nil, httperrors.NewInputParameterError("Invalid userdata: %v", err)
}
input.Project = ownerId.GetProjectId()
input.Domain = ownerId.GetProjectDomainId()
return input, nil
}
func (manager *SGuestManager) BatchCreateValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
input, err := manager.validateCreateData(ctx, userCred, ownerId, query, data)
if err != nil {
return nil, err
}
return input.JSON(input), nil
}
func (manager *SGuestManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
input, err := manager.validateCreateData(ctx, userCred, ownerId, query, data)
if err != nil {
return nil, err
}
if !input.IsSystem {
err = manager.checkCreateQuota(ctx, userCred, ownerId, input,
input.Backup)
_, err = manager.checkCreateQuota(ctx, userCred, ownerId, input, input.Backup, 1)
if err != nil {
return nil, err
}
}
input.Project = ownerId.GetProjectId()
input.Domain = ownerId.GetProjectDomainId()
return input.JSON(input), nil
}
@@ -1180,17 +1233,20 @@ func (self *SGuest) PostUpdate(ctx context.Context, userCred mcclient.TokenCrede
self.StartSyncTask(ctx, userCred, true, "")
}
func (manager *SGuestManager) checkCreateQuota(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, input *api.ServerCreateInput, hasBackup bool) error {
req := getGuestResourceRequirements(ctx, userCred, input, 1, hasBackup)
func (manager *SGuestManager) checkCreateQuota(
ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
input *api.ServerCreateInput, hasBackup bool, count int) (*SQuota, error) {
req := getGuestResourceRequirements(ctx, userCred, input, count, hasBackup)
quotaPlatform := make([]string, 0)
if len(input.Hypervisor) > 0 {
quotaPlatform = GetDriver(input.Hypervisor).GetQuotaPlatformID()
}
err := QuotaManager.CheckSetPendingQuota(ctx, userCred, rbacutils.ScopeProject, ownerId, quotaPlatform, &req)
if err != nil {
return httperrors.NewOutOfQuotaError(err.Error())
return nil, httperrors.NewOutOfQuotaError(err.Error())
} else {
return nil
return &req, nil
}
}
pkg/hostman/storageman/imagecache_local.go
+3
View File
@@ -71,6 +71,9 @@ func (l *SLocalImageCache) GetImageId() string {
}
func (l *SLocalImageCache) GetName() string {
if l.Desc != nil && len(l.Desc.Name) > 0 {
return l.Desc.Name
}
return l.imageId
}
pkg/hostman/storageman/imagecache_rbd.go
+6 -4
View File
@@ -33,9 +33,10 @@ import (
)
type SRbdImageCache struct {
imageId string
cond *sync.Cond
Manager IImageCacheManger
imageId string
imageName string
cond *sync.Cond
Manager IImageCacheManger
}
func NewRbdImageCache(imageId string, imagecacheManager IImageCacheManger) *SRbdImageCache {
@@ -72,6 +73,7 @@ func (r *SRbdImageCache) Acquire(ctx context.Context, zone, srcUrl, format strin
log.Errorf("failed to acquireimage %s ", r.imageId)
return false
}
r.imageName = localImageCache.GetName()
if !r.Load() {
log.Debugf("convert local image %s to rbd pool %s", r.imageId, r.Manager.GetPath())
@@ -112,7 +114,7 @@ func (r *SRbdImageCache) GetDesc() *remotefile.SImageDesc {
size := storage.getImageSizeMb(imageCacheManger.Pool, r.GetName())
return &remotefile.SImageDesc{
Size: int64(size),
Name: r.GetName(),
Name: r.imageName,
}
}
pkg/keystone/models/identity_provider.go
+1 -1
View File
@@ -704,7 +704,7 @@ func (self *SIdentityProvider) SyncOrCreateDomain(ctx context.Context, extId str
if err == nil {
if domain.Name != extName {
// sync domain name
newName, err := db.GenerateName2(DomainManager, nil, extName, domain)
newName, err := db.GenerateName2(DomainManager, nil, extName, domain, 1)
if err != nil {
log.Errorf("sync existing domain name (%s=%s) generate fail %s", domain.Name, extName, err)
} else {