Merge branch 'release/2.1.0' of ssh://git.yunion.io/~qiujian/onecloud into feature/qj-elastic-ip

pkg/cloudcommon/db/db_dispatcher.go

@@ -21,6 +21,7 @@ import (
 	"yunion.io/x/onecloud/pkg/mcclient/auth"
 	"yunion.io/x/onecloud/pkg/mcclient/modules"
 	"yunion.io/x/onecloud/pkg/util/httputils"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type DBModelDispatcher struct {
@@ -264,9 +265,9 @@ func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery
 				if cond != nil {
 					sq = sq.Filter(cond)
 					if filterAny {
-						q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("id"), sq)))
+						q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field(jfc.OriginKey), sq)))
 					} else {
-						q = q.Filter(sqlchemy.AND(sqlchemy.In(q.Field("id"), sq)))
+						q = q.Filter(sqlchemy.AND(sqlchemy.In(q.Field(jfc.OriginKey), sq)))
 					}
 				}
 			}
@@ -768,6 +769,7 @@ func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils
 		return nil, httperrors.NewGeneralError(err)
 	}
 	OpsLog.LogEvent(model, ACT_CREATE, model.GetShortDesc(), userCred)
+	logclient.AddActionLog(model, logclient.ACT_CREATE, "", userCred, true)
 	dispatcher.modelManager.OnCreateComplete(ctx, []IModel{model}, userCred, query, data)
 	return getItemDetails(dispatcher.modelManager, model, ctx, userCred, query)
 }
@@ -986,11 +988,13 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 
 	if err != nil {
 		log.Errorf("validate update condition error: %s", err)
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, err.Error(), userCred, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 
 	dataDict, ok := data.(*jsonutils.JSONDict)
 	if !ok {
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, "Invalid data JSONObject", userCred, false)
 		return nil, httperrors.NewInternalServerError("Invalid data JSONObject")
 	}
 
@@ -998,13 +1002,16 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 	if len(name) > 0 {
 		err = alterNameValidator(item, name)
 		if err != nil {
+			logclient.AddActionLog(item, logclient.ACT_UPDATE, err.Error(), userCred, false)
 			return nil, err
 		}
 	}
 
 	dataDict, err = item.ValidateUpdateData(ctx, userCred, query, dataDict)
 	if err != nil {
-		log.Errorf("validate update data error: %s", err)
+		errMsg := fmt.Sprintf("validate update data error: %s", err)
+		log.Errorf(errMsg)
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, errMsg, userCred, false)
 		return nil, httperrors.NewGeneralError(err)
 	}
 	item.PreUpdate(ctx, userCred, query, dataDict)
@@ -1012,7 +1019,9 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		filterData := dataDict.CopyIncludes(updateFields(manager, userCred)...)
 		err = filterData.Unmarshal(item)
 		if err != nil {
-			log.Errorf("unmarshal fail: %s", err)
+			errMsg := fmt.Sprintf("unmarshal fail: %s", err)
+			logclient.AddActionLog(item, logclient.ACT_UPDATE, errMsg, userCred, false)
+			log.Errorf(errMsg)
 			return httperrors.NewGeneralError(err)
 		}
 		return nil
@@ -1026,7 +1035,10 @@ func updateItem(manager IModelManager, item IModel, ctx context.Context, userCre
 		diffStr := sqlchemy.UpdateDiffString(diff)
 		if len(diffStr) > 0 {
 			OpsLog.LogEvent(item, ACT_UPDATE, diffStr, userCred)
+			logclient.AddActionLog(item, logclient.ACT_UPDATE, diffStr, userCred, true)
 		}
+	} else {
+		logclient.AddActionLog(item, logclient.ACT_UPDATE, "", userCred, true)
 	}
 	return getItemDetails(manager, item, ctx, userCred, query)
 }
@@ -1054,10 +1066,13 @@ func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IM
 		return item.MarkDelete()
 	})
 	if err != nil {
-		log.Errorf("save update error %s", err)
+		msg := fmt.Sprintf("save update error %s", err)
+		log.Errorf(msg)
+		logclient.AddActionLog(item, logclient.ACT_DELETE, msg, userCred, false)
 		return httperrors.NewGeneralError(err)
 	}
 	OpsLog.LogEvent(item, ACT_DELETE, item.GetShortDesc(), userCred)
+	logclient.AddActionLog(item, logclient.ACT_DELETE, item.GetShortDesc(), userCred, true)
 	return nil
 }
 

pkg/cloudcommon/db/opslog.go

@@ -10,6 +10,7 @@ import (
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/log"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 	"yunion.io/x/pkg/util/stringutils"
 	"yunion.io/x/sqlchemy"
 )
@@ -113,6 +114,9 @@ const (
 	ACT_UNCACHE_IMAGE_FAIL = "uncache_image_fail"
 	ACT_UNCACHED_IMAGE     = "uncached_image"
 
+	ACT_SYNC_CLOUD_DISK   = "sync_cloud_disk"
+	ACT_SYNC_CLOUD_SERVER = "sync_cloud_server"
+
 	ACT_SPLIT = "net_split"
 
 	ACT_PENDING_DELETE = "pending_delete"
@@ -304,6 +308,7 @@ func (manager *SOpsLogManager) SyncOwner(m IModel, former *STenant, userCred mcc
 	notes.Add(jsonutils.NewString(former.GetId()), "former_project_id")
 	notes.Add(jsonutils.NewString(former.GetName()), "form_project")
 	manager.LogEvent(m, ACT_CHANGE_OWNER, notes, userCred)
+	logclient.AddActionLog(m, logclient.ACT_CHANGE_OWNER, nil, userCred, true)
 }
 
 func (manager *SOpsLogManager) AllowListItems(ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) bool {

pkg/cloudcommon/db/resourcebase.go

@@ -13,7 +13,7 @@ type SResourceBase struct {
 	SModelBase
 
 	CreatedAt     time.Time `nullable:"false" created_at:"true" get:"user"`
-	UpdatedAt     time.Time `nullable:"false" updated_at:"true"`
+	UpdatedAt     time.Time `nullable:"false" updated_at:"true" list:"user"`
 	UpdateVersion int       `default:"0" nullable:"false" auto_version:"true"`
 	DeletedAt     time.Time ``
 	Deleted       bool      `nullable:"false" default:"false"`

pkg/cloudcommon/db/statusstandalone.go

@@ -3,9 +3,11 @@ package db
 import (
 	"context"
 	"fmt"
+	"strings"
 
 	"yunion.io/x/jsonutils"
 	"yunion.io/x/onecloud/pkg/mcclient"
+	"yunion.io/x/onecloud/pkg/util/logclient"
 )
 
 type SStatusStandaloneResourceBase struct {
@@ -40,6 +42,9 @@ func (model *SStatusStandaloneResourceBase) SetStatus(userCred mcclient.TokenCre
 			notes = fmt.Sprintf("%s: %s", notes, reason)
 		}
 		OpsLog.LogEvent(model, ACT_UPDATE_STATUS, notes, userCred)
+		if strings.Contains(notes, "fail") {
+			logclient.AddActionLog(model, logclient.ACT_VM_SYNC_STATUS, notes, userCred, false)
+		}
 	}
 	return nil
 }

pkg/cloudcommon/validators/choices.go

@@ -0,0 +1,31 @@
+package validators
+
+import (
+	"strings"
+)
+
+type Empty struct{}
+type Choices map[string]Empty
+
+func NewChoices(choices ...string) Choices {
+	cs := Choices{}
+	for _, choice := range choices {
+		cs[choice] = Empty{}
+	}
+	return cs
+}
+
+func (cs Choices) Has(choice string) bool {
+	_, ok := cs[choice]
+	return ok
+}
+
+func (cs Choices) String() string {
+	choices := make([]string, len(cs))
+	i := 0
+	for choice, _ := range cs {
+		choices[i] = choice
+		i++
+	}
+	return strings.Join(choices, "|")
+}

pkg/cloudcommon/validators/doc.go

@@ -0,0 +1 @@
+package validators // import "yunion.io/x/onecloud/pkg/cloudcommon/validators"

pkg/cloudcommon/validators/errors.go

@@ -0,0 +1,106 @@
+package validators
+
+import (
+	"fmt"
+)
+
+type ErrType uintptr
+
+const (
+	ERR_SUCCESS ErrType = iota
+	ERR_GENERAL         // uncategorized error
+	ERR_MISSING_KEY
+	ERR_INVALID_TYPE
+	ERR_INVALID_CHOICE
+	ERR_NOT_IN_RANGE
+	ERR_INVALID_VALUE
+	ERR_MODEL_MANAGER
+	ERR_MODEL_NOT_FOUND
+)
+
+var errTypeToString = map[ErrType]string{
+	ERR_SUCCESS:         "No error",
+	ERR_GENERAL:         "General error",
+	ERR_MISSING_KEY:     "Missing_key error",
+	ERR_INVALID_TYPE:    "Invalid type error",
+	ERR_INVALID_CHOICE:  "Invalid choice error",
+	ERR_NOT_IN_RANGE:    "Not in range error",
+	ERR_INVALID_VALUE:   "Invalid value error",
+	ERR_MODEL_MANAGER:   "Model manager error",
+	ERR_MODEL_NOT_FOUND: "Model not found error",
+}
+
+func (errType ErrType) String() string {
+	s, ok := errTypeToString[errType]
+	if ok {
+		return s
+	}
+	return "unknown error"
+}
+
+type ValidateError struct {
+	ErrType ErrType
+	Msg     string
+}
+
+func (ve *ValidateError) Error() string {
+	return ve.Msg
+}
+
+// TODO let each validator provide the error
+func newMissingKeyError(key string) error {
+	msg := fmt.Sprintf("missing %q", key)
+	return newError(ERR_MISSING_KEY, msg)
+}
+
+func newGeneralError(key string, err error) error {
+	msg := fmt.Sprintf("general error for %q: %s", key, err)
+	return newError(ERR_GENERAL, msg)
+}
+
+func newInvalidTypeError(key string, typ string, err error) error {
+	msg := fmt.Sprintf("expecting %s type for %q: %s", typ, key, err)
+	return newError(ERR_INVALID_TYPE, msg)
+}
+
+func newInvalidChoiceError(key string, choices Choices, choice string) error {
+	msg := fmt.Sprintf("invalid %q, want %s, got %s", key, choices, choice)
+	return newError(ERR_INVALID_CHOICE, msg)
+}
+
+func newNotInRangeError(key string, value, lower, upper int64) error {
+	msg := fmt.Sprintf("invalid %q: %d, want [%d,%d]", key, value, lower, upper)
+	return newError(ERR_NOT_IN_RANGE, msg)
+}
+
+func newInvalidValueError(key string, value string) error {
+	msg := fmt.Sprintf("invalid %q: %s", key, value)
+	return newError(ERR_INVALID_VALUE, msg)
+}
+
+func newModelManagerError(modelKeyword string) error {
+	msg := fmt.Sprintf("internal error: getting model manager for %q failed",
+		modelKeyword)
+	return newError(ERR_MODEL_MANAGER, msg)
+}
+
+func newModelNotFoundError(modelKeyword, idOrName string, err error) error {
+	msg := fmt.Sprintf("cannot find %q with id/name %q: %s",
+		modelKeyword, idOrName, err)
+	return newError(ERR_MODEL_NOT_FOUND, msg)
+}
+
+func newError(typ ErrType, msg string) error {
+	return &ValidateError{
+		ErrType: typ,
+		Msg:     msg,
+	}
+}
+
+func IsModelNotFoundError(err error) bool {
+	ve, ok := err.(*ValidateError)
+	if ok && ve.ErrType == ERR_MODEL_NOT_FOUND {
+		return true
+	}
+	return false
+}

pkg/cloudcommon/validators/validators.go

@@ -0,0 +1,427 @@
+package validators
+
+// TODO
+//
+// email
+// uuid
+// uri
+
+import (
+	"math"
+	"net"
+	"reflect"
+	"regexp"
+
+	"yunion.io/x/jsonutils"
+	"yunion.io/x/onecloud/pkg/cloudcommon/db"
+	"yunion.io/x/pkg/gotypes"
+	"yunion.io/x/pkg/util/regutils"
+	"yunion.io/x/sqlchemy"
+)
+
+type ValidatorFunc func(*jsonutils.JSONDict) error
+
+type IValidatorBase interface {
+	Validate(data *jsonutils.JSONDict) error
+}
+
+type IValidator interface {
+	IValidatorBase
+	getValue() interface{}
+	setDefault(data *jsonutils.JSONDict) bool
+}
+
+type Validator struct {
+	parent     IValidator
+	Key        string
+	optional   bool
+	defaultVal interface{}
+	value      jsonutils.JSONObject
+}
+
+func (v *Validator) Optional(optional bool) IValidator {
+	v.optional = optional
+	return v.parent
+}
+
+func (v *Validator) Default(defaultVal interface{}) IValidator {
+	v.defaultVal = defaultVal
+	return v.parent
+}
+
+func (v *Validator) getValue() interface{} {
+	return nil
+}
+
+func (v *Validator) setDefault(data *jsonutils.JSONDict) bool {
+	switch v.defaultVal.(type) {
+	case string:
+		s := v.defaultVal.(string)
+		v.value = jsonutils.NewString(s)
+		data.Set(v.Key, v.value)
+		return true
+	case int, int32, int64, uint, uint32, uint64:
+		value := reflect.ValueOf(v.defaultVal)
+		value64 := value.Convert(gotypes.Int64Type)
+		defaultVal64 := value64.Interface().(int64)
+		v.value = jsonutils.NewInt(defaultVal64)
+		data.Set(v.Key, v.value)
+		return true
+	}
+	return false
+}
+
+func (v *Validator) Validate(data *jsonutils.JSONDict) error {
+	err, _ := v.validateEx(data)
+	return err
+}
+
+func (v *Validator) validateEx(data *jsonutils.JSONDict) (err error, isSet bool) {
+	if !data.Contains(v.Key) {
+		if v.defaultVal != nil {
+			isSet = v.parent.setDefault(data)
+			return nil, isSet
+		}
+		if !v.optional {
+			err = newMissingKeyError(v.Key)
+			return
+		}
+		return
+	}
+	value, err := data.Get(v.Key)
+	if err != nil {
+		return
+	}
+	v.value = value
+	isSet = true
+	return
+}
+
+type ValidatorStringChoices struct {
+	Validator
+	Choices    Choices
+	defaultVal string
+	Value      string
+}
+
+func NewStringChoicesValidator(key string, choices Choices) *ValidatorStringChoices {
+	v := &ValidatorStringChoices{
+		Validator: Validator{Key: key},
+		Choices:   choices,
+	}
+	v.parent = v
+	return v
+}
+
+func (v *ValidatorStringChoices) Default(s string) IValidator {
+	if v.Choices.Has(s) {
+		return v.Validator.Default(s)
+	}
+	panic("invalid default for " + v.Key)
+}
+
+func (v *ValidatorStringChoices) getValue() interface{} {
+	return v.Value
+}
+
+func (v *ValidatorStringChoices) Validate(data *jsonutils.JSONDict) error {
+	if err, isSet := v.Validator.validateEx(data); err != nil || !isSet {
+		return err
+	}
+	s, err := v.value.GetString()
+	if err != nil {
+		return newGeneralError(v.Key, err)
+	}
+	if !v.Choices.Has(s) {
+		return newInvalidChoiceError(v.Key, v.Choices, s)
+	}
+	// in case it's stringified from v.value
+	data.Set(v.Key, jsonutils.NewString(s))
+	v.Value = s
+	return nil
+}
+
+type ValidatorRange struct {
+	Validator
+	Lower int64
+	Upper int64
+	Value int64
+}
+
+func (v *ValidatorRange) getValue() interface{} {
+	return v.Value
+}
+
+func (v *ValidatorRange) Default(i int64) IValidator {
+	if i >= v.Lower && i <= v.Upper {
+		return v.Validator.Default(i)
+	}
+	panic("invalid default for " + v.Key)
+}
+func (v *ValidatorRange) Validate(data *jsonutils.JSONDict) error {
+	if err, isSet := v.Validator.validateEx(data); err != nil || !isSet {
+		return err
+	}
+	i, err := v.value.Int()
+	if err != nil {
+		return newInvalidTypeError(v.Key, "integer", err)
+	}
+	if i < v.Lower || i > v.Upper {
+		return newNotInRangeError(v.Key, i, v.Lower, v.Upper)
+	}
+	data.Set(v.Key, jsonutils.NewInt(i))
+	v.Value = i
+	return nil
+}
+
+func NewRangeValidator(key string, lower int64, upper int64) *ValidatorRange {
+	v := &ValidatorRange{
+		Validator: Validator{Key: key},
+		Lower:     lower,
+		Upper:     upper,
+	}
+	v.parent = v
+	return v
+}
+
+func NewPortValidator(key string) *ValidatorRange {
+	return NewRangeValidator(key, 1, 65535)
+}
+
+func NewNonNegativeValidator(key string) *ValidatorRange {
+	return NewRangeValidator(key, 0, math.MaxInt64)
+}
+
+type ValidatorModelIdOrName struct {
+	Validator
+	ModelKeyword string
+	ProjectId    string
+	ModelManager db.IModelManager
+	Model        db.IModel
+	modelIdKey   string
+}
+
+func (v *ValidatorModelIdOrName) getValue() interface{} {
+	return v.Model
+}
+
+func NewModelIdOrNameValidator(key string, modelKeyword string, projectId string) *ValidatorModelIdOrName {
+	v := &ValidatorModelIdOrName{
+		Validator:    Validator{Key: key},
+		ProjectId:    projectId,
+		ModelKeyword: modelKeyword,
+		modelIdKey:   key + "_id",
+	}
+	v.parent = v
+	return v
+}
+
+func (v *ValidatorModelIdOrName) ModelIdKey(modelIdKey string) *ValidatorModelIdOrName {
+	v.modelIdKey = modelIdKey
+	return v
+}
+
+func (v *ValidatorModelIdOrName) validate(data *jsonutils.JSONDict) error {
+	if err, isSet := v.Validator.validateEx(data); err != nil || !isSet {
+		return err
+	}
+	modelIdOrName, err := v.value.GetString()
+	if err != nil {
+		return err
+	}
+
+	modelManager := db.GetModelManager(v.ModelKeyword)
+	if modelManager == nil {
+		return newModelManagerError(v.ModelKeyword)
+	}
+	v.ModelManager = modelManager
+	model, err := modelManager.FetchByIdOrName(v.ProjectId, modelIdOrName)
+	if err != nil {
+		return newModelNotFoundError(v.ModelKeyword, modelIdOrName, err)
+	}
+	v.Model = model
+	return nil
+}
+
+func (v *ValidatorModelIdOrName) Validate(data *jsonutils.JSONDict) error {
+	err := v.validate(data)
+	if err != nil {
+		return err
+	}
+	if v.Model != nil {
+		if len(v.modelIdKey) > 0 {
+			data.Remove(v.Key)
+			data.Set(v.modelIdKey, jsonutils.NewString(v.Model.GetId()))
+		}
+	}
+	return nil
+}
+
+func (v *ValidatorModelIdOrName) QueryFilter(q *sqlchemy.SQuery, data *jsonutils.JSONDict) (*sqlchemy.SQuery, error) {
+	err := v.validate(data)
+	if err != nil {
+		if IsModelNotFoundError(err) {
+			// hack
+			q = q.Equals(v.modelIdKey, "0")
+			q = q.Equals(v.modelIdKey, "1")
+			return q, nil
+		}
+		return nil, err
+	}
+	if v.Model != nil {
+		q = q.Equals(v.modelIdKey, v.Model.GetId())
+	}
+	return q, nil
+}
+
+type ValidatorRegexp struct {
+	Validator
+	Regexp     *regexp.Regexp
+	Value      string
+	allowEmpty bool
+}
+
+func (v *ValidatorRegexp) AllowEmpty(allowEmpty bool) *ValidatorRegexp {
+	v.allowEmpty = allowEmpty
+	return v
+}
+
+func (v *ValidatorRegexp) getValue() interface{} {
+	return v.Value
+}
+
+func (v *ValidatorRegexp) Validate(data *jsonutils.JSONDict) error {
+	if err, isSet := v.Validator.validateEx(data); err != nil || !isSet {
+		return err
+	}
+	value, err := v.value.GetString()
+	if err != nil {
+		return newInvalidTypeError(v.Key, "string", err)
+	}
+	if v.allowEmpty && len(value) == 0 {
+		return nil
+	}
+	if !v.Regexp.MatchString(value) {
+		return newInvalidValueError(v.Key, value)
+	}
+	v.Value = value
+	return nil
+}
+
+func NewRegexpValidator(key string, regexp *regexp.Regexp) *ValidatorRegexp {
+	v := &ValidatorRegexp{
+		Validator: Validator{Key: key},
+		Regexp:    regexp,
+	}
+	v.parent = v
+	return v
+}
+
+type ValidatorDomainName struct {
+	ValidatorRegexp
+}
+
+func NewDomainNameValidator(key string) *ValidatorDomainName {
+	v := &ValidatorDomainName{
+		ValidatorRegexp: *NewRegexpValidator(key, regutils.DOMAINNAME_REG),
+	}
+	v.parent = v
+	return v
+}
+
+type ValidatorURLPath struct {
+	ValidatorRegexp
+}
+
+// URI Path as defined in https://tools.ietf.org/html/rfc3986#section-3.3
+var regexpURLPath = regexp.MustCompile(`^(?:/[a-zA-Z0-9.%$&'()*+,;=!~_-]*)*$`)
+
+func NewURLPathValidator(key string) *ValidatorURLPath {
+	v := &ValidatorURLPath{
+		ValidatorRegexp: *NewRegexpValidator(key, regexpURLPath),
+	}
+	v.parent = v
+	return v
+}
+
+type ValidatorStruct struct {
+	Validator
+	Value interface{}
+}
+
+func (v *ValidatorStruct) getValue() interface{} {
+	return v.Value
+}
+
+func (v *ValidatorStruct) Validate(data *jsonutils.JSONDict) error {
+	if err, isSet := v.Validator.validateEx(data); err != nil || !isSet {
+		return err
+	}
+	err := v.value.Unmarshal(v.Value)
+	if err != nil {
+		return newGeneralError(v.Key, err)
+	}
+	if valueValidator, ok := v.Value.(IValidatorBase); ok {
+		err = valueValidator.Validate(data)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func NewStructValidator(key string, value interface{}) *ValidatorStruct {
+	v := &ValidatorStruct{
+		Validator: Validator{Key: key},
+		Value:     value,
+	}
+	v.parent = v
+	return v
+}
+
+type ValidatorIPv4Addr struct {
+	Validator
+	IP net.IP
+}
+
+func (v *ValidatorIPv4Addr) getValue() interface{} {
+	return v.IP
+}
+
+func (v *ValidatorIPv4Addr) setDefault(data *jsonutils.JSONDict) bool {
+	if v.defaultVal == nil {
+		return false
+	}
+	defaultIP, ok := v.defaultVal.(net.IP)
+	if !ok {
+		return false
+	}
+	value := jsonutils.NewString(defaultIP.String())
+	v.value = value
+	data.Set(v.Key, value)
+	return true
+}
+
+func (v *ValidatorIPv4Addr) Validate(data *jsonutils.JSONDict) error {
+	if err, isSet := v.Validator.validateEx(data); err != nil || !isSet {
+		return err
+	}
+	s, err := v.value.GetString()
+	if err != nil {
+		return newInvalidTypeError(v.Key, "string", err)
+	}
+	ip := net.ParseIP(s).To4()
+	if ip == nil {
+		return newInvalidValueError(v.Key, s)
+	}
+	v.IP = ip
+	return nil
+}
+
+func NewIPv4AddrValidator(key string) *ValidatorIPv4Addr {
+	v := &ValidatorIPv4Addr{
+		Validator: Validator{Key: key},
+	}
+	v.parent = v
+	return v
+}

pkg/cloudcommon/validators/validators_test.go

@@ -0,0 +1,500 @@
+package validators
+
+// TODO
+//
+//  - strict type, no implicit conversion
+//  - test model validator
+//  - invalid default for string choice, range
+
+import (
+	"net"
+	"reflect"
+	"testing"
+
+	"yunion.io/x/jsonutils"
+)
+
+func TestURLPathRegexp(t *testing.T) {
+	cases := []struct {
+		in    string
+		match bool
+	}{
+		{in: "", match: true},
+		{in: "/", match: true},
+		{in: "/p", match: true},
+		{in: "/p/", match: true},
+		{in: "p", match: false},
+		{in: "p/", match: false},
+		{in: "/p?", match: false},
+		{in: "/p#", match: false},
+	}
+	for _, c := range cases {
+		got := regexpURLPath.MatchString(c.in)
+		if got != c.match {
+			t.Errorf("%q match, want %v, got %v", c.in, c.match, got)
+		}
+	}
+}
+
+type C struct {
+	Name      string
+	In        string
+	Out       string
+	Optional  bool
+	Default   interface{}
+	ValueWant interface{}
+	Err       ErrType
+}
+
+func testS(t *testing.T, v IValidator, c *C) {
+	j, _ := jsonutils.ParseString(c.In)
+	jd := j.(*jsonutils.JSONDict)
+	err := v.Validate(jd)
+	if err != nil {
+		verr, ok := err.(*ValidateError)
+		if ok {
+			if verr.ErrType != c.Err {
+				t.Errorf("error want %q, got %q",
+					c.Err, verr.ErrType)
+			}
+		} else {
+			t.Errorf("want error type ValidateError")
+		}
+	} else {
+		if c.Err != ERR_SUCCESS {
+			t.Errorf("expect error: %s", c.Err)
+		}
+	}
+	jWant, _ := jsonutils.ParseString(c.Out)
+	if !reflect.DeepEqual(j, jWant) {
+		t.Errorf("json out want %s, got %s",
+			jWant.String(), j.String())
+	}
+	value := v.getValue()
+	if !reflect.DeepEqual(value, c.ValueWant) {
+		t.Errorf("value want %#v, got %#v", c.ValueWant, value)
+	}
+}
+
+func TestStringChoicesValidator(t *testing.T) {
+	choices := NewChoices("choice0", "choice1", "100")
+	cases := []*C{
+		{
+			Name:      "missing non-optional",
+			In:        `{}`,
+			Out:       `{}`,
+			Optional:  false,
+			Err:       ERR_MISSING_KEY,
+			ValueWant: "",
+		},
+		{
+			Name:      "missing optional",
+			In:        `{}`,
+			Out:       `{}`,
+			Optional:  true,
+			ValueWant: "",
+		},
+		{
+			Name:      "missing with default",
+			In:        `{}`,
+			Out:       `{s: "choice0"}`,
+			Default:   "choice0",
+			ValueWant: "choice0",
+		},
+		{
+			Name:      "stringified",
+			In:        `{"s": 100}`,
+			Out:       `{s: "100"}`,
+			ValueWant: "100",
+		},
+		{
+			Name:      "stringified invalid choice",
+			In:        `{"s": 101}`,
+			Out:       `{"s": 101}`,
+			Err:       ERR_INVALID_CHOICE,
+			ValueWant: "",
+		},
+		{
+			Name:      "good choice",
+			In:        `{"s": "choice1"}`,
+			Out:       `{"s": "choice1"}`,
+			ValueWant: "choice1",
+		},
+		{
+			Name:      "bad choice",
+			In:        `{"s": "badchoice"}`,
+			Out:       `{"s": "badchoice"}`,
+			Err:       ERR_INVALID_CHOICE,
+			ValueWant: "",
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			v := NewStringChoicesValidator("s", choices)
+			if c.Default != nil {
+				s := c.Default.(string)
+				v.Default(s)
+			}
+			if c.Optional {
+				v.Optional(true)
+			}
+			testS(t, v, c)
+		})
+	}
+}
+
+func TestRangeValidator(t *testing.T) {
+	cases := []*C{
+		{
+			Name:      "missing non-optional",
+			In:        `{}`,
+			Out:       `{}`,
+			Err:       ERR_MISSING_KEY,
+			ValueWant: int64(0),
+		},
+		{
+			Name:      "missing optional",
+			In:        `{}`,
+			Out:       `{}`,
+			Optional:  true,
+			ValueWant: int64(0),
+		},
+		{
+			Name:      "missing with default",
+			In:        `{}`,
+			Out:       `{s: 1}`,
+			Default:   int64(1),
+			ValueWant: int64(1),
+		},
+		{
+			Name:      "parsed",
+			In:        `{s: "100"}`,
+			Out:       `{s: 100}`,
+			ValueWant: int64(100),
+		},
+		{
+			Name:      "parsed invalid int",
+			In:        `{s: "abc"}`,
+			Out:       `{s: "abc"}`,
+			Err:       ERR_INVALID_TYPE,
+			ValueWant: int64(0),
+		},
+		{
+			Name:      "parsed not in range",
+			In:        `{s: "65536"}`,
+			Out:       `{s: "65536"}`,
+			Err:       ERR_NOT_IN_RANGE,
+			ValueWant: int64(0),
+		},
+		{
+			Name:      "in range",
+			In:        `{s: 100}`,
+			Out:       `{s: 100}`,
+			ValueWant: int64(100),
+		},
+		{
+			Name:      "not in range",
+			In:        `{s: 0}`,
+			Out:       `{s: 0}`,
+			Err:       ERR_NOT_IN_RANGE,
+			ValueWant: int64(0),
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			v := NewPortValidator("s")
+			if c.Default != nil {
+				i := c.Default.(int64)
+				v.Default(i)
+			}
+			if c.Optional {
+				v.Optional(true)
+			}
+			testS(t, v, c)
+		})
+	}
+}
+
+func TestRegexValidator(t *testing.T) {
+	type RegexC struct {
+		*C
+		AllowEmpty bool
+	}
+	cases := []*RegexC{
+		{
+			C: &C{
+				Name:      "missing non-optional",
+				In:        `{}`,
+				Out:       `{}`,
+				Err:       ERR_MISSING_KEY,
+				ValueWant: "",
+			},
+		},
+		{
+			C: &C{
+				Name:      "missing optional",
+				In:        `{}`,
+				Out:       `{}`,
+				Optional:  true,
+				ValueWant: "",
+			},
+		},
+		{
+			C: &C{
+				Name:      "missing with default",
+				In:        `{}`,
+				Out:       `{s: "example.com"}`,
+				Default:   "example.com",
+				ValueWant: "example.com",
+			},
+		},
+		{
+			C: &C{
+				Name:      "valid",
+				In:        `{s: "a.example.com"}`,
+				Out:       `{s: "a.example.com"}`,
+				ValueWant: "a.example.com",
+			},
+		},
+		{
+			C: &C{
+				Name:      "valid (allow empty)",
+				In:        `{s: ""}`,
+				Out:       `{s: ""}`,
+				ValueWant: "",
+			},
+			AllowEmpty: true,
+		},
+		{
+			C: &C{
+				Name:      "invalid",
+				In:        `{s: "/.example.com"}`,
+				Out:       `{s: "/.example.com"}`,
+				ValueWant: "",
+				Err:       ERR_INVALID_VALUE,
+			},
+		},
+		{
+			C: &C{
+				Name:      "invalid (disallow empty)",
+				In:        `{s: ""}`,
+				Out:       `{s: ""}`,
+				ValueWant: "",
+				Err:       ERR_INVALID_VALUE,
+			},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			v := NewDomainNameValidator("s")
+			if c.Default != nil {
+				i := c.Default.(string)
+				v.Default(i)
+			}
+			if c.Optional {
+				v.Optional(true)
+			}
+			if c.AllowEmpty {
+				v.AllowEmpty(true)
+			}
+			testS(t, v, c.C)
+		})
+	}
+}
+
+func TestIPv4Validator(t *testing.T) {
+	var nilIP net.IP
+	localIP := net.IPv4(127, 0, 0, 1).To4()
+	cases := []*C{
+		{
+			Name:      "missing non-optional",
+			In:        `{}`,
+			Out:       `{}`,
+			Err:       ERR_MISSING_KEY,
+			ValueWant: nilIP,
+		},
+		{
+			Name:      "missing optional",
+			In:        `{}`,
+			Out:       `{}`,
+			Optional:  true,
+			ValueWant: nilIP,
+		},
+		{
+			Name:      "missing with default",
+			In:        `{}`,
+			Out:       `{s: "127.0.0.1"}`,
+			Default:   localIP,
+			ValueWant: localIP,
+		},
+		{
+			Name:      "valid",
+			In:        `{s: "127.0.0.1"}`,
+			Out:       `{s: "127.0.0.1"}`,
+			ValueWant: localIP,
+		},
+		{
+			Name:      "valid (v4 in v6)",
+			In:        `{s: "::ffff:127.0.0.1"}`,
+			Out:       `{s: "::ffff:127.0.0.1"}`,
+			ValueWant: localIP,
+		},
+		{
+			Name:      "invalid",
+			In:        `{s: "127.0.0"}`,
+			Out:       `{s: "127.0.0"}`,
+			Err:       ERR_INVALID_VALUE,
+			ValueWant: nilIP,
+		},
+		{
+			Name:      "invalid (empty string)",
+			In:        `{s: ""}`,
+			Out:       `{s: ""}`,
+			Err:       ERR_INVALID_VALUE,
+			ValueWant: nilIP,
+		},
+		{
+			Name:      "invalid (wrong type)",
+			In:        `{s: 100}`,
+			Out:       `{s: 100}`,
+			Err:       ERR_INVALID_VALUE,
+			ValueWant: nilIP,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			v := NewIPv4AddrValidator("s")
+			if c.Default != nil {
+				i := c.Default.(net.IP)
+				v.Default(i)
+			}
+			if c.Optional {
+				v.Optional(true)
+			}
+			testS(t, v, c)
+		})
+	}
+}
+
+type TestStruct struct {
+	F0 string
+	F1 int
+	F2 bool
+}
+
+type TestVStruct TestStruct
+
+func (v *TestVStruct) Validate(data *jsonutils.JSONDict) error {
+	if v.F2 {
+		return nil
+	} else {
+		return newInvalidValueError("F2", v.F0)
+	}
+}
+
+func TestStructValidator(t *testing.T) {
+	type StructC struct {
+		*C
+		Value interface{}
+	}
+	defaultVal := &TestStruct{
+		F0: "holy",
+		F1: 100,
+		F2: true,
+	}
+	var defaultValMiss *TestStruct
+	{
+		defaultValMissCopy := *defaultVal
+		defaultValMissCopy.F2 = false
+		defaultValMiss = &defaultValMissCopy
+	}
+	defaultValJsonStr := `{s: {"F0": "holy", "F1": 100, "F2": true}}`
+	defaultValJsonStrMiss := `{s: {"F0": "holy", "F1": 100}}`
+	defaultValJsonStrMore := `{s: {"F0": "holy", "F1": 100, "F2": true, "Foo": "bar"}}`
+
+	defaultVVal := (*TestVStruct)(defaultVal)
+	var defaultVValBad *TestVStruct
+	{
+		defaultVValBadCopy := *defaultVVal
+		defaultVValBadCopy.F2 = false
+		defaultVValBad = &defaultVValBadCopy
+	}
+	defaultVValJsonStrBad := `{s: {"F0": "holy", "F1": 100, "F2": false}}`
+	cases := []*StructC{
+		{
+			C: &C{
+				Name:      "missing non-optional",
+				In:        `{}`,
+				Out:       `{}`,
+				Err:       ERR_MISSING_KEY,
+				ValueWant: &TestStruct{},
+			},
+			Value: &TestStruct{},
+		},
+		{
+			C: &C{
+				Name:      "missing optional",
+				In:        `{}`,
+				Out:       `{}`,
+				Optional:  true,
+				ValueWant: &TestStruct{},
+			},
+			Value: &TestStruct{},
+		},
+		{
+			C: &C{
+				Name:      "valid",
+				In:        defaultValJsonStr,
+				Out:       defaultValJsonStr,
+				ValueWant: defaultVal,
+			},
+			Value: &TestStruct{},
+		},
+		{
+			C: &C{
+				Name:      "valid (missing fields)",
+				In:        defaultValJsonStrMiss,
+				Out:       defaultValJsonStrMiss,
+				ValueWant: defaultValMiss,
+			},
+			Value: &TestStruct{},
+		},
+		{
+			C: &C{
+				Name:      "valid (more fields)",
+				In:        defaultValJsonStrMore,
+				Out:       defaultValJsonStrMore,
+				ValueWant: defaultVal,
+			},
+			Value: &TestStruct{},
+		},
+		{
+			C: &C{
+				Name:      "valid (struct says valid)",
+				In:        defaultValJsonStr,
+				Out:       defaultValJsonStr,
+				ValueWant: defaultVVal,
+			},
+			Value: &TestVStruct{},
+		},
+		{
+			C: &C{
+				Name:      "invalid (struct says invalid)",
+				In:        defaultVValJsonStrBad,
+				Out:       defaultVValJsonStrBad,
+				ValueWant: defaultVValBad,
+				Err:       ERR_INVALID_VALUE,
+			},
+			Value: &TestVStruct{},
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.Name, func(t *testing.T) {
+			v := NewStructValidator("s", c.Value)
+			if c.Optional {
+				v.Optional(true)
+			}
+			testS(t, v, c.C)
+		})
+	}
+}