From fd352dbdbbd2ff2a839329f936df0c546cb73762 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 8 Aug 2018 09:27:23 +0000 Subject: [PATCH 1/5] scheduler: fix test compilation --- pkg/scheduler/db/models/base_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/scheduler/db/models/base_test.go b/pkg/scheduler/db/models/base_test.go index 4a2deaf646..8c084510be 100644 --- a/pkg/scheduler/db/models/base_test.go +++ b/pkg/scheduler/db/models/base_test.go @@ -155,7 +155,8 @@ func TestGuestDisk(t *testing.T) { if err != nil { t.Fatal(err) } - size, err := gst.(*Guest).DiskSize() + onlyLocal := false + size, err := gst.(*Guest).DiskSize(onlyLocal) if err != nil { t.Fatal(err) } From 1bf9036c836ce074c01a4feadbbb4fdf85e6e2bf Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 9 Aug 2018 04:27:44 +0000 Subject: [PATCH 2/5] treewide: make explicit calls to JSONDict.Remove() and JSONDict.RemoveIgnoreCase --- Gopkg.lock | 5 ++-- Gopkg.toml | 2 +- pkg/cloudcommon/db/db_dispatcher.go | 4 +-- pkg/cloudcommon/db/opslog.go | 6 ++--- pkg/compute/models/guests.go | 2 +- pkg/mcclient/modules/mod_images.go | 2 +- pkg/mcclient/modules/mod_usages.go | 4 +-- .../github.com/yunionio/jsonutils/access.go | 27 ++++++++++++++++++- .../github.com/yunionio/jsonutils/length.go | 13 +++++++++ 9 files changed, 52 insertions(+), 13 deletions(-) create mode 100644 vendor/github.com/yunionio/jsonutils/length.go diff --git a/Gopkg.lock b/Gopkg.lock index 7011dc02ec..dd3ce49855 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -567,9 +567,10 @@ revision = "bbd99532a768d2fe369079ceda730e30726ae1a6" [[projects]] + branch = "release/2.0.0" name = "github.com/yunionio/jsonutils" packages = ["."] - revision = "6dd39f8579af6c6b61b971eb3a1ccf55e724b0ca" + revision = "0f3f30adfdac3dec6ffbbfb72116f6f908134f63" [[projects]] branch = "master" @@ -904,6 +905,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "1fdd79d4d8791f7e87a3740b3807ea84f6306ecac3095d32a1ccf39b48be1f18" + inputs-digest = "aa5af605c21bc592cbc237ad3539bc2cd9bbe5b8495c6c2d22e4a3bd8b148633" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index f30a127cfa..40b7a9bfdb 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -30,7 +30,7 @@ version = "0.2.2" [[constraint]] - revision = "6dd39f8579af6c6b61b971eb3a1ccf55e724b0ca" + branch = "release/2.0.0" name = "github.com/yunionio/jsonutils" [[constraint]] diff --git a/pkg/cloudcommon/db/db_dispatcher.go b/pkg/cloudcommon/db/db_dispatcher.go index 579680b22a..b0b73f77cc 100644 --- a/pkg/cloudcommon/db/db_dispatcher.go +++ b/pkg/cloudcommon/db/db_dispatcher.go @@ -665,7 +665,7 @@ func doCreateItem(manager IModelManager, ctx context.Context, userCred mcclient. generateName, _ := dataDict.GetString("generate_name") if len(generateName) > 0 { - dataDict.Remove("generate_name", true) + dataDict.Remove("generate_name") dataDict.Add(jsonutils.NewString(GenerateName(manager, ownerProjId, generateName)), "name") } else { name, _ := data.GetString("name") @@ -756,7 +756,7 @@ func expandMultiCreateParams(data jsonutils.JSONObject, count int) ([]jsonutils. return nil, httperrors.NewInputParameterError("Missing name or generate_name") } jsonDict.Add(jsonutils.NewString(name), "generate_name") - jsonDict.Remove("name", false) + jsonDict.RemoveIgnoreCase("name") } ret := make([]jsonutils.JSONObject, count) for i := 0; i < count; i += 1 { diff --git a/pkg/cloudcommon/db/opslog.go b/pkg/cloudcommon/db/opslog.go index 308ba46c27..2afaef2974 100644 --- a/pkg/cloudcommon/db/opslog.go +++ b/pkg/cloudcommon/db/opslog.go @@ -270,17 +270,17 @@ func (manager *SOpsLogManager) ListItemFilter(ctx context.Context, q *sqlchemy.S } objTypes := jsonutils.GetQueryStringArray(queryDict, "obj_type") if objTypes != nil && len(objTypes) > 0 { - queryDict.Remove("obj_type", false) + queryDict.RemoveIgnoreCase("obj_type") q = q.Filter(sqlchemy.In(q.Field("obj_type"), objTypes)) } objIds := jsonutils.GetQueryStringArray(queryDict, "obj_id") if objIds != nil && len(objIds) > 0 { - queryDict.Remove("obj_id", false) + queryDict.RemoveIgnoreCase("obj_id") q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field("obj_id"), objIds), sqlchemy.In(q.Field("obj_name"), objIds))) } action := jsonutils.GetQueryStringArray(queryDict, "action") if action != nil && len(action) > 0 { - queryDict.Remove("action", false) + queryDict.RemoveIgnoreCase("action") q = q.Filter(sqlchemy.In(q.Field("action"), action)) } if !userCred.IsSystemAdmin() { diff --git a/pkg/compute/models/guests.go b/pkg/compute/models/guests.go index d7490a5f42..f8aeed41f8 100644 --- a/pkg/compute/models/guests.go +++ b/pkg/compute/models/guests.go @@ -196,7 +196,7 @@ func (manager *SGuestManager) ListItemFilter(ctx context.Context, q *sqlchemy.SQ isBMstr, _ := queryDict.GetString("baremetal") if len(isBMstr) > 0 && utils.ToBool(isBMstr) { queryDict.Add(jsonutils.NewString(HYPERVISOR_BAREMETAL), "hypervisor") - queryDict.Remove("baremetal", true) + queryDict.Remove("baremetal") } hypervisor, _ := queryDict.GetString("hypervisor") if len(hypervisor) > 0 { diff --git a/pkg/mcclient/modules/mod_images.go b/pkg/mcclient/modules/mod_images.go index 5e676bf469..9264241e0d 100644 --- a/pkg/mcclient/modules/mod_images.go +++ b/pkg/mcclient/modules/mod_images.go @@ -119,7 +119,7 @@ func (this *ImageManager) List(session *mcclient.ClientSession, params jsonutils path = fmt.Sprintf("%s/detail", path) } dictparams, _ := params.(*jsonutils.JSONDict) - dictparams.Remove("details", false) + dictparams.RemoveIgnoreCase("details") qs := params.QueryString() if len(qs) > 0 { path = fmt.Sprintf("%s?%s", path, qs) diff --git a/pkg/mcclient/modules/mod_usages.go b/pkg/mcclient/modules/mod_usages.go index 7945dee8e3..ff9daa18ae 100644 --- a/pkg/mcclient/modules/mod_usages.go +++ b/pkg/mcclient/modules/mod_usages.go @@ -20,8 +20,8 @@ func (this *UsageManager) GetGeneralUsage(session *mcclient.ClientSession, param url = fmt.Sprintf("%s/%s/%s", url, range_type, range_id) } dict := params.(*jsonutils.JSONDict) - dict.Remove("range_type", true) - dict.Remove("range_id", true) + dict.Remove("range_type") + dict.Remove("range_id") qs := dict.QueryString() if len(qs) > 0 { url = fmt.Sprintf("%s?%s", url, qs) diff --git a/vendor/github.com/yunionio/jsonutils/access.go b/vendor/github.com/yunionio/jsonutils/access.go index e1e0c0d469..57642a0442 100644 --- a/vendor/github.com/yunionio/jsonutils/access.go +++ b/vendor/github.com/yunionio/jsonutils/access.go @@ -42,11 +42,36 @@ func NewFloat(val float64) *JSONFloat { return &JSONFloat{data: val} } +func NewBool(val bool) *JSONBool { + if val { + return JSONTrue + } + return JSONFalse +} + func (this *JSONDict) Set(key string, value JSONObject) { this.data[key] = value } -func (this *JSONDict) Remove(key string, caseSensitive bool) bool { +func (this *JSONDict) Remove(key string) bool { + return this.remove(key, true) +} + +func (this *JSONDict) RemoveIgnoreCase(key string) bool { + someRemoved := false + for { + removed := this.remove(key, false) + if !removed { + break + } + if !someRemoved { + someRemoved = true + } + } + return someRemoved +} + +func (this *JSONDict) remove(key string, caseSensitive bool) bool { _, rk, ok := dictGet(this.data, key, caseSensitive) if ok { delete(this.data, rk) diff --git a/vendor/github.com/yunionio/jsonutils/length.go b/vendor/github.com/yunionio/jsonutils/length.go new file mode 100644 index 0000000000..d47009cc41 --- /dev/null +++ b/vendor/github.com/yunionio/jsonutils/length.go @@ -0,0 +1,13 @@ +package jsonutils + +func (this *JSONString) Length() int { + return len(this.data) +} + +func (this *JSONDict) Length() int { + return len(this.data) +} + +func (this *JSONArray) Length() int { + return len(this.data) +} From 16b80dfefb075f95b86fcaeec0da827c84ef04d5 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 8 Aug 2018 13:47:56 +0000 Subject: [PATCH 3/5] cloudcommon: db: make master, slave id explict in joint resource base --- pkg/cloudcommon/db/interface.go | 2 +- pkg/cloudcommon/db/jointbase.go | 7 +++---- pkg/compute/models/hostwires.go | 11 +++++++++-- pkg/compute/models/storagecachedimages.go | 11 ++++++++++- 4 files changed, 23 insertions(+), 8 deletions(-) diff --git a/pkg/cloudcommon/db/interface.go b/pkg/cloudcommon/db/interface.go index fee8f2bf21..e9de71647c 100644 --- a/pkg/cloudcommon/db/interface.go +++ b/pkg/cloudcommon/db/interface.go @@ -116,7 +116,7 @@ type IJointModelManager interface { GetMasterManager() IStandaloneModelManager GetSlaveManager() IStandaloneModelManager - FetchByIds(id1 string, id2 string) (IJointModel, error) + FetchByIds(masterId string, slaveId string) (IJointModel, error) AllowListDescendent(ctx context.Context, userCred mcclient.TokenCredential, model IStandaloneModel, query jsonutils.JSONObject) bool AllowAttach(ctx context.Context, userCred mcclient.TokenCredential, master IStandaloneModel, slave IStandaloneModel) bool diff --git a/pkg/cloudcommon/db/jointbase.go b/pkg/cloudcommon/db/jointbase.go index e1cfb323bb..ed1a0e7156 100644 --- a/pkg/cloudcommon/db/jointbase.go +++ b/pkg/cloudcommon/db/jointbase.go @@ -65,7 +65,7 @@ func (manager *SJointResourceBaseManager) SlaveField(q *sqlchemy.SQuery) sqlchem return queryField(q, manager.GetSlaveManager()) } -func (manager *SJointResourceBaseManager) FetchByIds(id1 string, id2 string) (IJointModel, error) { +func (manager *SJointResourceBaseManager) FetchByIds(masterId string, slaveId string) (IJointModel, error) { obj, err := NewModelObject(manager) if err != nil { return nil, err @@ -83,9 +83,8 @@ func (manager *SJointResourceBaseManager) FetchByIds(id1 string, id2 string) (IJ if slaveField == nil { return nil, fmt.Errorf("cannot find slave id") } - cond1 := sqlchemy.AND(sqlchemy.Equals(masterField, id1), sqlchemy.Equals(slaveField, id2)) - cond2 := sqlchemy.AND(sqlchemy.Equals(slaveField, id1), sqlchemy.Equals(masterField, id2)) - q = q.Filter(sqlchemy.OR(cond1, cond2)) + cond := sqlchemy.AND(sqlchemy.Equals(masterField, masterId), sqlchemy.Equals(slaveField, slaveId)) + q = q.Filter(cond) count := q.Count() if count > 1 { return nil, sqlchemy.ErrDuplicateEntry diff --git a/pkg/compute/models/hostwires.go b/pkg/compute/models/hostwires.go index c9b684f9eb..e2e25bf009 100644 --- a/pkg/compute/models/hostwires.go +++ b/pkg/compute/models/hostwires.go @@ -18,8 +18,15 @@ var HostwireManager *SHostwireManager func init() { db.InitManager(func() { - HostwireManager = &SHostwireManager{SHostJointsManager: NewHostJointsManager(SHostwire{}, - "hostwires_tbl", "hostwire", "hostwires", WireManager)} + HostwireManager = &SHostwireManager{ + SHostJointsManager: NewHostJointsManager( + SHostwire{}, + "hostwires_tbl", + "hostwire", + "hostwires", + WireManager, + ), + } }) } diff --git a/pkg/compute/models/storagecachedimages.go b/pkg/compute/models/storagecachedimages.go index c40282b28b..26f706bbac 100644 --- a/pkg/compute/models/storagecachedimages.go +++ b/pkg/compute/models/storagecachedimages.go @@ -35,7 +35,16 @@ var StoragecachedimageManager *SStoragecachedimageManager func init() { db.InitManager(func() { - StoragecachedimageManager = &SStoragecachedimageManager{SJointResourceBaseManager: db.NewJointResourceBaseManager(SStoragecachedimage{}, "storagecachedimages_tbl", "storagecachedimage", "storagecachedimages", StoragecacheManager, CachedimageManager)} + StoragecachedimageManager = &SStoragecachedimageManager{ + SJointResourceBaseManager: db.NewJointResourceBaseManager( + SStoragecachedimage{}, + "storagecachedimages_tbl", + "storagecachedimage", + "storagecachedimages", + StoragecacheManager, + CachedimageManager, + ), + } }) } From c39644ba4a58cebc9ab6d5756a8ddb1c7efee13c Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Wed, 8 Aug 2018 13:49:13 +0000 Subject: [PATCH 4/5] cloudcommon: db: rename argument name --- pkg/cloudcommon/db/jointbase.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/cloudcommon/db/jointbase.go b/pkg/cloudcommon/db/jointbase.go index ed1a0e7156..0277563390 100644 --- a/pkg/cloudcommon/db/jointbase.go +++ b/pkg/cloudcommon/db/jointbase.go @@ -126,14 +126,14 @@ func (joint *SJointResourceBase) GetJointModelManager() IJointModelManager { return joint.SResourceBase.GetModelManager().(IJointModelManager) } -func getFieldValue(joint IJointModel, name1 string, name2 string) string { +func getFieldValue(joint IJointModel, keyword string, alias string) string { jointValue := reflect.Indirect(reflect.ValueOf(joint)) - idStr, find := reflectutils.FindStructFieldInterface(jointValue, fmt.Sprintf("%s_id", name1)) - if find { + idStr, ok := reflectutils.FindStructFieldInterface(jointValue, fmt.Sprintf("%s_id", keyword)) + if ok { return idStr.(string) } - idStr, find = reflectutils.FindStructFieldInterface(jointValue, fmt.Sprintf("%s_id", name2)) - if find { + idStr, ok = reflectutils.FindStructFieldInterface(jointValue, fmt.Sprintf("%s_id", alias)) + if ok { return idStr.(string) } return "" From f76609e91da3f668700b9ed7535e502bbf712401 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 9 Aug 2018 04:41:23 +0000 Subject: [PATCH 5/5] dep: remove constraint on github.com/coreos/etcd It's not used and dep ensure will not pull it into vendor/ --- Gopkg.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index 40b7a9bfdb..8a280cd277 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -69,10 +69,6 @@ name = "github.com/ugorji/go" revision = "f3cacc17c85ecb7f1b6a9e373ee85d1480919868" -[[constraint]] - name = "github.com/coreos/etcd" - version = "3.3.5" - [[constraint]] name = "github.com/coredns/coredns" version = "v1.2.0"