From 7f0d75cfa1f1e1267391a09d818c13bc06f9036d Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 16 Aug 2018 12:50:11 +0000 Subject: [PATCH 1/2] scheduler: fix wrong import path --- pkg/scheduler/algorithm/predicates/guest/network_predicate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go index e78ba0d99c..bd6ece34bf 100644 --- a/pkg/scheduler/algorithm/predicates/guest/network_predicate.go +++ b/pkg/scheduler/algorithm/predicates/guest/network_predicate.go @@ -9,8 +9,8 @@ import ( "yunion.io/x/onecloud/pkg/scheduler/api" "yunion.io/x/onecloud/pkg/scheduler/core" networks "yunion.io/x/onecloud/pkg/scheduler/db/models" + "yunion.io/x/pkg/util/sets" "yunion.io/x/pkg/utils" - "yunion.io/yunioncloud/pkg/util/sets" ) // NetworkPredicate will filter the current network information with From f65acc402b999fcbbca8c53d6f17080f0a0b5583 Mon Sep 17 00:00:00 2001 From: Yousong Zhou Date: Thu, 16 Aug 2018 13:06:31 +0000 Subject: [PATCH 2/2] dep: update for recent changes - Gopkg.toml: restore constraint on yunion.io/x/pkg to master branch - treewide: convert optional:bool to positional:!bool --- Gopkg.lock | 11 +- Gopkg.toml | 4 +- cmd/climc/shell/dnsrecords.go | 16 +-- cmd/climc/shell/servers.go | 2 +- vendor/yunion.io/x/jsonutils/marshal.go | 5 +- vendor/yunion.io/x/jsonutils/unmarshal.go | 59 +++++++-- vendor/yunion.io/x/pkg/gotypes/serialize.go | 30 ++--- .../x/pkg/util/reflectutils/reflectutils.go | 20 ++- .../yunion.io/x/pkg/util/secrules/secrules.go | 36 ++++-- vendor/yunion.io/x/sqlchemy/column.go | 7 +- vendor/yunion.io/x/sqlchemy/insert.go | 33 +++-- vendor/yunion.io/x/sqlchemy/parser.go | 15 +-- vendor/yunion.io/x/sqlchemy/query.go | 1 - vendor/yunion.io/x/sqlchemy/reflect.go | 31 +++-- vendor/yunion.io/x/sqlchemy/sync.go | 4 +- vendor/yunion.io/x/sqlchemy/table.go | 23 ++-- vendor/yunion.io/x/sqlchemy/tablealias.go | 8 +- vendor/yunion.io/x/sqlchemy/update.go | 6 +- vendor/yunion.io/x/structarg/structarg.go | 119 +++++++++++------- 19 files changed, 265 insertions(+), 165 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index eb4115111f..8323a741ba 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -844,7 +844,7 @@ branch = "master" name = "yunion.io/x/jsonutils" packages = ["."] - revision = "a86e16793be4a6f4c5ea9203e40c2a0a9361efb8" + revision = "41e805b221e8fcd9435b706b4dd473df3dd8cff4" [[projects]] branch = "master" @@ -856,6 +856,7 @@ revision = "a10b94c5038480920262e0f344656328434955a2" [[projects]] + branch = "master" name = "yunion.io/x/pkg" packages = [ "gotypes", @@ -889,23 +890,23 @@ "util/workqueue", "utils" ] - revision = "07cad0cb1df5155b3bfa15079594d01050931554" + revision = "30cdcc2f0c2ca0f3eef573c27eb7350bdd95a7dd" [[projects]] branch = "master" name = "yunion.io/x/sqlchemy" packages = ["."] - revision = "f6a47145f9b94ebff1517017e790f068698b5b5a" + revision = "8d8043b1fc6f8a9192f1d086520bb167a8423475" [[projects]] branch = "master" name = "yunion.io/x/structarg" packages = ["."] - revision = "5f84a68c40a2d8eb923f0607f809cbda8d5b944a" + revision = "adf929ce0f8bd62335ebe57d9b51f023c84e4d7a" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "cff7836c426e8d0c9f6d7220994a2fbcd8b3fa2f4604e1aba3e9a0f50a0da98f" + inputs-digest = "8b3cd1b8309273551ebd55dfb0cadeaaad0aecc1b3a94d680a4a1769d2f42512" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 3343180803..007f3aec01 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -34,7 +34,6 @@ name = "yunion.io/x/jsonutils" [[constraint]] - #revision = "a14641a90b9e5c72868b7a8358d62ea1d83eab53" branch = "master" name = "yunion.io/x/sqlchemy" @@ -43,8 +42,7 @@ name = "yunion.io/x/log" [[constraint]] - revision = "07cad0cb1df5155b3bfa15079594d01050931554" - #branch = "master" + branch = "master" name = "yunion.io/x/pkg" [[constraint]] diff --git a/cmd/climc/shell/dnsrecords.go b/cmd/climc/shell/dnsrecords.go index 45b7c923ff..05185894ef 100644 --- a/cmd/climc/shell/dnsrecords.go +++ b/cmd/climc/shell/dnsrecords.go @@ -10,12 +10,12 @@ import ( ) type DNSRecordOptions struct { - A []string `help:"DNS A record" metavar:"A_RECORD" optional:"true"` - AAAA []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" optional:"true"` - CNAME string `help:"DNS CNAME record" metavar:"CNAME_RECORD" optional:"true"` - SRVHost string `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" optional:"true"` - SRVPort int64 `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" optional:"true"` - PTR string `help:"DNS PTR record" metavar:"PTR_RECORD" optional:"true"` + A []string `help:"DNS A record" metavar:"A_RECORD" positional:"false"` + AAAA []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" positional:"false"` + CNAME string `help:"DNS CNAME record" metavar:"CNAME_RECORD" positional:"false"` + SRVHost string `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" positional:"false"` + SRVPort int64 `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" positional:"false"` + PTR string `help:"DNS PTR record" metavar:"PTR_RECORD" positional:"false"` } func parseDNSRecords(args *DNSRecordOptions, params *jsonutils.JSONDict) { @@ -52,7 +52,7 @@ func init() { type DNSCreateOptions struct { NAME string `help:"DNS name to create"` - TTL int64 `help:"TTL in seconds" optional:"true"` + TTL int64 `help:"TTL in seconds" positional:"false"` Desc string `help:"Description" metavar:"DESCRIPTION"` DNSRecordOptions } @@ -92,7 +92,7 @@ func init() { type DNSUpdateOptions struct { ID string `help:"ID of DNS record to update"` Name string `help:"Domain name"` - TTL int64 `help:"TTL in seconds" optional:"true"` + TTL int64 `help:"TTL in seconds" positional:"false"` Desc string `help:"Description"` } R(&DNSUpdateOptions{}, "dns-update", "Update details of a dns records", func(s *mcclient.ClientSession, args *DNSUpdateOptions) error { diff --git a/cmd/climc/shell/servers.go b/cmd/climc/shell/servers.go index e6bccf2e57..999afdfd5a 100644 --- a/cmd/climc/shell/servers.go +++ b/cmd/climc/shell/servers.go @@ -511,7 +511,7 @@ func init() { type ServerSecGroupOptions struct { ID string `help:"ID or Name of server" metavar:"Guest"` - SecGrp string `help:"ID of Security Group" metavar:"Security Group" optional:"false"` + SecGrp string `help:"ID of Security Group" metavar:"Security Group" positional:"true"` } R(&ServerSecGroupOptions{}, "server-assign-secgroup", "Assign security group to a VM", func(s *mcclient.ClientSession, args *ServerSecGroupOptions) error { diff --git a/vendor/yunion.io/x/jsonutils/marshal.go b/vendor/yunion.io/x/jsonutils/marshal.go index bedce745e9..aff79cf1d2 100644 --- a/vendor/yunion.io/x/jsonutils/marshal.go +++ b/vendor/yunion.io/x/jsonutils/marshal.go @@ -12,8 +12,8 @@ import ( "reflect" "time" - "yunion.io/x/pkg/gotypes" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/util/timeutils" @@ -58,6 +58,9 @@ func struct2JSONPairs(val reflect.Value) []JSONPair { objPairs = append(objPairs, newPairs...) } else { key := reflectutils.GetStructFieldName(&fieldType) // utils.CamelSplit(fieldType.Name, "_") + if key == "" { + continue + } val := marshalValue(val.Field(i)) if val != JSONNull { objPair := JSONPair{key: key, val: val} diff --git a/vendor/yunion.io/x/jsonutils/unmarshal.go b/vendor/yunion.io/x/jsonutils/unmarshal.go index 3a98a40490..e1f3b62144 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal.go @@ -47,8 +47,6 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error { } func (this *JSONValue) unmarshalValue(val reflect.Value) error { - // return fmt.Errorf("JSONValue: type mismatch") - // null value if val.CanSet() { zeroVal := reflect.New(val.Type()).Elem() val.Set(zeroVal) @@ -102,6 +100,11 @@ func (this *JSONInt) unmarshalValue(val reflect.Value) error { } case reflect.String: val.SetString(fmt.Sprintf("%d", this.data)) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONInt type mismatch: %s", val.Type()) } @@ -162,6 +165,11 @@ func (this *JSONBool) unmarshalValue(val reflect.Value) error { } else { val.SetString("false") } + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONBool type mismatch: %s", val.Type()) } @@ -231,6 +239,11 @@ func (this *JSONFloat) unmarshalValue(val reflect.Value) error { } case reflect.String: val.SetString(fmt.Sprintf("%f", this.data)) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONFloat type mismatch: %s", val.Type()) } @@ -314,6 +327,11 @@ func (this *JSONString) unmarshalValue(val reflect.Value) error { val.SetBool(utils.ToBool(this.data)) case reflect.String: val.SetString(this.data) + case reflect.Ptr: + if val.IsNil() { + val.Set(reflect.New(val.Type().Elem())) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONString type mismatch: %s", val.Type()) } @@ -339,15 +357,32 @@ func (this *JSONArray) unmarshalValue(val reflect.Value) error { case reflect.String: val.SetString(this.String()) return nil + case reflect.Ptr: + if val.IsNil() { + kind := val.Type().Elem().Kind() + if kind == reflect.Array || kind == reflect.Slice { + val.Set(reflect.New(val.Type().Elem())) + return this.unmarshalValue(val.Elem()) + } + } + return fmt.Errorf("JSONArray type mismatch %s", val.Type()) case reflect.Slice, reflect.Array: - for _, json := range this.data { - newEle := reflect.New(val.Type().Elem()).Elem() - err := json.unmarshalValue(newEle) + if val.Kind() == reflect.Array { + if val.Len() != len(this.data) { + return fmt.Errorf("JSONArray length unmatch %s: %d != %d", + val.Type(), val.Len(), len(this.data)) + } + } else if val.Kind() == reflect.Slice { + if val.Len() < len(this.data) { + newVal := reflect.MakeSlice(val.Type(), len(this.data), len(this.data)) + val.Set(newVal) + } + } + for i, json := range this.data { + err := json.unmarshalValue(val.Index(i)) if err != nil { return err } - newVal := reflect.Append(val, newEle) - val.Set(newVal) } default: return fmt.Errorf("JSONArray type mismatch: %s", val.Type()) @@ -376,6 +411,16 @@ func (this *JSONDict) unmarshalValue(val reflect.Value) error { return this.unmarshalMap(val) case reflect.Struct: return this.unmarshalStruct(val) + case reflect.Ptr: + if val.IsNil() { + kind := val.Type().Elem().Kind() + if kind != reflect.Struct && kind != reflect.Map { + return fmt.Errorf("JSONDict type mismatch: %s", val.Type()) + } + newVal := reflect.New(val.Type().Elem()) + val.Set(newVal) + } + return this.unmarshalValue(val.Elem()) default: return fmt.Errorf("JSONDict type mismatch: %s", val.Type()) } diff --git a/vendor/yunion.io/x/pkg/gotypes/serialize.go b/vendor/yunion.io/x/pkg/gotypes/serialize.go index 8cb8c41d07..fdbc271497 100644 --- a/vendor/yunion.io/x/pkg/gotypes/serialize.go +++ b/vendor/yunion.io/x/pkg/gotypes/serialize.go @@ -14,30 +14,32 @@ type ISerializable interface { type FuncSerializableAllocator func() ISerializable -var ISerializableType reflect.Type -var serializableAllocators map[reflect.Type]FuncSerializableAllocator -var ErrTypeNotSerializable error - -func init() { - ISerializableType = reflect.TypeOf((*ISerializable)(nil)).Elem() - serializableAllocators = make(map[reflect.Type]FuncSerializableAllocator) +var ( + ISerializableType = reflect.TypeOf((*ISerializable)(nil)).Elem() + serializableAllocators = map[reflect.Type]FuncSerializableAllocator{} ErrTypeNotSerializable = errors.New("Type not serializable") -} +) +// RegisterSerializable registers an allocator func for the specified serializable type. +// +// This is intended to be used when you have multiple implmenetations of an +// interface and you want to use only one of them to cover them all. +// TokenCredential and SSimpleToken is such a case. func RegisterSerializable(valType reflect.Type, alloc FuncSerializableAllocator) { + if !IsSerializable(valType) { + panic(valType.String() + " does not implement ISerializable") + } serializableAllocators[valType] = alloc } -func IsSerializable(objType reflect.Type) bool { - _, ok := serializableAllocators[objType] - return ok +func IsSerializable(valType reflect.Type) bool { + return valType.Implements(ISerializableType) } func NewSerializable(objType reflect.Type) (ISerializable, error) { deserFunc, ok := serializableAllocators[objType] - if !ok { - return nil, ErrTypeNotSerializable - } else { + if ok { return deserFunc(), nil } + return nil, ErrTypeNotSerializable } diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go index 6b58992180..55d9019d8e 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go @@ -2,6 +2,7 @@ package reflectutils import ( "reflect" + "strings" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/utils" @@ -15,14 +16,23 @@ func GetStructFieldName(field *reflect.StructField) string { return nameStr } else { jsonStr, _ := tagMap["json"] - if len(jsonStr) > 0 { - return jsonStr - } else { - return utils.CamelSplit(field.Name, "_") - } + return toJsonKey(field.Name, jsonStr) } } +func toJsonKey(fieldName, jsonTag string) string { + jsonTag = strings.Replace(jsonTag, "omitempty", "", -1) + words := utils.FindWords([]byte(jsonTag), 0) + if len(words) == 0 { + return utils.CamelSplit(fieldName, "_") + } + name := words[0] + if name == "-" { + return "" + } + return name +} + func FetchStructFieldNameValueInterfaces(dataValue reflect.Value) map[string]interface{} { fields := make(map[string]interface{}) fetchStructFieldNameValueInterfaces(dataValue.Type(), dataValue, fields) diff --git a/vendor/yunion.io/x/pkg/util/secrules/secrules.go b/vendor/yunion.io/x/pkg/util/secrules/secrules.go index 663ee8632f..4f249fdcd6 100644 --- a/vendor/yunion.io/x/pkg/util/secrules/secrules.go +++ b/vendor/yunion.io/x/pkg/util/secrules/secrules.go @@ -25,14 +25,15 @@ const ( ) type SecurityRule struct { - Priority int // [1, 100] - Action TSecurityRuleAction - IPNet *net.IPNet - Protocol string - Direction TSecurityRuleDirection - PortStart int - PortEnd int - Ports []int + Priority int // [1, 100] + Action TSecurityRuleAction + IPNet *net.IPNet + Protocol string + Direction TSecurityRuleDirection + PortStart int + PortEnd int + Ports []int + Description string } const ( @@ -62,6 +63,25 @@ var ( ErrInvalidPort = errors.New("invalid port") ) +type SecurityRuleSet []SecurityRule + +func (v SecurityRuleSet) Len() int { + return len(v) +} + +func (v SecurityRuleSet) Swap(i, j int) { + v[i], v[j] = v[j], v[i] +} + +func (v SecurityRuleSet) Less(i, j int) bool { + if v[i].Priority > v[j].Priority { + return true + } else if v[i].Priority == v[j].Priority { + return strings.Compare(v[i].String(), v[j].String()) <= 0 + } + return false +} + func parsePortString(ps string) (int, error) { p, err := strconv.ParseUint(ps, 10, 16) if err != nil || p == 0 { diff --git a/vendor/yunion.io/x/sqlchemy/column.go b/vendor/yunion.io/x/sqlchemy/column.go index 90ff609a11..8f65bdaf74 100644 --- a/vendor/yunion.io/x/sqlchemy/column.go +++ b/vendor/yunion.io/x/sqlchemy/column.go @@ -7,7 +7,6 @@ import ( "strings" "time" - "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/regutils" @@ -465,11 +464,11 @@ func (c *SDecimalColumn) IsZero(val interface{}) bool { func NewDecimalColumn(name string, tagmap map[string]string) SDecimalColumn { tagmap, v, ok := utils.TagPop(tagmap, TAG_PRECISION) if !ok { - log.Fatalf("Field %s of float should have precision tag", name) + panic(fmt.Sprintf("Field %q of float misses precision tag", name)) } prec, err := strconv.Atoi(v) if err != nil { - log.Fatalf("Field %s of float precision %s shoud be integer!", name, v) + panic(fmt.Sprintf("Field precision of %q shoud be integer (%q)", name, v)) } return SDecimalColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "DECIMAL", tagmap), Precision: prec} @@ -544,7 +543,7 @@ func NewTextColumn(name string, tagmap map[string]string) STextColumn { if len(charset) == 0 { charset = "utf8" } else if charset != "utf8" && charset != "ascii" { - log.Fatalf("Unsupported charset %s for %s", charset, name) + panic(fmt.Sprintf("Unsupported charset %s for %s", charset, name)) } return STextColumn{SBaseWidthColumn: NewBaseWidthColumn(name, sqltype, tagmap), Charset: charset} diff --git a/vendor/yunion.io/x/sqlchemy/insert.go b/vendor/yunion.io/x/sqlchemy/insert.go index 341ce91343..06e3518d81 100644 --- a/vendor/yunion.io/x/sqlchemy/insert.go +++ b/vendor/yunion.io/x/sqlchemy/insert.go @@ -42,7 +42,6 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { dtc, ok := c.(*SDateTimeColumn) ov := fields[k] - // log.Debugf("field %s value %s %s", k, ov, ov==nil) if ok && (dtc.IsCreatedAt || dtc.IsUpdatedAt) { createdAtFields = append(createdAtFields, k) names = append(names, fmt.Sprintf("`%s`", k)) @@ -54,15 +53,13 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { format = append(format, "?") } else if c.IsPrimary() { if isAutoInc { - if len(autoIncField) == 0 { - autoIncField = k - } else { - log.Fatalf("multiple auto_increment columns???") + if len(autoIncField) > 0 { + panic(fmt.Sprintf("multiple auto_increment columns: %q, %q", autoIncField, k)) } - } else { - return fmt.Errorf("fail to insert for null primary key `%s`", k) + autoIncField = k } - } else if ! c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value + return fmt.Errorf("cannot insert for null primary key %q", k) + } else if !c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value val := c.ConvertFromString(c.Default()) values = append(values, val) names = append(names, fmt.Sprintf("`%s`", k)) @@ -92,15 +89,15 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { } /* - if len(autoIncField) > 0 { - lastId, err := results.LastInsertId() - if err == nil { - val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField) - if ok { - gotypes.SetValue(val, fmt.Sprint(lastId)) + if len(autoIncField) > 0 { + lastId, err := results.LastInsertId() + if err == nil { + val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField) + if ok { + gotypes.SetValue(val, fmt.Sprint(lastId)) + } } } - } */ // query the value, so default value can be feedback into the object @@ -112,7 +109,7 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { if ok && nc.IsAutoIncrement { lastId, err := results.LastInsertId() if err != nil { - log.Errorf("Fail to fetch lastInsertId %s", err) + err := fmt.Errorf("fetching lastInsertId failed: %v", err) return err } else { q = q.Equals(c.Name(), lastId) @@ -124,9 +121,9 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error { } err = q.First(dt) if err != nil { - log.Errorf("query after insert failed %s", err) + err := fmt.Errorf("query after insert failed: %v", err) return err } - + return nil } diff --git a/vendor/yunion.io/x/sqlchemy/parser.go b/vendor/yunion.io/x/sqlchemy/parser.go index 2464f9c82c..c66f38bf7b 100644 --- a/vendor/yunion.io/x/sqlchemy/parser.go +++ b/vendor/yunion.io/x/sqlchemy/parser.go @@ -3,14 +3,13 @@ package sqlchemy import ( "reflect" - "yunion.io/x/log" "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/tristate" "yunion.io/x/pkg/util/reflectutils" "yunion.io/x/pkg/utils" ) -func fieldToColumnSpec(field *reflect.StructField) IColumnSpec { +func structField2ColumnSpec(field *reflect.StructField) IColumnSpec { fieldname := reflectutils.GetStructFieldName(field) tagmap := utils.TagMap(field.Tag) if _, ok := tagmap[TAG_IGNORE]; ok { @@ -85,13 +84,11 @@ func fieldToColumnSpec(field *reflect.StructField) IColumnSpec { col := NewJSONColumn(fieldname, tagmap) return &col*/ default: - if gotypes.IsSerializable(field.Type) { + if field.Type.Implements(gotypes.ISerializableType) { col := NewCompoundColumn(fieldname, tagmap) return &col - } else { - log.Fatalf("Unsupported type! %s", field.Type) - return nil } + panic("not supported type %s" + field.Type.Name()) } } @@ -101,9 +98,9 @@ func struct2TableSpec(st reflect.Type, table *STableSpec) { if f.Type.Kind() == reflect.Struct && f.Type != gotypes.TimeType { struct2TableSpec(f.Type, table) } else { - coldef := fieldToColumnSpec(&f) - if coldef != nil { - table.columns = append(table.columns, coldef) + column := structField2ColumnSpec(&f) + if column != nil { + table.columns = append(table.columns, column) } } } diff --git a/vendor/yunion.io/x/sqlchemy/query.go b/vendor/yunion.io/x/sqlchemy/query.go index 7b0f332e5b..619240006e 100644 --- a/vendor/yunion.io/x/sqlchemy/query.go +++ b/vendor/yunion.io/x/sqlchemy/query.go @@ -126,7 +126,6 @@ func (sq *SSubQuery) Field(id string, alias ...string) IQueryField { return &sqf } } - log.Errorf("field %s not found for subquery", id) return nil } diff --git a/vendor/yunion.io/x/sqlchemy/reflect.go b/vendor/yunion.io/x/sqlchemy/reflect.go index 7c7753a2ad..c85cb63a0e 100644 --- a/vendor/yunion.io/x/sqlchemy/reflect.go +++ b/vendor/yunion.io/x/sqlchemy/reflect.go @@ -25,7 +25,7 @@ func getStringValue(dat interface{}) string { case gotypes.IntType, gotypes.Int8Type, gotypes.Int16Type, gotypes.Int32Type, gotypes.Int64Type: return fmt.Sprintf("%d", value.Int()) case gotypes.UintType, gotypes.Uint8Type, gotypes.Uint16Type, gotypes.Uint32Type, gotypes.Uint64Type: - return fmt.Sprintf("%u", value.Uint()) + return fmt.Sprintf("%d", value.Uint()) case gotypes.Float32Type, gotypes.Float64Type: return fmt.Sprintf("%f", value.Float()) case gotypes.StringType: @@ -53,15 +53,12 @@ func getStringValue(dat interface{}) string { return string(rawBytes) } default: - if gotypes.IsSerializable(value.Type()) { - is, ok := value.Interface().(gotypes.ISerializable) - if !ok { - log.Errorf("%s fail to convert to JSONObject", value) - } else { - return is.String() - } + serializable, ok := value.Interface().(gotypes.ISerializable) + if !ok { + log.Errorf("cannot convert %v to string", value) + return "" } - log.Errorf("Cannot convert to string %s", value) + return serializable.String() } return "" } @@ -127,15 +124,15 @@ func setValueBySQLString(value reflect.Value, val string) error { gotypes.Float32SliceType, gotypes.Float64SliceType, gotypes.StringSliceType: reflect.Append(value, reflect.ValueOf(val)) default: - if gotypes.IsSerializable(value.Type()) { - is, err := jsonutils.JSONDeserialize(value.Type(), val) - if err != nil { - return err - } - value.Set(reflect.ValueOf(is)) - } else { - return fmt.Errorf("!!!Unsupported type: %s", value.Type) + valueType := value.Type() + if !valueType.Implements(gotypes.ISerializableType) { + return fmt.Errorf("not supported type: %s", valueType) } + serializable, err := jsonutils.JSONDeserialize(valueType, val) + if err != nil { + return err + } + value.Set(reflect.ValueOf(serializable)) } return nil } diff --git a/vendor/yunion.io/x/sqlchemy/sync.go b/vendor/yunion.io/x/sqlchemy/sync.go index 07b3df8a81..d6343129f1 100644 --- a/vendor/yunion.io/x/sqlchemy/sync.go +++ b/vendor/yunion.io/x/sqlchemy/sync.go @@ -212,10 +212,10 @@ func (ts *STableSpec) Sync() error { func (ts *STableSpec) CheckSync() { sqls := ts.SyncSQL() - if sqls != nil && len(sqls) > 0 { + if len(sqls) > 0 { for _, sql := range sqls { fmt.Println(sql) } - log.Fatalf("DB not in sync!!!") + log.Fatalf("DB table %q not in sync", ts.name) } } diff --git a/vendor/yunion.io/x/sqlchemy/table.go b/vendor/yunion.io/x/sqlchemy/table.go index 3f9813f5de..74eedc5a51 100644 --- a/vendor/yunion.io/x/sqlchemy/table.go +++ b/vendor/yunion.io/x/sqlchemy/table.go @@ -5,7 +5,6 @@ import ( "reflect" "strings" - "yunion.io/x/log" "yunion.io/x/pkg/utils" ) @@ -29,12 +28,15 @@ type STableField struct { func NewTableSpecFromStruct(s interface{}, name string) *STableSpec { st := reflect.TypeOf(s) if st.Kind() != reflect.Struct { - log.Fatalf("Invalid table struct, NOT a STRUCT!!!") - return nil + panic("expect Struct kind") } - table := STableSpec{columns: make([]IColumnSpec, 0), name: name, structType: st} - struct2TableSpec(st, &table) - return &table + table := &STableSpec{ + columns: []IColumnSpec{}, + name: name, + structType: st, + } + struct2TableSpec(st, table) + return table } func (ts *STableSpec) Name() string { @@ -88,12 +90,11 @@ func (ts *STableSpec) ColumnSpec(name string) IColumnSpec { func (tbl *STable) Field(name string, alias ...string) IQueryField { // name = reflectutils.StructFieldName(name) name = utils.CamelSplit(name, "_") - cSpec := tbl.spec.ColumnSpec(name) - if cSpec == nil { - log.Fatalf("Column %s not found", name) - return nil + spec := tbl.spec.ColumnSpec(name) + if spec == nil { + panic("column not found: " + name) } - col := STableField{table: tbl, spec: cSpec} + col := STableField{table: tbl, spec: spec} if len(alias) > 0 { col.Label(alias[0]) } diff --git a/vendor/yunion.io/x/sqlchemy/tablealias.go b/vendor/yunion.io/x/sqlchemy/tablealias.go index 47898e4ba6..eec3c89d78 100644 --- a/vendor/yunion.io/x/sqlchemy/tablealias.go +++ b/vendor/yunion.io/x/sqlchemy/tablealias.go @@ -6,14 +6,10 @@ import ( ) var ( - tableID = 0 - tableIDLock *sync.Mutex + tableID = 0 + tableIDLock *sync.Mutex = &sync.Mutex{} ) -func init() { - tableIDLock = &sync.Mutex{} -} - func getTableAliasName() string { tableIDLock.Lock() defer tableIDLock.Unlock() diff --git a/vendor/yunion.io/x/sqlchemy/update.go b/vendor/yunion.io/x/sqlchemy/update.go index b651f338aa..73cd0b8dfc 100644 --- a/vendor/yunion.io/x/sqlchemy/update.go +++ b/vendor/yunion.io/x/sqlchemy/update.go @@ -64,9 +64,9 @@ func UpdateDiffString(diff map[string]SUpdateDiff) string { } func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, error) { - beforeInsertFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate") - if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() { - beforeInsertFunc.Call([]reflect.Value{}) + beforeUpdateFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate") + if beforeUpdateFunc.IsValid() && !beforeUpdateFunc.IsNil() { + beforeUpdateFunc.Call([]reflect.Value{}) } // dataType := reflect.TypeOf(dt).Elem() diff --git a/vendor/yunion.io/x/structarg/structarg.go b/vendor/yunion.io/x/structarg/structarg.go index 2f64914771..7e4c9193d5 100644 --- a/vendor/yunion.io/x/structarg/structarg.go +++ b/vendor/yunion.io/x/structarg/structarg.go @@ -27,8 +27,8 @@ type Argument interface { AliasToken() string ShortToken() string MetaVar() string - IsOptional() bool IsPositional() bool + IsRequired() bool IsMulti() bool IsSubcommand() bool HelpString(indent string) string @@ -45,8 +45,8 @@ type SingleArgument struct { aliasToken string shortToken string metavar string - optional bool positional bool + required bool help string choices []string useDefault bool @@ -136,7 +136,13 @@ const ( A boolean value explicitly declare whether the argument is optional, the tag is optional */ - TAG_OPTIONAL = "optional" + TAG_POSITIONAL = "positional" + /* + A boolean value explicitly declare whether the argument is required. + The tag is optional. This is for optional arguments. Positional + arguments must be "required" + */ + TAG_REQUIRED = "required" /* A boolean value explicitly decalre whther the argument is an subcommand A subcommand argument must be the last positional argument. @@ -263,51 +269,64 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) } } } - var positional, optional bool + // heuristic guessing "positional" + var positional bool if f.Name == strings.ToUpper(f.Name) { positional = true - optional = false } else { positional = false - optional = true } - opt_val := tagMap[TAG_OPTIONAL] - if len(opt_val) > 0 { - if opt_val == "true" { - optional = true - positional = false - } else if opt_val == "false" { - optional = false + if positionalTag := tagMap[TAG_POSITIONAL]; len(positionalTag) > 0 { + switch positionalTag { + case "true": positional = true - } else { - return fmt.Errorf("Invalid optional value %s, neither true nor false", opt_val) + case "false": + positional = false + default: + return fmt.Errorf("Invalid positional tag %q, neither true nor false", positionalTag) } - // fmt.Println(token, "optional", opt_val, optional, positional) } - if positional && !optional && use_default { - return fmt.Errorf("A positional non-optional argument should not set default value") + required := positional + if requiredTag := tagMap[TAG_REQUIRED]; len(requiredTag) > 0 { + switch requiredTag { + case "true": + required = true + case "false": + required = false + default: + return fmt.Errorf("Invalid required tag %q, neither true nor false", requiredTag) + } } - subcommand, e := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) - if e != nil { + if positional { + if !required { + return fmt.Errorf("positional %s must not have required:false", token) + } + if use_default { + return fmt.Errorf("positional %s must not have default value", token) + } + } + subcommand, err := strconv.ParseBool(tagMap[TAG_SUBCOMMAND]) + if err != nil { subcommand = false } var defval_t reflect.Value if use_default { - defval_t, e = gotypes.ParseValue(defval, f.Type) - if e != nil { - return e + defval_t, err = gotypes.ParseValue(defval, f.Type) + if err != nil { + return err } } if subcommand { positional = true - optional = false } var arg Argument = nil ovalue := reflect.New(v.Type()).Elem() ovalue.Set(v) sarg := SingleArgument{token: token, shortToken: shorttoken, - optional: optional, positional: positional, - metavar: metavar, help: help, + positional: positional, + required: required, + metavar: metavar, + help: help, choices: choices, useDefault: use_default, aliasToken: alias, @@ -321,7 +340,7 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) subcommands: make(map[string]SubcommandArgumentData)} } else if f.Type.Kind() == reflect.Array || f.Type.Kind() == reflect.Slice { var min, max int64 - var e error + var err error nargs := tagMap[TAG_NARGS] if nargs == "*" { min = 0 @@ -333,13 +352,13 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) min = 1 max = -1 } else { - min, e = strconv.ParseInt(nargs, 10, 64) - if e == nil { + min, err = strconv.ParseInt(nargs, 10, 64) + if err == nil { max = min } else if positional { min = 1 max = -1 - } else if optional { + } else if !required { min = 0 max = -1 } @@ -349,7 +368,11 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value) } else { arg = &sarg } - return this.AddArgument(arg) + err = this.AddArgument(arg) + if err != nil { + return fmt.Errorf("AddArgument %s: %v", arg, err) + } + return nil } func (this *ArgumentParser) AddArgument(arg Argument) error { @@ -361,13 +384,25 @@ func (this *ArgumentParser) AddArgument(arg Argument) error { return fmt.Errorf("Cannot append positional argument after an array positional argument") case last_arg.IsSubcommand(): return fmt.Errorf("Cannot append positional argument after a subcommand argument") - case last_arg.IsOptional() && !arg.IsOptional(): - return fmt.Errorf("Cannot append positional argument after an optional positional argument") } } this.posArgs = append(this.posArgs, arg) } else { - this.optArgs = append(this.optArgs, arg) + // Put required at the end and try to be stable + if arg.IsRequired() { + this.optArgs = append(this.optArgs, arg) + } else { + var i int + var opt Argument + for i, opt = range this.optArgs { + if opt.IsRequired() { + break + } + } + this.optArgs = append(this.optArgs, nil) + copy(this.optArgs[i+1:], this.optArgs[i:]) + this.optArgs[i] = arg + } } return nil } @@ -432,12 +467,12 @@ func (this *SingleArgument) ShortToken() string { func (this *SingleArgument) String() string { var start, end byte - if this.IsOptional() { - start = '[' - end = ']' - } else { + if this.IsRequired() { start = '<' end = '>' + } else { + start = '[' + end = ']' } if this.IsPositional() { return fmt.Sprintf("%c%s%c", start, this.MetaVar(), end) @@ -450,8 +485,8 @@ func (this *SingleArgument) String() string { } } -func (this *SingleArgument) IsOptional() bool { - return this.optional +func (this *SingleArgument) IsRequired() bool { + return this.required } func (this *SingleArgument) IsPositional() bool { @@ -519,7 +554,7 @@ func (this *SingleArgument) SetDefault() { } func (this *SingleArgument) Validate() error { - if !this.optional && !this.isSet && !this.useDefault { + if this.required && !this.isSet && !this.useDefault { return fmt.Errorf("Non-optional argument %s not set", this.token) } return nil @@ -818,7 +853,7 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error } } } - if err == nil && pos_idx < len(this.posArgs) && !this.posArgs[pos_idx].IsOptional() { + if err == nil && pos_idx < len(this.posArgs) { err = fmt.Errorf("Not enough arguments, missing %s", this.posArgs[pos_idx]) } if err == nil {