Merge pull request #67 in YUNIONIO/onecloud from ~QIUJIAN/onecloud:hotfix/qj-resolve-conflict-2.1.0-20180817 to release/2.1.0

* commit '9d2090962515cb6c7df4d3929f015d1d11667809':
  dep: update for recent changes
  scheduler: fix wrong import path

Gopkg.lock

@@ -850,7 +850,7 @@
   branch = "master"
   name = "yunion.io/x/jsonutils"
   packages = ["."]
-  revision = "a86e16793be4a6f4c5ea9203e40c2a0a9361efb8"
+  revision = "41e805b221e8fcd9435b706b4dd473df3dd8cff4"
 
 [[projects]]
   branch = "master"
@@ -862,6 +862,7 @@
   revision = "a10b94c5038480920262e0f344656328434955a2"
 
 [[projects]]
+  branch = "master"
   name = "yunion.io/x/pkg"
   packages = [
     "gotypes",
@@ -895,23 +896,23 @@
     "util/workqueue",
     "utils"
   ]
-  revision = "07cad0cb1df5155b3bfa15079594d01050931554"
+  revision = "30cdcc2f0c2ca0f3eef573c27eb7350bdd95a7dd"
 
 [[projects]]
   branch = "master"
   name = "yunion.io/x/sqlchemy"
   packages = ["."]
-  revision = "f6a47145f9b94ebff1517017e790f068698b5b5a"
+  revision = "8d8043b1fc6f8a9192f1d086520bb167a8423475"
 
 [[projects]]
   branch = "master"
   name = "yunion.io/x/structarg"
   packages = ["."]
-  revision = "5f84a68c40a2d8eb923f0607f809cbda8d5b944a"
+  revision = "adf929ce0f8bd62335ebe57d9b51f023c84e4d7a"
 
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "63315737a1e02fba65e6ba256ff2571dfb9351af5fe446152e89adaf1c7129fb"
+  inputs-digest = "8b3cd1b8309273551ebd55dfb0cadeaaad0aecc1b3a94d680a4a1769d2f42512"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -34,7 +34,6 @@
   name = "yunion.io/x/jsonutils"
 
 [[constraint]]
-  #revision = "a14641a90b9e5c72868b7a8358d62ea1d83eab53"
   branch = "master"
   name = "yunion.io/x/sqlchemy"
 
@@ -43,8 +42,7 @@
   name = "yunion.io/x/log"
 
 [[constraint]]
-  revision = "07cad0cb1df5155b3bfa15079594d01050931554"
-  #branch = "master"
+  branch = "master"
   name = "yunion.io/x/pkg"
 
 [[constraint]]

cmd/climc/shell/dnsrecords.go

@@ -10,12 +10,12 @@ import (
 )
 
 type DNSRecordOptions struct {
-	A       []string `help:"DNS A record" metavar:"A_RECORD" optional:"true"`
-	AAAA    []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" optional:"true"`
-	CNAME   string   `help:"DNS CNAME record" metavar:"CNAME_RECORD" optional:"true"`
-	SRVHost string   `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" optional:"true"`
-	SRVPort int64    `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" optional:"true"`
-	PTR     string   `help:"DNS PTR record" metavar:"PTR_RECORD" optional:"true"`
+	A       []string `help:"DNS A record" metavar:"A_RECORD" positional:"false"`
+	AAAA    []string `help:"DNS AAAA record" metavar:"AAAA_RECORD" positional:"false"`
+	CNAME   string   `help:"DNS CNAME record" metavar:"CNAME_RECORD" positional:"false"`
+	SRVHost string   `help:"DNS SRV record, server of service" metavar:"SRV_RECORD_HOST" positional:"false"`
+	SRVPort int64    `help:"DNS SRV record, port of service" metavar:"SRV_RECORD_PORT" positional:"false"`
+	PTR     string   `help:"DNS PTR record" metavar:"PTR_RECORD" positional:"false"`
 }
 
 func parseDNSRecords(args *DNSRecordOptions, params *jsonutils.JSONDict) {
@@ -52,7 +52,7 @@ func init() {
 
 	type DNSCreateOptions struct {
 		NAME string `help:"DNS name to create"`
-		TTL  int64  `help:"TTL in seconds" optional:"true"`
+		TTL  int64  `help:"TTL in seconds" positional:"false"`
 		Desc string `help:"Description" metavar:"DESCRIPTION"`
 		DNSRecordOptions
 	}
@@ -92,7 +92,7 @@ func init() {
 	type DNSUpdateOptions struct {
 		ID   string `help:"ID of DNS record to update"`
 		Name string `help:"Domain name"`
-		TTL  int64  `help:"TTL in seconds" optional:"true"`
+		TTL  int64  `help:"TTL in seconds" positional:"false"`
 		Desc string `help:"Description"`
 	}
 	R(&DNSUpdateOptions{}, "dns-update", "Update details of a dns records", func(s *mcclient.ClientSession, args *DNSUpdateOptions) error {

cmd/climc/shell/servers.go

@@ -527,7 +527,7 @@ func init() {
 
 	type ServerSecGroupOptions struct {
 		ID     string `help:"ID or Name of server" metavar:"Guest"`
-		SecGrp string `help:"ID of Security Group" metavar:"Security Group" optional:"false"`
+		SecGrp string `help:"ID of Security Group" metavar:"Security Group" positional:"true"`
 	}
 
 	R(&ServerSecGroupOptions{}, "server-assign-secgroup", "Assign security group to a VM", func(s *mcclient.ClientSession, args *ServerSecGroupOptions) error {

pkg/scheduler/algorithm/predicates/guest/network_predicate.go

@@ -9,8 +9,8 @@ import (
 	"yunion.io/x/onecloud/pkg/scheduler/api"
 	"yunion.io/x/onecloud/pkg/scheduler/core"
 	networks "yunion.io/x/onecloud/pkg/scheduler/db/models"
+	"yunion.io/x/pkg/util/sets"
 	"yunion.io/x/pkg/utils"
-	"yunion.io/yunioncloud/pkg/util/sets"
 )
 
 // NetworkPredicate will filter the current network information with

vendor/yunion.io/x/jsonutils/marshal.go

@@ -12,8 +12,8 @@ import (
 	"reflect"
 	"time"
 
-	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/log"
+	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/reflectutils"
 	"yunion.io/x/pkg/util/timeutils"
@@ -58,6 +58,9 @@ func struct2JSONPairs(val reflect.Value) []JSONPair {
 			objPairs = append(objPairs, newPairs...)
 		} else {
 			key := reflectutils.GetStructFieldName(&fieldType) // utils.CamelSplit(fieldType.Name, "_")
+			if key == "" {
+				continue
+			}
 			val := marshalValue(val.Field(i))
 			if val != JSONNull {
 				objPair := JSONPair{key: key, val: val}

vendor/yunion.io/x/jsonutils/unmarshal.go

@@ -47,8 +47,6 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error {
 }
 
 func (this *JSONValue) unmarshalValue(val reflect.Value) error {
-	// return fmt.Errorf("JSONValue: type mismatch")
-	// null value
 	if val.CanSet() {
 		zeroVal := reflect.New(val.Type()).Elem()
 		val.Set(zeroVal)
@@ -102,6 +100,11 @@ func (this *JSONInt) unmarshalValue(val reflect.Value) error {
 		}
 	case reflect.String:
 		val.SetString(fmt.Sprintf("%d", this.data))
+	case reflect.Ptr:
+		if val.IsNil() {
+			val.Set(reflect.New(val.Type().Elem()))
+		}
+		return this.unmarshalValue(val.Elem())
 	default:
 		return fmt.Errorf("JSONInt type mismatch: %s", val.Type())
 	}
@@ -162,6 +165,11 @@ func (this *JSONBool) unmarshalValue(val reflect.Value) error {
 		} else {
 			val.SetString("false")
 		}
+	case reflect.Ptr:
+		if val.IsNil() {
+			val.Set(reflect.New(val.Type().Elem()))
+		}
+		return this.unmarshalValue(val.Elem())
 	default:
 		return fmt.Errorf("JSONBool type mismatch: %s", val.Type())
 	}
@@ -231,6 +239,11 @@ func (this *JSONFloat) unmarshalValue(val reflect.Value) error {
 		}
 	case reflect.String:
 		val.SetString(fmt.Sprintf("%f", this.data))
+	case reflect.Ptr:
+		if val.IsNil() {
+			val.Set(reflect.New(val.Type().Elem()))
+		}
+		return this.unmarshalValue(val.Elem())
 	default:
 		return fmt.Errorf("JSONFloat type mismatch: %s", val.Type())
 	}
@@ -314,6 +327,11 @@ func (this *JSONString) unmarshalValue(val reflect.Value) error {
 		val.SetBool(utils.ToBool(this.data))
 	case reflect.String:
 		val.SetString(this.data)
+	case reflect.Ptr:
+		if val.IsNil() {
+			val.Set(reflect.New(val.Type().Elem()))
+		}
+		return this.unmarshalValue(val.Elem())
 	default:
 		return fmt.Errorf("JSONString type mismatch: %s", val.Type())
 	}
@@ -339,15 +357,32 @@ func (this *JSONArray) unmarshalValue(val reflect.Value) error {
 	case reflect.String:
 		val.SetString(this.String())
 		return nil
+	case reflect.Ptr:
+		if val.IsNil() {
+			kind := val.Type().Elem().Kind()
+			if kind == reflect.Array || kind == reflect.Slice {
+				val.Set(reflect.New(val.Type().Elem()))
+				return this.unmarshalValue(val.Elem())
+			}
+		}
+		return fmt.Errorf("JSONArray type mismatch %s", val.Type())
 	case reflect.Slice, reflect.Array:
-		for _, json := range this.data {
-			newEle := reflect.New(val.Type().Elem()).Elem()
-			err := json.unmarshalValue(newEle)
+		if val.Kind() == reflect.Array {
+			if val.Len() != len(this.data) {
+				return fmt.Errorf("JSONArray length unmatch %s: %d != %d",
+					val.Type(), val.Len(), len(this.data))
+			}
+		} else if val.Kind() == reflect.Slice {
+			if val.Len() < len(this.data) {
+				newVal := reflect.MakeSlice(val.Type(), len(this.data), len(this.data))
+				val.Set(newVal)
+			}
+		}
+		for i, json := range this.data {
+			err := json.unmarshalValue(val.Index(i))
 			if err != nil {
 				return err
 			}
-			newVal := reflect.Append(val, newEle)
-			val.Set(newVal)
 		}
 	default:
 		return fmt.Errorf("JSONArray type mismatch: %s", val.Type())
@@ -376,6 +411,16 @@ func (this *JSONDict) unmarshalValue(val reflect.Value) error {
 		return this.unmarshalMap(val)
 	case reflect.Struct:
 		return this.unmarshalStruct(val)
+	case reflect.Ptr:
+		if val.IsNil() {
+			kind := val.Type().Elem().Kind()
+			if kind != reflect.Struct && kind != reflect.Map {
+				return fmt.Errorf("JSONDict type mismatch: %s", val.Type())
+			}
+			newVal := reflect.New(val.Type().Elem())
+			val.Set(newVal)
+		}
+		return this.unmarshalValue(val.Elem())
 	default:
 		return fmt.Errorf("JSONDict type mismatch: %s", val.Type())
 	}

vendor/yunion.io/x/pkg/gotypes/serialize.go

@@ -14,30 +14,32 @@ type ISerializable interface {
 
 type FuncSerializableAllocator func() ISerializable
 
-var ISerializableType reflect.Type
-var serializableAllocators map[reflect.Type]FuncSerializableAllocator
-var ErrTypeNotSerializable error
-
-func init() {
-	ISerializableType = reflect.TypeOf((*ISerializable)(nil)).Elem()
-	serializableAllocators = make(map[reflect.Type]FuncSerializableAllocator)
+var (
+	ISerializableType      = reflect.TypeOf((*ISerializable)(nil)).Elem()
+	serializableAllocators = map[reflect.Type]FuncSerializableAllocator{}
 	ErrTypeNotSerializable = errors.New("Type not serializable")
-}
+)
 
+// RegisterSerializable registers an allocator func for the specified serializable type.
+//
+// This is intended to be used when you have multiple implmenetations of an
+// interface and you want to use only one of them to cover them all.
+// TokenCredential and SSimpleToken is such a case.
 func RegisterSerializable(valType reflect.Type, alloc FuncSerializableAllocator) {
+	if !IsSerializable(valType) {
+		panic(valType.String() + " does not implement ISerializable")
+	}
 	serializableAllocators[valType] = alloc
 }
 
-func IsSerializable(objType reflect.Type) bool {
-	_, ok := serializableAllocators[objType]
-	return ok
+func IsSerializable(valType reflect.Type) bool {
+	return valType.Implements(ISerializableType)
 }
 
 func NewSerializable(objType reflect.Type) (ISerializable, error) {
 	deserFunc, ok := serializableAllocators[objType]
-	if !ok {
-		return nil, ErrTypeNotSerializable
-	} else {
+	if ok {
 		return deserFunc(), nil
 	}
+	return nil, ErrTypeNotSerializable
 }

vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go

@@ -2,6 +2,7 @@ package reflectutils
 
 import (
 	"reflect"
+	"strings"
 
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/utils"
@@ -15,14 +16,23 @@ func GetStructFieldName(field *reflect.StructField) string {
 		return nameStr
 	} else {
 		jsonStr, _ := tagMap["json"]
-		if len(jsonStr) > 0 {
-			return jsonStr
-		} else {
-			return utils.CamelSplit(field.Name, "_")
-		}
+		return toJsonKey(field.Name, jsonStr)
 	}
 }
 
+func toJsonKey(fieldName, jsonTag string) string {
+	jsonTag = strings.Replace(jsonTag, "omitempty", "", -1)
+	words := utils.FindWords([]byte(jsonTag), 0)
+	if len(words) == 0 {
+		return utils.CamelSplit(fieldName, "_")
+	}
+	name := words[0]
+	if name == "-" {
+		return ""
+	}
+	return name
+}
+
 func FetchStructFieldNameValueInterfaces(dataValue reflect.Value) map[string]interface{} {
 	fields := make(map[string]interface{})
 	fetchStructFieldNameValueInterfaces(dataValue.Type(), dataValue, fields)

vendor/yunion.io/x/sqlchemy/column.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"time"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/regutils"
@@ -465,11 +464,11 @@ func (c *SDecimalColumn) IsZero(val interface{}) bool {
 func NewDecimalColumn(name string, tagmap map[string]string) SDecimalColumn {
 	tagmap, v, ok := utils.TagPop(tagmap, TAG_PRECISION)
 	if !ok {
-		log.Fatalf("Field %s of float should have precision tag", name)
+		panic(fmt.Sprintf("Field %q of float misses precision tag", name))
 	}
 	prec, err := strconv.Atoi(v)
 	if err != nil {
-		log.Fatalf("Field %s of float precision %s shoud be integer!", name, v)
+		panic(fmt.Sprintf("Field precision of %q shoud be integer (%q)", name, v))
 	}
 	return SDecimalColumn{SBaseWidthColumn: NewBaseWidthColumn(name, "DECIMAL", tagmap),
 		Precision: prec}
@@ -544,7 +543,7 @@ func NewTextColumn(name string, tagmap map[string]string) STextColumn {
 	if len(charset) == 0 {
 		charset = "utf8"
 	} else if charset != "utf8" && charset != "ascii" {
-		log.Fatalf("Unsupported charset %s for %s", charset, name)
+		panic(fmt.Sprintf("Unsupported charset %s for %s", charset, name))
 	}
 	return STextColumn{SBaseWidthColumn: NewBaseWidthColumn(name, sqltype, tagmap),
 		Charset: charset}

vendor/yunion.io/x/sqlchemy/insert.go

@@ -42,7 +42,6 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error {
 		dtc, ok := c.(*SDateTimeColumn)
 		ov := fields[k]
 
-		// log.Debugf("field %s value %s %s", k, ov, ov==nil)
 		if ok && (dtc.IsCreatedAt || dtc.IsUpdatedAt) {
 			createdAtFields = append(createdAtFields, k)
 			names = append(names, fmt.Sprintf("`%s`", k))
@@ -54,15 +53,13 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error {
 			format = append(format, "?")
 		} else if c.IsPrimary() {
 			if isAutoInc {
-				if len(autoIncField) == 0 {
-					autoIncField = k
-				} else {
-					log.Fatalf("multiple auto_increment columns???")
+				if len(autoIncField) > 0 {
+					panic(fmt.Sprintf("multiple auto_increment columns: %q, %q", autoIncField, k))
 				}
-			} else {
-				return fmt.Errorf("fail to insert for null primary key `%s`", k)
+				autoIncField = k
 			}
-		} else if ! c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value
+			return fmt.Errorf("cannot insert for null primary key %q", k)
+		} else if !c.IsSupportDefault() && len(c.Default()) > 0 && ov != nil && c.IsZero(ov) { // empty text value
 			val := c.ConvertFromString(c.Default())
 			values = append(values, val)
 			names = append(names, fmt.Sprintf("`%s`", k))
@@ -92,15 +89,15 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error {
 	}
 
 	/*
-	if len(autoIncField) > 0 {
-		lastId, err := results.LastInsertId()
-		if err == nil {
-			val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField)
-			if ok {
-				gotypes.SetValue(val, fmt.Sprint(lastId))
+		if len(autoIncField) > 0 {
+			lastId, err := results.LastInsertId()
+			if err == nil {
+				val, ok := reflectutils.FindStructFieldValue(dataValue, autoIncField)
+				if ok {
+					gotypes.SetValue(val, fmt.Sprint(lastId))
+				}
 			}
 		}
-	}
 	*/
 
 	// query the value, so default value can be feedback into the object
@@ -112,7 +109,7 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error {
 			if ok && nc.IsAutoIncrement {
 				lastId, err := results.LastInsertId()
 				if err != nil {
-					log.Errorf("Fail to fetch lastInsertId %s", err)
+					err := fmt.Errorf("fetching lastInsertId failed: %v", err)
 					return err
 				} else {
 					q = q.Equals(c.Name(), lastId)
@@ -124,9 +121,9 @@ func (t *STableSpec) insert(dt interface{}, debug bool) error {
 	}
 	err = q.First(dt)
 	if err != nil {
-		log.Errorf("query after insert failed %s", err)
+		err := fmt.Errorf("query after insert failed: %v", err)
 		return err
 	}
-	
+
 	return nil
 }

vendor/yunion.io/x/sqlchemy/parser.go

@@ -3,14 +3,13 @@ package sqlchemy
 import (
 	"reflect"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/gotypes"
 	"yunion.io/x/pkg/tristate"
 	"yunion.io/x/pkg/util/reflectutils"
 	"yunion.io/x/pkg/utils"
 )
 
-func fieldToColumnSpec(field *reflect.StructField) IColumnSpec {
+func structField2ColumnSpec(field *reflect.StructField) IColumnSpec {
 	fieldname := reflectutils.GetStructFieldName(field)
 	tagmap := utils.TagMap(field.Tag)
 	if _, ok := tagmap[TAG_IGNORE]; ok {
@@ -85,13 +84,11 @@ func fieldToColumnSpec(field *reflect.StructField) IColumnSpec {
 		col := NewJSONColumn(fieldname, tagmap)
 		return &col*/
 	default:
-		if gotypes.IsSerializable(field.Type) {
+		if field.Type.Implements(gotypes.ISerializableType) {
 			col := NewCompoundColumn(fieldname, tagmap)
 			return &col
-		} else {
-			log.Fatalf("Unsupported type! %s", field.Type)
-			return nil
 		}
+		panic("not supported type %s" + field.Type.Name())
 	}
 }
 
@@ -101,9 +98,9 @@ func struct2TableSpec(st reflect.Type, table *STableSpec) {
 		if f.Type.Kind() == reflect.Struct && f.Type != gotypes.TimeType {
 			struct2TableSpec(f.Type, table)
 		} else {
-			coldef := fieldToColumnSpec(&f)
-			if coldef != nil {
-				table.columns = append(table.columns, coldef)
+			column := structField2ColumnSpec(&f)
+			if column != nil {
+				table.columns = append(table.columns, column)
 			}
 		}
 	}

vendor/yunion.io/x/sqlchemy/query.go

@@ -126,7 +126,6 @@ func (sq *SSubQuery) Field(id string, alias ...string) IQueryField {
 			return &sqf
 		}
 	}
-	log.Errorf("field %s not found for subquery", id)
 	return nil
 }
 

vendor/yunion.io/x/sqlchemy/reflect.go

@@ -25,7 +25,7 @@ func getStringValue(dat interface{}) string {
 	case gotypes.IntType, gotypes.Int8Type, gotypes.Int16Type, gotypes.Int32Type, gotypes.Int64Type:
 		return fmt.Sprintf("%d", value.Int())
 	case gotypes.UintType, gotypes.Uint8Type, gotypes.Uint16Type, gotypes.Uint32Type, gotypes.Uint64Type:
-		return fmt.Sprintf("%u", value.Uint())
+		return fmt.Sprintf("%d", value.Uint())
 	case gotypes.Float32Type, gotypes.Float64Type:
 		return fmt.Sprintf("%f", value.Float())
 	case gotypes.StringType:
@@ -53,15 +53,12 @@ func getStringValue(dat interface{}) string {
 			return string(rawBytes)
 		}
 	default:
-		if gotypes.IsSerializable(value.Type()) {
-			is, ok := value.Interface().(gotypes.ISerializable)
-			if !ok {
-				log.Errorf("%s fail to convert to JSONObject", value)
-			} else {
-				return is.String()
-			}
+		serializable, ok := value.Interface().(gotypes.ISerializable)
+		if !ok {
+			log.Errorf("cannot convert %v to string", value)
+			return ""
 		}
-		log.Errorf("Cannot convert to string %s", value)
+		return serializable.String()
 	}
 	return ""
 }
@@ -127,15 +124,15 @@ func setValueBySQLString(value reflect.Value, val string) error {
 		gotypes.Float32SliceType, gotypes.Float64SliceType, gotypes.StringSliceType:
 		reflect.Append(value, reflect.ValueOf(val))
 	default:
-		if gotypes.IsSerializable(value.Type()) {
-			is, err := jsonutils.JSONDeserialize(value.Type(), val)
-			if err != nil {
-				return err
-			}
-			value.Set(reflect.ValueOf(is))
-		} else {
-			return fmt.Errorf("!!!Unsupported type: %s", value.Type)
+		valueType := value.Type()
+		if !valueType.Implements(gotypes.ISerializableType) {
+			return fmt.Errorf("not supported type: %s", valueType)
 		}
+		serializable, err := jsonutils.JSONDeserialize(valueType, val)
+		if err != nil {
+			return err
+		}
+		value.Set(reflect.ValueOf(serializable))
 	}
 	return nil
 }

vendor/yunion.io/x/sqlchemy/sync.go

@@ -212,10 +212,10 @@ func (ts *STableSpec) Sync() error {
 
 func (ts *STableSpec) CheckSync() {
 	sqls := ts.SyncSQL()
-	if sqls != nil && len(sqls) > 0 {
+	if len(sqls) > 0 {
 		for _, sql := range sqls {
 			fmt.Println(sql)
 		}
-		log.Fatalf("DB not in sync!!!")
+		log.Fatalf("DB table %q not in sync", ts.name)
 	}
 }

vendor/yunion.io/x/sqlchemy/table.go

@@ -5,7 +5,6 @@ import (
 	"reflect"
 	"strings"
 
-	"yunion.io/x/log"
 	"yunion.io/x/pkg/utils"
 )
 
@@ -29,12 +28,15 @@ type STableField struct {
 func NewTableSpecFromStruct(s interface{}, name string) *STableSpec {
 	st := reflect.TypeOf(s)
 	if st.Kind() != reflect.Struct {
-		log.Fatalf("Invalid table struct, NOT a STRUCT!!!")
-		return nil
+		panic("expect Struct kind")
 	}
-	table := STableSpec{columns: make([]IColumnSpec, 0), name: name, structType: st}
-	struct2TableSpec(st, &table)
-	return &table
+	table := &STableSpec{
+		columns:    []IColumnSpec{},
+		name:       name,
+		structType: st,
+	}
+	struct2TableSpec(st, table)
+	return table
 }
 
 func (ts *STableSpec) Name() string {
@@ -88,12 +90,11 @@ func (ts *STableSpec) ColumnSpec(name string) IColumnSpec {
 func (tbl *STable) Field(name string, alias ...string) IQueryField {
 	// name = reflectutils.StructFieldName(name)
 	name = utils.CamelSplit(name, "_")
-	cSpec := tbl.spec.ColumnSpec(name)
-	if cSpec == nil {
-		log.Fatalf("Column %s not found", name)
-		return nil
+	spec := tbl.spec.ColumnSpec(name)
+	if spec == nil {
+		panic("column not found: " + name)
 	}
-	col := STableField{table: tbl, spec: cSpec}
+	col := STableField{table: tbl, spec: spec}
 	if len(alias) > 0 {
 		col.Label(alias[0])
 	}

vendor/yunion.io/x/sqlchemy/tablealias.go

@@ -6,14 +6,10 @@ import (
 )
 
 var (
-	tableID     = 0
-	tableIDLock *sync.Mutex
+	tableID                 = 0
+	tableIDLock *sync.Mutex = &sync.Mutex{}
 )
 
-func init() {
-	tableIDLock = &sync.Mutex{}
-}
-
 func getTableAliasName() string {
 	tableIDLock.Lock()
 	defer tableIDLock.Unlock()

vendor/yunion.io/x/sqlchemy/update.go

@@ -64,9 +64,9 @@ func UpdateDiffString(diff map[string]SUpdateDiff) string {
 }
 
 func (us *SUpdateSession) saveUpdate(dt interface{}) (map[string]SUpdateDiff, error) {
-	beforeInsertFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate")
-	if beforeInsertFunc.IsValid() && !beforeInsertFunc.IsNil() {
-		beforeInsertFunc.Call([]reflect.Value{})
+	beforeUpdateFunc := reflect.ValueOf(dt).MethodByName("BeforeUpdate")
+	if beforeUpdateFunc.IsValid() && !beforeUpdateFunc.IsNil() {
+		beforeUpdateFunc.Call([]reflect.Value{})
 	}
 
 	// dataType := reflect.TypeOf(dt).Elem()

vendor/yunion.io/x/structarg/structarg.go

@@ -27,8 +27,8 @@ type Argument interface {
 	AliasToken() string
 	ShortToken() string
 	MetaVar() string
-	IsOptional() bool
 	IsPositional() bool
+	IsRequired() bool
 	IsMulti() bool
 	IsSubcommand() bool
 	HelpString(indent string) string
@@ -45,8 +45,8 @@ type SingleArgument struct {
 	aliasToken string
 	shortToken string
 	metavar    string
-	optional   bool
 	positional bool
+	required   bool
 	help       string
 	choices    []string
 	useDefault bool
@@ -136,7 +136,13 @@ const (
 	   A boolean value explicitly declare whether the argument is optional,
 	   the tag is optional
 	*/
-	TAG_OPTIONAL = "optional"
+	TAG_POSITIONAL = "positional"
+	/*
+	   A boolean value explicitly declare whether the argument is required.
+	   The tag is optional.  This is for optional arguments.  Positional
+	   arguments must be "required"
+	*/
+	TAG_REQUIRED = "required"
 	/*
 	   A boolean value explicitly decalre whther the argument is an subcommand
 	   A subcommand argument must be the last positional argument.
@@ -263,51 +269,64 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value)
 			}
 		}
 	}
-	var positional, optional bool
+	// heuristic guessing "positional"
+	var positional bool
 	if f.Name == strings.ToUpper(f.Name) {
 		positional = true
-		optional = false
 	} else {
 		positional = false
-		optional = true
 	}
-	opt_val := tagMap[TAG_OPTIONAL]
-	if len(opt_val) > 0 {
-		if opt_val == "true" {
-			optional = true
-			positional = false
-		} else if opt_val == "false" {
-			optional = false
+	if positionalTag := tagMap[TAG_POSITIONAL]; len(positionalTag) > 0 {
+		switch positionalTag {
+		case "true":
 			positional = true
-		} else {
-			return fmt.Errorf("Invalid optional value %s, neither true nor false", opt_val)
+		case "false":
+			positional = false
+		default:
+			return fmt.Errorf("Invalid positional tag %q, neither true nor false", positionalTag)
 		}
-		// fmt.Println(token, "optional", opt_val, optional, positional)
 	}
-	if positional && !optional && use_default {
-		return fmt.Errorf("A positional non-optional argument should not set default value")
+	required := positional
+	if requiredTag := tagMap[TAG_REQUIRED]; len(requiredTag) > 0 {
+		switch requiredTag {
+		case "true":
+			required = true
+		case "false":
+			required = false
+		default:
+			return fmt.Errorf("Invalid required tag %q, neither true nor false", requiredTag)
+		}
 	}
-	subcommand, e := strconv.ParseBool(tagMap[TAG_SUBCOMMAND])
-	if e != nil {
+	if positional {
+		if !required {
+			return fmt.Errorf("positional %s must not have required:false", token)
+		}
+		if use_default {
+			return fmt.Errorf("positional %s must not have default value", token)
+		}
+	}
+	subcommand, err := strconv.ParseBool(tagMap[TAG_SUBCOMMAND])
+	if err != nil {
 		subcommand = false
 	}
 	var defval_t reflect.Value
 	if use_default {
-		defval_t, e = gotypes.ParseValue(defval, f.Type)
-		if e != nil {
-			return e
+		defval_t, err = gotypes.ParseValue(defval, f.Type)
+		if err != nil {
+			return err
 		}
 	}
 	if subcommand {
 		positional = true
-		optional = false
 	}
 	var arg Argument = nil
 	ovalue := reflect.New(v.Type()).Elem()
 	ovalue.Set(v)
 	sarg := SingleArgument{token: token, shortToken: shorttoken,
-		optional: optional, positional: positional,
-		metavar: metavar, help: help,
+		positional: positional,
+		required:   required,
+		metavar:    metavar,
+		help:       help,
 		choices:    choices,
 		useDefault: use_default,
 		aliasToken: alias,
@@ -321,7 +340,7 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value)
 			subcommands: make(map[string]SubcommandArgumentData)}
 	} else if f.Type.Kind() == reflect.Array || f.Type.Kind() == reflect.Slice {
 		var min, max int64
-		var e error
+		var err error
 		nargs := tagMap[TAG_NARGS]
 		if nargs == "*" {
 			min = 0
@@ -333,13 +352,13 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value)
 			min = 1
 			max = -1
 		} else {
-			min, e = strconv.ParseInt(nargs, 10, 64)
-			if e == nil {
+			min, err = strconv.ParseInt(nargs, 10, 64)
+			if err == nil {
 				max = min
 			} else if positional {
 				min = 1
 				max = -1
-			} else if optional {
+			} else if !required {
 				min = 0
 				max = -1
 			}
@@ -349,7 +368,11 @@ func (this *ArgumentParser) addArgument(f reflect.StructField, v reflect.Value)
 	} else {
 		arg = &sarg
 	}
-	return this.AddArgument(arg)
+	err = this.AddArgument(arg)
+	if err != nil {
+		return fmt.Errorf("AddArgument %s: %v", arg, err)
+	}
+	return nil
 }
 
 func (this *ArgumentParser) AddArgument(arg Argument) error {
@@ -361,13 +384,25 @@ func (this *ArgumentParser) AddArgument(arg Argument) error {
 				return fmt.Errorf("Cannot append positional argument after an array positional argument")
 			case last_arg.IsSubcommand():
 				return fmt.Errorf("Cannot append positional argument after a subcommand argument")
-			case last_arg.IsOptional() && !arg.IsOptional():
-				return fmt.Errorf("Cannot append positional argument after an optional positional argument")
 			}
 		}
 		this.posArgs = append(this.posArgs, arg)
 	} else {
-		this.optArgs = append(this.optArgs, arg)
+		// Put required at the end and try to be stable
+		if arg.IsRequired() {
+			this.optArgs = append(this.optArgs, arg)
+		} else {
+			var i int
+			var opt Argument
+			for i, opt = range this.optArgs {
+				if opt.IsRequired() {
+					break
+				}
+			}
+			this.optArgs = append(this.optArgs, nil)
+			copy(this.optArgs[i+1:], this.optArgs[i:])
+			this.optArgs[i] = arg
+		}
 	}
 	return nil
 }
@@ -432,12 +467,12 @@ func (this *SingleArgument) ShortToken() string {
 
 func (this *SingleArgument) String() string {
 	var start, end byte
-	if this.IsOptional() {
-		start = '['
-		end = ']'
-	} else {
+	if this.IsRequired() {
 		start = '<'
 		end = '>'
+	} else {
+		start = '['
+		end = ']'
 	}
 	if this.IsPositional() {
 		return fmt.Sprintf("%c%s%c", start, this.MetaVar(), end)
@@ -450,8 +485,8 @@ func (this *SingleArgument) String() string {
 	}
 }
 
-func (this *SingleArgument) IsOptional() bool {
-	return this.optional
+func (this *SingleArgument) IsRequired() bool {
+	return this.required
 }
 
 func (this *SingleArgument) IsPositional() bool {
@@ -519,7 +554,7 @@ func (this *SingleArgument) SetDefault() {
 }
 
 func (this *SingleArgument) Validate() error {
-	if !this.optional && !this.isSet && !this.useDefault {
+	if this.required && !this.isSet && !this.useDefault {
 		return fmt.Errorf("Non-optional argument %s not set", this.token)
 	}
 	return nil
@@ -818,7 +853,7 @@ func (this *ArgumentParser) ParseArgs(args []string, ignore_unknown bool) error
 			}
 		}
 	}
-	if err == nil && pos_idx < len(this.posArgs) && !this.posArgs[pos_idx].IsOptional() {
+	if err == nil && pos_idx < len(this.posArgs) {
 		err = fmt.Errorf("Not enough arguments, missing %s", this.posArgs[pos_idx])
 	}
 	if err == nil {