Merge pull request #5779 from swordqiu/automated-cherry-pick-of-#5756-upstream-release-3.1

Automated cherry pick of #5756: fix: 1. allow share proxysetting 2. claim proxsetting as domain resources
2026-09-24 16:03:43 +08:00 · 2020-04-13 15:45:16 +08:00
parent df37a12fc2 df00cdbd04
commit 8ef6b60a42
15 changed files with 202 additions and 27 deletions
@@ -129,9 +129,9 @@ require (
 	k8s.io/cluster-bootstrap v0.17.3
 	k8s.io/kubernetes v1.16.0
 	yunion.io/x/executor v0.0.0-20200227030256-a18417815e74
-	yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda
-	yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
-	yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
+	yunion.io/x/jsonutils v0.0.0-20200330063846-589d9924bb8b
+	yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
+	yunion.io/x/pkg v0.0.0-20200403115157-880d716ed624
 	yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e
 	yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57
 	yunion.io/x/structarg v0.0.0-20190809075558-115bed041de3
@@ -1015,16 +1015,18 @@ vbom.ml/util v0.0.0-20160121211510-db5cfe13f5cc/go.mod h1:so/NYdZXCz+E3ZpW0uAoCj
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74 h1:A15C6VdVRWvmQ9pAJHrUs9yan5qKlYH7uaRxHg1kRbk=
 yunion.io/x/executor v0.0.0-20200227030256-a18417815e74/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws=
 yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634=
-yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda h1:wSwQj3MDGchGYM2RAo1riYhrTz4apH+5XyBnchuDi84=
-yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
+yunion.io/x/jsonutils v0.0.0-20200330063846-589d9924bb8b h1:mt0TOKRk76yeH0whJfmKsceXBuudXLjvoj8NKGTqpEU=
+yunion.io/x/jsonutils v0.0.0-20200330063846-589d9924bb8b/go.mod h1:T7kxQJR13+t7z0TuT+Wzd7MTxBOk2H9c0pO1ONQSv90=
 yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d h1:59zrDL7Ft+hDukguJRmLr/Gdu/9V75x+yX99ovZwfaA=
 yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
+yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3 h1:5Wc5hkB8PtMudmHuzCyok960RuOa9I55imIGrigSdjs=
+yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U=
 yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/pkg v0.0.0-20200302034534-fdf44d54b070/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
-yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e h1:rBfX77+VEBVpe6Xxy2gDa4WB7qbtndWvXcrVKzleF84=
-yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
+yunion.io/x/pkg v0.0.0-20200403115157-880d716ed624 h1:yPayJlTJHOP0vKY6ovliZyTwJY9RvdWEF+QDqnxt/iw=
+yunion.io/x/pkg v0.0.0-20200403115157-880d716ed624/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e h1:v+EzIadodSwkdZ/7bremd7J8J50Cise/HCylsOJngmo=
 yunion.io/x/s3cli v0.0.0-20190917004522-13ac36d8687e/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo=
 yunion.io/x/sqlchemy v0.0.0-20200312002602-1177cd8fbc57 h1:KtQAuLJ00RSUVqkiRmJ1DiDABiw0U3xxXnzD3lGavaY=
@@ -40,6 +40,11 @@ func (ps *SProxySetting) IsZero() bool {
 	return false
 }

+type ProxySettingResourceInput struct {
+	// 代理配置
+	ProxySettingId string `json:"proxy_setting_id"`
+}
+
 type ProxySettingTestInput struct {
 	HttpProxy  string
 	HttpsProxy string
@@ -80,8 +80,7 @@ type CloudaccountCreateInput struct {
 	// 额外信息,例如账单的access key
 	Options *jsonutils.JSONObject `json:"options"`

-	// 代理配置
-	ProxySettingId string `json:"proxy_setting_id"`
+	proxyapi.ProxySettingResourceInput

 	cloudprovider.SCloudaccount
 	cloudprovider.SCloudaccountCredential
@@ -12,6 +12,7 @@ import (
 	"golang.org/x/net/http/httpproxy"

 	"yunion.io/x/jsonutils"
+	"yunion.io/x/pkg/errors"

 	proxyapi "yunion.io/x/onecloud/pkg/apis/cloudcommon/proxy"
 	"yunion.io/x/onecloud/pkg/cloudcommon/db"
@@ -203,3 +204,16 @@ var referrersMen []db.IModelManager
 func RegisterReferrer(man db.IModelManager) {
 	referrersMen = append(referrersMen, man)
 }
+
+func ValidateProxySettingResourceInput(userCred mcclient.TokenCredential, input proxyapi.ProxySettingResourceInput) (*SProxySetting, proxyapi.ProxySettingResourceInput, error) {
+	m, err := ProxySettingManager.FetchByIdOrName(userCred, input.ProxySettingId)
+	if err != nil {
+		if errors.Cause(err) == sql.ErrNoRows {
+			return nil, input, errors.Wrapf(httperrors.ErrResourceNotFound, "%s %s", ProxySettingManager.Keyword(), input.ProxySettingId)
+		} else {
+			return nil, input, errors.Wrapf(err, "ProxySettingManager.FetchByIdOrName")
+		}
+	}
+	input.ProxySettingId = m.GetId()
+	return m.(*SProxySetting), input, nil
+}
@@ -318,11 +318,30 @@ func (self *SCloudaccount) ValidateUpdateData(ctx context.Context, userCred mccl
 		"proxy_setting",
 		proxy.ProxySettingManager.Keyword(),
 		userCred,
-	).Optional(true)
+	)
+	v.Optional(true)
 	if err := v.Validate(data); err != nil {
 		return nil, err
 	}

+	proxySetting := v.Model.(*proxy.SProxySetting)
+
+	if proxySetting != nil {
+		// updated proxy setting, so do the check
+		proxyFunc := proxySetting.HttpTransportProxyFunc()
+		secret, _ := self.getPassword()
+		_, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{
+			Vendor:    self.Provider,
+			URL:       self.AccessUrl,
+			Account:   self.Account,
+			Secret:    secret,
+			ProxyFunc: proxyFunc,
+		})
+		if err != nil {
+			return nil, httperrors.NewInputParameterError("invalid proxy setting %s", err)
+		}
+	}
+
 	return self.SEnabledStatusStandaloneResourceBase.ValidateUpdateData(ctx, userCred, query, data)
 }

@@ -385,13 +404,11 @@ func (manager *SCloudaccountManager) ValidateCreateData(ctx context.Context, use
 		if input.ProxySettingId == "" {
 			input.ProxySettingId = proxyapi.ProxySettingId_DIRECT
 		}
-		m, err := proxy.ProxySettingManager.FetchByIdOrName(userCred, input.ProxySettingId)
+		var proxySetting *proxy.SProxySetting
+		proxySetting, input.ProxySettingResourceInput, err = proxy.ValidateProxySettingResourceInput(userCred, input.ProxySettingResourceInput)
 		if err != nil {
-			return input, httperrors.NewInputParameterError("fetch proxysetting %s: %s",
-				input.ProxySettingId, err)
+			return input, httperrors.NewInputParameterError("ValidateProxySettingResourceInput %s", err)
 		}
-		proxySetting := m.(*proxy.SProxySetting)
-		input.ProxySettingId = proxySetting.Id
 		proxyFunc = proxySetting.HttpTransportProxyFunc()
 	}
 	accountId, err := cloudprovider.IsValidCloudAccount(cloudprovider.ProviderConfig{
@@ -83,6 +83,8 @@ type SCloudAccountCreateBaseOptions struct {
 	SyncIntervalSeconds int `help:"Interval to synchronize if auto sync is enable" metavar:"SECONDS"`

 	ProjectDomain string `help:"domain for this account"`
+
+	ProxySettingId string `help:"proxy setting id or name" json:"proxy_setting_id"`
 }

 type SVMwareCloudAccountCreateOptions struct {
@@ -229,7 +231,7 @@ type SCloudAccountUpdateBaseOptions struct {

 	SyncIntervalSeconds int    `help:"auto synchornize interval in seconds"`
 	AutoCreateProject   *bool  `help:"automatically create local project for new remote project"`
-	ProxySetting        string `help:"proxy setting name or id"`
+	ProxySettingId      string `help:"proxy setting name or id" json:"proxy_setting_id"`

 	Desc string `help:"Description" json:"description" token:"desc"`
 }
@@ -924,12 +924,12 @@ sigs.k8s.io/yaml
 yunion.io/x/executor/apis
 yunion.io/x/executor/client
 yunion.io/x/executor/server
-# yunion.io/x/jsonutils v0.0.0-20200303051356-aa609aba0cda
+# yunion.io/x/jsonutils v0.0.0-20200330063846-589d9924bb8b
 yunion.io/x/jsonutils
-# yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d
+# yunion.io/x/log v0.0.0-20200313080802-57a4ce5966b3
 yunion.io/x/log
 yunion.io/x/log/hooks
-# yunion.io/x/pkg v0.0.0-20200304112442-9dae9351325e
+# yunion.io/x/pkg v0.0.0-20200403115157-880d716ed624
 yunion.io/x/pkg/errors
 yunion.io/x/pkg/gotypes
 yunion.io/x/pkg/prettytable
@@ -0,0 +1,19 @@
+// Copyright 2019 Yunion
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package jsonutils
+
+const (
+	TAG_DEPRECATED_BY = "deprecated-by"
+)
@@ -35,3 +35,46 @@ func (dict *JSONDict) UpdateDefault(json JSONObject) {
 		}
 	}
 }
+
+func Diff(a, b *JSONDict) (aNoB, aDiffB, aAndB, bNoA *JSONDict) {
+	keysA := a.SortedKeys()
+	keysB := b.SortedKeys()
+	aNoB = NewDict()
+	aDiffB = NewDict()
+	aAndB = NewDict()
+	bNoA = NewDict()
+
+	i := 0
+	j := 0
+	for i < len(keysA) || j < len(keysB) {
+		if i < len(keysA) && j < len(keysB) {
+			keyA := keysA[i]
+			keyB := keysB[j]
+			if keyA > keyB {
+				aNoB.data[keyA] = a.data[keyA]
+				i += 1
+			} else if keyA < keyB {
+				bNoA.data[keyB] = b.data[keyB]
+				j += 1
+			} else {
+				valA := a.data[keysA[i]].String()
+				valB := b.data[keysB[i]].String()
+				if valA != valB {
+					aDiffB.data[keyA] = NewArray(a.data[keyA], b.data[keyB])
+				} else {
+					aAndB.data[keyA] = a.data[keyA]
+				}
+				i += 1
+				j += 1
+			}
+		} else if i < len(keysA) {
+			aNoB.data[keysA[i]] = a.data[keysA[i]]
+			i = i + 1
+		} else if j < len(keysB) {
+			bNoA.data[keysB[j]] = b.data[keysB[j]]
+			j = j + 1
+		}
+	}
+
+	return
+}
@@ -83,21 +83,60 @@ func marshalStruct(val reflect.Value, info *reflectutils.SStructFieldInfo) JSONO
 	}
 }

+func findValueByKey(pairs []JSONPair, key string) JSONObject {
+	for i := range pairs {
+		if pairs[i].key == key {
+			return pairs[i].val
+		}
+	}
+	return nil
+}
+
 func struct2JSONPairs(val reflect.Value) []JSONPair {
 	fields := reflectutils.FetchStructFieldValueSet(val)
 	objPairs := make([]JSONPair, 0, len(fields))
+	depFields := make(map[string]string)
 	for i := 0; i < len(fields); i += 1 {
 		jsonInfo := fields[i].Info
 		if jsonInfo.Ignore {
 			continue
 		}
 		key := jsonInfo.MarshalName()
+		if deprecatedBy, ok := fields[i].Info.Tags[TAG_DEPRECATED_BY]; ok {
+			depFields[key] = deprecatedBy
+			continue
+		}
 		val := marshalValue(fields[i].Value, jsonInfo)
 		if val != nil && val != JSONNull {
 			objPair := JSONPair{key: key, val: val}
 			objPairs = append(objPairs, objPair)
 		}
 	}
+	depPairs := make([]JSONPair, 0, len(depFields))
+	for depKey, key := range depFields {
+		findLoop := false
+		for {
+			if okey, ok := depFields[key]; ok {
+				if okey == depKey {
+					// loop detected
+					findLoop = true
+					break
+				}
+				key = okey
+			} else {
+				break
+			}
+		}
+		if findLoop {
+			continue
+		}
+		val := findValueByKey(objPairs, key)
+		if val != nil {
+			objPair := JSONPair{key: depKey, val: val}
+			depPairs = append(depPairs, objPair)
+		}
+	}
+	objPairs = append(objPairs, depPairs...)
 	return objPairs
 }

@@ -539,7 +539,7 @@ func setStructFieldAt(key string, v JSONObject, fieldValues reflectutils.SStruct
 		if err != nil {
 			return errors.Wrap(err, "JSONDict.unmarshalStruct")
 		}
-		depInfo, ok := fieldValues[index].Info.Tags["deprecated-by"]
+		depInfo, ok := fieldValues[index].Info.Tags[TAG_DEPRECATED_BY]
 		if ok {
 			err := setStructFieldAt(depInfo, v, fieldValues, visited)
 			if err != nil {
@@ -149,9 +149,18 @@ func DisableColors() {
 }

 func AddHookFormatter(logger *logrus.Logger) {
-	pcs := make([]uintptr, 2)
-	runtime.Callers(0, pcs)
-	logrusPackage := hooks.GetPackageName(runtime.FuncForPC(pcs[1]).Name())
+	pcs := make([]uintptr, 1)
+	npcs := runtime.Callers(1, pcs)
+	frames := runtime.CallersFrames(pcs[:npcs])
+	var myName string
+	for {
+		f, more := frames.Next()
+		myName = f.Function
+		if !more {
+			break
+		}
+	}
+	logrusPackage := hooks.GetPackageName(myName)
 	logger.Hooks.Add(&hooks.CallerHook{logrusPackage})
 	logger.Formatter = &TextFormatter{
 		TimestampFormat: "060102 15:04:05",
@@ -33,3 +33,7 @@ func Wrapf(err error, format string, args ...interface{}) error {
 func Cause(err error) error {
 	return errors.Cause(err)
 }
+
+func Errorf(format string, args ...interface{}) error {
+	return errors.Errorf(format, args...)
+}
@@ -44,6 +44,10 @@ type SStructFieldInfo struct {
 	Tags        map[string]string
 }

+func (s *SStructFieldInfo) updateTags(k, v string) {
+	s.Tags[k] = v
+}
+
 func (s SStructFieldInfo) deepCopy() *SStructFieldInfo {
 	scopy := SStructFieldInfo{
 		Ignore:      s.Ignore,
@@ -134,11 +138,11 @@ type SStructFieldValue struct {
 type SStructFieldValueSet []SStructFieldValue

 func FetchStructFieldValueSet(dataValue reflect.Value) SStructFieldValueSet {
-	return fetchStructFieldValueSet(dataValue, false)
+	return fetchStructFieldValueSet(dataValue, false, nil)
 }

 func FetchStructFieldValueSetForWrite(dataValue reflect.Value) SStructFieldValueSet {
-	return fetchStructFieldValueSet(dataValue, true)
+	return fetchStructFieldValueSet(dataValue, true, nil)
 }

 type sStructFieldInfoMap map[string]SStructFieldInfo
@@ -182,7 +186,7 @@ func fetchStructFieldInfos(dataType reflect.Type) sStructFieldInfoMap {
 	return smap
 }

-func fetchStructFieldValueSet(dataValue reflect.Value, allocatePtr bool) SStructFieldValueSet {
+func fetchStructFieldValueSet(dataValue reflect.Value, allocatePtr bool, tags map[string]string) SStructFieldValueSet {
 	fields := SStructFieldValueSet{}
 	dataType := dataValue.Type()
 	fieldInfos := fetchCacheStructFieldInfos(dataType)
@@ -218,7 +222,8 @@ func fetchStructFieldValueSet(dataValue reflect.Value, allocatePtr bool) SStruct
 			// different from how encoding/json handles struct
 			// field of interface type.
 			if fv.Kind() == reflect.Struct && sf.Type != gotypes.TimeType {
-				subfields := fetchStructFieldValueSet(fv, allocatePtr)
+				anonymousTags := utils.TagMap(sf.Tag)
+				subfields := fetchStructFieldValueSet(fv, allocatePtr, anonymousTags)
 				fields = append(fields, subfields...)
 				continue
 			}
@@ -231,6 +236,23 @@ func fetchStructFieldValueSet(dataValue reflect.Value, allocatePtr bool) SStruct
 			})
 		}
 	}
+	if len(tags) > 0 {
+		for i := range fields {
+			fieldName := fields[i].Info.MarshalName()
+			for k, v := range tags {
+				target := ""
+				pos := strings.Index(k, "->")
+				if pos > 0 {
+					target = k[:pos]
+					k = k[pos+2:]
+				}
+				if len(target) > 0 && target != fieldName {
+					continue
+				}
+				fields[i].Info.updateTags(k, v)
+			}
+		}
+	}
 	return fields
 }