From 4abe94ca0b9d203bae21d84f343bef5472ffec97 Mon Sep 17 00:00:00 2001 From: Jian Qiu Date: Thu, 17 Sep 2026 14:15:29 +0800 Subject: [PATCH] fix(mcp-server): read the field tags through the reflectutils accessor (#25690) Co-authored-by: Qiu Jian --- go.mod | 10 +- go.sum | 26 +- pkg/mcp-server/climcgen/mcp_tags.go | 11 +- vendor/modules.txt | 10 +- .../cloudmux/pkg/multicloud/esxi/devtools.go | 1 + .../pkg/multicloud/esxi/virtualmachine.go | 134 ++++++++-- .../x/cloudmux/pkg/multicloud/google/disk.go | 125 ++++++++++ .../pkg/multicloud/google/instance.go | 45 ++-- .../pkg/multicloud/google/securitygroup.go | 15 +- vendor/yunion.io/x/jsonutils/Makefile | 2 +- vendor/yunion.io/x/jsonutils/access.go | 16 +- vendor/yunion.io/x/jsonutils/consts.go | 3 + vendor/yunion.io/x/jsonutils/equals.go | 10 +- vendor/yunion.io/x/jsonutils/errors.go | 6 + vendor/yunion.io/x/jsonutils/interface.go | 12 +- vendor/yunion.io/x/jsonutils/jsonutils.go | 175 +++++++++++-- vendor/yunion.io/x/jsonutils/marshal.go | 21 +- vendor/yunion.io/x/jsonutils/parse_session.go | 33 ++- vendor/yunion.io/x/jsonutils/querystring.go | 10 +- vendor/yunion.io/x/jsonutils/std_json.go | 8 +- vendor/yunion.io/x/jsonutils/string.go | 4 + vendor/yunion.io/x/jsonutils/unmarshal.go | 36 ++- .../x/jsonutils/unmarshal_session.go | 39 ++- vendor/yunion.io/x/jsonutils/write.go | 21 +- .../x/pkg/util/cloudinit/cloudconfig.go | 164 ++++++++++-- vendor/yunion.io/x/pkg/util/encode/encode.go | 7 +- vendor/yunion.io/x/pkg/util/httputils/fs.go | 11 +- .../x/pkg/util/httputils/httputils.go | 233 ++++++++++++++++-- vendor/yunion.io/x/pkg/util/netutils/http.go | 47 +++- vendor/yunion.io/x/pkg/util/netutils/ipv6.go | 4 + .../yunion.io/x/pkg/util/netutils/netutils.go | 95 ++++++- .../x/pkg/util/reflectutils/ambiguous.go | 97 ++++++-- .../yunion.io/x/pkg/util/reflectutils/doc.go | 24 ++ .../x/pkg/util/reflectutils/jsonfield.go | 119 ++++++++- .../x/pkg/util/reflectutils/reflectutils.go | 103 ++++++-- .../yunion.io/x/pkg/util/samlutils/encrypt.go | 37 ++- vendor/yunion.io/x/pkg/util/samlutils/util.go | 17 +- vendor/yunion.io/x/pkg/util/seclib/certs.go | 12 +- vendor/yunion.io/x/pkg/util/seclib/seclib.go | 38 ++- vendor/yunion.io/x/pkg/util/secrules/cut.go | 10 +- .../yunion.io/x/pkg/util/secrules/secrules.go | 18 +- .../x/pkg/util/streamutils/streamutils.go | 38 ++- vendor/yunion.io/x/pkg/utils/crypt.go | 156 ++++++++++++ vendor/yunion.io/x/pkg/utils/misc.go | 27 +- vendor/yunion.io/x/pkg/utils/net.go | 17 +- vendor/yunion.io/x/pkg/utils/pidfile.go | 4 +- vendor/yunion.io/x/pkg/utils/tags.go | 34 ++- vendor/yunion.io/x/sqlchemy/parser.go | 2 +- vendor/yunion.io/x/structarg/structarg.go | 4 +- 49 files changed, 1813 insertions(+), 278 deletions(-) diff --git a/go.mod b/go.mod index 4f1e69a886..270374028e 100644 --- a/go.mod +++ b/go.mod @@ -117,15 +117,15 @@ require ( k8s.io/cri-api v0.28.15 k8s.io/klog/v2 v2.90.1 moul.io/http2curl/v2 v2.3.0 - yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120442-091221507d42 + yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005 - yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c + yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 - yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f + yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758 yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 - yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc - yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c + yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 + yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac ) require ( diff --git a/go.sum b/go.sum index a2ad724442..4bc049bfea 100644 --- a/go.sum +++ b/go.sum @@ -1655,6 +1655,7 @@ golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1972,28 +1973,29 @@ sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120442-091221507d42 h1:UwlzQiM9AqN2p59h3XOuKZM4Zj1hnXJVGlqKnLgCARE= -yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120442-091221507d42/go.mod h1:IwmeqVkld+4XylS80cyVBXyN8fJ5ynAspd/6tcvqJuc= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf h1:weQS1w9PxBeOgQFhXDkhc9UUsvpY/6AWXkk6d3Pc9fk= +yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf/go.mod h1:IwmeqVkld+4XylS80cyVBXyN8fJ5ynAspd/6tcvqJuc= yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005 h1:3sWwcjGXGjG9mLBWa7AyLq+QSi0udTAx21pfVQRFMBE= yunion.io/x/executor v0.0.0-20260312022053-f538abd2b005/go.mod h1:Uxuou9WQIeJXNpy7t2fPLL0BYLvLiMvGQwY7Qc6aSws= -yunion.io/x/jsonutils v0.0.0-20190625054549-a964e1e8a051/go.mod h1:4N0/RVzsYL3kH3WE/H1BjUQdFiWu50JGCFQuuy+Z634= -yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c h1:FthZM1upg4/yF4vmYheo/J7a738sGPjWPKk5kY88sH0= -yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c/go.mod h1:VK4Z93dgiKgAijcSqbMKmGaBMJuHulR16Hz4K015ZPo= +yunion.io/x/jsonutils v0.0.0-20220106020632-953b71a4c3a8/go.mod h1:p0nyMqGA/apTxxyLIU/o1k4V7Vujl2O6ey30L594sYE= +yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea h1:lB7Kwanf7REFaA19YmDc7p0HVkssVRoNQejMlqtpecw= +yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea/go.mod h1:8ctyIiHT+Af1Ldr0xhvZZvuf+kyghlBUuvTy6pOKwbo= yunion.io/x/log v0.0.0-20190514041436-04ce53b17c6b/go.mod h1:+gauLs73omeJAPlsXcevLsJLKixV+sR/E7WSYTSx1fE= yunion.io/x/log v0.0.0-20190629062853-9f6483a7103d/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= +yunion.io/x/log v1.0.0/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 h1:inY5o3LDa/zgsIZuPN0HmpzKIsu/lLgsBmMttuDPGj4= yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91/go.mod h1:LC6f/4FozL0iaAbnFt2eDX9jlsyo3WiOUPm03d7+U4U= yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900 h1:Hu/4ERvoWaN6aiFs4h4/yvVBMeVzb3eDaYTxkVx9Kdo= yunion.io/x/ovsdb v0.0.0-20230306173834-f164f413a900/go.mod h1:0vLkNEhlmA64HViPBAnSTUMrx5QP1CLsxXmxDKQ80tc= -yunion.io/x/pkg v0.0.0-20190620104149-945c25821dbf/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= yunion.io/x/pkg v0.0.0-20190628082551-f4033ba2ea30/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= -yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f h1:eRWHOwgnEK4dMoq7TQ61qZxly3LF0s7tH6nlcahvZiA= -yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f/go.mod h1:0Bwxqd9MA3ACi119/l02FprY/o9gHahmYC2bsSbnVpM= +yunion.io/x/pkg v0.0.0-20200814072949-4f1b541857d6/go.mod h1:t6rEGG2sQ4J7DhFxSZVOTjNd0YO/KlfWQyK1W4tog+E= +yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758 h1:3p/GCOPvDfvlSfovOh9GKxsSa7PPqp//DUjGRX8Wyus= +yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758/go.mod h1:Crp6aUYB+Yq16DFjZPHhHq7ha5HL2DfyPvzCo2LJ+LU= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 h1:1KJ3YYinydPHpDEQRXdr/T8SYcKZ5Er+m489H+PnaQ4= yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1/go.mod h1:0iFKpOs1y4lbCxeOmq3Xx/0AcQoewVPwj62eRluioEo= -yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc h1:di0CJ1Fu/aeY0Lfvr52sQx8VTOm0JyZXJz0aC+7p6No= -yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc/go.mod h1:vCIZpqhZ5Jzaq3tFyrti/vv8BijQKtkzSgNT/uH4H5A= -yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c h1:QuLab2kSRECZRxo4Lo2KcYn6XjQFDGaZ1+x0pYDVVwQ= -yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c/go.mod h1:EP6NSv2C0zzqBDTKumv8hPWLb3XvgMZDHQRfyuOrQng= +yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 h1:xU8Z/Zg+zc199Y58eB3fA8mvOHsA28UCDfd7OuK3QKA= +yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12/go.mod h1:RNiIM7ZFF7VFrk18DbwAa8PYR00YP7Y23XjqKIlfr3I= +yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac h1:hJAthswj6YfWd3GCOBl9eM7mv0+4EUmjnD0b9jdBYi0= +yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac/go.mod h1:DyMCdRyfV5PdEe/BDz2vBbemnmALUF3Rj1JmUUlu3KY= zombiezen.com/go/sqlite v0.13.1 h1:qDzxyWWmMtSSEH5qxamqBFmqA2BLSSbtODi3ojaE02o= zombiezen.com/go/sqlite v0.13.1/go.mod h1:Ht/5Rg3Ae2hoyh1I7gbWtWAl89CNocfqeb/aAMTkJr4= diff --git a/pkg/mcp-server/climcgen/mcp_tags.go b/pkg/mcp-server/climcgen/mcp_tags.go index ea167e2e72..5224e9c078 100644 --- a/pkg/mcp-server/climcgen/mcp_tags.go +++ b/pkg/mcp-server/climcgen/mcp_tags.go @@ -87,17 +87,16 @@ func collectMcpFields(optionsProto interface{}) map[string]mcpFieldMeta { if info == nil { continue } - tagMap := info.Tags - mcpVal, ok := tagMap[TagMCP] + mcpVal, ok := info.Tag(TagMCP) if !ok || mcpVal == "" || mcpVal == "false" { continue } // 即便 json:"-"（Ignore）也保留：climc 仍可能用字段名作为 CLI token（如 MemSpec -> mem-spec） - token, tokOK := tagMap["token"] + token, tokOK := info.Tag("token") if !tokOK { - if jsonName := tagMap["json"]; jsonName != "" && jsonName != "-" { + if jsonName, _ := info.Tag("json"); jsonName != "" && jsonName != "-" { token = info.MarshalName() - } else if alias := tagMap["alias"]; alias != "" { + } else if alias, _ := info.Tag("alias"); alias != "" { token = alias } else { // Ignore 字段的 info.Name 为空，必须用 FieldName @@ -115,7 +114,7 @@ func collectMcpFields(optionsProto interface{}) map[string]mcpFieldMeta { out[fieldTok] = meta } // json 名与字段名不一致时（如 Region / prefer_region）两边都登记 - if jsonName := tagMap["json"]; jsonName != "" && jsonName != "-" { + if jsonName, _ := info.Tag("json"); jsonName != "" && jsonName != "-" { if jsonTok := utils.CamelSplit(info.MarshalName(), "-"); jsonTok != "" && jsonTok != cliToken { out[jsonTok] = meta } diff --git a/vendor/modules.txt b/vendor/modules.txt index 3c8f42f8f8..05b5c51fb9 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2676,7 +2676,7 @@ sigs.k8s.io/structured-merge-diff/v4/value # sigs.k8s.io/yaml v1.3.0 ## explicit; go 1.12 sigs.k8s.io/yaml -# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260904120442-091221507d42 +# yunion.io/x/cloudmux v0.3.10-0-alpha.1.0.20260916101519-866f1e3144cf ## explicit; go 1.24 yunion.io/x/cloudmux/pkg/apis yunion.io/x/cloudmux/pkg/apis/billing @@ -2776,7 +2776,7 @@ yunion.io/x/cloudmux/pkg/multicloud/zstack/provider yunion.io/x/executor/apis yunion.io/x/executor/client yunion.io/x/executor/server -# yunion.io/x/jsonutils v1.0.1-0.20260715075349-615cfb44ff7c +# yunion.io/x/jsonutils v1.0.1-0.20260917025845-3108cd9a32ea ## explicit; go 1.18 yunion.io/x/jsonutils # yunion.io/x/log v1.0.1-0.20240305175729-7cf2d6cd5a91 @@ -2788,7 +2788,7 @@ yunion.io/x/log/hooks yunion.io/x/ovsdb/cli_util yunion.io/x/ovsdb/schema/ovn_nb yunion.io/x/ovsdb/types -# yunion.io/x/pkg v1.10.4-0.20260722094907-0da70a14584f +# yunion.io/x/pkg v1.10.4-0.20260916163305-b7743fa5e758 ## explicit; go 1.18 yunion.io/x/pkg/appctx yunion.io/x/pkg/errors @@ -2842,7 +2842,7 @@ yunion.io/x/pkg/utils # yunion.io/x/s3cli v0.0.0-20241221171442-1c11599d28e1 ## explicit; go 1.12 yunion.io/x/s3cli -# yunion.io/x/sqlchemy v1.1.3-0.20260807102031-1d7e260d0dcc +# yunion.io/x/sqlchemy v1.1.3-0.20260917035232-d682485b3a12 ## explicit; go 1.17 yunion.io/x/sqlchemy yunion.io/x/sqlchemy/backends @@ -2850,7 +2850,7 @@ yunion.io/x/sqlchemy/backends/clickhouse yunion.io/x/sqlchemy/backends/dameng yunion.io/x/sqlchemy/backends/mysql yunion.io/x/sqlchemy/backends/sqlite -# yunion.io/x/structarg v0.0.0-20231017124457-df4d5009457c +# yunion.io/x/structarg v0.0.0-20260917033311-96c7653334ac ## explicit; go 1.12 yunion.io/x/structarg # zombiezen.com/go/sqlite v0.13.1 diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go index f277cc1353..47b174c111 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/devtools.go @@ -33,6 +33,7 @@ type SDiskConfig struct { UnitNumber int32 Key int32 ImagePath string + DestPath string IsRoot bool Datastore *SDatastore Preallocation string diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go index e162c82860..f072453492 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/esxi/virtualmachine.go @@ -285,17 +285,105 @@ func (svm *SVirtualMachine) DoRebuildRoot(ctx context.Context, imagePath string, return svm.rebuildDisk(ctx, &svm.vdisks[0], imagePath, uefi) } +func rebuildRootDiskBackupPath(filename string) string { + const suffix = ".vmdk" + if strings.HasSuffix(strings.ToLower(filename), suffix) { + return filename[:len(filename)-len(suffix)] + ".rebuild-bak.vmdk" + } + return filename + ".rebuild-bak.vmdk" +} + +func (svm *SVirtualMachine) moveVirtualDiskFile(ctx context.Context, ds *SDatastore, src, dst string) error { + obj, err := ds.getDatastoreObj(ctx) + if err != nil { + return errors.Wrapf(err, "getDatastoreObj") + } + fm := obj.NewFileManager(ds.datacenter.getObjectDatacenter(), true) + err = fm.Move(ctx, src, dst) + if err != nil { + return errors.Wrapf(err, "move %s -> %s", src, dst) + } + return nil +} + +func (svm *SVirtualMachine) reattachNonRootDisks(ctx context.Context) error { + for i := 1; i < len(svm.vdisks); i++ { + err := svm.doDetachDisk(ctx, &svm.vdisks[i], false) + if err != nil { + return errors.Wrapf(err, "doDetachDisk %d", i) + } + err = svm.doAttachDisk(ctx, &svm.vdisks[i]) + if err != nil { + return errors.Wrapf(err, "doAttachDisk: %d", i) + } + } + return nil +} + +func (svm *SVirtualMachine) attachDisksInOriginalOrder(ctx context.Context, root *SVirtualDisk) error { + for i := 1; i < len(svm.vdisks); i++ { + err := svm.doDetachDisk(ctx, &svm.vdisks[i], false) + if err != nil { + return errors.Wrapf(err, "doDetachDisk %d", i) + } + } + err := svm.doAttachDisk(ctx, root) + if err != nil { + for i := 1; i < len(svm.vdisks); i++ { + if attachErr := svm.doAttachDisk(ctx, &svm.vdisks[i]); attachErr != nil { + log.Errorf("reattach disk %d after root attach failed: %s", i, attachErr) + } + } + return errors.Wrap(err, "doAttachDisk root") + } + for i := 1; i < len(svm.vdisks); i++ { + err = svm.doAttachDisk(ctx, &svm.vdisks[i]) + if err != nil { + return errors.Wrapf(err, "doAttachDisk: %d", i) + } + } + return nil +} + +func (svm *SVirtualMachine) restoreRootDiskAfterRebuild(ctx context.Context, disk *SVirtualDisk, ds *SDatastore, origPath, backupPath string) error { + if err := ds.Delete2(ctx, origPath, false, true); err != nil { + log.Errorf("delete rebuilt root disk %s before restore: %s", origPath, err) + } + if err := svm.moveVirtualDiskFile(ctx, ds, backupPath, origPath); err != nil { + return errors.Wrap(err, "rename backup root disk") + } + return svm.attachDisksInOriginalOrder(ctx, disk) +} + func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk, imagePath string, uefi bool) error { uuid := disk.GetId() sizeMb := disk.GetDiskSizeMB() diskKey := disk.getKey() ctlKey := disk.getControllerKey() unitNumber := *disk.dev.GetVirtualDevice().UnitNumber + origPath := disk.GetFilename() + backupPath := rebuildRootDiskBackupPath(origPath) - err := svm.doDetachAndDeleteDisk(ctx, disk) + istorage, err := disk.GetIStorage() + if err != nil { + return errors.Wrap(err, "GetIStorage") + } + ds := istorage.(*SDatastore) + + err = svm.doDetachDisk(ctx, disk, false) if err != nil { return err } + + err = svm.moveVirtualDiskFile(ctx, ds, origPath, backupPath) + if err != nil { + if attachErr := svm.attachDisksInOriginalOrder(ctx, disk); attachErr != nil { + log.Errorf("reattach disks in original order after backup failed: %s", attachErr) + } + return errors.Wrapf(err, "backup root disk %s", origPath) + } + log.Infof("backup root disk %s -> %s", origPath, backupPath) + err = svm.createDiskInternal(ctx, SDiskConfig{ Uefi: uefi, SizeMb: int64(sizeMb), @@ -304,20 +392,21 @@ func (svm *SVirtualMachine) rebuildDisk(ctx context.Context, disk *SVirtualDisk, UnitNumber: unitNumber, Key: diskKey, ImagePath: imagePath, + DestPath: origPath, IsRoot: len(imagePath) > 0, + Datastore: ds, }, false) if err != nil { + if restoreErr := svm.restoreRootDiskAfterRebuild(ctx, disk, ds, origPath, backupPath); restoreErr != nil { + log.Errorf("restore root disk %s from %s failed: %s", origPath, backupPath, restoreErr) + } return errors.Wrapf(err, "createDiskInternal") } - for i := 1; i < len(svm.vdisks); i++ { - err = svm.doDetachDisk(ctx, &svm.vdisks[i], false) - if err != nil { - return errors.Wrapf(err, "doDetachDisk %d", i) - } - err = svm.doAttachDisk(ctx, &svm.vdisks[i]) - if err != nil { - return errors.Wrapf(err, "doAttachDisk: %d", i) - } + if err = svm.reattachNonRootDisks(ctx); err != nil { + return err + } + if err = ds.Delete2(ctx, backupPath, false, true); err != nil { + log.Errorf("delete backup root disk %s: %s", backupPath, err) } return nil } @@ -1256,10 +1345,24 @@ func (svm *SVirtualMachine) GetRootImagePath() (string, error) { return path, nil } -func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath string) (string, error) { - newImagePath, datastore, err := svm.getDatastoreAndRootImagePath(false) - if err != nil { - return "", errors.Wrapf(err, "GetRootImagePath") +func (svm *SVirtualMachine) CopyRootDisk(ctx context.Context, imagePath, destPath string, datastore *SDatastore) (string, error) { + var ( + newImagePath string + err error + ) + if len(destPath) > 0 { + newImagePath = destPath + if datastore == nil { + _, datastore, err = svm.getDatastoreAndRootImagePath(false) + if err != nil { + return "", errors.Wrapf(err, "GetRootImagePath") + } + } + } else { + newImagePath, datastore, err = svm.getDatastoreAndRootImagePath(false) + if err != nil { + return "", errors.Wrapf(err, "GetRootImagePath") + } } ds, err := datastore.getDatastoreObj(ctx) if err != nil { @@ -1278,7 +1381,7 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi // copy disk if len(config.ImagePath) > 0 { config.IsRoot = true - config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath) + config.ImagePath, err = svm.CopyRootDisk(ctx, config.ImagePath, config.DestPath, config.Datastore) if err != nil { return errors.Wrap(err, "unable to copyRootDisk") } @@ -1327,7 +1430,6 @@ func (svm *SVirtualMachine) createDiskWithDeviceChange(ctx context.Context, devi } func (svm *SVirtualMachine) createDiskInternal(ctx context.Context, config SDiskConfig, check bool) error { - return svm.createDiskWithDeviceChange(ctx, nil, config, check) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go index fbce32ac97..44400ae0d4 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/disk.go @@ -220,6 +220,130 @@ func (disk *SDisk) GetProjectId() string { return disk.storage.zone.region.GetProjectId() } +func maxInt(a, b int) int { + if a > b { + return a + } + return b +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +func diskTypeName(storageType string) string { + if idx := strings.LastIndex(storageType, "/"); idx >= 0 { + return storageType[idx+1:] + } + return storageType +} + +func inRange(val, minVal, maxVal int) bool { + return val >= minVal && val <= maxVal +} + +// hyperdiskBalancedIopsRange 计算 balanced / balanced-ha 的 IOPS 区间 +func hyperdiskBalancedIopsRange(sizeGb, maxIopsCap int) (minIops, maxIops int, ok bool) { + switch { + case sizeGb == 4: + return 2000, 2000, true + case sizeGb == 5: + return 2500, 2500, true + case sizeGb >= 6: + return 3000, minInt(500*sizeGb, maxIopsCap), true + default: + return 0, 0, false + } +} + +func hyperdiskBalancedDefaultIops(sizeGb, maxIopsCap int) int { + if sizeGb <= 6 { + return 500 * sizeGb + } + // 6 GiB ~ 26.666667 TiB ≈ 27307 GiB + if sizeGb <= 27307 { + return minInt(6*sizeGb+3000, maxIopsCap) + } + return maxIopsCap +} + +func hyperdiskBalancedThroughputRange(iops int) (minTp, maxTp int) { + return maxInt(140, iops/256), minInt(2400, iops/4) +} + +func setHyperdiskBalancedPerformance(params map[string]interface{}, sizeGb, iops, throughput, maxIopsCap int) { + minIops, maxIops, ok := hyperdiskBalancedIopsRange(sizeGb, maxIopsCap) + if !ok { + return + } + effectiveIops := iops + if iops > 0 && inRange(iops, minIops, maxIops) { + params["provisionedIops"] = iops + } else if throughput > 0 { + effectiveIops = hyperdiskBalancedDefaultIops(sizeGb, maxIopsCap) + } else { + return + } + if throughput > 0 { + minTp, maxTp := hyperdiskBalancedThroughputRange(effectiveIops) + if inRange(throughput, minTp, maxTp) { + params["provisionedThroughput"] = throughput + } + } +} + +// setDiskProvisionedPerformance 按 GCP 盘型区间+容量公式写入 IOPS/吞吐，超范围或不支持则丢弃该参数 +func setDiskProvisionedPerformance(params map[string]interface{}, storageType string, sizeGb, iops, throughput int) { + if iops <= 0 && throughput <= 0 { + return + } + diskType := diskTypeName(storageType) + switch diskType { + case api.STORAGE_GOOGLE_PD_EXTREME: + if iops > 0 && inRange(iops, 2500, 120000) { + params["provisionedIops"] = iops + } + case api.STORAGE_GOOGLE_HYPERDISK_EXTREME: + if sizeGb < 64 || iops <= 0 { + return + } + minIops := 2 * sizeGb + maxIops := 350000 + if sizeGb <= 291 { + maxIops = 1200 * sizeGb + } + if inRange(iops, minIops, maxIops) { + params["provisionedIops"] = iops + } + case api.STORAGE_GOOGLE_HYPERDISK_BALANCED: + setHyperdiskBalancedPerformance(params, sizeGb, iops, throughput, 160000) + case "hyperdisk-balanced-high-availability": + setHyperdiskBalancedPerformance(params, sizeGb, iops, throughput, 100000) + case api.STORAGE_GOOGLE_HYPERDISK_THROUGHPUT: + if throughput <= 0 || sizeGb < 2048 || sizeGb > 32768 { + return + } + sizeTiB := float64(sizeGb) / 1024.0 + minTp := maxInt(20, int(5*sizeTiB)) + maxTp := minInt(int(90*sizeTiB), 2400) + if inRange(throughput, minTp, maxTp) { + params["provisionedThroughput"] = throughput + } + case api.STORAGE_GOOGLE_HYPERDISK_ML: + if throughput <= 0 { + return + } + minTp := maxInt(400, int(0.12*float64(sizeGb))) + maxTp := minInt(2097152, 1600*sizeGb) + if inRange(throughput, minTp, maxTp) { + params["provisionedThroughput"] = throughput + } + } +} + func (region *SRegion) CreateDisk(zone string, storageType string, opts *cloudprovider.DiskCreateConfig) (*SDisk, error) { if !strings.HasPrefix(storageType, GOOGLE_COMPUTE_DOMAIN) { storageType = fmt.Sprintf("projects/%s/zones/%s/diskTypes/%s", region.GetProjectId(), zone, storageType) @@ -240,6 +364,7 @@ func (region *SRegion) CreateDisk(zone string, storageType string, opts *cloudpr if len(opts.ImageId) > 0 { body["sourceImage"] = opts.ImageId } + setDiskProvisionedPerformance(body, storageType, opts.SizeGb, opts.Iops, opts.Throughput) disk := &SDisk{} resource := fmt.Sprintf("zones/%s/disks", zone) err := region.Insert(resource, jsonutils.Marshal(body), disk) diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go index 2f391846c7..0226cf3d3c 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/instance.go @@ -581,30 +581,35 @@ func (region *SRegion) _createVM(zone string, desc *cloudprovider.SManagedVMCrea labels[encode.EncodeGoogleLabel(k)] = encode.EncodeGoogleLabel(v) } + sysDiskParams := map[string]interface{}{ + "diskName": normalizeString(desc.SysDisk.Name), + "sourceImage": desc.ExternalImageId, + "diskSizeGb": desc.SysDisk.SizeGB, + "diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, desc.SysDisk.StorageType), + "labels": labels, + } + setDiskProvisionedPerformance(sysDiskParams, desc.SysDisk.StorageType, desc.SysDisk.SizeGB, desc.SysDisk.Iops, desc.SysDisk.Throughput) disks = append(disks, map[string]interface{}{ - "boot": true, - "initializeParams": map[string]interface{}{ - "diskName": normalizeString(desc.SysDisk.Name), - "sourceImage": desc.ExternalImageId, - "diskSizeGb": desc.SysDisk.SizeGB, - "diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, desc.SysDisk.StorageType), - "labels": labels, - }, - "autoDelete": true, + "boot": true, + "initializeParams": sysDiskParams, + "autoDelete": true, }) - for _, disk := range desc.DataDisks { - if len(disk.Name) == 0 { - disk.Name = fmt.Sprintf("vdisk-%s-%d", desc.Name, time.Now().UnixNano()) + for i, disk := range desc.DataDisks { + name := disk.Name + if len(name) == 0 { + name = fmt.Sprintf("vdisk-%s-%d-%d", desc.Name, i, time.Now().UnixNano()) } + dataDiskParams := map[string]interface{}{ + "diskName": normalizeString(name), + "diskSizeGb": disk.SizeGB, + "diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.StorageType), + "labels": labels, + } + setDiskProvisionedPerformance(dataDiskParams, disk.StorageType, disk.SizeGB, disk.Iops, disk.Throughput) disks = append(disks, map[string]interface{}{ - "boot": false, - "initializeParams": map[string]interface{}{ - "diskName": normalizeString(disk.Name), - "diskSizeGb": disk.SizeGB, - "diskType": fmt.Sprintf("zones/%s/diskTypes/%s", zone, disk.StorageType), - "labels": labels, - }, - "autoDelete": true, + "boot": false, + "initializeParams": dataDiskParams, + "autoDelete": true, }) } diff --git a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/securitygroup.go b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/securitygroup.go index 047a4f6021..7a353b4813 100644 --- a/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/securitygroup.go +++ b/vendor/yunion.io/x/cloudmux/pkg/multicloud/google/securitygroup.go @@ -15,6 +15,7 @@ package google import ( + "crypto/sha1" "fmt" "regexp" "strings" @@ -202,14 +203,16 @@ func normalizeString(input string) string { cleaned = regexp.MustCompile(`-$`).ReplaceAllString(cleaned, "") // 5. 限制长度为63个字符（域名标签的最大长度） + // 超长时保留前缀并追加原文 hash，避免仅截断前缀导致不同名称冲突 if len(cleaned) > 63 { - cleaned = cleaned[:63] - // 确保截断后不以连字符结尾 - cleaned = regexp.MustCompile(`-$`).ReplaceAllString(cleaned, "") - // 如果截断后为空，添加一个数字1 - if len(cleaned) == 0 { - cleaned = "1" + sum := sha1.Sum([]byte(cleaned)) + suffix := fmt.Sprintf("%x", sum[:4]) // 8 hex chars + prefixLen := 63 - 1 - len(suffix) + prefix := strings.TrimRight(cleaned[:prefixLen], "-") + if len(prefix) == 0 { + prefix = "a" } + cleaned = prefix + "-" + suffix } return cleaned diff --git a/vendor/yunion.io/x/jsonutils/Makefile b/vendor/yunion.io/x/jsonutils/Makefile index f101cd8b04..2ab03a3a49 100644 --- a/vendor/yunion.io/x/jsonutils/Makefile +++ b/vendor/yunion.io/x/jsonutils/Makefile @@ -6,4 +6,4 @@ GOPROXY ?= direct mod: GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x \ go get -d $(patsubst %,%@master,$(shell GO111MODULE=on go mod edit -print | sed -n -e 's|.*\(yunion.io/x/[a-z].*\) v.*|\1|p')) - go mod tidy + GOPROXY=$(GOPROXY) GONOSUMDB=yunion.io/x go mod tidy diff --git a/vendor/yunion.io/x/jsonutils/access.go b/vendor/yunion.io/x/jsonutils/access.go index 7d687d3e13..f236cc956c 100644 --- a/vendor/yunion.io/x/jsonutils/access.go +++ b/vendor/yunion.io/x/jsonutils/access.go @@ -20,10 +20,18 @@ import ( "time" "yunion.io/x/pkg/errors" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" "yunion.io/x/pkg/util/timeutils" ) +func normalizeJSONObject(obj JSONObject) JSONObject { + if gotypes.IsNil(obj) { + return JSONNull + } + return obj +} + type JSONPair struct { key string val JSONObject @@ -40,7 +48,7 @@ func NewDict(objs ...JSONPair) *JSONDict { func NewArray(objs ...JSONObject) *JSONArray { arr := JSONArray{data: make([]JSONObject, 0, len(objs))} for _, o := range objs { - arr.data = append(arr.data, o) + arr.data = append(arr.data, normalizeJSONObject(o)) } return &arr } @@ -74,7 +82,7 @@ func NewBool(val bool) *JSONBool { } func (this *JSONDict) Set(key string, value JSONObject) { - this.data = sortedmap.Add(this.data, key, value) + this.data = sortedmap.Add(this.data, key, normalizeJSONObject(value)) } func (this *JSONDict) Remove(key string) bool { @@ -130,12 +138,12 @@ func (this *JSONDict) Add(o JSONObject, keys ...string) error { } func (this *JSONArray) SetAt(idx int, obj JSONObject) { - this.data[idx] = obj + this.data[idx] = normalizeJSONObject(obj) } func (this *JSONArray) Add(objs ...JSONObject) { for _, o := range objs { - this.data = append(this.data, o) + this.data = append(this.data, normalizeJSONObject(o)) } } diff --git a/vendor/yunion.io/x/jsonutils/consts.go b/vendor/yunion.io/x/jsonutils/consts.go index 53eb20fd69..7658de9df8 100644 --- a/vendor/yunion.io/x/jsonutils/consts.go +++ b/vendor/yunion.io/x/jsonutils/consts.go @@ -20,4 +20,7 @@ import ( const ( TAG_DEPRECATED_BY = reflectutils.TAG_DEPRECATED_BY + + // maxParseDepth is the maximum nesting level accepted by the parser + maxParseDepth = 10000 ) diff --git a/vendor/yunion.io/x/jsonutils/equals.go b/vendor/yunion.io/x/jsonutils/equals.go index b49cc0f7cb..ad341834fb 100644 --- a/vendor/yunion.io/x/jsonutils/equals.go +++ b/vendor/yunion.io/x/jsonutils/equals.go @@ -15,6 +15,7 @@ package jsonutils import ( + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" ) @@ -51,7 +52,14 @@ func (arr *JSONArray) Equals(json JSONObject) bool { return false } for i, v := range arr.data { - if !v.Equals(arr2.data[i]) { + oth := arr2.data[i] + if gotypes.IsNil(v) || gotypes.IsNil(oth) { + if gotypes.IsNil(v) != gotypes.IsNil(oth) { + return false + } + continue + } + if !v.Equals(oth) { return false } } diff --git a/vendor/yunion.io/x/jsonutils/errors.go b/vendor/yunion.io/x/jsonutils/errors.go index c6851ccacf..0038481bc9 100644 --- a/vendor/yunion.io/x/jsonutils/errors.go +++ b/vendor/yunion.io/x/jsonutils/errors.go @@ -26,10 +26,16 @@ const ( ErrInvalidHex = errors.Error("invalid hex") ErrInvalidRune = errors.Error("invalid 4 byte rune") + ErrInvalidNodeId = errors.Error("invalid node id") + ErrDuplicateNodeId = errors.Error("duplicate node id") + ErrNestedTooDeep = errors.Error("json nested too deep") + ErrNodeReferenceDisabled = errors.Error("node reference is not enabled, see ParseTrusted") + ErrTypeMismatch = errors.Error("unmarshal type mismatch") ErrArrayLengthMismatch = errors.Error("unmarshal array length mismatch") ErrInterfaceUnsupported = errors.Error("do not known how to deserialize json into this interface type") ErrMapKeyMustString = errors.Error("map key must be string") + ErrNodeNotFound = errors.Error("no node found for the reference") ErrMissingInputField = errors.Error("missing input field") ErrNilInputField = errors.Error("nil input field") diff --git a/vendor/yunion.io/x/jsonutils/interface.go b/vendor/yunion.io/x/jsonutils/interface.go index 7d2b7edcd3..af6edccd40 100644 --- a/vendor/yunion.io/x/jsonutils/interface.go +++ b/vendor/yunion.io/x/jsonutils/interface.go @@ -15,6 +15,7 @@ package jsonutils import ( + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" ) @@ -41,6 +42,10 @@ func (self *JSONString) Interface() interface{} { func (self *JSONArray) Interface() interface{} { ret := make([]interface{}, len(self.data)) for i := 0; i < len(self.data); i += 1 { + if gotypes.IsNil(self.data[i]) { + ret[i] = nil + continue + } ret[i] = self.data[i].Interface() } return ret @@ -51,7 +56,12 @@ func (self *JSONDict) Interface() interface{} { for iter := sortedmap.NewIterator(self.data); iter.HasMore(); iter.Next() { k, v := iter.Get() - mapping[k] = v.(JSONObject).Interface() + jo, _ := v.(JSONObject) + if gotypes.IsNil(jo) { + mapping[k] = nil + continue + } + mapping[k] = jo.Interface() } return mapping diff --git a/vendor/yunion.io/x/jsonutils/jsonutils.go b/vendor/yunion.io/x/jsonutils/jsonutils.go index 97e68830db..36f007b0b8 100644 --- a/vendor/yunion.io/x/jsonutils/jsonutils.go +++ b/vendor/yunion.io/x/jsonutils/jsonutils.go @@ -16,10 +16,13 @@ package jsonutils import ( "bytes" + "math" "reflect" + "sort" "strconv" "strings" "time" + "unicode/utf16" "unicode/utf8" "yunion.io/x/log" @@ -117,6 +120,11 @@ func skipEmpty(str []byte, offset int) int { return i } +// isFiniteFloat reports whether the value has a json representation +func isFiniteFloat(val float64) bool { + return !math.IsNaN(val) && !math.IsInf(val, 0) +} + func hexchar2num(v byte) (byte, error) { switch { case v >= '0' && v <= '9': @@ -183,9 +191,22 @@ ret: if e != nil { return "", i, NewJSONError(str, i, e.Error()) } + i += 4 + if utf16.IsSurrogate(r) { + // a character outside the BMP is written as a + // surrogate pair, e.g. 😀 + if i+6 <= len(str) && str[i] == '\\' && str[i+1] == 'u' { + r2, e2 := hexstr2rune(str[i+2 : i+6]) + if e2 == nil { + if combined := utf16.DecodeRune(r, r2); combined != utf8.RuneError { + r = combined + i += 6 + } + } + } + } runen = utf8.EncodeRune(runebytes, r) buffer = append(buffer, runebytes[0:runen]...) - i += 4 case 'x': i++ if i+2 >= len(str) { @@ -253,13 +274,23 @@ ret2: return string(str[offset:i]), false, i, nil } +// isNodeReference reports whether the token has the form of a node +// reference, that is a bare with an integer N +func isNodeReference(val string) bool { + if len(val) < 3 || val[0] != '<' || val[len(val)-1] != '>' { + return false + } + _, err := strconv.ParseInt(val[1:len(val)-1], 10, 64) + return err == nil +} + func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, int, error) { val, quote, i, e := parseString(str, offset) if e != nil { return nil, i, errors.Wrap(e, "parseString") } else if quote { return &JSONString{data: val}, i, nil - } else if val[0] == '<' && val[len(val)-1] == '>' { + } else if s.allowNodeReference && len(val) > 1 && val[0] == '<' && val[len(val)-1] == '>' { // Pointer val = val[1 : len(val)-1] ival, err := strconv.ParseInt(val, 10, 64) @@ -272,6 +303,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, } s.saveReferer(nodeId, ptr) return ptr, i, nil + } else if !s.allowNodeReference && isNodeReference(val) { + // a node reference can not be left unresolved and kept as a plain + // value: a caller could not tell it apart from a real string + return nil, i, errors.Wrap(ErrNodeReferenceDisabled, val) } else { lval := strings.ToLower(val) if len(lval) == 0 || lval == "null" || lval == "none" { @@ -288,9 +323,10 @@ func (s *sJsonParseSession) parseJSONValue(str []byte, offset int) (JSONObject, return &JSONInt{data: ival}, i, nil } fval, err := strconv.ParseFloat(val, 64) - if err == nil { + if err == nil && isFiniteFloat(fval) { return &JSONFloat{data: fval}, i, nil } + // nan and +-inf have no json representation, keep them as strings return &JSONString{data: val}, i, nil } } @@ -357,12 +393,33 @@ func escapeJsonChar(sb *strings.Builder, ch byte) { } } +// escapeJsonByte writes a byte that is not part of a valid utf-8 sequence +// as a \xXX escape, which parseQuoteString reads back unchanged +func escapeJsonByte(sb *strings.Builder, ch byte) { + const hexdigits = "0123456789abcdef" + sb.Write([]byte{'\\', 'x', hexdigits[ch>>4], hexdigits[ch&0xf]}) +} + func quoteString(str string) string { sb := &strings.Builder{} sb.Grow(len(str) + 2) sb.WriteByte('"') - for i := 0; i < len(str); i += 1 { - escapeJsonChar(sb, str[i]) + for i := 0; i < len(str); { + ch := str[i] + if ch < utf8.RuneSelf { + escapeJsonChar(sb, ch) + i++ + continue + } + r, size := utf8.DecodeRuneInString(str[i:]) + if r == utf8.RuneError && size <= 1 { + // keep a non utf-8 byte reversible instead of writing it out + escapeJsonByte(sb, ch) + i++ + continue + } + sb.WriteString(str[i : i+size]) + i += size } sb.WriteByte('"') return sb.String() @@ -431,6 +488,11 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted var e error = nil var key string var stop = false + // collect the keys first so that the sorted map can be built in key + // order: adding to a sorted map out of order shifts the whole tail + // on every insert + values := make(map[string]JSONObject) + keys := make([]string, 0) for !stop && i < len(str) { i = skipEmpty(str, i) if i >= len(str) { @@ -474,11 +536,18 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted if e != nil { return smap, i, nodeId, errors.Wrap(e, "parse misc") } - if key == jsonPointerKey { + if s.allowNodeReference && key == jsonPointerKey { // node id - nodeId = int(val.(*JSONInt).data) + jval, ok := val.(*JSONInt) + if !ok { + return smap, i, nodeId, errors.Wrap(ErrInvalidNodeId, jsonPointerKey) + } + nodeId = int(jval.data) } else { - smap = sortedmap.Add(smap, key, val) + if _, ok := values[key]; !ok { + keys = append(keys, key) + } + values[key] = val } i = skipEmpty(str, i) if i >= len(str) { @@ -494,6 +563,10 @@ func (s *sJsonParseSession) parseDict(str []byte, offset int) (sortedmap.SSorted return smap, i, nodeId, NewJSONError(str, i, "Unexpected char") } } + sort.Strings(keys) + for _, key := range keys { + smap = sortedmap.Add(smap, key, values[key]) + } return smap, i, nodeId, nil } @@ -552,12 +625,20 @@ func (s *sJsonParseSession) parseArray(str []byte, offset int) ([]JSONObject, in } func (this *JSONDict) parse(s *sJsonParseSession, str []byte, offset int) (int, error) { + e := s.enter() + if e != nil { + return offset, errors.Wrap(e, "enter") + } + defer s.leave() smap, i, nodeId, e := s.parseDict(str, offset) if e == nil { this.nodeId = nodeId this.data = smap if this.nodeId > 0 { - s.saveNode(nodeId, this) + e = s.saveNode(nodeId, this) + if e != nil { + return i, errors.Wrap(e, "saveNode") + } } return i, nil } @@ -591,17 +672,21 @@ func (this *JSONDict) prettyString(level int) string { buffer.WriteByte('\n') buffer.WriteString(tab) buffer.WriteString(" ") - buffer.WriteByte('"') - buffer.WriteString(k) - buffer.WriteString("\":") - _, okdict := v.(*JSONDict) - _, okarray := v.(*JSONArray) - if okdict || okarray { - buffer.WriteByte('\n') - buffer.WriteString(v.prettyString(level + 2)) - } else { + buffer.WriteString(quoteString(k)) + buffer.WriteByte(':') + if gotypes.IsNil(v) { buffer.WriteByte(' ') - buffer.WriteString(v.String()) + buffer.WriteString("null") + } else { + _, okdict := v.(*JSONDict) + _, okarray := v.(*JSONArray) + if okdict || okarray { + buffer.WriteByte('\n') + buffer.WriteString(v.prettyString(level + 2)) + } else { + buffer.WriteByte(' ') + buffer.WriteString(v.String()) + } } idx++ } @@ -614,6 +699,11 @@ func (this *JSONDict) prettyString(level int) string { } func (this *JSONArray) parse(s *sJsonParseSession, str []byte, offset int) (int, error) { + e := s.enter() + if e != nil { + return offset, errors.Wrap(e, "enter") + } + defer s.leave() val, i, e := s.parseArray(str, offset) if e == nil { this.data = val @@ -639,7 +729,12 @@ func (this *JSONArray) prettyString(level int) string { buffer.WriteString(",") } buffer.WriteByte('\n') - buffer.WriteString(v.prettyString(level + 1)) + if gotypes.IsNil(v) { + buffer.WriteString(tab) + buffer.WriteString(" null") + } else { + buffer.WriteString(v.prettyString(level + 1)) + } } if len(this.data) > 0 { buffer.WriteByte('\n') @@ -654,12 +749,46 @@ func ParseString(str string) (JSONObject, error) { } func Parse(str []byte) (JSONObject, error) { - json, _, err := ParseStream(str, 0) + json, offset, err := ParseStream(str, 0) + if err != nil { + return nil, err + } + if i := skipEmpty(str, offset); i < len(str) { + return nil, NewJSONError(str, i, "Unexpected content after the value") + } + return json, nil +} + +// ParseStream parses one value starting at offset, it returns the value and +// the offset just after it, so that a stream of concatenated values can be +// walked. Node references are not resolved, see ParseTrusted. +func ParseStream(str []byte, offset int) (JSONObject, int, error) { + return parseStream(str, offset, false) +} + +// ParseTrusted parses a document from a trusted source, resolving the node +// references that Marshal writes for a cyclic object: the ___jnid_ key inside +// an object and a bare value referring to it. +// +// A document from an untrusted source must be parsed with Parse instead. A +// resolved reference makes two fields of the target struct point at the same +// object, which is what the round trip of a cyclic object needs, but it also +// lets a forged document do the same. +// +// Note that Marshal needs this syntax to terminate on a cyclic object, so it +// keeps writing it either way. +func ParseTrusted(str []byte) (JSONObject, error) { + json, _, err := parseStream(str, 0, true) return json, err } -func ParseStream(str []byte, offset int) (JSONObject, int, error) { - s := newJsonParseSession() +// ParseTrustedString is ParseTrusted for a string +func ParseTrustedString(str string) (JSONObject, error) { + return ParseTrusted([]byte(str)) +} + +func parseStream(str []byte, offset int, allowNodeReference bool) (JSONObject, int, error) { + s := newJsonParseSession(allowNodeReference) i := offset i = skipEmpty(str, i) var val JSONObject = nil diff --git a/vendor/yunion.io/x/jsonutils/marshal.go b/vendor/yunion.io/x/jsonutils/marshal.go index 12d6c1e508..5b8f41ea53 100644 --- a/vendor/yunion.io/x/jsonutils/marshal.go +++ b/vendor/yunion.io/x/jsonutils/marshal.go @@ -48,7 +48,7 @@ func (s *sJsonMarshalSession) marshalSlice(val reflect.Value, info *reflectutils objs := make([]JSONObject, 0) for i := 0; i < val.Len(); i += 1 { val := s.marshalValue(val.Index(i), nil, omitEmpty) - if val != nil { + if !gotypes.IsNil(val) { objs = append(objs, val) } } @@ -84,7 +84,7 @@ func (s *sJsonMarshalSession) marshalMap(val reflect.Value, info *reflectutils.S for i := 0; i < len(keys); i += 1 { key := keys[i] val := s.marshalValue(val.MapIndex(key), nil, omitEmpty) - if val != nil { + if !gotypes.IsNil(val) { objPairs = append(objPairs, JSONPair{key: fmt.Sprintf("%s", key), val: val}) } } @@ -128,12 +128,12 @@ func (s *sJsonMarshalSession) struct2JSONPairs(val reflect.Value, omitEmpty bool continue } key := jsonInfo.MarshalName() - if deprecatedBy, ok := fields[i].Info.Tags[TAG_DEPRECATED_BY]; ok { + if deprecatedBy, ok := fields[i].Info.Tag(TAG_DEPRECATED_BY); ok { depFields[key] = deprecatedBy continue } val := s.marshalValue(fields[i].Value, jsonInfo, omitEmpty) - if val != nil { + if !gotypes.IsNil(val) { objPair := JSONPair{key: key, val: val} objPairs = append(objPairs, objPair) } @@ -181,6 +181,9 @@ func marshalFloat64(val float64, info *reflectutils.SStructFieldInfo, bit int, o return nil } else if info != nil && info.ForceString { return NewString(fmt.Sprintf("%f", val)) + } else if !isFiniteFloat(val) { + // nan and +-inf have no json representation + return JSONNull } else { return NewFloat64(val) } @@ -191,6 +194,9 @@ func marshalFloat32(val float32, info *reflectutils.SStructFieldInfo, bit int, o return nil } else if info != nil && info.ForceString { return NewString(fmt.Sprintf("%f", val)) + } else if !isFiniteFloat(float64(val)) { + // nan and +-inf have no json representation + return JSONNull } else { return NewFloat32(val) } @@ -243,6 +249,13 @@ func marshalTime(val time.Time, info *reflectutils.SStructFieldInfo, omitEmpty b } } +// Marshal converts an object to a JSONObject, which can be written out with +// String. +// +// A cyclic object is written with the node reference syntax, that is the +// ___jnid_ key inside an object plus a bare value referring to it. That +// syntax is not standard json and its result is not a valid json document, +// and Parse does not resolve it: read such a document back with ParseTrusted. func Marshal(obj interface{}) JSONObject { if obj == nil { return JSONNull diff --git a/vendor/yunion.io/x/jsonutils/parse_session.go b/vendor/yunion.io/x/jsonutils/parse_session.go index 632dce9d39..f9a0485b91 100644 --- a/vendor/yunion.io/x/jsonutils/parse_session.go +++ b/vendor/yunion.io/x/jsonutils/parse_session.go @@ -15,7 +15,7 @@ package jsonutils import ( - "fmt" + "yunion.io/x/pkg/errors" ) type sNodeReferer struct { @@ -25,14 +25,36 @@ type sNodeReferer struct { type sJsonParseSession struct { objectMap map[int]*sNodeReferer + + // depth is the current nesting level of the object/array being parsed + depth int + + // allowNodeReference tells whether the ___jnid_ key and a bare + // value are read as node references, see ParseTrusted + allowNodeReference bool } -func newJsonParseSession() *sJsonParseSession { +func newJsonParseSession(allowNodeReference bool) *sJsonParseSession { return &sJsonParseSession{ - objectMap: make(map[int]*sNodeReferer), + objectMap: make(map[int]*sNodeReferer), + allowNodeReference: allowNodeReference, } } +// enter records entering one more nesting level, it fails if the nesting +// level exceeds maxParseDepth +func (s *sJsonParseSession) enter() error { + if s.depth >= maxParseDepth { + return ErrNestedTooDeep + } + s.depth++ + return nil +} + +func (s *sJsonParseSession) leave() { + s.depth-- +} + func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) { if nr, ok := s.objectMap[nodeId]; ok { nr.pointers = append(nr.pointers, ptr) @@ -43,10 +65,10 @@ func (s *sJsonParseSession) saveReferer(nodeId int, ptr *sJSONPointer) { } } -func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) { +func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) error { if nr, ok := s.objectMap[nodeId]; ok { if nr.node != nil { - panic(fmt.Sprintf("nodeId %d alreayd exists: %s != %s", nodeId, nr.node, node)) + return errors.Wrapf(ErrDuplicateNodeId, "node id %d", nodeId) } else { nr.node = node } @@ -56,4 +78,5 @@ func (s *sJsonParseSession) saveNode(nodeId int, node JSONObject) { pointers: nil, } } + return nil } diff --git a/vendor/yunion.io/x/jsonutils/querystring.go b/vendor/yunion.io/x/jsonutils/querystring.go index faded210f6..91ce91a41d 100644 --- a/vendor/yunion.io/x/jsonutils/querystring.go +++ b/vendor/yunion.io/x/jsonutils/querystring.go @@ -55,7 +55,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO if err != nil { return nil, errors.Wrapf(err, "get jsondict %s with %s fail", body, key) } - addQueryStringSeg(next, segs[1:], val) + _, err = addQueryStringSeg(next, segs[1:], val) + if err != nil { + return nil, errors.Wrapf(err, "addQueryStringSeg %s with %s fail", body, key) + } } return jbody, nil case *JSONArray: @@ -66,7 +69,10 @@ func addQueryStringSeg(body JSONObject, segs []sTextNumber, val []string) (JSONO if err != nil { return nil, errors.Wrapf(err, "get jsonarray %s at %d fail", body, index) } - addQueryStringSeg(next, segs[1:], val) + _, err = addQueryStringSeg(next, segs[1:], val) + if err != nil { + return nil, errors.Wrapf(err, "addQueryStringSeg %s at %d fail", body, index) + } } else if arrSize == index { // new next, err := addQueryStringSeg(nil, segs[1:], val) diff --git a/vendor/yunion.io/x/jsonutils/std_json.go b/vendor/yunion.io/x/jsonutils/std_json.go index f70091d8d9..e5b4b9ea88 100644 --- a/vendor/yunion.io/x/jsonutils/std_json.go +++ b/vendor/yunion.io/x/jsonutils/std_json.go @@ -16,9 +16,9 @@ package jsonutils import ( "encoding/json" - "fmt" "reflect" + "yunion.io/x/log" "yunion.io/x/pkg/gotypes" ) @@ -110,11 +110,13 @@ func tryStdMarshal(v reflect.Value, marshalFunc func(v reflect.Value) JSONObject if m != nil { data, err := m.MarshalJSON() if err != nil { - panic(fmt.Sprintf("MarshalJSON of %q error: %v", v.String(), err)) + log.Errorf("MarshalJSON of %s error: %v", v.Type(), err) + return JSONNull } jo, err := Parse(data) if err != nil { - panic(fmt.Sprintf("Parse data %q to json of %q error: %v", data, v.String(), err)) + log.Errorf("Parse data %s to json of %s error: %v", data, v.Type(), err) + return JSONNull } return jo } diff --git a/vendor/yunion.io/x/jsonutils/string.go b/vendor/yunion.io/x/jsonutils/string.go index dfa791b59e..e033ccbfcc 100644 --- a/vendor/yunion.io/x/jsonutils/string.go +++ b/vendor/yunion.io/x/jsonutils/string.go @@ -33,6 +33,10 @@ func (this *JSONInt) String() string { } func (this *JSONFloat) String() string { + if !isFiniteFloat(this.data) { + // nan and +-inf have no json representation + return "null" + } if this.bit != 32 && this.bit != 64 { this.bit = 64 } diff --git a/vendor/yunion.io/x/jsonutils/unmarshal.go b/vendor/yunion.io/x/jsonutils/unmarshal.go index 4209253e72..fa088df9b4 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal.go @@ -62,12 +62,26 @@ func jsonUnmarshal(jo JSONObject, o interface{}, keys []string) error { return errors.Wrap(err, "Get") } } - s := newJsonUnmarshalSession() value := reflect.ValueOf(o) - err := jo.unmarshalValue(s, reflect.Indirect(value)) + if value.IsValid() && value.Kind() == reflect.Ptr && value.IsNil() { + return errors.Wrapf(ErrTypeMismatch, "unmarshal into a nil pointer of type %s", value.Type()) + } + target := reflect.Indirect(value) + // a non addressable value can not be written to, unless it is a + // non nil map, which is a reference type + if target.IsValid() && !target.CanAddr() && + (target.Kind() != reflect.Map || target.IsNil()) { + return errors.Wrapf(ErrTypeMismatch, "unmarshal into a value of type %s, a pointer is required", target.Type()) + } + s := newJsonUnmarshalSession() + err := jo.unmarshalValue(s, target) if err != nil { return errors.Wrap(err, "jo.unmarshalValue") } + err = s.checkUnboundNodes() + if err != nil { + return errors.Wrap(err, "checkUnboundNodes") + } return nil } @@ -385,6 +399,11 @@ func (this *JSONString) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Va if err != nil { return err } + if !isFiniteFloat(floatVal) { + // nan and +-inf have no json representation, they would + // silently break any comparison the field takes part in + return errors.Wrap(ErrInvalidJsonFloat, "not a finite number") + } val.SetFloat(floatVal) } case reflect.Bool: @@ -476,7 +495,10 @@ func (this *JSONArray) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Val func (this *JSONDict) unmarshalValue(s *sJsonUnmarshalSession, val reflect.Value) error { if this.nodeId > 0 && val.CanAddr() { - s.saveNodeValue(this.nodeId, val.Addr()) + err := s.saveNodeValue(this.nodeId, val.Addr()) + if err != nil { + return errors.Wrap(err, "saveNodeValue") + } } return tryStdUnmarshal(s, this, val, this._unmarshalValue) } @@ -510,7 +532,11 @@ func (this *JSONDict) _unmarshalValue(s *sJsonUnmarshalSession, val reflect.Valu return err } if objPtr == nil { - val.Set(reflect.ValueOf(this.data)) // ??? + dataVal := reflect.ValueOf(this.data) + if !dataVal.Type().AssignableTo(val.Type()) { + return errors.Wrapf(ErrInterfaceUnsupported, "JSONDict.unmarshalValue: %s", val.Type()) + } + val.Set(dataVal) return nil } err = this.unmarshalValue(s, reflect.ValueOf(objPtr)) @@ -600,7 +626,7 @@ func setStructFieldAt(s *sJsonUnmarshalSession, key string, v JSONObject, fieldV if err != nil { return errors.Wrap(err, "JSONDict.unmarshalStruct") } - depInfo, ok := fieldValues[index].Info.Tags[TAG_DEPRECATED_BY] + depInfo, ok := fieldValues[index].Info.Tag(TAG_DEPRECATED_BY) if ok { err := setStructFieldAt(s, depInfo, v, fieldValues, keyIndexMap, visited) if err != nil { diff --git a/vendor/yunion.io/x/jsonutils/unmarshal_session.go b/vendor/yunion.io/x/jsonutils/unmarshal_session.go index 1b5d89c34b..c0fe3833a2 100644 --- a/vendor/yunion.io/x/jsonutils/unmarshal_session.go +++ b/vendor/yunion.io/x/jsonutils/unmarshal_session.go @@ -16,6 +16,9 @@ package jsonutils import ( "reflect" + "sort" + + "yunion.io/x/pkg/errors" ) type sJsonNodeValues struct { @@ -34,13 +37,27 @@ func newJsonUnmarshalSession() *sJsonUnmarshalSession { } } -func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) { +// assignable reports whether the node value can be assigned to the value +// referring to that node +func assignable(nodeValue, refValue reflect.Value) bool { + if !nodeValue.IsValid() || !refValue.IsValid() { + return false + } + return nodeValue.Type().AssignableTo(refValue.Type()) +} + +func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) error { if nv, ok := s.objectMap[nodeId]; !ok { s.objectMap[nodeId] = &sJsonNodeValues{ nodeValue: val, nodeValueSet: true, } } else { + for i := range nv.targetValues { + if !assignable(val, nv.targetValues[i]) { + return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, nv.targetValues[i].Type()) + } + } nv.nodeValue = val nv.nodeValueSet = true for i := range nv.targetValues { @@ -48,10 +65,30 @@ func (s *sJsonUnmarshalSession) saveNodeValue(nodeId int, val reflect.Value) { } nv.targetValues = nil } + return nil +} + +// checkUnboundNodes reports the node ids that were referred to but never +// resolved, which would otherwise leave the referring field untouched +func (s *sJsonUnmarshalSession) checkUnboundNodes() error { + ids := make([]int, 0) + for nodeId, nv := range s.objectMap { + if !nv.nodeValueSet { + ids = append(ids, nodeId) + } + } + if len(ids) == 0 { + return nil + } + sort.Ints(ids) + return errors.Wrapf(ErrNodeNotFound, "node ids %v", ids) } func (s *sJsonUnmarshalSession) setPointerValue(nodeId int, val reflect.Value) error { if nv, ok := s.objectMap[nodeId]; ok && nv.nodeValueSet { + if !assignable(nv.nodeValue, val) { + return errors.Wrapf(ErrTypeMismatch, "node id %d vs %s", nodeId, val.Type()) + } val.Set(nv.nodeValue) } else if ok && !nv.nodeValueSet { nv.targetValues = append(nv.targetValues, val) diff --git a/vendor/yunion.io/x/jsonutils/write.go b/vendor/yunion.io/x/jsonutils/write.go index f9ebc4a709..3295be0a06 100644 --- a/vendor/yunion.io/x/jsonutils/write.go +++ b/vendor/yunion.io/x/jsonutils/write.go @@ -18,9 +18,18 @@ import ( "fmt" "strings" + "yunion.io/x/pkg/gotypes" "yunion.io/x/pkg/sortedmap" ) +func writeJSONObject(sb *strings.Builder, v JSONObject) { + if gotypes.IsNil(v) { + sb.WriteString("null") + return + } + v.buildString(sb) +} + type writeSource interface { buildString(sb *strings.Builder) } @@ -46,6 +55,10 @@ func (this *JSONBool) buildString(sb *strings.Builder) { } func (this *JSONDict) buildString(sb *strings.Builder) { + if this == nil { + sb.WriteString("null") + return + } sb.WriteByte('{') var idx = 0 if this.nodeId > 0 { @@ -63,19 +76,23 @@ func (this *JSONDict) buildString(sb *strings.Builder) { sb.WriteString(quoteString(k)) sb.WriteByte(':') - v.buildString(sb) + writeJSONObject(sb, v) idx++ } sb.WriteByte('}') } func (this *JSONArray) buildString(sb *strings.Builder) { + if this == nil { + sb.WriteString("null") + return + } sb.WriteByte('[') for idx, v := range this.data { if idx > 0 { sb.WriteByte(',') } - v.buildString(sb) + writeJSONObject(sb, v) } sb.WriteByte(']') } diff --git a/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go b/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go index c144074584..4bd64a8d6d 100644 --- a/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go +++ b/vendor/yunion.io/x/pkg/util/cloudinit/cloudconfig.go @@ -18,6 +18,7 @@ import ( "bytes" "encoding/base64" "fmt" + "path" "strings" "yunion.io/x/jsonutils" @@ -99,32 +100,127 @@ func NewWriteFile(path string, content string, perm string, owner string, isBase return f } +// shellQuote wraps s so that a POSIX shell treats it as one literal word. +// A single quote inside s is closed, escaped and reopened. +func shellQuote(s string) string { + return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'" +} + +// validUserName reports whether name can be used as a user name. +// +// The set is deliberately narrow. A name becomes a directory under /home, a +// file name under /etc/sudoers.d and an argument to useradd, so a slash or a +// ".." would move the files that are written, and a control character would +// split a generated line. Leading "-" is refused so the name cannot be read as +// an option. +func validUserName(name string) bool { + if len(name) == 0 || name == "." || name == ".." || name[0] == '-' { + return false + } + for i := 0; i < len(name); i++ { + c := name[i] + switch { + case c >= 'a' && c <= 'z', c >= 'A' && c <= 'Z', c >= '0' && c <= '9': + case c == '_' || c == '-' || c == '.': + default: + return false + } + } + return true +} + +// validWritePath reports whether path may be written by a generated script. +// +// Quoting keeps a path from running as a command, but a path that walks up out +// of its directory would still write somewhere else. A path carrying a ".." +// element is refused rather than normalised, so that what is written is what +// was asked for. +func validWritePath(filePath string) bool { + if len(filePath) == 0 { + return false + } + for _, elem := range strings.Split(filePath, "/") { + if elem == ".." { + return false + } + } + return true +} + +// escapePowerShell escapes a value for use inside a PowerShell double quoted +// string, where a backtick introduces an escape, $ introduces a variable, and +// a control character written literally would break the line it sits on. +func escapePowerShell(s string) string { + return strings.NewReplacer( + "`", "``", + `"`, "`\"", + "$", "`$", + "\r", "`r", + "\n", "`n", + "\t", "`t", + "\x00", "`0", + ).Replace(s) +} + +const heredocPrefix = "_YUNION_EOF_" + +// heredocTerminator returns a terminator that does not occur in content, so +// that a line of the content cannot end the heredoc early. Quote it at the use +// site so the shell does not expand the content. +// +// The generated value is checked against the content rather than assumed to be +// unique: utils.GenRequestId returns the empty string if the random source is +// unavailable, which would otherwise leave a fixed, guessable terminator. +func heredocTerminator(content string) string { + return uniqueTerminator(heredocPrefix+utils.GenRequestId(8), content) +} + +// uniqueTerminator returns term with underscores appended until it does not +// occur in content. Each pass lengthens it, so this ends once it is longer +// than the content and can no longer occur in it. +func uniqueTerminator(term, content string) string { + for strings.Contains(content, term) { + term += "_" + } + return term +} + func setFilePermission(path, permission, owner string) []string { cmds := []string{} if len(permission) > 0 { - cmds = append(cmds, fmt.Sprintf("chmod %s %s", permission, path)) + cmds = append(cmds, fmt.Sprintf("chmod %s %s", shellQuote(permission), shellQuote(path))) } if len(owner) > 0 { - cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", owner, owner, path)) + cmds = append(cmds, fmt.Sprintf("chown %s:%s %s", shellQuote(owner), shellQuote(owner), shellQuote(path))) } return cmds } +// mkWriteFileCmd builds the commands that create path with content. The +// redirection and the path are quoted, and the heredoc is quoted with a +// terminator that is not expected to occur in the content. +func mkWriteFileCmd(redirect, filePath, content, permission, owner string) []string { + terminator := heredocTerminator(content) + cmds := []string{ + fmt.Sprintf("mkdir -p %s", shellQuote(path.Dir(filePath))), + fmt.Sprintf("cat %s %s <<'%s'\n%s\n%s", redirect, shellQuote(filePath), terminator, content, terminator), + } + return append(cmds, setFilePermission(filePath, permission, owner)...) +} + func mkPutFileCmd(path string, content string, permission string, owner string) []string { - cmds := []string{} - cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path)) - cmds = append(cmds, fmt.Sprintf("cat > %s <<_END\n%s\n_END", path, content)) - return append(cmds, setFilePermission(path, permission, owner)...) + return mkWriteFileCmd(">", path, content, permission, owner) } func mkAppendFileCmd(path string, content string, permission string, owner string) []string { - cmds := []string{} - cmds = append(cmds, fmt.Sprintf("mkdir -p $(dirname %s)", path)) - cmds = append(cmds, fmt.Sprintf("cat >> %s <<_END\n%s\n_END", path, content)) - return append(cmds, setFilePermission(path, permission, owner)...) + return mkWriteFileCmd(">>", path, content, permission, owner) } func (wf *SWriteFile) ShellScripts() []string { + if !validWritePath(wf.Path) { + log.Errorf("cloudinit: skipping write_file with a path that walks up out of its directory: %q", wf.Path) + return nil + } content := wf.Content if wf.Encoding == "b64" { _content, _ := base64.StdEncoding.DecodeString(wf.Content) @@ -176,23 +272,36 @@ func (u *SUser) Password(passwd string) *SUser { } func (u *SUser) PowerShellScripts() []string { + if !validUserName(u.Name) { + log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name) + return nil + } + // Every line below is parsed by PowerShell first, so the name and the + // password are escaped the same way. Using the unescaped name for one of + // the lines would have them name different accounts. + name := escapePowerShell(u.Name) shells := []string{} - shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, u.Name)) - shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, u.Name)) + shells = append(shells, fmt.Sprintf(`New-LocalUser -Name "%s" -Description "A New Local Account Created By PowerShell" -NoPassword`, name)) + shells = append(shells, fmt.Sprintf(`Add-LocalGroupMember -Group "Administrators" -Member "%s"`, name)) if len(u.PlainTextPasswd) > 0 { - shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, u.Name, u.PlainTextPasswd)) + shells = append(shells, fmt.Sprintf(`net user "%s" "%s"`, name, escapePowerShell(u.PlainTextPasswd))) } // enable需要再设置密码之后，否则会出现Enable-LocalUser : Unable to update the password. The value provided for the new password does not meet the length, complexity, or history requirements of the domain - shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, u.Name)) + shells = append(shells, fmt.Sprintf(`Enable-LocalUser "%s"`, name)) return shells } func (u *SUser) ShellScripts() []string { + if !validUserName(u.Name) { + log.Errorf("cloudinit: skipping scripts for unusable user name %q", u.Name) + return nil + } + name := shellQuote(u.Name) shells := []string{} - shells = append(shells, fmt.Sprintf("useradd -m %s || true", u.Name)) + shells = append(shells, fmt.Sprintf("useradd -m %s || true", name)) if len(u.HashedPasswd) > 0 { - shells = append(shells, fmt.Sprintf("usermod -p '%s' %s", u.HashedPasswd, u.Name)) + shells = append(shells, fmt.Sprintf("usermod -p %s %s", shellQuote(u.HashedPasswd), name)) } home := "/" + u.Name @@ -202,7 +311,7 @@ func (u *SUser) ShellScripts() []string { keyPath := fmt.Sprintf("%s/.ssh/authorized_keys", home) shells = append(shells, mkAppendFileCmd(keyPath, strings.Join(u.SshAuthorizedKeys, "\n"), "600", u.Name)...) - shells = append(shells, fmt.Sprintf("chown -R %s:%s %s/.ssh", u.Name, u.Name, home)) + shells = append(shells, fmt.Sprintf("chown -R %s:%s %s", name, name, shellQuote(home+"/.ssh"))) if !utils.IsInStringArray(u.Sudo, []string{"", "False"}) { shells = append(shells, mkPutFileCmd("/etc/sudoers.d/"+u.Name, fmt.Sprintf("%s %s", u.Name, u.Sudo), "", "")...) @@ -212,6 +321,9 @@ func (u *SUser) ShellScripts() []string { } func (conf *SCloudConfig) UserData() string { + if conf == nil { + return "" + } var buf bytes.Buffer jsonConf := jsonutils.Marshal(conf).(*jsonutils.JSONDict) if jsonConf.Contains("users") { @@ -228,6 +340,9 @@ func (conf *SCloudConfig) UserData() string { } func (conf *SCloudConfig) UserDataScript() string { + if conf == nil { + return "" + } shells := []string{} for _, u := range conf.Users { shells = append(shells, u.ShellScripts()...) @@ -248,8 +363,11 @@ func (conf *SCloudConfig) UserDataScript() string { } for _, pkg := range conf.Packages { - shells = append(shells, "which yum &>/dev/null && yum install -y "+pkg) - shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+pkg) + // A package name is an argument, not part of the command line, so it + // is quoted rather than pasted in. + quoted := shellQuote(pkg) + shells = append(shells, "which yum &>/dev/null && yum install -y "+quoted) + shells = append(shells, "which apt-get &>/dev/null && apt-get install -y "+quoted) } for _, wf := range conf.WriteFiles { shells = append(shells, wf.ShellScripts()...) @@ -317,7 +435,13 @@ func ParseUserData(data string) (*SCloudConfig, error) { if err != nil { return nil, errors.Wrapf(err, "ParseYAML") } - jsonDict := jsonConf.(*jsonutils.JSONDict) + jsonDict, ok := jsonConf.(*jsonutils.JSONDict) + if !ok { + // Anything that is valid YAML but not a mapping, e.g. a list or a + // scalar, is not a usable cloud-config document. + return nil, errors.Wrapf(errors.ErrInvalidFormat, + "cloud-config must be a YAML mapping, got %s", jsonConf.String()) + } if jsonDict.Contains("users") { userArray := jsonutils.NewArray() users, _ := jsonConf.GetArray("users") diff --git a/vendor/yunion.io/x/pkg/util/encode/encode.go b/vendor/yunion.io/x/pkg/util/encode/encode.go index 1420309784..438e662711 100644 --- a/vendor/yunion.io/x/pkg/util/encode/encode.go +++ b/vendor/yunion.io/x/pkg/util/encode/encode.go @@ -40,8 +40,11 @@ func unhex(c rune) byte { return 0 } +// shouldEncode reports whether c must be escaped by EncodeGoogleLabel. +// Only runes in [0, 0xff] can be represented as "_" followed by two hex +// digits, so anything above that range is written through unchanged. func shouldEncode(c rune) bool { - if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c > 256 { + if (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c > 0xff { return false } return true @@ -69,7 +72,7 @@ func DecodeGoogleLable(label string) string { var t strings.Builder for j := 0; j < len(s); { c := s[j] - if c == rune('_') && j+2 <= len(s) && ishex(s[j+1]) && ishex(s[j+2]) { + if c == rune('_') && j+2 < len(s) && ishex(s[j+1]) && ishex(s[j+2]) { t.WriteByte(unhex(s[j+1])<<4 | unhex(s[j+2])) j += 3 } else { diff --git a/vendor/yunion.io/x/pkg/util/httputils/fs.go b/vendor/yunion.io/x/pkg/util/httputils/fs.go index ca6445d95a..4d92114ecb 100644 --- a/vendor/yunion.io/x/pkg/util/httputils/fs.go +++ b/vendor/yunion.io/x/pkg/util/httputils/fs.go @@ -35,11 +35,20 @@ func (fs FileSystem) Open(path string) (http.File, error) { } s, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } if s.IsDir() { index := strings.TrimSuffix(path, "/") + "/index.html" - if _, err := fs.fs.Open(index); err != nil { + indexFile, err := fs.fs.Open(index) + if err != nil { + // Without an index file a directory must not be listed. + f.Close() return nil, err } + // The index file is only checked for; it is not the file returned. + indexFile.Close() } return f, nil diff --git a/vendor/yunion.io/x/pkg/util/httputils/httputils.go b/vendor/yunion.io/x/pkg/util/httputils/httputils.go index 47b4a01cd1..75dd553acf 100644 --- a/vendor/yunion.io/x/pkg/util/httputils/httputils.go +++ b/vendor/yunion.io/x/pkg/util/httputils/httputils.go @@ -28,6 +28,7 @@ import ( "net/http/httputil" "net/url" "os" + "regexp" "strconv" "strings" "syscall" @@ -93,6 +94,173 @@ type sClient interface { Do(req *http.Request) (*http.Response, error) } +// Field and parameter names that carry a credential on their own, and +// fragments that mark a name as carrying one. The same lists are used for the +// fields of a request body and the parameters of a request URL, so that the +// two cannot drift apart. +var sensitiveBodyKeyNames = []string{ + "key", "code", "sig", "pwd", "auth", "ticket", "session", "sessionid", + // The plain name is a credential; "access_key_id" is an identifier and + // is deliberately not matched, so it stays readable in the message. + "access_key", +} + +var sensitiveBodyKeyParts = []string{ + "token", "secret", "password", "passwd", "signature", "credential", + "apikey", "api_key", "private_key", +} + +func isSensitiveBodyKey(name string) bool { + lower := strings.ToLower(name) + if utils.IsInStringArray(lower, sensitiveBodyKeyNames) { + return true + } + for _, part := range sensitiveBodyKeyParts { + if strings.Contains(lower, part) { + return true + } + } + return false +} + +// redactURL masks the values of query parameters that commonly carry a +// credential, so that an error message can name the request without +// reproducing the secret. A URL with nothing to mask is returned unchanged, +// byte for byte, so no existing message is reformatted. +func redactURL(urlStr string) string { + parsed, err := url.Parse(urlStr) + if err != nil || len(parsed.RawQuery) == 0 { + return urlStr + } + query := parsed.Query() + changed := false + for name := range query { + if isSensitiveBodyKey(name) { + query.Set(name, "*") + changed = true + } + } + if !changed { + return urlStr + } + parsed.RawQuery = query.Encode() + return parsed.String() +} + +// redactJSONObject returns obj with the values of credential carrying fields +// replaced, at any depth. +func redactJSONObject(obj jsonutils.JSONObject) jsonutils.JSONObject { + switch v := obj.(type) { + case *jsonutils.JSONDict: + redacted := jsonutils.NewDict() + for name, value := range v.Value() { + if isSensitiveBodyKey(name) { + redacted.Set(name, jsonutils.NewString("*")) + } else { + redacted.Set(name, redactJSONObject(value)) + } + } + return redacted + case *jsonutils.JSONArray: + redacted := jsonutils.NewArray() + for _, value := range v.Value() { + redacted.Add(redactJSONObject(value)) + } + return redacted + default: + return obj + } +} + +// redactFormBody masks the credential carrying fields of an urlencoded body. +// +// The body is walked pair by pair rather than through url.ParseQuery, which +// stops at the first pair it cannot decode: the pairs it did decode are still +// returned, but taking only the value it returns would either drop the rest of +// the body or, if the raw body were used instead, leave the credential in it. +func redactFormBody(body string) string { + parts := strings.Split(body, "&") + changed := false + for i, part := range parts { + eq := strings.IndexByte(part, '=') + if eq < 0 { + continue + } + name := part[:eq] + if decoded, err := url.QueryUnescape(name); err == nil { + name = decoded + } + if isSensitiveBodyKey(name) { + parts[i] = part[:eq+1] + "*" + changed = true + } + } + if !changed { + return body + } + return strings.Join(parts, "&") +} + +// A credential carrying name as it appears in a body that cannot be walked +// field by field, e.g. XML, or a JSON document that did not parse. +const sensitiveBodyNamePattern = `(?i)[A-Za-z0-9_.:-]*(?:token|secret|password|passwd|signature|credential|apikey|api_key|private_key)[A-Za-z0-9_.:-]*` + +var ( + // value, where the value may be a CDATA section + xmlElementSensitiveRe = regexp.MustCompile( + `(?is)(<` + sensitiveBodyNamePattern + `\b[^>]*>)(?:\s*|[^<]*)( maxRedactedBodyBytes { + return nil + } + switch { + case strings.Contains(contType, "json"): + if parsed, err := jsonutils.ParseString(body); err == nil { + return redactJSONObject(parsed) + } + // A JSON body that does not parse is still masked by field name, + // since the names survive in the text. + return jsonutils.NewString(redactBodyText(body)) + case strings.Contains(contType, "x-www-form-urlencoded"): + return jsonutils.NewString(redactFormBody(body)) + case strings.Contains(contType, "xml"): + return jsonutils.NewString(redactBodyText(body)) + } + return nil +} + // body might have been consumed, so body is provided separately func newJsonClientErrorFromRequest(req *http.Request, body string) *JSONClientError { return newJsonClientErrorFromRequest2(req.Method, req.URL.String(), req.Header, body) @@ -102,7 +270,7 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head jce := &JSONClientError{} jce.Request.Method = strings.ToUpper(method) - jce.Request.Url = urlStr + jce.Request.Url = redactURL(urlStr) jce.Request.Headers = make(map[string]string) excludeHdrs := []string{ "Accept", @@ -112,22 +280,13 @@ func newJsonClientErrorFromRequest2(method string, urlStr string, hdrs http.Head http.CanonicalHeaderKey("authorization"), http.CanonicalHeaderKey("x-auth-token"), http.CanonicalHeaderKey("x-subject-token"), + http.CanonicalHeaderKey("cookie"), + http.CanonicalHeaderKey("set-cookie"), } - const ( - MAX_BODY = 128 - FIRST_PART = 100 - ) switch jce.Request.Method { case "PUT", "POST", "PATCH": contType := hdrs.Get(http.CanonicalHeaderKey("content-type")) - if len(body) > MAX_BODY { - jce.Request.Body = jsonutils.NewString(body[:FIRST_PART] + "..." + body[len(body)-MAX_BODY+FIRST_PART+3:]) - } else if strings.Contains(contType, "json") { - jce.Request.Body, _ = jsonutils.ParseString(body) - } else if strings.Contains(contType, "xml") || - strings.Contains(contType, "x-www-form-urlencoded") { - jce.Request.Body = jsonutils.NewString(body) - } + jce.Request.Body = redactRequestBody(body, contType) default: excludeHdrs = append(excludeHdrs, http.CanonicalHeaderKey("content-type"), http.CanonicalHeaderKey("content-length")) } @@ -233,13 +392,10 @@ func NewJsonClient(client sClient) *JsonClient { return &JsonClient{client: client} } +// Error renders the error as JSON. The request it carries is already masked +// when the error is built, so this has no side effects and repeating it gives +// the same message. func (e *JSONClientError) Error() string { - if !gotypes.IsNil(e.Request.Body) { - if body, ok := e.Request.Body.(*jsonutils.JSONDict); ok && body.Contains("password") { - body.Set("password", jsonutils.NewString("***")) - e.Request.Body = body - } - } errMsg := JSONClientErrorMsg{Error: e} return jsonutils.Marshal(errMsg).String() } @@ -427,6 +583,42 @@ func GetClient(insecure bool, timeout time.Duration) *http.Client { return getClient(insecure, false, timeout) } +// Session headers that a caller of this package sets, and that must not +// travel to a host the request was not made to. net/http already drops +// Authorization, Cookie and the proxy headers in that case, but does not know +// about these. +var redirectSensitiveHeaders = []string{ + "X-Auth-Token", + "X-Subject-Token", +} + +// isSameOrSubdomain reports whether dest is initial, or a subdomain of it. +// A redirect to a parent domain is not covered, matching the rule net/http +// uses for the headers it strips. +func isSameOrSubdomain(initial, dest *url.URL) bool { + i := strings.ToLower(initial.Hostname()) + d := strings.ToLower(dest.Hostname()) + if i == d { + return true + } + return strings.HasSuffix(d, "."+i) +} + +// checkRedirect drops the session headers above when a redirect leaves the +// host the request was made to, and keeps the usual limit on how many +// redirects are followed. +func checkRedirect(req *http.Request, via []*http.Request) error { + if len(via) >= 10 { + return errors.Errorf("stopped after 10 redirects") + } + if len(via) > 0 && !isSameOrSubdomain(via[0].URL, req.URL) { + for _, h := range redirectSensitiveHeaders { + req.Header.Del(h) + } + } + return nil +} + func getClient(insecure bool, legacyTLS bool, timeout time.Duration) *http.Client { adaptive := false if timeout == 0 { @@ -434,7 +626,8 @@ func getClient(insecure bool, legacyTLS bool, timeout time.Duration) *http.Clien } tr := getTransport(insecure, legacyTLS, adaptive, timeout) return &http.Client{ - Transport: tr, + Transport: tr, + CheckRedirect: checkRedirect, // 一个完整http request的超时时间 // Timeout specifies a time limit for requests made by this // Client. The timeout includes connection time, any diff --git a/vendor/yunion.io/x/pkg/util/netutils/http.go b/vendor/yunion.io/x/pkg/util/netutils/http.go index cc9feb6435..6d3a75ebdd 100644 --- a/vendor/yunion.io/x/pkg/util/netutils/http.go +++ b/vendor/yunion.io/x/pkg/util/netutils/http.go @@ -15,26 +15,47 @@ package netutils import ( + "net" "net/http" "strings" ) +// parseIPLiteral returns s as a plain address if it is one, and "" otherwise. +func parseIPLiteral(s string) string { + s = strings.TrimSpace(s) + if len(s) == 0 { + return "" + } + if ip := net.ParseIP(s); ip != nil { + return ip.String() + } + return "" +} + +// GetHttpRequestIp returns the address of the client that made the request. +// +// The forwarded headers are consulted first, since a proxy in front of the +// service rewrites them. When no proxy is in place they are whatever the +// client sent, so each candidate is parsed and only an actual address is +// returned; a value that is not an address is skipped rather than passed on +// as though it were one. +// +// The connection address is used as the last resort, split with +// net.SplitHostPort so that an IPv6 peer is returned without its brackets and +// without its port. func GetHttpRequestIp(r *http.Request) string { - ipStr := r.Header.Get("X-Forwarded-For") - if len(ipStr) > 0 { - ipList := strings.Split(ipStr, ",") - if len(ipList) > 0 { - return ipList[0] + if forwarded := r.Header.Get("X-Forwarded-For"); len(forwarded) > 0 { + for _, part := range strings.Split(forwarded, ",") { + if ip := parseIPLiteral(part); len(ip) > 0 { + return ip + } } } - ipStr = r.Header.Get("X-Real-Ip") - if len(ipStr) > 0 { - return ipStr + if ip := parseIPLiteral(r.Header.Get("X-Real-Ip")); len(ip) > 0 { + return ip } - ipStr = r.RemoteAddr - colonPos := strings.Index(ipStr, ":") - if colonPos > 0 { - ipStr = ipStr[:colonPos] + if host, _, err := net.SplitHostPort(r.RemoteAddr); err == nil { + return host } - return ipStr + return r.RemoteAddr } diff --git a/vendor/yunion.io/x/pkg/util/netutils/ipv6.go b/vendor/yunion.io/x/pkg/util/netutils/ipv6.go index 4c20786f8a..1617171130 100644 --- a/vendor/yunion.io/x/pkg/util/netutils/ipv6.go +++ b/vendor/yunion.io/x/pkg/util/netutils/ipv6.go @@ -154,6 +154,10 @@ func normalizeIpv6Addr(addrStr string) ([8]uint16, error) { } } } + } else if len(parts) > expectPartCnt { + // More groups than the address can hold. A "::" run can only reduce + // the number of groups, so there is no valid reading of this input. + return addr, errors.Wrapf(errors.ErrInvalidFormat, "%s too many groups", addrStr) } else { for i := 0; i < len(parts); i++ { partStr := strings.TrimSpace(parts[i]) diff --git a/vendor/yunion.io/x/pkg/util/netutils/netutils.go b/vendor/yunion.io/x/pkg/util/netutils/netutils.go index 929520a1b8..97480dcf2d 100644 --- a/vendor/yunion.io/x/pkg/util/netutils/netutils.go +++ b/vendor/yunion.io/x/pkg/util/netutils/netutils.go @@ -181,12 +181,21 @@ func (ar IPV4AddrRange) ContainsRange(ar2 IPV4AddrRange) bool { return ar.start <= ar2.start && ar.end >= ar2.end } +// Random returns an address from the range, excluding end. A range that +// covers a single address returns that address. func (ar IPV4AddrRange) Random() IPV4Addr { - return IPV4Addr(uint32(ar.start) + uint32(rand.Intn(int(uint32(ar.end)-uint32(ar.start))))) + if ar.start >= ar.end { + return ar.start + } + // int64 so the span of a very large range cannot overflow the argument + // to the random source. + span := int64(uint32(ar.end) - uint32(ar.start)) + return IPV4Addr(uint32(ar.start) + uint32(rand.Int63n(span))) } func (ar IPV4AddrRange) AddressCount() int { - return int(uint32(ar.end) - uint32(ar.start) + 1) + // 64 bit arithmetic so a full range does not wrap around to zero. + return int(uint64(uint32(ar.end)) - uint64(uint32(ar.start)) + 1) } func (ar IPV4AddrRange) String() string { @@ -360,10 +369,18 @@ func (ar IPV4AddrRange) equals(ar2 IPV4AddrRange) bool { return ar.start == ar2.start && ar.end == ar2.end } +// Masklen2Mask returns the network mask for a prefix length. +// +// A length above 32 cannot be represented and is treated as 32, i.e. a single +// host, rather than shifting past the width of the mask and wrapping around to +// a match-all mask. func Masklen2Mask(maskLen int8) IPV4Addr { if maskLen < 0 { panic("negative masklen") } + if maskLen > 32 { + maskLen = 32 + } return IPV4Addr(^(uint32(1<<(32-uint8(maskLen))) - 1)) } @@ -392,6 +409,25 @@ func Mask2Len(mask IPV4Addr) int8 { return int8(bits.LeadingZeros32(^uint32(mask))) } +// ParsePrefix parses an IPv4 prefix written as "address/masklen", or as a +// bare address, which is taken to be a /32. An empty string parses as +// 0.0.0.0/32 rather than being rejected. +// +// The prefix length may also be written as a dotted-decimal mask, e.g. +// "10.0.0.0/255.0.0.0". A dotted mask is converted by counting its leading +// one bits, so the result always has a contiguous mask: +// "1.2.3.4/255.0.255.0" yields 1.0.0.0/8 rather than being rejected. +// +// The numeric form is read with strconv.Atoi, so it also accepts a leading +// sign and leading zeros: "+8" and "024" parse as 8 and 24. +// +// These accepted forms are more permissive than net.ParseCIDR. Note that +// regutils.MatchCIDR and regutils.MatchIP4Addr reject the dotted-mask and +// leading-zero spellings that this function accepts, so a value checked with +// one of those and then normalised here can end up covering something other +// than what was checked. Prefer net.ParseCIDR where the stricter grammar is +// wanted; the lenient spellings are kept for compatibility with prefixes +// already stored by callers. func ParsePrefix(prefix string) (IPV4Addr, int8, error) { slash := strings.IndexByte(prefix, '/') if slash > 0 { @@ -438,7 +474,15 @@ func NewIPV4Prefix(prefix string) (IPV4Prefix, error) { return pref, nil } +// NewIPV4PrefixFromAddr builds a prefix from an address and a prefix length. +// +// A length above 32 is clamped to 32 so that the stored MaskLen always agrees +// with the address and range it describes. A negative length keeps its +// existing behaviour of panicking. func NewIPV4PrefixFromAddr(addr IPV4Addr, masklen int8) IPV4Prefix { + if masklen > 32 { + masklen = 32 + } pref := IPV4Prefix{ Address: addr.NetAddr(masklen), MaskLen: masklen, @@ -465,14 +509,28 @@ const ( multicastPrefix = "224.0.0.0/4" ) +// Ranges that cannot be a public address but are not part of the private +// ranges used to classify guest addresses. They are consulted by +// IsExitAddress only, so that IsPrivate keeps its current meaning for the +// callers that use it for address allocation and DHCP. +var reservedPrefixes = []string{ + "0.0.0.0/8", // "this network" + "192.0.2.0/24", // TEST-NET-1 + "198.51.100.0/24", // TEST-NET-2 + "203.0.113.0/24", // TEST-NET-3 + "240.0.0.0/4", // reserved, includes the limited broadcast address +} + var privateIPRanges []IPV4AddrRange var customizedPrivateIPRanges []IPV4AddrRange +var reservedIPRanges []IPV4AddrRange var hostLocalIPRange IPV4AddrRange var linkLocalIPRange IPV4AddrRange var multicastIPRange IPV4AddrRange func init() { initPrivateIPRanges() + initReservedIPRanges() prefix, _ := NewIPV4Prefix(hostlocalPrefix) hostLocalIPRange = prefix.ToIPRange() @@ -502,6 +560,22 @@ func initPrivateIPRanges() { } } +func initReservedIPRanges() { + reservedIPRanges = make([]IPV4AddrRange, 0, len(reservedPrefixes)) + for _, prefix := range reservedPrefixes { + p, err := NewIPV4Prefix(prefix) + if err != nil { + continue + } + reservedIPRanges = append(reservedIPRanges, p.ToIPRange()) + } +} + +// GetReservedIPRanges returns the ranges that IsReserved consults. +func GetReservedIPRanges() []IPV4AddrRange { + return append([]IPV4AddrRange(nil), reservedIPRanges...) +} + func SetPrivatePrefixes(pref []string) { customizedPrivateIPRanges = make([]IPV4AddrRange, 0) for _, prefix := range pref { @@ -538,8 +612,23 @@ func IsMulticast(addr IPV4Addr) bool { return multicastIPRange.Contains(addr) } +// IsReserved reports whether addr is in a range that is reserved and can +// never be a public address. +func IsReserved(addr IPV4Addr) bool { + for _, ipRange := range reservedIPRanges { + if ipRange.Contains(addr) { + return true + } + } + return false +} + +// IsExitAddress reports whether addr can be a public address, i.e. one that a +// guest can reach directly. Reserved ranges are excluded along with the +// private, host local, link local and multicast ones. func IsExitAddress(addr IPV4Addr) bool { - return !IsPrivate(addr) && !IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr) + return !IsPrivate(addr) && !IsReserved(addr) && + !IsHostLocal(addr) && !IsLinkLocal(addr) && !IsMulticast(addr) } func MacUnpackHex(mac string) string { diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go b/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go index 7d422ecf60..61ea63b2dc 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/ambiguous.go @@ -24,34 +24,89 @@ const ( TAG_OLD_DEPRECATED_BY = "deprecated-by" ) +// expandAmbiguousPrefix prepends the prefix given by the +// TAG_AMBIGUOUS_PREFIX tag to the fields sharing a name with another field. +// +// Expanding a prefix can itself collide with a name that is already in use, so +// the expansion is repeated until the names settle. A field whose expanded +// name would take over a name owned by a field outside its own ambiguous +// group keeps its original name, so that the expansion never introduces a new +// ambiguity. func expandAmbiguousPrefix(fields SStructFieldValueSet) SStructFieldValueSet { - keyIndexMap := make(map[string][]int) - for i := range fields { - if fields[i].Info.Ignore { - continue + prefixed := make(map[int]bool) + for { + keyIndexMap := make(map[string][]int) + for i := range fields { + if fields[i].Info.Ignore { + continue + } + key := fields[i].Info.MarshalName() + values, ok := keyIndexMap[key] + if !ok { + values = make([]int, 0, 2) + } + keyIndexMap[key] = append(values, i) } - key := fields[i].Info.MarshalName() - values, ok := keyIndexMap[key] - if !ok { - values = make([]int, 0, 2) - } - keyIndexMap[key] = append(values, i) - } - for _, indexes := range keyIndexMap { - if len(indexes) > 1 { + changed := false + for _, indexes := range keyIndexMap { + if len(indexes) < 2 { + continue + } // ambiguous found for _, idx := range indexes { - if amPrefix, ok := fields[idx].Info.Tags[TAG_AMBIGUOUS_PREFIX]; ok { - fields[idx].Info.Name = fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name) - if depBy, ok := fields[idx].Info.Tags[TAG_DEPRECATED_BY]; ok { - fields[idx].Info.Tags[TAG_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy) - } - if depBy, ok := fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY]; ok { - fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy) - } + if prefixed[idx] { + continue } + amPrefix, ok := fields[idx].Info.Tags[TAG_AMBIGUOUS_PREFIX] + if !ok { + continue + } + expanded := fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Name) + if takenByOther(fields, expanded, indexes) { + continue + } + fields[idx].Info.Name = expanded + if depBy, ok := fields[idx].Info.Tags[TAG_DEPRECATED_BY]; ok { + fields[idx].Info.Tags[TAG_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy) + } + if depBy, ok := fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY]; ok { + fields[idx].Info.Tags[TAG_OLD_DEPRECATED_BY] = fmt.Sprintf("%s%s", amPrefix, depBy) + } + for i := range fields[idx].Info.Aliases { + fields[idx].Info.Aliases[i] = fmt.Sprintf("%s%s", amPrefix, fields[idx].Info.Aliases[i]) + } + prefixed[idx] = true + changed = true } } + if !changed { + break + } } return fields } + +// takenByOther reports whether name is already used by a field outside group +func takenByOther(fields SStructFieldValueSet, name string, group []int) bool { + for i := range fields { + if fields[i].Info.Ignore { + continue + } + if fields[i].Info.MarshalName() != name { + continue + } + if !containsIndex(group, i) { + return true + } + } + return false +} + +func containsIndex(indexes []int, idx int) bool { + for _, i := range indexes { + if i == idx { + return true + } + } + return false +} diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/doc.go b/vendor/yunion.io/x/pkg/util/reflectutils/doc.go index 8c0a818313..55acd8687a 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/doc.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/doc.go @@ -12,4 +12,28 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package reflectutils walks struct values by reflection, mainly to give the +// json encoder and decoder in yunion.io/x/jsonutils the field names and the +// tags to honour. +// +// # Field naming +// +// Fields are collected in declaration order, with the fields of an embedded +// struct expanded in place. A name is taken from the "name" tag, then from +// the "json" tag, and finally from the kebab form of the field name, so it can +// be looked up by any of those. +// +// Two collected fields can carry the same name. An embedded struct may bring +// in a field that another embedded struct also contributes, and unlike +// encoding/json an outer field does not hide a field of an embedded struct. +// All the fields sharing a name are reported, and a value decoded into the +// struct is written to every one of them, so that code reaching the field +// through any of the embedded paths sees it. Shadowing is deliberately not +// implemented: the list inputs of the resource APIs rely on this, e.g. +// ServerListInput carries several fields named "domain" coming from different +// embedded structs. +// +// To set such fields apart, tag an embedded struct with +// TAG_AMBIGUOUS_PREFIX, whose value is prepended to the names of the fields +// that embedded struct contributes. package reflectutils // import "yunion.io/x/pkg/util/reflectutils" diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go b/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go index 2f994669ec..892d367314 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/jsonfield.go @@ -29,12 +29,31 @@ import ( // This struct has unexported fields initialized by exported functions in this // package. Do not construct a literal or modify the exported fields in an // unmanaged way +// +// The tags of a field are meant to be read through Tag and TagMap rather than +// off the Tags map, which is not handed out to stay writable by whoever read +// it; a caller may modify what TagMap returns. type SStructFieldInfo struct { // True if the field has json tag `json:"-"` - Ignore bool + Ignore bool + + // True if an empty string, slice, map, struct or dict is left out of + // the json object. A number and a boolean are not governed by this, + // see OmitZero and OmitFalse. + // + // An empty value is left out by default, whether or not the json tag + // asks for it; `allowempty` turns that off and `omitempty` states it. OmitEmpty bool + + // True if a false boolean is left out of the json object. A false is + // written by default; `omitfalse` turns that off and `allowfalse` + // states it. OmitFalse bool - OmitZero bool + + // True if a zero number is left out of the json object. A zero is + // written by default; `omitzero` turns that off and `allowzero` states + // it. + OmitZero bool // Name can take the following values, in descreasing preference // @@ -42,12 +61,24 @@ type SStructFieldInfo struct { // 2. name of "json" tag, when it's not for ignoration // 3. kebab form of FieldName concatenated with "_" when Ignore is false // 4. empty string - Name string - FieldName string - kebabFieldName string - ForceString bool - Tags map[string]string + Name string + // FieldName is the name of the go struct field + FieldName string + + kebabFieldName string + + // True if the field has the "string" json tag option, which writes the + // value as a json string + ForceString bool + + // Tags holds the tags of the field keyed by tag name, a tag without a + // value being mapped to the empty string. Read it through Tag or + // TagMap rather than off here. + Tags map[string]string + + // Aliases are the other names the field is looked up by, taken from + // the "alias" tag Aliases []string } @@ -145,6 +176,23 @@ func (info *SStructFieldInfo) MarshalName() string { return info.kebabFieldName } +// Tag returns the value of the tag named name and whether the field has it. +// A tag without a value is reported as present with an empty value. +func (info *SStructFieldInfo) Tag(name string) (string, bool) { + val, ok := info.Tags[name] + return val, ok +} + +// TagMap returns a copy of the tags of the field, which the caller owns and +// is free to modify. +func (info *SStructFieldInfo) TagMap() map[string]string { + tags := make(map[string]string, len(info.Tags)) + for k, v := range info.Tags { + tags[k] = v + } + return tags +} + type SStructFieldValue struct { Info *SStructFieldInfo Value reflect.Value @@ -152,9 +200,41 @@ type SStructFieldValue struct { Parent *SEmbedStructFieldValue } +// SEmbedStructFieldValue ties the fields enumerated for a nil embedded struct +// to the struct they belong to. Field is the embedded pointer on the struct, +// Value is the value the fields were enumerated out of, and Parent is the +// entry of the enclosing embedded struct, if any. type SEmbedStructFieldValue struct { Field reflect.Value Value reflect.Value + + Parent *SEmbedStructFieldValue +} + +// adoptEmbeddedStruct assigns Value to Field for every nil embedded struct +// the field was enumerated through, which puts the enumerated fields back on +// the struct. The assignments are made from the outermost embedded struct +// inwards, so that each one lands on a struct that is already part of the +// real one. It reports whether the field is backed by the struct afterwards, +// which is what writing to it requires. +func (v *SStructFieldValue) adoptEmbeddedStruct() bool { + chain := make([]*SEmbedStructFieldValue, 0, 2) + for p := v.Parent; p != nil; p = p.Parent { + chain = append(chain, p) + } + for i := len(chain) - 1; i >= 0; i -= 1 { + p := chain[i] + if !p.Field.IsValid() || p.Field.Kind() != reflect.Ptr || !p.Field.IsNil() { + // nothing to adopt, the pointer is already there + continue + } + if !p.Field.CanSet() { + // the struct can not hold the embedded pointer + return false + } + p.Field.Set(p.Value) + } + return true } type SStructFieldValueSet []SStructFieldValue @@ -225,6 +305,19 @@ func fetchStructFieldValueSet2(dataValue reflect.Value, allocatePtr bool, tags m } func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags map[string]string, includeIgnore bool, parent *SEmbedStructFieldValue) SStructFieldValueSet { + if !dataValue.IsValid() || dataValue.Kind() != reflect.Struct { + // A zero Value, a nil or a non struct value has no field to + // enumerate. Report it as such rather than through a panic. + return SStructFieldValueSet{} + } + if allocatePtr && !dataValue.CanAddr() { + // The value can not be modified in place, so a nil embedded + // pointer can not be allocated on it. The fields of such a + // pointer are enumerated out of a value allocated on the side + // instead, the same way as when the caller does not ask for + // allocation; see the note below for how to write to them. + allocatePtr = false + } fields := SStructFieldValueSet{} dataType := dataValue.Type() fieldInfos := fetchCacheStructFieldInfos(dataType) @@ -252,9 +345,17 @@ func fetchStructFieldValueSet3(dataValue reflect.Value, allocatePtr bool, tags m if fv.Kind() == reflect.Ptr && allocatePtr { fv.Set(reflect.New(fv.Type().Elem())) } else if fv.Kind() == reflect.Ptr && !allocatePtr { + // The embedded pointer is nil, so the fields are + // enumerated out of a value allocated on the side. + // Value is a pointer to that value and the fields + // below are the fields of the value it points at, + // so assigning Value to Field puts them back on + // the struct. Callers writing to those fields + // must do so first. efv = &SEmbedStructFieldValue{ - Field: fv, - Value: reflect.New(fv.Type().Elem()), + Field: fv, + Value: reflect.New(fv.Type().Elem()), + Parent: parent, } fv = efv.Value } else { diff --git a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go index 4d61d9d46a..7f3370bd9b 100644 --- a/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go +++ b/vendor/yunion.io/x/pkg/util/reflectutils/reflectutils.go @@ -19,6 +19,7 @@ import ( "reflect" "yunion.io/x/log" + "yunion.io/x/pkg/gotypes" ) /* @@ -96,13 +97,19 @@ func fetchStructFieldNameValues(dataType reflect.Type, dataValue reflect.Value, } */ +// FindStructFieldValue returns the field of dataValue named name. The field +// has to be writable, so a field reached through a nil embedded pointer is +// only returned once that pointer has been put in place, which this does. func FindStructFieldValue(dataValue reflect.Value, name string) (reflect.Value, bool) { set := FetchStructFieldValueSet(dataValue) - val, find := set.GetValue(name) - if find && val.CanSet() { - return val, true + idx := set.GetStructFieldIndex(name) + if idx < 0 { + return reflect.Value{}, false } - return reflect.Value{}, false + if !set[idx].adoptEmbeddedStruct() || !set[idx].Value.CanSet() { + return reflect.Value{}, false + } + return set[idx].Value, true } func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{}, bool) { @@ -111,35 +118,53 @@ func FindStructFieldInterface(dataValue reflect.Value, name string) (interface{} } func FillEmbededStructValue(container reflect.Value, embed reflect.Value) bool { + if !container.IsValid() || container.Kind() != reflect.Struct || !embed.IsValid() { + return false + } containerType := container.Type() + embedType := embed.Type() for i := 0; i < containerType.NumField(); i += 1 { fieldType := containerType.Field(i) + if fieldType.Type.Kind() != reflect.Struct || !fieldType.Anonymous { + continue + } fieldValue := container.Field(i) - if fieldType.Type.Kind() == reflect.Struct && fieldType.Anonymous { - if fieldType.Type == embed.Type() { - fieldValue.Set(embed) - return true - } else { - filled := FillEmbededStructValue(fieldValue, embed) - if filled { - return true - } - } + if !fieldValue.CanSet() { + // an unexported embedded struct can not be assigned to, and + // neither can anything inside it + continue + } + if fieldType.Type == embedType { + fieldValue.Set(embed) + return true + } + if FillEmbededStructValue(fieldValue, embed) { + return true } - } return false } +// SetStructFieldValue sets the field of structValue named fieldName to val. +// A field reached through a nil embedded pointer is written only once that +// pointer has been put in place, which this does. func SetStructFieldValue(structValue reflect.Value, fieldName string, val reflect.Value) bool { set := FetchStructFieldValueSet(structValue) - target, find := set.GetValue(fieldName) - if !find { + idx := set.GetStructFieldIndex(fieldName) + if idx < 0 { return false } + if !set[idx].adoptEmbeddedStruct() { + return false + } + target := set[idx].Value if !target.CanSet() { return false } + if !val.IsValid() || !val.Type().AssignableTo(target.Type()) { + // report a failure instead of letting reflect.Value.Set panic + return false + } target.Set(val) return true } @@ -161,15 +186,34 @@ func ExpandInterface(val interface{}) []interface{} { func getAnonymouStructPointer(structValue reflect.Value, targetType reflect.Type) interface{} { structType := structValue.Type() if structType == targetType { + if !structValue.CanInterface() { + // the value was reached through an unexported field + return nil + } return structValue.Addr().Interface() } for i := 0; i < structValue.NumField(); i += 1 { fieldType := structType.Field(i) - if fieldType.Anonymous && fieldType.Type.Kind() == reflect.Struct { - ptr := getAnonymouStructPointer(structValue.Field(i), targetType) - if ptr != nil { - return ptr + if !fieldType.Anonymous || !gotypes.IsFieldExportable(fieldType.Name) { + // an unexported embedded struct can not be pointed at + continue + } + fieldValue := structValue.Field(i) + fieldT := fieldType.Type + if fieldT.Kind() == reflect.Ptr { + // an embedded pointer that is nil has nothing to point at + if fieldValue.IsNil() { + continue } + fieldValue = fieldValue.Elem() + fieldT = fieldT.Elem() + } + if fieldT.Kind() != reflect.Struct { + continue + } + ptr := getAnonymouStructPointer(fieldValue, targetType) + if ptr != nil { + return ptr } } return nil @@ -214,11 +258,18 @@ func StructContains(type1 reflect.Type, type2 reflect.Type) bool { } for i := 0; i < type1.NumField(); i += 1 { field := type1.Field(i) - if field.Anonymous && field.Type.Kind() == reflect.Struct { - contains := StructContains(field.Type, type2) - if contains { - return true - } + if !field.Anonymous { + continue + } + fieldType := field.Type + if fieldType.Kind() == reflect.Ptr { + fieldType = fieldType.Elem() + } + if fieldType.Kind() != reflect.Struct { + continue + } + if StructContains(fieldType, type2) { + return true } } return false diff --git a/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go b/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go index 7a6e1d7a53..37de0faaf2 100644 --- a/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go +++ b/vendor/yunion.io/x/pkg/util/samlutils/encrypt.go @@ -76,6 +76,9 @@ func (key EncryptedKey) decryptKey(privateKey *rsa.PrivateKey) ([]byte, error) { encAlg := key.EncryptionMethod.Algorithm switch encAlg { case "http://www.w3.org/2001/04/xmlenc#rsa-oaep-mgf1p": + if key.EncryptionMethod.DigestMethod == nil { + return nil, errors.Wrap(errors.ErrInvalidFormat, "missing DigestMethod") + } var shaAlg hash.Hash hashAlg := key.EncryptionMethod.DigestMethod.Algorithm switch hashAlg { @@ -99,6 +102,9 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error if err != nil { return nil, errors.Wrap(err, "base64.StdEncoding.DecodeString") } + if data.KeyInfo.EncryptedKey == nil { + return nil, errors.Wrap(errors.ErrInvalidFormat, "missing KeyInfo.EncryptedKey") + } key, err := data.KeyInfo.EncryptedKey.decryptKey(privateKey) if err != nil { return nil, errors.Wrap(err, "KeyInfo.EncryptedKey.decryptKey") @@ -112,7 +118,36 @@ func (data EncryptedData) decryptData(privateKey *rsa.PrivateKey) ([]byte, error } } +// stripPKCS7Padding removes the padding that XML Encryption appends to the +// last block of a CBC payload. +// +// Data that does not carry valid padding is returned unchanged rather than +// reported, so that this cannot be used to tell one payload from another. +func stripPKCS7Padding(data []byte, blockSize int) []byte { + if len(data) == 0 || len(data)%blockSize != 0 { + return data + } + n := int(data[len(data)-1]) + if n == 0 || n > blockSize || n > len(data) { + return data + } + for _, b := range data[len(data)-n:] { + if int(b) != n { + return data + } + } + return data[:len(data)-n] +} + func decryptAesCbc(key []byte, secret []byte) ([]byte, error) { + // The payload is an IV followed by whole ciphertext blocks. Anything + // shorter, or not a whole number of blocks, cannot be decrypted. + if len(secret) < 2*aes.BlockSize || len(secret)%aes.BlockSize != 0 { + return nil, errors.Wrapf(errors.ErrInvalidFormat, + "ciphertext of %d bytes is not a whole number of %d byte blocks preceded by an IV", + len(secret), aes.BlockSize) + } + c, err := aes.NewCipher(key) if err != nil { return nil, errors.Wrap(err, "aes.NewCipher") @@ -125,5 +160,5 @@ func decryptAesCbc(key []byte, secret []byte) ([]byte, error) { decrypter.CryptBlocks(data, data) - return data, nil + return stripPKCS7Padding(data, aes.BlockSize), nil } diff --git a/vendor/yunion.io/x/pkg/util/samlutils/util.go b/vendor/yunion.io/x/pkg/util/samlutils/util.go index 0e934ee0d3..dee143c4df 100644 --- a/vendor/yunion.io/x/pkg/util/samlutils/util.go +++ b/vendor/yunion.io/x/pkg/util/samlutils/util.go @@ -20,7 +20,9 @@ import ( "crypto/rsa" "encoding/base64" "fmt" + "html" "io" + "sort" "strings" "github.com/ma314smith/signedxml" @@ -92,14 +94,23 @@ func SAMLEncode(input []byte) (string, error) { return base64.StdEncoding.EncodeToString(comp), nil } +// SAMLForm renders an auto-submitting HTML form carrying the given +// parameters. Both the action and the values are escaped, and the keys are +// sorted so the output is stable for a given input. func SAMLForm(action string, attrs map[string]string) string { form := strings.Builder{} // form.WriteString(``) form.WriteString(`
`) - for k, v := range attrs { - form.WriteString(fmt.Sprintf("", k, v)) + keys := make([]string, 0, len(attrs)) + for k := range attrs { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + form.WriteString(fmt.Sprintf("", + html.EscapeString(k), html.EscapeString(attrs[k]))) } form.WriteString(``) form.WriteString("